A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.
Player: quicker loads and frames, a stricter format check, face tracking that yields the page
LivLiva0c115dc6729
diff --git a/API.md b/API.mdindex 6b67363..da498c3 100644--- a/API.md+++ b/API.md@@ -199,7 +199,7 @@ tracking.calibrate() // the user's current pose becomes "facing front" (it alsotracking.stop()```-A refused or missing camera rejects with the browser's error. `trace: 'synth'` plays a demo motion and asks for no camera,+A refused or missing camera rejects with the browser's error, and stops the download unless `preloadFaceModel()` asked for it. `trace: 'synth'` plays a demo motion and asks for no camera,which makes a good fallback (`example/camera.js` does all of this):```js@@ -216,6 +216,7 @@ const tracking = await startFaceTracking(player).catch((e) => {| `mirror` | `true` | move like a mirror image || `smoothing` | `0.5` | 0 quick and shakier .. 1 steady and slower || `follow` | `0` | how much leaning toward / away from the camera moves the character (0..2) |+| `maxFps` | `'auto'` | how often to look for the face, at most: a number, or as often as the camera allows until that would take over half of the main thread (the model runs there), then less, 5 a second at least || `remember` | | keep the calibration on this device under this name (localStorage) || `wasmUrl` | jsDelivr | folder with MediaPipe's `vision_wasm_internal.{js,wasm}` and the `nosimd` pair || `modelUrl` | Google's bucket | `face_landmarker.task` |@@ -320,7 +321,9 @@ an error and the model before stays. Before anything reaches the GPU:compressed entry stops inflating at the size its directory gives (a ZIP bomb goes no further); no paths outside thepackage, no encryption, no ZIP64.- `model.json` (any version): a canvas of 8192 px a side at most, 512 layers, 2048 images, every layer's box inside- twice the canvas, every image a path inside the package, numbers that are numbers.+ twice the canvas, every image a path inside the package, numbers that are numbers. What a mesh would cost is held too,+ counted from the boxes before anything is built: 2 097 152 vertices in all, 1 048 576 bent by chains (the largest models+ built have 46 000 and 40 000), and a layer names a chain once.- Images: 8192 px a side (or the GPU's limit, if lower) and 64 million pixels in all at most, checked once each image'ssize is known, before it is decoded in full.- `keyforms.json` that does not fit its grid is left out (the character plays on the rig alone).diff --git a/FORMAT.md b/FORMAT.mdindex 17aadbc..4de4043 100644--- a/FORMAT.md+++ b/FORMAT.md@@ -217,7 +217,9 @@ What readers do:4. Refuse a model whose layer image is missing; warn and play on when an optional file is missing.5. Hold packages to limits before decoding. The reference player's: `model.json` 4 MB; canvas 8192 px a side; 512layers; 2048 ZIP entries and 256 MB uncompressed; `keyforms.json` 32 MB; no entry with `..`, an absolute path or- encryption.+ encryption; and what the meshes would cost, from the boxes alone: 2 097 152 vertices on a lattice of+ `max(6, round(max(width, height) / 128))` px, 1 048 576 of them times the chains bending them, a chain named once per+ layer.6. Treat everything in the package as data: there is nothing to run and no URL to fetch, and every string (names,labels, credits) is shown as plain text.7. Draw the watermark the package asks for (§3).diff --git a/src/runtime/format.ts b/src/runtime/format.tsindex 0e33fd3..0f16483 100644--- a/src/runtime/format.ts+++ b/src/runtime/format.ts@@ -2,7 +2,7 @@// runtime has always drawn from (ModelDef, the v3 `livemaker-puppet` layout), and v3 itself, read as it is, for good.// The runtime below never sees a v4 file: what changed in v4 is names, places and what is left out, not how a model// is drawn (a v3 model and its v4 export draw the same pixels).-import type { LayerDef, ModelDef, Tuning } from './puppet'+import type { ChainDef, LayerDef, ModelDef, Tuning } from './puppet'/** model.json's `format` and the newest `version` of it this player reads. A v4 file names, in `requires`, the* features a player must have to draw it right (FEATURES: none in 4.0); fields it does not know it ignores */@@ -37,7 +37,15 @@ export const LIMITS = {texture: 8192, texels: 64 * 2 ** 20,/** model.json and keyforms.json (bytes) */modelJson: 4 * 2 ** 20, keyforms: 32 * 2 ** 20,+ /** a layer's mesh covers its box on one lattice (latticeCell), whatever its image: the vertices of all the boxes, and of+ * the layers bent by chains times how many bend them. The heaviest of 111 models here: 46 000 and 40 000 */+ vertices: 2 ** 21, skinned: 2 ** 20,}+/** no number in a model comes near this; past it sums overflow to Infinity and then NaN, and the character is gone */+const BIG = 1e6+const GROUPS = ['head', 'body', 'hair_back', 'hair_side', 'hair_front', 'hair_inner']+/** the lattice every layer's mesh sits on, in model px (the same for all: see Puppet.buildLayer) */+export const latticeCell = (w: number, h: number) => Math.max(6, Math.round(Math.max(w, h) / 128))const IMAGE = /^textures\/[A-Za-z0-9_][A-Za-z0-9_.-]*\.(?:webp|png)$/const FILE = /^(?!.*(?:^|\/)\.\.(?:\/|$))[A-Za-z0-9_][A-Za-z0-9_.-]*(?:\/[A-Za-z0-9_][A-Za-z0-9_.-]*)*$/@@ -85,14 +93,59 @@ function check(m: ModelDef) {if (!(num(w) && num(h) && w > 0 && h > 0 && w <= LIMITS.canvas && h <= LIMITS.canvas)) bad(`its canvas ${w} x ${h} is out of range`)if (!Array.isArray(m.layers) || !m.layers.length) bad('it has no layers')if (m.layers.length > LIMITS.layers) bad(`it has ${m.layers.length} layers, more than ${LIMITS.layers}`)- const A = m.anchors, hd = A?.head- if (!hd || ![hd.cx, hd.cy, hd.rx, hd.ry, A.neck_y, A.top_y, A.bottom_y, A.hair_root_y].every(num) || !(hd.rx > 0 && hd.ry > 0)- || !Array.isArray(A.face) || A.face.length !== 4 || !A.face.every(num)) bad('its anchors are missing or not numbers')- list(A.eyes, LIMITS.eyes, 'anchors.eyes'); list(A.brows, LIMITS.eyes, 'anchors.brows')- for (const line of list(A.brow_lines, LIMITS.eyes, 'anchors.brow_lines') as ({ x?: unknown } | null)[]) {- if (line) list(line.x, LIMITS.points, 'a brow line')+ const A = m.anchors, hd = A?.head, inRange = (v: unknown) => num(v) && Math.abs(v) <= BIG+ if (!hd || ![hd.cx, hd.cy, hd.rx, hd.ry, A.neck_y, A.top_y, A.bottom_y, A.hair_root_y].every(inRange) || !(hd.rx > 1e-3 && hd.ry > 1e-3)+ || !Array.isArray(A.face) || A.face.length !== 4 || !A.face.every(inRange)) bad('its anchors are missing or not numbers')+ // what the rig computes with: a number in range, a list of them, an object of them. Missing (or null: a build's None) is+ // the player's default. Anything else came out as NaN, a character gone for this player's whole life, or as an error+ // thrown once the model before it had been dropped+ const sane = (v: unknown, what: string) => { if (v != null && !inRange(v)) bad(`${what} is not a number in range`) }+ const nums = (v: unknown, lo: number, hi: number, what: string) => {+ if (v == null) return+ if (!Array.isArray(v) || v.length < lo || v.length > hi || !v.every(inRange)) bad(`${what} is not a list of ${lo} to ${hi} numbers`)}- let images = 0+ const fields = (o: unknown, names: string, what: string) => {+ if (o == null) return+ if (typeof o !== 'object') bad(`${what} is not an object`)+ for (const n of names.split(' ')) sane((o as Record<string, unknown>)[n], `${what}.${n}`)+ }+ const weights = (o: unknown, what: string) => {+ if (o == null) return+ if (typeof o !== 'object' || Array.isArray(o)) bad(`${what} is not an object`)+ for (const [k, v] of Object.entries(o as object)) sane(v, `${what}.${k.slice(0, 60)}`)+ }+ sane(A.collar_y, 'anchors.collar_y')+ fields(A.jaw, 'top chin half open round wide', 'anchors.jaw')+ fields(A.chest, 'cx cy rx ry', 'anchors.chest')+ for (const b of list(A.eyes, LIMITS.eyes, 'anchors.eyes')) { if (b == null) bad('anchors.eyes has an empty entry'); nums(b, 4, 4, 'anchors.eyes') }+ for (const b of list(A.brows, LIMITS.eyes, 'anchors.brows')) nums(b, 4, 4, 'anchors.brows')+ for (const line of list(A.brow_lines, LIMITS.eyes, 'anchors.brow_lines') as ({ x?: unknown; bottom?: unknown; lid?: unknown; cap?: unknown } | null)[]) {+ if (line == null) continue+ const n = list(line.x, LIMITS.points, 'a brow line').length+ if (!n) bad('a brow line has no points')+ nums(line.x, n, n, 'a brow line'); nums(line.bottom ?? [], n, n, 'a brow line'); nums(line.lid ?? [], n, n, 'a brow line')+ if (line.cap != null) nums(line.cap, n, n, 'a brow line')+ }+ fields(m.tuning, 'turn nod hair blink expr mouth breath hairFront hairBack', 'tuning')+ const R = m.rig+ fields(R, 'yaw pitch roll bulge bulge_shape hair_limit wind breath sway brow pitch_ease jaw', 'rig')+ nums(R?.bulge_extent, 3, 5, 'rig.bulge_extent'); nums(R?.drift, 2, 2, 'rig.drift'); nums(R?.hair_drive, 3, 3, 'rig.hair_drive')+ fields(R?.body, 'yaw roll breath bust', 'rig.body'); fields(R?.neck, 'twist reach', 'rig.neck')+ if (R?.hair != null) for (const k of ['front', 'back', 'side'] as const) nums(R.hair[k], 5, 5, `rig.hair.${k}`)+ for (const c of list(R?.chains, LIMITS.chains, 'rig.chains') as (ChainDef | null)[]) {+ if (!c || typeof c.id !== 'string') bad('a chain has no id')+ const what = `chain ${c!.id.slice(0, 60)}`, points = list(c!.points, LIMITS.points, `${what}'s points`)+ if (points.length < 2) bad(`${what} has fewer than two points`)+ for (const q of points) { if (q == null) bad(`${what} has an empty point`); nums(q, 2, 2, `${what}'s points`) }+ if (!c!.k) bad(`${what} has no k`)+ nums(c!.k, 2, 2, `${what}'s k`)+ for (const n of ['damp', 'drive', 'hang', 'wind'] as const) if (!num(c![n])) bad(`${what} has no ${n}`)+ fields(c, 'depth damp drive hang wind freq limit sigma reach', what)+ weights(c!.bias, `${what}'s bias`)+ }+ const cell = latticeCell(w, h)+ const bangs = m.layers.find((L) => L?.group === 'hair_front')?.bones+ let images = 0, vertices = 0, skinned = 0, read = 0for (const L of m.layers) {if (!L || typeof L !== 'object' || typeof L.id !== 'string') bad('a layer has no id')if (![L.x, L.y, L.w, L.h].every(num) || !(L.w > 0 && L.h > 0) || L.x < -w || L.y < -h || L.x + L.w > 2 * w || L.y + L.h > 2 * h) {@@ -104,14 +157,32 @@ function check(m: ModelDef) {if (typeof p !== 'string' || !FILE.test(p)) bad(`layer ${L.id.slice(0, 60)}: "${String(p).slice(0, 80)}" is not a path inside the package`)if (++images > LIMITS.images) bad(`it names more than ${LIMITS.images} images`)}+ const what = `layer ${L.id.slice(0, 60)}`+ if (!GROUPS.includes(L.group)) bad(`${what}'s group is not one a player knows`)+ for (const n of ['role', 'clip', 'param'] as const) if (L[n] != null && typeof L[n] !== 'string') bad(`${what}'s ${n} is not text`)+ if (L.visemes != null && !(Array.isArray(L.visemes) && L.visemes.every((v) => typeof v === 'string'))) bad(`${what}'s visemes are not text`)+ sane(L.depth, `${what}'s depth`); sane(L.fall, `${what}'s fall`); sane(L.fall_end, `${what}'s fall_end`)+ nums(L.pivot, 2, 2, `${what}'s pivot`); nums(L.range, 2, 2, `${what}'s range`); nums(L.rigid, 2, 2, `${what}'s rigid`)+ weights(L.blush, `${what}'s blush`); weights(L.show, `${what}'s show`); weights(L.hide, `${what}'s hide`)+ // what its mesh will cost, by its box alone: a 1 x 1 picture stretched over a big box was a 150 000 vertex layer+ const bones = list(L.bones, LIMITS.chains, `${what}'s bones`)+ if (new Set(bones).size !== bones.length) bad(`${what} names a chain twice`)+ if (!bones.every((b) => typeof b === 'string')) bad(`${what}'s bones are not chain ids`)+ const verts = (Math.ceil((L.x + L.w) / cell) - Math.floor(L.x / cell) + 1) * (Math.ceil((L.y + L.h) / cell) - Math.floor(L.y / cell) + 1)+ const overBangs = L.id === 'brows' || L.role === 'expr_brows' || !!L.brow_tex // these read their whole box back, and follow the bangs' chains+ vertices += verts+ skinned += verts * (bones.length + (overBangs && Array.isArray(bangs) ? bangs.length : 0))+ if (overBangs) read += L.w * L.h}- list(m.expressions, LIMITS.expressions, 'expressions')- for (const c of list(m.rig?.chains, LIMITS.chains, 'rig.chains') as { points?: unknown }[]) list(c?.points, LIMITS.points, 'a chain')+ if (vertices > LIMITS.vertices) bad(`its layers' meshes would have ${vertices} vertices, more than ${LIMITS.vertices}`)+ if (skinned > LIMITS.skinned) bad(`its chains would bend ${skinned} vertices, more than ${LIMITS.skinned}`)+ if (read > LIMITS.texels) bad(`its brows layers cover ${read} pixels, more than ${LIMITS.texels}`)+ for (const e of list(m.expressions, LIMITS.expressions, 'expressions') as ({ id?: unknown } | null)[]) if (!e || typeof e.id !== 'string') bad('an expression has no id')if (m.keyforms !== undefined && (typeof m.keyforms !== 'string' || !FILE.test(m.keyforms))) bad('keyforms is not a path inside the package')if (m.watermark != null && typeof m.watermark !== 'string') bad('its watermark is not text')}-interface V4Layer extends Omit<LayerDef, 'name'> { label?: Labels; expression?: string }+interface V4Layer extends Omit<LayerDef, 'name' | 'depth'> { depth?: number; label?: Labels; expression?: string }interface V4 {format: string; version: number; name: string; meta: ModelMeta; requires?: string[]; files?: ModelFilescanvas: [number, number]; profile?: { id: string; facing?: string; glasses?: boolean }@@ -133,7 +204,7 @@ function lower(m: V4, lang: string): ModelDef {if (!k.startsWith('x-') && (typeof p !== 'string' || !FILE.test(p))) throw new Error(`files.${k}: "${p}" is not a path in the package`)}const layers = m.layers.map(({ label, expression, ...L }) =>- ({ ...L, name: label ? pickLabel(label, 'en', L.id) : L.id, ...(expression ? { param: expression } : {}) }) as LayerDef)+ ({ ...L, depth: L.depth ?? 0, name: label ? pickLabel(label, 'en', L.id) : L.id, ...(expression ? { param: expression } : {}) }) as LayerDef)let tuning: Partial<Tuning> | undefinedif (m.tuning) {const { hair_front, hair_back, ...t } = m.tuningdiff --git a/src/runtime/puppet.ts b/src/runtime/puppet.tsindex ef1d580..428b785 100644--- a/src/runtime/puppet.ts+++ b/src/runtime/puppet.ts@@ -16,7 +16,7 @@// the `brows` layer alone where the build gave the brows one (drawn over the bangs); that layer moves with// the bangs where it lies over them// - body: lags the head turn / tilt, breathes (shoulders + chest), optional chest spring-import { LIMITS, readModel, type Labels, type ModelFiles, type ModelMeta } from './format'+import { LIMITS, latticeCell, readModel, type Labels, type ModelFiles, type ModelMeta } from './format'import { PARAM_RANGES, defaultParams, smooth, type Params } from './params'export { PARAM_RANGES, defaultParams, smooth, type Params }@@ -196,7 +196,7 @@ export const defaultTuning = (): Tuning => ({ turn: 1, nod: 1, hair: 1, blink: 1interface GLLayer {def: LayerDeftex: WebGLTexture; frames: WebGLTexture[]; mask?: WebGLTexture; talk?: WebGLTexture; browTex?: WebGLTexture- vao: WebGLVertexArrayObject; pos: WebGLBuffer; count: number+ vao: WebGLVertexArrayObject; pos: WebGLBuffer; uvs: WebGLBuffer; tris: WebGLBuffer; count: numberrest: Float32Array; out: Float32Array/** the mesh's triangles (vertex indices, the index buffer's) */tri: Uint32Array@@ -511,31 +511,27 @@ export class Puppet {tag = (h >>> 0).toString(36)}const v = `?v=${tag}${query ? `&${query}` : ''}`+ // alongside the images, not before them. Without it the character still plays, on the rig alone+ const keyforms = model.keyforms ? fetch(base + model.keyforms + v).then(async (k) => {+ const t = k.ok ? await k.text() : ''+ return t && t.length <= LIMITS.keyforms ? JSON.parse(t) as KeyformFile : undefined+ }).catch(() => undefined) : undefined// all of them in one file when the server has it: one request instead of ~30const pack = png || !usePack ? null : await loadPack(packUrl(baseUrl, tag))- let keyforms: KeyformFile | undefined- if (model.keyforms) { // without it the character still plays, on the rig alone- const k = await fetch(base + model.keyforms + v).catch(() => null)- const t = k?.ok ? await k.text().catch(() => '') : ''- if (t && t.length <= LIMITS.keyforms) { try { keyforms = JSON.parse(t) } catch { /* plays on the rig alone */ } }- }await this.loadModel(model, (f) => { const b = pack?.get(f); return b ? loadBlob(b) : loadImage(base + f + v) }, { png, keyforms })}/** a model from anywhere (the SDK: files the user picked): its model.json, and each image it names by its path there */async loadModel(raw: ModelDef, image: (path: string) => Promise<HTMLImageElement>,- { png = false, keyforms }: { png?: boolean; keyforms?: KeyformFile } = {}) {+ { png = false, keyforms: keyformsComing }: { png?: boolean; keyforms?: KeyformFile | Promise<KeyformFile | undefined> } = {}) {const model = readModel(raw)- const imgs = await Promise.all(model.layers.map(async (L) => {+ // every image at once: a layer's second image waiting for its first cost a round trip each on a slow path+ const [imgs, keyforms] = await Promise.all([Promise.all(model.layers.map(async (L) => {const load = (p: string) => image(L.webp && !png ? p.replace(/\.png$/, '.webp') : p)- return {- main: await load(L.texture),- frames: await Promise.all((L.frames || []).map(load)),- mask: L.mask ? await load(L.mask) : undefined,- talk: L.talk ? await load(L.talk) : undefined,- brows: L.brow_tex ? await load(L.brow_tex) : undefined,- }- }))+ const [main, frames, mask, talk, brows] = await Promise.all([load(L.texture), Promise.all((L.frames || []).map(load)),+ L.mask ? load(L.mask) : undefined, L.talk ? load(L.talk) : undefined, L.brow_tex ? load(L.brow_tex) : undefined])+ return { main, frames, mask, talk, brows }+ })), keyformsComing])// before anything reaches the GPU (an image's size is known once it loads, before it is decoded in full)const most = Math.min(LIMITS.texture, (this.gl?.getParameter(this.gl.MAX_TEXTURE_SIZE) as number) || LIMITS.texture)let texels = 0@@ -600,8 +596,9 @@ export class Puppet {this.expr = {}this.resetMouth(); this.talkLeft = 0this.cutBottom = model.anchors.bottom_y >= model.canvas[1] - 4- for (const c of Object.values(this.chains)) for (const s of c) Object.assign(s, spring())- this.drive.started = false+ // all that carries over from frame to frame starts again: nothing of the model before moves this one (and a NaN+ // in the torso's smoothing once stayed there for good, whatever was loaded next)+ this.resetPhysics()this.keys = []; this.kfPair = nullif (keyforms && keyformsFit(keyforms)) this.prepareKeyforms(keyforms)else if (keyforms) console.warn('LivLiv: keyforms.json is not one this player reads; the character plays on the rig alone')@@ -611,6 +608,19 @@ export class Puppet {* springs at rest, the torso settled and the face neutral, so at the key the character stands exactly as in the clip* and everything else (physics, blinks, mouth, brows, roll) keeps moving on top of it */private prepareKeyforms(K: KeyformFile) {+ // base64 to bytes, each field once (one field can serve every head layer). A plain loop: Uint8Array.from(text, mapper)+ // calls back per character, which took half of the refined Nana's load+ const seen = new Map<string, Uint8Array>()+ const bytes = (b64: string) => {+ let out = seen.get(b64)+ if (!out) {+ const text = atob(b64)+ out = new Uint8Array(text.length)+ for (let i = 0; i < text.length; i++) out[i] = text.charCodeAt(i)+ seen.set(b64, out)+ }+ return out+ }const save = { params: this.params, body: { ...this.body }, bust: { ...this.bust }, mouthW: this.mouthW, smile: this.smileShown,chains: Object.values(this.chains).map((c) => c.map((q) => ({ ...q }))) }const B = this.rig.body, tt = this.tuning.turn@@ -627,14 +637,14 @@ export class Puppet {for (const L of this.layers) {// "torso": one field for every body layer (a drawn view's neck and collar), where the file has oneconst has = (f: string) => !!key.fields[f] || !!key.same?.includes(f)- const g = key.fields[L.def.id] ? L.def.id : L.def.group === 'head' ? 'head' : L.def.group.startsWith('hair') ? 'hair'+ const g = Object.hasOwn(key.fields, L.def.id) ? L.def.id : L.def.group === 'head' ? 'head' : L.def.group.startsWith('hair') ? 'hair': L.def.group === 'body' && has('torso') ? 'torso' : L.def.idconst b64 = key.fields[g] ?? (key.same?.includes(g) ? key.fields.head : undefined)L.kf ||= []if (!b64) { L.kf[ki] = null; continue }- const raw = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0))+ const raw = bytes(b64)const f = new Int16Array(raw.buffer, raw.byteOffset, raw.byteLength >> 1)- const m = K.mask?.[g] ? Uint8Array.from(atob(K.mask[g]), (c) => c.charCodeAt(0)) : undefined+ const m = K.mask?.[g] ? bytes(K.mask[g]) : undefinedconst d = new Float32Array(L.rest.length)for (let i = 0; i < d.length; i += 2) {const [dx, dy] = sampleField(f, K, L.rest[i], L.rest[i + 1])@@ -659,7 +669,7 @@ export class Puppet {for (const L of this.layers) {gl.deleteTexture(L.tex); L.frames.forEach((t) => gl.deleteTexture(t)); if (L.mask) gl.deleteTexture(L.mask); if (L.talk) gl.deleteTexture(L.talk)if (L.browTex) gl.deleteTexture(L.browTex)- gl.deleteVertexArray(L.vao); gl.deleteBuffer(L.pos)+ gl.deleteVertexArray(L.vao); gl.deleteBuffer(L.pos); gl.deleteBuffer(L.uvs); gl.deleteBuffer(L.tris)}if (this.maskFb) { gl.deleteFramebuffer(this.maskFb); gl.deleteTexture(this.maskTex); this.maskFb = this.maskTex = null; this.maskSize = [0, 0] }this.layers = []@@ -687,7 +697,7 @@ export class Puppet {// every layer's vertices sit on ONE global lattice, so the piecewise-linear deformation is identical// for overlapping layers: patches (eyes, mouth, expressions) can never drift off the face at big anglesconst [MW, MH] = this.model.canvas- const cell = Math.max(6, Math.round(Math.max(MW, MH) / 128))+ const cell = latticeCell(MW, MH)const gx0 = Math.floor(L.x / cell), gy0 = Math.floor(L.y / cell)const cols = Math.ceil((L.x + L.w) / cell) - gx0, rows = Math.ceil((L.y + L.h) / cell) - gy0const occ = alphaGrid([img, ...frames], cols, rows, (L.x - gx0 * cell) / cell, (L.y - gy0 * cell) / cell, L.w / cell, L.h / cell)@@ -851,7 +861,7 @@ export class Puppet {return {def: L, tex: this.texture(img), frames: frames.map((f) => this.texture(f)), mask: mask ? this.texture(mask) : undefined,talk: talk ? this.texture(talk) : undefined, browTex: brows ? this.texture(brows) : undefined,- vao, pos: pb, count: idx.length, rest, out: new Float32Array(rest.length), tri, tw: img.naturalWidth || img.width, th: img.naturalHeight || img.height,+ vao, pos: pb, uvs: ub, tris: ib, count: idx.length, rest, out: new Float32Array(rest.length), tri, tw: img.naturalWidth || img.width, th: img.naturalHeight || img.height,wHead, wPhys1, wPhys2, dep, wBrow, browSide, browCap, wFront, wJaw, wNeck, wTwist, fol, folB, alt,outB: folB ? new Float32Array(rest.length) : undefined, stencilRef: 0, soft: 0, maskCh: -1, skin, altSkin,}@@ -1098,7 +1108,7 @@ export class Puppet {}const C = A.chestif (C && bust) {- const d = Math.hypot((x - C.cx) / C.rx, (y - C.cy) / C.ry)+ const ux = (x - C.cx) / C.rx, uy = (y - C.cy) / C.ry, d = Math.sqrt(ux * ux + uy * uy) // per vertex: hypot allocatesconst w = 1 - smooth(0.2, 1.15, d)if (w > 0) {dy -= br * 0.012 * h.ry * w * (0.4 + B.bust) // the chest lifts more than the shoulders when breathing indiff --git a/src/runtime/tracking.ts b/src/runtime/tracking.tsindex 137fe14..fbafafe 100644--- a/src/runtime/tracking.ts+++ b/src/runtime/tracking.ts@@ -23,21 +23,28 @@ export type StartStep = { step: 'download'; progress: number } | { step: 'preparlet shared: Promise<Landmarker> | null = nulllet progress = 0const watchers = new Set<(p: number) => void>()+/** the download while it runs, and who still wants it: the trackers waiting for their camera (by their ticket), or anyone+ * who asked ahead of time (`ahead`, for good: hovering a button says the model will be wanted) */+let download: AbortController | null = null+const waiting = new Set<object>()+let ahead = false/** The face model, loaded once per page and kept: every tracker and every camera toggle reuses it. Call it early* (hovering the camera button, the live dialog) and the camera opens without waiting for it. The files are read with* progress and land in the HTTP cache, where the live window, opened in another tab, finds them */-export function loadFaceModel(onProgress?: (p: number) => void): Promise<Landmarker> {+export function loadFaceModel(onProgress?: (p: number) => void, ticket?: object): Promise<Landmarker> {if (onProgress) { watchers.add(onProgress); onProgress(progress) }const done = () => { if (onProgress) watchers.delete(onProgress) }+ if (ticket) waiting.add(ticket); else ahead = trueif (!shared) {+ const { signal } = (download = new AbortController())shared = (async () => {const vision = await import('@mediapipe/tasks-vision')const [name, size] = (await vision.FilesetResolver.isSimdSupported()) ? RUNTIME.simd : RUNTIME.plainconst runtime = `${FILES.wasm.replace(/\/$/, '')}/${name}`const got = [0, 0], total = size + MODEL_SIZEconst read = async (url: string, i: number) => {- const r = await fetch(url)+ const r = await fetch(url, { signal })if (!r.ok || !r.body) throw new Error(`${url} ${r.status}`)const reader = r.body.getReader(), parts: Uint8Array[] = []for (;;) {@@ -54,17 +61,26 @@ export function loadFaceModel(onProgress?: (p: number) => void): Promise<Landmar// the runtime is read only to have it cached (MediaPipe loads it by URL again); the model is handed over as it isconst [, model] = await Promise.all([read(`${runtime}.wasm?v=${V}`, 0), read(`${FILES.model}?v=${V}`, 1)])progress = 1; watchers.forEach((w) => w(1))+ download = nullreturn vision.FaceLandmarker.createFromOptions({ wasmLoaderPath: `${runtime}.js?v=${V}`, wasmBinaryPath: `${runtime}.wasm?v=${V}` }, {baseOptions: { modelAssetBuffer: model, delegate: 'GPU' },runningMode: 'VIDEO', numFaces: 1,outputFaceBlendshapes: true, outputFacialTransformationMatrixes: true,})})()- shared.catch(() => { shared = null; progress = 0 }) // a failed download is tried again next time+ shared.catch(() => { shared = null; progress = 0; download = null }) // a failed download is tried again next time}shared.then(done, done)return shared}+/** a tracker no longer waits for the model (its camera was refused, or it was stopped): with nobody else waiting and nobody+ * having asked ahead, the download stops. It runs beside the camera's prompt to save time, and a refusal should not+ * cost 7 MB */+function dropFaceModel(ticket: object) {+ waiting.delete(ticket)+ if (!waiting.size && !ahead) download?.abort()+}+/** a face unseen this long (s) is lost: the player hands the character back to its idle motion */export const LOST_AFTER = 0.4@@ -94,6 +110,14 @@ export class FaceTracker {follow = 0.3/** process every n-th camera frame (power saving: 2) */every = 1+ /** detections per second at most. The model runs on the main thread, inside the frame: 14-16 ms each on an M4 with a+ * busy machine, so on a phone it can take the frame rate with it. 'auto': a detection takes at most half of the main+ * thread's time, by what the last ones took, and never under 5 a second (a face is lost after LOST_AFTER). Unset: every+ * new camera frame, as livliv.ai has always done */+ maxFps?: number | 'auto'+ /** how long a detection takes (ms, smoothed) and when the next may run */+ private detectMs = 0+ private detectAt = 0/** set: calibration is remembered on this device for this character (localStorage), and found again next time */calibId?: stringcalibrated = false@@ -161,13 +185,15 @@ export class FaceTracker {// the camera is asked for at once, so it warms up (and the user answers its prompt) while the model loadslet modelReady = falseconst say = () => this.onStep?.(!modelReady ? (progress < 1 ? { step: 'download', progress } : { step: 'prepare' }) : { step: 'camera' })- const model = loadFaceModel(() => say()).then((lm) => { modelReady = true; say(); return lm })+ const model = loadFaceModel(() => say(), this).then((lm) => { modelReady = true; say(); return lm })const camera = navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480, facingMode: 'user' } })say()let stream: MediaStreamtry {[this.lm, stream] = await Promise.all([model, camera])+ waiting.delete(this)} catch (e) {+ dropFaceModel(this)camera.then((s) => s.getTracks().forEach((t) => t.stop()), () => {}) // the model failed: let the camera gothrow e}@@ -183,6 +209,7 @@ export class FaceTracker {/** the camera goes off; the face model stays loaded for the next time (loadFaceModel) */stop() {this.stopped = true+ dropFaceModel(this)this.stream?.getTracks().forEach((t) => t.stop())this.ok = false}@@ -192,9 +219,17 @@ export class FaceTracker {const now = performance.now()let f: Frame | null = nullif (this.src) f = this.src.next(now)- else if (this.lm && this.video.readyState >= 2 && this.video.currentTime !== this.last) {+ else if (this.lm && this.video.readyState >= 2 && this.video.currentTime !== this.last && now >= this.detectAt) {this.last = this.video.currentTime- if (++this.frameNo % this.every === 0) { f = toFrame(this.lm.detectForVideo(this.video, now), now); this.rec?.push(f) }+ if (++this.frameNo % this.every === 0) {+ const r = this.lm.detectForVideo(this.video, now)+ if (this.maxFps) { // (the first takes far longer, setting things up: not what later ones will)+ const took = Math.min(200, performance.now() - now)+ this.detectMs = this.detectMs ? this.detectMs + (took - this.detectMs) * 0.15 : took / 8+ this.detectAt = now + (this.maxFps === 'auto' ? Math.min(200, 2 * this.detectMs) : 1000 / this.maxFps)+ }+ f = toFrame(r, now); this.rec?.push(f)+ }}if (f) this.read(f, now)this.lostFor = this.seenAt < 0 ? Infinity : (now - this.seenAt) / 1000diff --git a/src/tracking.ts b/src/tracking.tsindex fb20d57..61d4b5a 100644--- a/src/tracking.ts+++ b/src/tracking.ts@@ -23,6 +23,9 @@ export interface FaceTrackingOptions extends FaceModelFiles {smoothing?: number/** how much leaning toward / away from the camera moves the character, 0 (default) .. 2 */follow?: number+ /** how many times a second to look for the face at most. 'auto' (default): as often as the camera gives a new picture,+ * until that would take over half of the page's main thread (the model runs there); then less, 5 at least */+ maxFps?: number | 'auto'/** keep the calibration (the user's resting pose) on this device under this name (localStorage) */remember?: string/** what starting waits for: the download (progress 0..1, first time only), preparing the model, the camera */@@ -65,6 +68,7 @@ export async function startFaceTracking(player: Host, options: FaceTrackingOptioconst t = new FaceTracker()t.mirror = options.mirror ?? truet.setSmoothing(options.smoothing ?? 0.5)+ t.maxFps = options.maxFps ?? 'auto't.calibId = options.remembert.onStep = options.onProgresst.trace = options.trace ?? ''diff --git a/test/bodies.html b/test/bodies.htmlindex a220436..d2f4fd7 100644--- a/test/bodies.html+++ b/test/bodies.html@@ -5,7 +5,8 @@<script type="importmap">{ "imports": {"three": "/node_modules/three/build/three.module.js","three/examples/jsm/": "/node_modules/three/examples/jsm/",- "@pixiv/three-vrm": "/node_modules/@pixiv/three-vrm/lib/three-vrm.module.js"+ "@pixiv/three-vrm": "/node_modules/@pixiv/three-vrm/lib/three-vrm.module.js",+ "@mediapipe/tasks-vision": "/node_modules/@mediapipe/tasks-vision/vision_bundle.mjs"} }</script><canvas id="png" style="width: 200px; height: 240px"></canvas><canvas id="vrm" style="width: 200px; height: 300px"></canvas>@@ -17,7 +18,7 @@import { PngBody } from '../dist/png.js'import { VrmBody } from '../dist/vrm.js'import { createSpeech } from '../dist/voice.js'- import { startFaceTracking } from '../dist/tracking.js'+ import { preloadFaceModel, startFaceTracking } from '../dist/tracking.js'const g = document.createElement('canvas').getContext('2d', { willReadFrequently: true })const read = (canvas) => {@@ -45,6 +46,19 @@return new Blob([b.buffer], { type: 'audio/wav' })}+ /** the camera refused while the face model comes in (the slow route of smoke.mjs); ahead: it was asked for before */+ window.runRefused = async (ahead = false) => {+ const files = { wasmUrl: '/node_modules/@mediapipe/tasks-vision/wasm', modelUrl: '/__model.task' }+ if (ahead) preloadFaceModel(files).catch(() => {})+ const real = navigator.mediaDevices.getUserMedia+ navigator.mediaDevices.getUserMedia = async () => { await wait(150); throw new DOMException('refused', 'NotAllowedError') }+ const p = new BodyPlayer(document.createElement('canvas'), new PngBody(), { autoplay: false })+ const name = await startFaceTracking(p, files).then(() => 'started', (e) => e.name)+ navigator.mediaDevices.getUserMedia = real+ p.dispose()+ return name+ }+/** what a VRM player says of a file it cannot play, keeping the character before */window.runVrmRefusals = async (urls) => {const c = document.createElement('canvas')diff --git a/test/fuzz.test.ts b/test/fuzz.test.tsindex 106f692..2aad753 100644--- a/test/fuzz.test.ts+++ b/test/fuzz.test.ts@@ -154,3 +154,61 @@ describe('fuzz: .livliv', () => {await assert.rejects(unzip(new Blob([locked as BlobPart])), /encrypted/)})})++describe('what a mesh would cost', () => {+ /** a model of `n` layers, each a box of 3 x the canvas (legal by itself): a tiny file for a big mesh */+ const big = (n: number, more: object = {}) => {+ const m = example(), first = m.layers[0]+ return { ...m, canvas: [768, 768], layers: Array.from({ length: n }, (_, i) => ({ ...first, id: `l${i}`, x: -768, y: -768, w: 2304, h: 2304, ...more })) }+ }+ it('counts vertices from the boxes', () => {+ assert.throws(() => readModel(big(15)), /vertices/) // 15 x 148 225 > 2 097 152+ assert.doesNotThrow(() => readModel(big(14)))+ })+ it('counts vertices bent by chains, and refuses a chain named twice', () => {+ assert.throws(() => readModel(big(1, { bones: Array(3000).fill('c') })), /bones|chain/)+ assert.throws(() => readModel(big(1, { bones: ['c', 'c'] })), /twice/)+ assert.throws(() => readModel(big(8, { bones: ['a'] })), /bend/) // 8 x 148 225 > 1 048 576+ assert.doesNotThrow(() => readModel(big(7, { bones: ['a'] })))+ })+ it('counts what brows layers read back', () => {+ const m = example(), first = m.layers[0]+ const boxes = { ...m, canvas: [8000, 8000], layers: [{ ...first, id: 'brows', x: -1000, y: -1000, w: 9000, h: 9000 }] }+ assert.throws(() => readModel(boxes), /brows layers/)+ })+})++describe('what the rig computes with', () => {+ const first = () => example().layers[0]+ const refused: [string, (m: ReturnType<typeof example>) => void, RegExp][] = [+ ['a chain with no id', (m) => { m.rig = { chains: [{ points: [[0, 0], [1, 1]], k: [1, 1], damp: 1, drive: 1, hang: 0, wind: 0 }] } }, /no id/],+ ['an empty chain', (m) => { m.rig = { chains: [null] } }, /no id/],+ ['a chain with one point', (m) => { m.rig = { chains: [{ id: 'c', points: [[0, 0]], k: [1, 1], damp: 1, drive: 1, hang: 0, wind: 0 }] } }, /two points/],+ ['a chain with no k', (m) => { m.rig = { chains: [{ id: 'c', points: [[0, 0], [1, 1]], damp: 1, drive: 1, hang: 0, wind: 0 }] } }, /no k/],+ ['a brow line with no bottom', (m) => { m.anchors.brows = [[1, 1, 5, 5]]; m.anchors.brow_lines = [{ x: [1, 2], lid: [3, 4] }] }, /brow line/],+ ['rig.roll as text', (m) => { m.rig = { ...m.rig, roll: 'x' } }, /rig\.roll/],+ ['rig.body.roll past any sense', (m) => { m.rig = { ...m.rig, body: { roll: 1e308 } } }, /rig\.body\.roll/],+ ['a spring chain of the wrong length', (m) => { m.rig = { ...m.rig, hair: { front: [1, 2] } } }, /rig\.hair\.front/],+ ['tuning as text', (m) => { m.tuning = { turn: 'x' } }, /tuning\.turn/],+ ['a role that is no text', (m) => { m.layers[0].role = 7 }, /role/],+ ['an unknown group', (m) => { m.layers[0].group = 'wings' }, /group/],+ ['a pivot of one number', (m) => { m.layers[0].pivot = [1] }, /pivot/],+ ['a blush weight as text', (m) => { m.layers[0].blush = { smile: 'x' } }, /blush/],+ ['an empty eye box', (m) => { m.anchors.eyes = [null] }, /eyes/],+ ]+ for (const [name, change, why] of refused) {+ it(`refuses ${name}`, () => {+ const m = example()+ change(m)+ assert.throws(() => readModel(m), why)+ })+ }+ it('takes what is simply missing, and null where a build wrote None', () => {+ const m = example()+ delete m.tuning; delete m.rig+ m.anchors.brows = [null, [1, 1, 5, 5]]; m.anchors.collar_y = null+ const { depth: _, ...noDepth } = first()+ m.layers[0] = noDepth+ assert.equal(readModel(m).layers[0].depth, 0) // the schema's default+ })+})diff --git a/test/smoke.html b/test/smoke.htmlindex 9db659c..be2ffef 100644--- a/test/smoke.html+++ b/test/smoke.html@@ -172,6 +172,20 @@gl.readPixels(100, 100, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, px)Object.assign(out, { model: p.model, drawn: px[3], made, freed, glError: gl.getError() })}+ {+ // what several loads and a dispose leave on the GPU+ const p = new Player(c, { autoplay: false, followPointer: false }), gl = p.puppet.gl+ const left = { texture: 0, buffer: 0, vertexArray: 0 }+ for (const [kind, make, free] of [['texture', 'createTexture', 'deleteTexture'], ['buffer', 'createBuffer', 'deleteBuffer'], ['vertexArray', 'createVertexArray', 'deleteVertexArray']]) {+ const m = gl[make], f = gl[free]+ gl[make] = function () { left[kind]++; return m.call(gl) }+ gl[free] = function (t) { if (t) left[kind]--; return f.call(gl, t) }+ }+ for (let i = 0; i < 3; i++) await p.load(`../characters/${id}/`)+ p.dispose()+ for (const k of ['createTexture', 'deleteTexture', 'createBuffer', 'deleteBuffer', 'createVertexArray', 'deleteVertexArray']) delete gl[k]+ out.left = left+ }{const p = new Player(c, { followPointer: false })p.dispose()@@ -196,6 +210,64 @@return out}+ /** random changes to a model.json, in the real player: whatever happens, the player goes on working. Refused: the model+ * before is still drawn. Loaded: the frames do not throw. And a good model after any of them draws (nothing is poisoned) */+ window.runFuzz = async (id, keep, from, to) => {+ const canvas2 = document.createElement('canvas')+ canvas2.style.cssText = 'width: 160px; height: 160px'+ document.body.append(canvas2)+ let p = new Player(canvas2, { autoplay: false, followPointer: false })+ await p.load(`../characters/${keep}/`)+ const base = await json(id), files = await filesOf(id, base)+ let frameErrors = 0+ const onError = () => { frameErrors++ }+ addEventListener('error', onError)+ const rng = (seed) => () => { seed = (seed + 0x6d2b79f5) | 0; let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); t ^= t + Math.imul(t ^ (t >>> 7), 61 | t); return ((t ^ (t >>> 14)) >>> 0) / 2 ** 32 }+ const NASTY = [null, -1, 0, 1, 0.5, 2, 1e9, -1e9, 1e308, 'x', '', [], {}, true, [1, 2], [null], [[1]], { x: 1 }, 'smile']+ const mutate = (v, r) => {+ const pick = (xs) => xs[Math.floor(r() * xs.length)], root = { v }+ let parent = root, key = 'v'+ for (let d = 0; d < 8 && r() < 0.85; d++) {+ const cur = parent[key]+ if (!cur || typeof cur !== 'object') break+ const keys = Array.isArray(cur) ? cur.map((_, i) => i) : Object.keys(cur)+ if (!keys.length) break+ parent = cur; key = pick(keys)+ }+ if (r() < 0.25) { if (Array.isArray(parent)) parent.splice(key, 1); else delete parent[key] } else parent[key] = pick(NASTY)+ return root.v+ }+ const nan = () => p.puppet.layers.some((L) => L.out.length && Number.isNaN(L.out[0]))+ const counts = { refused: 0, loaded: 0, blank: 0 }, wrong = []+ for (let seed = from; seed <= to; seed++) {+ const r = rng(seed)+ let m = JSON.parse(JSON.stringify(base))+ for (let k = 1 + Math.floor(r() * 2); k > 0; k--) m = mutate(m, r)+ const map = new Map(files)+ map.set(`${id}/model.json`, new Blob([JSON.stringify(m)]))+ const before = frameErrors+ let refused = null+ try { await p.load(map) } catch (e) { refused = String(e.message || e) }+ try { p.draw(1 / 60); p.draw(1 / 60) } catch { frameErrors++ }+ if (refused) {+ counts.refused+++ if (p.puppet.model.name !== p.model.name || !p.puppet.layers.length) wrong.push(`#${seed} the model before is gone: ${refused.slice(0, 90)}`)+ } else {+ counts.loaded+++ if (nan()) counts.blank+++ }+ if (frameErrors > before) wrong.push(`#${seed} a frame threw`)+ if (refused === null || wrong.length) {+ await p.load(`../characters/${keep}/`)+ p.draw(1 / 60)+ if (nan()) wrong.push(`#${seed} the next good model draws NaN`)+ }+ }+ removeEventListener('error', onError)+ p.dispose(); canvas2.remove()+ return { counts, wrong: wrong.slice(0, 8) }+ }+/** a package's connection dropping (the flaky routes of smoke.mjs): once, it is tried again and plays; always, the page* gets our words, not the browser's; a 404 is not tried again */window.runDropped = async (id) => {@@ -219,6 +291,10 @@box: filesOf(id, { ...m, layers: [{ ...first, w: 1e9 }, ...m.layers.slice(1)] }),json: filesOf(id, '{ "format": "livliv-model", '),layers: filesOf(id, { ...m, layers: Array.from({ length: 600 }, () => first) }),+ chain: filesOf(id, { ...m, rig: { ...m.rig, chains: [{ points: [[0, 0], [1, 1]] }] } }),+ browLine: filesOf(id, { ...m, anchors: { ...m.anchors, brows: [[1, 1, 5, 5]], brow_lines: [{ x: [1, 2] }] } }),+ rig: filesOf(id, { ...m, rig: { ...m.rig, roll: 'x' } }),+ mesh: filesOf(id, { ...m, layers: Array.from({ length: 25 }, (_, i) => ({ ...first, id: `l${i}`, x: -m.canvas[0], y: -m.canvas[1], w: 3 * m.canvas[0], h: 3 * m.canvas[1] })) }),}const out = {}for (const [k, files] of Object.entries(cases)) {diff --git a/test/smoke.mjs b/test/smoke.mjsindex f1828d8..a93b47e 100644--- a/test/smoke.mjs+++ b/test/smoke.mjs@@ -22,7 +22,7 @@ const list = (dir, at = '') => readdirSync(join(dir, at), { withFileTypes: true.flatMap((d) => (d.isDirectory() ? list(dir, join(at, d.name)) : [join(at, d.name).split('\\').join('/')]))const files = Object.fromEntries(ids.map((id) => [id, list(join(chars, id))]))-const TYPES = { '.html': 'text/html', '.js': 'text/javascript', '.json': 'application/json', '.webp': 'image/webp', '.png': 'image/png', '.jpg': 'image/jpeg' }+const TYPES = { '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript', '.wasm': 'application/wasm', '.json': 'application/json', '.webp': 'image/webp', '.png': 'image/png', '.jpg': 'image/jpeg' }const vrm = tinyVrm()/** an image bomb: a 9000 x 9000 PNG (all black, 80 KB), past the player's 8192 px a side */function bombPng(side = 9000) {@@ -36,9 +36,23 @@ function bombPng(side = 9000) {chunk('IDAT', deflateSync(Buffer.alloc(side * (side + 1)))), chunk('IEND', Buffer.alloc(0))])}const bomb = bombPng()+/** a face model that takes a while to arrive, for what happens to its download (runDownload in bodies.html) */+const slowModel = { asked: 0, finished: 0 }/** requests per route of runDropped */const dropped = {}const server = createServer((req, res) => {+ if (req.url.startsWith('/__model.task')) {+ slowModel.asked+++ res.writeHead(200, { 'content-type': 'application/octet-stream' })+ let left = 25+ const tick = setInterval(() => {+ if (res.destroyed) { clearInterval(tick); return }+ res.write(Buffer.alloc(150_000))+ if (--left === 0) { clearInterval(tick); slowModel.finished++; res.end() }+ }, 40)+ res.on('close', () => clearInterval(tick))+ return+ }const drop = req.url.match(/^\/__dropped\/(once|always|missing)\/(.+)\.livliv$/)if (drop) {const [, how, id] = drop, n = (dropped[how] = (dropped[how] || 0) + 1)@@ -100,7 +114,7 @@ try {// hostile packages: refused with the model before kept; broken keyforms: played on the rig aloneconst kfId = ids.find((id) => JSON.parse(readFileSync(join(chars, id, 'model.json'), 'utf8')).files?.keyforms) ?? ids[0]const hostile = await page.evaluate(([a, b]) => window.runHostile(a, b), [ids[0], kfId])- const why = { bomb: /too large/, path: /not an image path|not a path/, canvas: /canvas/, box: /box/, json: /not a model\.json/, layers: /layers/ }+ const why = { bomb: /too large/, path: /not an image path|not a path/, canvas: /canvas/, box: /box/, json: /not a model\.json/, layers: /layers/, mesh: /vertices/, chain: /no id/, browLine: /brow line/, rig: /rig\.roll/ }for (const [k, re] of Object.entries(why)) {const r = hostile[k], ok = !r.loaded && re.test(r.refused) && !!r.keptif (!ok) failed++@@ -110,6 +124,11 @@ try {if (!kfOk) failed++console.log(`${kfOk ? 'ok ' : 'FAIL'} broken keyforms: ${hostile.keyforms.refused ?? `${hostile.keyforms.name} plays on the rig alone`}`)+ // random changes to a model.json in the real player (see runFuzz): the seeds of a run are the same every time+ const fuzzed = await page.evaluate(([a, b]) => window.runFuzz(a, b, 1, 150), [ids.find((i) => i === 'k02') ?? ids[0], ids[0]])+ const fuzzOk = !fuzzed.wrong.length+ if (!fuzzOk) failed+++ console.log(`${fuzzOk ? 'ok ' : 'FAIL'} 150 changed models: ${fuzzed.counts.refused} refused, ${fuzzed.counts.loaded} loaded, of them ${fuzzed.counts.blank} blank${fuzzOk ? '' : `\n ${fuzzed.wrong.join('\n ')}`}`)// a package whose connection drops (see runDropped)const drops = await page.evaluate((id) => window.runDropped(id), ids[0])const dropsOk = /^played/.test(drops.once) && dropped.once === 2 && /could not load/.test(drops.always) && !/Failed to fetch/.test(drops.always) && dropped.always === 2@@ -121,8 +140,9 @@ try {const life = await page.evaluate((id) => window.runLifecycle(id), ids[0])const lifeOk = /disposed/.test(life.load) && life.model === null && life.drawn === 0 && life.freed >= life.made && life.glError === 0&& life.driver === 1 && !!life.again && life.mic === 'ended' && life.supported === true+ && Object.values(life.left).every((n) => n <= 0)if (!lifeOk) failed++- console.log(`${lifeOk ? 'ok ' : 'FAIL'} dispose while running: load "${life.load}", ${life.made} textures made, ${life.freed} freed, drawn ${life.drawn}, GL error ${life.glError}, driver disposed ${life.driver}, a new player ${life.again}, microphone ${life.mic}`)+ console.log(`${lifeOk ? 'ok ' : 'FAIL'} dispose while running: load "${life.load}", ${life.made} textures made, ${life.freed} freed, drawn ${life.drawn}, GL error ${life.glError}, driver disposed ${life.driver}, a new player ${life.again}, microphone ${life.mic}, left on the GPU after 3 loads and dispose ${JSON.stringify(life.left)}`)// bodies: a PNGTuber shows each picture and follows the drivers; a VRM loads, draws and takes its expressionsawait page.goto(`http://127.0.0.1:${server.address().port}/test/bodies.html`)@@ -170,6 +190,20 @@ try {if (!vaOk) failed++console.log(`${vaOk ? 'ok ' : 'FAIL'} VRM, loaded again on a wide canvas: aspect ${vrmAgain?.aspect1}, ${vrmAgain?.aspect2}; a failed load kept ${vrmAgain?.kept}, covers ${(vrmAgain?.share * 100).toFixed(1)} %; on a 2D canvas: ${vrmOn2d}`)if (errors.length) { failed++; console.log(`FAIL page errors:\n ${errors.join('\n ')}`) }+ // a camera that is refused: the face model's download stops with it, unless someone asked for the model ahead of time+ await page.goto(`http://127.0.0.1:${server.address().port}/test/bodies.html`)+ await page.waitForFunction(() => typeof window.runBodies === 'function')+ const refused = await page.evaluate(() => window.runRefused())+ await new Promise((r) => setTimeout(r, 1500))+ const refusedOk = refused === 'NotAllowedError' && slowModel.finished === 0+ if (!refusedOk) failed+++ console.log(`${refusedOk ? 'ok ' : 'FAIL'} camera refused (${refused}): the face model asked ${slowModel.asked} time(s), finished ${slowModel.finished}`)+ await page.evaluate(() => window.runRefused(true)).catch(() => {})+ await new Promise((r) => setTimeout(r, 2000))+ const aheadOk = slowModel.finished === 1+ if (!aheadOk) failed+++ console.log(`${aheadOk ? 'ok ' : 'FAIL'} camera refused after a preload: the download goes on (finished ${slowModel.finished})`)+// the example pages: the cover in the canvas's box until the character is ready, then the character over itfor (const name of ['index', 'demo', 'gallery']) {const p = await browser.newPage()