A lightweight 2.5D character player for the web, with lip sync and webcam face tracking.
Fix face-tracking roll direction while preserving yaw
neverbiasuc08382cc6be5
diff --git a/src/runtime/tracking.ts b/src/runtime/tracking.tsindex fbafafe..584b833 100644--- a/src/runtime/tracking.ts+++ b/src/runtime/tracking.ts@@ -279,7 +279,9 @@ export class FaceTracker {const x = this.f.x.filter(this.raw.x, dt) - this.zero.xthis.target.angleX = clamp(x, -30, 30)this.target.angleY = clamp(this.f.y.filter(this.raw.y, dt) - this.zero.y, -30, 30)- this.target.angleZ = clamp(this.f.z.filter(this.raw.z, dt) - this.zero.z, -30, 30)+ // Player roll is Y-down, opposite to MediaPipe's Y-up rotation. Convert after calibration so stored poses keep+ // their existing convention; yaw and horizontal position are unchanged.+ this.target.angleZ = clamp(this.zero.z - this.f.z.filter(this.raw.z, dt), -30, 30)this.target.bodyX = clamp(x * 0.2 + this.pos.x * 0.25 * this.follow, -10, 10)}if (b) {diff --git a/test/tracking.test.ts b/test/tracking.test.tsnew file mode 100644index 0000000..1e9cebc--- /dev/null+++ b/test/tracking.test.ts@@ -0,0 +1,84 @@+// Numerical regression tests for the camera-matrix -> player-parameter boundary, not a substitute for real-camera checks.+import assert from 'node:assert/strict'+import { describe, it, type TestContext } from 'node:test'+import { FaceTracker } from '../src/runtime/tracking'++// MediaPipe's column-major, Y-up face transform: Rz(roll) Ry(yaw), with no pitch.+// Positive roll moves the top of the face toward camera-image left. Positive player angleZ moves it screen-right+// (the player's model coordinates are Y-down), so roll and yaw need different signs at the parameter boundary.+function matrix(yaw: number, roll: number) {+ const y = yaw * Math.PI / 180, z = roll * Math.PI / 180+ const cy = Math.cos(y), sy = Math.sin(y), cz = Math.cos(z), sz = Math.sin(z)+ return [cz * cy, sz * cy, -sy, 0, -sz, cz, 0, 0, cz * sy, sz * sy, cy, 0, 0, 0, -45, 1]+}++function camera(t: TestContext, mirror: boolean) {+ // Only the video element and optional light-check canvas need a DOM; no camera, model or GPU is started here.+ const previous = Object.getOwnPropertyDescriptor(globalThis, 'document')+ Object.defineProperty(globalThis, 'document', { configurable: true, value: {+ createElement: (tag: string) => tag === 'video' ? { readyState: 2, currentTime: 0 } : { getContext: () => null },+ } })+ t.after(() => {+ if (previous) Object.defineProperty(globalThis, 'document', previous)+ else Reflect.deleteProperty(globalThis, 'document')+ })+ let now = 0, pose = matrix(0, 0)+ t.mock.method(performance, 'now', () => now)+ const tracker = new FaceTracker()+ tracker.setMirror(mirror)+ tracker.lm = { detectForVideo: () => ({+ faceLandmarks: [[]], facialTransformationMatrixes: [{ data: pose }], faceBlendshapes: [],+ }) } as unknown as NonNullable<FaceTracker['lm']>+ const read = (yaw: number, roll: number, frames = 1) => {+ pose = matrix(yaw, roll)+ for (let i = 0; i < frames; i++) {+ now += 1000 / 30+ Object.assign(tracker.video, { currentTime: now / 1000 })+ assert.equal(tracker.poll(), true)+ }+ return { ...tracker.target }+ }+ return { tracker, read }+}++const near = (value: number | undefined, expected: number, tolerance = 1e-6) =>+ assert.ok(value !== undefined && Math.abs(value - expected) < tolerance, `${value} ≈ ${expected}`)++describe('camera roll mapping', () => {+ for (const mirror of [true, false]) {+ for (const direction of [-1, 1]) {+ it(`mirror=${mirror}, rotation sign ${direction}: roll has the screen-coordinate sign; yaw is preserved`, (t) => {+ const { read } = camera(t, mirror)+ const p = read(direction * 10, direction * 20)+ near(p.angleX, (mirror ? -13 : 13) * direction)+ near(p.bodyX, (mirror ? -2.6 : 2.6) * direction)+ near(p.angleZ, (mirror ? 20 : -20) * direction)+ })+ }++ it(`mirror=${mirror}: a nonzero calibrated pose remains centered, then tracks relative roll and yaw`, (t) => {+ const { tracker, read } = camera(t, mirror)+ read(4, 10, 20)+ tracker.calibrate()+ const centered = read(4, 10)+ near(centered.angleZ, 0)+ near(centered.angleX, 0)+ const moved = read(14, 22, 90)+ near(moved.angleX, mirror ? -13 : 13, 0.1)+ near(moved.angleZ, mirror ? 12 : -12, 0.1)+ })++ it(`mirror=${mirror}: toggling mirror keeps the calibrated center and reverses relative roll and yaw`, (t) => {+ const { tracker, read } = camera(t, mirror)+ read(4, 10, 20)+ tracker.calibrate()+ tracker.setMirror(!mirror)+ const centered = read(4, 10)+ near(centered.angleZ, 0)+ near(centered.angleX, 0)+ const moved = read(14, 22, 90)+ near(moved.angleX, mirror ? 13 : -13, 0.1)+ near(moved.angleZ, mirror ? -12 : 12, 0.1)+ })+ }+})