Turn one illustration into an animated 2D mesh avatar with a coding agent and a local editor
Port mesh deformation, physics and sprites to the native core
shinshin86360de2e8440f
diff --git a/native/conformance/deformation.mjs b/native/conformance/deformation.mjsnew file mode 100644index 0000000..a9438cd--- /dev/null+++ b/native/conformance/deformation.mjs@@ -0,0 +1,144 @@+import { writeFile } from 'node:fs/promises';+import { PARAMS, createRig } from '../../packages/runtime/src/engine/rig.js';+import { createPhysics } from '../../packages/runtime/src/engine/physics.js';+import { createSpriteModule } from '../../packages/runtime/src/engine/sprites.js';++const defaults = Object.fromEntries(PARAMS.map(p => [p.id, p.def]));+const poses = [+ {}, ...[-30, -15, 15, 30].map(angleX => ({ angleX })),+ ...[-15, 15].map(angleY => ({ angleY })), ...[-15, 15].map(angleZ => ({ angleZ })),+ { angleX: -20, angleY: 10, angleZ: 15, bodyAngleZ: 8 }, { bodyAngleX: 10, breath: 1 },+ { browY: 1, browAngle: 1, mouthOpen: 1 }, { handAngle: 10, armAngle: 10, fingerTap: 1 },+ { angleX: 20, angleY: -10, eyeLOpen: 0, eyeROpen: 0.5, eyeSmile: 1 },+];+const parts = ['ball', 'low', 'crease', 'lash'];+const alphaOf = image => Uint8Array.from({ length: image.width * image.height }, (_, i) => image.data[i * 4 + 3]);+const json = (directory, name, value) => writeFile(new URL(name, directory), JSON.stringify(value) + '\n');++function deformTassel(ch, mesh) {+ const [px, py] = ch.t.pivot, d = ch.dir, P = ch.pts;+ const sstep = (a, b, x) => { const t = Math.min(1, Math.max(0, (x - a) / (b - a))); return t * t * (3 - 2 * t); };+ const c1 = Math.cos(ch.phi[0]), s1 = Math.sin(ch.phi[0]), c2 = Math.cos(ch.phi[1]), s2 = Math.sin(ch.phi[1]);+ const out = new Float32Array(mesh.rest.length);+ for (let i = 0; i < mesh.rest.length; i += 2) {+ const vx = mesh.rest[i] - px, vy = mesh.rest[i + 1] - py;+ const along = vx * d[0] + vy * d[1];+ const ax = P[0][0] + vx * c1 - vy * s1, ay = P[0][1] + vx * s1 + vy * c1;+ const ux = vx - d[0] * ch.L1, uy = vy - d[1] * ch.L1;+ const bx = P[1][0] + ux * c2 - uy * s2, by = P[1][1] + ux * s2 + uy * c2;+ const w = sstep(ch.L1 * 0.55, ch.L1 * 1.35, along);+ out[i] = ax + (bx - ax) * w; out[i + 1] = ay + (by - ay) * w;+ }+ return Array.from(out);+}++export async function dumpDeformation({ rig, engine, grids, images, sprites, buildGrid, destination }) {+ const fixed = { gain: 0.8, bunL: [1.5, -2], bunR: [-3, 1],+ strands: rig.strands.map((_, i) => [[0, 0], [i * 0.2, -0.5], [1, i * 0.1], [-1.3, 2]]) };+ const points = [];+ for (let y = 0; y <= rig.image.height; y += 7) for (let x = 0; x <= rig.image.width; x += 7) points.push([x, y]);+ for (const { mesh } of grids) for (let i = 0; i < mesh.rest.length; i += 2) points.push([mesh.rest[i], mesh.rest[i + 1]]);+ const hair = images['hairmask.png'];+ const weights = points.map(([x, y]) => engine.baseWeights(x, y,+ hair.data[(Math.min(hair.height - 1, Math.round(y)) * hair.width + Math.min(hair.width - 1, Math.round(x))) * 4] / 255));+ const hands = points.map(([x, y]) => engine.handWeights(x, y));+ const files = [];+ const tmp = [0, 0];+ for (const [index, pose] of poses.entries()) for (const gain of [0, fixed.gain]) {+ const P = { ...defaults, ...pose }, phys = { ...fixed, gain }, frame = engine.handFrame(P);+ const base = [], hand = [], eyes = [];+ for (const [i, [x, y]] of points.entries()) {+ base.push(...engine.deformBase(x, y, weights[i], P, phys, tmp));+ hand.push(...engine.deformHand(x, y, hands[i], P, frame, tmp));+ }+ for (const [eye, e] of rig.eyes.entries()) for (const part of parts) {+ const open = eye === 0 ? P.eyeROpen : P.eyeLOpen;+ const positions = points.flatMap(([x, y], i) => Array.from(engine.deformBase(x,+ engine.eyePartY(e, part, x, y, open, P.eyeSmile), weights[i], P, phys, tmp)));+ eyes.push({ eye, part, alpha: engine.eyePartAlpha(part, open, P.eyeSmile), positions });+ }+ const file = `pose-${index}-${gain === 0 ? 'zero' : 'fixed'}.json`;+ await json(destination, file, { parameters: P, physics: phys, base, hand, eyes });+ files.push(file);+ }+ await json(destination, 'deformation.json', { version: 1, parameters: PARAMS, points, files });++ const { Physics } = createPhysics(engine, rig);+ const traces = [];+ for (const gain of [0, 0.8, 1, 2]) {+ const physics = new Physics(); physics.gain = gain;+ const frames = [];+ for (let frame = 0; frame < 120; frame++) {+ const P = { ...defaults, angleX: Math.sin(frame / 20) * 30, angleZ: Math.cos(frame / 15) * 15 };+ const dt = frame === 0 ? 0 : frame === 60 ? 0.08 : 1 / 60;+ const out = structuredClone(physics.step(P, dt));+ const chains = physics.chains.map(ch => ({ name: ch.t.name, points: ch.pts, phi: ch.phi, anchor: ch.anchor,+ positions: deformTassel(ch, grids.find(grid => grid.name === ch.t.name).mesh) }));+ frames.push({ parameters: P, dt, out, chains });+ }+ traces.push({ gain, frames });+ }+ await json(destination, 'physics.json', { version: 1, traces });++ const spriteImages = Object.fromEntries(Object.keys(sprites.layers).map(name => [name, images[`sprites/${name}.png`]]));+ const { createSprites } = createSpriteModule(engine, rig);+ const spriteFiles = [];+ for (const gain of [0, 0.8]) {+ const items = new Map();+ const renderer = { addLayer(name, _image, mesh) { const item = { name, mesh, visible: true, alpha: 1 }; items.set(name, item); return item; } };+ const current = createSprites(renderer, sprites, spriteImages, buildGrid, alphaOf);+ const updates = [];+ const update = (parameters, dt, mood = null) => {+ const phys = { ...fixed, gain };+ const coverage = current.update(parameters, phys, dt, mood);+ updates.push({ parameters, physics: phys, dt, mood, coverage,+ items: [...items.values()].map(item => ({ name: item.name, visible: item.visible, alpha: item.alpha,+ positions: item.visible ? Array.from(item.mesh.pos) : null })) });+ };+ for (const pose of poses) for (const mouthForm of [-1, -0.3, 0, 0.5, 1]) for (const mouthOpen of [0.25, 0.8]) {+ for (const eyeOpen of [0, 0.5, 1]) update({ ...defaults, ...pose, mouthForm, mouthOpen,+ eyeLOpen: eyeOpen, eyeROpen: eyeOpen, eyeSmile: eyeOpen === 0 ? (mouthForm > 0 ? 1 : 0) : 0 }, 0.1);+ }+ for (const pose of poses) for (const mood of ['angry', 'sad', 'surprise']) {+ for (const eyeOpen of [0, 0.5, 1]) update({ ...defaults, ...pose, eyeLOpen: eyeOpen, eyeROpen: eyeOpen,+ mouthForm: 0, mouthOpen: mood === 'sad' ? 0 : 0.18 }, 0.1, mood);+ }+ // Cross-fades in both directions, paused redraws, asymmetric eyes and mouth hysteresis.+ for (const mood of [null, 'sad', 'surprise', 'angry']) for (const eyeOpen of [1, 0.5, 0, 1.25, 0.2]) {+ for (const dt of [1 / 120, 1 / 120, 0, 1 / 60]) update({ ...defaults, eyeROpen: eyeOpen,+ eyeLOpen: 1 - Math.min(1, eyeOpen), eyeSmileL: 1, mouthOpen: 0.17, mouthForm: 0.32 }, dt, mood);+ }+ for (const mouthForm of [-0.56, -0.5, -0.48, -0.19, -0.25, -0.12, 0.36, 0.3, 0.43, 0.74, 0.8, 0.83]) {+ update({ ...defaults, mouthOpen: 0.8, mouthForm }, 1 / 60);+ }+ const file = `sprites-${gain === 0 ? 'zero' : 'fixed'}.json`;+ await json(destination, file, { version: 1, updates }); spriteFiles.push(file);+ }+ await json(destination, 'sprites.json', { version: 1, files: spriteFiles });+ const variants = [];+ for (const kind of ['absent-features', 'resampled-strands']) {+ const variant = structuredClone(rig);+ if (kind === 'absent-features') {+ for (const field of ['buns', 'strands', 'accessories', 'hand']) delete variant[field];+ } else {+ variant.strands.forEach((s, i) => {+ if (i % 3 === 0) s.nodes = [s.nodes[0], s.nodes.at(-1)];+ else if (i % 3 === 1) s.nodes.splice(1, 1);+ else s.nodes.splice(1, 0, s.nodes[0].map((v, a) => (v + s.nodes[1][a]) / 2));+ });+ }+ const engine = createRig(variant), { Physics } = createPhysics(engine, variant), physics = new Physics();+ const weights = [[440, 450, 0], [600, 500, 1], [600, 350, 1], [200, 900, 0.5]].map(([x, y, hair]) =>+ ({ x, y, hair, weights: engine.baseWeights(x, y, hair) }));+ const frames = [];+ for (let frame = 0; frame < 30; frame++) {+ const parameters = { ...defaults, angleX: Math.sin(frame / 20) * 30, angleY: -12, angleZ: Math.cos(frame / 15) * 15 };+ const out = structuredClone(physics.step(parameters, 1 / 60));+ const positions = weights.map(({ x, y, weights }) => Array.from(engine.deformBase(x, y, weights, parameters, out, tmp)));+ frames.push({ parameters, dt: 1 / 60, out, positions });+ }+ variants.push({ rig: variant, weights, frames });+ }+ await json(destination, 'variants.json', { version: 1, variants });+ console.log(`Wrote ${files.length} deformation poses (${points.length} points each), ${traces.length} physics traces and sprite sequences`);+}diff --git a/native/conformance/dump.mjs b/native/conformance/dump.mjsindex 884899f..e10a470 100644--- a/native/conformance/dump.mjs+++ b/native/conformance/dump.mjs@@ -4,6 +4,7 @@ import { readFile, mkdir, writeFile } from 'node:fs/promises';import { registerHooks } from 'node:module';import { fileURLToPath } from 'node:url';import { PNG } from 'pngjs';+import { dumpDeformation } from './deformation.mjs';// Match the runtime's bundler-style extensionless lighting imports in plain Node.const hooks = registerHooks({ resolve(specifier, context, next) {@@ -25,14 +26,17 @@ const json = async path => JSON.parse(await readFile(new URL(path, sample), 'utfconst rig = parseRig(await json('rig.json'));const metadata = await json('built/layers.json');const sprites = await json('built/sprites/sprites.json');-const { buildGrid } = createRenderer(createRig(rig), rig);+const engine = createRig(rig);+const { buildGrid } = createRenderer(engine, rig);const plain = mesh => Object.fromEntries(Object.entries(mesh).map(([key, value]) =>[key, ArrayBuffer.isView(value) ? Array.from(value) : value]));const grids = [];+const images = {};for (const [sprite, rectangles] of [[false, { base: [0, 0, rig.image.width, rig.image.height], ...metadata.layers }], [true, sprites.layers]]) {for (const [name, rect] of Object.entries(rectangles)) {const texture = `${sprite ? 'sprites/' : ''}${name}.png`;const image = PNG.sync.read(await readFile(new URL(`built/${texture}`, sample)));+ images[texture] = image;if (image.width !== rect[2] || image.height !== rect[3]) throw new Error(`PNG dimensions do not match ${texture}`);const cell = sprite ? rig.mesh.spriteCell : name === 'base' ? rig.mesh.baseCell: name === 'hand' ? rig.mesh.handCell : name.startsWith('tassel') ? rig.mesh.tasselCell@@ -58,3 +62,5 @@ const destination = new URL('../target/conformance/', import.meta.url);await mkdir(destination, { recursive: true });await writeFile(new URL('grids.json', destination), JSON.stringify({ version: 1, grids, cases }) + '\n');console.log(`Wrote ${grids.length} layer/sprite grids and ${cases.length} edge cases to ${fileURLToPath(new URL('grids.json', destination))}`);+images['hairmask.png'] = PNG.sync.read(await readFile(new URL('built/hairmask.png', sample)));+await dumpDeformation({ rig, engine, grids, images, sprites, buildGrid, destination });diff --git a/native/crates/mesh-avatar-core/src/engine/mod.rs b/native/crates/mesh-avatar-core/src/engine/mod.rsindex f7a1b46..8e28625 100644--- a/native/crates/mesh-avatar-core/src/engine/mod.rs+++ b/native/crates/mesh-avatar-core/src/engine/mod.rs@@ -1 +1,4 @@pub mod grid;+pub mod physics;+pub mod rig;+pub mod sprites;diff --git a/native/crates/mesh-avatar-core/src/engine/physics.rs b/native/crates/mesh-avatar-core/src/engine/physics.rsnew file mode 100644index 0000000..892f2d1--- /dev/null+++ b/native/crates/mesh-avatar-core/src/engine/physics.rs@@ -0,0 +1,339 @@+//! Secondary hair motion and two-joint tassel pendulums.+use super::{+ grid::Grid,+ rig::{rotate, sstep, Parameters, PhysicsOffsets, RigEngine},+};+use crate::rig::{Accessory, Point, Rig, Strand};+use anyhow::{Context, Result};++#[derive(Clone)]+struct StrandState {+ config: Strand,+ nodes: [Point; 4],+ p: Option<[Point; 4]>,+ v: [Point; 4],+}+struct BunState {+ index: usize,+ at: Point,+ p: Option<Point>,+ v: Point,+}+#[derive(Debug, Clone)]+struct Pendulum {+ th: Point,+ w: Point,+ pos: Point,+ vel: Point,+ acc: Point,+}++#[derive(Debug, Clone)]+pub struct TasselChain {+ pub accessory: Accessory,+ pub dir: Point,+ pub length1: f64,+ pub length2: f64,+ pub points: Option<[Point; 3]>,+ pub phi: Point,+ pub anchor: Point,+ state: Option<Pendulum>,+}++impl TasselChain {+ pub fn deform_grid(&self, mesh: &mut Grid) -> Result<()> {+ let points = self+ .points+ .context("step physics before deforming a tassel")?;+ let [px, py] = self.accessory.pivot;+ let [c1, s1] = [self.phi[0].cos(), self.phi[0].sin()];+ let [c2, s2] = [self.phi[1].cos(), self.phi[1].sin()];+ for (r, o) in mesh+ .rest+ .as_chunks::<2>()+ .0+ .iter()+ .zip(mesh.pos.as_chunks_mut::<2>().0)+ {+ let vx = f64::from(r[0]) - px;+ let vy = f64::from(r[1]) - py;+ let along = vx * self.dir[0] + vy * self.dir[1];+ let ax = points[0][0] + vx * c1 - vy * s1;+ let ay = points[0][1] + vx * s1 + vy * c1;+ let ux = vx - self.dir[0] * self.length1;+ let uy = vy - self.dir[1] * self.length1;+ let bx = points[1][0] + ux * c2 - uy * s2;+ let by = points[1][1] + ux * s2 + uy * c2;+ let w = sstep(self.length1 * 0.55, self.length1 * 1.35, along);+ *o = [(ax + (bx - ax) * w) as f32, (ay + (by - ay) * w) as f32];+ }+ Ok(())+ }+}++pub struct Physics {+ pub gain: f64,+ pub out: PhysicsOffsets,+ pub chains: Vec<TasselChain>,+ pub time: f64,+ strands: Vec<StrandState>,+ buns: Vec<BunState>,+ fringe: usize,+}++fn hair_anchor(engine: &RigEngine<'_>, at: Point, p: &Parameters) -> Point {+ let mut point = at;+ engine.apply_head(&mut point, 1.0, p, 1.0);+ engine.apply_body(&mut point, at[1], p, 0.0, 1.0, 0.0);+ point+}++fn soft_limit(v: Point, m: f64) -> Point {+ let l = v[0].hypot(v[1]);+ if l < 1e-6 {+ return v;+ }+ let k = m * (l / m).tanh() / l;+ [v[0] * k, v[1] * k]+}++impl Physics {+ pub fn new(rig: &Rig) -> Self {+ let strands: Vec<_> = rig+ .strands+ .iter()+ .map(|s| {+ let nodes = std::array::from_fn(|i| {+ if s.nodes.len() == 4 {+ return s.nodes[i];+ }+ let f = i as f64 / 3.0 * (s.nodes.len() - 1) as f64;+ let k = (f.floor() as usize).min(s.nodes.len() - 2);+ let t = f - k as f64;+ std::array::from_fn(|a| s.nodes[k][a] + (s.nodes[k + 1][a] - s.nodes[k][a]) * t)+ });+ StrandState {+ config: s.clone(),+ nodes,+ p: None,+ v: [[0.0; 2]; 4],+ }+ })+ .collect();+ let fringe = rig+ .strands+ .iter()+ .filter(|s| s.name.starts_with("bang"))+ .count()+ .min(6);+ let mut buns = vec![];+ if let Some(b) = &rig.buns {+ for (index, e) in [&b.bun_l, &b.bun_r].into_iter().enumerate() {+ if let Some(e) = e {+ buns.push(BunState {+ index,+ at: [e.cx, e.cy],+ p: None,+ v: [0.0; 2],+ });+ }+ }+ }+ let chains = rig+ .accessories+ .iter()+ .map(|t| {+ let dx = t.tip[0] - t.pivot[0];+ let dy = t.tip[1] - t.pivot[1];+ let len = dx.hypot(dy);+ TasselChain {+ accessory: t.clone(),+ dir: [dx / len, dy / len],+ length1: len * t.split,+ length2: len * (1.0 - t.split),+ points: None,+ phi: [0.0; 2],+ anchor: [0.0; 2],+ state: None,+ }+ })+ .collect();+ Self {+ gain: 1.0,+ out: PhysicsOffsets {+ strands: vec![[[0.0; 2]; 4]; strands.len()],+ ..Default::default()+ },+ chains,+ time: 0.0,+ strands,+ buns,+ fringe,+ }+ }++ /// dt is finite, non-negative seconds. Integration substeps and the 0.05-second+ /// cap follow the JS engine; the caller owns elapsed time.+ pub fn step(&mut self, engine: &RigEngine<'_>, p: &Parameters, dt: f64) -> &PhysicsOffsets {+ assert!(+ dt.is_finite() && dt >= 0.0,+ "dt must be finite and non-negative"+ );+ let sub = ((dt / (1.0 / 240.0)).ceil() as usize).max(1);+ let h = dt.min(0.05) / sub as f64;+ let roll = -p.angle_z / 30.0 * engine.rig.head.max_roll+ - p.body_angle_z / 10.0 * engine.rig.body.max_roll;+ let g = self.gain;+ for (si, st) in self.strands.iter_mut().enumerate() {+ let target = st.nodes.map(|n| hair_anchor(engine, n, p));+ let positions = st.p.get_or_insert(target);+ for i in 0..sub {+ let time = self.time + i as f64 * h;+ for (j, &stiffness) in [0.0, 300.0, 200.0, 140.0].iter().enumerate().skip(1) {+ let hang =+ (st.nodes[j][0] - st.nodes[0][0]).hypot(st.nodes[j][1] - st.nodes[0][1]);+ let wind = ((time * 1.3).sin() * 0.5+ + (time * 3.1).sin() * 0.3+ + (time * 4.7 + si as f64 * 1.7).sin() * 0.2)+ * st.config.max+ * 0.1+ * j as f64+ / 3.0;+ let parent = [+ positions[j - 1][0] - target[j - 1][0],+ positions[j - 1][1] - target[j - 1][1],+ ];+ let tx =+ target[j][0] + parent[0] * 0.9 * 0.85 - roll.sin() * hang * 0.12 + wind;+ let ty = target[j][1] + parent[1] * 0.9 * 0.85;+ let k = stiffness * st.config.k * 1.7 / g.max(0.35);+ let c = 7.0 * 1.6 * 1.7_f64.sqrt() / g.sqrt().max(0.5);+ let v = &mut st.v[j];+ let q = &mut positions[j];+ v[0] += (k * (tx - q[0]) - c * v[0]) * h;+ v[1] += (k * (ty - q[1]) - c * v[1]) * h;+ q[0] += v[0] * h;+ q[1] += v[1] * h;+ }+ positions[0] = target[0];+ }+ for j in 0..4 {+ let o = [+ positions[j][0] - target[j][0],+ positions[j][1] - target[j][1],+ ];+ self.out.strands[si][j] = soft_limit(+ o,+ (st.config.max * 0.9 * g.max(1.0) * j as f64 / 3.0).max(0.5),+ );+ }+ }+ for s in &mut self.buns {+ let a = hair_anchor(engine, s.at, p);+ let wind = (self.time * 1.3 + s.at[0] * 0.01).sin() * 0.6+ + (self.time * 2.9 + s.at[1] * 0.02).sin() * 0.4;+ let tx = a[0] - roll.sin() * 3.0 + wind * 4.0 * 0.12;+ let ty = a[1];+ let pos = s.p.get_or_insert([tx, ty]);+ let k = 260.0 / g.max(0.35);+ let c = 11.0 / g.sqrt().max(0.5);+ for _ in 0..sub {+ s.v[0] += (k * (tx - pos[0]) - c * s.v[0]) * h;+ s.v[1] += (k * (ty - pos[1]) - c * s.v[1]) * h;+ pos[0] += s.v[0] * h;+ pos[1] += s.v[1] * h;+ }+ let mut o = [pos[0] - a[0], pos[1] - a[1]];+ let l = o[0].hypot(o[1]);+ let m = 4.0 * g.max(1.0);+ if l > m {+ o[0] *= m / l;+ o[1] *= m / l;+ }+ if s.index == 0 {+ self.out.bun_l = o;+ } else {+ self.out.bun_r = o;+ }+ }+ self.out.gain = 1.0;+ for _ in 0..4 {+ for j in 1..4 {+ let snap: Vec<_> = self+ .out+ .strands+ .iter()+ .take(self.fringe)+ .map(|o| o[j])+ .collect();+ for i in 0..self.fringe {+ let l = snap[i.saturating_sub(1)];+ let r = snap[(i + 1).min(self.fringe - 1)];+ for a in 0..2 {+ self.out.strands[i][j][a] = snap[i][a] * 0.5 + (l[a] + r[a]) * 0.25;+ }+ }+ }+ }+ for ch in &mut self.chains {+ let anchor = engine.tassel_anchor(&ch.accessory, p, &self.out);+ let st = ch.state.get_or_insert(Pendulum {+ th: [0.0; 2],+ w: [0.0; 2],+ pos: anchor,+ vel: [0.0; 2],+ acc: [0.0; 2],+ });+ let vel = [+ (anchor[0] - st.pos[0]) / dt.max(1e-3),+ (anchor[1] - st.pos[1]) / dt.max(1e-3),+ ];+ let kf = 1.0 - (-dt * 6.0).exp();+ let acc = [+ (vel[0] - st.vel[0]) / dt.max(1e-3),+ (vel[1] - st.vel[1]) / dt.max(1e-3),+ ];+ for a in 0..2 {+ st.acc[a] += (acc[a] - st.acc[a]) * kf;+ st.vel[a] += (vel[a] - st.vel[a]) * kf;+ }+ st.pos = anchor;+ let normal = [-ch.dir[1], ch.dir[0]];+ let push =+ -(st.acc[0] * normal[0] + st.acc[1] * normal[1]) / (ch.length1 + ch.length2) * 0.32;+ let wind = ((self.time * 0.9 + ch.accessory.pivot[0]).sin() * 0.6+ + (self.time * 0.37 + ch.accessory.pivot[1]).sin() * 0.4)+ * 0.03;+ let target = -roll * 0.85 + wind;+ let w1 = 2.0 * std::f64::consts::PI * 0.7 / g.max(0.4).sqrt();+ let w2 = w1 * 1.35;+ for _ in 0..sub {+ let a1 = -w1 * w1 * (st.th[0] - target) - 2.0 * 0.22 * w1 * st.w[0] + push;+ st.w[0] += a1 * h;+ st.th[0] += st.w[0] * h;+ let a2 = -w2 * w2 * (st.th[1] - st.th[0]) - 2.0 * 0.22 * w2 * st.w[1]+ + push * 0.4+ + a1 * 0.5;+ st.w[1] += a2 * h;+ st.th[1] += st.w[1] * h;+ }+ st.th[0] = st.th[0].clamp(-0.55, 0.55);+ st.th[1] = st.th[1].clamp(-0.55 * 1.4, 0.55 * 1.4);+ ch.phi = st.th;+ let d1 = rotate(ch.dir, [0.0; 2], st.th[0].cos(), st.th[0].sin());+ let d2 = rotate(ch.dir, [0.0; 2], st.th[1].cos(), st.th[1].sin());+ let p1 = [+ anchor[0] + d1[0] * ch.length1,+ anchor[1] + d1[1] * ch.length1,+ ];+ ch.points = Some([+ anchor,+ p1,+ [p1[0] + d2[0] * ch.length2, p1[1] + d2[1] * ch.length2],+ ]);+ ch.anchor = anchor;+ }+ self.time += dt;+ &self.out+ }+}diff --git a/native/crates/mesh-avatar-core/src/engine/rig.rs b/native/crates/mesh-avatar-core/src/engine/rig.rsnew file mode 100644index 0000000..f16b1ec--- /dev/null+++ b/native/crates/mesh-avatar-core/src/engine/rig.rs@@ -0,0 +1,439 @@+//! CPU weights and deformation, following engine/rig.js.+use crate::rig::{Accessory, Ellipse, Eye, Point, Rig};+use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, Copy, Serialize)]+pub struct ParameterDefinition {+ pub id: &'static str,+ pub label: &'static str,+ pub min: f64,+ pub max: f64,+ #[serde(rename = "def")]+ pub default: f64,+ pub group: &'static str,+}++macro_rules! parameters {+ ($($field:ident => ($id:literal, $label:literal, $min:expr, $max:expr, $default:expr, $group:literal)),* $(,)?) => {+ #[derive(Debug, Clone, Serialize, Deserialize)]+ #[serde(default)]+ pub struct Parameters {+ $(#[serde(rename = $id)] pub $field: f64,)*+ #[serde(rename = "eyeSmileL")]+ pub eye_smile_l: Option<f64>,+ }+ impl Default for Parameters {+ fn default() -> Self { Self { $($field: $default,)* eye_smile_l: None } }+ }+ pub const PARAMS: &[ParameterDefinition] = &[+ $(ParameterDefinition { id: $id, label: $label, min: $min, max: $max, default: $default, group: $group },)*+ ];+ }+}++parameters! {+ angle_x => ("angleX", "Face angle X", -30.0, 30.0, 0.0, "Head and body"),+ angle_y => ("angleY", "Face angle Y", -30.0, 30.0, 0.0, "Head and body"),+ angle_z => ("angleZ", "Face roll", -30.0, 30.0, 0.0, "Head and body"),+ body_angle_x => ("bodyAngleX", "Body angle X", -10.0, 10.0, 0.0, "Head and body"),+ body_angle_z => ("bodyAngleZ", "Body roll", -10.0, 10.0, 0.0, "Head and body"),+ breath => ("breath", "Breath", 0.0, 1.0, 0.0, "Head and body"),+ eye_l_open => ("eyeLOpen", "Left eye open", 0.0, 1.25, 1.0, "Eyes and brows"),+ eye_r_open => ("eyeROpen", "Right eye open", 0.0, 1.25, 1.0, "Eyes and brows"),+ eye_smile => ("eyeSmile", "Eye smile", 0.0, 1.0, 0.0, "Eyes and brows"),+ gaze_x => ("gazeX", "Gaze X", -1.0, 1.0, 0.0, "Eyes and brows"),+ gaze_y => ("gazeY", "Gaze Y", -1.0, 1.0, 0.0, "Eyes and brows"),+ brow_y => ("browY", "Brow Y", -1.0, 1.0, 0.0, "Eyes and brows"),+ brow_angle => ("browAngle", "Brow angle", -1.0, 1.0, 0.0, "Eyes and brows"),+ mouth_open => ("mouthOpen", "Mouth open", 0.0, 1.0, 0.0, "Mouth and cheeks"),+ mouth_form => ("mouthForm", "Mouth form (i to o)", -1.0, 1.0, 0.0, "Mouth and cheeks"),+ blush => ("blush", "Blush", 0.0, 1.0, 0.0, "Mouth and cheeks"),+ arm_angle => ("armAngle", "Arm angle", -10.0, 10.0, 0.0, "Hand"),+ hand_angle => ("handAngle", "Wrist angle", -10.0, 10.0, 0.0, "Hand"),+ finger_tap => ("fingerTap", "Finger tap", 0.0, 1.0, 0.0, "Hand"),+}++pub(crate) fn sstep(a: f64, b: f64, x: f64) -> f64 {+ let t = ((x - a) / (b - a)).clamp(0.0, 1.0);+ t * t * (3.0 - 2.0 * t)+}++fn gaussian(x: f64, y: f64, area: Option<&Ellipse>) -> f64 {+ area.map_or(0.0, |a| {+ (-(((x - a.cx) / a.rx).powi(2) + ((y - a.cy) / a.ry).powi(2))).exp()+ })+}++pub(crate) fn rotate(p: Point, origin: Point, c: f64, s: f64) -> Point {+ let dx = p[0] - origin[0];+ let dy = p[1] - origin[1];+ [origin[0] + dx * c - dy * s, origin[1] + dx * s + dy * c]+}++fn sampled(arr: &[f64; 24], u: f64) -> f64 {+ let f = u.clamp(0.0, 1.0) * 23.0;+ let i = (f.floor() as usize).min(22);+ arr[i] + (arr[i + 1] - arr[i]) * (f - i as f64)+}++#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum EyePart {+ Ball,+ Low,+ Crease,+ Lash,+}++fn eye_lids_raw(e: &Eye, u: f64, open: f64, smile: f64) -> Point {+ let u = u.clamp(0.0, 1.0);+ let top = sampled(&e.top, u);+ let bot = sampled(&e.bot, u);+ let b = 1.0 - open;+ let hump = 4.0 * u * (1.0 - u);+ let h = sampled(&e.bot, 0.5) - sampled(&e.top, 0.5);+ let arc = e.top[0] + (e.top[23] - e.top[0]) * u - 0.3 * h * hump;+ let closed = top.max(bot + 1.0 + (arc - bot - 1.0) * smile);+ [+ (closed - top) * b.max(-0.07),+ if b > 0.0 {+ (closed - bot) * b * smile+ } else {+ 0.0+ },+ ]+}++pub fn eye_part_y(e: &Eye, part: EyePart, x: f64, y: f64, open: f64, smile: f64) -> f64 {+ let mut du = 0.0;+ let mut db = 0.0;+ for (k, w) in [(-2.0, 1.0), (-1.0, 2.0), (0.0, 3.0), (1.0, 2.0), (2.0, 1.0)] {+ let [a, c] = eye_lids_raw(e, (x + k * 5.0 - e.x0) / (e.x1 - e.x0), open, smile);+ du += a * w / 9.0;+ db += c * w / 9.0;+ }+ let u = ((x - e.x0) / (e.x1 - e.x0)).clamp(0.0, 1.0);+ let top = sampled(&e.top, u);+ let lid = top + du;+ match part {+ EyePart::Lash => lid - (top - y) * (1.0 - 0.3 * (1.0 - open).clamp(0.0, 1.0)),+ EyePart::Low => y + (sampled(&e.bot, u) + db - sampled(&e.bot, u)),+ EyePart::Crease => y + (lid - top) * 0.22,+ EyePart::Ball => y,+ }+}++pub fn eye_part_alpha(part: EyePart, open: f64, smile: f64) -> f64 {+ let b = (1.0 - open).clamp(0.0, 1.0);+ match part {+ EyePart::Low => 1.0 - (b * smile * 1.2).min(1.0),+ EyePart::Crease => 1.0 - 0.45 * b,+ _ => 1.0,+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct BaseWeights {+ pub head: f64,+ pub turn: f64,+ pub strands: Vec<(usize, f64, f64)>,+ pub bun_l: f64,+ pub bun_r: f64,+ pub brow: f64,+ pub jaw: f64,+ pub nose: f64,+ pub mouth: f64,+ pub eye_a: f64,+ pub eye_b: f64,+ pub ear_r: f64,+ pub ear_l: f64,+ pub chest: f64,+ pub breath: f64,+ pub shoulder: f64,+}++#[derive(Debug, Clone, Default, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct HandWeights {+ pub arm: f64,+ pub wrist: f64,+ pub hand: f64,+ pub finger: f64,+ pub pin_bottom: f64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct PhysicsOffsets {+ pub gain: f64,+ pub bun_l: Point,+ pub bun_r: Point,+ #[serde(default)]+ pub strands: Vec<[Point; 4]>,+}++impl Default for PhysicsOffsets {+ fn default() -> Self {+ Self {+ gain: 1.0,+ bun_l: [0.0; 2],+ bun_r: [0.0; 2],+ strands: vec![],+ }+ }+}++#[derive(Debug, Clone)]+pub struct HandFrame {+ elbow: Point,+ translated_elbow: Point,+ c: f64,+ s: f64,+ pub w: Point,+ pub b: f64,+ pub k: f64,+}++impl HandFrame {+ pub fn transform(&self, point: Point) -> Point {+ let q = rotate(point, self.elbow, self.c, self.s);+ [+ q[0] + self.translated_elbow[0] - self.elbow[0],+ q[1] + self.translated_elbow[1] - self.elbow[1],+ ]+ }+}++pub struct RigEngine<'a> {+ pub rig: &'a Rig,+}++impl<'a> RigEngine<'a> {+ pub fn new(rig: &'a Rig) -> Self {+ Self { rig }+ }+ pub fn head_weight(&self, _x: f64, y: f64) -> f64 {+ let b = self.rig.head.weight_band;+ 1.0 - sstep(b[0], b[1], y)+ }+ pub fn turn_weight(&self, _x: f64, y: f64) -> f64 {+ let b = self.rig.head.turn_band;+ 1.0 - sstep(b[0], b[1], y)+ }++ pub fn base_weights(&self, x: f64, y: f64, hair: f64) -> BaseWeights {+ let r = self.rig;+ let mut hits = Vec::new();+ let mut sum = 0.0;+ if hair >= 0.02 {+ for (i, s) in r.strands.iter().enumerate() {+ let mut best = f64::INFINITY;+ let mut bt = 0.0;+ let n = s.nodes.len() - 1;+ for j in 0..n {+ let [ax, ay] = s.nodes[j];+ let vx = s.nodes[j + 1][0] - ax;+ let vy = s.nodes[j + 1][1] - ay;+ let u = (((x - ax) * vx + (y - ay) * vy) / (vx * vx + vy * vy)).clamp(0.0, 1.0);+ let d = (x - ax - vx * u).hypot(y - ay - vy * u);+ if d < best {+ best = d;+ bt = (j as f64 + u) / n as f64;+ }+ }+ let g = (-(best / s.sigma).powi(2)).exp();+ if g < 0.02 {+ continue;+ }+ hits.push((i, g, bt));+ sum += g;+ }+ }+ let norm = sum.max(1.0);+ for (_, weight, t) in &mut hits {+ *weight = hair * *weight / norm * sstep(0.04, 0.55, *t);+ }+ let g = |a: &Ellipse| gaussian(x, y, Some(a));+ BaseWeights {+ head: self.head_weight(x, y),+ turn: self.turn_weight(x, y),+ strands: hits,+ bun_l: gaussian(x, y, r.buns.as_ref().and_then(|b| b.bun_l.as_ref())),+ bun_r: gaussian(x, y, r.buns.as_ref().and_then(|b| b.bun_r.as_ref())),+ brow: g(&r.face.brow.ellipse)+ * (1.0 - sstep(r.face.brow.band[0], r.face.brow.band[1], y)),+ jaw: g(&r.face.jaw.ellipse) * sstep(r.face.jaw.band[0], r.face.jaw.band[1], y),+ nose: g(&r.face.nose),+ mouth: g(&r.face.mouth),+ eye_a: g(&r.face.eye_a),+ eye_b: g(&r.face.eye_b),+ ear_r: g(&r.face.ear_r),+ ear_l: g(&r.face.ear_l),+ chest: g(&r.body.chest),+ breath: 1.0 - sstep(r.body.breath_band[0], r.body.breath_band[1], y),+ shoulder: r.body.shoulders.iter().map(g).sum(),+ }+ }++ pub fn apply_head(&self, p: &mut Point, w: f64, params: &Parameters, w_turn: f64) {+ let h = &self.rig.head;+ let nx = (p[0] - h.ellipse.cx) / h.ellipse.rx;+ let ny = (p[1] - h.ellipse.cy) / h.ellipse.ry;+ let t = 1.0 - nx * nx - ny * ny;+ if t > 0.0 {+ let d = sstep(0.0, 0.84, t);+ p[0] += params.angle_x / 30.0 * h.shift_x * d * w_turn;+ p[1] += -params.angle_y / 30.0 * h.shift_y * d * w_turn;+ }+ if w <= 0.0 {+ return;+ }+ let a = -params.angle_z / 30.0 * h.max_roll * w;+ *p = rotate(*p, [h.pivot_x, h.pivot_y], a.cos(), a.sin());+ }++ pub fn apply_body(+ &self,+ p: &mut Point,+ rest_y: f64,+ params: &Parameters,+ chest: f64,+ breath_w: f64,+ shoulder: f64,+ ) {+ let body = &self.rig.body;+ p[1] -= params.breath * (5.0 * breath_w + 4.0 * shoulder);+ p[0] += (p[0] - body.chest.cx) * 0.012 * params.breath * chest;+ p[0] += params.body_angle_x / 10.0 * (7.0 + 9.0 * chest);+ let a = -params.body_angle_z / 10.0+ * body.max_roll+ * (1.0 - sstep(body.roll_band[0], body.roll_band[1], rest_y));+ *p = rotate(*p, [body.pivot_x, body.pivot_y], a.cos(), a.sin());+ }++ pub fn deform_base(+ &self,+ x: f64,+ y: f64,+ w: &BaseWeights,+ params: &Parameters,+ phys: &PhysicsOffsets,+ ) -> Point {+ let mut p = [x, y];+ for &(i, wi, t) in &w.strands {+ let Some(o) = phys.strands.get(i) else {+ continue;+ };+ let f = t * 3.0;+ let k = (f.floor() as usize).min(2);+ let u = f - k as f64;+ p[0] += (o[k][0] + (o[k + 1][0] - o[k][0]) * u) * wi * phys.gain;+ p[1] += (o[k][1] + (o[k + 1][1] - o[k][1]) * u) * wi * phys.gain;+ }+ p[0] += (phys.bun_l[0] * w.bun_l + phys.bun_r[0] * w.bun_r) * phys.gain;+ p[1] += (phys.bun_l[1] * w.bun_l + phys.bun_r[1] * w.bun_r) * phys.gain;+ p[1] -= params.brow_y * 7.0 * w.brow;+ if w.brow > 0.01 {+ p[1] += (x - self.rig.face.brow.ellipse.cx) * (params.brow_angle * 0.12 * w.brow).sin();+ }+ p[1] += params.mouth_open * 3.5 * w.jaw;+ let ax = params.angle_x / 30.0;+ let ay = params.angle_y / 30.0;+ let f = &self.rig.face;+ p[0] += ax * (8.0 * w.nose + 5.0 * w.mouth) + (x - f.eye_a.cx) * 0.12 * ax * w.eye_a+ - (x - f.eye_b.cx) * 0.12 * ax * w.eye_b+ - ax * 9.0 * w.ear_r+ + ax * 4.0 * w.ear_l+ - ax * 10.0 * (w.bun_l + w.bun_r);+ p[1] -= ay * (6.0 * w.nose + 3.0 * w.mouth);+ p[1] += (y - f.eye_a.cy) * -0.06 * ay.abs() * w.eye_a+ + (y - f.eye_b.cy) * -0.06 * ay.abs() * w.eye_b;+ self.apply_head(&mut p, w.head, params, w.turn);+ self.apply_body(&mut p, y, params, w.chest, w.breath, w.shoulder);+ p+ }++ pub fn hand_weights(&self, x: f64, y: f64) -> HandWeights {+ let Some(h) = &self.rig.hand else {+ return HandWeights::default();+ };+ let at = |b: Point, v| sstep(b[0], b[1], v);+ HandWeights {+ arm: 1.0 - at(h.arm_band, y),+ wrist: 1.0 - at(h.wrist_band, y),+ hand: 1.0 - at(h.hand_band, y),+ finger: at(h.finger_x_band, x) * (1.0 - at(h.finger_y_band, y)),+ pin_bottom: at(h.pin_band, y),+ }+ }++ pub fn hand_frame(&self, params: &Parameters) -> HandFrame {+ let h = self.rig.hand.as_ref();+ let elbow = h.map_or([0.0; 2], |h| h.elbow);+ let wrist = h.map_or([0.0; 2], |h| h.wrist);+ let contact = h.map_or([0.0; 2], |h| h.contact);+ let mut c = contact;+ self.apply_head(+ &mut c,+ self.head_weight(contact[0], contact[1]),+ params,+ self.turn_weight(contact[0], contact[1]),+ );+ self.apply_body(&mut c, contact[1], params, 0.0, 1.0, 0.0);+ let mut e = elbow;+ self.apply_body(&mut e, elbow[1], params, 0.0, 0.0, 0.0);+ let full =+ (c[1] - e[1]).atan2(c[0] - e[0]) - (contact[1] - elbow[1]).atan2(contact[0] - elbow[0]);+ let a = full * h.map_or(0.0, |h| h.forearm_share);+ let mut frame = HandFrame {+ elbow,+ translated_elbow: e,+ c: a.cos(),+ s: a.sin(),+ w: [0.0; 2],+ b: 0.0,+ k: 1.0,+ };+ frame.w = frame.transform(wrist);+ let c1 = frame.transform(contact);+ let w = frame.w;+ frame.b = (c[1] - w[1]).atan2(c[0] - w[0]) - (c1[1] - w[1]).atan2(c1[0] - w[0]);+ frame.k = ((c[0] - w[0]).hypot(c[1] - w[1]) / (c1[0] - w[0]).hypot(c1[1] - w[1]))+ .clamp(0.97, 1.03);+ frame+ }++ pub fn deform_hand(+ &self,+ x: f64,+ y: f64,+ w: &HandWeights,+ p: &Parameters,+ frame: &HandFrame,+ ) -> Point {+ let h = self.rig.hand.as_ref();+ let a = p.finger_tap * 0.07 * w.finger;+ let mut q = rotate([x, y], h.map_or([0.0; 2], |h| h.knuckle), a.cos(), a.sin());+ let ha = -p.hand_angle / 10.0 * 0.045 * w.wrist;+ let aa = -p.arm_angle / 10.0 * 0.015 * w.arm;+ q = rotate(q, h.map_or([0.0; 2], |h| h.wrist), ha.cos(), ha.sin());+ q = rotate(q, h.map_or([0.0; 2], |h| h.elbow), aa.cos(), aa.sin());+ q = frame.transform(q);+ let bw = frame.b * w.hand;+ let kw = 1.0 + (frame.k - 1.0) * w.hand;+ q = rotate(q, frame.w, bw.cos() * kw, bw.sin() * kw);+ q[1] += (y - q[1]) * w.pin_bottom;+ q+ }++ pub fn tassel_anchor(&self, t: &Accessory, p: &Parameters, phys: &PhysicsOffsets) -> Point {+ self.deform_base(+ t.pivot[0],+ t.pivot[1],+ &self.base_weights(t.pivot[0], t.pivot[1], 1.0),+ p,+ phys,+ )+ }+}diff --git a/native/crates/mesh-avatar-core/src/engine/sprites.rs b/native/crates/mesh-avatar-core/src/engine/sprites.rsnew file mode 100644index 0000000..a0e65e3--- /dev/null+++ b/native/crates/mesh-avatar-core/src/engine/sprites.rs@@ -0,0 +1,364 @@+//! Drawn eyes, vowel shapes, expression drawings and time-based cross-fades.+use super::{+ grid::{build_grid, Grid},+ rig::{sstep, BaseWeights, Parameters, PhysicsOffsets, RigEngine},+};+use crate::{+ format::{SpriteMetadata, Texture},+ rig::Rig,+};+use anyhow::{Context, Result};+use serde::{Deserialize, Serialize};+use std::collections::BTreeMap;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum MouthShape {+ A,+ AHalf,+ I,+ E,+ O,+ U,+}++impl MouthShape {+ fn definition(self) -> (&'static str, f64, f64) {+ match self {+ Self::A => ("mouth_a", 0.9, 1.0),+ Self::AHalf => ("mouth_a_half", 0.55, 1.0),+ Self::I => ("mouth_i", 0.55, 1.0),+ Self::E => ("mouth_a_half", 0.65, 1.06),+ Self::O => ("mouth_o", 0.75, 1.0),+ Self::U => ("mouth_o", 0.75, 0.78),+ }+ }+}++pub fn mouth_shape(open: f64, form: f64, prev: Option<MouthShape>) -> Option<MouthShape> {+ if open < 0.16 {+ return None;+ }+ let regions = [+ (MouthShape::I, -9.0, -0.55),+ (MouthShape::E, -0.55, -0.2),+ (MouthShape::A, -0.2, 0.35),+ (MouthShape::O, 0.35, 0.75),+ (MouthShape::U, 0.75, 9.0),+ ];+ let previous = prev.map(|p| {+ if p == MouthShape::AHalf {+ MouthShape::A+ } else {+ p+ }+ });+ let vowel = regions+ .iter()+ .find(|&&(v, lo, hi)| Some(v) == previous && form >= lo - 0.07 && form < hi + 0.07)+ .or_else(|| regions.iter().find(|&&(_, lo, hi)| form >= lo && form < hi))+ .map(|&(v, _, _)| v)?;+ Some(if vowel == MouthShape::A && open < 0.6 {+ MouthShape::AHalf+ } else {+ vowel+ })+}++pub fn eye_sprite(open: f64, smile: f64, mood: Option<&str>) -> Option<&str> {+ if mood.is_some() && open >= 0.3 {+ return mood;+ }+ if open >= 0.75 {+ return None;+ }+ Some(if open < 0.3 {+ if smile >= 0.5 {+ "eyes_smile"+ } else {+ "eyes_closed"+ }+ } else {+ "eyes_half"+ })+}++pub fn eye_mood(overlay: Option<&str>, expression: Option<&str>) -> Option<&'static str> {+ let value = overlay+ .filter(|s| !s.is_empty() && *s != "neutral")+ .or(expression)?;+ match value {+ "angry" | "angryTalk" => Some("angry"),+ "sad" | "sadTalk" => Some("sad"),+ "surprise" | "surprisedTalk" => Some("surprise"),+ _ => None,+ }+}++fn shut(name: &str) -> bool {+ name.starts_with("eyes_closed") || name.starts_with("eyes_smile")+}+fn order(name: &str) -> u8 {+ if shut(name) {+ 1+ } else if name.starts_with("eyes_") {+ 0+ } else if matches!(name, "mouth_angry" | "mouth_sad") {+ 2+ } else {+ 3+ }+}+fn rank(name: &str) -> i8 {+ if name == "open" {+ -1+ } else if shut(name) {+ 1+ } else {+ 0+ }+}++#[derive(Debug, Clone)]+pub struct SpriteItem {+ pub name: String,+ pub mesh: Grid,+ pub visible: bool,+ pub alpha: f64,+ weights: Vec<BaseWeights>,+ inner: Vec<f64>,+}++#[derive(Clone)]+struct EyeState {+ cur: String,+ prev: String,+ t: f64,+}++#[derive(Debug, Serialize, Deserialize, PartialEq)]+pub struct SpriteCoverage {+ pub eyes: [bool; 2],+ pub mouth: Option<MouthShape>,+}++pub struct Sprites {+ pub items: Vec<SpriteItem>,+ last_shape: Option<MouthShape>,+ eye_state: [EyeState; 2],+}++fn mouth_inner(rig: &Rig, x: f64, y: f64) -> f64 {+ let area = &rig.mouth.area;+ let e = &area.ellipse;+ let c = (-area.angle).cos();+ let s = (-area.angle).sin();+ let dx = x - e.cx;+ let dy = y - e.cy;+ let u = (dx * c - dy * s) / e.rx;+ let v = (dx * s + dy * c) / e.ry;+ 1.0 - sstep(0.55, 0.95, u.hypot(v))+}++fn place(+ item: &mut SpriteItem,+ engine: &RigEngine<'_>,+ p: &Parameters,+ phys: &PhysicsOffsets,+ squash: f64,+ width: f64,+) {+ let mouth = &engine.rig.mouth;+ for (k, (r, o)) in item+ .mesh+ .rest+ .as_chunks::<2>()+ .0+ .iter()+ .zip(item.mesh.pos.as_chunks_mut::<2>().0)+ .enumerate()+ {+ let mut x = f64::from(r[0]);+ let mut y = f64::from(r[1]);+ let w = item.inner[k];+ if w > 0.0 {+ let line = mouth.cy + (x - mouth.cx) * mouth.angle.tan();+ y = line + (y - line) * (1.0 + (squash - 1.0) * w);+ let cx = mouth.area.ellipse.cx;+ x = cx + (x - cx) * (1.0 + (width - 1.0) * w);+ }+ let point = engine.deform_base(x, y, &item.weights[k], p, phys);+ *o = [point[0] as f32, point[1] as f32];+ }+}++impl Sprites {+ pub fn new(+ engine: &RigEngine<'_>,+ sheet: &SpriteMetadata,+ textures: &BTreeMap<String, Texture>,+ ) -> Result<Self> {+ let mut names: Vec<_> = sheet.layer_order.iter().map(String::as_str).collect();+ if names.is_empty() {+ names.extend(sheet.layers.keys().map(String::as_str));+ }+ names.sort_by_key(|name| order(name));+ let mut items = vec![];+ for name in names {+ let rect = sheet.layers[name].map(f64::from);+ let image = textures+ .get(&format!("sprites/{name}.png"))+ .with_context(|| format!("missing sprite texture: {name}"))?;+ let mesh = build_grid(+ rect,+ engine.rig.mesh.sprite_cell as f64,+ &image.alpha(),+ image.width as usize,+ None,+ )?;+ let weights = mesh+ .rest+ .as_chunks::<2>()+ .0+ .iter()+ .map(|p| engine.base_weights(p[0] as f64, p[1] as f64, 0.0))+ .collect();+ let inner = mesh+ .rest+ .as_chunks::<2>()+ .0+ .iter()+ .map(|p| {+ if name.starts_with("mouth") {+ mouth_inner(engine.rig, p[0] as f64, p[1] as f64)+ } else {+ 0.0+ }+ })+ .collect();+ items.push(SpriteItem {+ name: name.into(),+ mesh,+ weights,+ inner,+ visible: false,+ alpha: 1.0,+ });+ }+ Ok(Self {+ items,+ last_shape: None,+ eye_state: std::array::from_fn(|_| EyeState {+ cur: "open".into(),+ prev: "open".into(),+ t: 1.0,+ }),+ })+ }++ fn show(+ &mut self,+ name: &str,+ alpha: f64,+ engine: &RigEngine<'_>,+ p: &Parameters,+ phys: &PhysicsOffsets,+ ) {+ if alpha <= 0.001 {+ return;+ }+ if let Some(item) = self.items.iter_mut().find(|item| item.name == name) {+ item.visible = true;+ item.alpha = alpha;+ place(item, engine, p, phys, 1.0, 1.0);+ }+ }++ pub fn update(+ &mut self,+ engine: &RigEngine<'_>,+ p: &Parameters,+ phys: &PhysicsOffsets,+ dt: f64,+ mood: Option<&str>,+ ) -> SpriteCoverage {+ for item in &mut self.items {+ item.visible = false;+ }+ let mut covered = [false; 2];+ for (i, (open, smile)) in [+ (p.eye_r_open, p.eye_smile),+ (p.eye_l_open, p.eye_smile_l.unwrap_or(p.eye_smile)),+ ]+ .into_iter()+ .enumerate()+ {+ let drawn = mood+ .filter(|&m| {+ matches!(m, "sad" | "surprise")+ && self+ .items+ .iter()+ .any(|it| it.name == format!("eyes_{m}_{i}"))+ })+ .map(|m| format!("eyes_{m}"));+ let want = eye_sprite(open, smile, drawn.as_deref()).unwrap_or("open");+ let st = &mut self.eye_state[i];+ if want != st.cur {+ st.prev = st.cur.clone();+ st.cur = want.into();+ st.t = 0.0;+ }+ st.t = if dt > 0.0 {+ (st.t + dt / 0.06).min(1.0)+ } else {+ 1.0+ };+ let k = sstep(0.0, 1.0, st.t);+ let cur = st.cur.clone();+ let prev = st.prev.clone();+ let t = st.t;+ covered[i] = cur != "open" && k >= 0.999;+ let mut show = |name: &str, alpha| {+ if name != "open" {+ self.show(&format!("{name}_{i}"), alpha, engine, p, phys);+ }+ };+ if t >= 1.0 {+ show(&cur, 1.0);+ } else if rank(&cur) > rank(&prev) {+ show(&prev, 1.0);+ show(&cur, k);+ } else {+ show(&cur, 1.0);+ show(&prev, 1.0 - k);+ }+ }+ let shape = mouth_shape(p.mouth_open, p.mouth_form, self.last_shape);+ self.last_shape = shape;+ let def = shape.map(MouthShape::definition);+ let mouth = def.and_then(|(name, _, _)| self.items.iter().position(|it| it.name == name));+ if let Some(mood) = mood.filter(|&m| matches!(m, "angry" | "sad")) {+ if mouth.is_none() || p.mouth_open < 0.16 + 0.04 {+ self.show(&format!("mouth_{mood}"), 1.0, engine, p, phys);+ }+ }+ if let (Some(index), Some((_, reference, width))) = (mouth, def) {+ let item = &mut self.items[index];+ item.visible = true;+ item.alpha = sstep(0.16, 0.16 + 0.04, p.mouth_open);+ place(+ item,+ engine,+ p,+ phys,+ (p.mouth_open / reference).clamp(0.5, 1.1),+ width,+ );+ }+ SpriteCoverage {+ eyes: covered,+ mouth: shape,+ }+ }+}diff --git a/native/crates/mesh-avatar-core/src/format.rs b/native/crates/mesh-avatar-core/src/format.rsindex b4b15dc..76c0b32 100644--- a/native/crates/mesh-avatar-core/src/format.rs+++ b/native/crates/mesh-avatar-core/src/format.rs@@ -98,12 +98,66 @@ pub struct LayerMetadata {pub eyes: [EyeCurves; 2],}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, Serialize)]pub struct SpriteMetadata {- #[serde(default = "legacy_version")]pub version: u32,pub build: String,pub layers: BTreeMap<String, Rect>,+ /// JSON insertion order, used for stable sprite cross-fade draw ordering.+ #[serde(skip)]+ pub layer_order: Vec<String>,+}++impl<'de> Deserialize<'de> for SpriteMetadata {+ fn deserialize<D: serde::Deserializer<'de>>(+ deserializer: D,+ ) -> std::result::Result<Self, D::Error> {+ struct OrderedLayers(Vec<(String, Rect)>);+ impl<'de> Deserialize<'de> for OrderedLayers {+ fn deserialize<D: serde::Deserializer<'de>>(+ deserializer: D,+ ) -> std::result::Result<Self, D::Error> {+ struct Visitor;+ impl<'de> serde::de::Visitor<'de> for Visitor {+ type Value = OrderedLayers;+ fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str("a layer rectangle map")+ }+ fn visit_map<M: serde::de::MapAccess<'de>>(+ self,+ mut map: M,+ ) -> std::result::Result<Self::Value, M::Error> {+ let mut entries = Vec::new();+ while let Some(entry) = map.next_entry()? {+ entries.push(entry);+ }+ Ok(OrderedLayers(entries))+ }+ }+ deserializer.deserialize_map(Visitor)+ }+ }+ #[derive(Deserialize)]+ struct Metadata {+ #[serde(default = "legacy_version")]+ version: u32,+ build: String,+ layers: OrderedLayers,+ }+ let value = Metadata::deserialize(deserializer)?;+ let layer_order = value+ .layers+ .0+ .iter()+ .map(|(name, _)| name.clone())+ .collect();+ Ok(Self {+ version: value.version,+ build: value.build,+ layers: value.layers.0.into_iter().collect(),+ layer_order,+ })+ }}#[derive(Debug, Clone)]diff --git a/native/crates/mesh-avatar-core/tests/deformation.rs b/native/crates/mesh-avatar-core/tests/deformation.rsnew file mode 100644index 0000000..63a6338--- /dev/null+++ b/native/crates/mesh-avatar-core/tests/deformation.rs@@ -0,0 +1,399 @@+use mesh_avatar_core::{+ engine::{+ grid::build_grid,+ physics::Physics,+ rig::{eye_part_alpha, eye_part_y, EyePart, Parameters, PhysicsOffsets, RigEngine},+ sprites::{SpriteCoverage, Sprites},+ },+ format::AvatarPackage,+ rig::Point,+};+use serde::{de::DeserializeOwned, Deserialize};+use std::{+ fs,+ path::{Path, PathBuf},+};++fn directory() -> PathBuf {+ Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/conformance")+}+fn golden<T: DeserializeOwned>(name: &str) -> Option<T> {+ match fs::read(directory().join(name)) {+ Ok(bytes) => Some(serde_json::from_slice(&bytes).unwrap()),+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {+ eprintln!("{name} absent; skipping. Run node native/conformance/dump.mjs first.");+ None+ }+ Err(e) => panic!("could not read {name}: {e}"),+ }+}+fn avatar() -> AvatarPackage {+ AvatarPackage::open(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../samples/miko-qipao"))+ .unwrap()+}+fn compare(actual: f64, expected: f64, max: &mut f64) {+ assert!(+ actual.is_finite() && expected.is_finite(),+ "non-finite conformance output"+ );+ let delta = (actual - expected).abs();+ *max = max.max(delta);+ assert!(+ delta <= 1e-3,+ "actual {actual}, expected {expected}, delta {delta}"+ );+}+fn compare_point(actual: Point, expected: Point, max: &mut f64) {+ for i in 0..2 {+ compare(actual[i], expected[i], max);+ }+}++#[derive(Deserialize)]+struct Deformations {+ version: u32,+ parameters: Vec<serde_json::Value>,+ points: Vec<Point>,+ files: Vec<String>,+}+#[derive(Deserialize)]+struct EyeOutput {+ eye: usize,+ part: EyePart,+ alpha: f64,+ positions: Vec<f64>,+}+#[derive(Deserialize)]+struct Pose {+ parameters: Parameters,+ physics: PhysicsOffsets,+ base: Vec<f64>,+ hand: Vec<f64>,+ eyes: Vec<EyeOutput>,+}++#[test]+fn every_regression_point_and_eye_part_matches_fixed_and_zero_physics() {+ let Some(goldens) = golden::<Deformations>("deformation.json") else {+ return;+ };+ assert_eq!(goldens.version, 1);+ compare_json(+ &serde_json::to_value(mesh_avatar_core::engine::rig::PARAMS).unwrap(),+ &serde_json::to_value(&goldens.parameters).unwrap(),+ &mut 0.0,+ );+ assert_eq!(goldens.files.len(), 28);+ let avatar = avatar();+ let engine = RigEngine::new(&avatar.rig);+ let hair = &avatar.textures["hairmask.png"];+ let weights: Vec<_> = goldens+ .points+ .iter()+ .map(|&[x, y]| {+ let ix = (x.round() as u32).min(hair.width - 1);+ let iy = (y.round() as u32).min(hair.height - 1);+ engine.base_weights(+ x,+ y,+ hair.rgba[((iy * hair.width + ix) * 4) as usize] as f64 / 255.0,+ )+ })+ .collect();+ let hands: Vec<_> = goldens+ .points+ .iter()+ .map(|&[x, y]| engine.hand_weights(x, y))+ .collect();+ let mut max = 0.0;+ for file in &goldens.files {+ let pose: Pose = golden(file).expect("missing pose golden");+ let p = &pose.parameters;+ let frame = engine.hand_frame(p);+ assert_eq!(pose.base.len(), goldens.points.len() * 2);+ assert_eq!(pose.hand.len(), pose.base.len());+ assert_eq!(pose.eyes.len(), 8);+ for (i, &[x, y]) in goldens.points.iter().enumerate() {+ compare_point(+ engine.deform_base(x, y, &weights[i], p, &pose.physics),+ [pose.base[2 * i], pose.base[2 * i + 1]],+ &mut max,+ );+ compare_point(+ engine.deform_hand(x, y, &hands[i], p, &frame),+ [pose.hand[2 * i], pose.hand[2 * i + 1]],+ &mut max,+ );+ }+ for eye in &pose.eyes {+ let open = if eye.eye == 0 {+ p.eye_r_open+ } else {+ p.eye_l_open+ };+ compare(+ eye_part_alpha(eye.part, open, p.eye_smile),+ eye.alpha,+ &mut max,+ );+ assert_eq!(eye.positions.len(), pose.base.len());+ for (i, &[x, y]) in goldens.points.iter().enumerate() {+ let y = eye_part_y(&avatar.rig.eyes[eye.eye], eye.part, x, y, open, p.eye_smile);+ compare_point(+ engine.deform_base(x, y, &weights[i], p, &pose.physics),+ [eye.positions[2 * i], eye.positions[2 * i + 1]],+ &mut max,+ );+ }+ }+ }+ eprintln!(+ "Deformation: {} poses, {} points each, max delta {max} px",+ goldens.files.len(),+ goldens.points.len()+ );+}++fn compare_json(actual: &serde_json::Value, expected: &serde_json::Value, max: &mut f64) {+ use serde_json::Value;+ match (actual, expected) {+ (Value::Number(a), Value::Number(b)) => {+ compare(a.as_f64().unwrap(), b.as_f64().unwrap(), max)+ }+ (Value::Array(a), Value::Array(b)) => {+ assert_eq!(a.len(), b.len());+ for (a, b) in a.iter().zip(b) {+ compare_json(a, b, max);+ }+ }+ (Value::Object(a), Value::Object(b)) => {+ assert_eq!(a.len(), b.len());+ for (key, value) in a {+ compare_json(value, &b[key], max);+ }+ }+ _ => assert_eq!(actual, expected),+ }+}++#[derive(Deserialize)]+struct Variants {+ version: u32,+ variants: Vec<Variant>,+}+#[derive(Deserialize)]+struct Variant {+ rig: mesh_avatar_core::rig::Rig,+ weights: Vec<WeightedPoint>,+ frames: Vec<VariantFrame>,+}+#[derive(Deserialize)]+struct WeightedPoint {+ x: f64,+ y: f64,+ hair: f64,+ weights: serde_json::Value,+}+#[derive(Deserialize)]+struct VariantFrame {+ parameters: Parameters,+ dt: f64,+ out: PhysicsOffsets,+ positions: Vec<Point>,+}++#[test]+fn absent_optional_features_and_resampled_strands_match_javascript() {+ let Some(goldens) = golden::<Variants>("variants.json") else {+ return;+ };+ assert_eq!(goldens.version, 1);+ let mut max = 0.0;+ for variant in goldens.variants {+ variant.rig.validate().unwrap();+ let engine = RigEngine::new(&variant.rig);+ let mut weights = Vec::new();+ for point in &variant.weights {+ let w = engine.base_weights(point.x, point.y, point.hair);+ compare_json(&serde_json::to_value(&w).unwrap(), &point.weights, &mut max);+ weights.push(w);+ }+ let mut physics = Physics::new(&variant.rig);+ for frame in variant.frames {+ let out = physics.step(&engine, &frame.parameters, frame.dt);+ compare_json(+ &serde_json::to_value(out).unwrap(),+ &serde_json::to_value(frame.out).unwrap(),+ &mut max,+ );+ for (i, (point, weight)) in variant.weights.iter().zip(&weights).enumerate() {+ compare_point(+ engine.deform_base(point.x, point.y, weight, &frame.parameters, out),+ frame.positions[i],+ &mut max,+ );+ }+ }+ }+ eprintln!("Optional features and strand resampling: max delta {max} px");+}++#[derive(Deserialize)]+struct PhysicsGoldens {+ version: u32,+ traces: Vec<Trace>,+}+#[derive(Deserialize)]+struct Trace {+ gain: f64,+ frames: Vec<PhysicsFrame>,+}+#[derive(Deserialize)]+struct PhysicsFrame {+ parameters: Parameters,+ dt: f64,+ out: PhysicsOffsets,+ chains: Vec<Chain>,+}+#[derive(Deserialize)]+struct Chain {+ name: String,+ points: [Point; 3],+ phi: Point,+ anchor: Point,+ positions: Vec<f32>,+}++#[test]+fn hair_and_tassel_traces_and_skinned_vertices_match_javascript() {+ let Some(goldens) = golden::<PhysicsGoldens>("physics.json") else {+ return;+ };+ assert_eq!(goldens.version, 1);+ let avatar = avatar();+ let engine = RigEngine::new(&avatar.rig);+ let mut max = 0.0;+ for trace in goldens.traces {+ let mut physics = Physics::new(&avatar.rig);+ physics.gain = trace.gain;+ let mut meshes: Vec<_> = physics+ .chains+ .iter()+ .map(|ch| {+ let name = &ch.accessory.name;+ let image = &avatar.textures[&format!("{name}.png")];+ build_grid(+ avatar.layers.layers[name].map(f64::from),+ avatar.rig.mesh.tassel_cell as f64,+ &image.alpha(),+ image.width as usize,+ None,+ )+ .unwrap()+ })+ .collect();+ for frame in trace.frames {+ let out = physics.step(&engine, &frame.parameters, frame.dt);+ compare(out.gain, frame.out.gain, &mut max);+ compare_point(out.bun_l, frame.out.bun_l, &mut max);+ compare_point(out.bun_r, frame.out.bun_r, &mut max);+ assert_eq!(out.strands.len(), frame.out.strands.len());+ for (actual, expected) in out.strands.iter().zip(&frame.out.strands) {+ for j in 0..4 {+ compare_point(actual[j], expected[j], &mut max);+ }+ }+ assert_eq!(physics.chains.len(), frame.chains.len());+ for ((actual, expected), mesh) in+ physics.chains.iter().zip(&frame.chains).zip(&mut meshes)+ {+ assert_eq!(actual.accessory.name, expected.name);+ compare_point(actual.phi, expected.phi, &mut max);+ compare_point(actual.anchor, expected.anchor, &mut max);+ for (a, b) in actual.points.unwrap().into_iter().zip(expected.points) {+ compare_point(a, b, &mut max);+ }+ actual.deform_grid(mesh).unwrap();+ assert_eq!(mesh.pos.len(), expected.positions.len());+ for (&a, &b) in mesh.pos.iter().zip(&expected.positions) {+ compare(a as f64, b as f64, &mut max);+ }+ }+ }+ }+ eprintln!("Physics and tassel skinning: max delta {max} px");+}++#[derive(Deserialize)]+struct SpriteFiles {+ version: u32,+ files: Vec<String>,+}+#[derive(Deserialize)]+struct SpriteGoldens {+ version: u32,+ updates: Vec<SpriteFrame>,+}+#[derive(Deserialize)]+struct SpriteFrame {+ parameters: Parameters,+ physics: PhysicsOffsets,+ dt: f64,+ mood: Option<String>,+ coverage: SpriteCoverage,+ items: Vec<SpriteOutput>,+}+#[derive(Deserialize)]+struct SpriteOutput {+ name: String,+ visible: bool,+ alpha: f64,+ positions: Option<Vec<f32>>,+}++#[test]+fn sprite_geometry_visibility_alpha_order_and_crossfades_match_javascript() {+ let Some(files) = golden::<SpriteFiles>("sprites.json") else {+ return;+ };+ assert_eq!(files.version, 1);+ let avatar = avatar();+ let engine = RigEngine::new(&avatar.rig);+ let mut max = 0.0;+ let mut count = 0;+ for file in files.files {+ let frames: SpriteGoldens = golden(&file).expect("missing sprite golden");+ assert_eq!(frames.version, 1);+ let mut sprites =+ Sprites::new(&engine, avatar.sprites.as_ref().unwrap(), &avatar.textures).unwrap();+ for frame in frames.updates {+ count += 1;+ let coverage = sprites.update(+ &engine,+ &frame.parameters,+ &frame.physics,+ frame.dt,+ frame.mood.as_deref(),+ );+ assert_eq!(coverage, frame.coverage);+ assert_eq!(sprites.items.len(), frame.items.len());+ for (actual, expected) in sprites.items.iter().zip(&frame.items) {+ assert_eq!(actual.name, expected.name, "sprite draw order");+ assert_eq!(+ actual.visible, expected.visible,+ "{} visibility at frame {count}",+ actual.name+ );+ compare(actual.alpha, expected.alpha, &mut max);+ if let Some(pos) = &expected.positions {+ assert!(actual.visible);+ assert_eq!(actual.mesh.pos.len(), pos.len());+ for (&a, &b) in actual.mesh.pos.iter().zip(pos) {+ compare(a as f64, b as f64, &mut max);+ }+ }+ }+ }+ }+ eprintln!("Sprites: {count} updates, max delta {max} px");+}