Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat(face): the face layer — video-face cuts a talking clip's lower face into smile and neutral sets and anchors it on every body frame; face-bake bakes a line with the pack's own player, sprite-gen-face-v1.mjs

Soohong Kim0f197f050315
diff --git a/pyproject.toml b/pyproject.toml
index 4ed2a38..7bf77f8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -101,4 +101,4 @@ include = ["sprite_gen*"]
# not feared: adding curator/src/deep/new.js under the old patterns built a wheel without it.
# `curator/**/*` derives the answer from the filesystem, the same fix as packages.find above.
[tool.setuptools.package-data]
-sprite_gen = ["serve/curator/**/*", "serve/composer/**/*", "audio/*.mjs"]
+sprite_gen = ["serve/curator/**/*", "serve/composer/**/*", "audio/*.mjs", "video/*.mjs"]
diff --git a/sprite_gen/_modules.py b/sprite_gen/_modules.py
index dd526a7..3f1c66b 100644
--- a/sprite_gen/_modules.py
+++ b/sprite_gen/_modules.py
@@ -41,6 +41,8 @@ MODULE_DOMAIN = {
'source': 'video',
'rife_install': 'video',
'clip_prompt': 'video',
+ 'face_pack': 'video',
+ 'face_bake': 'video',
'prepare': 'gen',
'extract': 'frames',
'cutout': 'frames',
diff --git a/sprite_gen/cli.py b/sprite_gen/cli.py
index f105da8..6425e8b 100644
--- a/sprite_gen/cli.py
+++ b/sprite_gen/cli.py
@@ -33,6 +33,7 @@ from sprite_gen.video import compare as video_compare
from sprite_gen.video import restoration as video_restoration
from sprite_gen.video import source as video_source
from sprite_gen.video import rife_install
+from sprite_gen.video import face_bake, face_pack
from sprite_gen.effects import breathe, recolor, shadow
from sprite_gen.scene import render, inspect_scene
from sprite_gen.serve import serve_compose, serve_curation
@@ -402,6 +403,18 @@ COMMANDS: dict[str, tuple[str, Callable[[argparse.ArgumentParser], None], Callab
video_stop.add_arguments,
video_stop.run,
),
+ # The face layer (docs/face-parts.md): a talking clip's lower face cut to be laid on body motions, and a line baked
+ # with the pack's own player.
+ "video-face": (
+ "A face pack: a talking clip's lower face (mouth, jaw, cheeks) cut into smile and neutral sets, anchored on every frame of the body motions it is laid on, with the player that picks its frames to a voice.",
+ face_pack.add_arguments,
+ face_pack.run,
+ ),
+ "face-bake": (
+ "Bake a line ahead: a body motion with the face layer talking to a voice, picked by the pack's player (node), as frames, sound and a report; a jump the laying put in is refused.",
+ face_bake.add_arguments,
+ face_bake.run,
+ ),
"rife": (
"install: download the pinned RIFE (rife-ncnn-vulkan, sha256-checked) into the user data directory, where the loop repair finds it.",
rife_install.add_arguments,
diff --git a/sprite_gen/video/face_bake.py b/sprite_gen/video/face_bake.py
new file mode 100644
index 0000000..f4835cc
--- /dev/null
+++ b/sprite_gen/video/face_bake.py
@@ -0,0 +1,279 @@
+# SPDX-License-Identifier: Apache-2.0
+"""`sprite-gen face-bake` — a line baked ahead: a body motion with the face layer talking to a voice, as frames, a
+sound track and a report.
+
+The frames are picked by the pack's own player (`sprite-gen-face-v1.mjs`, run with node through `face_plan.mjs`:
+`mouthTrack`, then `planScene`), the same functions a live page calls, so a baked line and a live one show the same frames. The voice is
+folded to mono first and padded after (`--lead`, `--tail`: silence before and after it), so a stereo voice is never
+delayed on one channel alone. With `--play` a body clip plays from its start with its own sound until its own_mouth
+span ends, and the line follows; without it the line is said on the loop from `--start-cell`.
+
+Each frame is laid as a live page draws it (`face_layer.lay`: the body cell, the face sprite over it through the
+cell's anchor, what of the body stays on top over that) and read where the face is (`face_layer.step_features`):
+
+- **outside**: pixels changed where the layer's weight, through the cell's anchor, is 0. Must be 0.
+- **jumps**: a frame whose step from the frame before is over the jump line times the larger of the body's own step,
+ the layer's own step and the body's median step is over the line (`over_line`). The body and the layer moving in
+ the same frame can each be within the line and still add up past it; a frame over the line and over the sum of the
+ two steps as well is one the laying put in (`over_sum`), and a bake with one is refused.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+import subprocess
+import tempfile
+import wave
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+
+from sprite_gen._deps import np
+from sprite_gen.util.console import utf8_streams
+from sprite_gen.video import face_layer as fl
+from sprite_gen.video import face_pack
+
+PROG = "face-bake"
+RATE = 48000
+MATTE = "2a2430"
+
+PLAN = Path(__file__).with_name("face_plan.mjs")
+
+
+def load_pack(folder: Path) -> dict[str, Any]:
+ """The pack's manifest, every file it lists checked against its digest."""
+ try:
+ pack = json.loads((folder / "face-pack.json").read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as exc:
+ raise SystemExit(f"{PROG}: --pack {folder}: {exc}") from None
+ if pack.get("kind") != face_pack.PACK_KIND or pack.get("version") != face_pack.VERSION:
+ raise SystemExit(f"{PROG}: --pack {folder}: not a {face_pack.PACK_KIND} version {face_pack.VERSION}")
+ for name, digest in pack["files"].items():
+ path = folder / name
+ if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != digest:
+ raise SystemExit(f"{PROG}: --pack {folder}: {name} is missing or not the file the pack was made with")
+ return pack
+
+
+def decode(path: Path, seconds: float | None = None) -> np.ndarray:
+ """A sound as mono float samples at RATE (ffmpeg folds the channels), cut or padded to `seconds` when given."""
+ if shutil.which("ffmpeg") is None:
+ raise SystemExit(f"{PROG}: ffmpeg is required on PATH to read {path}")
+ proc = subprocess.run(["ffmpeg", "-v", "error", "-nostdin", "-i", str(path), "-map", "0:a:0", "-ac", "1", "-ar", str(RATE),
+ "-f", "f32le", "pipe:1"], capture_output=True)
+ if proc.returncode:
+ raise SystemExit(f"{PROG}: ffmpeg could not read the sound of {path}")
+ x = np.frombuffer(proc.stdout, dtype="<f4").astype(np.float32)
+ if seconds is not None:
+ n = int(round(seconds * RATE))
+ x = np.concatenate([x[:n], np.zeros(max(0, n - len(x)), np.float32)])
+ return x
+
+
+def write_wav(path: Path, x: np.ndarray) -> None:
+ with wave.open(str(path), "wb") as w:
+ w.setnchannels(1)
+ w.setsampwidth(2)
+ w.setframerate(RATE)
+ w.writeframes(np.clip(np.round(x * 32767), -32768, 32767).astype("<i2").tobytes())
+
+
+def plan_line(folder: Path, voice: np.ndarray, play: str | None, start: int, tags: list[str]) -> dict[str, Any]:
+ node = shutil.which("node")
+ if node is None:
+ raise SystemExit(f"{PROG}: node is required on PATH: the pack's player (sprite-gen-face-v1.mjs) picks the frames")
+ with tempfile.TemporaryDirectory() as tmp_name:
+ tmp = Path(tmp_name)
+ (tmp / "samples.f32").write_bytes(voice.astype("<f4").tobytes())
+ (tmp / "args.json").write_text(json.dumps({"rate": RATE, "tags": tags, "play": play, "start": start}), encoding="utf-8")
+ proc = subprocess.run([node, str(PLAN), str(folder / face_pack.PLAYER_PATH.name), str(folder / "face-pack.json"),
+ str(tmp / "samples.f32"), str(tmp / "args.json"), str(tmp / "plan.json")], capture_output=True, encoding="utf-8")
+ if proc.returncode:
+ raise SystemExit(f"{PROG}: the pack's player failed: {proc.stderr.strip()[-600:]}")
+ return json.loads((tmp / "plan.json").read_text(encoding="utf-8"))
+
+
+def _matrix(six: list[float]) -> np.ndarray:
+ return np.array([six[:3], six[3:], [0.0, 0.0, 1.0]])
+
+
+def bake(pack_dir: Path, voice_path: Path, out: Path, *, play: str | None = None, start: int = 0, tags: list[str] | None = None,
+ lead: float = 0.5, tail: float = 0.6) -> dict[str, Any]:
+ pack = load_pack(pack_dir)
+ tags = list(tags or [])
+ bodies = {b["name"]: b for b in pack["body"]}
+ if play is not None and play not in bodies:
+ raise SystemExit(f"{PROG}: --play {play}: the pack's body clips are {', '.join(bodies)}")
+ if lead < 0 or tail < 0:
+ raise SystemExit(f"{PROG}: --lead and --tail are seconds of silence, 0 or more")
+ if out.exists() and any(out.iterdir()):
+ raise SystemExit(f"{PROG}: --out {out} is not empty")
+ fps = float(pack["fps"])
+ voice = decode(voice_path)
+ padded = np.concatenate([np.zeros(int(round(lead * RATE)), np.float32), voice, np.zeros(int(round(tail * RATE)), np.float32)])
+ result = plan_line(pack_dir, padded, play, start, tags)
+ plan, track = result["plan"], result["track"]
+
+ geom = fl.Geometry.read(pack["face"]["geometry"], tuple(pack["size"]))
+ w0 = np.load(pack_dir / "weight.npy")
+ sprites: dict[int, fl.Sprite] = {}
+ keeps: dict[tuple[str, int], Any] = {}
+ out.mkdir(parents=True, exist_ok=True)
+ frames_dir = out / "frames"
+ frames_dir.mkdir()
+
+ def sprite(j: int) -> fl.Sprite:
+ if j not in sprites:
+ row = pack["face"]["frames"][j]
+ sprites[j] = fl.Sprite(Image.open(pack_dir / row["file"]).convert("RGBA"), _matrix(row["to_ref"]))
+ return sprites[j]
+
+ def keep(name: str, k: int):
+ if (name, k) not in keeps:
+ rec = bodies[name]["keep"][k]
+ keeps[name, k] = None if rec is None else (Image.open(pack_dir / rec["file"]).convert("RGBA"), tuple(rec["at"]))
+ return keeps[name, k]
+
+ rows, prev = [], None
+ for t, f in enumerate(plan["frames"]):
+ b = bodies[f["body"]]
+ cell = Image.open(pack_dir / b["cells"][f["cell"]]).convert("RGBA")
+ to_body = _matrix(b["to_body"][f["cell"]])
+ sp = None if f["face"] is None else sprite(f["face"])
+ laid = fl.lay(cell, sp, to_body, keep(f["body"], f["cell"]))
+ laid.save(frames_dir / f"frame-{t:05d}.png")
+ comp, body = np.asarray(laid, np.float32), np.asarray(cell, np.float32)
+ w_ell = fl.warp(w0, np.linalg.inv(to_body), cell.size)
+ outside = int(np.any(comp[w_ell <= 0] != body[w_ell <= 0], axis=1).sum()) if sp is not None else 0
+ feats = [fl.step_features(fl.warp(comp, to_body, geom.size), geom), fl.step_features(fl.warp(body, to_body, geom.size), geom)]
+ feats.append(fl.step_features(fl.warp(np.asarray(sp.image, np.float32), np.linalg.inv(sp.to_ref), geom.size), geom)
+ if sp is not None else feats[1])
+ row = {"t": t, **f, "outside_changed_px": outside}
+ if prev is not None:
+ row["step"] = {k: float(np.abs(a - p).mean()) for k, a, p in zip(("comp", "body", "layer"), feats, prev)}
+ prev = feats
+ rows.append(row)
+ med = float(np.median([r["step"]["body"] for r in rows[1:]])) if len(rows) > 1 else 0.0
+ over_line, over_sum = [], []
+ for r in rows[1:]:
+ s = r["step"]
+ if s["comp"] > fl.JUMP_LINE * max(s["body"], s["layer"], med):
+ over_line.append(r["t"])
+ if s["comp"] > s["body"] + s["layer"]:
+ over_sum.append(r["t"])
+
+ own = plan["line_start"] or 0
+ sound_parts = []
+ if play is not None and own:
+ src = bodies[play].get("sound")
+ sound_parts.append(decode(pack_dir / src, own / fps) if src else np.zeros(int(round(own / fps * RATE)), np.float32))
+ sound_parts.append(padded)
+ track_audio = np.concatenate(sound_parts)
+ frames = len(rows)
+ track_audio = np.concatenate([track_audio, np.zeros(max(0, int(round(frames / fps * RATE)) - len(track_audio)), np.float32)])
+ write_wav(out / "voice-baked.wav", track_audio)
+ write_wav(out / "voice-padded.wav", padded)
+ refused = []
+ if over_sum:
+ refused.append(f"frames {over_sum} step over the jump line and over the body's and the layer's steps together")
+ if any(r["outside_changed_px"] for r in rows):
+ refused.append("pixels changed outside the layer's weight")
+ if plan["layer_from"] is not None and plan["layer_from"] >= frames:
+ refused.append("the layer found no frame within the jump line of its rest to come on")
+ report = {"pack": str(pack_dir), "voice": str(voice_path), "play": play, "start_cell": start, "tags": tags, "lead_s": lead, "tail_s": tail,
+ "fps": fps, "frames": frames, "set": plan["set"], "rest": plan["rest"], "line_start": plan["line_start"],
+ "layer_from": plan["layer_from"], "levels": track["levels"], "syllables": len(track["syllables"]),
+ "talk_frames": sum(1 for r in rows if r["layer"] == "talk"), "rest_frames": sum(1 for r in rows if r["layer"] == "rest"),
+ "off_frames": sum(1 for r in rows if r["layer"] == "off"),
+ "outside_changed_px_max": max((r["outside_changed_px"] for r in rows), default=0),
+ "jump_line": fl.JUMP_LINE, "body_median_step": med, "over_line": over_line, "over_sum": over_sum,
+ "verdict": "refused" if refused else "pass", "refused": refused, "rows": rows}
+ (out / "bake.json").write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
+ return report
+
+
+def video(out: Path, report: dict[str, Any], beside: Path | None, height: int | None, matte: str) -> Path:
+ """The baked frames (and, beside them on the left, another line's frames of the same count) over a flat matte,
+ with the baked sound, as an mp4."""
+ if shutil.which("ffmpeg") is None:
+ raise SystemExit(f"{PROG}: ffmpeg is required on PATH for --video")
+ frames = report["frames"]
+ w, h = Image.open(out / "frames" / "frame-00000.png").size
+ with tempfile.TemporaryDirectory() as tmp_name:
+ inputs = []
+ if beside is not None:
+ left = sorted(beside.glob("frame-*.png"))
+ if len(left) != frames:
+ raise SystemExit(f"{PROG}: --beside {beside} holds {len(left)} frames, the bake {frames}: the two lines must be the same length")
+ link = Path(tmp_name) / "left"
+ link.mkdir()
+ for k, p in enumerate(left):
+ (link / f"frame-{k:05d}.png").symlink_to(p.resolve())
+ inputs += ["-framerate", str(report["fps"]), "-i", str(link / "frame-%05d.png")]
+ inputs += ["-framerate", str(report["fps"]), "-i", str(out / "frames" / "frame-%05d.png"), "-i", str(out / "voice-baked.wav")]
+ panels = 2 if beside is not None else 1
+ chain = f"color=c=0x{matte}:s={w * panels}x{h}:r={report['fps']}[bg]"
+ last = "bg"
+ for k in range(panels):
+ chain += f";[{last}][{k}:v]overlay={k * w}:0:shortest=1[p{k}]"
+ last = f"p{k}"
+ scale = f",scale=-2:{height}:flags=lanczos" if height else ""
+ chain += f";[{last}]format=yuv420p{scale}[v]"
+ target = out / ("beside.mp4" if beside is not None else "baked.mp4")
+ proc = subprocess.run(["ffmpeg", "-v", "error", "-nostdin", "-y", *inputs, "-filter_complex", chain, "-map", "[v]",
+ "-map", f"{panels}:a", "-c:v", "libx264", "-crf", "20", "-preset", "slow", "-c:a", "aac", "-b:a", "128k",
+ "-shortest", str(target)], capture_output=True, encoding="utf-8")
+ if proc.returncode:
+ raise SystemExit(f"{PROG}: ffmpeg could not write {target.name}: {proc.stderr.strip()[-400:]}")
+ return target
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument("--pack", type=Path, required=True, help="a face pack folder (sprite-gen video-face)")
+ parser.add_argument("--voice", type=Path, required=True, help="the line's voice (any sound ffmpeg reads)")
+ parser.add_argument("--out", type=Path, required=True, help="the folder to write (new or empty)")
+ parser.add_argument("--play", help="a body clip of the pack to play from its start before the line (its own mouth and sound first)")
+ parser.add_argument("--start-cell", type=int, default=0, help="without --play: the loop cell the line starts on (default 0)")
+ parser.add_argument("--tags", nargs="*", default=[], help="the line's tags, e.g. [surprised]")
+ parser.add_argument("--lead", type=float, default=0.5, help="seconds of silence before the voice (default 0.5)")
+ parser.add_argument("--tail", type=float, default=0.6, help="seconds of silence after the voice (default 0.6)")
+ parser.add_argument("--video", action="store_true", help="also write baked.mp4 (the frames over a flat matte, with the sound)")
+ parser.add_argument("--beside", type=Path, help="also write beside.mp4: these frames (frame-*.png, as many) on the left, the bake on the right")
+ parser.add_argument("--video-height", type=int, help="the mp4's height (default the frames')")
+ parser.add_argument("--matte", default=MATTE, help=f"the mp4's background, six hex digits (default {MATTE})")
+
+
+def run(**kwargs: object) -> int:
+ out = Path(str(kwargs["out"]))
+ report = bake(Path(str(kwargs["pack"])), Path(str(kwargs["voice"])), out, play=kwargs.get("play"), # type: ignore[arg-type]
+ start=int(kwargs.get("start_cell") or 0), tags=list(kwargs.get("tags") or []), # type: ignore[arg-type]
+ lead=float(0.5 if kwargs.get("lead") is None else kwargs["lead"]), # type: ignore[arg-type]
+ tail=float(0.6 if kwargs.get("tail") is None else kwargs["tail"])) # type: ignore[arg-type]
+ made = []
+ if kwargs.get("video"):
+ made.append(str(video(out, report, None, kwargs.get("video_height"), str(kwargs.get("matte") or MATTE)))) # type: ignore[arg-type]
+ if kwargs.get("beside") is not None:
+ made.append(str(video(out, report, Path(str(kwargs["beside"])), kwargs.get("video_height"), str(kwargs.get("matte") or MATTE)))) # type: ignore[arg-type]
+ print(json.dumps({"out": str(out), "frames": report["frames"], "set": report["set"], "rest": report["rest"],
+ "line_start": report["line_start"], "layer_from": report["layer_from"], "talk_frames": report["talk_frames"],
+ "over_line": report["over_line"], "over_sum": report["over_sum"], "outside_changed_px_max": report["outside_changed_px_max"],
+ "verdict": report["verdict"], "videos": made}, ensure_ascii=False, indent=2))
+ if report["verdict"] != "pass":
+ print(f"{PROG}: refused: " + "; ".join(report["refused"]))
+ return 1
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ utf8_streams()
+ parser = argparse.ArgumentParser(prog=f"sprite-gen {PROG}", description=__doc__)
+ add_arguments(parser)
+ return run(**vars(parser.parse_args(argv)))
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/sprite_gen/video/face_layer.py b/sprite_gen/video/face_layer.py
new file mode 100644
index 0000000..101f077
--- /dev/null
+++ b/sprite_gen/video/face_layer.py
@@ -0,0 +1,675 @@
+# SPDX-License-Identifier: Apache-2.0
+"""The face layer: the lower face of a talking clip laid over any body motion, frame by frame.
+
+A body motion (an idle loop, a greeting, a reaction) and the face that talks are two layers. The face layer is the
+whole lower face — mouth, jaw line, cheeks and chin down to the neck line — cut from a talking clip, so the body's own
+jaw, which a talking clip moves, is never seen under a closed mouth. Everything is read in the coordinates of one
+reference cell (the body's rest pose, the still the clips were made from), on which the request names the face: the
+mouth, both eyes, the lower face's top edge across the cheeks, the box a lower face can be in and the skin points it is
+grown from (`Geometry`).
+
+- **Anchor** (`fit`): per frame, the similarity — place, size, tilt — that lays the reference cell's face on the frame.
+ It is read on the band round the reference's own lower face (cheeks under the eyes, the nose, the hair's edge, the
+ headset; not the jaw, which moves as the face talks, and nothing below `band_bottom`, where the neck and choker are
+ the body's): the engine's masked-NCC match (`auto_motion._match_costs`, `_best`) reads the place within `SEARCH` px,
+ and again within twice that where the best sits on the search's rim; every size and tilt of a grid is then laid
+ through it and the place read again near it, the least cost wins, a parabola through its neighbours refines size,
+ tilt and place, and a best on the grid's rim is read again on a grid twice as wide. A reading further from its
+ window's median than three scaled MADs is a misreading and takes that median (`hampel`).
+- **Weight** (`weight`): the lower faces of the reference and of every frame of every clip, read into the reference
+ through their anchors, joined (`union`; a frame adds nothing past `REACH` px of the reference's own lower face, so a
+ hand or sleeve the skin grows into stays out), and holes filled; flat 1 over it grown by `MARGIN`, a cos² feather
+ over `FEATHER` px outside it, 0 above the top edge rising to 1 over `RAMP`, the eyes cut out.
+- **Sprite** (`face_sprite`): a talking frame's lower face at the clip's own resolution, its alpha the weight less
+ what must stay the body's where the face meets it — the dark grey and black that reach in from the boundary (the
+ headset's boom, the choker) and the lines drawn in the feather — its colour moved onto the reference's by the median
+ over the feather.
+- **Keep** (`body_keep`): per body frame, what of the body stays on top of the layer: its own dark grey and black
+ reaching in from the boundary, its lines in the feather, and a hand (`hand_reach`): what differs from the reference
+ face and reaches in from outside the layer far into the face above the chin.
+- **Lay** (`lay`): the body frame, the sprite over it through the frame's anchor (one `resample.transform_cell`,
+ after one `resample.resize_cell` where the sprite is shrunk), then the keep over that. Both are plain source-over,
+ so a live player draws the same with two `drawImage` calls; nothing outside the layer's weight changes.
+
+docs/face-parts.md owns the rules; the numbers below are read at a 960 px reference on which the lower face is about
+190 px across.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Any
+
+from PIL import Image
+
+from sprite_gen._deps import np
+from sprite_gen.util import resample
+from sprite_gen.video import auto_motion, motion_anchor
+from sprite_gen.video import loop as loop_mod
+from sprite_gen.video import parts as parts_mod
+from sprite_gen.video.stop import SEAM_MAX
+
+LUMA = np.array([0.299, 0.587, 0.114], np.float32)
+SKIN_LUMA = 175 # skin is opaque and lighter than this
+RAMP = 14 # the weight rises from 0 at the top edge to 1 this far below it
+MARGIN, FEATHER = 6, 10 # the union grown by MARGIN px is flat; the weight falls to 0 over FEATHER px outside it
+EYE_GROW = 6
+REACH = 12 # a frame's lower face counts this far past the reference's own (a talking jaw drops ~7 px)
+BAND = 30 # the anchor band: this far outside the flat part, above band_bottom, less the eyes
+SEARCH = 48 # px each way the place is searched; a best on the rim is searched again at twice this
+FINE = 3 # px each way the place is read again at each size and tilt
+RECENTRE_ROUNDS = 4 # times the grid is read again round a place further than FINE - 1 from the last
+SCALES = np.array([0.96, 0.98, 1.0, 1.02, 1.04])
+THETAS = np.arange(-4.0, 4.01, 1.0)
+WIDE_SCALES = np.round(np.arange(0.92, 1.0801, 0.02), 3)
+WIDE_THETAS = np.arange(-8.0, 8.01, 1.0)
+HAMPEL_WINDOW = 5
+HAMPEL_FLOOR = {"tx": 0.5, "ty": 0.5, "s": 0.01, "th": 0.3}
+DARK_LUMA, DARK_CHROMA, DARK_GROW = 110, 45, 3
+LINE_LUMA, LINE_GROW = 150, 2
+FOREIGN = 48 # a body pixel this far (any channel, 0..255) from the reference face is foreign
+HAND_MIN = 400 # px of a hand's skin on the flat weight above the chin before it is a hand
+HAND_GROW = 3
+SKIN_WARM = 12 # a hand's skin is red over blue by this much (0..255)
+SHRINK_FROM = 0.9 # a sprite laid smaller than this is resized first (resample.resize_cell), then mapped
+JUMP_LINE = SEAM_MAX # a step over this many of the body's median steps is a jump
+# the mouth reading (`mouth_reading`): darker than the box's skin by DROP, off the headset's grey
+DROP = 18
+CLOSED_HEIGHT = 5.0 # a mouth this tall or less is closed
+SMILE_FROM = 0.35 # px of lift from which a closed mouth smiles
+OPEN_LUMA = 40 # the open mouth's area: pixels round the mouth this much off the reference
+
+
+@dataclass(frozen=True)
+class Geometry:
+ """The face on the reference cell (its pixel coordinates); docs/face-parts.md "The request"."""
+
+ size: tuple[int, int]
+ mouth: tuple[float, float]
+ eyes: tuple[tuple[int, int, int, int], ...]
+ top: int
+ box: tuple[int, int, int, int]
+ seeds: tuple[tuple[int, int], ...]
+ band_bottom: int
+
+ @classmethod
+ def read(cls, value: dict[str, Any], size: tuple[int, int]) -> Geometry:
+ def ints(v: Any, n: int, what: str) -> tuple[int, ...]:
+ if not isinstance(v, (list, tuple)) or len(v) != n or any(isinstance(x, bool) or not isinstance(x, (int, float)) for x in v):
+ raise ValueError(f"face.{what} must be {n} numbers")
+ return tuple(int(round(x)) for x in v)
+
+ if not isinstance(value, dict):
+ raise ValueError("the request needs a face object")
+ lower = value.get("lower_face")
+ if not isinstance(lower, dict):
+ raise ValueError("face.lower_face must be an object with top, box and seeds")
+ eyes = value.get("eyes")
+ seeds = lower.get("seeds")
+ if not isinstance(eyes, list) or len(eyes) != 2:
+ raise ValueError("face.eyes must be two boxes x0,y0,x1,y1")
+ if not isinstance(seeds, list) or not seeds:
+ raise ValueError("face.lower_face.seeds must name at least one skin point")
+ geom = cls(size=size, mouth=tuple(float(v) for v in ints(value.get("mouth"), 2, "mouth")),
+ eyes=tuple(ints(e, 4, "eyes[]") for e in eyes), top=int(lower.get("top", -1)),
+ box=ints(lower.get("box"), 4, "lower_face.box"), seeds=tuple(ints(s, 2, "lower_face.seeds[]") for s in seeds),
+ band_bottom=int(value.get("band_bottom", -1)))
+ w, h = size
+ x0, y0, x1, y1 = geom.box
+ if not (0 <= x0 < x1 <= w and 0 <= y0 < y1 <= h):
+ raise ValueError(f"face.lower_face.box {geom.box} is not inside the {w}x{h} reference")
+ if not y0 <= geom.top < y1:
+ raise ValueError("face.lower_face.top must lie inside its box")
+ if not geom.top < geom.band_bottom < h:
+ raise ValueError("face.band_bottom must lie below the lower face's top edge")
+ if not all(x0 <= x < x1 and y0 <= y < y1 for x, y in geom.seeds):
+ raise ValueError("face.lower_face.seeds must lie inside its box")
+ return geom
+
+ def around_mouth(self, left: int, up: int, right: int, down: int) -> tuple[int, int, int, int]:
+ mx, my = (int(round(v)) for v in self.mouth)
+ w, h = self.size
+ return max(mx - left, 0), max(my - up, 0), min(mx + right, w), min(my + down, h)
+
+ @property
+ def mouth_box(self) -> tuple[int, int, int, int]:
+ """Where the mouth line can be: the corners and the lift are read here (`mouth_reading`)."""
+ return self.around_mouth(46, 31, 49, 39)
+
+ @property
+ def open_zone(self) -> tuple[int, int, int, int]:
+ """Where an open mouth differs from the reference's closed one (`open_area`)."""
+ return self.around_mouth(45, 30, 45, 45)
+
+ @property
+ def step_box(self) -> tuple[int, int, int, int]:
+ """The face round the mouth on which a step between two frames is read (`step_features`)."""
+ return self.around_mouth(58, 41, 64, 79)
+
+
+# --- geometry -----------------------------------------------------------------------------------------------------
+
+def luma(a: np.ndarray) -> np.ndarray:
+ return a[..., :3] @ LUMA
+
+
+def sim(tx: float, ty: float, s: float, th: float, centre: tuple[float, float]) -> np.ndarray:
+ """A(p) = c + T + s R(th) (p - c): the reference's pixel p to the frame's (pixel indices, centres on integers)."""
+ a = math.radians(th)
+ rot = np.array([[math.cos(a), -math.sin(a)], [math.sin(a), math.cos(a)]]) * s
+ c = np.asarray(centre, float)
+ m = np.eye(3)
+ m[:2, :2] = rot
+ m[:2, 2] = c + np.array([tx, ty]) - rot @ c
+ return m
+
+
+def shift(dx: float, dy: float) -> np.ndarray:
+ m = np.eye(3)
+ m[:2, 2] = (dx, dy)
+ return m
+
+
+def scale(k: float) -> np.ndarray:
+ m = np.eye(3)
+ m[0, 0] = m[1, 1] = k
+ return m
+
+
+def pil_data(m: np.ndarray) -> tuple[float, ...]:
+ """Pillow's AFFINE data (output -> input, read on continuous coordinates, pixel centres at +0.5) for a map `m`
+ given on pixel indices."""
+ a, b = m[:2, :2], m[:2, 2]
+ b = b + 0.5 - a @ np.array([0.5, 0.5])
+ return (float(a[0, 0]), float(a[0, 1]), float(b[0]), float(a[1, 0]), float(a[1, 1]), float(b[1]))
+
+
+def warp(arr: np.ndarray, m: np.ndarray, size: tuple[int, int]) -> np.ndarray:
+ """Float channels of `arr` sampled at m(p) for every output pixel p (output -> input), bilinear, 0 outside."""
+ data = pil_data(m)
+ flat = arr[..., None] if arr.ndim == 2 else arr
+ out = np.stack([np.asarray(Image.fromarray(np.ascontiguousarray(flat[..., k], np.float32), "F").transform(
+ size, Image.Transform.AFFINE, data, resample=Image.Resampling.BILINEAR), np.float32) for k in range(flat.shape[2])], -1)
+ return out[..., 0] if arr.ndim == 2 else out
+
+
+def placement(place: tuple[int, int, int], native: tuple[int, int]) -> np.ndarray:
+ """The map from a clip frame laid on the reference (`to_reference`) to the clip's own pixels: a clip that shows the
+ reference's square at (x, y), `side` px across, at its own size."""
+ x, y, side = place
+ return scale(native[0] / side) @ shift(-x, -y)
+
+
+def to_reference(frame: Image.Image, place: tuple[int, int, int], size: tuple[int, int]) -> Image.Image:
+ """A clip frame on the reference's pixels: scaled to the square it shows (`resample.resize_cell`) and put there.
+ A clip of the whole reference (place 0, 0, its width) is the frame as it is."""
+ x, y, side = place
+ if (x, y) == (0, 0) and frame.size == (side, side) == size:
+ return frame
+ out = Image.new("RGBA", size)
+ out.paste(resample.resize_cell(frame, (side, side)), (x, y))
+ return out
+
+
+def features(image: Image.Image) -> np.ndarray:
+ return motion_anchor._features(image).astype(np.float64)
+
+
+def step_features(arr: np.ndarray, geom: Geometry) -> np.ndarray:
+ """The engine's analysis features (`loop._small_features`) of a picture in reference coordinates, on the face
+ round the mouth (`Geometry.step_box`): what a step between two frames is read on."""
+ x0, y0, x1, y1 = geom.step_box
+ crop = np.clip(arr[y0:y1, x0:x1] + 0.5, 0, 255).astype(np.uint8)
+ return loop_mod._small_features(Image.fromarray(crop, "RGBA")).astype(np.float64)
+
+
+# --- masks --------------------------------------------------------------------------------------------------------
+
+def reach(mask: np.ndarray, seeds: np.ndarray) -> np.ndarray:
+ """The pieces of `mask` (4-connected) that hold a seed."""
+ lab, sizes = parts_mod._pieces(mask)
+ if not sizes.size:
+ return np.zeros_like(mask)
+ ids = np.unique(lab[seeds & (lab >= 0)])
+ return np.isin(lab, ids) if ids.size else np.zeros_like(mask)
+
+
+def fill_holes(mask: np.ndarray) -> np.ndarray:
+ """`mask` with every hole filled: what the outside of the picture cannot reach without crossing it."""
+ border = np.zeros_like(mask)
+ border[[0, -1], :] = True
+ border[:, [0, -1]] = True
+ return ~reach(~mask, border & ~mask)
+
+
+def soft_grow(mask: np.ndarray, grow: int) -> np.ndarray:
+ """1 on `mask` grown by `grow` px, then cos² to 0 over as many px again."""
+ grown = parts_mod._dilate(mask, grow)
+ soft = grown.astype(np.float32)
+ outer = grown
+ for k in range(1, grow + 1):
+ ring_ = parts_mod._dilate(grown, k)
+ soft[ring_ & ~outer] = math.cos(k / (grow + 1) * math.pi / 2) ** 2
+ outer = ring_
+ return soft
+
+
+def eye_mask(geom: Geometry) -> np.ndarray:
+ w, h = geom.size
+ yy, xx = np.mgrid[0:h, 0:w]
+ m = np.zeros((h, w), bool)
+ for ex0, ey0, ex1, ey1 in geom.eyes:
+ m |= (xx >= ex0 - EYE_GROW) & (xx < ex1 + EYE_GROW) & (yy >= ey0 - EYE_GROW) & (yy < ey1 + EYE_GROW)
+ return m
+
+
+def lower_face(arr: np.ndarray, geom: Geometry) -> np.ndarray:
+ """The lower face of a picture already in reference coordinates: opaque skin grown from the seeds inside the box
+ below the top edge; the face's outline and the mouth's lines stop it."""
+ x0, y0, x1, y1 = geom.box
+ sub = arr[y0:y1, x0:x1]
+ skin = (luma(sub) > SKIN_LUMA) & (sub[..., 3] > 200)
+ seeds = np.zeros_like(skin)
+ for x, y in geom.seeds:
+ seeds[y - y0, x - x0] = True
+ out = np.zeros(arr.shape[:2], bool)
+ out[y0:y1, x0:x1] = reach(skin, seeds & skin)
+ return out
+
+
+def union(reference_face: np.ndarray, faces: list[np.ndarray]) -> np.ndarray:
+ """The lower faces joined, each held within REACH px of the reference's own, the holes (open mouths, lines)
+ filled."""
+ room = parts_mod._dilate(reference_face, REACH)
+ out = reference_face.copy()
+ for face in faces:
+ out |= face & room
+ return fill_holes(out)
+
+
+def weight(face_union: np.ndarray, geom: Geometry) -> np.ndarray:
+ """1 on the union grown by MARGIN px, cos² to 0 over FEATHER px outside it; 0 above the top edge rising to 1 over
+ RAMP px; 0 on the eyes, feathered over half the feather."""
+ flat = parts_mod._dilate(face_union, MARGIN)
+ w = flat.astype(np.float32)
+ inner = flat
+ for k in range(1, FEATHER + 1):
+ ring_ = parts_mod._dilate(flat, k)
+ w[ring_ & ~inner] = math.cos(k / (FEATHER + 1) * math.pi / 2) ** 2
+ inner = ring_
+ h = geom.size[1]
+ rise = np.clip((np.arange(h, dtype=np.float32) - geom.top) / RAMP, 0, 1)
+ w *= (np.sin(rise * math.pi / 2) ** 2)[:, None]
+ w *= 1 - soft_grow(eye_mask(geom), FEATHER // 2)
+ return w.astype(np.float32)
+
+
+def dark(arr: np.ndarray) -> np.ndarray:
+ """Dark grey and black: a headset, its boom, a choker (not a dark red mouth)."""
+ rgb = arr[..., :3]
+ return (luma(arr) < DARK_LUMA) & (arr[..., 3] > 128) & (rgb.max(-1) - rgb.min(-1) < DARK_CHROMA)
+
+
+def dark_reach(arr: np.ndarray, w: np.ndarray) -> np.ndarray:
+ """The dark grey and black of a picture that reaches in from where its weight `w` is under 0.5, grown DARK_GROW px
+ and feathered as far: the headset's boom and the choker cross the layer's boundary and stay; a dark mouth inside
+ the face does not reach it."""
+ d = dark(arr)
+ m = reach(d, d & (w < 0.5)) & parts_mod._dilate(w > 0, DARK_GROW + 1)
+ return soft_grow(m, DARK_GROW) if m.any() else np.zeros(w.shape, np.float32)
+
+
+def line_band(arr: np.ndarray, w: np.ndarray) -> np.ndarray:
+ """The lines a picture draws in the feather (0 < w < 0.98: the jaw line, the nose, hair), grown LINE_GROW px: a
+ line blended across the feather over the other picture's would read as a second, fainter line."""
+ band = (w > 0) & (w < 0.98)
+ m = (luma(arr) < LINE_LUMA) & (arr[..., 3] > 128) & band
+ return soft_grow(m, LINE_GROW) if m.any() else np.zeros(w.shape, np.float32)
+
+
+def hand_reach(body_ref: np.ndarray, reference: np.ndarray, w0: np.ndarray, geom: Geometry) -> np.ndarray:
+ """A hand over the face, in reference coordinates. What differs from the reference face (FOREIGN) is cut into
+ pieces, and a piece that holds a pixel outside the layer reaches in from outside. The frame's own skin (light and
+ warm: teeth and white streaks are not skin) is cut into pieces too, its drawn lines the walls: the piece that lies
+ most on the reference's own lower face is the face; any other skin piece a reaching piece touches is the hand's (a
+ palm, a finger, the sleeve under it), with the lines round it (HAND_GROW px of the reaching piece). It is a hand
+ where HAND_MIN px or more of those skin pieces lie on the layer's flat part above the chin (`band_bottom`); grown
+ HAND_GROW px and feathered. A body's own mouth is not skin and is kept only where it meets a hand; the face's own
+ skin, however far its outline moves off the reference's, is never a hand's."""
+ near = parts_mod._dilate(w0 > 0, HAND_GROW + FEATHER)
+ ys, xs = np.nonzero(near)
+ x0, y0, x1, y1 = int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1
+ b, r = body_ref[y0:y1, x0:x1], reference[y0:y1, x0:x1]
+ solid = b[..., 3] > 128
+ differ = ((np.abs(b[..., :3] - r[..., :3]).max(-1) > FOREIGN) | (np.abs(b[..., 3] - r[..., 3]) > 64)) & solid
+ differ &= near[y0:y1, x0:x1]
+ sub_w = w0[y0:y1, x0:x1]
+ lab, sizes = parts_mod._pieces(differ)
+ if not sizes.size:
+ return np.zeros(w0.shape, np.float32)
+ reaching = np.isin(lab, np.unique(lab[differ & (sub_w <= 0) & (lab >= 0)]))
+ warm = (b[..., 0] - b[..., 2] >= SKIN_WARM) & (luma(b) > SKIN_LUMA) & solid
+ skin_lab, skin_sizes = parts_mod._pieces(warm)
+ if not skin_sizes.size or not reaching.any():
+ return np.zeros(w0.shape, np.float32)
+ own = lower_face(reference, geom)[y0:y1, x0:x1]
+ face = int(np.argmax(np.bincount(skin_lab[own & (skin_lab >= 0)], minlength=skin_sizes.size))) if (own & (skin_lab >= 0)).any() else -1
+ touched = np.unique(skin_lab[parts_mod._dilate(reaching, 2) & (skin_lab >= 0)])
+ hand_skin = np.isin(skin_lab, touched[touched != face])
+ inside = (sub_w > 0.98) & (np.arange(y0, y1)[:, None] < geom.band_bottom)
+ if int((hand_skin & inside).sum()) < HAND_MIN:
+ return np.zeros(w0.shape, np.float32)
+ hand = hand_skin | (reaching & parts_mod._dilate(hand_skin, HAND_GROW))
+ out = np.zeros(w0.shape, bool)
+ out[y0:y1, x0:x1] = hand & (sub_w > 0)
+ return soft_grow(out, HAND_GROW)
+
+
+# --- anchors ------------------------------------------------------------------------------------------------------
+
+@dataclass
+class Band:
+ """The anchor reference: auto_motion's masked-NCC reference tuple on the band round the layer, and the
+ reference's features on the band's box."""
+
+ reference: tuple
+ features: np.ndarray
+
+ @classmethod
+ def read(cls, reference: Image.Image, w0: np.ndarray, geom: Geometry) -> Band:
+ ref = features(reference)
+ rows = np.arange(geom.size[1])[:, None] < geom.band_bottom
+ band = parts_mod._dilate(w0 > 0, BAND) & ~(w0 > 0.98) & rows & ~eye_mask(geom)
+ ys, xs = np.nonzero(band)
+ x0, y0, x1, y1 = int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1
+ patch = ref[y0:y1, x0:x1]
+ mask = band[y0:y1, x0:x1] & (patch[:, :, 3] > 0.05)
+ flat = patch[mask]
+ c = flat - flat.mean(axis=0)
+ return cls(((x0, y0, x1, y1), mask, c, float(np.sqrt((c * c).sum()))), patch.copy())
+
+
+def register(reference: tuple, feat: np.ndarray, centre: tuple[int, int] = (0, 0), radius: int = SEARCH) -> dict[str, Any]:
+ """The shift (dx, dy) that lays `feat` over the reference: the engine's integer match (`auto_motion._match_costs`,
+ `auto_motion._best`) and a parabola through the cost at the best offset and its neighbours on each axis."""
+ (x, y, _, _), _, _, _ = reference
+ costs, xs, ys = auto_motion._match_costs(reference, feat, centre, radius)
+ iy, ix = auto_motion._best(costs, xs, ys, reference, centre)
+ dx, dy = x - (xs + ix), y - (ys + iy)
+
+ def vertex(a: float, b: float, c: float) -> float:
+ den = a - 2 * b + c
+ return 0.5 * (a - c) / den if den > 1e-12 else 0.0
+
+ fx = vertex(costs[iy, ix - 1], costs[iy, ix], costs[iy, ix + 1]) if 0 < ix < costs.shape[1] - 1 else 0.0
+ fy = vertex(costs[iy - 1, ix], costs[iy, ix], costs[iy + 1, ix]) if 0 < iy < costs.shape[0] - 1 else 0.0
+ return {"sx": float(dx - fx), "sy": float(dy - fy), "cost": float(costs[iy, ix]),
+ "edge": bool(abs(dx - centre[0]) == radius or abs(dy - centre[1]) == radius)}
+
+
+def _local(reference: tuple, pad: int) -> tuple[tuple, tuple[int, int], tuple[int, int]]:
+ (x0, y0, x1, y1), mask, c, norm = reference
+ return ((pad, pad, pad + x1 - x0, pad + y1 - y0), mask, c, norm), (x0 - pad, y0 - pad), (x1 - x0 + 2 * pad, y1 - y0 + 2 * pad)
+
+
+def matrix(a: dict[str, float], geom: Geometry) -> np.ndarray:
+ return sim(a["tx"], a["ty"], a["s"], a["th"], geom.mouth)
+
+
+def fit(band: Band, feat: np.ndarray, geom: Geometry) -> dict[str, Any]:
+ """The anchor of one frame, read on the band (module docstring "Anchor")."""
+ first = register(band.reference, feat)
+ search = SEARCH
+ if first["edge"]:
+ search = 2 * SEARCH
+ first = register(band.reference, feat, (0, 0), search)
+ loc, origin, size = _local(band.reference, FINE + 1)
+ c = np.asarray(geom.mouth)
+
+ def at(t: np.ndarray, s: float, th: float) -> tuple[float, np.ndarray, bool]:
+ m = sim(t[0], t[1], s, th, geom.mouth)
+ wf = warp(feat, m @ shift(*origin), size)
+ rr = register(loc, wf.astype(np.float64), (0, 0), FINE)
+ return rr["cost"], m @ shift(-rr["sx"], -rr["sy"]), rr["edge"]
+
+ def best_on(t: np.ndarray, scales: np.ndarray, thetas: np.ndarray) -> dict[str, Any]:
+ costs = np.array([[at(t, s, th)[0] for th in thetas] for s in scales])
+ i, j = np.unravel_index(np.argmin(costs), costs.shape)
+
+ def vertex(a: float, b: float, c: float) -> float:
+ den = a - 2 * b + c
+ return float(np.clip(0.5 * (a - c) / den, -0.5, 0.5)) if den > 1e-12 else 0.0
+
+ ds = vertex(costs[i - 1, j], costs[i, j], costs[i + 1, j]) if 0 < i < len(scales) - 1 else 0.0
+ dt = vertex(costs[i, j - 1], costs[i, j], costs[i, j + 1]) if 0 < j < len(thetas) - 1 else 0.0
+ s = float(scales[i] + ds * (scales[1] - scales[0]))
+ th = float(thetas[j] + dt * (thetas[1] - thetas[0]))
+ cost, m, edge = at(t, s, th)
+ tt = m[:2, 2] - c + m[:2, :2] @ c
+ return {"tx": float(tt[0]), "ty": float(tt[1]), "s": s, "th": th, "cost": float(cost), "fine_edge": bool(edge),
+ "grid_edge": bool(i in (0, len(scales) - 1) or j in (0, len(thetas) - 1))}
+
+ t0 = np.array([-first["sx"], -first["sy"]])
+ rec = best_on(t0, SCALES, THETAS)
+ rec["wide_grid"] = False
+ if rec["grid_edge"]:
+ rec = {**best_on(t0, WIDE_SCALES, WIDE_THETAS), "wide_grid": True}
+ # The place a size and tilt need can lie further from the first, place-only reading than FINE: the place-only
+ # match settles where the band's tilted edges agree best, not where the tilt turns about. The grid is read again
+ # round the place, size and tilt it found until the place stays.
+ def moved(r: dict[str, Any], t: np.ndarray) -> bool:
+ return math.hypot(r["tx"] - t[0], r["ty"] - t[1]) > FINE - 1
+
+ t, rounds, again = t0, 0, rec["fine_edge"] or moved(rec, t0)
+ while again and rounds < RECENTRE_ROUNDS:
+ t = np.array([rec["tx"], rec["ty"]])
+ local = best_on(t, rec["s"] + np.array([-1.0, 0.0, 1.0]) * (SCALES[1] - SCALES[0]),
+ rec["th"] + np.array([-1.0, 0.0, 1.0]) * (THETAS[1] - THETAS[0]))
+ again = local["fine_edge"] or local["grid_edge"] or moved(local, t)
+ rec = {**local, "grid_edge": rec["grid_edge"], "wide_grid": rec["wide_grid"]}
+ rounds += 1
+ rec["recentred"] = rounds
+ rec["search_px"] = search
+ rec["edge"] = bool(first["edge"])
+ rec["place_only"] = [float(t0[0]), float(t0[1])]
+ return rec
+
+
+def band_residual(band: Band, feat: np.ndarray, m: np.ndarray) -> float:
+ """Mean |frame - reference| (0..255, colour matched by the median) on the band where both are opaque, the frame
+ read through `m`: how well the anchor lays the reference's face on the frame."""
+ (x0, y0, x1, y1), mask, _, _ = band.reference
+ w = warp(feat, m @ shift(x0, y0), (x1 - x0, y1 - y0))
+ r = band.features
+ keep = mask & (w[..., 3] > 0.99) & (r[..., 3] > 0.99)
+ d = (w[keep][:, :3] - r[keep][:, :3]) * 255
+ return float(np.abs(d - np.median(d, 0)).mean()) if keep.any() else float("nan")
+
+
+def hampel(rows: list[dict[str, Any]], cyclic: bool) -> list[dict[str, Any]]:
+ """Each anchor number against the median of HAMPEL_WINDOW frames round it (wrapping on a loop): a reading further
+ from it than three scaled MADs (and than HAMPEL_FLOOR) is a misreading, and the frame takes the window's medians
+ for all four numbers (its place was read at the misread size and tilt). Every other reading stays as read: a head's
+ real bounce is not flattened."""
+ n, h = len(rows), HAMPEL_WINDOW // 2
+ out = []
+ for t in range(n):
+ idx = [(t + k) % n for k in range(-h, h + 1)] if cyclic else list(range(max(0, t - h), min(n, t + h + 1)))
+ med, bad = {}, False
+ for k in ("tx", "ty", "s", "th"):
+ v = np.array([rows[q][k] for q in idx])
+ med[k] = float(np.median(v))
+ mad = 1.4826 * float(np.median(np.abs(v - med[k])))
+ bad |= abs(rows[t][k] - med[k]) > max(3 * mad, HAMPEL_FLOOR[k])
+ out.append({**(med if bad else {k: float(rows[t][k]) for k in ("tx", "ty", "s", "th")}), "misread": bool(bad)})
+ return out
+
+
+# --- the mouth ----------------------------------------------------------------------------------------------------
+
+def mouth_reading(arr: np.ndarray, geom: Geometry) -> dict[str, Any] | None:
+ """How a mouth in reference coordinates smiles: the dark line or opening nearest the mouth point (DROP under the
+ box's skin, off the headset's grey), its corners its leftmost and rightmost pixels. `lift` is how far its top edge
+ in the middle third sits below the chord between the corners (px; + the corners turned up: a smile, - turned down:
+ an O or a pout); `height` the mean of its bottom less top there, plus one. None where no mouth line is found."""
+ x0, y0, x1, y1 = geom.mouth_box
+ a = arr[y0:y1, x0:x1]
+ lum = luma(a)
+ solid = a[..., 3] > 200
+ if not solid.any():
+ return None
+ skin = float(np.median(lum[solid]))
+ grey = (a[..., :3].max(-1) - a[..., :3].min(-1) < DARK_CHROMA) & (lum < 120)
+ m = (lum < skin - DROP) & solid & ~parts_mod._dilate(grey, 2)
+ yy, xx = np.mgrid[0:a.shape[0], 0:a.shape[1]]
+ cx, cy = geom.mouth[0] - x0, geom.mouth[1] - y0
+ near = m & ((xx - cx) ** 2 / 30 ** 2 + (yy - cy) ** 2 / 16 ** 2 < 1)
+ if not near.any():
+ return None
+ g = reach(m, near)
+ ys, xs = np.nonzero(g)
+ if len(xs) < 6:
+ return None
+ left, right = int(xs.min()), int(xs.max())
+ width = right - left + 1
+ if width < 8:
+ return None
+ yl, yr = float(ys[xs <= left + 1].mean()), float(ys[xs >= right - 1].mean())
+ cols = [x for x in range(int(left + width / 3), int(right - width / 3) + 1) if (xs == x).any()]
+ if not cols:
+ return None
+ top = np.array([ys[xs == x].min() for x in cols], float)
+ bottom = np.array([ys[xs == x].max() for x in cols], float)
+ chord = yl + (np.array(cols, float) - left) / max(right - left, 1) * (yr - yl)
+ return {"lift": round(float((top - chord).mean()), 2), "width": int(width),
+ "height": round(float(bottom.mean() - top.mean() + 1), 2), "px": int(g.sum())}
+
+
+def expression(reading: dict[str, Any] | None) -> str | None:
+ """A closed mouth's expression: 'smile' with SMILE_FROM px of lift or more, 'neutral' under it; None for an open
+ mouth or none read. One line, not a gap between two: a closed frame left out between two of the same expression
+ would cut a run of that expression into two turns."""
+ if reading is None or reading["height"] > CLOSED_HEIGHT:
+ return None
+ return "smile" if reading["lift"] >= SMILE_FROM else "neutral"
+
+
+def open_area(arr: np.ndarray, reference: np.ndarray, geom: Geometry) -> int:
+ """How big a mouth reads: pixels round the mouth (`Geometry.open_zone`) that differ from the reference's closed
+ mouth by more than OPEN_LUMA."""
+ x0, y0, x1, y1 = geom.open_zone
+ return int((np.abs(luma(arr[y0:y1, x0:x1]) - luma(reference[y0:y1, x0:x1])) > OPEN_LUMA).sum())
+
+
+# --- laying -------------------------------------------------------------------------------------------------------
+
+@dataclass
+class Sprite:
+ """A talking frame's lower face at the clip's own resolution: `image` (RGBA, alpha = the weight less the holes)
+ and `to_ref`, the map from its pixels to the reference's."""
+
+ image: Image.Image
+ to_ref: np.ndarray
+
+
+def colour_offset(reference: np.ndarray, picture: np.ndarray, w: np.ndarray) -> np.ndarray:
+ """The median of reference - picture over the feather (0.02 < w < 0.98) where both are opaque."""
+ band = (w > 0.02) & (w < 0.98) & (picture[..., 3] > 254) & (reference[..., 3] > 254)
+ return np.median(reference[band][:, :3] - picture[band][:, :3], 0) if band.any() else np.zeros(3)
+
+
+def face_sprite(native: Image.Image, to_native: np.ndarray, w0: np.ndarray, reference: np.ndarray) -> tuple[Sprite, dict[str, Any]]:
+ """The sprite of one talking frame. `to_native` maps the reference's pixels to the frame's own (the frame's
+ anchor, then its placement)."""
+ nat = np.asarray(native.convert("RGBA"), np.float32)
+ size = (w0.shape[1], w0.shape[0])
+ face_ref = warp(nat, to_native, size)
+ hole = np.maximum(dark_reach(face_ref, w0), line_band(face_ref, w0))
+ w_face = w0 * (1 - hole)
+ off = colour_offset(reference, face_ref, w_face)
+ back = np.linalg.inv(to_native)
+ w_nat = warp(w_face, back, native.size)
+ w_nat[w_nat < 1.0 / 512] = 0
+ ys, xs = np.nonzero(w_nat > 0)
+ x0, y0 = max(int(xs.min()) - 2, 0), max(int(ys.min()) - 2, 0)
+ x1, y1 = min(int(xs.max()) + 3, native.size[0]), min(int(ys.max()) + 3, native.size[1])
+ crop = nat[y0:y1, x0:x1].copy()
+ crop[..., :3] = np.clip(crop[..., :3] + off, 0, 255)
+ crop[..., 3] *= w_nat[y0:y1, x0:x1]
+ image = Image.fromarray(np.clip(crop + 0.5, 0, 255).astype(np.uint8), "RGBA")
+ return Sprite(image, back @ shift(x0, y0)), {"colour_offset": [round(float(v), 2) for v in off],
+ "hole_px": int((hole > 0.5).sum()), "origin": [x0, y0], "size": [x1 - x0, y1 - y0]}
+
+
+def body_keep(body: np.ndarray, to_body: np.ndarray, w0: np.ndarray, reference: np.ndarray, geom: Geometry) -> tuple[np.ndarray, dict[str, Any]]:
+ """What of a body frame stays on top of the layer (module docstring "Keep"), in the frame's own pixels, and how
+ much of it each part is. `to_body` maps the reference's pixels to the frame's (its anchor and placement)."""
+ size = (body.shape[1], body.shape[0])
+ back = np.linalg.inv(to_body)
+ w_ell = warp(w0, back, size)
+ w_ell[w_ell < 1e-4] = 0
+ held = dark_reach(body, w_ell)
+ lines = line_band(body, w_ell)
+ body_ref = warp(body, to_body, (w0.shape[1], w0.shape[0]))
+ hand_ref = hand_reach(body_ref, reference, w0, geom)
+ hand = warp(hand_ref, back, size) if hand_ref.any() else np.zeros(w_ell.shape, np.float32)
+ keep = np.maximum(np.maximum(held, lines), hand) * (w_ell > 0)
+ return keep.astype(np.float32), {"dark_px": int((held > 0.5).sum()), "line_px": int((lines > 0.5).sum()),
+ "hand_px": int((hand > 0.5).sum())}
+
+
+def keep_overlay(body: np.ndarray, keep: np.ndarray) -> tuple[Image.Image, tuple[int, int]] | None:
+ """The body's own pixels where they stay on top, as a picture to draw over the layer, and where it goes."""
+ alpha = body[..., 3] * keep
+ ys, xs = np.nonzero(np.clip(alpha + 0.5, 0, 255) >= 1)
+ if not len(xs):
+ return None
+ x0, y0, x1, y1 = int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1
+ crop = body[y0:y1, x0:x1].copy()
+ crop[..., 3] = alpha[y0:y1, x0:x1]
+ return Image.fromarray(np.clip(crop + 0.5, 0, 255).astype(np.uint8), "RGBA"), (x0, y0)
+
+
+def over(dst: np.ndarray, src: np.ndarray, at: tuple[int, int] = (0, 0)) -> np.ndarray:
+ """Source-over (straight alpha) of `src` on `dst` with its corner at `at`; float RGBA 0..255, in place."""
+ x0, y0 = at
+ h, w = src.shape[:2]
+ d = dst[y0:y0 + h, x0:x0 + w]
+ sa, da = src[..., 3:4] / 255, d[..., 3:4] / 255
+ a = sa + da * (1 - sa)
+ rgb = np.where(a > 0, (src[..., :3] * sa + d[..., :3] * da * (1 - sa)) / np.maximum(a, 1e-6), 0)
+ d[..., :3] = rgb
+ d[..., 3:4] = a * 255
+ return dst
+
+
+def placed_sprite(sprite: Sprite, to_body: np.ndarray, size: tuple[int, int]) -> tuple[Image.Image, tuple[int, int]]:
+ """The sprite on a body frame's pixels, and where its box goes: one `resample.transform_cell` through
+ to_body . to_ref, after one `resample.resize_cell` where it is laid smaller than SHRINK_FROM."""
+ m = to_body @ sprite.to_ref
+ k = math.sqrt(abs(float(np.linalg.det(m[:2, :2]))))
+ image = sprite.image
+ if k < SHRINK_FROM:
+ w, h = image.size
+ nw, nh = max(1, round(w * k)), max(1, round(h * k))
+ image = resample.resize_cell(image, (nw, nh))
+ m = m @ np.diag([w / nw, h / nh, 1.0])
+ w, h = image.size
+ corners = m @ np.array([[-1, w, -1, w], [-1, -1, h, h], [1, 1, 1, 1]], float)
+ x0, y0 = max(int(math.floor(corners[0].min())) - 1, 0), max(int(math.floor(corners[1].min())) - 1, 0)
+ x1, y1 = min(int(math.ceil(corners[0].max())) + 2, size[0]), min(int(math.ceil(corners[1].max())) + 2, size[1])
+ return resample.transform_cell(image, (x1 - x0, y1 - y0), pil_data(np.linalg.inv(m) @ shift(x0, y0))), (x0, y0)
+
+
+def lay(body: Image.Image, sprite: Sprite | None, to_body: np.ndarray, keep: tuple[Image.Image, tuple[int, int]] | None) -> Image.Image:
+ """A body frame with the face layer: the sprite over it, then what of the body stays on top over that. No sprite
+ is the body frame as it is."""
+ if sprite is None:
+ return body.convert("RGBA")
+ out = np.asarray(body.convert("RGBA"), np.float32).copy()
+ placed, at = placed_sprite(sprite, to_body, body.size)
+ over(out, np.asarray(placed, np.float32), at)
+ if keep is not None:
+ over(out, np.asarray(keep[0], np.float32), keep[1])
+ return Image.fromarray(np.clip(out + 0.5, 0, 255).astype(np.uint8), "RGBA")
diff --git a/sprite_gen/video/face_pack.py b/sprite_gen/video/face_pack.py
new file mode 100644
index 0000000..e0e5bf3
--- /dev/null
+++ b/sprite_gen/video/face_pack.py
@@ -0,0 +1,466 @@
+# SPDX-License-Identifier: Apache-2.0
+"""`sprite-gen video-face` — a face pack: the lower face of a talking clip, cut to be laid on body motions.
+
+The request (docs/face-parts.md "The request") names a reference cell and the face on it, a talking clip (keyed
+frames, and where on the reference its frames show) and the body motions the face is laid on: one loop the others
+come back to, and clips that play once. The pack is a folder a player reads (`sprite-gen-face-v1.mjs`, copied in) and
+`face-bake` bakes from:
+
+- **Anchors** for every frame of every clip, read on the band round the reference's own lower face (so a clip's
+ anchors are the same whatever else the request names), and the **weight**: the lower faces the anchors lay on the
+ reference, joined (`face_layer`).
+- **Face frames**: each talking frame's mouth read in reference coordinates (`face_layer.mouth_reading`), its open
+ area (`face_layer.open_area`) and its step to every other frame (`face_layer.step_features`, over the clip's median
+ step). A closed mouth that smiles is in the smile set, one that does not in the neutral set; an open run takes the
+ set of the closed frames on both its sides, and a run between a smile and a neutral frame turns from one to the
+ other and is in neither. A frame in a set is cut as a sprite (`face_layer.face_sprite`).
+- **Body clips**: each frame's cell, its anchor and what of it stays on top of the layer (`face_layer.body_keep`).
+ A clip that says something with its own mouth has an `own_mouth` span, from its start to the frame its mouth has
+ closed after its own sound: given as seconds, or read off its mouth timeline (the first `closed` frame after the
+ last `voiced` one). The layer stays off there; the clip's own sound plays.
+- **Entry**: where the layer comes on — a clip's `own_mouth` end, or a clip's first frame — the engine reads the
+ clip's own mouth and enters in the set its lift reads by the talking frames' own line (`face_layer.SMILE_FROM`), at
+ the rest (a closed frame of that set the line opens from and closes back to) nearest in lift, the least switch
+ among those within LIFT_TIE px of it.
+- **Switch**: per body frame and per rest the pack uses, how far laying that rest moves the frame's face, over the
+ larger of the clip's median step and the talking clip's (a body that only breathes steps far less than a face that
+ talks); the layer comes on and goes off only where that is within the jump line.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import os
+import shutil
+import tempfile
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+
+from sprite_gen._deps import np
+from sprite_gen.util.console import utf8_streams
+from sprite_gen.video import face_layer as fl
+
+PROG = "video-face"
+REQUEST_KIND = "sprite-gen-face-request"
+PACK_KIND = "sprite-gen-face-pack"
+VERSION = 1
+PLAYER_PATH = Path(__file__).with_name("sprite-gen-face-v1.mjs")
+LIFT_TIE = 0.5 # px: rests this near the nearest lift are told apart by their switch
+
+# The voice -> mouth level rule the player runs (sprite-gen-face-v1.mjs `mouthTrack`): the broadcast mouth's numbers,
+# docs/face-parts.md "The line".
+MOUTH_RULES = {"closed_under": 0.12, "prominence": 0.05, "min_gap_s": 1 / 12, "open_lo_hi": [2, 5], "surprise_at": 0.85,
+ "lead_frames": 1, "max_step": 3, "normal_top": 6, "surprise_top": 6, "wide_from": 4,
+ "surprise_tags": ["[surprised]", "[gasps]"]}
+# The face frame pick (`pickFaces`): a hold, a jump within a line and an entry from or exit to the rest cost these
+# (in mouth levels); a mouth opened less than the line wants weighs UNDER_WEIGHT times one opened more.
+PICK_RULES = {"hold_cost": 0.6, "jump_cost": 1.5, "edge_cost": 1.0, "under_weight": 3.0}
+
+
+@dataclass
+class Clip:
+ name: str
+ files: list[Path]
+ place: tuple[int, int, int]
+ native: tuple[int, int]
+ loop: bool = False
+ own_mouth: dict[str, Any] | None = None
+ sound: Path | None = None
+ rows: list[dict[str, Any]] = field(default_factory=list)
+ used: list[dict[str, Any]] = field(default_factory=list)
+
+ def frame(self, k: int) -> Image.Image:
+ return Image.open(self.files[k]).convert("RGBA")
+
+ def on_reference(self, k: int, size: tuple[int, int]) -> Image.Image:
+ return fl.to_reference(self.frame(k), self.place, size)
+
+ def to_native(self, k: int, geom: fl.Geometry) -> np.ndarray:
+ """The reference's pixels to frame k's own: its anchor, then its placement."""
+ return fl.placement(self.place, self.native) @ fl.matrix(self.used[k], geom)
+
+
+# --- the request --------------------------------------------------------------------------------------------------
+
+def _frames(folder: Path, what: str) -> list[Path]:
+ files = sorted(folder.glob("frame-*.png"))
+ if len(files) < 6:
+ raise SystemExit(f"{PROG}: {what} {folder} holds {len(files)} frame-*.png; a clip needs six or more")
+ return files
+
+
+def _place(value: Any, size: tuple[int, int], what: str) -> tuple[int, int, int]:
+ if value is None:
+ return (0, 0, size[0])
+ if not isinstance(value, list) or len(value) != 3 or any(isinstance(v, bool) or not isinstance(v, int) for v in value) or value[2] <= 0:
+ raise SystemExit(f"{PROG}: {what}.place must be [x, y, side]: integers, where on the reference the clip's square frame shows")
+ return tuple(value) # type: ignore[return-value]
+
+
+def own_mouth_from_timeline(path: Path, frames: int, fps: float) -> dict[str, Any]:
+ """The span a clip talks with its own mouth, off its mouth timeline (a TSV with `frame`, `state` and `sound`
+ columns, one row a frame): from its first frame to the first `closed` frame after its last `voiced` one."""
+ with open(path, newline="", encoding="utf-8") as fh:
+ rows = list(csv.DictReader(fh, delimiter="\t"))
+ if not rows or not {"frame", "state", "sound"} <= set(rows[0]):
+ raise SystemExit(f"{PROG}: {path} is not a mouth timeline: it needs the tab-separated columns frame, state and sound")
+ if [int(r["frame"]) for r in rows] != list(range(frames)):
+ raise SystemExit(f"{PROG}: {path} has {len(rows)} rows for frames {rows[0]['frame']}..{rows[-1]['frame']}; the clip has {frames} frames from 0")
+ voiced = [int(r["frame"]) for r in rows if r["sound"] == "voiced"]
+ if not voiced:
+ raise SystemExit(f"{PROG}: {path} has no voiced frame; a clip that says nothing has no own_mouth span")
+ last = voiced[-1]
+ end = next((int(r["frame"]) for r in rows[last + 1:] if r["state"] == "closed"), None)
+ if end is None:
+ raise SystemExit(f"{PROG}: {path}: the mouth never reads closed after the last voiced frame {last}; the face layer has no frame to come on")
+ return {"start_frame": 0, "end_frame": end, "start_s": 0.0, "end_s": round(end / fps, 4), "last_voiced_frame": last,
+ "from": "mouth_timeline", "timeline": str(path)}
+
+
+def read_request(path: Path) -> tuple[dict[str, Any], fl.Geometry, Image.Image, Clip, list[Clip]]:
+ try:
+ req = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as exc:
+ raise SystemExit(f"{PROG}: --request {path}: {exc}") from None
+ if not isinstance(req, dict) or req.get("kind") != REQUEST_KIND or req.get("version") != VERSION:
+ raise SystemExit(f"{PROG}: --request {path}: a {REQUEST_KIND} version {VERSION}")
+ base = path.parent
+ fps = req.get("fps", 24)
+ if isinstance(fps, bool) or not isinstance(fps, (int, float)) or fps <= 0:
+ raise SystemExit(f"{PROG}: fps must be a positive number")
+ reference = Image.open(base / str(req.get("reference"))).convert("RGBA")
+ try:
+ geom = fl.Geometry.read(req.get("face"), reference.size)
+ except ValueError as exc:
+ raise SystemExit(f"{PROG}: {exc}") from None
+
+ def clip(value: Any, what: str, name: str) -> Clip:
+ if not isinstance(value, dict) or not isinstance(value.get("frames"), str):
+ raise SystemExit(f"{PROG}: {what} needs frames (a folder of keyed frame-*.png)")
+ files = _frames(base / value["frames"], what)
+ native = Image.open(files[0]).size
+ if native[0] != native[1]:
+ raise SystemExit(f"{PROG}: {what} frames are {native[0]}x{native[1]}; a clip's frames are square")
+ place = _place(value.get("place"), reference.size, what)
+ if value.get("place") is None and native != reference.size:
+ raise SystemExit(f"{PROG}: {what} frames are {native[0]}x{native[1]}, the reference {reference.size[0]}x{reference.size[1]}; "
+ "give place [x, y, side] for a clip of part of it")
+ return Clip(name, files, place, native)
+
+ talk = clip(req.get("talk"), "talk", "talk")
+ bodies = []
+ names = set()
+ for k, value in enumerate(req.get("body") or []):
+ name = value.get("name") if isinstance(value, dict) else None
+ if not isinstance(name, str) or not name or name in names or not all(c.isalnum() or c in "-_" for c in name):
+ raise SystemExit(f"{PROG}: body[{k}].name must be a new name of letters, digits, - and _")
+ names.add(name)
+ c = clip(value, f"body[{k}]", name)
+ c.loop = bool(value.get("loop", False))
+ timeline, span = value.get("mouth_timeline"), value.get("own_mouth")
+ if timeline is not None and span is not None:
+ raise SystemExit(f"{PROG}: body {name}: own_mouth or mouth_timeline, not both")
+ if c.loop and (timeline is not None or span is not None):
+ raise SystemExit(f"{PROG}: body {name} loops; the loop the others come back to talks only with the face layer")
+ if timeline is not None:
+ c.own_mouth = own_mouth_from_timeline(base / str(timeline), len(c.files), fps)
+ elif span is not None:
+ if not isinstance(span, list) or len(span) != 2 or not all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in span) or span[0] != 0 or span[1] <= 0:
+ raise SystemExit(f"{PROG}: body {name}: own_mouth must be [0, end seconds]: the clip talks with its own mouth from its start")
+ end = int(round(span[1] * fps))
+ if end >= len(c.files):
+ raise SystemExit(f"{PROG}: body {name}: own_mouth ends at frame {end}, past the clip's last frame {len(c.files) - 1}")
+ c.own_mouth = {"start_frame": 0, "end_frame": end, "start_s": 0.0, "end_s": round(end / fps, 4), "from": "own_mouth"}
+ if value.get("sound") is not None:
+ c.sound = base / str(value["sound"])
+ if not c.sound.is_file():
+ raise SystemExit(f"{PROG}: body {name}: sound {c.sound} is not a file")
+ bodies.append(c)
+ loops = [c for c in bodies if c.loop]
+ if len(loops) != 1:
+ raise SystemExit(f"{PROG}: the request needs exactly one body with loop true (the rest motion the others come back to); it has {len(loops)}")
+ return req, geom, reference, talk, bodies
+
+
+# --- reading ------------------------------------------------------------------------------------------------------
+
+def each(fn, items, jobs: int) -> list[Any]:
+ """fn over items, in their order, on `jobs` threads (every frame is read on its own)."""
+ items = list(items)
+ if jobs <= 1:
+ return [fn(x) for x in items]
+ with ThreadPoolExecutor(max_workers=jobs) as pool:
+ return list(pool.map(fn, items))
+
+
+_WORK: dict[str, Any] = {}
+
+
+def _work_init(band: fl.Band, geom: fl.Geometry) -> None:
+ _WORK["band"], _WORK["geom"] = band, geom
+
+
+def _anchor(task: tuple[str, tuple[int, int, int]]) -> dict[str, Any]:
+ """One frame's anchor and how well it lays the reference's face on the frame (a worker's task)."""
+ band, geom = _WORK["band"], _WORK["geom"]
+ path, place = task
+ feat = fl.features(fl.to_reference(Image.open(path).convert("RGBA"), place, geom.size))
+ a = fl.fit(band, feat, geom)
+ a["band_residual"] = fl.band_residual(band, feat, fl.matrix(a, geom))
+ a["band_residual_place_only"] = fl.band_residual(band, feat, fl.sim(*a["place_only"], 1.0, 0.0, geom.mouth))
+ a["band_residual_fixed"] = fl.band_residual(band, feat, np.eye(3))
+ return a
+
+
+def read_anchors(clips: list[Clip], band: fl.Band, geom: fl.Geometry, jobs: int) -> None:
+ """Every frame's anchor, read on `jobs` processes (the reading is most of a pack's time), then each clip's
+ misreadings taken out (`face_layer.hampel`)."""
+ tasks = [(str(p), c.place) for c in clips for p in c.files]
+ if jobs <= 1:
+ _work_init(band, geom)
+ rows = [_anchor(t) for t in tasks]
+ else:
+ with ProcessPoolExecutor(max_workers=jobs, initializer=_work_init, initargs=(band, geom)) as pool:
+ rows = list(pool.map(_anchor, tasks, chunksize=2))
+ at = 0
+ for c in clips:
+ c.rows = rows[at:at + len(c.files)]
+ c.used = fl.hampel(c.rows, cyclic=c.loop)
+ at += len(c.files)
+
+
+def lower_faces(clip: Clip, geom: fl.Geometry, jobs: int) -> list[np.ndarray]:
+ def one(k: int) -> np.ndarray:
+ arr = np.asarray(clip.on_reference(k, geom.size), np.float32)
+ return fl.lower_face(fl.warp(arr, fl.matrix(clip.used[k], geom), geom.size), geom)
+
+ return each(one, range(len(clip.files)), jobs)
+
+
+def face_sets(readings: list[dict[str, Any] | None]) -> dict[str, list[int]]:
+ """The smile and neutral sets of the talking frames (module docstring "Face frames")."""
+ tags = {k: fl.expression(r) for k, r in enumerate(readings) if fl.expression(r) is not None}
+ closed = sorted(tags)
+ sets: dict[str, list[int]] = {"smile": [], "neutral": [], "turn": []}
+ for k in closed:
+ sets[tags[k]].append(k) # type: ignore[index]
+ for a, b in zip(closed, closed[1:]):
+ run = list(range(a + 1, b))
+ if run:
+ sets[tags[a] if tags[a] == tags[b] else "turn"] += run # type: ignore[index]
+ return {k: sorted(v) for k, v in sets.items()}
+
+
+def _digest(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _six(m: np.ndarray) -> list[float]:
+ return [round(float(v), 6) for v in (m[0, 0], m[0, 1], m[0, 2], m[1, 0], m[1, 1], m[1, 2])]
+
+
+def _summary(values: list[float]) -> dict[str, float]:
+ v = np.asarray([x for x in values if x == x], float)
+ return {"median": round(float(np.median(v)), 3), "p90": round(float(np.percentile(v, 90)), 3), "max": round(float(v.max()), 3)} if v.size else {}
+
+
+def build(request: Path, out: Path, jobs: int = 0) -> dict[str, Any]:
+ req, geom, reference, talk, bodies = read_request(request)
+ if out.exists():
+ raise SystemExit(f"{PROG}: --out {out} exists; a pack is written whole into a new folder")
+ fps = float(req.get("fps", 24))
+ jobs = jobs or os.cpu_count() or 1
+ ref = np.asarray(reference, np.float32)
+ clips = [talk, *bodies]
+
+ # anchors, read on the band round the reference's own lower face: a clip's anchors are the same whatever else
+ # the request names; then the lower faces they lay on the reference, joined, give the weight
+ ref_face = fl.lower_face(ref, geom)
+ if not ref_face.any():
+ raise SystemExit(f"{PROG}: no lower face grows from face.lower_face.seeds on the reference: the seeds must sit on its skin")
+ band = fl.Band.read(reference, fl.weight(fl.fill_holes(ref_face), geom), geom)
+ read_anchors(clips, band, geom, jobs)
+ faces = []
+ for c in clips:
+ faces.extend(lower_faces(c, geom, jobs))
+ face_union = fl.union(ref_face, faces)
+ del faces
+ w0 = fl.weight(face_union, geom)
+
+ # the talking frames
+ def talking(k: int) -> tuple[Any, int, np.ndarray]:
+ face_ref = fl.warp(np.asarray(talk.frame(k), np.float32), talk.to_native(k, geom), geom.size)
+ return fl.mouth_reading(face_ref, geom), fl.open_area(face_ref, ref, geom), fl.step_features(face_ref, geom)
+
+ readings, areas, feats = (list(v) for v in zip(*each(talking, range(len(talk.files)), jobs)))
+ F = np.stack(feats)
+ steps = np.abs(np.diff(F, axis=0)).mean(1)
+ med = float(np.median(steps))
+ D = np.stack([np.abs(F - F[a]).mean(1) for a in range(len(F))]) / med
+ sets = face_sets(readings)
+ in_set = {k: s for s in ("smile", "neutral") for k in sets[s]}
+ if not sets["smile"] and not sets["neutral"]:
+ raise SystemExit(f"{PROG}: no talking frame reads a closed mouth: the face clip has no rest to open from")
+
+ tmp = Path(tempfile.mkdtemp(prefix=f".{out.name}.", dir=out.parent if out.parent.exists() else None))
+ try:
+ (tmp / "face").mkdir()
+ sprites: dict[int, fl.Sprite] = {}
+
+ def face_row(k: int) -> dict[str, Any]:
+ row: dict[str, Any] = {"index": k, "set": in_set.get(k), "open_px": areas[k],
+ "closed": fl.expression(readings[k]) is not None, "reading": readings[k]}
+ if k in in_set:
+ sprite, rec = fl.face_sprite(talk.frame(k), talk.to_native(k, geom), w0, ref)
+ name = f"face/frame-{k:05d}.png"
+ sprite.image.save(tmp / name)
+ # laid from the saved picture, as a player reads it
+ sprites[k] = fl.Sprite(Image.open(tmp / name).convert("RGBA"), sprite.to_ref)
+ row.update({"file": name, "to_ref": _six(sprite.to_ref), "sprite": rec})
+ return row
+
+ face_rows = each(face_row, range(len(talk.files)), jobs)
+
+ def lift(k: int) -> float:
+ return float(readings[k]["lift"]) # type: ignore[index]
+
+ closed_in = {s: [k for k in sets[s] if face_rows[k]["closed"]] for s in ("smile", "neutral")}
+
+ # the body clips
+ body_rows, rests_used = [], set()
+ prepared = []
+ for c in bodies:
+ folder = tmp / "body" / c.name
+ folder.mkdir(parents=True)
+
+ def body_frame(k: int, c: Clip = c, folder: Path = folder) -> tuple[Any, ...]:
+ cell = folder / f"frame-{k:05d}.png"
+ shutil.copyfile(c.files[k], cell)
+ body = np.asarray(c.frame(k), np.float32)
+ m = c.to_native(k, geom)
+ keep, krec = fl.body_keep(body, m, w0, ref, geom)
+ overlay = fl.keep_overlay(body, keep)
+ kept = None
+ if overlay is not None:
+ kname = folder / f"keep-{k:05d}.png"
+ overlay[0].save(kname)
+ kept = {"file": kname.relative_to(tmp).as_posix(), "at": list(overlay[1]), **krec}
+ body_ref = fl.warp(body, m, geom.size)
+ return cell.relative_to(tmp).as_posix(), kept, m, fl.mouth_reading(body_ref, geom), fl.step_features(body_ref, geom)
+
+ cells, keeps, to_body, own, steps_b = (list(v) for v in zip(*each(body_frame, range(len(c.files)), jobs)))
+ Fb = np.stack(steps_b)
+ med_b = float(np.median(np.abs(np.diff(Fb, axis=0)).mean(1)))
+ entry_frame = c.own_mouth["end_frame"] if c.own_mouth else 0
+ prepared.append((c, folder, cells, keeps, to_body, Fb, med_b, own, entry_frame))
+
+ def switch(c: Clip, to_body: list[np.ndarray], keeps: list[Any], Fb: np.ndarray, med_b: float, k: int, rest: int) -> float:
+ kept = None if keeps[k] is None else (Image.open(tmp / keeps[k]["file"]).convert("RGBA"), tuple(keeps[k]["at"]))
+ laid = fl.lay(c.frame(k), sprites[rest], to_body[k], kept)
+ comp = fl.warp(np.asarray(laid, np.float32), to_body[k], geom.size)
+ return float(np.abs(fl.step_features(comp, geom) - Fb[k]).mean() / max(med_b, med))
+
+ for c, folder, cells, keeps, to_body, Fb, med_b, own, entry_frame in prepared:
+ reading = own[entry_frame]
+ if reading is None:
+ raise SystemExit(f"{PROG}: body {c.name}: no mouth reads at frame {entry_frame}, where the face layer comes on")
+ target = float(reading["lift"])
+ entry_set = "smile" if target >= fl.SMILE_FROM else "neutral"
+ candidates = [(abs(lift(k) - target), k) for k in closed_in[entry_set]]
+ if not candidates:
+ raise SystemExit(f"{PROG}: body {c.name} enters in the {entry_set} set, which has no closed talking frame to rest on")
+ nearest = min(candidates)[0]
+ tied = [k for d, k in candidates if d <= nearest + LIFT_TIE]
+ switches = dict(zip(tied, each(lambda k: switch(c, to_body, keeps, Fb, med_b, entry_frame, k), tied, jobs)))
+ rest = min(switches, key=lambda k: (round(switches[k], 6), k))
+ rests_used.add(rest)
+ c_entry = {"frame": entry_frame, "own_lift": target, "set": entry_set, "rest": rest,
+ "candidates": {str(k): {"lift": lift(k), "switch": round(v, 3)} for k, v in switches.items()}}
+ body_rows.append({"name": c.name, "loop": c.loop, "cells": cells, "keep": keeps,
+ "to_body": [_six(m) for m in to_body], "own_mouth": c.own_mouth, "entry": c_entry,
+ "median_step": med_b})
+
+ for row, (c, folder, cells, keeps, to_body, Fb, med_b, own, entry_frame) in zip(body_rows, prepared):
+ row["switch"] = {str(r): [round(v, 3) for v in each(lambda k, r=r: switch(c, to_body, keeps, Fb, med_b, k, r), range(len(c.files)), jobs)]
+ for r in sorted(rests_used)}
+ if c.sound is not None:
+ dest = folder / ("sound" + c.sound.suffix.lower())
+ shutil.copyfile(c.sound, dest)
+ row["sound"] = dest.relative_to(tmp).as_posix()
+ row["own_lift"] = [None if r is None else r["lift"] for r in own]
+
+ top = max(areas)
+ levels = MOUTH_RULES["normal_top"]
+ rules = {**MOUTH_RULES, **PICK_RULES, "levels": levels,
+ "open_px": [round(top * level / levels, 1) for level in range(levels + 1)],
+ "jump_line": fl.JUMP_LINE, "enter_line": round(float(steps.max() / med), 3)}
+ shutil.copyfile(PLAYER_PATH, tmp / PLAYER_PATH.name)
+ np.save(tmp / "weight.npy", w0)
+ pack = {"kind": PACK_KIND, "version": VERSION, "fps": fps, "size": list(geom.size), "rules": rules,
+ "face": {"geometry": req["face"], "frames": face_rows, "sets": {s: {"frames": sets[s], "closed": closed_in[s]} for s in ("smile", "neutral")},
+ "steps": [[round(float(v), 3) for v in row] for row in D], "median_step": med},
+ "body": body_rows, "player": {"path": PLAYER_PATH.name, "export": "createFacePlayer"}}
+ files = sorted(p for p in tmp.rglob("*") if p.is_file())
+ pack["files"] = {p.relative_to(tmp).as_posix(): _digest(p) for p in files}
+ (tmp / "face-pack.json").write_text(json.dumps(pack, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+
+ report = {"request": str(request), "pack": str(out), "weight_px": int((w0 > 0).sum()), "union_px": int(face_union.sum()),
+ "talk": _clip_report(talk) | {"sets": {k: v for k, v in sets.items()}, "median_step": med,
+ "open_px_max": top, "enter_line": rules["enter_line"]},
+ "body": [{**_clip_report(c), "own_mouth": row["own_mouth"], "entry": row["entry"],
+ "hand_frames": [k for k, kp in enumerate(row["keep"]) if kp and kp["hand_px"] > 0],
+ "switch_within_line": {r: sum(v <= fl.JUMP_LINE for v in vals) for r, vals in row["switch"].items()}}
+ for c, row in zip(bodies, body_rows)]}
+ (tmp / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
+ os.replace(tmp, out)
+ except BaseException:
+ shutil.rmtree(tmp, ignore_errors=True)
+ raise
+ return report
+
+
+def _clip_report(c: Clip) -> dict[str, Any]:
+ rows, used = c.rows, c.used
+ return {"name": c.name, "frames": len(c.files), "place": list(c.place),
+ "anchor": {k: [round(min(u[k] for u in used), 3), round(max(u[k] for u in used), 3)] for k in ("tx", "ty", "s", "th")},
+ "band_residual": _summary([r["band_residual"] for r in rows]),
+ "band_residual_place_only": _summary([r["band_residual_place_only"] for r in rows]),
+ "band_residual_fixed": _summary([r["band_residual_fixed"] for r in rows]),
+ "misread": [k for k, u in enumerate(used) if u["misread"]],
+ "wide_grid": [k for k, r in enumerate(rows) if r["wide_grid"]],
+ "wide_search": [k for k, r in enumerate(rows) if r["search_px"] > fl.SEARCH],
+ "search_edge": [k for k, r in enumerate(rows) if r["edge"]]}
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument("--request", type=Path, required=True,
+ help="the face request JSON: the reference and its face, the talking clip, the body clips (docs/face-parts.md)")
+ parser.add_argument("--out", type=Path, required=True, help="the pack folder to write (must not exist)")
+ parser.add_argument("--jobs", type=int, default=0, help="threads that read frames at once (default: one a CPU)")
+
+
+def run(**kwargs: object) -> int:
+ report = build(Path(str(kwargs["request"])), Path(str(kwargs["out"])), jobs=int(kwargs.get("jobs") or 0)) # type: ignore[arg-type]
+ print(json.dumps({"pack": report["pack"], "sets": {k: len(v) for k, v in report["talk"]["sets"].items()},
+ "body": [{"name": b["name"], "own_mouth": b["own_mouth"] and [b["own_mouth"]["start_s"], b["own_mouth"]["end_s"]],
+ "entry": {k: b["entry"][k] for k in ("frame", "set", "rest")}, "hand_frames": len(b["hand_frames"])}
+ for b in report["body"]]}, ensure_ascii=False, indent=2))
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ utf8_streams()
+ parser = argparse.ArgumentParser(prog=f"sprite-gen {PROG}", description=__doc__)
+ add_arguments(parser)
+ return run(**vars(parser.parse_args(argv)))
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/sprite_gen/video/face_plan.mjs b/sprite_gen/video/face_plan.mjs
new file mode 100644
index 0000000..3dcd0ba
--- /dev/null
+++ b/sprite_gen/video/face_plan.mjs
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// `sprite-gen face-bake`'s door to a pack's player: node face_plan.mjs <player.mjs> <face-pack.json> <samples.f32>
+// <args.json> <out.json>. It runs the pack's own copy of the player (the one a live page loads): the voice's mouth
+// levels (`mouthTrack`), then what every frame shows (`planScene`).
+import { readFileSync, writeFileSync } from "node:fs";
+import { pathToFileURL } from "node:url";
+
+const [player, packPath, samplesPath, argsPath, outPath] = process.argv.slice(2);
+const { mouthTrack, planScene } = await import(pathToFileURL(player).href);
+const pack = JSON.parse(readFileSync(packPath, "utf8"));
+const args = JSON.parse(readFileSync(argsPath, "utf8"));
+const bytes = readFileSync(samplesPath);
+const samples = new Float32Array(bytes.buffer, bytes.byteOffset, bytes.length / 4);
+const track = mouthTrack(samples, args.rate, pack.rules, { fps: pack.fps, tags: args.tags });
+const plan = planScene(pack, { play: args.play, start: args.start, levels: track.levels });
+writeFileSync(outPath, JSON.stringify({ track, plan }));
diff --git a/sprite_gen/video/sprite-gen-face-v1.mjs b/sprite_gen/video/sprite-gen-face-v1.mjs
new file mode 100644
index 0000000..e6a9bdd
--- /dev/null
+++ b/sprite_gen/video/sprite-gen-face-v1.mjs
@@ -0,0 +1,368 @@
+// SPDX-License-Identifier: Apache-2.0
+// sprite-gen face player v1: the one place the face layer's rules run. A live page (a browser source) and
+// `sprite-gen face-bake` (through node) call the same functions, so a baked line and a live one pick the same
+// frames. docs/face-parts.md owns the rules; the numbers are the pack's (`face-pack.json` rules).
+//
+// mouthTrack(samples, sampleRate, rules, { fps, tags }) voice -> a mouth level per frame
+// pickFaces(pack, levels, set, rest, { from }) levels -> a talking frame (or the rest) per frame
+// planScene(pack, { play, start, levels, tags }) a body clip and a line -> what each frame shows
+// loadFacePack(url), createFacePlayer(canvas, pack, { audioContext }) the live page
+
+// Python's round(): halves to the even neighbour (the rule was measured with it).
+function roundEven(x) {
+ const f = Math.floor(x);
+ const d = x - f;
+ if (d > 0.5) return f + 1;
+ if (d < 0.5) return f;
+ return f % 2 === 0 ? f : f + 1;
+}
+
+// The broadcast mouth's syllable rule: loudness over 1/48 s windows every 1/96 s, lightly smoothed, against the
+// line's loudest; a syllable is a peak over closed_under standing prominence above the lower of its neighbouring
+// dips, min_gap_s apart (the louder stays). Each opens open_lo_hi by its loudness (a surprise line's loudest to
+// surprise_top) on the frame lead_frames ahead of it, half of it the frame before, two fifths the frame after, closed
+// otherwise; no frame moves more than max_step levels, and a level over normal_top is entered from wide_from.
+export function mouthTrack(samples, sampleRate, rules, { fps = 24, tags = [] } = {}) {
+ const n = samples.length;
+ const frames = Math.ceil((n / sampleRate) * fps);
+ const hop = Math.floor(sampleRate / 96);
+ const raw = [];
+ for (let i = 0; i < n - 2 * hop; i += hop) {
+ let s = 0;
+ for (let k = i; k < i + 2 * hop; k++) s += samples[k] * samples[k];
+ raw.push(Math.sqrt(s / (2 * hop) + 1e-12));
+ }
+ const env = raw.map((_, i) => ((raw[i - 1] ?? 0) + raw[i] + (raw[i + 1] ?? 0)) / 3);
+ const peak = env.reduce((a, b) => Math.max(a, b), 0);
+ for (let i = 0; i < env.length; i++) env[i] = peak > 0 ? env[i] / peak : 0;
+ const cand = [];
+ for (let i = 1; i < env.length - 1; i++) {
+ if (env[i] >= env[i - 1] && env[i] > env[i + 1] && env[i] > rules.closed_under) {
+ const lo = Math.max(0, i - 10), hi = Math.min(env.length, i + 11);
+ let left = Infinity, right = Infinity;
+ for (let k = lo; k < i; k++) left = Math.min(left, env[k]);
+ for (let k = i + 1; k < hi; k++) right = Math.min(right, env[k]);
+ if (env[i] - Math.max(left, right) > rules.prominence) cand.push(i);
+ }
+ }
+ const syl = [];
+ for (const i of cand) {
+ if (syl.length && (i - syl[syl.length - 1]) / 96 < rules.min_gap_s) {
+ if (env[i] > env[syl[syl.length - 1]]) syl[syl.length - 1] = i;
+ } else syl.push(i);
+ }
+ const surprise = tags.some((t) => rules.surprise_tags.includes(t));
+ const top = surprise ? rules.surprise_top : rules.normal_top;
+ const louds = syl.map((i) => env[i]);
+ const [loL, hiL] = louds.length ? [Math.min(...louds), Math.max(...louds)] : [0, 1];
+ const [openLo, openHi] = rules.open_lo_hi;
+ const levels = new Array(frames).fill(0);
+ const peaks = [];
+ syl.forEach((i, k) => {
+ const f = roundEven((i / 96) * fps) - rules.lead_frames;
+ if (f < 0 || f >= frames) return;
+ const l = louds[k];
+ const size = surprise && l >= rules.surprise_at ? top
+ : Math.trunc(roundEven(openLo + ((l - loL) / Math.max(hiL - loL, 1e-6)) * (openHi - openLo)));
+ peaks.push({ frame: f, size });
+ levels[f] = Math.max(levels[f], size);
+ if (f > 0) levels[f - 1] = Math.max(levels[f - 1], Math.ceil(size * 0.5));
+ if (f + 1 < frames) levels[f + 1] = Math.max(levels[f + 1], Math.trunc(size * 0.4));
+ });
+ for (let pass = 0; pass < 20; pass++) {
+ let changed = false;
+ for (let i = 1; i < frames; i++) {
+ if (levels[i] - levels[i - 1] > rules.max_step) { levels[i - 1] = levels[i] - rules.max_step; changed = true; }
+ if (levels[i - 1] - levels[i] > rules.max_step) { levels[i] = levels[i - 1] - rules.max_step; changed = true; }
+ if (levels[i] > rules.normal_top && levels[i - 1] < rules.wide_from) { levels[i - 1] = rules.wide_from; changed = true; }
+ }
+ if (!changed) break;
+ }
+ return { levels, syllables: syl.map((i) => ({ t_s: i / 96, loud: env[i] })), peaks };
+}
+
+// The talking frames of one expression set, picked to a line's mouth levels: Viterbi over (frame of the set | the
+// rest) per output frame, starting and ending on the rest. A frame's next is the clip's next; a hold, a jump to a
+// frame within the jump line of the clip's next, an entry from the rest and an exit to it (both within the enter
+// line: the clip's own largest step) cost the pack's numbers. A frame costs how far its open area is from the
+// level's (a mouth opened less than wanted under_weight times more), in one level's area; the rest costs the area
+// wanted. Before `from` only the rest is allowed (the layer is not on yet).
+export function pickFaces(pack, levels, set, rest, { from = 0 } = {}) {
+ const rules = pack.rules;
+ const D = pack.face.steps;
+ const n = D.length;
+ const frames = pack.face.sets[set].frames;
+ const member = new Set(frames);
+ const area = pack.face.frames.map((f) => f.open_px);
+ const want = levels.map((l) => rules.open_px[Math.min(l, rules.open_px.length - 1)]);
+ const unit = rules.open_px[1];
+ const S = frames.length;
+ const REST = S;
+ const pos = new Map(frames.map((j, k) => [j, k]));
+ const enter = frames.map((j) => D[rest][j] <= rules.enter_line);
+ const leave = frames.map((j) => D[j][rest] <= rules.enter_line);
+ let cost = new Array(S + 1).fill(Infinity);
+ cost[REST] = 0;
+ const back = [];
+ for (let t = 0; t < want.length; t++) {
+ const next = new Array(S + 1).fill(Infinity);
+ const arg = new Array(S + 1).fill(-1);
+ for (let k = 0; k < S; k++) {
+ const j = frames[k];
+ if (t < from) continue;
+ let best = Infinity, how = -1;
+ if (j > 0 && member.has(j - 1) && cost[pos.get(j - 1)] < best) { best = cost[pos.get(j - 1)]; how = pos.get(j - 1); }
+ if (cost[k] + rules.hold_cost < best) { best = cost[k] + rules.hold_cost; how = k; }
+ if (enter[k] && cost[REST] + rules.edge_cost < best) { best = cost[REST] + rules.edge_cost; how = REST; }
+ let jb = Infinity, ji = -1;
+ for (let q = 0; q < S; q++) {
+ const after = Math.min(frames[q] + 1, n - 1);
+ if (D[after][j] <= rules.jump_line) {
+ const c = cost[q] + rules.jump_cost + D[after][j];
+ if (c < jb) { jb = c; ji = q; }
+ }
+ }
+ if (ji >= 0 && jb < best) { best = jb; how = ji; }
+ const d = area[j] - want[t];
+ next[k] = best + (d < 0 ? -d * rules.under_weight : d) / unit;
+ arg[k] = how;
+ }
+ let lb = Infinity, li = -1;
+ for (let k = 0; k < S; k++) if (leave[k] && cost[k] + rules.edge_cost < lb) { lb = cost[k] + rules.edge_cost; li = k; }
+ if (cost[REST] <= lb) { next[REST] = cost[REST] + want[t] / unit; arg[REST] = REST; }
+ else { next[REST] = lb + want[t] / unit; arg[REST] = li; }
+ cost = next;
+ back.push(arg);
+ }
+ const path = [REST];
+ for (let t = want.length - 1; t > 0; t--) path.push(back[t][path[path.length - 1]]);
+ path.reverse();
+ return { path: path.map((k) => (k === REST ? null : frames[k])), cost: cost[REST], want };
+}
+
+function body(pack, name) {
+ const b = pack.body.find((x) => x.name === name);
+ if (!b) throw new RangeError(`face pack has no body clip ${name}`);
+ return b;
+}
+
+function loopBody(pack) {
+ return pack.body.find((x) => x.loop);
+}
+
+
+function mod(a, n) {
+ return ((a % n) + n) % n;
+}
+
+// What every frame shows for a body clip played from its start (`play`), or the loop from a cell (`start`), with a
+// line said at once (its mouth `levels`, one a frame). The line waits for the end of the clip's own_mouth span: its
+// own mouth and sound play till then, the layer off. The layer comes on in the set and at the rest the body names at
+// that frame (`entry`), at the first frame from there whose switch to that rest is within the jump line — or at once,
+// in its own set and rest, when it is already on (`layer`) — and the line's frames are picked from there. A clip
+// that plays once goes on to the loop from its first cell.
+export function planScene(pack, { play = null, start = 0, levels = [], layer = null } = {}) {
+ const loop = loopBody(pack);
+ const first = play ? body(pack, play) : loop;
+ const own = play && first.own_mouth ? first.own_mouth.end_frame : 0;
+ const total = (levels.length ? own : first.cells.length) + levels.length;
+ const cellAt = (t) => {
+ if (!play) return { body: loop.name, cell: mod(start + t, loop.cells.length) };
+ if (t < first.cells.length) return { body: first.name, cell: t };
+ return { body: loop.name, cell: mod(t - first.cells.length, loop.cells.length) };
+ };
+ const frames = [];
+ if (!levels.length) {
+ for (let t = 0; t < total; t++) frames.push({ ...cellAt(t), face: null, layer: "off", level: 0 });
+ return { frames, set: null, rest: null, line_start: null, layer_from: null };
+ }
+ const { set, rest } = layer ?? body(pack, cellAt(own).body).entry;
+ let from = 0;
+ if (!layer) {
+ while (from < levels.length) {
+ const c = cellAt(own + from);
+ if (body(pack, c.body).switch[String(rest)][c.cell] <= pack.rules.jump_line) break;
+ from++;
+ }
+ }
+ const pick = pickFaces(pack, levels, set, rest, { from });
+ for (let t = 0; t < total; t++) {
+ const c = cellAt(t);
+ if (t < own + from) { frames.push({ ...c, face: null, layer: "off", level: t >= own ? levels[t - own] : 0 }); continue; }
+ const j = pick.path[t - own];
+ frames.push({ ...c, face: j ?? rest, layer: j === null ? "rest" : "talk", level: levels[t - own] });
+ }
+ return { frames, set, rest, line_start: own, layer_from: own + from, want: pick.want, cost: pick.cost };
+}
+
+// --- the live page --------------------------------------------------------------------------------------------------
+
+function loadImage(url) {
+ return new Promise((resolve, reject) => {
+ const im = new Image();
+ im.onload = () => resolve(im);
+ im.onerror = () => reject(new Error(`face pack picture did not load: ${url}`));
+ im.src = url;
+ });
+}
+
+// The pack and its pictures: the face and keep pictures loaded whole, a body cell when it is first drawn.
+export async function loadFacePack(url) {
+ const at = (p) => new URL(p, new URL(url, globalThis.location?.href)).href;
+ const pack = await (await fetch(url)).json();
+ if (pack.kind !== "sprite-gen-face-pack" || pack.version !== 1) throw new RangeError("not a sprite-gen-face-pack version 1");
+ pack.url = at;
+ pack.images = { face: {}, keep: {} };
+ await Promise.all(pack.face.frames.filter((f) => f.file).map(async (f) => { pack.images.face[f.index] = await loadImage(at(f.file)); }));
+ await Promise.all(pack.body.map(async (b) => {
+ pack.images.keep[b.name] = await Promise.all(b.keep.map((k) => (k ? loadImage(at(k.file)) : null)));
+ }));
+ return pack;
+}
+
+function mono(buffer) {
+ const out = new Float32Array(buffer.length);
+ for (let c = 0; c < buffer.numberOfChannels; c++) {
+ const data = buffer.getChannelData(c);
+ for (let i = 0; i < data.length; i++) out[i] += data[i] / buffer.numberOfChannels;
+ }
+ return out;
+}
+
+function chain(m1, m2) {
+ // m1 . m2 for affine maps given as six numbers [a b c; d e f]
+ const [a, b, c, d, e, f] = m1, [g, h, i, j, k, l] = m2;
+ return [a * g + b * j, a * h + b * k, a * i + b * l + c, d * g + e * j, d * h + e * k, d * i + e * l + f];
+}
+
+function canvasTransform(m, k) {
+ // a map on pixel indices (centres on integers) as a canvas transform (centres at +0.5), scaled by k
+ const [a, b, c, d, e, f] = m;
+ return [k * a, k * d, k * b, k * e, k * (c + 0.5 - (a + b) * 0.5), k * (f + 0.5 - (d + e) * 0.5)];
+}
+
+// The live player: the loop runs on one frame clock (the audio context's), a body clip plays from its start when
+// asked (`play`), a line talks the face layer (`say`). Requests run one after another. After a line the layer stays at
+// its rest until a loop frame within the jump line of it, then goes off.
+export function createFacePlayer(canvas, pack, { audioContext } = {}) {
+ const ctx = canvas.getContext("2d");
+ const audio = audioContext ?? new AudioContext();
+ const fps = pack.fps;
+ const loop = loopBody(pack);
+ const cells = new Map();
+ let origin = 0;
+ let loopPhase = 0;
+ let scene = null;
+ let resting = null;
+ let queue = Promise.resolve();
+ let running = false;
+ let shown = -1;
+
+ const now = () => Math.floor((audio.currentTime - origin) * fps);
+ const cell = (name, k) => {
+ const key = `${name}/${k}`;
+ if (!cells.has(key)) {
+ const im = new Image();
+ im.src = pack.url(body(pack, name).cells[k]);
+ cells.set(key, im);
+ }
+ return cells.get(key);
+ };
+
+ function draw(f) {
+ const k = canvas.width / pack.size[0];
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ const im = cell(f.body, f.cell);
+ if (im.complete) ctx.drawImage(im, 0, 0, canvas.width, canvas.height);
+ if (f.face === null || f.face === undefined) return;
+ const b = body(pack, f.body);
+ ctx.imageSmoothingEnabled = true;
+ ctx.imageSmoothingQuality = "high";
+ ctx.setTransform(...canvasTransform(chain(b.to_body[f.cell], pack.face.frames[f.face].to_ref), k));
+ ctx.drawImage(pack.images.face[f.face], 0, 0);
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ const keep = b.keep[f.cell];
+ const kim = pack.images.keep[f.body][f.cell];
+ if (keep) ctx.drawImage(kim, keep.at[0] * k, keep.at[1] * k, kim.width * k, kim.height * k);
+ }
+
+ function finish(s) {
+ const end = s.plan.frames[s.plan.frames.length - 1];
+ const after = s.t0 + s.plan.frames.length;
+ loopPhase = end.body === loop.name ? after - (end.cell + 1) : after;
+ resting = end.face !== null && s.plan.set ? { set: s.plan.set, rest: s.plan.rest } : null;
+ scene = null;
+ s.resolve();
+ }
+
+ function frameAt(t) {
+ if (scene && t - scene.t0 >= scene.plan.frames.length) finish(scene);
+ if (scene && t >= scene.t0) return scene.plan.frames[t - scene.t0];
+ const c = { body: loop.name, cell: mod(t - loopPhase, loop.cells.length) };
+ if (resting && body(pack, c.body).switch[String(resting.rest)][c.cell] <= pack.rules.jump_line) resting = null;
+ return { ...c, face: resting ? resting.rest : null };
+ }
+
+ function tick() {
+ if (!running) return;
+ const t = now();
+ if (t !== shown) { shown = t; draw(frameAt(t)); }
+ requestAnimationFrame(tick);
+ }
+
+ function sound(buffer, t, seconds) {
+ const src = audio.createBufferSource();
+ src.buffer = buffer;
+ src.connect(audio.destination);
+ src.start(origin + t / fps, 0, seconds);
+ }
+
+ function run(make) {
+ const job = queue.then(() => new Promise((resolve) => {
+ const t0 = now() + 1;
+ const s = make(t0);
+ scene = { ...s, t0, resolve };
+ }));
+ queue = job.catch(() => {});
+ return job;
+ }
+
+ async function decode(voice) {
+ if (voice instanceof AudioBuffer) return voice;
+ const bytes = typeof voice === "string" ? await (await fetch(voice)).arrayBuffer() : voice;
+ return audio.decodeAudioData(bytes);
+ }
+
+ return {
+ start() { origin = audio.currentTime; loopPhase = 0; running = true; requestAnimationFrame(tick); },
+ stop() { running = false; },
+ // A body clip from its start, with its own sound; a line said while its own mouth talks waits for its end.
+ async play(name) {
+ const b = body(pack, name);
+ const own = b.sound ? await decode(pack.url(b.sound)) : null;
+ return run((t0) => {
+ resting = null;
+ if (own) sound(own, t0, (b.own_mouth ? b.own_mouth.end_frame : b.cells.length) / fps);
+ return { plan: planScene(pack, { play: name }), play: name };
+ });
+ },
+ // A line: the voice plays and the face layer talks to it.
+ async say(voice, { tags = [] } = {}) {
+ const data = await decode(voice);
+ const { levels } = mouthTrack(mono(data), data.sampleRate, pack.rules, { fps, tags });
+ if (scene && scene.play && body(pack, scene.play).own_mouth && now() < scene.t0 + body(pack, scene.play).own_mouth.end_frame) {
+ // said while a clip talks with its own mouth: the clip's plan again with the line after its own mouth
+ const s = scene;
+ s.plan = planScene(pack, { play: s.play, levels });
+ sound(data, s.t0 + s.plan.line_start);
+ return new Promise((resolve) => { const done = s.resolve; s.resolve = () => { done(); resolve(); }; });
+ }
+ return run((t0) => {
+ const plan = planScene(pack, { start: mod(t0 - loopPhase, loop.cells.length), levels, layer: resting });
+ sound(data, t0);
+ return { plan };
+ });
+ },
+ };
+}
diff --git a/tests/video/test_face_layer.py b/tests/video/test_face_layer.py
index 0e9cda0..a01091e 100644
--- a/tests/video/test_face_layer.py
+++ b/tests/video/test_face_layer.py
@@ -79,7 +79,7 @@ def test_the_entry_set_is_read_off_the_clip_mouth_where_the_layer_comes_on(case)
idle, greet = body(pack, "idle")["entry"], body(pack, "greet")["entry"]
assert idle["frame"] == 0 and idle["set"] == "neutral" and idle["rest"] in pack["face"]["sets"]["neutral"]["closed"]
assert greet["frame"] == tf.GREET_OWN_END and greet["set"] == "smile" and greet["rest"] in pack["face"]["sets"]["smile"]["closed"]
- assert greet["own_lift"] >= fl.SMILE_LIFT > fl.NEUTRAL_LIFT >= idle["own_lift"]
+ assert greet["own_lift"] >= fl.SMILE_FROM > idle["own_lift"]
def test_own_mouth_runs_to_the_first_closed_frame_after_the_last_sound(case):