Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

fix(face): video-face reads a body frame's place again on the ring of skin round its closed mouth, size and tilt staying the band's — a head that turns moves its hair, headset and cheeks further than its mouth and the band's place carried the layer off it; a read move is checked against its neighbours as an anchor is, and a frame whose mouth is open takes the moves of the read frames round it

Soohong Kim0daa1f106fce
diff --git a/sprite_gen/video/face_layer.py b/sprite_gen/video/face_layer.py
index 2dc5a91..f56a6b5 100644
--- a/sprite_gen/video/face_layer.py
+++ b/sprite_gen/video/face_layer.py
@@ -69,6 +69,9 @@ WIDE_SCALES = np.round(np.arange(0.92, 1.0801, 0.02), 3)
WIDE_THETAS = np.arange(-8.0, 8.01, 1.0)
HAMPEL_WINDOW = 5
HAMPEL_FLOOR = {"tx": 0.5, "ty": 0.5, "s": 0.01, "th": 0.3}
+RING_IN, RING_OUT = (30, 14), (60, 42) # px half-axes of the ellipses round the mouth the ring lies between
+RING_SEARCH = 16 # px each way a body frame's place is read again on the ring; twice this where the best is on the rim
+RING_OPAQUE = 200 / 255 # the ring reads the reference's lower face where it is this opaque
DARK_LUMA, DARK_CHROMA, DARK_GROW = 110, 45, 3
LINE_LUMA, LINE_GROW = 150, 2
FOREIGN = 48 # a body pixel this far (any channel, 0..255) from the reference face is foreign
@@ -387,21 +390,38 @@ def hand_reach(body_ref: np.ndarray, reference: np.ndarray, w0: np.ndarray, geom
@dataclass
class Band:
- """The anchor reference: auto_motion's masked-NCC reference tuple on the band round the layer, and the
- reference's features on the band's box."""
+ """An anchor reference: auto_motion's masked-NCC reference tuple on a region of the reference (the band round
+ the layer, or the ring round the mouth), and the reference's features on the region's box."""
reference: tuple
features: np.ndarray
@classmethod
def read(cls, reference: Image.Image, w0: np.ndarray, geom: Geometry) -> Band:
- ref = features(reference)
+ """The band round the layer, on which a frame's place, size and tilt are read (`fit`)."""
rows = np.arange(geom.size[1])[:, None] < geom.band_bottom
- band = parts_mod._dilate(w0 > 0, BAND) & ~(w0 > 0.98) & rows & ~eye_mask(geom)
- ys, xs = np.nonzero(band)
+ return cls.on(features(reference), parts_mod._dilate(w0 > 0, BAND) & ~(w0 > 0.98) & rows & ~eye_mask(geom), 0.05)
+
+ @classmethod
+ def ring(cls, reference: Image.Image, w0: np.ndarray, geom: Geometry) -> Band:
+ """The ring round the mouth, on which a body frame's place is read again (`place_on`): the reference's own
+ lower face (`w0`, its weight) between the ellipses RING_IN and RING_OUT round the mouth, where it is opaque.
+ The mouth inside it is left out: a body's closed mouth smiles more or less than the reference's."""
+ h, w = w0.shape
+ yy, xx = np.mgrid[0:h, 0:w]
+ mx, my = geom.mouth
+
+ def inside(rx: float, ry: float) -> np.ndarray:
+ return ((xx - mx) / rx) ** 2 + ((yy - my) / ry) ** 2 < 1
+
+ return cls.on(features(reference), (w0 > 0) & inside(*RING_OUT) & ~inside(*RING_IN), RING_OPAQUE)
+
+ @classmethod
+ def on(cls, ref: np.ndarray, region: np.ndarray, opaque: float) -> Band:
+ ys, xs = np.nonzero(region)
x0, y0, x1, y1 = int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1
patch = ref[y0:y1, x0:x1]
- mask = band[y0:y1, x0:x1] & (patch[:, :, 3] > 0.05)
+ mask = region[y0:y1, x0:x1] & (patch[:, :, 3] > opaque)
flat = patch[mask]
c = flat - flat.mean(axis=0)
return cls(((x0, y0, x1, y1), mask, c, float(np.sqrt((c * c).sum()))), patch.copy())
@@ -493,6 +513,61 @@ def fit(band: Band, feat: np.ndarray, geom: Geometry) -> dict[str, Any]:
return rec
+def place_on(ring: Band, feat: np.ndarray, a: dict[str, Any], geom: Geometry) -> dict[str, Any]:
+ """Anchor `a` with its place read again on the ring round the mouth (`Band.ring`), its size and tilt kept: the
+ frame laid through `a` is matched to the ring within RING_SEARCH px (`register`), and within twice that where the
+ best sits on the search's rim."""
+ m = matrix(a, geom)
+ for radius in (RING_SEARCH, 2 * RING_SEARCH):
+ loc, origin, size = _local(ring.reference, radius + 1)
+ rr = register(loc, warp(feat, m @ shift(*origin), size).astype(np.float64), (0, 0), radius)
+ if not rr["edge"]:
+ break
+ moved = m @ shift(-rr["sx"], -rr["sy"])
+ c = np.asarray(geom.mouth)
+ t = moved[:2, 2] - c + moved[:2, :2] @ c
+ return {**a, "tx": float(t[0]), "ty": float(t[1])}
+
+
+def on_ring(used: list[dict[str, Any]], placed: list[dict[str, Any] | None], cyclic: bool) -> list[dict[str, Any]]:
+ """Each anchor with the place read on the ring round the mouth: `placed` is `place_on`'s anchor where the frame's
+ own mouth is closed and None where it is not (an open mouth moves the jaw and the skin round it). A read frame's
+ move from band to ring is checked as `hampel` checks an anchor, against the moves read among the HAMPEL_WINDOW
+ frames round it (three or more), and a misreading takes their median. A frame read on the ring takes its move; any
+ other takes the moves of the read frames round it, linear between the nearest on each side (wrapping on a loop),
+ the nearest one's past the first or the last. A clip with no frame read keeps its places. Each anchor records its
+ move (`ring`) and whether it was read (`ring_read`)."""
+ n, h = len(used), HAMPEL_WINDOW // 2
+ read_moves = {k: np.array([p["tx"] - used[k]["tx"], p["ty"] - used[k]["ty"]]) for k, p in enumerate(placed) if p is not None}
+ moves = {}
+ for k, d in read_moves.items():
+ near = [read_moves[j % n] for j in range(k - h, k + h + 1) if (cyclic or 0 <= j < n) and j % n in read_moves]
+ if len(near) >= 3:
+ v = np.stack(near)
+ med = np.median(v, 0)
+ mad = 1.4826 * np.median(np.abs(v - med), 0)
+ floor = np.array([HAMPEL_FLOOR["tx"], HAMPEL_FLOOR["ty"]])
+ d = med if (np.abs(d - med) > np.maximum(3 * mad, floor)).any() else d
+ moves[k] = d
+ read = sorted(moves)
+ out = []
+ for k, a in enumerate(used):
+ if k in moves or not read:
+ d = moves.get(k, np.zeros(2))
+ else:
+ before = [j for j in read if j < k] or ([j - n for j in read] if cyclic else [])
+ after = [j for j in read if j > k] or ([j + n for j in read] if cyclic else [])
+ if before and after:
+ j0, j1 = max(before), min(after)
+ t = (k - j0) / (j1 - j0)
+ d = (1 - t) * moves[j0 % n] + t * moves[j1 % n]
+ else:
+ d = moves[(max(before) if before else min(after)) % n]
+ out.append({**a, "tx": a["tx"] + float(d[0]), "ty": a["ty"] + float(d[1]),
+ "ring": [round(float(d[0]), 3), round(float(d[1]), 3)], "ring_read": k in moves})
+ return out
+
+
def band_residual(band: Band, feat: np.ndarray, m: np.ndarray) -> float:
"""Mean |frame - reference| (0..255, colour matched by the median) on the band where both are opaque, the frame
read through `m`: how well the anchor lays the reference's face on the frame."""
diff --git a/sprite_gen/video/face_pack.py b/sprite_gen/video/face_pack.py
index ab06bae..b6f2b39 100644
--- a/sprite_gen/video/face_pack.py
+++ b/sprite_gen/video/face_pack.py
@@ -7,8 +7,10 @@ come back to, and clips that play once. The pack is a folder a player reads (`sp
`face-bake` bakes from:
- **Anchors** for every frame of every clip, read on the band round the reference's own lower face (so a clip's
- anchors are the same whatever else the request names), and the **weight**: the lower faces the anchors lay on the
- reference, joined (`face_layer`).
+ anchors are the same whatever else the request names); a body frame whose own mouth is closed has its place read
+ again on the ring of skin round the mouth, its size and tilt the band's (a head that turns moves its hair, headset
+ and cheeks further than its mouth), and its other frames take the move of the read ones round them. The **weight**:
+ the lower faces the anchors lay on the reference, joined (`face_layer`).
- **Face frames**: each talking frame's mouth read in reference coordinates (`face_layer.mouth_reading`), its open
area (`face_layer.open_area`) and its step to every other frame (`face_layer.step_features`, over the clip's median
step). A closed mouth that smiles is in the smile set, one that does not in the neutral set; an open run takes the
@@ -203,8 +205,8 @@ def each(fn, items, jobs: int) -> list[Any]:
_WORK: dict[str, Any] = {}
-def _work_init(band: fl.Band, geom: fl.Geometry) -> None:
- _WORK["band"], _WORK["geom"] = band, geom
+def _work_init(work: dict[str, Any]) -> None:
+ _WORK.update(work)
def _anchor(task: tuple[str, tuple[int, int, int]]) -> dict[str, Any]:
@@ -219,16 +221,30 @@ def _anchor(task: tuple[str, tuple[int, int, int]]) -> dict[str, Any]:
return a
+def _on_ring(task: tuple[str, tuple[int, int, int], tuple[int, int], dict[str, Any]]) -> dict[str, Any] | None:
+ """One body frame's anchor with its place read on the ring round the mouth, or None where its own mouth, read
+ through the anchor, is not closed (a worker's task)."""
+ ring, geom = _WORK["ring"], _WORK["geom"]
+ path, place, native, a = task
+ frame = Image.open(path).convert("RGBA")
+ body_ref = fl.warp(np.asarray(frame, np.float32), fl.placement(place, native) @ fl.matrix(a, geom), geom.size)
+ if fl.expression(fl.mouth_reading(body_ref, geom)) is None:
+ return None
+ return fl.place_on(ring, fl.features(fl.to_reference(frame, place, geom.size)), a, geom)
+
+
+def _pool_map(fn, tasks: list[Any], work: dict[str, Any], jobs: int) -> list[Any]:
+ if jobs <= 1:
+ _work_init(work)
+ return [fn(t) for t in tasks]
+ with ProcessPoolExecutor(max_workers=jobs, initializer=_work_init, initargs=(work,)) as pool:
+ return list(pool.map(fn, tasks, chunksize=2))
+
+
def read_anchors(clips: list[Clip], band: fl.Band, geom: fl.Geometry, jobs: int) -> None:
"""Every frame's anchor, read on `jobs` processes (the reading is most of a pack's time), then each clip's
misreadings taken out (`face_layer.hampel`)."""
- tasks = [(str(p), c.place) for c in clips for p in c.files]
- if jobs <= 1:
- _work_init(band, geom)
- rows = [_anchor(t) for t in tasks]
- else:
- with ProcessPoolExecutor(max_workers=jobs, initializer=_work_init, initargs=(band, geom)) as pool:
- rows = list(pool.map(_anchor, tasks, chunksize=2))
+ rows = _pool_map(_anchor, [(str(p), c.place) for c in clips for p in c.files], {"band": band, "geom": geom}, jobs)
at = 0
for c in clips:
c.rows = rows[at:at + len(c.files)]
@@ -236,6 +252,18 @@ def read_anchors(clips: list[Clip], band: fl.Band, geom: fl.Geometry, jobs: int)
at += len(c.files)
+def read_places(bodies: list[Clip], ring: fl.Band, geom: fl.Geometry, jobs: int) -> None:
+ """Every body frame's place read again on the ring round the mouth where its own mouth is closed, and moved with
+ the read frames round it where it is not (`face_layer.on_ring`). The talking clip keeps the band's: its jaw moves
+ as it talks, and the layer is cut from it."""
+ tasks = [(str(p), c.place, c.native, a) for c in bodies for p, a in zip(c.files, c.used)]
+ placed = _pool_map(_on_ring, tasks, {"ring": ring, "geom": geom}, jobs)
+ at = 0
+ for c in bodies:
+ c.used = fl.on_ring(c.used, placed[at:at + len(c.files)], cyclic=c.loop)
+ at += len(c.files)
+
+
def lower_faces(clip: Clip, geom: fl.Geometry, jobs: int) -> list[np.ndarray]:
def one(k: int) -> np.ndarray:
arr = np.asarray(clip.on_reference(k, geom.size), np.float32)
@@ -285,8 +313,9 @@ def build(request: Path, out: Path, jobs: int = 0) -> dict[str, Any]:
ref_face = fl.lower_face(ref, geom)
if not ref_face.any():
raise SystemExit(f"{PROG}: no lower face grows from face.lower_face.seeds on the reference: the seeds must sit on its skin")
- band = fl.Band.read(reference, fl.weight(fl.fill_holes(ref_face), geom), geom)
- read_anchors(clips, band, geom, jobs)
+ w_ref = fl.weight(fl.fill_holes(ref_face), geom)
+ read_anchors(clips, fl.Band.read(reference, w_ref, geom), geom, jobs)
+ read_places(bodies, fl.Band.ring(reference, w_ref, geom), geom, jobs)
faces = []
for c in clips:
faces.extend(lower_faces(c, geom, jobs))
@@ -384,8 +413,9 @@ def build(request: Path, out: Path, jobs: int = 0) -> dict[str, Any]:
c_entry = {"frame": entry_frame, "own_lift": target, "set": entry_set, "rest": rest,
"candidates": {str(k): {"lift": lift(k), "switch": round(v, 3)} for k, v in switches.items()}}
body_rows.append({"name": c.name, "loop": c.loop, "cells": cells, "keep": keeps,
- "to_body": [_six(m) for m in to_body], "own_mouth": c.own_mouth, "entry": c_entry,
- "median_step": med_b})
+ "to_body": [_six(m) for m in to_body], "ring": [u["ring"] for u in c.used],
+ "ring_read": [k for k, u in enumerate(c.used) if u["ring_read"]],
+ "own_mouth": c.own_mouth, "entry": c_entry, "median_step": med_b})
for row, (c, folder, cells, keeps, to_body, Fb, med_b, own, entry_frame) in zip(body_rows, prepared):
row["switch"] = {str(r): [round(v, 3) for v in each(lambda k, r=r: switch(c, to_body, keeps, Fb, med_b, k, r), range(len(c.files)), jobs)]
@@ -434,6 +464,8 @@ def _clip_report(c: Clip) -> dict[str, Any]:
"band_residual_place_only": _summary([r["band_residual_place_only"] for r in rows]),
"band_residual_fixed": _summary([r["band_residual_fixed"] for r in rows]),
"misread": [k for k, u in enumerate(used) if u["misread"]],
+ **({"ring_read": sum(u["ring_read"] for u in used), "ring_move": _summary([float(np.hypot(*u["ring"])) for u in used])}
+ if used and "ring" in used[0] else {}),
"wide_grid": [k for k, r in enumerate(rows) if r["wide_grid"]],
"wide_search": [k for k, r in enumerate(rows) if r["search_px"] > fl.SEARCH],
"search_edge": [k for k, r in enumerate(rows) if r["edge"]]}