Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.
fix(face): video-face reads a body frame's place again on the ring of skin round its closed mouth, size and tilt staying the band's — a head that turns moves its hair, headset and cheeks further than its mouth and the band's place carried the layer off it; a read move is checked against its neighbours as an anchor is, and a frame whose mouth is open takes the moves of the read frames round it
Soohong Kim0daa1f106fce
diff --git a/sprite_gen/video/face_layer.py b/sprite_gen/video/face_layer.pyindex 2dc5a91..f56a6b5 100644--- a/sprite_gen/video/face_layer.py+++ b/sprite_gen/video/face_layer.py@@ -69,6 +69,9 @@ WIDE_SCALES = np.round(np.arange(0.92, 1.0801, 0.02), 3)WIDE_THETAS = np.arange(-8.0, 8.01, 1.0)HAMPEL_WINDOW = 5HAMPEL_FLOOR = {"tx": 0.5, "ty": 0.5, "s": 0.01, "th": 0.3}+RING_IN, RING_OUT = (30, 14), (60, 42) # px half-axes of the ellipses round the mouth the ring lies between+RING_SEARCH = 16 # px each way a body frame's place is read again on the ring; twice this where the best is on the rim+RING_OPAQUE = 200 / 255 # the ring reads the reference's lower face where it is this opaqueDARK_LUMA, DARK_CHROMA, DARK_GROW = 110, 45, 3LINE_LUMA, LINE_GROW = 150, 2FOREIGN = 48 # a body pixel this far (any channel, 0..255) from the reference face is foreign@@ -387,21 +390,38 @@ def hand_reach(body_ref: np.ndarray, reference: np.ndarray, w0: np.ndarray, geom@dataclassclass Band:- """The anchor reference: auto_motion's masked-NCC reference tuple on the band round the layer, and the- reference's features on the band's box."""+ """An anchor reference: auto_motion's masked-NCC reference tuple on a region of the reference (the band round+ the layer, or the ring round the mouth), and the reference's features on the region's box."""reference: tuplefeatures: np.ndarray@classmethoddef read(cls, reference: Image.Image, w0: np.ndarray, geom: Geometry) -> Band:- ref = features(reference)+ """The band round the layer, on which a frame's place, size and tilt are read (`fit`)."""rows = np.arange(geom.size[1])[:, None] < geom.band_bottom- band = parts_mod._dilate(w0 > 0, BAND) & ~(w0 > 0.98) & rows & ~eye_mask(geom)- ys, xs = np.nonzero(band)+ return cls.on(features(reference), parts_mod._dilate(w0 > 0, BAND) & ~(w0 > 0.98) & rows & ~eye_mask(geom), 0.05)++ @classmethod+ def ring(cls, reference: Image.Image, w0: np.ndarray, geom: Geometry) -> Band:+ """The ring round the mouth, on which a body frame's place is read again (`place_on`): the reference's own+ lower face (`w0`, its weight) between the ellipses RING_IN and RING_OUT round the mouth, where it is opaque.+ The mouth inside it is left out: a body's closed mouth smiles more or less than the reference's."""+ h, w = w0.shape+ yy, xx = np.mgrid[0:h, 0:w]+ mx, my = geom.mouth++ def inside(rx: float, ry: float) -> np.ndarray:+ return ((xx - mx) / rx) ** 2 + ((yy - my) / ry) ** 2 < 1++ return cls.on(features(reference), (w0 > 0) & inside(*RING_OUT) & ~inside(*RING_IN), RING_OPAQUE)++ @classmethod+ def on(cls, ref: np.ndarray, region: np.ndarray, opaque: float) -> Band:+ ys, xs = np.nonzero(region)x0, y0, x1, y1 = int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1patch = ref[y0:y1, x0:x1]- mask = band[y0:y1, x0:x1] & (patch[:, :, 3] > 0.05)+ mask = region[y0:y1, x0:x1] & (patch[:, :, 3] > opaque)flat = patch[mask]c = flat - flat.mean(axis=0)return cls(((x0, y0, x1, y1), mask, c, float(np.sqrt((c * c).sum()))), patch.copy())@@ -493,6 +513,61 @@ def fit(band: Band, feat: np.ndarray, geom: Geometry) -> dict[str, Any]:return rec+def place_on(ring: Band, feat: np.ndarray, a: dict[str, Any], geom: Geometry) -> dict[str, Any]:+ """Anchor `a` with its place read again on the ring round the mouth (`Band.ring`), its size and tilt kept: the+ frame laid through `a` is matched to the ring within RING_SEARCH px (`register`), and within twice that where the+ best sits on the search's rim."""+ m = matrix(a, geom)+ for radius in (RING_SEARCH, 2 * RING_SEARCH):+ loc, origin, size = _local(ring.reference, radius + 1)+ rr = register(loc, warp(feat, m @ shift(*origin), size).astype(np.float64), (0, 0), radius)+ if not rr["edge"]:+ break+ moved = m @ shift(-rr["sx"], -rr["sy"])+ c = np.asarray(geom.mouth)+ t = moved[:2, 2] - c + moved[:2, :2] @ c+ return {**a, "tx": float(t[0]), "ty": float(t[1])}+++def on_ring(used: list[dict[str, Any]], placed: list[dict[str, Any] | None], cyclic: bool) -> list[dict[str, Any]]:+ """Each anchor with the place read on the ring round the mouth: `placed` is `place_on`'s anchor where the frame's+ own mouth is closed and None where it is not (an open mouth moves the jaw and the skin round it). A read frame's+ move from band to ring is checked as `hampel` checks an anchor, against the moves read among the HAMPEL_WINDOW+ frames round it (three or more), and a misreading takes their median. A frame read on the ring takes its move; any+ other takes the moves of the read frames round it, linear between the nearest on each side (wrapping on a loop),+ the nearest one's past the first or the last. A clip with no frame read keeps its places. Each anchor records its+ move (`ring`) and whether it was read (`ring_read`)."""+ n, h = len(used), HAMPEL_WINDOW // 2+ read_moves = {k: np.array([p["tx"] - used[k]["tx"], p["ty"] - used[k]["ty"]]) for k, p in enumerate(placed) if p is not None}+ moves = {}+ for k, d in read_moves.items():+ near = [read_moves[j % n] for j in range(k - h, k + h + 1) if (cyclic or 0 <= j < n) and j % n in read_moves]+ if len(near) >= 3:+ v = np.stack(near)+ med = np.median(v, 0)+ mad = 1.4826 * np.median(np.abs(v - med), 0)+ floor = np.array([HAMPEL_FLOOR["tx"], HAMPEL_FLOOR["ty"]])+ d = med if (np.abs(d - med) > np.maximum(3 * mad, floor)).any() else d+ moves[k] = d+ read = sorted(moves)+ out = []+ for k, a in enumerate(used):+ if k in moves or not read:+ d = moves.get(k, np.zeros(2))+ else:+ before = [j for j in read if j < k] or ([j - n for j in read] if cyclic else [])+ after = [j for j in read if j > k] or ([j + n for j in read] if cyclic else [])+ if before and after:+ j0, j1 = max(before), min(after)+ t = (k - j0) / (j1 - j0)+ d = (1 - t) * moves[j0 % n] + t * moves[j1 % n]+ else:+ d = moves[(max(before) if before else min(after)) % n]+ out.append({**a, "tx": a["tx"] + float(d[0]), "ty": a["ty"] + float(d[1]),+ "ring": [round(float(d[0]), 3), round(float(d[1]), 3)], "ring_read": k in moves})+ return out++def band_residual(band: Band, feat: np.ndarray, m: np.ndarray) -> float:"""Mean |frame - reference| (0..255, colour matched by the median) on the band where both are opaque, the frameread through `m`: how well the anchor lays the reference's face on the frame."""diff --git a/sprite_gen/video/face_pack.py b/sprite_gen/video/face_pack.pyindex ab06bae..b6f2b39 100644--- a/sprite_gen/video/face_pack.py+++ b/sprite_gen/video/face_pack.py@@ -7,8 +7,10 @@ come back to, and clips that play once. The pack is a folder a player reads (`sp`face-bake` bakes from:- **Anchors** for every frame of every clip, read on the band round the reference's own lower face (so a clip's- anchors are the same whatever else the request names), and the **weight**: the lower faces the anchors lay on the- reference, joined (`face_layer`).+ anchors are the same whatever else the request names); a body frame whose own mouth is closed has its place read+ again on the ring of skin round the mouth, its size and tilt the band's (a head that turns moves its hair, headset+ and cheeks further than its mouth), and its other frames take the move of the read ones round them. The **weight**:+ the lower faces the anchors lay on the reference, joined (`face_layer`).- **Face frames**: each talking frame's mouth read in reference coordinates (`face_layer.mouth_reading`), its openarea (`face_layer.open_area`) and its step to every other frame (`face_layer.step_features`, over the clip's medianstep). A closed mouth that smiles is in the smile set, one that does not in the neutral set; an open run takes the@@ -203,8 +205,8 @@ def each(fn, items, jobs: int) -> list[Any]:_WORK: dict[str, Any] = {}-def _work_init(band: fl.Band, geom: fl.Geometry) -> None:- _WORK["band"], _WORK["geom"] = band, geom+def _work_init(work: dict[str, Any]) -> None:+ _WORK.update(work)def _anchor(task: tuple[str, tuple[int, int, int]]) -> dict[str, Any]:@@ -219,16 +221,30 @@ def _anchor(task: tuple[str, tuple[int, int, int]]) -> dict[str, Any]:return a+def _on_ring(task: tuple[str, tuple[int, int, int], tuple[int, int], dict[str, Any]]) -> dict[str, Any] | None:+ """One body frame's anchor with its place read on the ring round the mouth, or None where its own mouth, read+ through the anchor, is not closed (a worker's task)."""+ ring, geom = _WORK["ring"], _WORK["geom"]+ path, place, native, a = task+ frame = Image.open(path).convert("RGBA")+ body_ref = fl.warp(np.asarray(frame, np.float32), fl.placement(place, native) @ fl.matrix(a, geom), geom.size)+ if fl.expression(fl.mouth_reading(body_ref, geom)) is None:+ return None+ return fl.place_on(ring, fl.features(fl.to_reference(frame, place, geom.size)), a, geom)+++def _pool_map(fn, tasks: list[Any], work: dict[str, Any], jobs: int) -> list[Any]:+ if jobs <= 1:+ _work_init(work)+ return [fn(t) for t in tasks]+ with ProcessPoolExecutor(max_workers=jobs, initializer=_work_init, initargs=(work,)) as pool:+ return list(pool.map(fn, tasks, chunksize=2))++def read_anchors(clips: list[Clip], band: fl.Band, geom: fl.Geometry, jobs: int) -> None:"""Every frame's anchor, read on `jobs` processes (the reading is most of a pack's time), then each clip'smisreadings taken out (`face_layer.hampel`)."""- tasks = [(str(p), c.place) for c in clips for p in c.files]- if jobs <= 1:- _work_init(band, geom)- rows = [_anchor(t) for t in tasks]- else:- with ProcessPoolExecutor(max_workers=jobs, initializer=_work_init, initargs=(band, geom)) as pool:- rows = list(pool.map(_anchor, tasks, chunksize=2))+ rows = _pool_map(_anchor, [(str(p), c.place) for c in clips for p in c.files], {"band": band, "geom": geom}, jobs)at = 0for c in clips:c.rows = rows[at:at + len(c.files)]@@ -236,6 +252,18 @@ def read_anchors(clips: list[Clip], band: fl.Band, geom: fl.Geometry, jobs: int)at += len(c.files)+def read_places(bodies: list[Clip], ring: fl.Band, geom: fl.Geometry, jobs: int) -> None:+ """Every body frame's place read again on the ring round the mouth where its own mouth is closed, and moved with+ the read frames round it where it is not (`face_layer.on_ring`). The talking clip keeps the band's: its jaw moves+ as it talks, and the layer is cut from it."""+ tasks = [(str(p), c.place, c.native, a) for c in bodies for p, a in zip(c.files, c.used)]+ placed = _pool_map(_on_ring, tasks, {"ring": ring, "geom": geom}, jobs)+ at = 0+ for c in bodies:+ c.used = fl.on_ring(c.used, placed[at:at + len(c.files)], cyclic=c.loop)+ at += len(c.files)++def lower_faces(clip: Clip, geom: fl.Geometry, jobs: int) -> list[np.ndarray]:def one(k: int) -> np.ndarray:arr = np.asarray(clip.on_reference(k, geom.size), np.float32)@@ -285,8 +313,9 @@ def build(request: Path, out: Path, jobs: int = 0) -> dict[str, Any]:ref_face = fl.lower_face(ref, geom)if not ref_face.any():raise SystemExit(f"{PROG}: no lower face grows from face.lower_face.seeds on the reference: the seeds must sit on its skin")- band = fl.Band.read(reference, fl.weight(fl.fill_holes(ref_face), geom), geom)- read_anchors(clips, band, geom, jobs)+ w_ref = fl.weight(fl.fill_holes(ref_face), geom)+ read_anchors(clips, fl.Band.read(reference, w_ref, geom), geom, jobs)+ read_places(bodies, fl.Band.ring(reference, w_ref, geom), geom, jobs)faces = []for c in clips:faces.extend(lower_faces(c, geom, jobs))@@ -384,8 +413,9 @@ def build(request: Path, out: Path, jobs: int = 0) -> dict[str, Any]:c_entry = {"frame": entry_frame, "own_lift": target, "set": entry_set, "rest": rest,"candidates": {str(k): {"lift": lift(k), "switch": round(v, 3)} for k, v in switches.items()}}body_rows.append({"name": c.name, "loop": c.loop, "cells": cells, "keep": keeps,- "to_body": [_six(m) for m in to_body], "own_mouth": c.own_mouth, "entry": c_entry,- "median_step": med_b})+ "to_body": [_six(m) for m in to_body], "ring": [u["ring"] for u in c.used],+ "ring_read": [k for k, u in enumerate(c.used) if u["ring_read"]],+ "own_mouth": c.own_mouth, "entry": c_entry, "median_step": med_b})for row, (c, folder, cells, keeps, to_body, Fb, med_b, own, entry_frame) in zip(body_rows, prepared):row["switch"] = {str(r): [round(v, 3) for v in each(lambda k, r=r: switch(c, to_body, keeps, Fb, med_b, k, r), range(len(c.files)), jobs)]@@ -434,6 +464,8 @@ def _clip_report(c: Clip) -> dict[str, Any]:"band_residual_place_only": _summary([r["band_residual_place_only"] for r in rows]),"band_residual_fixed": _summary([r["band_residual_fixed"] for r in rows]),"misread": [k for k, u in enumerate(used) if u["misread"]],+ **({"ring_read": sum(u["ring_read"] for u in used), "ring_move": _summary([float(np.hypot(*u["ring"])) for u in used])}+ if used and "ring" in used[0] else {}),"wide_grid": [k for k, r in enumerate(rows) if r["wide_grid"]],"wide_search": [k for k, r in enumerate(rows) if r["search_px"] > fl.SEARCH],"search_edge": [k for k, r in enumerate(rows) if r["edge"]]}