Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

test(parts): a tail behind the body read on every frame and laid without changing a solid body pixel; a tail in front, an empty mask and no order not laid (red)

Soohong Kimceb5a9c36c1a
diff --git a/tests/video/test_parts.py b/tests/video/test_parts.py
new file mode 100644
index 0000000..a87e369
--- /dev/null
+++ b/tests/video/test_parts.py
@@ -0,0 +1,337 @@
+# SPDX-License-Identifier: Apache-2.0
+"""A part on its own beat told apart from the body (sprite_gen/video/parts.py): the tail's mask read from the clip's
+frames a cycle apart, on frames filmed and on frames made between two; whether it lies in front of the body or behind
+it; the tail of another drawing laid behind the body in place of a frame's own without changing a pixel of the body; and
+what is not laid — a tail in front of the body, a frame or a tail with no mask, an order read from nothing.
+
+Three walkers are drawn here, 26 frames a cycle with a bob, legs and arms swinging, over a clip of three cycles. Their
+sleeves and hands are the tail's orange; every part is outlined in a darker shade of its colour, as cel art is, and
+drawn at four times the size and averaged, so every edge is as soft as a keyed frame's. The tail, five segments from a
+root at the back, sways on its own beat of 41 frames. Seen from the side (`side`) it hangs behind the body and the near
+hand swings over its root. Seen from behind it hangs over the legs and swings over the hands, in front of them: a thin
+tail rooted at the small of the back (`back`), which on three frames of the clip lies wholly over a leg, and a bushy one
+rooted lower with the hands hanging lower (`back_bushy`), which crosses a hand on a third of the frames and a leg on
+all of them. Each pixel's share of the tail is known, and the mask is read against the pixels the tail shows in. A frame
+made between two filmed ones is the walker drawn at the time between them, as an optical-flow in-between draws it.
+Synthetic frames only."""
+
+from __future__ import annotations
+
+import functools
+import math
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from sprite_gen.video import auto_motion, parts
+
+N, BEAT = 26, 41 # frames a walk cycle; frames the tail takes to sway and back
+CLIP = range(-N, 2 * N) # the clip on the loop's clock: the loop is 0..N-1, a cycle of the clip either side of it
+READ = range(-N, 2 * N, 2) # the clip's frames the order is read from
+SS, H, W = 4, 240, 200
+ORANGE, CREAM = (0.9, 0.5, 0.15), (0.95, 0.9, 0.75)
+LINE = 1.5 # px: every part's outline, a darker shade of its colour
+# The tail on each walker: its first segment's angle (degrees) and how far it sways, its root, its segments' radii.
+TAILS = {"side": dict(base=160, sway=38, root=(120, 88), radii=(9, 8, 7, 6, 5)),
+ "back": dict(base=270, sway=68, root=(118, 100), radii=(9, 8, 7, 6, 5)),
+ "back_bushy": dict(base=270, sway=68, root=(138, 100), radii=(10, 12, 12, 11, 8))}
+REACH = {"back": 54, "back_bushy": 72} # px from the shoulder to the hand, seen from behind
+MADE = (2.5, 6.5, 9.4, 17.6, 21.5) # times of frames made between two filmed ones
+HIDDEN = (-18, 3, 44) # the frames the thin tail seen from behind lies wholly over a leg on
+# The mask against the tail as drawn, over the solid pixels: each frame's overlap with it (IoU) and the solid pixels it
+# takes that are not the tail's (leak) or leaves that are (miss), shares of the tail's. Where the tail meets the hand of
+# its own colour a pixel or two of either may go to the other side. Measured on the walker seen from the side, every
+# frame of the clip and every frame made: IoU 0.865 at least, leak 0.095 at most, miss 0.066.
+IOU_MIN, IOU_MEDIAN = 0.83, 0.87
+LEAK_MAX, MISS_MAX = 0.16, 0.1
+OFF = 0.15 # premultiplied RGBA distance: a pixel of the frame laid this far from the drawing is off it
+
+_yy, _xx = np.mgrid[0:H * SS, 0:W * SS].astype(np.float32)
+_yy, _xx = (_yy + 0.5) / SS, (_xx + 0.5) / SS
+
+
+def _seg(p: tuple[float, float], q: tuple[float, float]) -> np.ndarray:
+ (py, px), (qy, qx) = p, q
+ vy, vx = qy - py, qx - px
+ t = np.clip(((_yy - py) * vy + (_xx - px) * vx) / max(vy * vy + vx * vx, 1e-9), 0, 1)
+ return np.hypot(_yy - (py + t * vy), _xx - (px + t * vx))
+
+
+def _cover(inside: np.ndarray) -> np.ndarray:
+ return inside.astype(np.float32).reshape(H, SS, W, SS).mean(axis=(1, 3))
+
+
+def _shape(e: np.ndarray, rgb: np.ndarray | tuple, line: tuple | None = None) -> tuple[np.ndarray, np.ndarray]:
+ """A shape (its signed distance `e`, inside at or below 0) filled with `rgb`, outlined LINE px in `line` (a darker
+ shade of its colour where none is given): its coverage and colour per pixel."""
+ a = _cover(e <= 0)
+ fill = np.clip(_cover(e <= -LINE) / np.maximum(a, 1e-6), 0, 1)[..., None]
+ rgb = np.broadcast_to(np.asarray(rgb, np.float32), (H, W, 3))
+ line = 0.35 * np.asarray(rgb if line is None else line, np.float32)
+ return a, fill * rgb + (1 - fill) * line
+
+
+def _capsule(p, q, r, rgb):
+ return _shape(_seg(p, q) - r, rgb)
+
+
+def _tail(at: float, bob: float, walker: str) -> tuple[np.ndarray, np.ndarray]:
+ """The tail at time `at` of its beat, its root on a body bobbed `bob` px: five segments, a wave running down them
+ from the root, orange to a cream tip."""
+ shape = TAILS[walker]
+ phase = 2 * math.pi * at / BEAT
+ angles = [shape["base"] + shape["sway"] * math.sin(phase)]
+ for k in range(1, 5):
+ angles.append(angles[-1] + 14 * math.sin(phase - 0.7 * k))
+ p = (shape["root"][0] + bob, shape["root"][1])
+ e, tip = np.full(_yy.shape, np.inf, np.float32), None
+ for k, (angle, r) in enumerate(zip(angles, shape["radii"])):
+ q = (p[0] - 13 * math.sin(math.radians(angle)), p[1] + 13 * math.cos(math.radians(angle)))
+ ek = _seg(p, q) - r
+ if k == 4:
+ tip = ek < e
+ e = np.minimum(e, ek)
+ p = q
+ tipped = np.clip(_cover(tip & (e <= 0)) / np.maximum(_cover(e <= 0), 1e-6), 0, 1)[..., None]
+ return _shape(e, (1 - tipped) * np.array(ORANGE, np.float32) + tipped * np.array(CREAM, np.float32), (0.3, 0.2, 0.1))
+
+
+def _bob(body: float) -> float:
+ return 2.5 * math.cos(4 * math.pi * body / N)
+
+
+def _arm(shoulder, hand, r, hand_r):
+ return _shape(np.minimum(_seg(shoulder, hand) - r, _seg(hand, hand) - hand_r), ORANGE)
+
+
+def _layers(body: float, tail: float | None, walker: str, tail_bob: float | None = None
+ ) -> list[tuple[bool, np.ndarray, np.ndarray]]:
+ """The walker's layers back to front, (is the tail, coverage, colour): its body at time `body`, its tail at `tail`
+ on a root bobbed as the body is (or `tail_bob`), or none."""
+ phase = 2 * math.pi * body / N
+ bob = _bob(body)
+ if walker == "side":
+ leg, arm = 0.42 * math.sin(phase), 0.5 * math.sin(phase)
+ hip, shoulder = (128 + bob, 100), (78 + bob, 104)
+ foot = lambda a: (hip[0] + 80 * math.cos(a), hip[1] + 80 * math.sin(a)) # noqa: E731
+ hand = lambda a: (shoulder[0] + 52 * math.cos(a), shoulder[1] + 52 * math.sin(a)) # noqa: E731
+ far_hand, near_hand, far_foot, near_foot = hand(arm), hand(-arm), foot(-leg), foot(leg)
+ body_ = [_arm(shoulder, far_hand, 5, 6.5),
+ _capsule(hip, far_foot, 7, (0.15, 0.16, 0.26)), _capsule(far_foot, (far_foot[0], far_foot[1] + 11), 5, (0.12, 0.12, 0.2)),
+ _capsule((70 + bob, 104), (126 + bob, 100), 16, (0.25, 0.4, 0.75)),
+ _capsule((44 + bob, 112), (44 + bob, 112), 18, (0.95, 0.8, 0.65)),
+ _capsule((40 + bob, 120), (40 + bob, 120), 3, (0.1, 0.1, 0.15)),
+ _capsule((96 + bob, 90), (96 + bob, 118), 2.5, (0.1, 0.15, 0.3)),
+ _capsule(hip, near_foot, 7, (0.2, 0.22, 0.35)), _capsule(near_foot, (near_foot[0], near_foot[1] + 11), 5, (0.16, 0.16, 0.25)),
+ _arm(shoulder, near_hand, 5, 6.5)]
+ else: # seen from behind: the legs, the back and the head, the arms
+ lift = (7 * max(0.0, math.sin(phase)), 7 * max(0.0, -math.sin(phase)))
+ body_ = []
+ for x, up, rgb in ((86, lift[0], (0.15, 0.16, 0.26)), (114, lift[1], (0.2, 0.22, 0.35))):
+ hip, foot = (130 + bob, x), (210 - up, x + 2 * math.sin(phase))
+ body_ += [_capsule(hip, foot, 8, rgb), _capsule(foot, foot, 7, (0.12, 0.12, 0.2))]
+ body_ += [_capsule((72 + bob, 100), (124 + bob, 100), 24, (0.25, 0.4, 0.75)),
+ _capsule((44 + bob, 100), (44 + bob, 100), 19, (0.95, 0.8, 0.65)),
+ _capsule((112 + bob, 80), (112 + bob, 120), 2.5, (0.1, 0.15, 0.3))]
+ for x, out, sign in ((72, -10, 1), (128, 10, -1)):
+ shoulder = (66 + bob, x)
+ hand = (shoulder[0] + REACH[walker] - 5 * sign * math.sin(phase), x + out + 3 * sign * math.sin(phase))
+ body_.append(_arm(shoulder, hand, 5.5, 7))
+ rest = [(False, a, c) for a, c in body_]
+ if tail is None:
+ return rest
+ tail_ = [(True, *_tail(tail, bob if tail_bob is None else tail_bob, walker))]
+ return tail_ + rest if walker == "side" else rest + tail_
+
+
+def _compose(layers) -> tuple[np.ndarray, np.ndarray]:
+ """Straight RGBA, and each pixel's coverage by the tail as it shows."""
+ premul, alpha, tail = np.zeros((H, W, 3), np.float32), np.zeros((H, W), np.float32), np.zeros((H, W), np.float32)
+ for is_tail, a, rgb in layers:
+ premul = rgb * a[..., None] + premul * (1 - a[..., None])
+ tail = (a if is_tail else 0) + tail * (1 - a)
+ alpha = a + alpha * (1 - a)
+ return np.concatenate([premul / np.maximum(alpha, 1e-6)[..., None], alpha[..., None]], -1), tail
+
+
+def _image(rgba: np.ndarray) -> Image.Image:
+ return Image.fromarray(np.clip(rgba * 255 + 0.5, 0, 255).astype(np.uint8), "RGBA")
+
+
[email protected]_cache(maxsize=None)
+def _drawn(body: float, tail: float | None, walker: str, tail_bob: float | None = None
+ ) -> tuple[Image.Image, np.ndarray, np.ndarray]:
+ """The walker drawn: (the frame, the pixels the tail shows in — its share of their colour over half — and the solid
+ pixels)."""
+ rgba, tail_share = _compose(_layers(body, tail, walker, tail_bob))
+ alpha = rgba[..., 3]
+ return _image(rgba), (tail_share > 0.5 * alpha) & (alpha > parts.SOFT), alpha > parts.SOLID_GROW
+
+
+def _frame(t: float, walker: str) -> Image.Image:
+ return _drawn(t, t, walker)[0]
+
+
[email protected]_cache(maxsize=None)
+def _masker(walker: str) -> parts.Masker:
+ clip = {t: _frame(t, walker) for t in CLIP}
+ regions, _ = auto_motion.discover([clip[t] for t in range(N)], reference_index=0)
+ return parts.Masker(clip, N, regions)
+
+
+def _read(mask: np.ndarray, t: float, walker: str) -> dict[str, float]:
+ _, tail, solid = _drawn(t, t, walker)
+ tail, mask = tail & solid, mask & solid
+ return {"iou": (mask & tail).sum() / (mask | tail).sum(), "leak": (mask & ~tail).sum() / tail.sum(),
+ "miss": (tail & ~mask).sum() / tail.sum()}
+
+
+def _crossed(t: float) -> bool:
+ """Whether the near hand and the tail of the walker seen from the side, each drawn alone, overlap at time t."""
+ a = -0.5 * math.sin(2 * math.pi * t / N)
+ hand = (78 + _bob(t) + 52 * math.cos(a), 104 + 52 * math.sin(a))
+ return bool(((_cover(_seg(hand, hand) <= 6.5) > 0.5) & (_tail(t, _bob(t), "side")[0] > 0.5)).any())
+
+
+def _bytes(frames) -> list[bytes]:
+ return [np.asarray(f).tobytes() for f in frames]
+
+
+def test_a_tail_behind_the_body_is_read_where_it_is_drawn_on_every_frame():
+ # Every frame of the clip, and frames made between two read at the time they show: none left out of the bounds.
+ masker = _masker("side")
+ reads = [_read(masker.clip_mask(t), t, "side") for t in CLIP]
+ reads += [_read(masker.mask(_frame(t, "side"), t), t, "side") for t in MADE]
+ iou, leak, miss = (np.array([r[k] for r in reads]) for k in ("iou", "leak", "miss"))
+ assert iou.size == 83
+ assert iou.min() >= IOU_MIN and np.median(iou) >= IOU_MEDIAN, iou.round(3)
+ assert leak.max() <= LEAK_MAX and miss.max() <= MISS_MAX, (leak.round(3), miss.round(3))
+ # and the near hand, the tail's colour, swings over the tail's root on some frames of the loop
+ assert sum(_crossed(t) for t in range(N)) >= 4
+
+
[email protected]("walker, rare", [("side", parts.RARE), ("back", parts.RARE), ("back_bushy", parts.RARE),
+ ("side", parts.RARE / 2), ("side", 2 * parts.RARE),
+ ("back_bushy", parts.RARE / 2), ("back_bushy", 2 * parts.RARE)])
+def test_in_front_of_the_body_or_behind_it_is_read_from_the_clip(walker, rare):
+ # The order's mask seeds the body with the colours the clip's rims rarely show; half the share and twice it read
+ # each walker the same way.
+ read = _masker(walker).order(READ, rare)
+ if walker == "side":
+ assert read["order"] == "behind" and read["front_share"] < parts.FRONT - 0.1, read
+ else:
+ assert read["order"] == "front" and read["front_share"] > parts.FRONT + 0.05, read
+
+
+def test_a_tail_laid_behind_the_body_changes_none_of_its_solid_pixels():
+ # The eased retime's case: the tail of a frame made a frame on, laid behind the body of the frame at its own time.
+ masker = _masker("side")
+ times = range(0, N, 3)
+ lays = [(_frame(t, "side"), t, _frame(t + 1.0, "side"), t + 1.0) for t in times]
+ made, record = parts.lay_behind(masker, READ, lays)
+ assert record["refused"] is None and record["order"]["order"] == "behind" and len(made) == len(lays)
+ for (frame, t, source, at), laid, done in zip(lays, made, record["lays"]):
+ mask, source_mask = masker.clip_mask(t), masker.mask(source, at)
+ f, out = np.asarray(frame), np.asarray(laid)
+ body = ~mask & (f[..., 3] > 0)
+ solid = body & (f[..., 3] >= round(parts.SOLID * 255))
+ assert (out[solid] == f[solid]).all(), t
+ # a body pixel less than solid shows the new tail through it, and is itself as drawn: laid over the tail's layer
+ layer = np.asarray(source).copy()
+ layer[~source_mask] = 0
+ under = np.asarray(Image.alpha_composite(Image.fromarray(layer), frame))
+ assert (out[body & ~solid] == under[body & ~solid]).all(), t
+ assert done["refused"] is None and done["laid_px"] == int((layer[..., 3] > 0).sum())
+ # and what it makes is the walker with its body at t and its tail a frame on, but where the masks miss or overreach
+ drawn = _drawn(t, at, "side", _bob(at))[0]
+ assert _off(laid, drawn).sum() <= LEAK_MAX * _drawn(t, t, "side")[1].sum(), t
+
+
[email protected]("walker", ["back", "back_bushy"])
+def test_a_tail_in_front_of_the_body_is_not_laid_and_no_frame_is_read_for_it(walker, monkeypatch):
+ # Laid behind the body, the tail would go under the legs and the hands it hangs over. The order is read first; no
+ # frame's mask is read, and every frame is as it was given.
+ masker = _masker(walker)
+ monkeypatch.setattr(masker, "mask", lambda frame, at: pytest.fail(f"a mask read at {at} after the order"))
+ lays = [(_frame(t, walker), t, _frame(t + 0.5, walker), t + 0.5) for t in range(0, N, 5)]
+ made, record = parts.lay_behind(masker, READ, lays)
+ assert record["refused"] == "front" and record["order"]["order"] == "front" and record["lays"] == []
+ assert _bytes(made) == _bytes(frame for frame, _, _, _ in lays)
+
+
+def test_a_tail_lying_wholly_over_a_leg_has_no_mask_and_nothing_is_laid_with_it():
+ # Where the thin tail lies wholly over a leg, no frame of its walk phase a cycle away leaves clear what it covers:
+ # there is no rim to read it from, and its mask is empty. Laid on with that mask, the frame would keep its own tail
+ # beside the one laid; laid from it, the frame would lose its own and get none. Nothing is laid, whichever the order.
+ masker = _masker("back")
+ for t in HIDDEN:
+ frame, tail, solid = _drawn(t, t, "back")
+ assert (tail & solid).sum() > 1000 and not masker.clip_mask(t).any(), t
+ other = t + N if t + N in CLIP else t - N
+ source, source_mask = _frame(other, "back"), masker.clip_mask(other)
+ for order in ("behind", "front"):
+ for args in ((frame, masker.clip_mask(t), source, source_mask), (source, source_mask, frame, masker.clip_mask(t))):
+ made, record = parts.lay(*args, order)
+ assert record["refused"] == "empty" and record["laid_px"] == 0, (t, order)
+ assert made.tobytes() == args[0].tobytes()
+
+
+def test_a_tail_with_no_mask_refuses_every_frame_laid_with_it():
+ # A tail hidden wholly behind the body shows nothing of itself to read: laid from such a frame, the frame laid on
+ # would lose its own tail and get none. The frames laid before it are not laid either.
+ masker = _masker("side")
+ hidden = _drawn(18.0, None, "side")[0]
+ lays = [(_frame(t, "side"), t, _frame(t + 1.0, "side"), t + 1.0) for t in (0, 9)] + [(_frame(18, "side"), 18, hidden, 18.0)]
+ made, record = parts.lay_behind(masker, READ, lays)
+ assert record["refused"] == "empty" and record["order"]["order"] == "behind"
+ assert [(done["refused"], done["source_px"] > 0) for done in record["lays"]] == [(None, True), (None, True), ("empty", False)]
+ assert _bytes(made) == _bytes(frame for frame, _, _, _ in lays)
+
+
+def test_frames_none_of_which_has_one_of_its_walk_phase_a_cycle_away_read_no_order():
+ # One cycle of the clip has no frame a cycle from another, so nothing reads the part in front of the body or behind
+ # it; read as behind, a tail in front would be laid under the body. Nothing is laid.
+ masker = _masker("side")
+ assert masker.order(range(N)) == {"order": None, "front_share": None, "part_px": 0}
+ lays = [(_frame(t, "side"), t, _frame(t + 1.0, "side"), t + 1.0) for t in (0, 9)]
+ made, record = parts.lay_behind(masker, range(N), lays)
+ assert record["refused"] == "no pairs" and record["lays"] == []
+ assert _bytes(made) == _bytes(frame for frame, _, _, _ in lays)
+
+
+def _straight(im: Image.Image) -> np.ndarray:
+ return np.asarray(im).astype(np.float32) / 255
+
+
+def _off(made: Image.Image, drawn: Image.Image) -> np.ndarray:
+ """The pixels of `made` off `drawn` (premultiplied)."""
+ a, b = _straight(made), _straight(drawn)
+ a[..., :3] *= a[..., 3:]
+ b[..., :3] *= b[..., 3:]
+ return np.abs(a - b).max(-1) > OFF
+
+
[email protected]("walker", ["side", "back_bushy"])
+def test_with_the_tails_own_masks_the_frame_laid_is_the_drawing(walker):
+ # Laying alone: given the pixels each tail shows in, the frame laid is the walker drawn with the frame's body and the
+ # other drawing's tail, but at the edges where a soft pixel of the one met the other, and behind the body where the
+ # other drawing's own hand hid some of its tail and the frame's hand does not. In front of the body, the frame's own
+ # tail's place is taken from the frames of its walk phase a cycle or two away.
+ order = "behind" if walker == "side" else "front"
+ for t in range(0, N, 3):
+ frame, mask, _ = _drawn(t, t, walker)
+ other = t + 1.0 if order == "behind" else t + N
+ source, source_mask, _ = _drawn(other, other, walker)
+ plates = [_drawn(s, s, walker)[:2] for s in (t - N, t + N, t + 2 * N)] if order == "front" else []
+ laid, record = parts.lay(frame, mask, source, source_mask, order, plates=plates)
+ assert record["refused"] is None
+ drawn = _drawn(t, other, walker, _bob(other))[0]
+ assert _off(laid, drawn).sum() <= 0.1 * mask.sum(), t
+
+
+def test_an_order_neither_in_front_nor_behind_is_refused():
+ frame = _frame(0, "side")
+ none = np.zeros((H, W), bool)
+ with pytest.raises(ValueError, match="front"):
+ parts.lay(frame, none, frame, none, "over")