Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.
fix(video): normalize the canvas background to the exact key; tell residual background from a clipped subject
Soohong Kimc2dfce20c68c
diff --git a/sprite_gen/frames/cutout.py b/sprite_gen/frames/cutout.pyindex 01e2bc5..e2b6bec 100644--- a/sprite_gen/frames/cutout.py+++ b/sprite_gen/frames/cutout.py@@ -270,13 +270,15 @@ def _matte_route(}-def _extract_route(image: Image.Image, kind: str) -> tuple[Image.Image, dict[str, Any]]:+def extract_route(image: Image.Image, kind: str) -> tuple[Image.Image, dict[str, Any]]:"""Magenta/green key background → reuse the verified `extract` chroma engine (no drift).The engine keys from the background colour it detects on the borders(`detect_background_key_rgb`) as well as the pure key, so the brightnessthe model happened to paint does not decide whether the cut lands. Thedetected colour is reported as `chroma_key_painted` for the audit trail.+ Public because `video-canvas` normalizes a still's background through this+ exact matte, so the canvas and `video-frames` agree by construction."""from sprite_gen.frames.extract import detect_background_key_rgb, remove_chroma_background@@ -318,7 +320,7 @@ def cutout(route = _detect_key_kind(_corner_average(image)) if key == "auto" else keyif route in ("magenta", "green"):- result, route_stats = _extract_route(image, route)+ result, route_stats = extract_route(image, route)else:result, route_stats = _matte_route(image, input_path, strength, band, erode, tolerance)diff --git a/sprite_gen/video/canvas.py b/sprite_gen/video/canvas.pyindex 2e5f1b2..45301ee 100644--- a/sprite_gen/video/canvas.py+++ b/sprite_gen/video/canvas.py@@ -8,9 +8,17 @@ attack or projectile needs room in front (wide), everything else stays square.The state -> canvas table below is the single owner of that rule; `--shape`overrides it per call.-The padding is filled with the still's own corner colour (its chroma key) so the-clip stays keyable end to end. A still whose corners disagree is refused — a-non-flat background cannot be extended without guessing.+The padding is filled with the chroma key so the clip stays keyable end to end.+A still whose corners disagree is refused — a non-flat background cannot be+extended without guessing.++Image models paint "#00FF00" a little differently every time ((8, 166, 25) on+2026-09-11), and the video model reproduces the input colour almost exactly. So+when the corners are a green/magenta key at *any* brightness, the flat+background is repainted to the exact declared key here — the pixels the+`cutout` chroma matte erases become the pure key, the subject is untouched —+and the padding uses that same pure key. A white/ivory base is padded with its+own corner colour as before (no chroma key to normalize to)."""from __future__ import annotations@@ -23,6 +31,9 @@ from typing import Anyfrom PIL import Image+from sprite_gen._deps import np+from sprite_gen.frames.cutout import KEY_TARGETS, extract_route+from sprite_gen.frames.extract import is_key_familyfrom sprite_gen.spec.runio import atomic_write_textSHAPE_SQUARE = "square"@@ -54,6 +65,7 @@ SHAPE_DEFAULTS: dict[str, CanvasProfile] = {SHAPE_WIDE: STATE_CANVAS["attack"],}CORNER_TOLERANCE = 24 # max per-channel spread across the four corners for a "flat" background+KEYS = ("auto", "green", "magenta", "white") # auto: the corners decide; white: no chroma key, pad with the corner colourdef profile_for(state: str | None, shape: str | None = None) -> CanvasProfile:@@ -81,6 +93,61 @@ def corner_key(image: Image.Image) -> tuple[int, int, int]:return tuple(round(sum(c[i] for c in corners) / 4) for i in range(3)) # type: ignore[return-value]+def resolve_key(corner: tuple[int, int, int], key: str) -> str | None:+ """Which chroma key the still is on: `green` | `magenta`, or None for a non-key (white/ivory) base.++ `auto` classifies the flat corner colour with the engine's own family rule+ (`is_key_family` — the key's hue at any brightness). An explicit green/magenta+ that the corners are not is refused rather than repainted blindly.+ """+ if key not in KEYS:+ raise SystemExit(f"video-canvas: unknown --key {key!r}; expected one of {', '.join(KEYS)}")+ if key == "white":+ return None+ if key == "auto":+ for kind, target in KEY_TARGETS.items():+ if is_key_family(corner, target):+ return kind+ return None+ if not is_key_family(corner, KEY_TARGETS[key]):+ raise SystemExit(+ f"video-canvas: --key {key} but the still's corners are {corner}, not a {key} key family colour; "+ "pass --key auto to let the corners decide or --key white for a non-chroma base"+ )+ return key+++def normalize_key(image: Image.Image, kind: str) -> tuple[Image.Image, dict[str, Any]]:+ """Repaint the still's flat background to the exact declared key; the subject stays byte-identical.++ The mask is the `cutout` chroma matte's own alpha-0 set (keyed from the+ painted background colour, see `extract.detect_background_key_rgb`), so the+ pixels this repaints are exactly the pixels `video-frames` will erase. Fails+ loud when a corner survives the matte — then the still is not on a key the+ engine can cut and padding it would only hide that.+ """+ target = KEY_TARGETS[kind]+ rgb = image.convert("RGB")+ keyed, stats = extract_route(rgb.convert("RGBA"), kind)+ erased = np.array(keyed, dtype=np.uint8)[..., 3] == 0+ data = np.array(rgb, dtype=np.uint8)+ h, w = erased.shape+ corners = ((0, 0), (0, w - 1), (h - 1, 0), (h - 1, w - 1))+ if not all(erased[y, x] for y, x in corners):+ raise SystemExit(+ f"video-canvas: the still's corners survived the {kind} key matte "+ f"(painted {tuple(stats['chroma_key_painted'])}); the background is not a {kind} key the engine can cut"+ )+ data[erased] = target+ report = {+ "key": kind,+ "key_painted": stats["chroma_key_painted"],+ "normalized_px": int(erased.sum()),+ "normalized_pct": round(100 * float(erased.mean()), 2),+ }+ return Image.fromarray(data, "RGB"), report++def pad_canvas(image: Image.Image,profile: CanvasProfile,@@ -88,10 +155,20 @@ def pad_canvas(facing: str = "right",headroom: float | None = None,lead: float | None = None,+ key: str = "auto",) -> tuple[Image.Image, dict[str, Any]]:- """Return (padded RGB image, placement report). Never downsizes the still."""- key = corner_key(image)- src = image.convert("RGB")+ """Return (padded RGB image, placement report). Never downsizes the still.++ On a green/magenta base the background is normalized to the exact key and+ the padding is that key; otherwise the padding is the corner colour.+ """+ corner = corner_key(image)+ kind = resolve_key(corner, key)+ if kind is None:+ src, fill, key_report = image.convert("RGB"), corner, {"key": None, "key_painted": list(corner), "normalized_px": 0, "normalized_pct": 0.0}+ else:+ src, key_report = normalize_key(image, kind)+ fill = KEY_TARGETS[kind]w, h = src.sizehead = profile.headroom if headroom is None else headroomfront = profile.lead if lead is None else lead@@ -112,7 +189,7 @@ def pad_canvas(canvas_h = max(h, round(canvas_w / profile.ratio))y = canvas_h - hx = 0 if facing == "right" else canvas_w - w- canvas = Image.new("RGB", (canvas_w, canvas_h), key)+ canvas = Image.new("RGB", (canvas_w, canvas_h), fill)canvas.paste(src, (x, y))report = {"shape": profile.shape,@@ -123,7 +200,9 @@ def pad_canvas("headroom": head,"lead": front,"facing": facing,- "key_rgb": list(key),+ "key_rgb": list(fill),+ "corner_rgb": list(corner),+ **key_report,"why": profile.why,}return canvas, report@@ -139,12 +218,13 @@ def run_canvas(headroom: float | None,lead: float | None,report_path: Path | None,+ key: str = "auto",) -> dict[str, Any]:still = still.expanduser().resolve()if not still.is_file():raise SystemExit(f"video-canvas: still not found: {still}")profile = profile_for(state, shape)- canvas, report = pad_canvas(Image.open(still), profile, facing=facing, headroom=headroom, lead=lead)+ canvas, report = pad_canvas(Image.open(still), profile, facing=facing, headroom=headroom, lead=lead, key=key)out = out.expanduser().resolve()out.parent.mkdir(parents=True, exist_ok=True)tmp = out.with_name(out.name + ".part")@@ -164,6 +244,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:parser.add_argument("--facing", choices=("right", "left"), default="right", help="which way the subject faces (wide canvases add room in front)")parser.add_argument("--headroom", type=float, help="tall: empty fraction above the subject (default from the profile)")parser.add_argument("--lead", type=float, help="wide: empty fraction in front of the subject (default from the profile)")+ parser.add_argument("--key", choices=KEYS, default="auto", help="chroma key of the still (auto reads the corners; green/magenta are normalized to the exact key; white pads with the corner colour)")parser.add_argument("--report", type=Path, help="write the canvas report JSON here")@@ -172,6 +253,7 @@ def run(**kwargs: object) -> int:Path(str(kwargs["still"])), Path(str(kwargs["out"])),state=kwargs.get("state"), shape=kwargs.get("shape"), facing=str(kwargs.get("facing") or "right"), # type: ignore[arg-type]headroom=kwargs.get("headroom"), lead=kwargs.get("lead"), report_path=kwargs.get("report"), # type: ignore[arg-type]+ key=str(kwargs.get("key") or "auto"),)print(json.dumps(payload, ensure_ascii=False, indent=2))return 0diff --git a/sprite_gen/video/frames.py b/sprite_gen/video/frames.pyindex e754fc5..9b9d57e 100644--- a/sprite_gen/video/frames.py+++ b/sprite_gen/video/frames.py@@ -8,6 +8,15 @@ per-frame alpha coverage and an **edge-contact check**: a frame whose subjecttouches the top/left/right edge means the model framed too tight (a jump whosehair left the frame, 2026-09-08) and the run fails loud — the fix is a taller orwider canvas (`video-canvas`), not a quiet crop.++An opaque edge pixel is not always the subject. When the model painted the key+a little off and part of that background survives the matte, the edge band is+"touched" by leftover *background* (2026-09-11: a (8, 162, 24) green read as+"framed too tight"). The check therefore classifies every contact pixel by its+raw colour — the declared key's hue family (`extract.is_key_family`) is+**residual background**, anything else is the **subject** — and the two fail+with different messages: residual points at `video-canvas` normalization of the+base still, subject contact at a taller/wider canvas."""from __future__ import annotations@@ -22,6 +31,7 @@ from typing import Anyfrom PIL import Imagefrom sprite_gen.frames.cutout import cutout+from sprite_gen.frames.extract import is_key_familyfrom sprite_gen.spec.runio import atomic_write_textEDGE_ROWS = 4 # rows/cols inspected at each edge@@ -78,6 +88,31 @@ def edge_contact(image: Image.Image) -> dict[str, int]:return {"top": top, "left": left, "right": right}+def classify_edge_contact(raw: Image.Image, keyed: Image.Image, chroma_key: tuple[int, int, int] | None) -> dict[str, int]:+ """Split the opaque edge-band pixels into `subject` and `residual` (leftover key background).++ A contact pixel whose *raw* colour is the declared key's hue family is+ background the matte failed to erase, not the subject. Without a chroma key+ (matte route) every contact is the subject.+ """+ alpha = keyed.convert("RGBA").getchannel("A").load()+ rgb = raw.convert("RGB").load()+ w, h = keyed.size+ band = set()+ band.update((x, y) for y in range(min(EDGE_ROWS, h)) for x in range(w))+ band.update((x, y) for x in range(min(EDGE_ROWS, w)) for y in range(h))+ band.update((x, y) for x in range(max(0, w - EDGE_ROWS), w) for y in range(h))+ subject = residual = 0+ for x, y in band:+ if alpha[x, y] <= 0:+ continue+ if chroma_key is not None and is_key_family(rgb[x, y], chroma_key):+ residual += 1+ else:+ subject += 1+ return {"subject": subject, "residual": residual}++def key_frames(raw_files: list[Path],keyed_dir: Path,@@ -98,9 +133,12 @@ def key_frames(row = {"frame": src.name, "alpha_zero_pct": alpha_zero_pct, "route": stats.get("route")}if check_edges:contact = edge_contact(image)- row["edge"] = contactif any(v > EDGE_MAX_PIXELS for v in contact.values()):+ chroma = stats.get("chroma_key")+ contact.update(classify_edge_contact(Image.open(src), image, tuple(chroma) if chroma else None))+ contact["key_painted"] = stats.get("chroma_key_painted")contacts.append({"frame": src.name, **contact})+ row["edge"] = contactif hist[0] == 0:raise SystemExit(f"video-frames: {src.name} keyed to 0% transparency — the background is not the chroma key")rows.append(row)@@ -112,16 +150,34 @@ def key_frames("rows": rows,}if contacts and check_edges:- worst = max(contacts, key=lambda c: c["top"] + c["left"] + c["right"])- raise SystemExit(- f"video-frames: the subject touches the frame edge in {len(contacts)} frame(s) "- f"(worst {worst['frame']}: top {worst['top']} / left {worst['left']} / right {worst['right']} px) — "- "the clip was framed too tight; regenerate from a taller/wider canvas (`sprite-gen video-canvas`) "- "or pass --allow-edge-contact to accept the clipping"- )+ raise SystemExit(_edge_contact_message(contacts))return report+def _edge_contact_message(contacts: list[dict[str, Any]]) -> str:+ """Residual background and a clipped subject are different defects with different fixes."""+ worst = max(contacts, key=lambda c: c["top"] + c["left"] + c["right"])+ where = f"(worst {worst['frame']}: top {worst['top']} / left {worst['left']} / right {worst['right']} px)"+ subject_frames = [c for c in contacts if c["subject"] > 0]+ residual_frames = [c for c in contacts if c["residual"] > 0]+ if not subject_frames:+ painted = tuple(worst["key_painted"]) if worst.get("key_painted") else None+ return (+ f"video-frames: leftover chroma background reaches the frame edge in {len(contacts)} frame(s) {where} — "+ f"the key survived the matte (painted background {painted}); this is background, not the subject: "+ "normalize the base still with `sprite-gen video-canvas` (repaints the flat background to the exact key) "+ "and regenerate the clip"+ )+ message = (+ f"video-frames: the subject touches the frame edge in {len(subject_frames)} frame(s) {where} — "+ "the clip was framed too tight; regenerate from a taller/wider canvas (`sprite-gen video-canvas`) "+ "or pass --allow-edge-contact to accept the clipping"+ )+ if residual_frames:+ message += f"; {len(residual_frames)} frame(s) also carry leftover chroma background at the edge (`residual` in the report)"+ return message++def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool, report_path: Path | None) -> dict[str, Any]:clip = clip.expanduser().resolve()if not clip.is_file():diff --git a/tests/video/test_canvas_key_normalize.py b/tests/video/test_canvas_key_normalize.pynew file mode 100644index 0000000..f2d9ca9--- /dev/null+++ b/tests/video/test_canvas_key_normalize.py@@ -0,0 +1,162 @@+# SPDX-License-Identifier: Apache-2.0+"""`video-canvas` normalizes an off-key flat background to the exact key, and `video-frames`+tells leftover key background at the edge apart from a subject that left the frame+(plan sprite-gen/chroma-key-relative-threshold, step 3)."""++from __future__ import annotations++from pathlib import Path++import pytest+from PIL import Image++from sprite_gen._deps import np+from sprite_gen.video import canvas as canvas_mod+from sprite_gen.video import frames as frames_mod++GREEN = (0, 255, 0)+MAGENTA = (255, 0, 255)+DARK_GREEN = (8, 162, 24) # 96.38 from pure green: the 2026-09-11 cliff colour+DARK_MAGENTA = (170, 8, 180)+IVORY = (248, 247, 242)+SUBJECT = (200, 40, 40)+SIZE = (120, 160)+SUBJECT_BOX = (40, 40, 80, 150) # x0, y0, x1, y1+++def _still(tmp_path: Path, bg: tuple[int, int, int], name: str = "still.png") -> Path:+ im = Image.new("RGB", SIZE, bg)+ x0, y0, x1, y1 = SUBJECT_BOX+ for y in range(y0, y1):+ for x in range(x0, x1):+ im.putpixel((x, y), SUBJECT)+ p = tmp_path / name+ im.save(p)+ return p+++def _corners(im: Image.Image) -> list[tuple[int, ...]]:+ w, h = im.size+ return [im.getpixel(p) for p in ((0, 0), (w - 1, 0), (0, h - 1), (w - 1, h - 1))]++[email protected]("bg,kind,key", [(DARK_GREEN, "green", GREEN), ((10, 150, 30), "green", GREEN), (DARK_MAGENTA, "magenta", MAGENTA)])+def test_dark_key_still_is_repainted_to_the_exact_key_and_not_refused(tmp_path: Path, bg, kind, key) -> None:+ still = Image.open(_still(tmp_path, bg))+ out, rep = canvas_mod.pad_canvas(still, canvas_mod.profile_for("jump")) # tall: real padding above+ assert _corners(out) == [key] * 4 and rep["key_rgb"] == list(key) and rep["key"] == kind+ assert rep["corner_rgb"] == list(bg) and rep["key_painted"] == list(bg)+ ox, oy = rep["offset"]+ data = np.array(out)+ inner = data[oy:oy + SIZE[1], ox:ox + SIZE[0]]+ x0, y0, x1, y1 = SUBJECT_BOX+ subject = np.zeros(inner.shape[:2], dtype=bool)+ subject[y0:y1, x0:x1] = True+ assert (inner[subject] == SUBJECT).all(), "subject pixels must stay byte-identical"+ assert (inner[~subject] == key).all(), "every background pixel of the still becomes the exact key"+ assert rep["normalized_px"] == int((~subject).sum())+ # the padding outside the still is the exact key too+ outside = np.ones(data.shape[:2], dtype=bool)+ outside[oy:oy + SIZE[1], ox:ox + SIZE[0]] = False+ assert (data[outside] == key).all()+++def test_exact_key_still_is_unchanged_and_reports_zero_painted_drift(tmp_path: Path) -> None:+ still = Image.open(_still(tmp_path, GREEN))+ out, rep = canvas_mod.pad_canvas(still, canvas_mod.profile_for("walk"))+ assert rep["key"] == "green" and rep["key_painted"] == list(GREEN) and _corners(out) == [GREEN] * 4+ ox, oy = rep["offset"]+ assert (np.array(out)[oy:oy + SIZE[1], ox:ox + SIZE[0]] == np.array(still)).all()+++def test_non_key_base_keeps_the_corner_colour_padding(tmp_path: Path) -> None:+ still = Image.open(_still(tmp_path, IVORY))+ out, rep = canvas_mod.pad_canvas(still, canvas_mod.profile_for("jump"))+ assert rep["key"] is None and rep["key_rgb"] == list(IVORY) and rep["normalized_px"] == 0+ assert _corners(out) == [IVORY] * 4+++def test_explicit_key_that_the_corners_are_not_is_refused(tmp_path: Path) -> None:+ still = Image.open(_still(tmp_path, IVORY))+ with pytest.raises(SystemExit, match="not a green key family colour"):+ canvas_mod.pad_canvas(still, canvas_mod.profile_for("walk"), key="green")+ with pytest.raises(SystemExit, match="unknown --key"):+ canvas_mod.pad_canvas(still, canvas_mod.profile_for("walk"), key="blue")+ # --key white opts out of normalization even on a green base+ out, rep = canvas_mod.pad_canvas(Image.open(_still(tmp_path, DARK_GREEN, "g.png")), canvas_mod.profile_for("walk"), key="white")+ assert rep["key"] is None and _corners(out) == [DARK_GREEN] * 4+++def test_run_canvas_accepts_key_and_records_it(tmp_path: Path) -> None:+ rep = canvas_mod.run_canvas(_still(tmp_path, DARK_GREEN), tmp_path / "o" / "c.png", state="walk", shape=None, facing="right", headroom=None, lead=None, report_path=None, key="green")+ assert rep["key"] == "green" and rep["key_painted"] == list(DARK_GREEN)+ assert _corners(Image.open(tmp_path / "o" / "c.png")) == [GREEN] * 4+++def test_normalized_dark_still_keys_like_the_pure_key_still(tmp_path: Path) -> None:+ """The canvas output of a dark-green still and of the same still on pure green must key to the+ same alpha in `video-frames` (the double safety: normalization + relative key both close it)."""+ dark, _ = canvas_mod.pad_canvas(Image.open(_still(tmp_path, DARK_GREEN, "d.png")), canvas_mod.profile_for("walk"))+ pure, _ = canvas_mod.pad_canvas(Image.open(_still(tmp_path, GREEN, "p.png")), canvas_mod.profile_for("walk"))+ (tmp_path / "raw").mkdir()+ dark.save(tmp_path / "raw" / "frame-0000.png")+ pure.save(tmp_path / "raw" / "frame-0001.png")+ rep = frames_mod.key_frames(sorted((tmp_path / "raw").glob("*.png")), tmp_path / "keyed", key="green")+ assert rep["edge_contacts"] == [] and rep["alpha_zero_pct_min"] == rep["alpha_zero_pct_max"]+ a = np.array(Image.open(tmp_path / "keyed" / "frame-0000.png").convert("RGBA"))[..., 3]+ b = np.array(Image.open(tmp_path / "keyed" / "frame-0001.png").convert("RGBA"))[..., 3]+ assert (a == b).all()+++# --- frames: residual background vs subject at the edge ---------------------------+++def _frames(tmp_path: Path, *, residual: bool, touch_top: bool) -> list[Path]:+ d = tmp_path / "raw"+ d.mkdir(exist_ok=True)+ files = []+ for k in range(3):+ im = Image.new("RGB", (80, 100), GREEN)+ top = 0 if touch_top else 20+ for y in range(top, 90):+ for x in range(25 + k, 55 + k):+ im.putpixel((x, y), (180, 60, 30))+ if residual: # a dark-green patch of leftover background hugging the left edge+ for y in range(40, 60):+ for x in range(0, 10):+ im.putpixel((x, y), DARK_GREEN)+ p = d / f"frame-{k:04d}.png"+ im.save(p)+ files.append(p)+ return files+++def test_leftover_key_background_at_the_edge_is_reported_as_residual_not_framing(tmp_path: Path) -> None:+ files = _frames(tmp_path, residual=True, touch_top=False)+ with pytest.raises(SystemExit, match="leftover chroma background") as info:+ frames_mod.key_frames(files, tmp_path / "keyed", key="green")+ assert "framed too tight" not in str(info.value) and "video-canvas" in str(info.value)+ rep = frames_mod.key_frames(files, tmp_path / "keyed2", key="green", check_edges=False)+ assert rep["frames"] == 3+++def test_subject_contact_is_still_framing_and_carries_the_split(tmp_path: Path) -> None:+ files = _frames(tmp_path, residual=False, touch_top=True)+ with pytest.raises(SystemExit, match="framed too tight") as info:+ frames_mod.key_frames(files, tmp_path / "keyed", key="green")+ assert "leftover" not in str(info.value)+ raw = Image.open(files[0])+ keyed = Image.open(tmp_path / "keyed" / files[0].name)+ split = frames_mod.classify_edge_contact(raw, keyed, GREEN)+ assert split["subject"] > 0 and split["residual"] == 0+++def test_mixed_contact_names_both(tmp_path: Path) -> None:+ files = _frames(tmp_path, residual=True, touch_top=True)+ with pytest.raises(SystemExit, match="framed too tight") as info:+ frames_mod.key_frames(files, tmp_path / "keyed", key="green")+ assert "also carry leftover chroma background" in str(info.value)+ raw = Image.open(files[0])+ keyed = Image.open(tmp_path / "keyed" / files[0].name)+ split = frames_mod.classify_edge_contact(raw, keyed, GREEN)+ assert split["subject"] > 0 and split["residual"] > 0