Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.
feat(video): a magenta plate for the subject to clear, painted by video-canvas --prop-plate and erased by video-frames --prop-plate from the one box the canvas report holds
Soohong Kimbd94190772f4
diff --git a/sprite_gen/video/canvas.py b/sprite_gen/video/canvas.pyindex 0dd2094..324a7c0 100644--- a/sprite_gen/video/canvas.py+++ b/sprite_gen/video/canvas.py@@ -29,6 +29,10 @@ background is repainted to the exact declared key here — the pixels the`cutout` chroma matte erases become the pure key, the subject is untouched —and the padding uses that same pure key. A white/ivory base is padded with itsown corner colour as before (no chroma key to normalize to).++`--prop-plate` paints a flat magenta plate on the green key in front of the subject, a low wall for it to clear+(`prop_plate`, sized by the subject's height), and writes its box into the report, where `video-frames --prop-plate`+reads it to erase the plate from the clip's keyed frames."""from __future__ import annotations@@ -48,6 +52,7 @@ from sprite_gen.frames.extract import is_border_key_candidatefrom sprite_gen.gen.handedness import VIEWSfrom sprite_gen.spec.runio import atomic_write_textfrom sprite_gen.util.console import utf8_streams+from sprite_gen.video import prop_plate as plate_modSHAPE_SQUARE = "square"SHAPE_TALL = "tall"@@ -420,14 +425,19 @@ def run_canvas(trail: float | None = None,fit: str = "state",view: str | None = None,+ prop_plate: bool = False,) -> dict[str, Any]:still = still.expanduser().resolve()if not still.is_file():raise SystemExit(f"video-canvas: still not found: {still}")if fit == "tight" and (shape is not None or headroom is not None or lead is not None or trail is not None):raise SystemExit("video-canvas: --fit tight picks its own shape and adds no room; drop --shape/--headroom/--lead/--trail")+ if prop_plate and report_path is None:+ raise SystemExit("video-canvas: --prop-plate needs --report: the plate's box is written there for video-frames --prop-plate")if state == "effect":from . import effect+ if prop_plate:+ raise SystemExit("video-canvas: an effect keeps its source canvas; drop --prop-plate")if fit != "state" or any(v is not None for v in (shape, headroom, lead, trail)):raise SystemExit("video-canvas: an effect keeps its source canvas and center; drop framing overrides")with Image.open(still) as image:@@ -435,6 +445,10 @@ def run_canvas(else:profile = profile_for(state, shape)canvas, report = pad_canvas(Image.open(still), profile, facing=facing, headroom=headroom, lead=lead, trail=trail, key=key, fit=fit, view=view)+ if prop_plate:+ if report["key"] != "green":+ raise SystemExit(f"video-canvas: --prop-plate paints a magenta plate on a green key; the still's key is {report['key'] or 'not a chroma key'}")+ report["prop_plate"] = plate_mod.paint(canvas, subject_box(canvas, tuple(report["key_rgb"]), 0), facing=facing) # type: ignore[arg-type]out = out.expanduser().resolve()out.parent.mkdir(parents=True, exist_ok=True)tmp = out.with_name(out.name + ".part")@@ -460,6 +474,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:parser.add_argument("--key", choices=KEYS, default="auto", help="chroma key of the still (auto reads the corners; green/magenta are normalized to the exact key; white pads with the corner colour)")parser.add_argument("--fit", choices=FITS, default="state", help="state: the state's room for the motion (default); tight: drop the empty rows around the subject and add no room, so it fills the frame and a motion that leaves it is clipped")parser.add_argument("--view", choices=VIEWS, help="the still's view; a walk or run from the side has the still's room around its silhouette read, and its square widened by the room it lacks (`room` in the report). Left out, the room is not read")+ parser.add_argument("--prop-plate", action="store_true", help="paint a flat magenta plate, a low wall for the subject to clear, on the ground in front of it (the side it faces), sized by its height; green key only, needs --report, where its box is written for `video-frames --prop-plate` to erase")parser.add_argument("--report", type=Path, help="write the canvas report JSON here")@@ -469,6 +484,7 @@ def run(**kwargs: object) -> int:state=kwargs.get("state"), shape=kwargs.get("shape"), facing=str(kwargs.get("facing") or "right"), # type: ignore[arg-type]headroom=kwargs.get("headroom"), lead=kwargs.get("lead"), trail=kwargs.get("trail"), report_path=kwargs.get("report"), # type: ignore[arg-type]key=str(kwargs.get("key") or "auto"), fit=str(kwargs.get("fit") or "state"), view=kwargs.get("view"), # type: ignore[arg-type]+ prop_plate=bool(kwargs.get("prop_plate")),)print(json.dumps(payload, ensure_ascii=False, indent=2))return 0diff --git a/sprite_gen/video/frames.py b/sprite_gen/video/frames.pyindex 27fe3d3..2fe475c 100644--- a/sprite_gen/video/frames.py+++ b/sprite_gen/video/frames.py@@ -51,6 +51,11 @@ than `SPECK_APART` of the body's height from it. Its pixels are every opaque onecheck reads), corners joining them. Near the body, small detached pieces are the subject's ownand stay: the outline's loose pixels one transparent pixel off, a shadow drawn under a shoe, ashoe's part the matte cut loose. How many each frame lost is in its report row (`specks`).++A **plate** painted into the canvas (`video-canvas --prop-plate`, a low wall for the subject to clear) is taken out of+every keyed frame of a run that passed: `--prop-plate` names that canvas's report, whose box is the plate's+(`prop_plate.read`), and the erase runs on the frames before they are handed over (`prop_plate.erase`; the report's+`prop_plate`). A refused run's frames keep it."""from __future__ import annotations@@ -74,6 +79,7 @@ from sprite_gen.frames.extract import _SPILL_FULL_MIN_TINT, DEFAULT_UNMIX_REACHfrom sprite_gen.frames.extract import BORDER_MIN_FRACTION, FRAME_SIDES, own_borderfrom sprite_gen.spec.runio import atomic_write_text, new_stage, refuse_overlapfrom sprite_gen.util.console import utf8_streams+from sprite_gen.video import prop_plate as plate_modEDGE_ROWS = 4 # rows/cols inspected at each edge# Spill (key colour the video model painted INTO the subject — a reflection on@@ -442,8 +448,8 @@ def _withdraw(keyed_dir: Path) -> Path:def _refuse_overlap(clip: Path, reference: Path | None, out_dir: Path, target: Path, report_given: bool,- receipt: Path | None) -> None:- """A run never writes over, moves or deletes what it reads: `--clip` and `--reference`, each with every link or+ receipt: Path | None, plate: Path | None = None) -> None:+ """A run never writes over, moves or deletes what it reads: `--clip`, `--reference` and `--prop-plate`, each with every link orfolder its path looks up, against what it replaces in `--out-dir` (`raw/`'s frames, `keyed/`, `keyed.refused/`),its report and its source manifest (`runio.refuse_overlap`). Refused before anything is made."""out = out_dir.expanduser().resolve()@@ -451,7 +457,7 @@ def _refuse_overlap(clip: Path, reference: Path | None, out_dir: Path, target: Poutputs["--report" if report_given else f"{target.name} in --out-dir"] = targetif receipt is not None:outputs["--source-manifest"] = Path(receipt).expanduser().resolve()- refuse_overlap({"--clip": clip, "--reference": reference}, outputs, verb="video-frames")+ refuse_overlap({"--clip": clip, "--reference": reference, "--prop-plate": plate}, outputs, verb="video-frames")def key_frames(@@ -464,10 +470,12 @@ def key_frames(allow_subject: bool = False,decontam: str = DECONTAM_DEFAULT,alpha_mode: str = "chroma",+ plate: dict[str, Any] | None = None,) -> dict[str, Any]:"""Key `raw_files` into `keyed_dir` and read their edges; the report. The frames are keyed in a working folderbeside it and handed over as `keyed_dir` when the run passes; a run that does not pass leaves them as- `refused_dir(keyed_dir)` and no `keyed_dir`. A refusal raises `KeyingRefused`, carrying the report."""+ `refused_dir(keyed_dir)` and no `keyed_dir`. A refusal raises `KeyingRefused`, carrying the report. A `plate`+ (`prop_plate.read`) is erased from the frames of a run that passed before they are handed over."""from . import effectif alpha_mode not in effect.ALPHA_MODES:raise SystemExit(f"video-frames: unknown --alpha-mode {alpha_mode!r}")@@ -482,6 +490,8 @@ def key_frames(report = (effect.key_frames(raw_files, work) if alpha_mode == "brightness" else_key_chroma(raw_files, work, key=key, check_edges=check_edges, spill=spill, allow_subject=allow_subject,decontam=decontam))+ if plate is not None:+ report["prop_plate"] = plate_mod.erase([work / f.name for f in raw_files], plate)except BaseException:shutil.rmtree(refused_dir(keyed_dir), ignore_errors=True)work.rename(refused_dir(keyed_dir))@@ -688,7 +698,7 @@ def _edge_contact_message(contacts: list[dict[str, Any]]) -> str:return message-def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool, report_path: Path | None, spill: str = "small", reference: Path | None = None, allow_subject_edge_contact: bool = False, decontam: str = DECONTAM_DEFAULT, source_manifest: Path | None = None, alpha_mode: str = "chroma") -> dict[str, Any]:+def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool, report_path: Path | None, spill: str = "small", reference: Path | None = None, allow_subject_edge_contact: bool = False, decontam: str = DECONTAM_DEFAULT, source_manifest: Path | None = None, alpha_mode: str = "chroma", prop_plate: Path | None = None) -> dict[str, Any]:from . import effectif alpha_mode not in effect.ALPHA_MODES:raise SystemExit(f"video-frames: unknown --alpha-mode {alpha_mode!r}")@@ -713,7 +723,8 @@ def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool,clip = clip.expanduser().resolve()if not clip.is_file():raise SystemExit(f"video-frames: clip not found: {clip}")- _refuse_overlap(given, reference, out_dir, target, report_path is not None, source_manifest)+ _refuse_overlap(given, reference, out_dir, target, report_path is not None, source_manifest, prop_plate)+ plate = plate_mod.read(prop_plate) if prop_plate is not None else Noneout_dir = out_dir.expanduser().resolve()raw_dir = out_dir / "raw"keyed_dir = out_dir / "keyed"@@ -725,7 +736,7 @@ def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool,files = extract(clip, raw_dir, meta["stream_index"])head = {"kind": "sprite-gen-video-frames-report", "clip": str(clip), "out_dir": str(out_dir), "raw_dir": str(raw_dir)}try:- report = key_frames(files, keyed_dir, key=key, check_edges=not allow_edge_contact, spill=decision["mode"], allow_subject=allow_subject_edge_contact, decontam=decontam, alpha_mode=alpha_mode)+ report = key_frames(files, keyed_dir, key=key, check_edges=not allow_edge_contact, spill=decision["mode"], allow_subject=allow_subject_edge_contact, decontam=decontam, alpha_mode=alpha_mode, plate=plate)except KeyingRefused as refused:# The run's record, written whatever it found: the frames it keyed are in `refused_dir`, and nothing is handed over.payload = {**head, "status": "refused", "error": str(refused), "keyed_dir": None, "refused_dir": str(refused_dir(keyed_dir)),@@ -759,6 +770,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:parser.add_argument("--source-manifest", type=Path, help="write exact clip/canvas/keyed-file digests and source timestamps (requires --reference)")parser.add_argument("--spill", choices=SPILL_MODES, default="small", help="small: correct only small key-tinted clusters (default); full: every key tint in the subject is spill; auto: decide from --reference")parser.add_argument("--reference", type=Path, help="the still the clip was made from (required by --spill auto)")+ parser.add_argument("--prop-plate", type=Path, help="the `video-canvas --report` of a canvas painted with --prop-plate: the plate's box it holds is erased from every keyed frame (the subject crossing in front of it stays); the report's `prop_plate` says what went")parser.add_argument("--decontam", choices=DECONTAM_MODES, default=DECONTAM_DEFAULT, help="auto (default): re-explain key-tinted edges with the subject's own colours (one palette per clip, video fit) where it applies, the reason recorded where not (a white key, a subject with no interior; brightness alpha is off); palette: the same, failing where it cannot run; off: the matte as is")@@ -768,8 +780,8 @@ def run(**kwargs: object) -> int:spill=str(kwargs.get("spill") or "small"), reference=kwargs.get("reference"), # type: ignore[arg-type]allow_subject_edge_contact=bool(kwargs.get("allow_subject_edge_contact")),decontam=str(kwargs.get("decontam") or DECONTAM_DEFAULT), source_manifest=kwargs.get("source_manifest"),- alpha_mode=str(kwargs.get("alpha_mode") or "chroma")) # type: ignore[arg-type]- summary = {k: payload[k] for k in ("clip", "keyed_dir", "width", "height", "fps", "frames", "codec", "pix_fmt", "audio_streams", "alpha_zero_pct_min", "alpha_zero_pct_max", "edge_contacts", "edge_policy", "spill", "report")}+ alpha_mode=str(kwargs.get("alpha_mode") or "chroma"), prop_plate=kwargs.get("prop_plate")) # type: ignore[arg-type]+ summary = {k: payload[k] for k in ("clip", "keyed_dir", "width", "height", "fps", "frames", "codec", "pix_fmt", "audio_streams", "alpha_zero_pct_min", "alpha_zero_pct_max", "edge_contacts", "edge_policy", "spill", "report") + (("prop_plate",) if "prop_plate" in payload else ())}print(json.dumps(summary, ensure_ascii=False, indent=2))line = decontam_warning(payload["decontam"])if line:diff --git a/sprite_gen/video/prop_plate.py b/sprite_gen/video/prop_plate.pynew file mode 100644index 0000000..349e73a--- /dev/null+++ b/sprite_gen/video/prop_plate.py@@ -0,0 +1,127 @@+# SPDX-License-Identifier: Apache-2.0+"""A flat magenta plate painted into a clip's still for the subject to clear, and taken out of its keyed frames+again (`video-canvas --prop-plate`, `video-frames --prop-plate`).++A vault needs something to vault. Asked in words for a wall in the background colour, a video model draws a black+slab, and keying leaves the slab in the frames. So the wall is drawn into the still instead, as a plate the key+does not take and the erase does: flat magenta on the green key, standing on the ground in front of the subject.++One box is the plate. `place` sets it on the canvas, and `video-canvas` paints it and writes it into its report+(`prop_plate.box`, canvas px, right and bottom one past). `video-frames` reads it from that report (`read`) and+erases inside it on every keyed frame (`erase`), the box carried onto the clip's frames by the ratio of their size+to the canvas's where the video model returned another size.++The plate, of the subject's height on the canvas (`canvas.subject_box`): a wall as high as the thighs (HEIGHT), a+fifth of the height thick (WIDTH), its near side GAP ahead of the subject's front (the side it faces), its foot on+the subject's foot row.++The erase: inside the box and a rim of RIM px, every pixel led by red and blue over green (by more than+MAGENTA_LEAD, red and blue within MAGENTA_BALANCE of each other: the plate, its rim blended into the key too) goes,+as does every pixel within STILL_TOLERANCE on each channel of what that pixel shows over the whole clip (the median:+the plate's keyed rim, a ground line drawn under it), since the subject crosses the box for part of the clip only.+Alpha is one of those channels, so a pixel keyed out over most of the clip is not read as black: the subject's dark+outline crossing the rim stays, as does the rest of the subject crossing in front of the plate.++A plate the video model draws a pixel off in a frame leaves its edge there, the codec's blend of magenta and green+(grey, so neither rule takes it), a line a pixel wide down each side.+"""++from __future__ import annotations++import json+from pathlib import Path+from typing import Any++from PIL import Image++from sprite_gen._deps import np++RGB = (255, 0, 255)+HEIGHT = 0.49 # of the subject's height+WIDTH = 0.20+GAP = 0.84+RIM = 3 # px added round the box on each side, on the clip's frames+MAGENTA_LEAD = 35+MAGENTA_BALANCE = 90+STILL_TOLERANCE = 40+CANVAS_REPORT = "sprite-gen-video-canvas-report"+++def place(subject: tuple[int, int, int, int], *, facing: str, size: tuple[int, int]) -> tuple[int, int, int, int]:+ """The plate's box (left, top, right, bottom; right and bottom one past) on a canvas of `size`, in front of the+ `subject` box facing `facing`. Refused where it would leave the canvas."""+ left, top, right, bottom = subject+ height = bottom - top+ width, gap = round(WIDTH * height), round(GAP * height)+ if facing == "right":+ x0 = right + gap+ x1 = x0 + width+ else:+ x1 = left - gap+ x0 = x1 - width+ if x0 < 0 or x1 > size[0]:+ raise SystemExit(+ f"video-canvas: --prop-plate: the plate ({x0}..{x1}) falls past the canvas's {facing} edge ({size[0]} px wide); "+ "frame the subject with room in front (--shape wide, --lead)"+ )+ return x0, bottom - round(HEIGHT * height), x1, bottom+++def paint(canvas: Image.Image, subject: tuple[int, int, int, int], *, facing: str) -> dict[str, Any]:+ """Paint the plate in front of `subject` on `canvas` (in place); the report's `prop_plate`."""+ box = place(subject, facing=facing, size=canvas.size)+ canvas.paste(RGB, box)+ return {"box": list(box), "rgb": list(RGB), "subject": list(subject),+ "of_subject_height": {"height": HEIGHT, "width": WIDTH, "gap": GAP}}+++def read(report: Path) -> dict[str, Any]:+ """The plate a `video-canvas --report` holds: its `box` and the `canvas` size it is on, and the report's path."""+ path = Path(report).expanduser().resolve()+ try:+ data = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, ValueError) as e:+ raise SystemExit(f"video-frames: --prop-plate: cannot read the canvas report {path}: {e}") from e+ if not isinstance(data, dict) or data.get("kind") != CANVAS_REPORT:+ raise SystemExit(f"video-frames: --prop-plate: {path} is not a video-canvas report ({CANVAS_REPORT})")+ plate = data.get("prop_plate")+ if not isinstance(plate, dict):+ raise SystemExit(f"video-frames: --prop-plate: {path} holds no plate; the canvas was made without --prop-plate")+ return {"report": str(path), "box": [int(v) for v in plate["box"]], "canvas": [int(v) for v in data["canvas"]]}+++def frame_box(box: list[int], canvas: list[int], frame: tuple[int, int]) -> tuple[int, int, int, int]:+ """The canvas `box` on a frame of size `frame`: scaled by frame / canvas each way, rounded outward, widened by+ RIM px, kept on the frame."""+ (x0, y0, x1, y1), (cw, ch), (fw, fh) = box, canvas, frame+ x0, y0 = x0 * fw // cw, y0 * fh // ch+ x1, y1 = -(-x1 * fw // cw), -(-y1 * fh // ch)+ return max(0, x0 - RIM), max(0, y0 - RIM), min(fw, x1 + RIM), min(fh, y1 + RIM)+++def erase(files: list[Path], plate: dict[str, Any]) -> dict[str, Any]:+ """Erase the plate from every keyed frame in `files` (in place); the frames report's `prop_plate`."""+ with Image.open(files[0]) as first:+ x0, y0, x1, y1 = frame_box(plate["box"], plate["canvas"], first.size)+ crops = []+ for f in files:+ with Image.open(f) as im:+ crops.append(np.asarray(im.convert("RGBA"))[y0:y1, x0:x1])+ still = np.median(np.stack(crops), axis=0).astype(np.int32)+ changed = erased = 0+ for f in files:+ with Image.open(f) as im:+ a = np.array(im.convert("RGBA"))+ reg = a[y0:y1, x0:x1]+ rgba = reg.astype(np.int32)+ r, g, b = rgba[..., 0], rgba[..., 1], rgba[..., 2]+ magenta = (np.minimum(r, b) - g > MAGENTA_LEAD) & (np.abs(r - b) < MAGENTA_BALANCE)+ gone = magenta | (np.abs(rgba - still).max(axis=2) < STILL_TOLERANCE)+ if not reg[gone].any():+ continue+ erased += int((reg[gone][:, 3] > 0).sum())+ reg[gone] = 0+ Image.fromarray(a, "RGBA").save(f)+ changed += 1+ return {"canvas_report": plate["report"], "erase_box": [x0, y0, x1, y1], "rim": RIM, "frames": len(files),+ "frames_changed": changed, "erased_px": erased}