Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat(video): a magenta plate for the subject to clear, painted by video-canvas --prop-plate and erased by video-frames --prop-plate from the one box the canvas report holds

Soohong Kimbd94190772f4
diff --git a/sprite_gen/video/canvas.py b/sprite_gen/video/canvas.py
index 0dd2094..324a7c0 100644
--- a/sprite_gen/video/canvas.py
+++ b/sprite_gen/video/canvas.py
@@ -29,6 +29,10 @@ background is repainted to the exact declared key here — the pixels the
`cutout` chroma matte erases become the pure key, the subject is untouched —
and the padding uses that same pure key. A white/ivory base is padded with its
own corner colour as before (no chroma key to normalize to).
+
+`--prop-plate` paints a flat magenta plate on the green key in front of the subject, a low wall for it to clear
+(`prop_plate`, sized by the subject's height), and writes its box into the report, where `video-frames --prop-plate`
+reads it to erase the plate from the clip's keyed frames.
"""
from __future__ import annotations
@@ -48,6 +52,7 @@ from sprite_gen.frames.extract import is_border_key_candidate
from sprite_gen.gen.handedness import VIEWS
from sprite_gen.spec.runio import atomic_write_text
from sprite_gen.util.console import utf8_streams
+from sprite_gen.video import prop_plate as plate_mod
SHAPE_SQUARE = "square"
SHAPE_TALL = "tall"
@@ -420,14 +425,19 @@ def run_canvas(
trail: float | None = None,
fit: str = "state",
view: str | None = None,
+ prop_plate: bool = False,
) -> dict[str, Any]:
still = still.expanduser().resolve()
if not still.is_file():
raise SystemExit(f"video-canvas: still not found: {still}")
if fit == "tight" and (shape is not None or headroom is not None or lead is not None or trail is not None):
raise SystemExit("video-canvas: --fit tight picks its own shape and adds no room; drop --shape/--headroom/--lead/--trail")
+ if prop_plate and report_path is None:
+ raise SystemExit("video-canvas: --prop-plate needs --report: the plate's box is written there for video-frames --prop-plate")
if state == "effect":
from . import effect
+ if prop_plate:
+ raise SystemExit("video-canvas: an effect keeps its source canvas; drop --prop-plate")
if fit != "state" or any(v is not None for v in (shape, headroom, lead, trail)):
raise SystemExit("video-canvas: an effect keeps its source canvas and center; drop framing overrides")
with Image.open(still) as image:
@@ -435,6 +445,10 @@ def run_canvas(
else:
profile = profile_for(state, shape)
canvas, report = pad_canvas(Image.open(still), profile, facing=facing, headroom=headroom, lead=lead, trail=trail, key=key, fit=fit, view=view)
+ if prop_plate:
+ if report["key"] != "green":
+ raise SystemExit(f"video-canvas: --prop-plate paints a magenta plate on a green key; the still's key is {report['key'] or 'not a chroma key'}")
+ report["prop_plate"] = plate_mod.paint(canvas, subject_box(canvas, tuple(report["key_rgb"]), 0), facing=facing) # type: ignore[arg-type]
out = out.expanduser().resolve()
out.parent.mkdir(parents=True, exist_ok=True)
tmp = out.with_name(out.name + ".part")
@@ -460,6 +474,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--key", choices=KEYS, default="auto", help="chroma key of the still (auto reads the corners; green/magenta are normalized to the exact key; white pads with the corner colour)")
parser.add_argument("--fit", choices=FITS, default="state", help="state: the state's room for the motion (default); tight: drop the empty rows around the subject and add no room, so it fills the frame and a motion that leaves it is clipped")
parser.add_argument("--view", choices=VIEWS, help="the still's view; a walk or run from the side has the still's room around its silhouette read, and its square widened by the room it lacks (`room` in the report). Left out, the room is not read")
+ parser.add_argument("--prop-plate", action="store_true", help="paint a flat magenta plate, a low wall for the subject to clear, on the ground in front of it (the side it faces), sized by its height; green key only, needs --report, where its box is written for `video-frames --prop-plate` to erase")
parser.add_argument("--report", type=Path, help="write the canvas report JSON here")
@@ -469,6 +484,7 @@ def run(**kwargs: object) -> int:
state=kwargs.get("state"), shape=kwargs.get("shape"), facing=str(kwargs.get("facing") or "right"), # type: ignore[arg-type]
headroom=kwargs.get("headroom"), lead=kwargs.get("lead"), trail=kwargs.get("trail"), report_path=kwargs.get("report"), # type: ignore[arg-type]
key=str(kwargs.get("key") or "auto"), fit=str(kwargs.get("fit") or "state"), view=kwargs.get("view"), # type: ignore[arg-type]
+ prop_plate=bool(kwargs.get("prop_plate")),
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0
diff --git a/sprite_gen/video/frames.py b/sprite_gen/video/frames.py
index 27fe3d3..2fe475c 100644
--- a/sprite_gen/video/frames.py
+++ b/sprite_gen/video/frames.py
@@ -51,6 +51,11 @@ than `SPECK_APART` of the body's height from it. Its pixels are every opaque one
check reads), corners joining them. Near the body, small detached pieces are the subject's own
and stay: the outline's loose pixels one transparent pixel off, a shadow drawn under a shoe, a
shoe's part the matte cut loose. How many each frame lost is in its report row (`specks`).
+
+A **plate** painted into the canvas (`video-canvas --prop-plate`, a low wall for the subject to clear) is taken out of
+every keyed frame of a run that passed: `--prop-plate` names that canvas's report, whose box is the plate's
+(`prop_plate.read`), and the erase runs on the frames before they are handed over (`prop_plate.erase`; the report's
+`prop_plate`). A refused run's frames keep it.
"""
from __future__ import annotations
@@ -74,6 +79,7 @@ from sprite_gen.frames.extract import _SPILL_FULL_MIN_TINT, DEFAULT_UNMIX_REACH
from sprite_gen.frames.extract import BORDER_MIN_FRACTION, FRAME_SIDES, own_border
from sprite_gen.spec.runio import atomic_write_text, new_stage, refuse_overlap
from sprite_gen.util.console import utf8_streams
+from sprite_gen.video import prop_plate as plate_mod
EDGE_ROWS = 4 # rows/cols inspected at each edge
# Spill (key colour the video model painted INTO the subject — a reflection on
@@ -442,8 +448,8 @@ def _withdraw(keyed_dir: Path) -> Path:
def _refuse_overlap(clip: Path, reference: Path | None, out_dir: Path, target: Path, report_given: bool,
- receipt: Path | None) -> None:
- """A run never writes over, moves or deletes what it reads: `--clip` and `--reference`, each with every link or
+ receipt: Path | None, plate: Path | None = None) -> None:
+ """A run never writes over, moves or deletes what it reads: `--clip`, `--reference` and `--prop-plate`, each with every link or
folder its path looks up, against what it replaces in `--out-dir` (`raw/`'s frames, `keyed/`, `keyed.refused/`),
its report and its source manifest (`runio.refuse_overlap`). Refused before anything is made."""
out = out_dir.expanduser().resolve()
@@ -451,7 +457,7 @@ def _refuse_overlap(clip: Path, reference: Path | None, out_dir: Path, target: P
outputs["--report" if report_given else f"{target.name} in --out-dir"] = target
if receipt is not None:
outputs["--source-manifest"] = Path(receipt).expanduser().resolve()
- refuse_overlap({"--clip": clip, "--reference": reference}, outputs, verb="video-frames")
+ refuse_overlap({"--clip": clip, "--reference": reference, "--prop-plate": plate}, outputs, verb="video-frames")
def key_frames(
@@ -464,10 +470,12 @@ def key_frames(
allow_subject: bool = False,
decontam: str = DECONTAM_DEFAULT,
alpha_mode: str = "chroma",
+ plate: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Key `raw_files` into `keyed_dir` and read their edges; the report. The frames are keyed in a working folder
beside it and handed over as `keyed_dir` when the run passes; a run that does not pass leaves them as
- `refused_dir(keyed_dir)` and no `keyed_dir`. A refusal raises `KeyingRefused`, carrying the report."""
+ `refused_dir(keyed_dir)` and no `keyed_dir`. A refusal raises `KeyingRefused`, carrying the report. A `plate`
+ (`prop_plate.read`) is erased from the frames of a run that passed before they are handed over."""
from . import effect
if alpha_mode not in effect.ALPHA_MODES:
raise SystemExit(f"video-frames: unknown --alpha-mode {alpha_mode!r}")
@@ -482,6 +490,8 @@ def key_frames(
report = (effect.key_frames(raw_files, work) if alpha_mode == "brightness" else
_key_chroma(raw_files, work, key=key, check_edges=check_edges, spill=spill, allow_subject=allow_subject,
decontam=decontam))
+ if plate is not None:
+ report["prop_plate"] = plate_mod.erase([work / f.name for f in raw_files], plate)
except BaseException:
shutil.rmtree(refused_dir(keyed_dir), ignore_errors=True)
work.rename(refused_dir(keyed_dir))
@@ -688,7 +698,7 @@ def _edge_contact_message(contacts: list[dict[str, Any]]) -> str:
return message
-def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool, report_path: Path | None, spill: str = "small", reference: Path | None = None, allow_subject_edge_contact: bool = False, decontam: str = DECONTAM_DEFAULT, source_manifest: Path | None = None, alpha_mode: str = "chroma") -> dict[str, Any]:
+def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool, report_path: Path | None, spill: str = "small", reference: Path | None = None, allow_subject_edge_contact: bool = False, decontam: str = DECONTAM_DEFAULT, source_manifest: Path | None = None, alpha_mode: str = "chroma", prop_plate: Path | None = None) -> dict[str, Any]:
from . import effect
if alpha_mode not in effect.ALPHA_MODES:
raise SystemExit(f"video-frames: unknown --alpha-mode {alpha_mode!r}")
@@ -713,7 +723,8 @@ def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool,
clip = clip.expanduser().resolve()
if not clip.is_file():
raise SystemExit(f"video-frames: clip not found: {clip}")
- _refuse_overlap(given, reference, out_dir, target, report_path is not None, source_manifest)
+ _refuse_overlap(given, reference, out_dir, target, report_path is not None, source_manifest, prop_plate)
+ plate = plate_mod.read(prop_plate) if prop_plate is not None else None
out_dir = out_dir.expanduser().resolve()
raw_dir = out_dir / "raw"
keyed_dir = out_dir / "keyed"
@@ -725,7 +736,7 @@ def run_frames(clip: Path, out_dir: Path, *, key: str, allow_edge_contact: bool,
files = extract(clip, raw_dir, meta["stream_index"])
head = {"kind": "sprite-gen-video-frames-report", "clip": str(clip), "out_dir": str(out_dir), "raw_dir": str(raw_dir)}
try:
- report = key_frames(files, keyed_dir, key=key, check_edges=not allow_edge_contact, spill=decision["mode"], allow_subject=allow_subject_edge_contact, decontam=decontam, alpha_mode=alpha_mode)
+ report = key_frames(files, keyed_dir, key=key, check_edges=not allow_edge_contact, spill=decision["mode"], allow_subject=allow_subject_edge_contact, decontam=decontam, alpha_mode=alpha_mode, plate=plate)
except KeyingRefused as refused:
# The run's record, written whatever it found: the frames it keyed are in `refused_dir`, and nothing is handed over.
payload = {**head, "status": "refused", "error": str(refused), "keyed_dir": None, "refused_dir": str(refused_dir(keyed_dir)),
@@ -759,6 +770,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--source-manifest", type=Path, help="write exact clip/canvas/keyed-file digests and source timestamps (requires --reference)")
parser.add_argument("--spill", choices=SPILL_MODES, default="small", help="small: correct only small key-tinted clusters (default); full: every key tint in the subject is spill; auto: decide from --reference")
parser.add_argument("--reference", type=Path, help="the still the clip was made from (required by --spill auto)")
+ parser.add_argument("--prop-plate", type=Path, help="the `video-canvas --report` of a canvas painted with --prop-plate: the plate's box it holds is erased from every keyed frame (the subject crossing in front of it stays); the report's `prop_plate` says what went")
parser.add_argument("--decontam", choices=DECONTAM_MODES, default=DECONTAM_DEFAULT, help="auto (default): re-explain key-tinted edges with the subject's own colours (one palette per clip, video fit) where it applies, the reason recorded where not (a white key, a subject with no interior; brightness alpha is off); palette: the same, failing where it cannot run; off: the matte as is")
@@ -768,8 +780,8 @@ def run(**kwargs: object) -> int:
spill=str(kwargs.get("spill") or "small"), reference=kwargs.get("reference"), # type: ignore[arg-type]
allow_subject_edge_contact=bool(kwargs.get("allow_subject_edge_contact")),
decontam=str(kwargs.get("decontam") or DECONTAM_DEFAULT), source_manifest=kwargs.get("source_manifest"),
- alpha_mode=str(kwargs.get("alpha_mode") or "chroma")) # type: ignore[arg-type]
- summary = {k: payload[k] for k in ("clip", "keyed_dir", "width", "height", "fps", "frames", "codec", "pix_fmt", "audio_streams", "alpha_zero_pct_min", "alpha_zero_pct_max", "edge_contacts", "edge_policy", "spill", "report")}
+ alpha_mode=str(kwargs.get("alpha_mode") or "chroma"), prop_plate=kwargs.get("prop_plate")) # type: ignore[arg-type]
+ summary = {k: payload[k] for k in ("clip", "keyed_dir", "width", "height", "fps", "frames", "codec", "pix_fmt", "audio_streams", "alpha_zero_pct_min", "alpha_zero_pct_max", "edge_contacts", "edge_policy", "spill", "report") + (("prop_plate",) if "prop_plate" in payload else ())}
print(json.dumps(summary, ensure_ascii=False, indent=2))
line = decontam_warning(payload["decontam"])
if line:
diff --git a/sprite_gen/video/prop_plate.py b/sprite_gen/video/prop_plate.py
new file mode 100644
index 0000000..349e73a
--- /dev/null
+++ b/sprite_gen/video/prop_plate.py
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: Apache-2.0
+"""A flat magenta plate painted into a clip's still for the subject to clear, and taken out of its keyed frames
+again (`video-canvas --prop-plate`, `video-frames --prop-plate`).
+
+A vault needs something to vault. Asked in words for a wall in the background colour, a video model draws a black
+slab, and keying leaves the slab in the frames. So the wall is drawn into the still instead, as a plate the key
+does not take and the erase does: flat magenta on the green key, standing on the ground in front of the subject.
+
+One box is the plate. `place` sets it on the canvas, and `video-canvas` paints it and writes it into its report
+(`prop_plate.box`, canvas px, right and bottom one past). `video-frames` reads it from that report (`read`) and
+erases inside it on every keyed frame (`erase`), the box carried onto the clip's frames by the ratio of their size
+to the canvas's where the video model returned another size.
+
+The plate, of the subject's height on the canvas (`canvas.subject_box`): a wall as high as the thighs (HEIGHT), a
+fifth of the height thick (WIDTH), its near side GAP ahead of the subject's front (the side it faces), its foot on
+the subject's foot row.
+
+The erase: inside the box and a rim of RIM px, every pixel led by red and blue over green (by more than
+MAGENTA_LEAD, red and blue within MAGENTA_BALANCE of each other: the plate, its rim blended into the key too) goes,
+as does every pixel within STILL_TOLERANCE on each channel of what that pixel shows over the whole clip (the median:
+the plate's keyed rim, a ground line drawn under it), since the subject crosses the box for part of the clip only.
+Alpha is one of those channels, so a pixel keyed out over most of the clip is not read as black: the subject's dark
+outline crossing the rim stays, as does the rest of the subject crossing in front of the plate.
+
+A plate the video model draws a pixel off in a frame leaves its edge there, the codec's blend of magenta and green
+(grey, so neither rule takes it), a line a pixel wide down each side.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+
+from sprite_gen._deps import np
+
+RGB = (255, 0, 255)
+HEIGHT = 0.49 # of the subject's height
+WIDTH = 0.20
+GAP = 0.84
+RIM = 3 # px added round the box on each side, on the clip's frames
+MAGENTA_LEAD = 35
+MAGENTA_BALANCE = 90
+STILL_TOLERANCE = 40
+CANVAS_REPORT = "sprite-gen-video-canvas-report"
+
+
+def place(subject: tuple[int, int, int, int], *, facing: str, size: tuple[int, int]) -> tuple[int, int, int, int]:
+ """The plate's box (left, top, right, bottom; right and bottom one past) on a canvas of `size`, in front of the
+ `subject` box facing `facing`. Refused where it would leave the canvas."""
+ left, top, right, bottom = subject
+ height = bottom - top
+ width, gap = round(WIDTH * height), round(GAP * height)
+ if facing == "right":
+ x0 = right + gap
+ x1 = x0 + width
+ else:
+ x1 = left - gap
+ x0 = x1 - width
+ if x0 < 0 or x1 > size[0]:
+ raise SystemExit(
+ f"video-canvas: --prop-plate: the plate ({x0}..{x1}) falls past the canvas's {facing} edge ({size[0]} px wide); "
+ "frame the subject with room in front (--shape wide, --lead)"
+ )
+ return x0, bottom - round(HEIGHT * height), x1, bottom
+
+
+def paint(canvas: Image.Image, subject: tuple[int, int, int, int], *, facing: str) -> dict[str, Any]:
+ """Paint the plate in front of `subject` on `canvas` (in place); the report's `prop_plate`."""
+ box = place(subject, facing=facing, size=canvas.size)
+ canvas.paste(RGB, box)
+ return {"box": list(box), "rgb": list(RGB), "subject": list(subject),
+ "of_subject_height": {"height": HEIGHT, "width": WIDTH, "gap": GAP}}
+
+
+def read(report: Path) -> dict[str, Any]:
+ """The plate a `video-canvas --report` holds: its `box` and the `canvas` size it is on, and the report's path."""
+ path = Path(report).expanduser().resolve()
+ try:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError) as e:
+ raise SystemExit(f"video-frames: --prop-plate: cannot read the canvas report {path}: {e}") from e
+ if not isinstance(data, dict) or data.get("kind") != CANVAS_REPORT:
+ raise SystemExit(f"video-frames: --prop-plate: {path} is not a video-canvas report ({CANVAS_REPORT})")
+ plate = data.get("prop_plate")
+ if not isinstance(plate, dict):
+ raise SystemExit(f"video-frames: --prop-plate: {path} holds no plate; the canvas was made without --prop-plate")
+ return {"report": str(path), "box": [int(v) for v in plate["box"]], "canvas": [int(v) for v in data["canvas"]]}
+
+
+def frame_box(box: list[int], canvas: list[int], frame: tuple[int, int]) -> tuple[int, int, int, int]:
+ """The canvas `box` on a frame of size `frame`: scaled by frame / canvas each way, rounded outward, widened by
+ RIM px, kept on the frame."""
+ (x0, y0, x1, y1), (cw, ch), (fw, fh) = box, canvas, frame
+ x0, y0 = x0 * fw // cw, y0 * fh // ch
+ x1, y1 = -(-x1 * fw // cw), -(-y1 * fh // ch)
+ return max(0, x0 - RIM), max(0, y0 - RIM), min(fw, x1 + RIM), min(fh, y1 + RIM)
+
+
+def erase(files: list[Path], plate: dict[str, Any]) -> dict[str, Any]:
+ """Erase the plate from every keyed frame in `files` (in place); the frames report's `prop_plate`."""
+ with Image.open(files[0]) as first:
+ x0, y0, x1, y1 = frame_box(plate["box"], plate["canvas"], first.size)
+ crops = []
+ for f in files:
+ with Image.open(f) as im:
+ crops.append(np.asarray(im.convert("RGBA"))[y0:y1, x0:x1])
+ still = np.median(np.stack(crops), axis=0).astype(np.int32)
+ changed = erased = 0
+ for f in files:
+ with Image.open(f) as im:
+ a = np.array(im.convert("RGBA"))
+ reg = a[y0:y1, x0:x1]
+ rgba = reg.astype(np.int32)
+ r, g, b = rgba[..., 0], rgba[..., 1], rgba[..., 2]
+ magenta = (np.minimum(r, b) - g > MAGENTA_LEAD) & (np.abs(r - b) < MAGENTA_BALANCE)
+ gone = magenta | (np.abs(rgba - still).max(axis=2) < STILL_TOLERANCE)
+ if not reg[gone].any():
+ continue
+ erased += int((reg[gone][:, 3] > 0).sum())
+ reg[gone] = 0
+ Image.fromarray(a, "RGBA").save(f)
+ changed += 1
+ return {"canvas_report": plate["report"], "erase_box": [x0, y0, x1, y1], "rim": RIM, "frames": len(files),
+ "frames_changed": changed, "erased_px": erased}