Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat(video-cycle-align): --between-cache keeps the frames made between source frames and a later pass takes them

Soohong Kime6510a1335ee
diff --git a/sprite_gen/video/align.py b/sprite_gen/video/align.py
index c44996f..c265182 100644
--- a/sprite_gen/video/align.py
+++ b/sprite_gen/video/align.py
@@ -73,6 +73,12 @@ says which own foot lands on the first candidate, and that loop alone starts as
A set that needs a made frame where no RIFE is installed raises `rife.RifeNotInstalled` with
nothing rewritten: this command fails on it, `video-set` skips the alignment with a warning.
+
+A caller that aligns a set more than once — again with the feet told, or with a retake in place of
+one loop — names one directory for every pass (`--between-cache`): each frame made between two source
+frames is kept there (`rife.Cache`), and a later pass takes it instead of making it again where both
+source frames, the fraction and the RIFE that made it are byte for byte the same. The report counts
+the frames taken and made (`between_cache`). Nothing is kept without it.
"""
from __future__ import annotations
@@ -772,7 +778,7 @@ def _ghost_line(name: str, g: dict[str, Any]) -> str:
def align_set(loop_dirs: list[Path], *, length: int | None = None, interpolate: rife_mod.Interpolate | None = None,
report_path: Path | None = None, between: str = DEFAULT_BETWEEN, views: list[str | None] | None = None,
start_foot: str = START_FOOT, multi_cycle: str = "fail", cycles: dict[str, int] | None = None,
- state: str | None = None, feet: dict[str, str] | None = None) -> dict[str, Any]:
+ state: str | None = None, feet: dict[str, str] | None = None, between_cache: Path | None = None) -> dict[str, Any]:
"""Resample every loop of a set to one length (default: the median), turned to a foot strike —
where `views` (one per loop: `front`, `back`, `side@right`, `front_diagonal@left`, …) lets it,
the strike of the same own foot (`start_foot`) in every loop.
@@ -788,9 +794,18 @@ def align_set(loop_dirs: list[Path], *, length: int | None = None, interpolate:
whoever looked saw it (`--foot`): that loop starts where `start_foot` lands, recorded
`start_foot_source` "given" (`foot_strike`). A loop whose foot nobody named is listed under the
report's `unnamed_feet`, with its two strike frames in the rebuilt `cycle/` (`candidates`) and the
- `--foot` arguments that settle it."""
+ `--foot` arguments that settle it.
+
+ `between_cache` is a directory every frame made between two source frames is kept in and taken from
+ (`rife.Cache`), for a caller that aligns the same set again: a frame is taken only for the same two
+ source frames, fraction and interpolator (its `identity()`, which an injected one must have). The
+ report's `between_cache` counts the frames taken (`hits`), made (`misses`) and kept but unreadable;
+ it is None without a directory, and nothing is kept."""
if len(loop_dirs) < 1:
raise SystemExit("video-cycle-align: at least one --loop-dir")
+ if between_cache is not None and interpolate is not None and not callable(getattr(interpolate, "identity", None)):
+ raise SystemExit("video-cycle-align: --between-cache keeps each frame under what made it, and the interpolator given "
+ "names no identity (`identity()`, as rife.Rife names its binary, model and machine)")
if views is not None and len(views) != len(loop_dirs):
raise SystemExit(f"video-cycle-align: {len(views)} --view for {len(loop_dirs)} --loop-dir; give one per loop, in the same order")
if start_foot not in handed_mod.SIDES:
@@ -801,6 +816,10 @@ def align_set(loop_dirs: list[Path], *, length: int | None = None, interpolate:
if v is not None:
name, _, facing = v.partition("@")
handed_mod.validate_view(name, facing or None)
+ try:
+ cache = rife_mod.Cache(between_cache) if between_cache is not None else None
+ except OSError as exc:
+ raise SystemExit(f"video-cycle-align: --between-cache {between_cache}: {exc}") from exc
loops = []
for d in loop_dirs:
d = d.expanduser().resolve()
@@ -857,6 +876,7 @@ def align_set(loop_dirs: list[Path], *, length: int | None = None, interpolate:
*(f"--view {v}" for v in (views or []) if v is not None), *([f"--length {length}"] if length is not None else []),
*([f"--between {between}"] if between != DEFAULT_BETWEEN else []), *([f"--start-foot {start_foot}"] if start_foot != START_FOOT else []),
*([f"--state {state}"] if state else []), *([f"--report {report_path}"] if report_path is not None else []),
+ *([f"--between-cache {cache.directory}"] if cache is not None else []),
*(f"--cycles {loops[i][0]}={k}" for i, k in sorted(given.items())),
*(f"--foot {loops[i][0]}={f}" for i, f in sorted(told.items())),
*(f"--cycles {e['dir']}=<{'|'.join(s.rpartition('=')[2] for s in e['settle'])}>" for e in stopped)]
@@ -887,7 +907,7 @@ def align_set(loop_dirs: list[Path], *, length: int | None = None, interpolate:
found = rife_mod.Rife()
located.append(found.describe())
interpolate = found
- return interpolate(a, b, t)
+ return cache.frame(interpolate, a, b, t) if cache is not None else interpolate(a, b, t)
# Every loop is resampled before any is rewritten: a loop that cannot be made leaves the set as it was. The cut
# each is aligned from is read first, while its strip is still the cut's.
@@ -967,6 +987,7 @@ def align_set(loop_dirs: list[Path], *, length: int | None = None, interpolate:
"fps": round(fps, 4), "cycle_seconds": round(target / fps, 4),
"interpolator": ({"kind": "rife-ncnn-vulkan", **located[0]} if located else
{"kind": "injected"} if any(r.get("smear") for r in rows) else None),
+ "between_cache": cache.record() if cache is not None else None,
"made_by_rife": sum(r["made_by_rife"] for r in rows),
"replaced": sum(1 for r in rows for m in r.get("smear", []) if m["method"] == "nearest"), "loops": rows}
if report_path is not None:
@@ -1005,6 +1026,12 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
"measured in the report (`smear`); under auto and nearest a filmed ghost frame gives way to a clean source "
"frame beside it (`ghost_at`); a made frame kept that cross-fades two drawings inside its outline is kept "
"and named to look at (`look`)")
+ parser.add_argument("--between-cache", type=Path, metavar="DIR",
+ help="a directory every frame RIFE makes between two source frames is kept in, for aligning the same set "
+ "again: a later alignment given the same DIR (the feet told, a retake in place of one loop) takes a "
+ "kept frame instead of making it, only where both source frames, the fraction, and the RIFE binary, "
+ "model, engine code and machine are byte for byte the same; the report counts `between_cache` hits "
+ "and misses. Give each alignment its own DIR and remove it after; nothing is kept without it")
def parse_cycles(values: list[str]) -> dict[str, int]:
@@ -1039,7 +1066,8 @@ def run(**kwargs: object) -> int:
between=str(kwargs.get("between") or DEFAULT_BETWEEN), views=kwargs.get("view"), # type: ignore[arg-type]
start_foot=str(kwargs.get("start_foot") or START_FOOT),
multi_cycle=str(kwargs.get("multi_cycle") or "fail"), cycles=parse_cycles(list(kwargs.get("cycles") or [])), # type: ignore[arg-type]
- state=kwargs.get("state"), feet=parse_feet(list(kwargs.get("foot") or []))) # type: ignore[arg-type]
+ state=kwargs.get("state"), feet=parse_feet(list(kwargs.get("foot") or [])), # type: ignore[arg-type]
+ between_cache=kwargs.get("between_cache")) # type: ignore[arg-type]
except rife_mod.RifeNotInstalled as exc:
# Asked for by name, so no RIFE is a failure, never a quiet skip (video-set skips with a warning).
raise SystemExit(f"video-cycle-align: {exc}; frames between source frames are made by RIFE — "
@@ -1047,7 +1075,7 @@ def run(**kwargs: object) -> int:
for line in report["warnings"]:
print(f"video-cycle-align: warning: {line}", file=sys.stderr)
print(json.dumps({k: report[k] for k in ("length", "lengths", "cycles_given", "feet_given", "between", "made_by_rife", "replaced", "retake",
- "unnamed_feet", "cycle_seconds")}
+ "unnamed_feet", "cycle_seconds", "between_cache")}
| {"loops": [{k: r[k] for k in ("name", "from", "to", "made_by_rife", "turned_by", "turned_on", "start_foot", "start_foot_source",
"seam_ratio")} for r in report["loops"]]},
ensure_ascii=False, indent=2))
diff --git a/sprite_gen/video/rife.py b/sprite_gen/video/rife.py
index d7f53e5..312a565 100644
--- a/sprite_gen/video/rife.py
+++ b/sprite_gen/video/rife.py
@@ -25,11 +25,20 @@ RIFE that is found and then fails is `RifeUnavailable` and always an error.
The interpolator is a plain callable `(a, b, t) -> frame`, so a caller can be handed a stand-in
(`Interpolate`), and a test needs no binary.
+
+A frame made between two frames can be kept and taken again (`Cache`): an alignment run twice on
+the same source frames — again with the feet told — makes the same frames a second time, two RIFE
+calls each (section 1 of docs/loop-repair.md has their cost; section 4 the cache). It is kept under
+everything it is made from, the interpolator's `identity()` included, so it is taken only where
+making it again would give the same bytes.
"""
from __future__ import annotations
+import hashlib
+import json
import os
+import platform
import shutil
import subprocess
import sys
@@ -37,7 +46,8 @@ import tempfile
from pathlib import Path
from typing import Callable
-from PIL import Image, ImageFilter
+import PIL
+from PIL import Image, ImageFilter, PngImagePlugin
from sprite_gen._deps import np
@@ -69,6 +79,10 @@ CROSSFADE_RANGE = 7
CROSSFADE_WINDOW = 0.08
CROSSFADE_KEEP = 0.6
CALL_TIMEOUT_SECONDS = 120
+# `Cache`: the first thing a kept frame's key is made from — another way of keying or keeping frames is another
+# name, so no frame kept the old way is taken — and the PNG text a kept frame names its own pixels' sha256 under.
+CACHE_FORMAT = "sprite-gen frame between, 1"
+PIXELS_TEXT = "sprite-gen-pixels-sha256"
Interpolate = Callable[[Image.Image, Image.Image, float], Image.Image]
@@ -313,6 +327,14 @@ def crossfade(made: Image.Image, a: Image.Image, b: Image.Image, t: float) -> fl
return round(lost / max(1, int(solid.sum())), 5)
+def _file_sha256(path: Path) -> str:
+ h = hashlib.sha256()
+ with path.open("rb") as f:
+ for block in iter(lambda: f.read(1 << 20), b""):
+ h.update(block)
+ return h.hexdigest()
+
+
class Rife:
"""A located RIFE as an `Interpolate` callable. Counts the frames it made (`made`)."""
@@ -321,6 +343,7 @@ class Rife:
binary, model = locate()
self.binary, self.model = binary, model
self.made = 0
+ self._identity: str | None = None
def __call__(self, a: Image.Image, b: Image.Image, t: float) -> Image.Image:
with tempfile.TemporaryDirectory(prefix="sprite-gen-rife-") as td:
@@ -330,3 +353,92 @@ class Rife:
def describe(self) -> dict[str, str]:
return {"binary": str(self.binary), "model": self.model.name}
+
+ def identity(self) -> str:
+ """What a frame this RIFE makes depends on besides its two frames and t, as one string (`Cache` keys
+ on it): the binary's and the model's files by sha256, this module's code by sha256 (it turns a frame
+ into RIFE's two runs and back), NumPy's and Pillow's versions, and the machine — the Apple GPU and
+ llvmpipe are only measured to agree to the third decimal of an error (docs/loop-repair.md section 1),
+ so a frame made on one is not taken on another."""
+ if self._identity is None:
+ model = sorted(p for p in self.model.iterdir() if p.is_file())
+ self._identity = json.dumps({
+ "binary": _file_sha256(self.binary), "model": {p.name: _file_sha256(p) for p in model},
+ "code": _file_sha256(Path(__file__)), "numpy": np.__version__, "pillow": PIL.__version__,
+ "machine": [platform.node(), platform.machine(), sys.platform]}, sort_keys=True)
+ return self._identity
+
+
+def _kept(path: Path, size: tuple[int, int]) -> Image.Image | None:
+ """The frame kept at `path`, or None where it is not one: not a PNG, not RGBA at `size`, cut short, or
+ its pixels not the ones its PIXELS_TEXT names."""
+ try:
+ with Image.open(path) as im:
+ if im.format != "PNG" or im.mode != "RGBA" or im.size != size:
+ return None
+ im.load()
+ data, named = im.tobytes(), im.info.get(PIXELS_TEXT)
+ except (OSError, SyntaxError, ValueError, EOFError):
+ return None
+ if named != hashlib.sha256(data).hexdigest():
+ return None
+ return Image.frombytes("RGBA", size, data)
+
+
+class Cache:
+ """Frames made between two frames, kept in `directory` and taken from there when the same frame is
+ asked for again (`video-cycle-align --between-cache`, docs/loop-repair.md section 4).
+
+ A frame is kept under the sha256 of everything it is made from: CACHE_FORMAT, the interpolator's
+ `identity()`, each of the two frames' mode, size and bytes, and `t` exactly. Another frame, fraction,
+ RIFE or machine is another key, so a frame kept for other inputs is never taken. A kept frame is a
+ PNG naming its own pixels' sha256, written whole or not at all; a file that does not read back as
+ that frame (cut short, written over) is made again and kept over it, counted `unreadable`. A frame
+ taken is a new image of the kept pixels, as the interpolator's own frame is.
+
+ Nothing is kept unless a caller names the directory, and nothing here removes it: it belongs to the
+ caller, who gives each alignment its own. It is created readable by this user only. Counts the frames
+ taken (`hits`) and made (`misses`)."""
+
+ def __init__(self, directory: Path):
+ self.directory = Path(directory).expanduser().resolve()
+ self.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
+ self.hits = self.misses = self.unreadable = 0
+
+ def key(self, interpolate: Interpolate, a: Image.Image, b: Image.Image, t: float) -> str:
+ h = hashlib.sha256()
+ for part in (CACHE_FORMAT, interpolate.identity(), a.mode, *map(str, a.size), b.mode, *map(str, b.size), float(t).hex()): # type: ignore[attr-defined]
+ h.update(part.encode() + b"\0")
+ h.update(a.tobytes())
+ h.update(b.tobytes())
+ return h.hexdigest()
+
+ def frame(self, interpolate: Interpolate, a: Image.Image, b: Image.Image, t: float) -> Image.Image:
+ """The frame at `t` between `a` and `b`: the one kept for exactly these, else `interpolate`'s, kept."""
+ path = self.directory / f"{self.key(interpolate, a, b, t)}.png"
+ if path.exists():
+ kept = _kept(path, a.size)
+ if kept is not None:
+ self.hits += 1
+ return kept
+ self.unreadable += 1
+ made = interpolate(a, b, t)
+ if made.mode != "RGBA" or made.size != a.size:
+ raise ValueError(f"rife: a frame between is kept as RGBA at its frames' size {a.size}, not {made.mode} at {made.size}")
+ self.misses += 1
+ # saved from a copy of its pixels, so the frame handed on carries nothing of this save
+ pixels = made.tobytes()
+ info = PngImagePlugin.PngInfo()
+ info.add_text(PIXELS_TEXT, hashlib.sha256(pixels).hexdigest())
+ fd, staged = tempfile.mkstemp(prefix=".", suffix=".tmp", dir=self.directory)
+ try:
+ with os.fdopen(fd, "wb") as out:
+ Image.frombytes("RGBA", made.size, pixels).save(out, format="PNG", pnginfo=info)
+ os.replace(staged, path)
+ except BaseException:
+ Path(staged).unlink(missing_ok=True)
+ raise
+ return made
+
+ def record(self) -> dict[str, object]:
+ return {"dir": str(self.directory), "hits": self.hits, "misses": self.misses, "unreadable": self.unreadable}