Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.
# SPDX-License-Identifier: Apache-2.0
"""Root motion (sprite_gen/video/root_motion.py, docs/video-pipeline.md "Root motion"): a cut played once carries
where its body is in each cell (the mass centre), what each cell touches and its air, in the strip sidecar; with
`--air-retime` the air is given a length, a gravity and a speed of its own, its cells made between the clip's frames
(held_fill.at_times), the take-off and the landing as filmed and the contacts where they were. Without the handle the
cut is the same pictures as before (tests/picture_digest.py) and its sidecar only grows a key (and the record of its
cells, `cell_record`, as every cut's).
The jump is synthetic (tests/video/jump_inputs.py), its mass centre on a parabola under a known gravity; the
interpolator is the stand-in that draws it again at the time asked for (`Between`)."""
from __future__ import annotations
import hashlib
import json
import math
from pathlib import Path
import pytest
from PIL import Image
from sprite_gen.video import evidence
from sprite_gen.video import held_fill
from sprite_gen.video import loop as loop_mod
from sprite_gen.video import nonloop
from sprite_gen.video import root_motion as rm
from sprite_gen.video import sway
from tests.video import jump_inputs as J
from tests.video.held_fill_inputs import Between
from tests.video.test_held_fill import UNHELD, _no_rife, _pictures
def _run(tmp_path: Path, frames: list[Image.Image], name: str = "j", **kw) -> dict:
keyed = J.save(frames, tmp_path / f"{name}-keyed")
return loop_mod.run_loop(keyed, tmp_path / f"{name}-out", fps=J.FPS, min_len=None, max_len=None, n_out=None,
seam_max=loop_mod.SEAM_RATIO_MAX, name=name, report_path=None, **kw)
def _meta(rep: dict) -> dict:
return json.loads(Path(rep["strip"]["path"]).with_suffix("").with_suffix(".strip.json").read_text())
def _cells(rep: dict) -> list[Image.Image]:
meta = _meta(rep)
strip = Image.open(rep["strip"]["path"]).convert("RGBA")
return [strip.crop((k * meta["w"], 0, (k + 1) * meta["w"], meta["h"])) for k in range(meta["frames"])]
def _read(frames: list[Image.Image], declared=()) -> dict:
floor = max(nonloop.body_box(f)[3] for f in frames)
roots = [rm.root(f) for f in frames]
touch = rm.touching(frames, floor=floor, times=range(len(frames)), declared=declared)
return {"roots": roots, "touch": touch, **rm.read(roots, touch, [k / J.FPS for k in range(len(frames))])}
def test_the_root_of_a_body_flying_under_gravity_is_read_on_its_parabola():
frames = J.jump()
got = _read(frames)
height = max(b[3] - b[1] for b in map(nonloop.body_box, frames))
worst = max(max(abs(got["roots"][k][0] - J.path(k - J.TAKEOFF)[0]), abs(got["roots"][k][1] - J.path(k - J.TAKEOFF)[1]))
for k in range(J.TAKEOFF, J.LANDING + 1))
assert worst <= 0.5 and worst / height < 0.05 # a whole pixel's placing, under half a pixel
(air,) = got["air"]
assert air["cells"] == [J.TAKEOFF + 1, J.LANDING - 1]
assert (air["takeoff"], air["landing"]) == (J.TAKEOFF, J.LANDING)
assert air["seconds"] == pytest.approx(J.AIR, abs=1e-4)
assert air["apex_px"] == pytest.approx(J.APEX, abs=1.0)
assert got["contacts"] == [{"by": "ground", "cells": [0, J.TAKEOFF]}, {"by": "ground", "cells": [J.LANDING, J.FRAMES - 1]}]
def test_a_contact_the_floor_does_not_show_is_named_over_the_clip_s_frames():
frames = J.jump()
declared = rm.parse_contacts(["hand=6:8"], len(frames))
got = _read(frames, declared)
assert {"by": "hand", "cells": [6, 8]} in got["contacts"]
(air,) = got["air"]
assert (air["takeoff"], air["cells"], air["landing"]) == (8, [9, 14], J.LANDING)
with pytest.raises(ValueError, match="ground"):
rm.parse_contacts(["ground=1:2"], 10)
with pytest.raises(ValueError, match="numbered from 0"):
rm.parse_contacts(["hand=4:40"], 10)
def test_the_retime_keys_are_read_and_a_wrong_one_is_refused():
assert rm.parse_retime("gravity=9.81,air=0.6,speed=4") == {"gravity": 9.81, "air": 0.6, "speed": 4.0}
assert rm.parse_retime("air=1.2") == {"air": 1.2}
for bad in ("air", "air=0", "lift=2", "air=1,air=2", "gravity=-1", "speed=x"):
with pytest.raises(ValueError):
rm.parse_retime(bad)
def _retimed(**asked):
frames = J.jump()
between = Between(J.draw, range(J.FRAMES))
floor = max(nonloop.body_box(f)[3] for f in frames)
out, record, plan = rm.retime(frames, between, fps=J.FPS, rate=J.FPS, floor=floor, times=range(J.FRAMES), declared=(),
asked=asked, meters_per_px=J.METERS_PER_PX)
return frames, out, record, plan, between
def test_the_air_twice_as_long_rises_g_t2_over_8_its_cells_made_between_and_the_contacts_kept():
frames, out, record, plan, between = _retimed(gravity=9.81, air=2 * J.AIR)
assert record["applied"]
(air,) = record["air"]
cells = round(2 * J.AIR * J.FPS)
assert len(out) == J.FRAMES + cells - (J.LANDING - J.TAKEOFF) # the air holds twice the cells
assert air["seconds"] == pytest.approx(2 * J.AIR, abs=1e-4) and air["gravity"] == pytest.approx(9.81, abs=1e-3)
T = 2 * J.AIR
assert air["apex_px"] == pytest.approx(J.GRAVITY * T * T / 8, abs=0.01)
# read on the cells delivered: the air spans the new cells, rises g·t²/8 and lands where the parabola does
ox, oy = plan["offset"]
roots = [rm.root(f) for f in out]
measured = rm.read(roots, plan["touch"], [k / J.FPS for k in range(len(out))])
(got,) = measured["air"]
assert got["cells"] == [J.TAKEOFF + 1, J.TAKEOFF + cells - 1] and got["seconds"] == pytest.approx(T, abs=1e-4)
assert got["apex_px"] == pytest.approx(J.GRAVITY * T * T / 8, abs=1.0)
# every air cell on the parabola, within a whole pixel's placing
xa, ya = roots[J.TAKEOFF]
for j in range(1, cells):
t = j / J.FPS
x = xa + 2 * (J.X_LANDING - J.X_TAKEOFF) * j / cells # the clip's own speed, for twice as long
y = ya - J.GRAVITY * t * (T - t) / 2
assert abs(roots[J.TAKEOFF + j][0] - x) <= 0.5 and abs(roots[J.TAKEOFF + j][1] - y) <= 0.5
# the poses: the clip's at the same share of its air, every other one made between two frames
assert record["made"] == cells // 2 and record["shown_again"] == []
assert sorted({round(c[2], 4) for c in between.calls}) == [0.5]
# no picture shown twice in the air
same = held_fill.same_steps(out)
assert not any(same[J.TAKEOFF:J.TAKEOFF + cells])
# the take-off, the cells before it and the landing block are the frames as filmed, moved whole
moved = air["moved_along_px"]
assert moved == 2 * (J.X_LANDING - J.X_TAKEOFF) - (J.X_LANDING - J.X_TAKEOFF)
for k in range(J.TAKEOFF + 1):
assert out[k].crop((ox, oy, ox + J.W, oy + J.H)).tobytes() == frames[k].tobytes()
for k in range(J.LANDING, J.FRAMES):
o = J.TAKEOFF + cells + (k - J.LANDING)
assert out[o].crop((ox + moved, oy, ox + moved + J.W, oy + J.H)).tobytes() == frames[k].tobytes()
# the ground cells stand on the floor, moved, and the air cells touch nothing
floor = J.FLOOR + 1 + oy
for k, f in enumerate(out):
on = nonloop.on_floor(nonloop.body_box(f), floor)
assert on == (not J.TAKEOFF < k < J.TAKEOFF + cells), k
def test_the_air_given_only_a_length_keeps_the_clip_s_own_gravity_and_speed():
frames, out, record, plan, _ = _retimed(air=J.AIR)
(air,) = record["air"]
assert air["gravity_before_px"] == pytest.approx(J.GRAVITY, rel=0.02)
assert air["seconds"] == pytest.approx(J.AIR, abs=1e-4) and air["moved_along_px"] == 0
assert record["made"] == 0 # every cell a frame: nothing to make, no RIFE asked for
def test_a_cut_with_no_air_is_not_retimed_and_says_why():
frames = [J.draw(0)] * 2 + J.jump()[:J.TAKEOFF + 1]
floor = max(nonloop.body_box(f)[3] for f in frames)
out, record, plan = rm.retime(frames, None, fps=J.FPS, rate=J.FPS, floor=floor, times=range(len(frames)),
declared=(), asked={"air": 1.0}, meters_per_px=None)
assert out is frames and plan is None and record["applied"] is False and record["why"] == "no air in the cut"
def test_video_loop_carries_the_root_motion_of_a_jump_and_retimes_its_air(tmp_path):
rep = _run(tmp_path, J.jump(), "plain", state="jump", cycle_mode="once")
meta = _meta(rep)
root = meta["root_motion"]
assert len(root["root"]) == meta["frames"]
start = meta["source_cut"]["start"]
(air,) = root["air"]
assert (air["takeoff"], air["landing"]) == (J.TAKEOFF - start, J.LANDING - start)
assert air["seconds"] == pytest.approx(J.AIR, abs=1e-3)
assert air["apex_px"] == pytest.approx(J.APEX * meta["scale"], abs=1.0)
assert "air_retime" not in meta and rep["root_motion"]["air"] == root["air"]
between = Between(J.draw, range(J.FRAMES))
rep = _run(tmp_path, J.jump(), "long", state="jump", cycle_mode="once", air_retime=f"gravity=9.81,air={2 * J.AIR}",
meters_per_px=J.METERS_PER_PX, interpolate=between)
meta = _meta(rep)
record = meta["air_retime"]
assert record["applied"] and record["made"] == 10 and meta["frames"] == rep["cycle"]["length"] + 10
(air,) = meta["root_motion"]["air"]
T = 2 * J.AIR
assert air["seconds"] == pytest.approx(T, abs=1e-3)
assert air["apex_m"] == pytest.approx(9.81 * T * T / 8, rel=0.03)
assert "times" in meta["source_cut"] and "out_fps" not in meta
# no picture shown twice in the air: the held runs left are the end pose the cut holds
first, last = air["cells"]
assert all(r["start"] > last or r["start"] + r["length"] <= first for r in meta["held_fill"]["held_runs"])
assert record["poses"] == meta["held_fill"]["poses"] and record["cells"] == meta["frames"]
def test_an_air_whose_frames_between_show_a_frame_beside_them_again_names_those_cells(tmp_path, capsys):
# An interpolator that gives back the frame before makes no air cell between two frames: each is refused (`held`),
# tried again only later within half a cell, and is the nearer frame, named in the record and in a warning line.
calls: list[float] = []
def stuck(a, b, t):
calls.append(t)
return a
rep = _run(tmp_path, J.jump(), "stuck", state="jump", cycle_mode="once", air_retime=f"air={2 * J.AIR}",
meters_per_px=J.METERS_PER_PX, interpolate=stuck)
record = _meta(rep)["air_retime"]
assert record["applied"] and record["made"] == 0 and len(record["shown_again"]) == 10
assert all(t["faults"] == ["held"] and t["t"] >= 0.5 for m in rep["air_retime"]["frames"] for t in m["tries"])
assert len(calls) == 10 * 5 # 0.5, then 0.55 to 0.7: half a cell is a quarter of a frame
cells = ", ".join(map(str, record["shown_again"]))
assert f"10 cells of the retimed air ({cells}) show the frame beside them again" in capsys.readouterr().err
@pytest.mark.real_rife
@pytest.mark.timeout(900) # 10 calls of the binary at about 210 x 250, each judged: seconds on a GPU, minutes on a CPU's Vulkan driver
def test_real_rife_makes_the_air_twice_as_long_with_no_frame_shown_again(tmp_path):
"""The same jump through rife-ncnn-vulkan located as a cut locates it: every between made and judged, none shown
again, the air on its parabola."""
rep = _run(tmp_path, J.jump(), "real", state="jump", cycle_mode="once", air_retime=f"air={2 * J.AIR}",
meters_per_px=J.METERS_PER_PX)
meta = _meta(rep)
record = meta["air_retime"]
assert record["applied"] and record["made"] == 10 and record["shown_again"] == []
(air,) = meta["root_motion"]["air"]
T = 2 * J.AIR
assert air["seconds"] == pytest.approx(T, abs=1e-3)
assert air["apex_m"] == pytest.approx(9.81 * T * T / 8, rel=0.03)
first, last = air["cells"]
assert all(r["start"] > last or r["start"] + r["length"] <= first for r in meta["held_fill"]["held_runs"])
def test_no_rife_leaves_the_air_as_filmed_and_says_which_motion(tmp_path, monkeypatch, capsys):
_no_rife(monkeypatch, tmp_path)
rep = _run(tmp_path, J.jump(), "leap", state="jump", cycle_mode="once", air_retime=f"air={2 * J.AIR}")
err = capsys.readouterr().err
meta = _meta(rep)
assert meta["air_retime"]["applied"] is False and meta["air_retime"]["why"] == "RIFE is not installed"
assert "leap" in err and "was not retimed" in err and "rife install" in err
assert meta["frames"] == rep["cycle"]["length"] # as filmed
assert meta["root_motion"]["air"][0]["seconds"] == pytest.approx(J.AIR, abs=1e-3)
def test_a_retimed_cut_records_each_cell_as_the_frame_it_is_and_a_cell_made_or_moved_as_not_filmed(tmp_path):
"""The strip's record of its cells (`cell_record`, video-sway --read-on-cut) of the jump with its air half as long
again, on the clip's own canvas: a cell held_fill.at_times takes as a frame (`source_frames`) is that frame, one made between
two frames reads as the frame before its time; the cells up to the take-off are their frames as filmed, and no air
cell (made, or moved onto the parabola) nor any from the landing on (moved along) is, so --read-on-cut does not
pair it."""
between = Between(J.draw, range(J.FRAMES))
rep = _run(tmp_path, J.jump(), "long", state="jump", cycle_mode="once", air_retime=f"air={1.5 * J.AIR}",
interpolate=between)
meta = _meta(rep)
record = meta["air_retime"]
(air,) = record["air"]
assert record["applied"] and record["canvas_offset"] == [0, 0] and air["moved_along_px"] != 0
assert {im.size for im in (Image.open(p) for p in (Path(rep["out_dir"]) / "cycle").glob("frame-*.png"))} == {(J.W, J.H)}
sha = hashlib.sha256(Path(rep["strip"]["path"]).read_bytes()).hexdigest()
cells, times = sway.cell_record(meta, sha), meta["source_cut"]["times"]
assert len(cells) == meta["frames"] == len(times)
made = [c for c, t in enumerate(times) if abs(t - round(t)) > held_fill.SNAP]
assert len(made) == record["made"] == 10 and all(air["takeoff"] < c < air["landing"] for c in made)
for c, (cell, t) in enumerate(zip(cells, times)):
assert cell["source_frame"] == (math.floor(t) if c in made else round(t)), c
assert cell["as_filmed"] is (c <= air["takeoff"]), c
# Read on a cut of the same clip from frame 6: the retimed cut's first cell of frame 6 is made between 6 and 7.
read = _run(tmp_path, J.jump(), "from6", state="jump", cycle_mode="once", start=6, length=10)
first = [c["source_frame"] for c in cells].index(6)
assert first in made
with pytest.raises(SystemExit, match=rf"cell {first} is not that frame as filmed: video-loop did not cut it as filmed "
rf"\(cell_record\.cells\[{first}\]\.as_filmed\)"):
sway.recut_pair(meta, _meta(read), hashlib.sha256(Path(read["strip"]["path"]).read_bytes()).hexdigest(),
strip_sha256=sha)
def test_a_retimed_cut_whose_frames_between_show_a_frame_again_records_those_cells_as_the_nearer_frame(tmp_path):
"""The same air half as long again with an interpolator that gives back the frame before: no cell between two
frames is made (each refused, `held`), each is the nearer frame (`air_retime.shown_again`), and the strip's record of
its cells names that frame (`source_frames`, held_fill.at_times'), not the frame before its time."""
rep = _run(tmp_path, J.jump(), "stuck", state="jump", cycle_mode="once", air_retime=f"air={1.5 * J.AIR}",
interpolate=lambda a, b, t: a)
meta = _meta(rep)
record = meta["air_retime"]
between = [c for c, t in enumerate(meta["source_cut"]["times"]) if abs(t - round(t)) > held_fill.SNAP]
assert record["applied"] and record["made"] == 0 and record["shown_again"] == between and len(between) == 10
sha = hashlib.sha256(Path(rep["strip"]["path"]).read_bytes()).hexdigest()
cells, times = sway.cell_record(meta, sha), meta["source_cut"]["times"]
assert len(cells) == meta["frames"] == len(times)
assert [cell["source_frame"] for cell in cells] == [round(t) for t in times]
assert any(round(times[c]) != math.floor(times[c]) for c in between)
def test_gravity_or_speed_without_metres_per_pixel_is_refused(tmp_path):
with pytest.raises(SystemExit, match="--meters-per-px"):
_run(tmp_path, J.jump(), "g", state="jump", cycle_mode="once", air_retime="gravity=9.81")
with pytest.raises(SystemExit, match="--contact"):
_run(tmp_path, J.jump(), "c", state="jump", cycle_mode="once", contacts=["hand=3:99"])
# The base engine's sidecars of the six inputs the held-cell tests pin (559d498, v3.8.0), every key: without the handle
# the cut keeps every one of them, value for value, a cut played once grows `root_motion`, and every cut the record of
# its cells (`cell_record`, bound to its strip by the PNG file's sha, so read against this run's strip). The keyed clip's
# identity (`source_cut.source.sha256`) is the sha of the input PNG files, other bytes on Linux than on macOS (its
# zlib): it is read against this run's own files, and pinned without.
SIDECARS = {
"hop-one-shot": "b546c1762af5aee76faa7ff83752a50b2accbc2de4b5daf3fd610a8d3eef914e",
"swing-once-return": "991343cce794a6f90c30b606f461ed6eeaf8d804e0ecadac3f51ab631b0fb813",
"swing-pinned": "88ecedeca1ff148799ac226b946c5c7d60d4a73dee347e2c6b4c33ca98eeeec3",
"topple-once": "78f0c731c7f6c72ea27fa15890652899d45ad8efd1e55ba8a66907b2d447f84e",
"walk-auto": "edf29b6fde312a55e0e4813ca1571511d855b603a78c12b75a9d5121f9e9448d",
"walk-fixed": "b9d991735bd05268e04ffae9c2660a21271075f5c0a0c281b2d0e7fbbe4c84de",
}
@pytest.mark.parametrize("case", sorted(UNHELD))
def test_without_the_handle_the_cut_is_the_same_pictures_as_before_and_its_sidecar_only_grows(case, tmp_path, monkeypatch):
_no_rife(monkeypatch, tmp_path)
frames, kw, pinned = UNHELD[case]
keyed = J.save(frames(), tmp_path / f"{case}-keyed")
rep = loop_mod.run_loop(keyed, tmp_path / f"{case}-out", fps=24.0, min_len=None, max_len=None, n_out=None,
seam_max=loop_mod.SEAM_RATIO_MAX, name=case, report_path=None, **kw)
assert _pictures(Path(rep["out_dir"]), case) == pinned
meta = _meta(rep)
grown = meta.pop("root_motion", None)
cells = meta.pop(loop_mod.CELL_RECORD)
assert cells["strip_sha256"] == hashlib.sha256(Path(rep["strip"]["path"]).read_bytes()).hexdigest()
assert len(cells["cells"]) == meta["frames"]
source = meta["source_cut"]["source"]
assert source.pop("sha256") == evidence.source_identity(sorted(keyed.glob("*.png")), 24.0)["sha256"]
# 4.0.0 adds the strip's exact rate and the cells it shows again on purpose; the rest is as the base wrote it.
assert meta.pop("fps") == pytest.approx(meta["frames"] / meta["cycle_seconds"], rel=1e-4)
holds = meta.pop("holds")
rows = ((meta.get("held_fill") or {}).get("before") or {}).get("held_runs", [])
kept = [row.pop("keep") for row in rows if "keep" in row]
assert [h["kind"] for h in holds] == kept and set(kept) <= set(held_fill.KEEPS)
assert hashlib.sha256(json.dumps(meta, sort_keys=True).encode()).hexdigest() == SIDECARS[case]
assert (grown is not None) == (meta["kind"] in rm.KINDS)
if grown is not None:
assert len(grown["root"]) == meta["frames"] and grown["point"] == "mass-centre"
def test_video_loop_and_video_set_take_the_switches():
from tests.conftest import help_options
loop = help_options("-m", "sprite_gen.cli", "video-loop")
assert {"--air-retime", "--meters-per-px", "--contact"} <= loop
assert {"--air-retime", "--meters-per-px"} <= help_options("-m", "sprite_gen.cli", "video-set")
def test_video_set_hands_the_switches_to_each_cut_and_says_each_air(monkeypatch, tmp_path):
from sprite_gen.video import batch
seen: list[dict] = []
monkeypatch.setattr(batch.frames_mod, "run_frames", lambda clip, out, **kw: {
"fps": 24, "frames": 10, "alpha_zero_pct_min": 50, "alpha_zero_pct_max": 60, "keyed_dir": str(out)})
def run_loop(*a, **kw):
seen.append(kw)
return {"cycle": {"length": 10, "period_global": None, "ratio": 0.0, "kind": "once"}, "resampled_seam_ratio": None,
"n_out": 10, "gif": {"file": "loop.gif"}, "webp": {"file": "loop.webp"}, "strip": {"path": "strip.png"},
"held_fill": {"cells": 10, "poses": 10, "held_runs": [], "made": [],
"before": {"cells": 10, "poses": 10, "held_runs": []}},
"root_motion": {"air": [{"cells": [3, 6], "takeoff": 2, "landing": 7, "seconds": 0.2083, "apex_px": 31.5,
"apex_m": 0.42}]},
"air_retime": {"applied": False, "why": "RIFE is not installed", "before": []}}
def video(image, prompt, out, report, **kw):
out.write_bytes(b"test-video")
report.write_text(json.dumps({"prompt": prompt}))
return 0
monkeypatch.setattr(batch.loop_mod, "run_loop", run_loop)
monkeypatch.setattr(batch, "_staggered_start", lambda gap: None)
still = tmp_path / "side.png"
Image.new("RGBA", (64, 64), (200, 80, 30, 255)).save(still)
result = batch.run_set(bases={"side": still}, states=["jump"], root=tmp_path / "set", character=None, duration=3,
resolution="720p", key="green", concurrency=1, force=False, gap=0, video_runner=video,
align_cycles="off", air_retime="gravity=9.81,air=0.6,speed=4", meters_per_px=0.005)
assert [(kw["air_retime"], kw["meters_per_px"]) for kw in seen] == [("gravity=9.81,air=0.6,speed=4", 0.005)]
assert result["items"][0]["loop"]["air"] == ["air 0.21 s, apex 0.42 m (cells 3-6)"]
assert any("air not retimed" in w for w in result["warnings"])
with pytest.raises(SystemExit, match="--meters-per-px"):
batch.run_set(bases={"side": still}, states=["jump"], root=tmp_path / "set2", character=None, duration=3,
resolution="720p", key="green", concurrency=1, force=False, gap=0, video_runner=video,
align_cycles="off", air_retime="speed=4")