Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat(video-loop): root motion on every cut played once, and its air drawn again on a given gravity, length and speed

Soohong Kimcb1971a86ec1
diff --git a/sprite_gen/video/batch.py b/sprite_gen/video/batch.py
index 0292388..815aa5d 100644
--- a/sprite_gen/video/batch.py
+++ b/sprite_gen/video/batch.py
@@ -38,6 +38,7 @@ from sprite_gen.video import canvas as canvas_mod
from sprite_gen.video import frames as frames_mod
from sprite_gen.video import loop as loop_mod
from sprite_gen.video import held_fill as held_fill_mod
+from sprite_gen.video import root_motion as root_motion_mod
from sprite_gen.video import align as align_mod
from sprite_gen.video import body_plan as body_mod
from sprite_gen.video.body_plan import Body
@@ -723,6 +724,8 @@ def run_item(
motion: str | None = None,
fill_held: str = "auto",
out_fps: float | None = None,
+ air_retime: str | None = None,
+ meters_per_px: float | None = None,
) -> dict[str, Any]:
"""One (direction, state) item, canvas to loop. `play` is the request's play mode for it (`play.MODES`; None, the
state's preset) and `motion` the caller's own motion text in place of the state's sentence."""
@@ -863,7 +866,7 @@ def run_item(
lp = loop_mod.run_loop(Path(fr["keyed_dir"]), item_dir / "loop", fps=float(fr["fps"]), state=state, min_len=None, max_len=None, n_out=None, seam_max=loop_mod.SEAM_RATIO_MAX, name=item, report_path=item_dir / "loop.report.json", anchor=anchor, body_height=body_height,
cycle_mode=cycle_for(state, direction, play, pinned=pins),
facing=facing if direction in handed_mod.LATERAL_VIEWS else "right",
- fill_held=fill_held, out_fps=out_fps)
+ fill_held=fill_held, out_fps=out_fps, air_retime=air_retime, meters_per_px=meters_per_px)
result["loop"] = {"kind": lp["cycle"].get("kind", "periodic"), "cycle": lp["cycle"]["length"], "period": lp["cycle"]["period_global"], "cycle_ratio": round(lp["cycle"]["ratio"], 3), "seam_ratio": lp["resampled_seam_ratio"], "n_out": lp["n_out"], "drift_px": lp["strip"].get("drift_px", 0), "gif": lp["gif"]["file"], "webp": lp["webp"]["file"], "strip": lp["strip"]["path"]}
result["loop"]["review_recommended"] = lp["cycle"].get("review_recommended", False)
result["loop"]["half_period_guard"] = lp["cycle"].get("half_period_guard")
@@ -886,6 +889,12 @@ def run_item(
result["loop"]["held_fill"] = {"before": held_fill_mod.said(held["before"]), "held_runs": held["held_runs"],
"made": held["made"], **({"unmade": held["unmade"]} if "unmade" in held else {}),
**({"out_fps": held["out_fps"]} if "out_fps" in held else {})}
+ if "root_motion" in lp:
+ # A cut played once: each air's seconds and apex (docs/video-pipeline.md, "Root motion").
+ result["loop"]["air"] = root_motion_mod.said(lp["root_motion"])
+ if "air_retime" in lp:
+ result["loop"]["air_retime"] = {k: lp["air_retime"][k] for k in ("applied", "why", "made", "shown_again", "poses", "cells")
+ if k in lp["air_retime"]}
if "rife" in (lp.get("jump_repair") or {}):
# cut as filmed for want of RIFE: the set report lists it under `warnings` (a wrap the seam bridge read carries
# its `seam`, not the flow's `off_max_before`)
@@ -1103,6 +1112,8 @@ def run_set(
motion: dict[str, str] | None = None,
fill_held: str = "auto",
out_fps: float | None = None,
+ air_retime: str | None = None,
+ meters_per_px: float | None = None,
) -> dict[str, Any]:
"""`bases` maps a view to its still; `facing` is right, left or `right,left` (`facings_of`), and a view filmed
both ways is keyed `view@right` and `view@left` (`resolve_bases`). `handed` (`handedness.parse`) puts each
@@ -1144,6 +1155,13 @@ def run_set(
raise SystemExit("video-set: non-loop states need --fit state and --anchor none")
aligned_items = [i for i, _, s, _, _ in items if align_cycles == "auto" and loop_mod.profile_for(s).gait
and sum(1 for _, _, s2, _, _ in items if s2 == s) >= 2]
+ if air_retime is not None:
+ try:
+ asked_air = root_motion_mod.parse_retime(air_retime)
+ except ValueError as exc:
+ raise SystemExit(f"video-set: {exc}") from None
+ if meters_per_px is None and {"gravity", "speed"} & set(asked_air):
+ raise SystemExit("video-set: --air-retime gravity= and speed= are metres: give --meters-per-px")
if out_fps is not None and aligned_items:
raise SystemExit(f"video-set: --out-fps gives each cut at a higher rate, and the cycle alignment of {', '.join(aligned_items)} "
"makes its own frames between to one length — pass --align-cycles off with it")
@@ -1168,7 +1186,7 @@ def run_set(
return prepare_side
with ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex:
- futures = {ex.submit(run_item, item=i, direction=d, state=s, base=b, root=root, character=character, duration=duration, resolution=resolution, key=key, force=force, gap=gap, video_runner=video_runner, shape=shape, anchor=anchor, spill=spill, facing=f, facing_fix=facing_fix, prepare_side=side_preparer(f), body_height=body_height, fit=fit, decontam=decontam, walk_start=walk_start, redraw_runner=redraw_runner, still_provider=still_provider, handed=handed, body_plan=body_plan, **({"end_frame": end_frames[i]} if end_frames and i in end_frames else {}), **({"play": plays[i]} if i in plays else {}), **({"motion": motions[i]} if i in motions else {}), fill_held=fill_held, out_fps=out_fps): i for i, d, s, f, b in items}
+ futures = {ex.submit(run_item, item=i, direction=d, state=s, base=b, root=root, character=character, duration=duration, resolution=resolution, key=key, force=force, gap=gap, video_runner=video_runner, shape=shape, anchor=anchor, spill=spill, facing=f, facing_fix=facing_fix, prepare_side=side_preparer(f), body_height=body_height, fit=fit, decontam=decontam, walk_start=walk_start, redraw_runner=redraw_runner, still_provider=still_provider, handed=handed, body_plan=body_plan, **({"end_frame": end_frames[i]} if end_frames and i in end_frames else {}), **({"play": plays[i]} if i in plays else {}), **({"motion": motions[i]} if i in motions else {}), fill_held=fill_held, out_fps=out_fps, air_retime=air_retime, meters_per_px=meters_per_px): i for i, d, s, f, b in items}
for fut in as_completed(futures):
r = fut.result()
results.append(r)
@@ -1183,6 +1201,8 @@ def run_set(
for r in results if r.get("ok") and "unmade" in r["loop"]["held_fill"]]
+ [f"{r['item']}: --out-fps {r['loop']['held_fill']['out_fps']['asked']:g} not met — {r['loop']['held_fill']['out_fps']['why']}"
for r in results if r.get("ok") and not r["loop"]["held_fill"].get("out_fps", {"applied": True})["applied"]]
+ + [f"{r['item']}: air not retimed — RIFE is not installed" for r in results
+ if r.get("ok") and r["loop"].get("air_retime", {}).get("why") == "RIFE is not installed"]
+ [f"cycle-align:{st}: not aligned — {a['why']}" for st, a in aligned.items() if a.get("reason") == "rife-not-installed"])
if warnings:
warnings.append(f"install RIFE with `{rife_mod.install_command()}` (docs/loop-repair.md)")
@@ -1306,6 +1326,12 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--out-fps", type=float, default=None,
help="give each cut at this rate, above its clip's, made of frames between (video-loop --out-fps); with "
"a walk or run filmed in two or more directions, pass --align-cycles off")
+ parser.add_argument("--air-retime", default=None, metavar="gravity=G,air=T,speed=V",
+ help="each cut played once with an air in it: that air given a length, a gravity and a speed of its own "
+ "(video-loop --air-retime); each item's `loop.air` says each air's seconds and apex")
+ parser.add_argument("--meters-per-px", type=float, default=None,
+ help="the metres a pixel of the clips as filmed holds (video-loop --meters-per-px): the root motion in "
+ "metres, and --air-retime's gravity and speed")
parser.add_argument("--force", action="store_true", help="regenerate clips that already exist")
@@ -1329,6 +1355,7 @@ def run(**kwargs: object) -> int:
play=_parse_keyed(list(kwargs.get("play") or []), "--play") or None, # type: ignore[arg-type]
motion=_parse_keyed(list(kwargs.get("motion") or []), "--motion") or None, # type: ignore[arg-type]
fill_held=str(kwargs.get("fill_held") or "auto"), out_fps=kwargs.get("out_fps"), # type: ignore[arg-type]
+ air_retime=kwargs.get("air_retime"), meters_per_px=kwargs.get("meters_per_px"), # type: ignore[arg-type]
)
return 0 if not payload["failed"] else 1
diff --git a/sprite_gen/video/loop.py b/sprite_gen/video/loop.py
index a6b0cfd..4d2de95 100644
--- a/sprite_gen/video/loop.py
+++ b/sprite_gen/video/loop.py
@@ -54,6 +54,7 @@ from sprite_gen.video import cadence as cadence_mod
from sprite_gen.video import crossings as crossings_mod
from sprite_gen.video import held as held_mod
from sprite_gen.video import held_fill as held_fill_mod
+from sprite_gen.video import root_motion as root_motion_mod
from sprite_gen.video import body_turn as body_turn_mod
from sprite_gen.video import legs as legs_mod
from sprite_gen.video import period as period_mod
@@ -1366,6 +1367,40 @@ def _upsample(frames: list[Image.Image], interpolate: rife_mod.Interpolate | Non
return out, record
+def _air_retime(frames: list[Image.Image], interpolate: rife_mod.Interpolate | None, **kw: Any
+ ) -> tuple[list[Image.Image], dict[str, Any], dict[str, Any] | None]:
+ """`root_motion.retime` on a cut played once, with RIFE located only once a frame is to be made: an air given its
+ own length at the clip's rate makes none. A frame to make and no RIFE installed: the cut is delivered as filmed, and
+ the record says which air and how to install it."""
+ made_by: list[dict[str, str]] = []
+ lazy = _located(interpolate, made_by)
+ try:
+ frames_out, record, plan = root_motion_mod.retime(frames, lazy, **kw)
+ except rife_mod.RifeNotInstalled as exc:
+ roots, touch = root_motion_mod.reading(frames, floor=kw["floor"], times=kw["times"], declared=kw["declared"])
+ before = root_motion_mod.read(roots, touch, [k / kw["fps"] for k in range(len(frames))])["air"]
+ return frames, {"asked": dict(kw["asked"]), "applied": False, "before": before, "why": "RIFE is not installed",
+ "rife": str(exc), "install": rife_mod.install_command()}, None
+ if record["applied"] and record["made"]:
+ record["interpolator"] = {**made_by[0], **lazy.counts()} if made_by else {"kind": "injected"}
+ return frames_out, record, plan
+
+
+def _air_lines(name: str, record: dict[str, Any]) -> list[str]:
+ """What became of `--air-retime`, in words: the air not retimed and why, or the cells of it that show the frame
+ beside them again."""
+ if not record["applied"]:
+ if record["why"] == "RIFE is not installed":
+ spans = ", ".join(f"cells {s['cells'][0]}-{s['cells'][1]}" for s in record["before"] if s["seconds"] is not None)
+ return [f"the air of {name} ({spans}) was not retimed — RIFE is not installed, so no frame between was made; "
+ f"run `{record['install']}` and cut it again (docs/video-pipeline.md, \"Root motion\")"]
+ return [f"--air-retime: {record['why']}: cut as filmed"]
+ again = record["shown_again"]
+ return [f"{len(again)} cell{'s' if len(again) > 1 else ''} of the retimed air ({', '.join(map(str, again))}) show the "
+ "frame beside them again: no frame between was made there without a fault (docs/video-pipeline.md, \"Root "
+ "motion\")"] if again else []
+
+
def _fill_lines(record: dict[str, Any]) -> list[str]:
"""What became of the cut's held cells, in words (`video-loop: warning:` lines): the cells left for want of RIFE, the
runs a frame between was refused for, the frames made to look at. A run kept as the motion's own says nothing."""
@@ -2203,6 +2238,9 @@ def _cut_loop(
body_turn: str = "warn",
fill_held: str = "auto",
out_fps: float | None = None,
+ air_retime: str | None = None,
+ meters_per_px: float | None = None,
+ contacts: list[str] | None = None,
) -> dict[str, Any]:
state = state.strip().lower() if state else state
if state == effect.STATE:
@@ -2251,6 +2289,17 @@ def _cut_loop(
"rate than it was filmed (its frames between made), a lower one is its preview's (--gif-fps)")
if cycle_mode == "lifetime":
raise SystemExit("video-loop: --out-fps: an effect's lifetime plays as filmed")
+ asked_air = None
+ if air_retime is not None:
+ try:
+ asked_air = root_motion_mod.parse_retime(air_retime)
+ except ValueError as exc:
+ raise SystemExit(f"video-loop: {exc}") from None
+ if meters_per_px is None and {"gravity", "speed"} & set(asked_air):
+ raise SystemExit("video-loop: --air-retime gravity= and speed= are metres: give --meters-per-px, the metres a "
+ "pixel of the clip as filmed holds (the standing height in metres over the strip's body_src_h)")
+ if meters_per_px is not None and not (math.isfinite(meters_per_px) and meters_per_px > 0):
+ raise SystemExit(f"video-loop: --meters-per-px {meters_per_px:g}: a number over 0")
if anchor is None:
# gaits drift a few pixels over a cycle and the wrap shows it; in-place states do not
anchor = "body" if profile_for(state).gait else "none"
@@ -2279,6 +2328,10 @@ def _cut_loop(
_refuse_overlap(given, files, out_dir, name, target, report_path is not None)
prof = profile_for(state)
n = len(files)
+ try:
+ declared = root_motion_mod.parse_contacts(contacts or [], n)
+ except ValueError as exc:
+ raise SystemExit(f"video-loop: {exc}") from None
# A walk or run may open on a lead-in (gait_fallback.lead_in): the video model reframing a small
# subject before it walks at the clip's size. An automatic search reads the clip after it as a clip
# of its own, and what it finds is said in the clip's own frame numbers (`_from_lead_in`); a cut the
@@ -2778,7 +2831,39 @@ def _cut_loop(
"of it is named a ghost, here or in video-cycle-align, so look at cycle/ (docs/loop-repair.md)", file=sys.stderr)
held_record = report_base["held_fill"]
upsampled = None
- if out_fps is not None:
+ # A cut played once carries its root motion (docs/video-pipeline.md, "Root motion"), read against the clip's floor.
+ floor = root_motion_mod.clip_floor(files) if cycle["kind"] in root_motion_mod.KINDS else None
+ retimed = None
+ if asked_air is not None:
+ if cycle["kind"] not in root_motion_mod.KINDS:
+ report_base["air_retime"] = {"asked": asked_air, "applied": False,
+ "why": f"a {cycle['kind']} cut: the air of a cut played once is retimed"}
+ else:
+ # The air given its own length, gravity and speed: its cells made again between the clip's frames at the
+ # cut's rate (held_fill.at_times), each moved onto the parabola, the landing on along.
+ try:
+ frames, report_base["air_retime"], retimed = _air_retime(
+ frames, interpolate, fps=fps, rate=out_fps or fps, floor=floor, times=[i + k for k in range(L)],
+ declared=declared, asked=asked_air, meters_per_px=meters_per_px, facing=facing)
+ except (ValueError, rife_mod.RifeUnavailable) as exc:
+ error = (f"video-loop: {exc}; --air-retime makes the frames between the clip's — fix RIFE, or cut it "
+ "without --air-retime (docs/video-pipeline.md, \"Root motion\")")
+ refuse(target, out_dir, name, {**report_base, "status": "failed", "error": error, "cycle": cycle})
+ raise SystemExit(error) from exc
+ for line in _air_lines(name, report_base["air_retime"]):
+ print(f"video-loop: warning: {name}: {line}", file=sys.stderr)
+ if retimed is not None:
+ upsampled = retimed["times"]
+ report_base["seam_measurement"] = "rendered-cells"
+ as_filmed = {} # the cells are each made at a time of their own: read as they play
+ cycle_seconds = len(frames) / (out_fps or fps)
+ n_out = min(len(frames), max(n_asked, round(cycle_seconds * gif_fps)))
+ if out_fps is not None:
+ record = report_base["air_retime"]
+ held_record["out_fps"] = {"asked": out_fps, "fps": fps, "applied": True, "cells": len(frames),
+ "times": [round(x, 4) for x in upsampled], "made": record["made"],
+ "shown_again": record["shown_again"]}
+ if out_fps is not None and retimed is None:
# A rate above the clip's: every cell at its own time, made between the frames beside it (held_fill.upsample).
try:
frames, held_record["out_fps"] = _upsample(frames, interpolate, ring=ring, fps=fps, out_fps=out_fps)
@@ -2832,9 +2917,13 @@ def _cut_loop(
strip_meta["source_cut"] = {"source": report_base["source"], "start": i, "length": L,
**({"samples": [i + k for k in strip_meta["sample_indices"]]} if upsampled is None else
{"times": [round(i + upsampled[k], 4) for k in strip_meta["sample_indices"]]})}
- if upsampled is not None:
+ if upsampled is not None and out_fps is not None:
# Each cell is a time of its own, at a rate above the clip's: the cells are not the clip's frames.
strip_meta.update(out_fps=out_fps, source_fps=fps)
+ if "air_retime" in report_base:
+ # The air given its own length, or why it was not: a retimed cut's cells are times of their own, and the air's
+ # are moved onto its parabola.
+ strip_meta["air_retime"] = root_motion_mod.public(report_base["air_retime"])
if cycle["kind"] in nonloop.KINDS:
strip_meta.update(nonloop.metadata(cycle, strip_meta["sample_indices"] if upsampled is None else
[upsampled[k] for k in strip_meta["sample_indices"]]))
@@ -2855,6 +2944,18 @@ def _cut_loop(
strip_meta["clip_feet"] = report_base["clip_feet"]
# The cut's held cells, what became of them, and its poses over its cells (docs/loop-repair.md section 7).
strip_meta["held_fill"] = held_fill_mod.public(held_record)
+ if cycle["kind"] in root_motion_mod.KINDS:
+ # Where the body is in each cell, what each touches and the air (docs/video-pipeline.md, "Root motion"): read on
+ # the cells' frames as delivered, each at its time as it plays.
+ cells_of = strip_meta["sample_indices"]
+ played = len(frames) / cycle_seconds
+ shown = [frames[k] for k in cells_of]
+ touch = ([retimed["touch"][k] for k in cells_of] if retimed is not None else root_motion_mod.touching(
+ shown, floor=floor, times=[i + (k if upsampled is None else upsampled[k]) for k in cells_of], declared=declared))
+ report_base["root_motion"] = strip_meta["root_motion"] = root_motion_mod.sidecar(
+ shown, rect=strip_meta["source_rect"], size=(strip_meta["w"], strip_meta["h"]),
+ floor=None if floor is None else floor + (retimed["offset"][1] if retimed is not None else 0),
+ seconds=[k / played for k in cells_of], touch=touch, meters_per_px=meters_per_px)
# A cut frame that is no longer the frame as filmed for a filmed ghost, and one kept with no clean frame beside
# it: `cycle/` is the alignment's material (`cycle.source/`), whose screen no longer sees a ghost given way, so the
# strip says it, and an alignment carries it over (frames of the cut).
@@ -3150,6 +3251,23 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
"between the frames beside it — never a frame shown again. A loop keeps its cycle's seconds, a "
"cut played once runs from its first frame to its last; the strip metadata says `out_fps`. With no "
"RIFE installed the cut is delivered at its own rate, with a warning (docs/loop-repair.md section 7)")
+ parser.add_argument("--air-retime", default=None, metavar="gravity=G,air=T,speed=V",
+ help="a cut played once: give each air with its take-off and landing in the cut (the frames between the "
+ "last contact and the next) a length of T seconds, a gravity of G m/s² and a speed along of V m/s, "
+ "any of them, each the clip's own where not given: its cells made again at the cut's rate, each "
+ "showing the clip's pose at the same share of its air, made by RIFE between the frames where it "
+ "falls between them, and moved onto the parabola; the take-off, the landing and the contacts as "
+ "filmed, the landing moved along. G and V need --meters-per-px. The strip metadata says "
+ "`air_retime`; with no RIFE installed the air is cut as filmed, with a warning "
+ "(docs/video-pipeline.md, \"Root motion\")")
+ parser.add_argument("--meters-per-px", type=float, default=None,
+ help="the metres a pixel of the clip as filmed holds (the standing height in metres over the strip's "
+ "`body_src_h`): the root motion in metres too (`root_m`, each apex's `apex_m`), and --air-retime's "
+ "gravity and speed")
+ parser.add_argument("--contact", action="append", default=None, metavar="NAME=SPANS",
+ help="a contact the floor does not show (a hand on a wall the clip does not draw), over zero-based frames "
+ "of the clip, as inspect-motion's spans: hand=12:20 (repeatable). The ground is read off the floor; "
+ "the air is the frames that touch neither")
parser.add_argument("--name", default="loop", help="basename for strip/gif/webp outputs")
parser.add_argument("--report", type=Path)
@@ -3173,6 +3291,8 @@ def run(**kwargs: object) -> int:
seam_pop_max=kwargs.get("seam_pop_max", repair_mod.SEAM_POP_REFERENCE), # type: ignore[arg-type]
body_turn=str(kwargs.get("body_turn") or "warn"),
fill_held=str(kwargs.get("fill_held") or "auto"), out_fps=kwargs.get("out_fps"), # type: ignore[arg-type]
+ air_retime=kwargs.get("air_retime"), meters_per_px=kwargs.get("meters_per_px"), # type: ignore[arg-type]
+ contacts=kwargs.get("contact"), # type: ignore[arg-type]
)
summary = {k: payload[k] for k in ("state", "frames_total", "window", "cycle_seconds", "n_out", "delay_ms", "resampled_seam_ratio", "specks_dropped", "report")}
summary["cycle"] = {k: payload["cycle"].get(k) for k in ("kind", "start", "length", "period_global", "ratio", "review_recommended", "half_period_guard", "fundamental")}
@@ -3245,6 +3365,12 @@ def run(**kwargs: object) -> int:
"held_runs": held["held_runs"], "made": held["made"],
**({"unmade": held["unmade"]["cells"]} if "unmade" in held else {}),
**({"out_fps": {k: held["out_fps"][k] for k in ("asked", "applied", "cells")}} if "out_fps" in held else {})}
+ if "root_motion" in payload:
+ # Each air of a cut played once, as a person is told it: its seconds and its apex.
+ summary["air"] = root_motion_mod.said(payload["root_motion"])
+ if "air_retime" in payload:
+ summary["air_retime"] = {k: payload["air_retime"][k] for k in ("applied", "why", "made", "shown_again", "cells", "poses")
+ if k in payload["air_retime"]}
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0
diff --git a/sprite_gen/video/root_motion.py b/sprite_gen/video/root_motion.py
new file mode 100644
index 0000000..5de31de
--- /dev/null
+++ b/sprite_gen/video/root_motion.py
@@ -0,0 +1,340 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Root motion: where the body is in each cell of a cut played once, what it stands on and its air (`read`,
+`sidecar`); and the air given a length, a gravity and a speed of its own (`retime`). docs/video-pipeline.md, "Root
+motion".
+
+The root is the subject's mass centre: the mean of its pixels (alpha over 0), the point inspect-motion reads as a
+frame's `centroid_px` (`qa.motion.centroid`). No body part is named. In the air it is the point that falls on a parabola
+under gravity whatever the arms and legs do, so it is the point a game moves: a cell is drawn with its root on the
+body's place. A frame stands on the ground where its lowest body row is within nonloop.GROUND_PX of the clip's floor,
+the lowest row any frame of the clip reaches (`nonloop.on_floor`, the rule the strip's standing height reads). A
+contact the floor does not show, a hand on a wall the clip does not draw, is named by the caller over frames of the clip
+in inspect-motion's spans (`--contact hand=12:20`, `qa.motion.contact_spans`). The air is each run of frames touching
+nothing: its take-off is the frame before it, its landing the frame after, its seconds those between the two, and its
+apex how far the root rises over the take-off's.
+
+`retime` gives each air with a take-off and a landing a length (`air`, seconds), a gravity (`gravity`) and a speed
+along (`speed`), each the clip's own where not given. The cut is made again at its rate: the frames up to the take-off
+and from the landing at the clip's pace, the air's cells spread evenly over it, each showing the clip's pose at the same
+share of the clip's air, made between the two frames it falls between where it falls between them (held_fill.at_times,
+the cells the 3.8.0 fill makes). Each air cell is then moved whole, by whole pixels, until its root stands on the
+parabola from the take-off's root to the landing's; the landing and every cell after it is moved along by as much
+further as the air now carries the body, so a planted foot stays planted. The take-off and the landing are the frames
+as filmed, and so is every cell up to the take-off.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+
+from sprite_gen._deps import np
+from sprite_gen.qa import motion as qa_motion
+from sprite_gen.video import held_fill as held_fill_mod
+from sprite_gen.video import nonloop
+from sprite_gen.video import rife as rife_mod
+
+KINDS = ("once", "once-return", "one-shot") # the cuts played once that carry a body (not an effect's lifetime)
+RETIME_KEYS = ("gravity", "air", "speed")
+GROUND = "ground"
+SNAP = held_fill_mod.SNAP # a time within this of a frame is that frame
+
+
+def parse_retime(text: str) -> dict[str, float]:
+ """`gravity=G,air=T,speed=V`, any of them, each once: metres a second squared, seconds, metres a second along."""
+ out: dict[str, float] = {}
+ for part in text.split(","):
+ key, eq, value = (s.strip() for s in part.partition("="))
+ if not eq or key not in RETIME_KEYS or key in out:
+ raise ValueError(f"--air-retime {text!r}: KEY=VALUE pairs of {', '.join(RETIME_KEYS)}, each once "
+ "(gravity=9.81,air=0.6,speed=4)")
+ try:
+ number = float(value)
+ except ValueError:
+ raise ValueError(f"--air-retime {key}={value}: not a number") from None
+ if not math.isfinite(number) or number < 0 or (number == 0 and key != "speed"):
+ raise ValueError(f"--air-retime {key}={value}: " + ("0 or more" if key == "speed" else "more than 0"))
+ out[key] = number
+ return out
+
+
+def parse_contacts(values: Sequence[str], frames: int) -> list[tuple[str, list[list[int]]]]:
+ """`NAME=SPANS` (`hand=12:20`): a contact the floor does not show, over zero-based frames of the clip, in
+ inspect-motion's spans (`qa.motion.contact_spans`). `ground` is the floor's own."""
+ out: list[tuple[str, list[list[int]]]] = []
+ for value in values:
+ name, eq, spans = (s.strip() for s in value.partition("="))
+ if not eq or not name or name == GROUND or any(name == n for n, _ in out):
+ raise ValueError(f"--contact {value!r}: NAME=SPANS, a name of its own (not {GROUND!r}) and the clip's frames "
+ "it touches through, e.g. hand=12:20")
+ try:
+ out.append((name, qa_motion.contact_spans(spans, frames)))
+ except ValueError as exc:
+ raise ValueError(f"--contact {value!r}: {exc} (the clip has {frames} frames, numbered from 0)") from None
+ return out
+
+
+def clip_floor(files: Sequence[Path]) -> int | None:
+ """The clip's floor: the lowest body row any of its frames reaches (`nonloop.body_box`), as filmed."""
+ bottoms = [box[3] for f in files if (box := nonloop.body_box(Image.open(f).convert("RGBA"))) is not None]
+ return max(bottoms) if bottoms else None
+
+
+def root(frame: Image.Image) -> list[float] | None:
+ """A frame's root: its subject's mass centre, [x, y] px (`qa.motion.centroid`)."""
+ return qa_motion.centroid(np.asarray(frame.getchannel("A")) > 0)
+
+
+def touching(frames: Sequence[Image.Image], *, floor: int | None, times: Sequence[float],
+ declared: Sequence[tuple[str, list[list[int]]]] = ()) -> list[list[str]]:
+ """What each frame touches: the ground where its body stands on the floor, and each declared contact whose span
+ holds its time (in frames of the clip)."""
+ out = []
+ for frame, t in zip(frames, times):
+ box = nonloop.body_box(frame)
+ on = [GROUND] if floor is not None and box is not None and nonloop.on_floor(box, floor) else []
+ on += [name for name, spans in declared if any(a - SNAP <= t <= b + SNAP for a, b in spans)]
+ out.append(on)
+ return out
+
+
+def reading(frames: Sequence[Image.Image], *, floor: int | None, times: Sequence[float],
+ declared: Sequence[tuple[str, list[list[int]]]] = ()) -> tuple[list[list[float] | None], list[list[str]]]:
+ """Each frame's root and what it touches (`root`, `touching`)."""
+ return [root(f) for f in frames], touching(frames, floor=floor, times=times, declared=declared)
+
+
+def _runs(flags: Sequence[bool]) -> list[list[int]]:
+ """Each run of true flags: [first, last]."""
+ found: list[list[int]] = []
+ for k, flag in enumerate(flags):
+ if flag and found and found[-1][1] == k - 1:
+ found[-1][1] = k
+ elif flag:
+ found.append([k, k])
+ return found
+
+
+def read(roots: Sequence[list[float] | None], touch: Sequence[list[str]], seconds: Sequence[float]) -> dict[str, Any]:
+ """The contacts (each name's runs of frames) and the air (each run of frames touching nothing: its take-off and
+ landing frames, the seconds between them, and its apex, how far the root rises over the take-off's, px up)."""
+ n = len(touch)
+ names = list(dict.fromkeys(name for on in touch for name in on))
+ contacts = [{"by": name, "cells": run} for name in names for run in _runs([name in on for on in touch])]
+ air = []
+ for first, last in _runs([not on for on in touch]):
+ takeoff = first - 1 if first > 0 else None
+ landing = last + 1 if last < n - 1 else None
+ span: dict[str, Any] = {"cells": [first, last], "takeoff": takeoff, "landing": landing,
+ "seconds": round(seconds[landing] - seconds[takeoff], 4) if None not in (takeoff, landing) else None}
+ ys = [(roots[k][1], k) for k in range(first, last + 1) if roots[k] is not None]
+ if takeoff is not None and roots[takeoff] is not None and ys:
+ top, at = min(ys)
+ span.update(apex_px=round(roots[takeoff][1] - top, 2), apex_cell=at)
+ air.append(span)
+ return {"contacts": sorted(contacts, key=lambda c: (c["cells"][0], c["by"])), "air": air}
+
+
+def sidecar(frames: Sequence[Image.Image], *, rect: Sequence[int] | None, size: tuple[int, int], floor: int | None,
+ seconds: Sequence[float], touch: Sequence[list[str]], meters_per_px: float | None) -> dict[str, Any]:
+ """The strip's `root_motion`: each cell's root in cell px from the cell's top-left (draw the cell with that point on
+ the body's place), the floor's row, the contacts and the air in cells; with `meters_per_px` (of the frames as cut)
+ the roots in metres from the first cell's, up positive, and each apex in metres. `frames` are the cells' frames as
+ cut, `rect` the source rectangle the strip crops them to, `size` a cell's (w, h)."""
+ if rect is None:
+ return {"why": "the cells are each moved to a foot line of their own (--anchor feet): no one rectangle maps a frame to its cell"}
+ left, top, right, bottom = rect
+ sx, sy = size[0] / (right - left), size[1] / (bottom - top)
+ at = [root(f) for f in frames]
+ measured = read(at, touch, seconds)
+ for span in measured["air"]:
+ if "apex_px" in span:
+ span["apex_px"] = round(span["apex_px"] * sy, 2)
+ out: dict[str, Any] = {
+ "point": "mass-centre",
+ "root": [None if r is None else [round((r[0] - left) * sx, 2), round((r[1] - top) * sy, 2)] for r in at],
+ "ground_y": None if floor is None else round((floor - top) * sy, 2),
+ **measured,
+ }
+ if meters_per_px is not None:
+ per_cell = meters_per_px / sy
+ out["meters_per_px"] = round(per_cell, 8)
+ first = next((r for r in out["root"] if r is not None), None)
+ out["root_m"] = [None if r is None or first is None else
+ [round((r[0] - first[0]) * per_cell, 4), round((first[1] - r[1]) * per_cell, 4)] for r in out["root"]]
+ for span in out["air"]:
+ if "apex_px" in span:
+ span["apex_m"] = round(span["apex_px"] * per_cell, 4)
+ return out
+
+
+def said(record: dict[str, Any]) -> list[str]:
+ """Each air of a cut as a person is told it: its seconds and its apex."""
+ lines = []
+ for span in record.get("air", []):
+ if span.get("seconds") is None:
+ lines.append(f"cells {span['cells'][0]}-{span['cells'][1]} in the air, no "
+ f"{'take-off' if span['takeoff'] is None else 'landing'} in the cut")
+ continue
+ apex = f"{span['apex_m']:.2f} m" if "apex_m" in span else f"{span['apex_px']:g} px"
+ lines.append(f"air {span['seconds']:.2f} s, apex {apex} (cells {span['cells'][0]}-{span['cells'][1]})")
+ return lines
+
+
+def _gravity(ys: Sequence[float], a: int, b: int, fps: float) -> float | None:
+ """The clip's own gravity over an air from frame `a` to `b` (px a second squared, down): the least squares fit of
+ the root's rise over the line from take-off to landing to g·t·(T - t)/2. None where the root is unread."""
+ T = (b - a) / fps
+ num = den = 0.0
+ for k in range(a + 1, b):
+ t = (k - a) / fps
+ s = t * (T - t) / 2
+ num += (ys[a] + (ys[b] - ys[a]) * t / T - ys[k]) * s
+ den += s * s
+ return num / den if den > 0 else None
+
+
+def apex(rise: float, gravity: float, seconds: float) -> float:
+ """The highest a root rises over its take-off's on the parabola that ends `rise` px higher after `seconds` under
+ `gravity` (px a second squared): g·T²/8 where it lands at its own height."""
+ t = min(seconds, max(0.0, seconds / 2 + rise / (gravity * seconds)))
+ return rise * t / seconds + gravity * t * (seconds - t) / 2
+
+
+def _pieces(L: int, spans: Sequence[dict[str, Any]], *, fps: float, rate: float,
+ airs: Sequence[float]) -> list[tuple[float, float, int, int | None]]:
+ """The cut as pieces (from frame, to frame, cells, which air or None): the ground at the clip's pace, each air in
+ as many cells as its new seconds hold at `rate`."""
+ pieces: list[tuple[float, float, int, int | None]] = []
+ at = 0
+ for which, span in enumerate(spans):
+ a, b = span["takeoff"], span["landing"]
+ if a > at:
+ pieces.append((at, a, max(1, round((a - at) * rate / fps)), None))
+ pieces.append((a, b, max(1, round(airs[which] * rate)), which))
+ at = b
+ if L - 1 > at:
+ pieces.append((at, L - 1, max(1, round((L - 1 - at) * rate / fps)), None))
+ return pieces
+
+
+def retime(frames: list[Image.Image], interpolate: rife_mod.Interpolate, *, fps: float, rate: float, floor: int | None,
+ times: Sequence[float], declared: Sequence[tuple[str, list[list[int]]]], asked: dict[str, float],
+ meters_per_px: float | None, facing: str = "right") -> tuple[list[Image.Image], dict[str, Any], dict[str, Any] | None]:
+ """The cut with each air that has a take-off and a landing retimed (see the module), at `rate` cells a second.
+ `times` are the frames' times in frames of the clip (for the declared contacts). Returns the frames, the record, and
+ what the cut's reading needs where it was retimed (None where not): each cell's time in frames of the cut (`times`),
+ what each cell touches (`touch`), and the canvas the cells were moved on (`offset`, where the frames' (0, 0) is)."""
+ L = len(frames)
+ roots, touch = reading(frames, floor=floor, times=times, declared=declared)
+ before = read(roots, touch, [k / fps for k in range(L)])
+ record: dict[str, Any] = {"asked": dict(asked), "applied": False, "before": before["air"]}
+ if meters_per_px is not None:
+ record["meters_per_px"] = meters_per_px
+ spans = [s for s in before["air"] if s["seconds"] is not None]
+ if not spans:
+ record["why"] = ("no air in the cut" if not before["air"] else
+ "no air with both its take-off and its landing in the cut")
+ return frames, record, None
+ if any(roots[s["takeoff"]] is None or roots[s["landing"]] is None for s in spans):
+ record["why"] = "a take-off or a landing frame shows no body"
+ return frames, record, None
+ plans = []
+ for span in spans:
+ a, b = span["takeoff"], span["landing"]
+ T0 = (b - a) / fps
+ ys = [r[1] if r is not None else math.nan for r in roots]
+ g0 = _gravity(ys, a, b, fps)
+ g = asked["gravity"] / meters_per_px if "gravity" in asked else g0
+ if g is None or not g > 0:
+ record["why"] = (f"the clip's own air (frames {a}-{b}) does not rise over the line from its take-off to its "
+ "landing (no gravity to read): give gravity=")
+ return frames, record, None
+ along = roots[b][0] - roots[a][0]
+ sign = (1 if along > 0 else -1) if abs(along) >= 1 else (1 if facing == "right" else -1)
+ speed = asked["speed"] / meters_per_px if "speed" in asked else abs(along) / T0
+ T1 = asked.get("air", T0)
+ plans.append({"a": a, "b": b, "T0": T0, "g0": g0, "g": g, "speed": speed, "sign": sign, "T1": T1})
+ pieces = _pieces(L, spans, fps=fps, rate=rate, airs=[p["T1"] for p in plans])
+ at, spacing, piece_of = [0.0], [1.0], [None]
+ for start, end, cells, which in pieces:
+ for j in range(1, cells + 1):
+ at.append(start + (end - start) * j / cells)
+ spacing.append((end - start) / cells)
+ piece_of.append((which, j, cells))
+ made_frames, made = held_fill_mod.at_times(frames, interpolate, at, ring=False, spacing=spacing)
+ # Each cell moved: none up to the first take-off; an air cell onto its parabola; from a landing on, along.
+ moves: list[tuple[int, int]] = []
+ along_px = 0
+ for k, of in enumerate(piece_of):
+ which = None if of is None else of[0]
+ if which is not None and of[1] == 1:
+ plans[which].update(moved_from=along_px, cells=of[2], seconds=of[2] / rate, first_cell=k - 1)
+ if which is None or of[1] == of[2]:
+ if which is not None:
+ # the landing: as far along as the air carried the body from its take-off
+ p = plans[which]
+ xa = roots[p["a"]][0] + p["moved_from"]
+ along_px = round(xa + p["sign"] * p["speed"] * p["seconds"] - roots[p["b"]][0])
+ p["moved_px"] = along_px - p["moved_from"]
+ moves.append((along_px, 0))
+ continue
+ p = plans[which]
+ T, t, w = p["seconds"], of[1] / rate, of[1] / of[2]
+ xa, ya = roots[p["a"]][0] + p["moved_from"], roots[p["a"]][1]
+ x = xa + p["sign"] * p["speed"] * T * w
+ y = ya + (roots[p["b"]][1] - ya) * w - p["g"] * t * (T - t) / 2
+ here = root(made_frames[k])
+ moves.append((0, 0) if here is None else (round(x - here[0]), round(y - here[1])))
+ boxes = [nonloop.body_box(f) for f in made_frames]
+ W, H = frames[0].size
+ lo_x = min([0] + [b[0] + m[0] for b, m in zip(boxes, moves) if b is not None])
+ lo_y = min([0] + [b[1] + m[1] for b, m in zip(boxes, moves) if b is not None])
+ hi_x = max([W] + [b[2] + m[0] for b, m in zip(boxes, moves) if b is not None])
+ hi_y = max([H] + [b[3] + m[1] for b, m in zip(boxes, moves) if b is not None])
+ offset, size = (-lo_x, -lo_y), (hi_x - lo_x, hi_y - lo_y)
+ out = []
+ for f, (dx, dy) in zip(made_frames, moves):
+ if (dx, dy) == (0, 0) and offset == (0, 0) and size == (W, H):
+ out.append(f)
+ continue
+ canvas = Image.new("RGBA", size, (0, 0, 0, 0))
+ canvas.paste(f, (offset[0] + dx, offset[1] + dy))
+ out.append(canvas)
+ cell_touch = []
+ for k, of in enumerate(piece_of):
+ if of is not None and of[0] is not None and of[1] < of[2]:
+ cell_touch.append([])
+ else:
+ cell_touch.append(touch[min(L - 1, round(at[k]))])
+ airs = []
+ for span, p in zip(spans, plans):
+ rise = roots[p["a"]][1] - roots[p["b"]][1]
+ row = {"cells": [p["first_cell"] + 1, p["first_cell"] + p["cells"] - 1],
+ "takeoff": p["first_cell"], "landing": p["first_cell"] + p["cells"],
+ "frames": [p["a"], p["b"]], "seconds_before": round(p["T0"], 4), "seconds": round(p["seconds"], 4),
+ "apex_before_px": span.get("apex_px"), "apex_px": round(apex(rise, p["g"], p["seconds"]), 2),
+ "gravity_px": round(p["g"], 2), "speed_px": round(p["speed"], 2), "moved_along_px": p["moved_px"]}
+ if p["g0"] is not None:
+ row["gravity_before_px"] = round(p["g0"], 2)
+ if meters_per_px is not None:
+ row.update(apex_m=round(row["apex_px"] * meters_per_px, 4), gravity=round(p["g"] * meters_per_px, 4),
+ speed=round(p["speed"] * meters_per_px, 4))
+ if "air" in asked and abs(p["seconds"] - asked["air"]) > 1e-9:
+ row["asked_seconds"] = asked["air"]
+ airs.append(row)
+ same = held_fill_mod.same_steps(out)
+ record.update(applied=True, rate=rate, air=airs, made=made["made"], shown_again=made["shown_again"],
+ canvas_offset=list(offset), **held_fill_mod.reading(same))
+ record["frames"] = made["frames"]
+ return out, record, {"times": at, "touch": cell_touch, "offset": offset}
+
+
+def public(record: dict[str, Any]) -> dict[str, Any]:
+ """The retime as the strip metadata carries it: what was asked, each air before and after, the cells made and the
+ cells shown again, the poses over the cells — not how each frame was made."""
+ return {k: v for k, v in record.items() if k != "frames"}
diff --git a/sprite_gen/video/source.py b/sprite_gen/video/source.py
index 959e8a8..0dc1c54 100644
--- a/sprite_gen/video/source.py
+++ b/sprite_gen/video/source.py
@@ -207,9 +207,10 @@ def project(baseline: Any, source: Source) -> tuple[Projection | None, str | Non
# A held cut is read again only where its report records each frame's move, and the cut's frames before repair to
# prove every one of them on (`source_projection`, the moves of the cells the repair replaced included).
held = bool(hold.get("applied"))
- # A cut whose held cells were made again (`held_fill`), or whose cells are times of their own (`out_fps`), shows
- # frames the source does not hold.
+ # A cut whose held cells were made again (`held_fill`), or whose cells are times of their own (`out_fps`, an air
+ # retimed: `air_retime`), shows frames the source does not hold.
if (any(k in m for k in ("cycle_align", "motion_anchor", "wrap_blend", "out_fps")) or (m.get("held_fill") or {}).get("made")
+ or (m.get("air_retime") or {}).get("applied")
or held and ("cut_frames" not in hold or report.get("source_projection") is None)):
return None, "source-processing-unsupported"
moves, pad = _cut_moves(hold, n) if held else (None, None)