Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat(video): idle stands still and closes on its first frame

Soohong Kimb2613621b437
diff --git a/docs/video-pipeline.md b/docs/video-pipeline.md
index cf8e5da..59154b0 100644
--- a/docs/video-pipeline.md
+++ b/docs/video-pipeline.md
@@ -92,6 +92,16 @@ frame, and asks for crisp frames without motion blur. `video-set` asks attack cl
and pins the clip to end on the frame it starts from: `--last-frame` is the canvas itself
(first-last mode, see [video.md](video.md)), so the strike has to come back to the still.
+An idle holds its feet and is pinned too. `MOTION_TEXT["idle"]` keeps both feet planted flat for
+the whole clip, limits the motion to breathing, a settle of the arms, hair and loose cloth and
+one blink, and names walking and marching in place as what not to do — a side-view full body
+asked for "a subtle weight sway" and an evenly paced loop tends to step in place. Its template
+(`PINNED_LOOP_TEXT`) replaces the "evenly paced motion so the animation loops" line with the
+return: the last frame comes back to the exact pose of the first. `video-set` pins idle clips
+to the canvas (`PIN_LAST_FRAME_STATES`) and, because such a clip starts and ends on the same
+frame, cuts it with `video-loop --cycle pinned` (`PINNED_LOOP_STATES`) instead of searching
+it for a repeat.
+
`build_prompt(direction, state, character, facing, motion=...)` takes a caller's own motion
paragraph — whole sentences about the subject, such as a request interpreter writes per
request — in place of the built-in state sentence. The frame, camera, background and design
@@ -314,6 +324,15 @@ cycle is known (the 2026-09-09 reel jump: 2.3 hops in 145 frames). It is an expl
instruction, not a failover: the report says `kind = "fixed"`, and the seam gate still
applies.
+`--cycle pinned` is for a clip pinned to end on its first frame (`video --last-frame` set to
+the start image). The cycle is every frame but the last, which re-renders the first, so the
+wrap plays like the step into that last frame. The gate is the pin, not the seam ratio: the
+last frame has to land within `max(seam_max x the mean step, PIN_NOISE_MAX)` of the first, on
+the analysis thumbnail. A near-still clip moves so little per frame that the ratio reads the
+re-render noise of the pinned frame as a jump; a clip that ends in another pose still fails,
+with its own message (`the pinned clip does not end on its first frame`). The report says
+`kind = "pinned"` and records `pin_error` and `pin_tolerance`.
+
Outputs:
- `cycle/frame-NNN.png` — the cycle frames, RGB under alpha 0 scrubbed, detached specks
@@ -352,7 +371,7 @@ runs canvas → video → frames → loop for every (direction, state). The xAI
is **2 requests per second** (five parallel starts produced two HTTP 429s): starts are
staggered (`--start-gap 2`) and a 429 gets a bounded, logged retry (15 s, 30 s). Clip length
is each state's own default (3 s, attack 4 s) unless `--duration` sets one for all, and an
-attack clip is pinned to end on its canvas. States use differently shaped canvases (square, tall, wide),
+attack or idle clip is pinned to end on its canvas (an idle is then cut whole, `--cycle pinned`). States use differently shaped canvases (square, tall, wide),
so the same character films at different pixel heights; `--body-height N` gives every state's loop the
same standing-height target and keeps the character one size across the set. At a low
`--resolution`, `--fit tight` frames every item without room so that target is reached by
diff --git a/sprite_gen/video/batch.py b/sprite_gen/video/batch.py
index 2c79476..55324c9 100644
--- a/sprite_gen/video/batch.py
+++ b/sprite_gen/video/batch.py
@@ -40,9 +40,13 @@ DEFAULT_DURATION_SECONDS = 3
# them, a longer one stretches them.
STATE_DURATION_SECONDS = {"attack": 4}
# States whose clip is pinned to end on the frame it starts from (first-last mode): the model
-# has to come back to the still, which is what closes a one-shot action into a loop instead of
-# leaving it wherever the strike ended.
-PIN_LAST_FRAME_STATES = frozenset({"attack"})
+# has to come back to the still, which closes a one-shot action instead of leaving it wherever
+# the strike ended, and closes an idle without asking for a repeating rhythm.
+PIN_LAST_FRAME_STATES = frozenset({"attack", "idle"})
+# Pinned states whose whole clip is the loop: it starts and ends on the canvas, so `video-loop
+# --cycle pinned` keeps every frame but the last (a re-render of the first). An attack is
+# pinned too but is cut as the one-shot it is.
+PINNED_LOOP_STATES = frozenset({"idle"})
RETRY_BACKOFF_SECONDS = (15, 30)
@@ -55,7 +59,14 @@ VIEW_TEXT = {
"back": "seen from directly behind, facing away from the viewer",
}
MOTION_TEXT = {
- "idle": "holds a relaxed idle in place: slow gentle breathing, a subtle weight sway, one natural blink if the face has eyes. The ground contact never slides.",
+ # A side-view full body asked for "a subtle weight sway" and an evenly paced loop steps in place
+ # more often than not, so the feet are held and walking is named as what not to do.
+ "idle": (
+ "stands still in a relaxed idle pose with both feet planted flat on the ground for the whole clip: "
+ "slow, gentle breathing that softly rises and falls in the chest and shoulders, a slight settle of the arms, "
+ "hair and loose cloth, and one natural blink if the face has eyes. The feet never lift, step, shuffle or slide "
+ "— no walking, no marching in place, no turning."
+ ),
"walk": "moves in place on a treadmill: a steady locomotion cycle for this body type with clear repeating ground contacts and an even left-right or front-back rhythm the body already has.",
"run": "moves in place on a treadmill: a fast locomotion cycle for this body type with a bounding rhythm and clear repeating ground contacts.",
"jump": "performs a modest vertical hop in place over and over: compress, spring up about half the body height, land softly, return to the exact starting stance, repeat at an even rhythm. Same height every time.",
@@ -77,6 +88,13 @@ COMMON_TEXT = (
"proportions exactly as in the image. Consistent, evenly paced motion so the animation loops."
)
+# A pinned loop (`PINNED_LOOP_STATES`) closes because the clip ends on its first frame, not by
+# repeating a motion at an even rhythm — so it asks for the return instead of the rhythm.
+PINNED_LOOP_TEXT = COMMON_TEXT.replace(
+ "Consistent, evenly paced motion so the animation loops.",
+ "The last frame returns to the exact pose of the first frame, so the animation loops seamlessly.",
+)
+
# One-shot actions pinned to their first frame (`PIN_LAST_FRAME_STATES`). No "evenly paced" line:
# a strike is fast and a windup is not, so the timing lives in the motion sentence instead. What the
# character holds is kept inside the frame too, and the fast frames are asked to stay crisp.
@@ -112,7 +130,12 @@ def build_prompt(direction: str, state: str, character: str | None, facing: str
"""
validate_facing(facing)
view = VIEW_TEXT.get(direction, f"seen from the {direction}").format(facing=facing)
- template = ACTION_COMMON_TEXT if state in PIN_LAST_FRAME_STATES else COMMON_TEXT
+ if state in PINNED_LOOP_STATES:
+ template = PINNED_LOOP_TEXT
+ elif state in PIN_LAST_FRAME_STATES:
+ template = ACTION_COMMON_TEXT
+ else:
+ template = COMMON_TEXT
if motion is not None:
motion = " ".join(motion.split())
if not motion:
@@ -240,7 +263,8 @@ def run_item(
fr = frames_mod.run_frames(clip, item_dir / "frames", key=key, allow_edge_contact=False, report_path=item_dir / "frames.report.json", spill=spill, reference=canvas_png, allow_subject_edge_contact=fit == "tight")
result["frames"] = {k: fr[k] for k in ("fps", "frames", "alpha_zero_pct_min", "alpha_zero_pct_max")}
result["frames"]["spill"] = fr.get("spill", {}).get("mode")
- lp = loop_mod.run_loop(Path(fr["keyed_dir"]), item_dir / "loop", fps=float(fr["fps"]), state=state, min_len=None, max_len=None, n_out=None, seam_max=loop_mod.SEAM_RATIO_MAX, name=item, report_path=item_dir / "loop.report.json", anchor=anchor, body_height=body_height)
+ lp = loop_mod.run_loop(Path(fr["keyed_dir"]), item_dir / "loop", fps=float(fr["fps"]), state=state, min_len=None, max_len=None, n_out=None, seam_max=loop_mod.SEAM_RATIO_MAX, name=item, report_path=item_dir / "loop.report.json", anchor=anchor, body_height=body_height,
+ cycle_mode="pinned" if state in PINNED_LOOP_STATES else "auto")
result["loop"] = {"kind": lp["cycle"].get("kind", "periodic"), "cycle": lp["cycle"]["length"], "period": lp["cycle"]["period_global"], "cycle_ratio": round(lp["cycle"]["ratio"], 3), "seam_ratio": lp["resampled_seam_ratio"], "n_out": lp["n_out"], "drift_px": lp["strip"].get("drift_px", 0), "gif": lp["gif"]["file"], "webp": lp["webp"]["file"], "strip": lp["strip"]["path"]}
result["loop"]["review_recommended"] = lp["cycle"].get("review_recommended", False)
result["loop"]["half_period_guard"] = lp["cycle"].get("half_period_guard")
diff --git a/sprite_gen/video/loop.py b/sprite_gen/video/loop.py
index 8159ba2..f98d84e 100644
--- a/sprite_gen/video/loop.py
+++ b/sprite_gen/video/loop.py
@@ -58,7 +58,11 @@ ONE_SHOT_MIN_MOVED = 0.4
ONE_SHOT_MIN_COHERENCE = 3.0 # departure must exceed three ordinary playback steps
ONE_SHOT_MIN_ACTIVE = 4 # isolated keying/jitter spikes are not a performed action
ONE_SHOT_PAD = 2 # one-shot: rest frames kept on each side of the excursion so the seam is rest -> rest
-CYCLE_MODES = ("auto", "periodic", "one-shot", "fixed")
+CYCLE_MODES = ("auto", "periodic", "one-shot", "fixed", "pinned")
+# --cycle pinned: the clip was asked to end on its first frame (first-last mode). A re-rendered
+# first frame is never byte-identical to its source; on the analysis thumbnail (mean absolute
+# RGBA difference, 0..1) it lands within this of it, well under one frame of visible motion.
+PIN_NOISE_MAX = 0.005
ONE_SHOT_MIN_LEN = 4 # a one-shot's length is the clip's own fact; only a degenerate cut is refused
# Gait half-period guard. A walk cycle is two steps; when the two halves look alike in
# pixels (legs hidden by a costume, near/far leg not distinguishable) the half period dips
@@ -422,6 +426,27 @@ def fixed_cycle(D: np.ndarray, *, start: int, length: int) -> dict[str, Any]:
"ratio": seam / inner if inner > 0 else math.inf, "period_global": None, "periodicity": None}
+def pinned_cycle(D: np.ndarray, *, seam_max: float) -> dict[str, Any]:
+ """The whole clip as one cycle, for a clip pinned to end on its first frame.
+
+ Every frame but the last is kept: the last re-renders the first, so playing it too would show
+ that pose twice at the wrap. The wrap (second-to-last -> first) then plays like the step into
+ that last frame, and closes when the last frame really is the first again — so the gate is
+ the pin error, allowed the same `seam_max` steps as any seam or the re-render noise, whichever
+ is larger. A near-still clip moves so little per frame that the seam ratio alone reads its
+ re-render noise as a jump.
+ """
+ n = D.shape[0]
+ length = n - 1
+ adjacent = np.array([D[i, i + 1] for i in range(length - 1)])
+ inner = float(adjacent.mean())
+ seam = float(D[length - 1, 0])
+ pin_error = float(D[n - 1, 0])
+ return {"kind": "pinned", "start": 0, "length": length, "seam": seam, "inner_mean_adjacent": inner,
+ "ratio": seam / inner if inner > 0 else math.inf, "pin_error": pin_error,
+ "pin_tolerance": max(seam_max * inner, PIN_NOISE_MAX), "period_global": None, "periodicity": None}
+
+
def _drop_specks(image: Image.Image, min_fraction: float) -> tuple[Image.Image, int]:
"""Erase detached alpha components smaller than `min_fraction` of the largest one."""
a = np.asarray(image)[..., 3] > 16
@@ -816,6 +841,8 @@ def run_loop(
cycle = fixed_cycle(D, start=start, length=length)
elif cycle_mode == "one-shot":
cycle = detect_one_shot(D, min_len=lo, max_len=hi, frame_mass=masses)
+ elif cycle_mode == "pinned":
+ cycle = pinned_cycle(D, seam_max=seam_max)
else:
gait_floor = round(prof.min_seconds * fps) if prof.gait and prof.min_seconds > 0 else None
cycle = detect_cycle(D, min_len=lo, max_len=hi, gait_floor=gait_floor)
@@ -926,7 +953,19 @@ def run_loop(
delay_ms = max(20, round(1000 * cycle_seconds / n_out))
gif_path = out_dir / f"{name}.gif"
webp_path = out_dir / f"{name}.webp"
- if seam_ratio > seam_max:
+ if cycle["kind"] == "pinned":
+ if cycle["pin_error"] > cycle["pin_tolerance"]:
+ error = (
+ f"video-loop: the pinned clip does not end on its first frame (pin error {cycle['pin_error']:.4f} "
+ f"exceeds {cycle['pin_tolerance']:.4f}); the loop does not close — regenerate the clip with "
+ "--last-frame set to its first frame"
+ )
+ write_loop_report(target, {**report_base, "status": "failed", "error": error, "cycle": cycle,
+ "resampled_seam": resampled_seam,
+ "resampled_inner_mean_adjacent": resampled_adjacent,
+ "resampled_seam_ratio": seam_ratio})
+ raise SystemExit(error)
+ elif seam_ratio > seam_max:
error = (
f"video-loop: loop seam ratio {seam_ratio:.2f} exceeds {seam_max} "
f"({cycle['kind']} length {L} frames from {i}); the cycle does not close — "
@@ -987,7 +1026,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--n-out", type=int, help="frames in the GIF/WebP (default: cycle seconds x --gif-fps)")
parser.add_argument("--gif-fps", type=float, default=GIF_FPS_DEFAULT, help=f"GIF/WebP playback density (default {GIF_FPS_DEFAULT:g}); every state plays at this rate regardless of cycle length")
parser.add_argument("--seam-max", type=float, default=SEAM_RATIO_MAX, help=f"loop seam gate (default {SEAM_RATIO_MAX})")
- parser.add_argument("--cycle", choices=CYCLE_MODES, default="auto", help="auto: periodic first, one-shot failover for action states (recorded in the report); periodic / one-shot force one detector; fixed cuts exactly --start/--length (no detection, reported as kind=fixed)")
+ parser.add_argument("--cycle", choices=CYCLE_MODES, default="auto", help="auto: periodic first, one-shot failover for action states (recorded in the report); periodic / one-shot force one detector; fixed cuts exactly --start/--length (no detection, reported as kind=fixed); pinned: the whole clip but its last frame, for a clip pinned to end on its first frame (the gate is that pin)")
parser.add_argument("--start", type=int, help="fixed cut: first keyed frame of the cycle (with --cycle fixed)")
parser.add_argument("--length", type=int, help="fixed cut: cycle length in frames (with --cycle fixed)")
parser.add_argument("--strip-height", type=int, default=STRIP_MAX_HEIGHT, help=f"cell/strip/GIF height cap in px (default {STRIP_MAX_HEIGHT}); the cycle is scaled down to fit, and never up unless --body-height asks for it")
diff --git a/tests/video/test_idle_still.py b/tests/video/test_idle_still.py
new file mode 100644
index 0000000..b1acd38
--- /dev/null
+++ b/tests/video/test_idle_still.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Idle holds its feet and is pinned: the clip ends on its first frame and `video-loop --cycle
+pinned` keeps the whole clip as the loop, gated on that pin rather than on the seam ratio."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from sprite_gen.video import batch as batch_mod
+from sprite_gen.video import loop as loop_mod
+
+
+def test_the_idle_prompt_holds_the_feet_and_asks_for_the_return_not_a_rhythm() -> None:
+ prompt = batch_mod.build_prompt("side", "idle", None)
+ assert "both feet planted flat on the ground" in prompt
+ assert "no walking, no marching in place" in prompt
+ assert "weight sway" not in prompt
+ assert "evenly paced" not in prompt
+ assert prompt.endswith("The last frame returns to the exact pose of the first frame, so the animation loops seamlessly.")
+ # the rest of the frame rules are the ordinary ones, not the attack's
+ assert "anything it holds" not in prompt
+ assert prompt.replace(batch_mod.MOTION_TEXT["idle"], "{m}") == batch_mod.PINNED_LOOP_TEXT.format(
+ motion="{m}", view=batch_mod.VIEW_TEXT["side"].format(facing="right"))
+
+
+def test_idle_is_pinned_and_its_whole_clip_is_the_loop() -> None:
+ assert "idle" in batch_mod.PIN_LAST_FRAME_STATES
+ assert batch_mod.PINNED_LOOP_STATES == frozenset({"idle"})
+ # an attack is pinned too, but cut as a one-shot with its own template
+ assert "attack" not in batch_mod.PINNED_LOOP_STATES
+ assert "Crisp, clean frames" in batch_mod.build_prompt("side", "attack", None)
+ # the other states keep the evenly paced loop
+ assert "evenly paced" in batch_mod.build_prompt("side", "walk", None)
+
+
+def _still_frames(tmp_path: Path, *, n: int = 48, breath: int = 1, end_offset: int = 0) -> Path:
+ """A standing body that breathes by `breath` px; the last frame is the first moved by `end_offset`."""
+ d = tmp_path / "keyed"
+ d.mkdir()
+ for t in range(n):
+ im = Image.new("RGBA", (64, 96), (0, 0, 0, 0))
+ lift = round(breath * (1 - np.cos(2 * np.pi * t / (n - 1))) / 2)
+ dx = end_offset if t == n - 1 else 0
+ for y in range(20 - lift, 90):
+ for x in range(24 + dx, 40 + dx):
+ im.putpixel((x, y), (180, 60, 60, 255))
+ # a real clip never repeats a frame byte for byte; GIF writers merge frames that do
+ im.putpixel((26 + t % 12, 40 + t // 12), (90, 200, 90, 255))
+ im.save(d / f"frame-{t:04d}.png")
+ return d
+
+
+def _loop(tmp_path: Path, keyed: Path, name: str = "idle") -> dict:
+ return loop_mod.run_loop(keyed, tmp_path / "out", fps=24.0, state="idle", min_len=None, max_len=None, n_out=None,
+ seam_max=loop_mod.SEAM_RATIO_MAX, name=name, report_path=None, cycle_mode="pinned")
+
+
+def test_a_pinned_still_clip_is_one_whole_loop(tmp_path: Path) -> None:
+ rep = _loop(tmp_path, _still_frames(tmp_path))
+ cycle = rep["cycle"]
+ assert (cycle["kind"], cycle["start"], cycle["length"]) == ("pinned", 0, 47) # every frame but the re-rendered first
+ assert cycle["pin_error"] <= cycle["pin_tolerance"]
+ assert rep["strip"]["kind"] == "pinned"
+
+
+def test_a_near_still_clip_is_not_refused_for_its_seam_ratio(tmp_path: Path) -> None:
+ """Breathing moves almost nothing per frame, so the ratio of the wrap to one step can be large
+ while the wrap itself is invisible; the pin decides, with a re-render noise floor."""
+ frames = _still_frames(tmp_path, breath=1)
+ D = loop_mod.distance_matrix(sorted(frames.glob("*.png")))
+ cycle = loop_mod.pinned_cycle(D, seam_max=loop_mod.SEAM_RATIO_MAX)
+ assert cycle["pin_tolerance"] == max(loop_mod.SEAM_RATIO_MAX * cycle["inner_mean_adjacent"], loop_mod.PIN_NOISE_MAX)
+ assert cycle["pin_error"] <= cycle["pin_tolerance"]
+
+
+def test_a_clip_that_does_not_come_back_to_its_first_frame_is_refused(tmp_path: Path) -> None:
+ keyed = _still_frames(tmp_path, end_offset=10)
+ with pytest.raises(SystemExit, match="the pinned clip does not end on its first frame"):
+ _loop(tmp_path, keyed)
+ report = json.loads((tmp_path / "out" / "idle.loop.report.json").read_text())
+ assert report["status"] == "failed" and report["cycle"]["kind"] == "pinned"
+
+
+def test_the_cli_takes_pinned() -> None:
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ loop_mod.add_arguments(parser)
+ assert parser.parse_args(["--frames-dir", "k", "--out-dir", "o", "--fps", "24", "--cycle", "pinned"]).cycle == "pinned"
+
+
+def test_video_set_pins_idle_and_cuts_it_whole(tmp_path: Path, monkeypatch) -> None:
+ seen: dict[str, object] = {}
+
+ def fake_canvas(base, out, *, state, shape, facing, headroom, lead, report_path, fit="state"):
+ Image.new("RGB", (32, 32), (0, 255, 0)).save(out)
+ return {"fit": fit, "shape": "square", "canvas": [32, 32], "offset": [0, 0]}
+
+ def fake_frames(clip, out_dir, **kw):
+ keyed = Path(out_dir) / "keyed"
+ keyed.mkdir(parents=True, exist_ok=True)
+ return {"fps": 24.0, "frames": 48, "alpha_zero_pct_min": 0.0, "alpha_zero_pct_max": 0.0, "keyed_dir": str(keyed), "spill": {"mode": "small"}}
+
+ def fake_loop(frames_dir, out_dir, **kw):
+ seen["cycle_mode"] = kw.get("cycle_mode")
+ return {"cycle": {"kind": "pinned", "length": 47, "period_global": None, "ratio": 1.0}, "resampled_seam_ratio": 1.0, "n_out": 47,
+ "gif": {"file": "g"}, "webp": {"file": "w"}, "strip": {"path": "s", "drift_px": 0}}
+
+ def video(image, prompt, out, report, **kw):
+ seen["last_frame"] = kw.get("last_frame")
+ seen["prompt"] = prompt
+ Path(out).write_bytes(b"x")
+ Path(report).write_text("{}")
+ Path(kw["log"]).write_text("")
+ return 0
+
+ monkeypatch.setattr(batch_mod.facing_mod.vision, "grok_inspect", lambda *a, **kw: ("right", {}))
+ monkeypatch.setattr(batch_mod.canvas_mod, "run_canvas", fake_canvas)
+ monkeypatch.setattr(batch_mod.frames_mod, "run_frames", fake_frames)
+ monkeypatch.setattr(batch_mod.loop_mod, "run_loop", fake_loop)
+ base = tmp_path / "base.png"
+ Image.new("RGB", (32, 32), (0, 255, 0)).save(base)
+ payload = batch_mod.run_set(bases={"side": base}, states=["idle"], root=tmp_path / "set", character=None, duration=None,
+ resolution="480p", key="green", concurrency=1, force=False, gap=0.0, video_runner=video)
+ assert payload["ok"] == 1
+ assert seen["cycle_mode"] == "pinned"
+ assert Path(str(seen["last_frame"])).name == "canvas.png"
+ assert seen["prompt"] == batch_mod.build_prompt("side", "idle", None)