Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

perf(video-loop): motion-auto computes its masked NCC with FFTs

Soohong Kim3c42d37bc71e
diff --git a/docs/loop-review.md b/docs/loop-review.md
index 66447b0..fd44f37 100644
--- a/docs/loop-review.md
+++ b/docs/loop-review.md
@@ -73,6 +73,13 @@ textured regions. It ranks their normalized cross-correlation across time and
requires both to track throughout the sequence. These are stable image regions,
not semantic detections of a face, torso, species or limb. Untrackable patches are
counted as rejected candidates; absence of a usable pair is an explicit failure.
+The correlation at every offset of a search window comes out of FFTs: per channel,
+the masked patch sum, its squared sum and its product with the centered reference are
+window correlations, so the masked patch is never copied out once per offset. The
+costs agree with that direct form to about 1e-6, and two offsets that tie within it
+may swap. On three 3 s walk clips at 544 px the region search took 1.0 to 1.4 s of
+CPU instead of 22.6 to 37.4 s, and full loops cut from the same frames gave the same
+cycle and byte-identical strips.
For selection only, the measured horizontal trajectory is locally fitted and the
vertical linear trend is removed. Local lag minima can then identify a repeat in
diff --git a/sprite_gen/video/auto_motion.py b/sprite_gen/video/auto_motion.py
index 116d192..b47e2af 100644
--- a/sprite_gen/video/auto_motion.py
+++ b/sprite_gen/video/auto_motion.py
@@ -60,8 +60,23 @@ def _reference(a, box):
return box, mask, centered, norm
+def _correlate(region, kernel, rows, cols):
+ """Valid-window cross-correlation of one 2-D channel with a 2-D kernel through one FFT pair.
+
+ The transform is the size of the region, so the circular product only wraps for offsets
+ past the last valid window; those are cut off and no wrapped pixel reaches a result.
+ """
+ spectrum = np.fft.rfft2(region) * np.conj(np.fft.rfft2(kernel, s=region.shape))
+ return np.fft.irfft2(spectrum, s=region.shape)[:rows, :cols]
+
+
def _match(reference, moving, center, radius):
- """Bounded masked NCC; vectorized across translations, never wrap pixels."""
+ """Bounded masked NCC at every translation, never wrap pixels.
+
+ Per channel, the masked patch sum, squared sum and product with the centered reference
+ are window correlations, so all offsets come out of a few FFTs instead of one gathered
+ copy of the masked patch per offset (that copy was most of a walk loop's time).
+ """
(x, y, right, bottom), mask, feature, norm = reference
cx, cy = center
height, width = mask.shape
@@ -69,12 +84,23 @@ def _match(reference, moving, center, radius):
ys, ye = max(0, y-cy-radius), min(moving.shape[0]-height, y-cy+radius)
if xs > xe or ys > ye:
raise ValueError("automatic motion anchor has no search overlap")
- windows = np.lib.stride_tricks.sliding_window_view(
- moving[ys:ye+height, xs:xe+width], (height, width), axis=(0, 1))
- patches = windows.transpose(0, 1, 3, 4, 2)[:, :, mask, :]
- centered = patches - patches.mean(axis=2, keepdims=True)
- denominator = np.sqrt((centered * centered).sum(axis=(2, 3))) * norm
- costs = 1 - (centered * feature).sum(axis=(2, 3)) / np.maximum(1e-10, denominator)
+ region = moving[ys:ye+height, xs:xe+width].astype(np.float64)
+ rows, cols = ye-ys+1, xe-xs+1
+ weights = mask.astype(np.float64)
+ count = weights.sum()
+ kernel = np.zeros((height, width, region.shape[2]))
+ kernel[mask] = feature
+ product = np.zeros((rows, cols))
+ energy = np.zeros((rows, cols))
+ for channel in range(region.shape[2]):
+ values = region[:, :, channel]
+ total = _correlate(values, weights, rows, cols)
+ # sum((p - mean p) * f) = sum(p * f) - mean p * sum(f); the reference was centered in
+ # float32, so its sum is near zero but not zero, and is kept rather than assumed away.
+ product += _correlate(values, kernel[:, :, channel], rows, cols) - total * kernel[:, :, channel].sum() / count
+ energy += _correlate(values * values, weights, rows, cols) - total * total / count
+ denominator = np.sqrt(np.maximum(energy, 0.0)) * norm
+ costs = 1 - product / np.maximum(1e-10, denominator)
iy, ix = np.unravel_index(costs.argmin(), costs.shape)
dx, dy = x-(xs+int(ix)), y-(ys+int(iy))
if abs(dx-cx) == radius or abs(dy-cy) == radius:
diff --git a/tests/video/test_auto_motion.py b/tests/video/test_auto_motion.py
index 471e653..56f3a1d 100644
--- a/tests/video/test_auto_motion.py
+++ b/tests/video/test_auto_motion.py
@@ -50,6 +50,51 @@ def test_symmetric_steps_keep_two_phase_occurrences_and_respect_window():
periodicity_min=.15, double_tolerance=.25, double_search=3)
+def gathered_match(reference, moving, center, radius):
+ """The masked NCC written out directly: one gathered masked patch per offset."""
+ (x, y, _, _), mask, feature, norm = reference
+ cx, cy = center
+ height, width = mask.shape
+ xs, xe = max(0, x-cx-radius), min(moving.shape[1]-width, x-cx+radius)
+ ys, ye = max(0, y-cy-radius), min(moving.shape[0]-height, y-cy+radius)
+ windows = np.lib.stride_tricks.sliding_window_view(
+ moving[ys:ye+height, xs:xe+width], (height, width), axis=(0, 1))
+ patches = windows.transpose(0, 1, 3, 4, 2)[:, :, mask, :]
+ centered = patches - patches.mean(axis=2, keepdims=True)
+ denominator = np.sqrt((centered * centered).sum(axis=(2, 3))) * norm
+ costs = 1 - (centered * feature).sum(axis=(2, 3)) / np.maximum(1e-10, denominator)
+ return {(x-(xs+ix), y-(ys+iy)): float(costs[iy, ix]) for iy in range(costs.shape[0]) for ix in range(costs.shape[1])}
+
+
+def test_match_equals_the_gathered_masked_ncc():
+ rng = np.random.default_rng(7)
+ unique = tied = 0
+ for k in range(0, 40, 3):
+ a = auto_motion.motion_anchor._features(walker(0))
+ moving = auto_motion.motion_anchor._features(walker(k))
+ for box in [(48, 30, 70, 50), (44, 60, 72, 84), (50, 100, 66, 126)]:
+ reference = auto_motion._reference(a, box)
+ if reference is None:
+ continue
+ center = tuple(int(v) for v in rng.integers(-3, 4, size=2) + np.array([-k, -round(k*.2)]))
+ try:
+ got = auto_motion._match(reference, moving, center, 12)
+ except ValueError:
+ continue
+ expected = gathered_match(reference, moving, center, 12)
+ best = min(expected.values())
+ # The flat synthetic torso matches itself at several vertical shifts; float noise picks
+ # among an exact tie in either implementation, so any member of the tie is right.
+ ties = [offset for offset, cost in expected.items() if cost <= best + 1e-5]
+ assert tuple(got[0].tolist()) in ties
+ assert abs(got[1]-expected[tuple(got[0].tolist())]) < 1e-6
+ if len(ties) == 1:
+ unique += 1
+ else:
+ tied += 1
+ assert unique >= 12 and tied >= 1
+
+
def test_region_discovery_refuses_blank_input():
with pytest.raises(ValueError, match='visible foreground'):
auto_motion.discover([Image.new('RGBA', (100, 100)) for _ in range(6)], reference_index=0)