Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

fix(frames): key chroma from the painted background colour, not only the pure key

Soohong Kimbfde97a3b382
diff --git a/sprite_gen/frames/cutout.py b/sprite_gen/frames/cutout.py
index 0b50b88..01e2bc5 100644
--- a/sprite_gen/frames/cutout.py
+++ b/sprite_gen/frames/cutout.py
@@ -112,15 +112,16 @@ def _corner_average(image: Image.Image) -> tuple[int, int, int]:
def _detect_key_kind(color: tuple[int, int, int]) -> str:
"""Classify a corner background colour into a route: magenta | green | white.
- Saturated magenta/green keys route to the extract engine; everything else
- (bright achromatic ivory/white, or any low-saturation background) routes to
- the position-based matte.
+ Saturated magenta/green keys — at whatever brightness the model painted
+ them (`extract.is_key_family`, the same rule the engine keys with) — route
+ to the extract engine; everything else (bright achromatic ivory/white, or
+ any low-saturation background) routes to the position-based matte.
"""
- r, g, b = color
- if r >= 128 and b >= 128 and g <= 100 and (r - g) >= 60 and (b - g) >= 60:
- return "magenta"
- if g >= 128 and r <= 100 and b <= 100 and (g - r) >= 60 and (g - b) >= 60:
- return "green"
+ from sprite_gen.frames.extract import is_key_family
+
+ for kind, target in KEY_TARGETS.items():
+ if is_key_family(color, target):
+ return kind
return "white"
@@ -270,14 +271,25 @@ def _matte_route(
def _extract_route(image: Image.Image, kind: str) -> tuple[Image.Image, dict[str, Any]]:
- """Magenta/green key background → reuse the verified `extract` chroma engine (no drift)."""
- from sprite_gen.frames.extract import remove_chroma_background
+ """Magenta/green key background → reuse the verified `extract` chroma engine (no drift).
+
+ The engine keys from the background colour it detects on the borders
+ (`detect_background_key_rgb`) as well as the pure key, so the brightness
+ the model happened to paint does not decide whether the cut lands. The
+ detected colour is reported as `chroma_key_painted` for the audit trail.
+ """
+ from sprite_gen.frames.extract import detect_background_key_rgb, remove_chroma_background
target = KEY_TARGETS[kind]
+ painted = detect_background_key_rgb(image, target)
result = remove_chroma_background(
image, target, _EXTRACT_KEY_THRESHOLD, _EXTRACT_FRINGE_THRESHOLD, _EXTRACT_FRINGE_DELTA
)
- return result.convert("RGBA"), {"route": f"extract:{kind}", "chroma_key": list(target)}
+ return result.convert("RGBA"), {
+ "route": f"extract:{kind}",
+ "chroma_key": list(target),
+ "chroma_key_painted": list(painted),
+ }
def cutout(
diff --git a/sprite_gen/frames/extract.py b/sprite_gen/frames/extract.py
index 93e887c..3606aba 100644
--- a/sprite_gen/frames/extract.py
+++ b/sprite_gen/frames/extract.py
@@ -169,6 +169,115 @@ def _grow_chebyshev(mask: np.ndarray) -> np.ndarray:
return spread
+# --- background key detection (RGB path) -------------------------------------
+# Image models asked for #00FF00 / #FF00FF hand back a slightly different flat
+# colour every run (measured 2026-09-11: Grok painted (8, 162, 24) for green).
+# Its absolute distance to the pure key was 96.38 — a hair over the 96 hard-cut
+# radius — so the whole background survived, while (7, 163, 24) at 95.34 keyed
+# out. The rule below reads the *actual* background from the frame borders and,
+# when that colour is the declared key's hue family, measures the hard cut from
+# it as well. Raising the radius instead would only move the cliff to the next
+# darker green; this removes the cliff for any brightness the model picks.
+_KEY_FAMILY_MIN_KEYED = 64 # a keyed channel must at least read as lit (same bar as _key_channel_split's dark side)
+_KEY_FAMILY_MAX_UNKEYED_RATIO = 0.35 # unkeyed channels stay below this fraction of the dimmest keyed one
+_KEY_FAMILY_KEYED_BALANCE = 0.8 # for two-channel keys (magenta) the dimmer keyed channel keeps this share of the brighter
+_KEY_DETECT_CORNER_DIV = 5 # corner patch = width/5 x height/5 (same sampling as the YCbCr path)
+_KEY_DETECT_MIN_FRACTION = 0.12 # family share of the samples needed to trust the detection
+_KEY_DETECT_BIN_SHIFT = 3 # RGB histogram bin width 8 — the mode bin, never a mean, picks the colour
+
+
+def is_key_family(color: tuple[int, int, int], chroma_key: tuple[int, int, int]) -> bool:
+ """True when `color` is the key's hue at any brightness — a *variant of the key*.
+
+ Brightness is free; saturation and hue are not. Every channel the key
+ saturates must read as lit (>= 64), the channels the key leaves dark must
+ stay under 35% of the dimmest keyed channel, and a two-channel key keeps
+ its keyed channels within 20% of each other. (8, 162, 24) and (170, 8, 180)
+ are family; hot pink (250, 77, 150), purple (213, 112, 246) and cyan are
+ subject colours, not keys. Degenerate keys have no family.
+ """
+ keyed_channels, unkeyed_channels = _key_channel_split(chroma_key)
+ if not keyed_channels:
+ return False
+ keyed_min = min(color[index] for index in keyed_channels)
+ keyed_max = max(color[index] for index in keyed_channels)
+ unkeyed_max = max(color[index] for index in unkeyed_channels)
+ return (
+ keyed_min >= _KEY_FAMILY_MIN_KEYED
+ and unkeyed_max <= keyed_min * _KEY_FAMILY_MAX_UNKEYED_RATIO
+ and keyed_min >= keyed_max * _KEY_FAMILY_KEYED_BALANCE
+ )
+
+
+def _key_detect_sample_mask(height: int, width: int) -> np.ndarray:
+ """Corner patches (w/5 x h/5) plus the 1-px border — where the background lives."""
+ mask = np.zeros((height, width), dtype=bool)
+ if height == 0 or width == 0:
+ return mask
+ corner_w = width // _KEY_DETECT_CORNER_DIV
+ corner_h = height // _KEY_DETECT_CORNER_DIV
+ if corner_w < 2:
+ corner_w = width
+ if corner_h < 2:
+ corner_h = height
+ mask[:corner_h, :corner_w] = True
+ mask[:corner_h, width - corner_w:] = True
+ mask[height - corner_h:, :corner_w] = True
+ mask[height - corner_h:, width - corner_w:] = True
+ mask[0, :] = mask[-1, :] = True
+ mask[:, 0] = mask[:, -1] = True
+ return mask
+
+
+def detect_background_key_rgb(
+ image: Image.Image, chroma_key: tuple[int, int, int]
+) -> tuple[int, int, int]:
+ """The flat background colour as the model actually painted it, or the declared key.
+
+ Samples the corner patches and the border, keeps the opaque pixels that are
+ the declared key's hue family (`is_key_family`), and — when they make up at
+ least 12% of the samples — returns the mean of the most populated 8-wide
+ RGB histogram bin among them. The mode, never the mean of everything: a
+ handful of antialiased fringe pixels on the border must not drag an
+ exact-key background off the key, and a two-cluster border must resolve to
+ the dominant cluster, not a midpoint that mattes neither. Otherwise — no
+ such background, the subject crowds every border, a degenerate key — the
+ declared key comes back unchanged, which keeps `remove_chroma_background`
+ byte-identical to its pre-detection behaviour on exact-key backgrounds.
+ """
+ keyed_channels, unkeyed_channels = _key_channel_split(chroma_key)
+ if not keyed_channels:
+ return tuple(chroma_key) # type: ignore[return-value]
+ rgba = image if image.mode == "RGBA" else image.convert("RGBA")
+ data = np.array(rgba, dtype=np.uint8)
+ height, width = data.shape[:2]
+ sample = _key_detect_sample_mask(height, width) & (data[..., 3] != 0)
+ if not sample.any():
+ return tuple(chroma_key) # type: ignore[return-value]
+ rgb = data[..., :3].astype(np.int32)
+ # Vector form of `is_key_family` — same three inequalities, same constants.
+ keyed_min = rgb[..., keyed_channels].min(axis=-1)
+ keyed_max = rgb[..., keyed_channels].max(axis=-1)
+ unkeyed_max = rgb[..., unkeyed_channels].max(axis=-1)
+ family = (
+ sample
+ & (keyed_min >= _KEY_FAMILY_MIN_KEYED)
+ & (unkeyed_max <= keyed_min * _KEY_FAMILY_MAX_UNKEYED_RATIO)
+ & (keyed_min >= keyed_max * _KEY_FAMILY_KEYED_BALANCE)
+ )
+ family_count = int(np.count_nonzero(family))
+ if family_count < int(np.count_nonzero(sample)) * _KEY_DETECT_MIN_FRACTION:
+ return tuple(chroma_key) # type: ignore[return-value]
+ colors = rgb[family] # (N, 3)
+ bins = (colors >> _KEY_DETECT_BIN_SHIFT).astype(np.int64)
+ slots = (bins[:, 0] << 16) | (bins[:, 1] << 8) | bins[:, 2]
+ unique, inverse, counts = np.unique(slots, return_inverse=True, return_counts=True)
+ mode = int(np.argmax(counts)) # first max = lowest slot on a tie: deterministic
+ members = colors[inverse == mode]
+ mean = members.sum(axis=0, dtype=np.int64) // len(members)
+ return (int(mean[0]), int(mean[1]), int(mean[2]))
+
+
# remove_chroma_background pixel classes, decided once on the source colors.
_KEYED = 0 # erased: transparent input or hard key cut
_SUBJECT = 1 # not key-tinted — never touched
@@ -192,7 +301,15 @@ def remove_chroma_background(
*,
unmix_reach: int = 4,
spill_max_fraction: float = 0.005,
+ background_key: tuple[int, int, int] | None = None,
) -> Image.Image:
+ """Key `chroma_key` out of `image` (hard cut + soft-alpha fringe unmix + trapped-spill despill).
+
+ `background_key`: the background colour as actually painted. None (the
+ default) detects it from the borders with `detect_background_key_rgb`;
+ passing `chroma_key` itself pins the single-key behaviour (the frozen
+ byte-identity gate does that).
+ """
rgba = image.convert("RGBA")
width, height = rgba.size
data = np.array(rgba, dtype=np.uint8) # (H, W, 4); written back at the end
@@ -200,12 +317,27 @@ def remove_chroma_background(
keyed_channels, unkeyed_channels = _key_channel_split(chroma_key)
unseen = 255
+ # The background as painted. Distances are taken to the declared key *and*
+ # to the detected family colour and the smaller one wins, so an exact-key
+ # background reproduces the single-key result bit for bit while a darker or
+ # paler key of the same hue is cut from where it actually sits. The blend
+ # model (despill / unmix) also mixes against the painted colour — that is
+ # the colour the antialiased fringe was actually blended with. Which
+ # channels count as "keyed" stays a property of the declared key.
+ painted_key = (
+ detect_background_key_rgb(rgba, chroma_key)
+ if background_key is None
+ else (int(background_key[0]), int(background_key[1]), int(background_key[2]))
+ )
+
# Classification, decided on the source colors before anything is erased.
# np.select takes the first condition that holds, so the condlist order below
# *is* the if/elif order it replaces — swapping the subject and in-band rows
# changes the output wherever both hold at once (a wide --fringe-key-threshold
# makes that reachable, and the gate has a case for it).
key_distance = _key_distance_field(source_rgb, chroma_key)
+ if painted_key != tuple(chroma_key):
+ key_distance = np.minimum(key_distance, _key_distance_field(source_rgb, painted_key))
source_tint = _key_tint_field(source_rgb, keyed_channels, unkeyed_channels)
keyed_mask = (data[..., 3] == 0) | (key_distance <= threshold)
classes = np.select(
@@ -218,7 +350,7 @@ def remove_chroma_background(
depths = np.full((height, width), unseen, dtype=np.uint8) # chebyshev distance to keyed region
depths[keyed_mask] = 0
- key_tint = key_tint_score(chroma_key, chroma_key)
+ key_tint = key_tint_score(painted_key, chroma_key)
max_reach = min(unseen - 1, unmix_reach if key_tint > 0 else 0)
# Geometric distance to the nearest keyed-out pixel — outer background
@@ -255,7 +387,7 @@ def remove_chroma_background(
red, green, blue, alpha = (int(value) for value in data[y, x])
color = (red, green, blue)
out_red, out_green, out_blue, out_alpha = unmix_key_blend(
- color, alpha, chroma_key, key_tint, key_tint_score(color, chroma_key)
+ color, alpha, painted_key, key_tint, key_tint_score(color, chroma_key)
)
# The scalar path wrote this through `pixels[x, y] = (...)`, and PIL
# clamped every channel to 0..255 on the way in. A uint8 array does
@@ -323,7 +455,7 @@ def remove_chroma_background(
red, green, blue, alpha = (int(value) for value in data[y, x])
color = (red, green, blue)
coverage, despilled = despill_color(
- color, chroma_key, key_tint, key_tint_score(color, chroma_key)
+ color, painted_key, key_tint, key_tint_score(color, chroma_key)
)
if coverage > 0:
data[y, x] = (*despilled, alpha)
diff --git a/tests/frames/test_chroma_key_relative.py b/tests/frames/test_chroma_key_relative.py
index 5e52ec5..98af3f3 100644
--- a/tests/frames/test_chroma_key_relative.py
+++ b/tests/frames/test_chroma_key_relative.py
@@ -139,3 +139,116 @@ def test_boundary_pair_must_not_split(tmp_path: Path) -> None:
)
outcomes[background] = audit(result, source)
assert outcomes[(7, 163, 24)] == outcomes[(8, 162, 24)] == (0, 0), outcomes
+
+
+# --- the detection layer itself ----------------------------------------------
+# `remove_chroma_background` keys from the smaller of the distance to the
+# declared key and to the background colour it detects on the borders. These
+# pin the detector's contract: what counts as "the key's family", that the mode
+# wins over the mean, and that the opt-out reproduces the single-key result.
+
+from sprite_gen.frames.extract import detect_background_key_rgb, is_key_family # noqa: E402
+
+GREEN = (0, 255, 0)
+MAGENTA = (255, 0, 255)
+HOT_PINK = (250, 77, 150) # ~129 from magenta — subject material the trap-band tests protect
+PURPLE = (213, 112, 246) # ~153 from magenta — same
+
+
+ ("color", "key", "expected"),
+ [
+ ((8, 162, 24), GREEN, True),
+ ((10, 150, 30), GREEN, True),
+ ((20, 120, 25), GREEN, True), # brightness is free
+ ((3, 248, 5), GREEN, True),
+ ((170, 8, 180), MAGENTA, True),
+ ((251, 2, 248), MAGENTA, True),
+ ((0, 255, 255), GREEN, False), # cyan: blue is not dark
+ ((120, 255, 120), GREEN, False), # washed out: unkeyed channels too high
+ ((30, 60, 30), GREEN, False), # too dark to read as a lit key channel
+ (HOT_PINK, MAGENTA, False),
+ (PURPLE, MAGENTA, False),
+ ((255, 0, 128), MAGENTA, False), # keyed channels out of balance
+ ((248, 247, 242), GREEN, False),
+ ((8, 162, 24), (128, 128, 128), False), # degenerate key has no family
+ ],
+)
+def test_is_key_family(color: tuple[int, int, int], key: tuple[int, int, int], expected: bool) -> None:
+ assert is_key_family(color, key) is expected
+
+
+def test_detect_returns_declared_key_when_background_is_exact_despite_border_fringe() -> None:
+ """A few antialiased fringe pixels on the border must not drag the key off — mode, not mean."""
+ img = Image.new("RGBA", (64, 48), GREEN + (255,))
+ px = img.load()
+ for x in range(0, 64, 3): # sparse family-coloured fringe along the top border
+ px[x, 0] = (10, 200, 20, 255)
+ assert detect_background_key_rgb(img, GREEN) == GREEN
+
+
+def test_detect_picks_dominant_cluster_not_midpoint() -> None:
+ """Two family clusters on the border: the dominant one wins outright."""
+ img = Image.new("RGBA", (60, 60), (8, 162, 24, 255))
+ px = img.load()
+ for y in range(60):
+ for x in range(0, 10): # minority stripe of a different green
+ px[x, y] = (5, 230, 10, 255)
+ assert detect_background_key_rgb(img, GREEN) == (8, 162, 24)
+
+
+def test_detect_ignores_family_coloured_subject_when_background_dominates() -> None:
+ """A green-family subject touching the borders does not become the key."""
+ img = Image.new("RGBA", (120, 120), GREEN + (255,))
+ px = img.load()
+ for y in range(0, 120): # subject column touching top and bottom borders
+ for x in range(50, 70):
+ px[x, y] = (20, 140, 30, 255)
+ assert detect_background_key_rgb(img, GREEN) == GREEN
+
+
+def test_detect_returns_declared_key_without_family_background() -> None:
+ img = Image.new("RGBA", (40, 40), (248, 247, 242, 255))
+ assert detect_background_key_rgb(img, GREEN) == GREEN
+ assert detect_background_key_rgb(img, (128, 128, 128)) == (128, 128, 128)
+
+
+def test_detect_skips_transparent_border_pixels() -> None:
+ img = Image.new("RGBA", (40, 40), (0, 0, 0, 0))
+ px = img.load()
+ for y in range(8, 32):
+ for x in range(8, 32):
+ px[x, y] = (8, 162, 24, 255)
+ # Only transparent pixels are sampled → nothing to detect from.
+ assert detect_background_key_rgb(img, GREEN) == GREEN
+
+
+def test_background_key_opt_out_reproduces_single_key_behaviour() -> None:
+ """`background_key=<declared>` pins the pre-detection result (the byte-identity gate relies on it)."""
+ source = make_flat_key_still((8, 162, 24))
+ pinned = remove_chroma_background(
+ source, GREEN, _EXTRACT_KEY_THRESHOLD, _EXTRACT_FRINGE_THRESHOLD, _EXTRACT_FRINGE_DELTA,
+ background_key=GREEN,
+ )
+ assert audit(pinned, source)[0] == SIZE * SIZE - 64 * 64 # every background pixel survives
+
+
+def test_cutout_auto_routes_dark_green_corners_to_extract(tmp_path: Path) -> None:
+ source = make_flat_key_still((8, 162, 24))
+ src_path = tmp_path / "dark-green.png"
+ source.save(src_path)
+ stats = cutout(src_path, tmp_path / "out.png", key="auto")
+ assert stats["route"] == "extract:green"
+ assert stats["chroma_key"] == [0, 255, 0]
+ assert stats["chroma_key_painted"] == [8, 162, 24]
+ assert audit(Image.open(tmp_path / "out.png"), source) == (0, 0)
+
+
+def test_cutout_auto_route_uses_the_engine_family_rule() -> None:
+ """Route detection and the engine share one family rule: key variants → extract, subject colours → matte."""
+ from sprite_gen.frames.cutout import _detect_key_kind
+
+ assert _detect_key_kind((8, 162, 24)) == "green"
+ assert _detect_key_kind((170, 8, 180)) == "magenta"
+ assert _detect_key_kind(HOT_PINK) == "white"
+ assert _detect_key_kind((248, 247, 242)) == "white"
diff --git a/tests/gen/test_chroma_extraction.py b/tests/gen/test_chroma_extraction.py
index 97169b1..70c8145 100644
--- a/tests/gen/test_chroma_extraction.py
+++ b/tests/gen/test_chroma_extraction.py
@@ -96,22 +96,33 @@ def _open_moe(name: str) -> Image.Image:
def _fringe_peel_candidates(image: Image.Image, chroma_key: tuple[int, int, int]) -> list[tuple[int, int]]:
- """Pixels the legacy in-band peel would erase before soft-alpha unmix."""
+ """Pixels the legacy in-band peel would erase before soft-alpha unmix.
+
+ Key distance is what the engine measures: the smaller of the distance to the
+ declared key and to the background colour actually painted on the borders
+ (`detect_background_key_rgb`) — the moe fixtures were painted a few units
+ off the pure key, and the hard-cut sphere sits where the paint is.
+ """
width, height = image.size
pixels = image.load()
+ painted_key = extract.detect_background_key_rgb(image, chroma_key)
classes = bytearray(width * height)
keyed: list[int] = []
+
+ def key_distance(color: tuple[int, int, int]) -> float:
+ return min(extract.color_distance(color, chroma_key), extract.color_distance(color, painted_key))
+
for y in range(height):
for x in range(width):
red, green, blue, alpha = pixels[x, y]
index = y * width + x
color = (red, green, blue)
- if alpha == 0 or extract.color_distance(color, chroma_key) <= KEY_THRESHOLD:
+ if alpha == 0 or key_distance(color) <= KEY_THRESHOLD:
classes[index] = extract._KEYED
keyed.append(index)
elif extract.key_tint_score(color, chroma_key) < FRINGE_DELTA:
classes[index] = extract._SUBJECT
- elif extract.color_distance(color, chroma_key) <= FRINGE_THRESHOLD:
+ elif key_distance(color) <= FRINGE_THRESHOLD:
classes[index] = extract._BLEND_IN_BAND
else:
classes[index] = extract._BLEND_OUT_OF_BAND
diff --git a/tests/gen/test_chroma_rcb_byte_identity.py b/tests/gen/test_chroma_rcb_byte_identity.py
index 22f841e..46d8275 100644
--- a/tests/gen/test_chroma_rcb_byte_identity.py
+++ b/tests/gen/test_chroma_rcb_byte_identity.py
@@ -1138,10 +1138,15 @@ def _run_production(case, image=None):
"""`image` 를 주면 **그 이미지 객체 그대로** 넘긴다 — 입력 불변 단정은 자기가
들고 있는 객체가 들어가야 성립한다 (노드 3: 매번 새로 빌드하면 공허해진다)."""
_, builder, key, threshold, fringe_threshold, fringe_delta, reach, spill = case
+ # `background_key=key` 는 단일키(선언키) 계약을 고정한다. 2026-09-11 이후 엔진은
+ # 기본값(None)에서 보더 검출색(`detect_background_key_rgb`)까지 기준으로 키잉하므로,
+ # 여기 픽스처(오프키 노이즈 배경·moe 실사)에서는 기본값이 출력을 바꾼다. 이 게이트가
+ # 잠그는 것은 벡터화 == 동결 스칼라 이고, 검출 층은 tests/frames/test_chroma_key_relative.py
+ # 가 따로 잠근다.
return extract.remove_chroma_background(
builder() if image is None else image,
key, threshold, fringe_threshold, fringe_delta,
- unmix_reach=reach, spill_max_fraction=spill,
+ unmix_reach=reach, spill_max_fraction=spill, background_key=key,
)