Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

14.1 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
"""The measured background matte: a flat background of any colour, keyed from what the image painted.

The chroma engine (`extract.remove_chroma_background`) keys magenta and green backgrounds it
recognises as a key (`extract.is_border_key_candidate`), and the white matte (`cutout._matte_route`)
keys bright achromatic ones. A model asked for a key background often paints neither: a muted
yellow-green, an off-cyan, a grey, with noise, a gradient, a darker same-colour shadow round the
subject and thin dark strips along the edges. This matte keys those from a measurement of the image
itself, so no colour needs a route of its own (docs/chroma-alpha.md "The measured background").

1. Edge strips: thin lines along an edge in one colour that is not the background (a letterbox bar).
2. The background colour and its spread, read on a ring inside the strips (`measure_background`).
3. Background-like pixels: within the spread of the colour, or, on a saturated background, its hue
   in shade (`background_like`), so a drop shadow and a gradient are background too.
4. What is keyed: the background-like pixels connected to the edges, the strips, and, on a
   saturated background, enclosed pockets of it thick enough not to be a speck of the subject.
5. The edge: within `band` px of the keyed set, each pixel is unmixed between the local background
   and the nearest solid subject colour, giving partial alpha and the subject's own colour back.

Numpy only, no per-pixel Python loop, so a clip's frames key at the speed of a still.
"""

from __future__ import annotations

from typing import Any

from PIL import Image, ImageFilter

from sprite_gen._deps import np

STRIP_MAX_FRACTION = 0.03  # an edge strip is at most this share of the side it runs across (min 2 px)
STRIP_LINE_SHARE = 0.6  # a strip line: this share of it is one colour
RING_WIDTH = 8  # px of the measuring ring, inside the deepest a strip may go
MODE_BIN = 8  # RGB histogram bin of the background mode
INLIER_TOLERANCE = 24  # max-channel distance from the mode that counts as the background when measuring it
MIN_BACKGROUND_SHARE = 0.2  # the background holds at least this share of the ring
TOLERANCE_FLOOR = 26  # max-channel distance from the colour that is background (the white matte's 26)
SPREAD_SCALE = 4.0  # ... or this many spreads of the measured background, if wider
SATURATED_CHROMA = 40  # max - min channel at which a background's colour is its own (shade is then read)
SHADE_HUE = 30.0  # degrees of hue from the background's that are its shade (a model shades by hue, not by scale)
SHADE_SATURATION = 0.5  # ... at this share of its saturation or more
SHADE_CHROMA = (20.0, 0.15)  # ... and at least max(this, this share of its chroma): near-black lineart is not shade
HOLE_THICKNESS = 2  # an enclosed pocket survives this many erosions to be keyed (5 px across)
EDGE_BAND = 2  # px of edge unmixed outside the keyed set
ALPHA_FLOOR = 4  # alpha below this (of 255) is keyed outright
FULL_BACKGROUND_SHARE = 0.999  # keyed share at which nothing of a subject is left


def _shift_or(mask: np.ndarray) -> np.ndarray:
    """The mask grown by one px in the four directions."""
    grown = mask.copy()
    grown[1:] |= mask[:-1]
    grown[:-1] |= mask[1:]
    grown[:, 1:] |= mask[:, :-1]
    grown[:, :-1] |= mask[:, 1:]
    return grown


def _reconstruct(seed: np.ndarray, allowed: np.ndarray) -> np.ndarray:
    """The pixels of `allowed` connected (4-way) to `seed` inside it: geodesic reconstruction."""
    mask = seed & allowed
    while True:
        before = mask
        for _ in range(16):
            mask = _shift_or(mask) & allowed
        if np.array_equal(mask, before):
            return mask


def _erode(mask: np.ndarray, times: int) -> np.ndarray:
    for _ in range(times):
        inner = mask.copy()
        inner[1:] &= mask[:-1]
        inner[:-1] &= mask[1:]
        inner[:, 1:] &= mask[:, :-1]
        inner[:, :-1] &= mask[:, 1:]
        mask = inner
    return mask


def _filled_mean(rgb: np.ndarray, weight: np.ndarray, radii: tuple[int, ...]) -> tuple[np.ndarray, np.ndarray]:
    """Per pixel the mean colour of the weighted pixels nearby, at the smallest radius that has any.

    Returns (colour, found): `found` is False where no radius reached a weighted pixel."""
    height, width = weight.shape
    colour = np.zeros((height, width, 3), dtype=np.float32)
    found = np.zeros((height, width), dtype=bool)
    w8 = Image.fromarray((weight * 255).astype(np.uint8))
    channels = [Image.fromarray((rgb[..., c] * weight).astype(np.uint8)) for c in range(3)]
    for radius in radii:
        bw = np.asarray(w8.filter(ImageFilter.BoxBlur(radius))).astype(np.float32) / 255
        new = ~found & (bw > 0.02)
        if not new.any():
            continue
        for c in range(3):
            bc = np.asarray(channels[c].filter(ImageFilter.BoxBlur(radius))).astype(np.float32)
            colour[..., c][new] = (bc / np.maximum(bw, 1e-4))[new]
        found |= new
    return colour, found


def _hue_chroma_saturation(rgb: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """HSV hue (degrees), chroma (max - min) and saturation (chroma / max) of RGB in 0..255."""
    high, low = rgb.max(axis=-1), rgb.min(axis=-1)
    chroma = high - low
    safe = np.maximum(chroma, 1e-4)
    r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2]
    hue = np.where(high == r, ((g - b) / safe) % 6, np.where(high == g, (b - r) / safe + 2, (r - g) / safe + 4)) * 60
    return hue, chroma, chroma / np.maximum(high, 1.0)


def background_like(rgb: np.ndarray, colour: np.ndarray, tolerance: float, shade: bool) -> np.ndarray:
    """Pixels that are the background `colour` (one RGB, or one per pixel): within `tolerance` on
    every channel, or, when `shade`, its hue in shade: within `SHADE_HUE` of its hue, at
    `SHADE_SATURATION` of its saturation and with chroma over `SHADE_CHROMA`, at any brightness.
    A model darkens a green background toward a deeper, purer green, not toward black along the
    same RGB line, so the shade is read in hue."""
    like = np.abs(rgb - colour).max(axis=-1) <= tolerance
    if shade:
        hue, chroma, saturation = _hue_chroma_saturation(rgb)
        key_hue, key_chroma, key_saturation = _hue_chroma_saturation(np.broadcast_to(colour, rgb.shape))
        turn = np.abs(hue - key_hue) % 360
        like |= ((np.minimum(turn, 360 - turn) <= SHADE_HUE) & (saturation >= SHADE_SATURATION * key_saturation)
                 & (chroma >= np.maximum(SHADE_CHROMA[0], SHADE_CHROMA[1] * key_chroma)))
    return like


def _strip_depth(height: int, width: int) -> int:
    return max(2, round(min(height, width) * STRIP_MAX_FRACTION))


def _ring(height: int, width: int) -> np.ndarray:
    inset = _strip_depth(height, width) + 2
    ring = np.zeros((height, width), dtype=bool)
    outer = min(inset + RING_WIDTH, height // 2, width // 2)
    if outer <= inset:  # a tiny image: the whole border
        ring[:2] = ring[-2:] = True
        ring[:, :2] = ring[:, -2:] = True
        return ring
    ring[inset:outer, inset:width - inset] = True
    ring[height - outer:height - inset, inset:width - inset] = True
    ring[inset:height - inset, inset:outer] = True
    ring[inset:height - inset, width - outer:width - inset] = True
    return ring


def measure_background(rgb: np.ndarray) -> dict[str, Any] | None:
    """The flat background's colour and spread, read on a ring just inside where an edge strip may
    reach: the mode of the ring's colours, then the mean and spread of the ring pixels within
    `INLIER_TOLERANCE` of it. None when no colour holds `MIN_BACKGROUND_SHARE` of the ring."""
    height, width = rgb.shape[:2]
    samples = rgb[_ring(height, width)]
    if not len(samples):
        return None
    bins = samples.astype(np.int32) // MODE_BIN
    slots = (bins[:, 0] << 10) | (bins[:, 1] << 5) | bins[:, 2]
    counts = np.bincount(slots)
    mode = int(np.argmax(counts))
    seed = samples[slots == mode].mean(axis=0)
    inliers = samples[np.abs(samples - seed).max(axis=-1) <= INLIER_TOLERANCE]
    share = len(inliers) / len(samples)
    if share < MIN_BACKGROUND_SHARE:
        return None
    colour = inliers.mean(axis=0)
    spread = float(inliers.std(axis=0).max())
    chroma = float(colour.max() - colour.min())
    return {"colour": colour, "spread": spread, "share": share, "chroma": chroma,
            "shade": chroma >= SATURATED_CHROMA,
            "tolerance": max(float(TOLERANCE_FLOOR), SPREAD_SCALE * spread)}


def _edge_strips(rgb: np.ndarray, like: np.ndarray) -> tuple[np.ndarray, dict[str, int]]:
    """Lines along each edge in one colour that is not the background: (their pixels, depth per side).

    A line is a strip line when `STRIP_LINE_SHARE` of it is within `TOLERANCE_FLOOR` of its median
    and that colour is not mostly background; the strip is the run of such lines from the edge, at
    most `_strip_depth` deep. Only the strip colour's pixels (and the background's) are taken, so a
    subject drawn over the strip keeps its own pixels."""
    height, width = like.shape
    depth_max = _strip_depth(height, width)
    strip = np.zeros((height, width), dtype=bool)
    depths: dict[str, int] = {}
    views = {"top": (rgb, like, strip), "bottom": (rgb[::-1], like[::-1], strip[::-1]),
             "left": (rgb.transpose(1, 0, 2), like.T, strip.T),
             "right": (rgb.transpose(1, 0, 2)[::-1], like.T[::-1], strip.T[::-1])}
    for side, (lines, line_like, out) in views.items():
        depth = 0
        for k in range(min(depth_max, len(lines))):
            line = lines[k]
            if line_like[k].mean() >= STRIP_LINE_SHARE:
                break
            median = np.median(line, axis=0)
            same = np.abs(line - median).max(axis=-1) <= TOLERANCE_FLOOR
            if same.mean() < STRIP_LINE_SHARE:
                break
            out[k] |= same | line_like[k]
            depth = k + 1
        if depth:
            depths[side] = depth
    return strip, depths


def measured_matte(image: Image.Image, *, band: int = EDGE_BAND) -> tuple[Image.Image, dict[str, Any]]:
    """Key ``image``'s flat background of whatever colour it was painted. Returns (RGBA, stats).

    Raises SystemExit when no flat background is found on the ring, or when keying it would leave
    nothing of a subject."""
    rgba = np.asarray(image.convert("RGBA"))
    rgb = rgba[..., :3].astype(np.float32)
    height, width = rgb.shape[:2]
    measured = measure_background(rgb)
    if measured is None:
        raise SystemExit("cutout: no flat background colour holds the image's border ring; "
                         "is the background one colour?")
    colour, tolerance, shade = measured["colour"], measured["tolerance"], measured["shade"]

    like = background_like(rgb, colour, tolerance, shade) | (rgba[..., 3] == 0)
    strip, strips = _edge_strips(rgb, like)
    edge_seed = np.zeros_like(like)
    reach = _strip_depth(height, width) + 2 + RING_WIDTH
    edge_seed[:reach] = edge_seed[-reach:] = True
    edge_seed[:, :reach] = edge_seed[:, -reach:] = True
    connected = _reconstruct((edge_seed & like) | strip, like | strip)

    # a gradient: the background read again against its own colour nearby, the subject kept out
    local, found = _filled_mean(rgb, connected.astype(np.float32), (8, 32, 128))
    local = np.where(found[..., None], local, colour)
    like = background_like(rgb, local, tolerance, shade) | (rgba[..., 3] == 0)
    connected = _reconstruct((edge_seed & like) | strip, like | strip)

    # enclosed pockets of the background (between an arm and the body), thick enough not to be a speck;
    # only a saturated background's colour is its own: a pale one's pockets may be the subject's own
    # highlight or skin, so it is keyed by position alone (the white matte's rule)
    pockets = like & ~connected if shade else np.zeros_like(like)
    pockets = _reconstruct(_erode(pockets, HOLE_THICKNESS), pockets) if pockets.any() else pockets
    keyed = connected | pockets

    keyed_share = float(keyed.mean())
    if keyed_share >= FULL_BACKGROUND_SHARE:
        raise SystemExit(f"cutout: the measured background {tuple(int(v) for v in colour)} covers "
                         f"{keyed_share:.1%} of the image; there is no subject to cut out")

    # the edge: unmixed between the local background and the nearest solid subject colour
    near = keyed.copy()
    for _ in range(band):
        near = _shift_or(near)
    edge = near & ~keyed
    solid = ~near
    background, found = _filled_mean(rgb, keyed.astype(np.float32), (2, 8, 32, 128))
    background = np.where(found[..., None], background, colour)
    subject, found = _filled_mean(rgb, solid.astype(np.float32), (1, 2, 4, 8, 16, 64))
    subject = np.where(found[..., None], subject, rgb)
    toward = subject - background
    offset = rgb - background
    span = (toward * toward).sum(axis=-1)
    alpha = np.where(span > 900.0, (offset * toward).sum(axis=-1) / np.maximum(span, 1.0),
                     (np.abs(offset).max(axis=-1) - tolerance) / 60.0)
    alpha = np.clip(alpha, 0.0, 1.0)
    unmixed = np.clip((rgb - (1.0 - alpha[..., None]) * background) / np.maximum(alpha[..., None], 1e-3), 0, 255)
    edge_rgb = alpha[..., None] * unmixed + (1.0 - alpha[..., None]) * subject

    out = rgba.astype(np.float32).copy()
    out[edge, :3] = edge_rgb[edge]
    out[edge, 3] = np.minimum(out[edge, 3], np.round(alpha[edge] * 255))
    out[keyed] = 0
    faint = out[..., 3] < ALPHA_FLOOR
    out[faint] = 0
    result = Image.fromarray(np.clip(np.round(out), 0, 255).astype(np.uint8), "RGBA")

    residue = int((background_like(np.asarray(result)[..., :3].astype(np.float32), background, tolerance, False)
                   & (np.asarray(result)[..., 3] >= 128) & near).sum())
    return result, {
        "route": "measured",
        "background": [int(round(v)) for v in colour],
        "background_spread": round(measured["spread"], 2),
        "background_ring_share": round(measured["share"], 4),
        "shade": shade,
        "tolerance": round(tolerance, 1),
        "edge_strips": strips,
        "keyed_pixels": int((out[..., 3] == 0).sum()),
        "pocket_pixels": int(pockets.sum()),
        "edge_pixels": int(edge.sum()),
        "band": band,
        "key_residue_px": residue,
    }