Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

10.8 KB
Raw Blame
"""Lossless RGBA Cloud Optimized GeoTIFF and PNG exports using Pillow and the stdlib.

Directories precede all pixels; tiled overviews precede full-resolution data.
Compressed blocks are spooled to disk, bounding COG decoding to small windows.
The PNG reuses those blocks, decoding one tile row at a time.
No GDAL or libtiff encoder is required.
"""

import math
import struct
import tempfile
import zlib
from array import array

from PIL import Image, ImageChops, ImageDraw

from .common import atomic_path, contained
from .geo import MERCATOR_RADIUS, pixel, tiles

BLOCK = 256
TIFF_LIMIT = 2**32


def region(box, read_tile):
    """Assemble a small pixel window from 256-pixel tiles, including alpha."""
    left, top, right, bottom = box
    image = Image.new("RGBA", (right - left, bottom - top))
    try:
        for y in range(top // BLOCK, (bottom - 1) // BLOCK + 1):
            for x in range(left // BLOCK, (right - 1) // BLOCK + 1):
                with read_tile(x, y) as tile:
                    image.paste(tile, (x * BLOCK - left, y * BLOCK - top))
        return image
    except BaseException:
        image.close()
        raise


class Level:
    def __init__(self, width, height):
        self.width, self.height = width, height
        self.columns = math.ceil(width / BLOCK)
        self.rows = math.ceil(height / BLOCK)
        self.offsets, self.sizes = array("Q"), array("I")

    def read(self, stream, x, y):
        i = y * self.columns + x
        stream.seek(self.offsets[i])
        raw = zlib.decompress(stream.read(self.sizes[i]))
        return Image.frombytes("RGBA", (BLOCK, BLOCK), raw)

    def write(self, stream, image):
        with Image.new("RGBA", (BLOCK, BLOCK)) as tile:
            tile.paste(image, (0, 0))
            data = zlib.compress(tile.tobytes())
        stream.seek(0, 2)
        self.offsets.append(stream.tell())
        self.sizes.append(len(data))
        stream.write(data)


def header(levels, grid, *, big=False):
    """Encode just the TIFF directories and GeoKeys, with final block offsets.

    Classic TIFF uses 32-bit offsets; BigTIFF uses 64-bit offsets. Pixel data
    follows this header in smallest-to-largest level order, row-major per level.
    """
    offset_format, count_format = ("Q", "Q") if big else ("I", "H")
    inline = 8 if big else 4
    directories = []
    scale = 2 * math.pi * MERCATOR_RADIUS / (BLOCK * 2**grid["zoom"])
    half = math.pi * MERCATOR_RADIUS
    for i, level in enumerate(levels):
        # (TIFF type, values). ExtraSamples=2 means unassociated alpha.
        tags = {
            254: (4, [int(i != 0)]), 256: (4, [level.width]), 257: (4, [level.height]),
            258: (3, [8, 8, 8, 8]), 259: (3, [8]), 262: (3, [2]),
            277: (3, [4]), 284: (3, [1]), 322: (4, [BLOCK]), 323: (4, [BLOCK]),
            324: (16 if big else 4, [0] * len(level.sizes)),
            325: (4, level.sizes), 338: (3, [2]),
        }
        if i == 0:
            tags.update({
                33550: (12, [scale, scale, 0]),
                33922: (12, [0, 0, 0, grid["left"] * scale - half,
                            half - grid["top"] * scale, 0]),
                34735: (3, [1, 1, 0, 3, 1024, 0, 1, 1, 1025, 0, 1, 1, 3072, 0, 1, 3857]),
            })
        directories.append(tags)

    data = bytearray(struct.pack("<2sHHHQ", b"II", 43, 8, 0, 16) if big
                     else struct.pack("<2sHI", b"II", 42, 8))
    locations = []
    for tags in directories:
        data.extend(b"\0" * (-len(data) % 8))
        locations.append(len(data))
        data.extend(b"\0" * ((8 if big else 2) + len(tags) * (20 if big else 12) + inline))
    deferred = []
    offset_fields = []
    for i, tags in enumerate(directories):
        cursor = locations[i]
        struct.pack_into("<" + count_format, data, cursor, len(tags))
        cursor += 8 if big else 2
        for code, (kind, values) in sorted(tags.items()):
            struct.pack_into("<HH" + offset_format, data, cursor, code, kind, len(values))
            field = cursor + (12 if big else 8)
            raw = struct.pack("<" + {3: "H", 4: "I", 12: "d", 16: "Q"}[kind] * len(values), *values)
            if len(raw) <= inline:
                data[field:field + len(raw)] = raw
                if code == 324:
                    offset_fields.append((i, field))
            else:
                deferred.append((code in (324, 325), i, code, field, raw))
            cursor += 20 if big else 12
        struct.pack_into("<" + offset_format, data, cursor,
                         locations[i + 1] if i + 1 < len(locations) else 0)
    # Keep metadata before potentially large tile offset/count arrays.
    for _, i, code, field, raw in sorted(deferred, key=lambda item: item[0]):
        data.extend(b"\0" * (-len(data) % 8))
        struct.pack_into("<" + offset_format, data, field, len(data))
        if code == 324:
            offset_fields.append((i, len(data)))
        data.extend(raw)
    data.extend(b"\0" * (-len(data) % 8))
    starts, cursor = {}, len(data)
    for i in reversed(range(len(levels))):
        starts[i] = cursor
        cursor += sum(levels[i].sizes)
    if not big and cursor >= TIFF_LIMIT:
        return header(levels, grid, big=True)
    for i, field in offset_fields:
        cursor = starts[i]
        for size in levels[i].sizes:
            struct.pack_into("<" + offset_format, data, field, cursor)
            cursor += size
            field += inline
    return data


def write_png(path, level, spool, progress):
    """Stream the full-resolution pixels as PNG, decoding one tile row at a time."""
    with path.open("wb") as output:
        def chunk(kind, data):
            output.write(struct.pack(">I", len(data)))
            output.write(kind)
            output.write(data)
            output.write(struct.pack(">I", zlib.crc32(data, zlib.crc32(kind))))

        output.write(b"\x89PNG\r\n\x1a\n")
        chunk(b"IHDR", struct.pack(">2I5B", level.width, level.height, 8, 6, 0, 0, 0))
        compressor = zlib.compressobj()
        progress("Building satellite.png", 0, level.rows)
        for row, top in enumerate(range(0, level.height, BLOCK), 1):
            with region((0, top, level.width, min(top + BLOCK, level.height)),
                        lambda x, y: level.read(spool, x, y)) as strip:
                for y in range(strip.height):
                    with strip.crop((0, y, level.width, y + 1)) as scanline:
                        data = compressor.compress(b"\0" + scanline.tobytes())
                    if data:
                        chunk(b"IDAT", data)
            progress("Building satellite.png", row, level.rows)
        chunk(b"IDAT", compressor.flush())
        chunk(b"IEND", b"")


def clear(image, outline):
    """Make pixels outside a polygon in image pixels transparent, keeping those its edges pass through."""
    with Image.new("L", image.size) as inside:
        ImageDraw.Draw(inside).polygon(outline, fill=255, outline=255)
        with image.getchannel("A") as alpha, ImageChops.multiply(alpha, inside) as kept:
            image.putalpha(kept)


def merge(stage, folder, progress, polygon):
    """Write satellite.tif and satellite.png, retaining transparent gaps and clearing pixels outside the polygon."""
    grid = stage["grid"]
    levels = [Level(grid["width"], grid["height"])]
    while max(levels[-1].width, levels[-1].height) > BLOCK:
        previous = levels[-1]
        levels.append(Level((previous.width + 1) // 2, (previous.height + 1) // 2))
    count = sum(level.columns * level.rows for level in levels)
    done = 0
    progress("Building satellite.tif", done, count)

    saved = {}
    for planned, tile in zip(tiles(grid), stage["results"]):
        if (tile["x"], tile["y"], tile["zoom"]) != (planned["x"], planned["y"], grid["zoom"]):
            raise ValueError("Invalid saved satellite tile order.")
        saved[tile["x"], tile["y"]] = tile["filename"]
    outline = [(x - grid["left"], y - grid["top"]) for x, y in (pixel(point, grid["zoom"]) for point in polygon)]

    def source(x, y):
        if (x, y) not in saved:
            return Image.new("RGBA", (BLOCK, BLOCK))
        with Image.open(contained(folder, saved[x, y])) as image:
            if image.format not in ("JPEG", "PNG") or image.size != (BLOCK, BLOCK):
                raise ValueError("Expected a 256 × 256 satellite JPEG or PNG.")
            return image.convert("RGBA")

    with (
        atomic_path(folder / "satellite.tif") as temporary,
        atomic_path(folder / "satellite.png") as png_temporary,
        tempfile.TemporaryFile(dir=folder) as spool,
    ):
        for i, level in enumerate(levels):
            for y in range(0, level.height, BLOCK):
                for x in range(0, level.width, BLOCK):
                    width, height = min(BLOCK, level.width - x), min(BLOCK, level.height - y)
                    if i == 0:
                        left, top = grid["left"] + x, grid["top"] + y
                        with region((left, top, left + width, top + height), source) as image:
                            clear(image, [(u - x, v - y) for u, v in outline])
                            level.write(spool, image)
                    else:
                        previous = levels[i - 1]
                        # Fractional windows preserve the extent for odd dimensions.
                        sx, sy = previous.width / level.width, previous.height / level.height
                        box = (x * sx, y * sy, (x + width) * sx, (y + height) * sy)
                        window = (math.floor(box[0]), math.floor(box[1]),
                                  math.ceil(box[2]), math.ceil(box[3]))
                        local = tuple(v - window[j % 2] for j, v in enumerate(box))
                        with (
                            region(window, lambda tx, ty: previous.read(spool, tx, ty)) as image,
                            image.resize((width, height), Image.Resampling.BOX, box=local) as smaller,
                        ):
                            level.write(spool, smaller)
                    done += 1
                    progress("Building satellite.tif", done, count)
        with temporary.open("wb") as output:
            output.write(header(levels, grid))
            total_bytes = sum(sum(level.sizes) for level in levels)
            copied = 0
            progress("Saving satellite.tif", copied, total_bytes, "bytes")
            for level in reversed(levels):
                spool.seek(level.offsets[0])
                remaining = sum(level.sizes)
                while remaining:
                    data = spool.read(min(1024 * 1024, remaining))
                    if not data:
                        raise OSError("Truncated satellite export spool.")
                    output.write(data)
                    remaining -= len(data)
                    copied += len(data)
                    progress("Saving satellite.tif", copied, total_bytes, "bytes")
        write_png(png_temporary, levels[0], spool, progress)