Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

9.2 KB
Raw Blame
"""Join terrain GeoTIFFs, copying 256-pixel blocks and splitting larger ones.

Sources use Int16 heights up to zoom 12 and Float32 from zoom 13.
"""

import math
import struct
import zlib

from .common import atomic_path
from .geo import MERCATOR_RADIUS

TYPES = {1: "B", 2: "c", 3: "H", 4: "I", 5: "II", 11: "f", 12: "d"}
COMPATIBLE = (258, 259, 262, 277, 284, 317, 339, 34735, 34736, 34737, 42113)
TIFF_LIMIT = 2**32


class TIFF:
    def __init__(self, stream):
        self.stream = stream
        stream.seek(0, 2)
        self.size = stream.tell()
        magic = self.read(0, 8)
        if magic[:4] not in (b"II*\0", b"MM\0*"):
            raise ValueError("Unsupported terrain TIFF header.")
        self.order = "<" if magic[:2] == b"II" else ">"
        offset = struct.unpack(self.order + "I", magic[4:])[0]
        count = struct.unpack(self.order + "H", self.read(offset, 2))[0]
        if count > 128:
            raise ValueError("Unexpected terrain TIFF directory.")
        self.tags = {}
        for i in range(count):
            entry = self.read(offset + 2 + i * 12, 12)
            code, kind, length, location = struct.unpack(self.order + "HHII", entry)
            if kind not in TYPES or code in self.tags:
                raise ValueError("Unsupported terrain TIFF tag.")
            size = struct.calcsize(self.order + TYPES[kind]) * length
            raw = entry[8 : 8 + size] if size <= 4 else self.read(location, size)
            self.tags[code] = kind, length, raw
        self.float = (self.value(258), self.value(339)) == (32, 3)
        if ((not self.float and (self.value(258), self.value(339)) != (16, 2))
                or self.value(277) != 1 or self.value(262) != 1 or self.value(259) not in (1, 8, 32946)):
            raise ValueError("Expected tiled Float32 or Int16 terrain.")
        # Only Float32 sources use 512-pixel blocks, which blocks() splits.
        if (self.value(322), self.value(323)) not in ((256, 256), (512, 512) if self.float else (256, 256)):
            raise ValueError("Unexpected terrain block size.")
        if self.value(274, 1) != 1 or self.value(284, 1) != 1 or self.value(317, 1) not in ((1, 3) if self.float else (1, 2)):
            raise ValueError("Unsupported terrain layout.")
        if any(key not in self.tags for key in (33550, 33922, 34735, 42113, 324, 325)):
            raise ValueError("Missing terrain georeferencing or blocks.")
        keys = self.values(34735)
        self.keys = {keys[i]: keys[i + 3] for i in range(4, len(keys) - 3, 4) if keys[i + 1:i + 3] == (0, 1)}
        if self.keys.get(3072) != 3857 or self.keys.get(1025) not in (1, 2):
            raise ValueError("Expected EPSG:3857 terrain.")

    def read(self, offset, size):
        if offset < 0 or offset + size > self.size:
            raise ValueError("Truncated terrain TIFF.")
        self.stream.seek(offset)
        data = self.stream.read(size)
        if len(data) != size:
            raise ValueError("Truncated terrain TIFF.")
        return data

    def values(self, code):
        kind, count, raw = self.tags[code]
        return struct.unpack(self.order + TYPES[kind] * count, raw)

    def value(self, code, default=None):
        return self.values(code)[0] if code in self.tags else default

    def blocks(self):
        """Yield four 256-pixel blocks, preserving the source pixel encoding."""
        if self.value(322) == 256:
            for offset, size in zip(self.values(324), self.values(325)):
                yield self.read(offset, size)
            return
        data = self.read(self.value(324), self.value(325))
        compressed = self.value(259) != 1
        if compressed:
            data = zlib.decompress(data)
        if len(data) != 512 * 512 * 4:
            raise ValueError("Invalid terrain block size.")
        blocks = [bytearray() for _ in range(4)]
        for y in range(512):
            row = data[y * 2048:(y + 1) * 2048]
            left, right = blocks[y // 256 * 2:y // 256 * 2 + 2]
            if self.value(317, 1) == 3:
                # Float prediction differences span four byte planes per row.
                # Rebase only the first byte of each half-plane after splitting.
                carry = 0
                for plane in range(4):
                    a = bytearray(row[plane * 512:plane * 512 + 256])
                    b = bytearray(row[plane * 512 + 256:(plane + 1) * 512])
                    a_sum, b_sum = sum(a), sum(b)
                    a[0] = (a[0] + carry) & 255
                    b[0] = (b[0] + a_sum) & 255
                    carry = b_sum
                    left.extend(a)
                    right.extend(b)
            else:
                left.extend(row[:1024])
                right.extend(row[1024:])
        for block in blocks:
            yield zlib.compress(block) if compressed else block


def header(tags, order):
    directory = bytearray((b"II" if order == "<" else b"MM") + struct.pack(order + "HIH", 42, 8, len(tags)))
    data = bytearray()
    start = 8 + 2 + len(tags) * 12 + 4
    for code, (kind, count, raw) in sorted(tags.items()):
        directory.extend(struct.pack(order + "HHI", code, kind, count))
        if len(raw) <= 4:
            directory.extend(raw.ljust(4, b"\0"))
        else:
            data.extend(b"\0" * (-(start + len(data)) % 8))
            directory.extend(struct.pack(order + "I", start + len(data)))
            data.extend(raw)
    return directory + b"\0" * 4 + data


def origin(source, tile):
    scale = 2 * math.pi * MERCATOR_RADIUS / 2 ** tile["zoom"] / 512
    half = math.pi * MERCATOR_RADIUS
    shift = scale / 2 if source.keys[1025] == 2 else 0
    return scale, -half + tile["x"] * scale * 512 + shift, half - tile["y"] * scale * 512 - shift


def validate(source, tile):
    scale, west, north = origin(source, tile)
    pixel, tie = source.values(33550), source.values(33922)
    if not ((source.value(256), source.value(257)) == (512, 512) and len(pixel) == 3 and len(tie) == 6
            and all(math.isfinite(v) for v in pixel + tie)
            and all(abs(pixel[i] - scale) <= scale * 1e-8 for i in (0, 1))
            and tie[:3] == (0, 0, 0) and abs(tie[3] - west) <= scale * 1e-5 and abs(tie[4] - north) <= scale * 1e-5):
        raise ValueError("Terrain georeferencing does not match the download grid.")
    offsets, sizes = source.values(324), source.values(325)
    count = (512 // source.value(322)) ** 2
    if len(offsets) != count or len(sizes) != count:
        raise ValueError("Invalid terrain block directory.")
    if any(not offset or not size or offset + size > source.size for offset, size in zip(offsets, sizes)):
        raise ValueError("Missing or truncated saved terrain block.")


def merge(path, folder, results, grid, progress):
    """One source file open at a time; usual 256-pixel blocks are copied as-is."""
    columns, rows = grid["columns"], grid["rows"]
    if len(results) != columns * rows:
        raise ValueError("Download all terrain tiles before building terrain.tif.")
    offsets, sizes = [0] * (columns * rows * 4), [0] * (columns * rows * 4)
    tags, compatible, order = {}, None, None

    def set_tag(code, kind, values):
        tags[code] = kind, len(values), struct.pack(order + TYPES[kind] * len(values), *values)

    progress("Building terrain.tif", 0, len(results))
    with atomic_path(path) as temporary, temporary.open("wb") as output:
        for i, tile in enumerate(results):
            row, column = divmod(i, columns)
            if (tile["x"], tile["y"], tile["zoom"]) != (grid["x0"] + column, grid["y0"] + row, grid["zoom"]):
                raise ValueError("Invalid saved terrain tile order.")
            with (folder / tile["filename"]).open("rb") as stream:
                source = TIFF(stream)
                validate(source, tile)
                signature = {key: source.tags[key] for key in COMPATIBLE if key in source.tags}
                if compatible is None:
                    compatible, order = signature, source.order
                    tags.update(signature)
                    scale, west, north = origin(source, tile)
                    set_tag(256, 4, [columns * 512])
                    set_tag(257, 4, [rows * 512])
                    set_tag(322, 4, [256])
                    set_tag(323, 4, [256])
                    set_tag(33550, 12, [scale, scale, 0])
                    set_tag(33922, 12, [0, 0, 0, west, north, 0])
                    set_tag(324, 4, offsets)
                    set_tag(325, 4, sizes)
                    output.write(header(tags, order))
                elif order != source.order or compatible != signature:
                    raise ValueError("Terrain tile formats differ.")
                for j, data in enumerate(source.blocks()):
                    block = (row * 2 + j // 2) * columns * 2 + column * 2 + j % 2
                    size = len(data)
                    if output.tell() + size >= TIFF_LIMIT:
                        raise ValueError("Terrain exceeds the 4 GiB TIFF limit. Use a smaller area or lower terrain zoom.")
                    offsets[block], sizes[block] = output.tell(), size
                    output.write(data)
            progress("Building terrain.tif", i + 1, len(results))
        set_tag(324, 4, offsets)
        set_tag(325, 4, sizes)
        output.seek(0)
        output.write(header(tags, order))