Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
15.3 KB
Raw Blame
"""The tiled GeoTIFF reader, checked against files this module writes itself.

The writer below is the smallest one that makes the layouts the reader must
handle: tiles, planar or chunky samples, deflate or none, the horizontal and the
floating-point predictors, little- or big-endian, classic or BigTIFF. Each file
is written in memory, so no test touches the disk except the one that opens a
local file.
"""

from __future__ import annotations

import struct
import zlib
from pathlib import Path

import numpy as np
import pytest

from azimut.engine.cog import GeoTiff, _lzw_decode

TILE = 16
ORIGIN_X, ORIGIN_Y = 547664.0, 7262632.0
NAMES = ["building_presence", "building_height", "confidence"]

SHORT, LONG, ASCII, DOUBLE, LONG8 = 3, 4, 2, 12, 16
SAMPLE_FORMAT = {"u": 1, "i": 2, "f": 3}


# -- a small TIFF writer ---------------------------------------------------------------


def lzw_encode(data: bytes) -> bytes:
    """TIFF LZW with the early change, the mirror of what the reader decodes."""
    table = {bytes((i,)): i for i in range(256)}
    codes: list[tuple[int, int]] = [(256, 9)]
    width, next_code = 9, 258
    word = b""
    for byte in data:
        candidate = word + bytes((byte,))
        if candidate in table:
            word = candidate
            continue
        codes.append((table[word], width))
        table[candidate] = next_code
        next_code += 1
        if next_code >= 1 << width and width < 12:
            width += 1
        if next_code >= 4094:  # the table is full: clear it and start again
            codes.append((256, width))
            table = {bytes((i,)): i for i in range(256)}
            width, next_code = 9, 258
        word = bytes((byte,))
    if word:
        codes.append((table[word], width))
    if next_code + 1 >= 1 << width and width < 12:
        width += 1
    codes.append((257, width))
    return _pack_codes(codes)


def _pack_codes(codes: list[tuple[int, int]]) -> bytes:
    acc = 0
    have = 0
    out = bytearray()
    for code, width in codes:
        acc = (acc << width) | code
        have += width
        while have >= 8:
            have -= 8
            out.append((acc >> have) & 0xFF)
        acc &= (1 << have) - 1
    if have:
        out.append((acc << (8 - have)) & 0xFF)
    return bytes(out)


def _encode_tile(tile: np.ndarray, predictor: int, endian: str) -> bytes:
    """One tile, (rows, columns, samples), as the writer lays it on disk."""
    rows, cols, nch = tile.shape
    dtype = tile.dtype.newbyteorder(endian)
    if predictor == 2:
        diff = tile.copy()
        diff[:, 1:, :] = tile[:, 1:, :] - tile[:, :-1, :]  # wraps in the sample type
        return diff.astype(dtype).tobytes()
    if predictor == 3:
        size = tile.dtype.itemsize
        samples = tile.reshape(rows, cols * nch).astype(dtype)
        cut = samples.view(np.uint8).reshape(rows, cols * nch, size)
        # Most significant byte of every sample first, then the next plane, and so on.
        msb_first = cut if endian == ">" else cut[:, :, ::-1]
        planes = msb_first.transpose(0, 2, 1).reshape(rows, cols * nch * size)
        diff = planes.copy()
        diff[:, nch:] = planes[:, nch:] - planes[:, :-nch]  # difference, `nch` bytes back
        return diff.tobytes()
    return tile.astype(dtype).tobytes()


def _compress(raw: bytes, compression: int) -> bytes:
    if compression == 8:
        return zlib.compress(raw)
    if compression == 5:
        return lzw_encode(raw)
    return raw


def _values(typ: int, values: object, endian: str) -> bytes:
    if typ == ASCII:
        return str(values).encode("ascii") + b"\0"
    code = {SHORT: "H", LONG: "I", LONG8: "Q", DOUBLE: "d"}[typ]
    items = list(values)  # type: ignore[call-overload]
    return struct.pack(f"{endian}{len(items)}{code}", *items)


def write_tiff(
    data: np.ndarray,
    *,
    planar: int = 2,
    predictor: int = 1,
    compression: int = 8,
    endian: str = "<",
    big: bool = False,
    names: list[str] | None = None,
    nodata: float | None = -99.0,
    strips: bool = False,
    epsg: int = 32723,
) -> bytes:
    """A tiled GeoTIFF of `data` (bands, height, width), in memory."""
    bands, height, width = data.shape
    across, down = -(-width // TILE), -(-height // TILE)
    padded = np.zeros((bands, down * TILE, across * TILE), dtype=data.dtype)
    padded[:, :height, :width] = data

    # Planar: one run of tiles per band. Chunky: every band in each tile.
    planes = [padded[b : b + 1] for b in range(bands)] if planar == 2 else [padded]
    blocks = [
        plane[:, ty * TILE : (ty + 1) * TILE, tx * TILE : (tx + 1) * TILE].transpose(1, 2, 0)
        for plane in planes
        for ty in range(down)
        for tx in range(across)
    ]
    blobs = [_compress(_encode_tile(block, predictor, endian), compression) for block in blocks]

    out = bytearray(b"II" if endian == "<" else b"MM")
    if big:
        out += struct.pack(endian + "HHHQ", 43, 8, 0, 0)
    else:
        out += struct.pack(endian + "HI", 42, 0)
    offsets, counts = [], []
    for blob in blobs:
        offsets.append(len(out))
        counts.append(len(blob))
        out += blob

    offset_type = LONG8 if big else LONG
    entries: list[tuple[int, int, object]] = [
        (256, LONG, [width]),
        (257, LONG, [height]),
        (258, SHORT, [data.dtype.itemsize * 8] * bands),
        (259, SHORT, [compression]),
        (277, SHORT, [bands]),
        (284, SHORT, [planar]),
        (317, SHORT, [predictor]),
        (339, SHORT, [SAMPLE_FORMAT[data.dtype.kind]] * bands),
        (33550, DOUBLE, [0.5, 0.5, 0.0]),
        (33922, DOUBLE, [0.0, 0.0, 0.0, ORIGIN_X, ORIGIN_Y, 0.0]),
        (34735, SHORT, [1, 1, 0, 3, 1024, 0, 1, 1, 1025, 0, 1, 1, 3072, 0, 1, epsg]),
    ]
    if names:
        items = "".join(
            f'<Item name="DESCRIPTION" sample="{i}" role="description">{name}</Item>'
            for i, name in enumerate(names)
        )
        entries.append((42112, ASCII, f"<GDALMetadata>{items}</GDALMetadata>"))
    if nodata is not None:
        entries.append((42113, ASCII, f"{nodata:g}"))
    if strips:
        entries += [(273, offset_type, offsets), (279, offset_type, counts)]
    else:
        entries += [
            (322, LONG, [TILE]),
            (323, LONG, [TILE]),
            (324, offset_type, offsets),
            (325, offset_type, counts),
        ]

    inline = 8 if big else 4
    fields = bytearray()
    for tag, typ, values in sorted(entries, key=lambda entry: entry[0]):
        raw = _values(typ, values, endian)
        count = len(raw) if typ == ASCII else len(list(values))  # type: ignore[call-overload]
        if len(raw) <= inline:
            field = raw.ljust(inline, b"\0")
        else:
            if len(out) % 2:
                out += b"\0"
            where = len(out)
            out += raw
            field = struct.pack(endian + ("Q" if big else "I"), where)
        fields += struct.pack(endian + ("HHQ" if big else "HHI"), tag, typ, count) + field

    if len(out) % 2:
        out += b"\0"
    ifd = len(out)
    out += struct.pack(endian + ("Q" if big else "H"), len(entries)) + fields
    out += struct.pack(endian + ("Q" if big else "I"), 0)
    if big:
        struct.pack_into(endian + "Q", out, 8, ifd)
    else:
        struct.pack_into(endian + "I", out, 4, ifd)
    return bytes(out)


class CountingReader:
    """A reader over bytes in memory that remembers every request it was asked for."""

    def __init__(self, blob: bytes) -> None:
        self.blob = blob
        self.requests: list[tuple[int, int]] = []

    def __call__(self, offset: int, length: int) -> bytes:
        self.requests.append((offset, length))
        return self.blob[offset : offset + length]


def samples(bands: int, height: int, width: int, dtype: str, seed: int = 0) -> np.ndarray:
    rng = np.random.default_rng(seed)
    if np.dtype(dtype).kind == "f":
        return rng.normal(0.0, 50.0, (bands, height, width)).astype(dtype)
    info = np.iinfo(dtype)
    return rng.integers(info.min, info.max, (bands, height, width), endpoint=True, dtype=dtype)


# -- the tags, and the windows, of the case the sample file is -------------------------


def test_float_planar_deflate_predictor3_reads_its_tags_and_windows():
    data = samples(3, 32, 32, "float32")
    blob = write_tiff(data, names=NAMES, predictor=3)

    tiff = GeoTiff(CountingReader(blob))

    assert (tiff.width, tiff.height, tiff.bands) == (32, 32, 3)
    assert tiff.dtype == np.float32
    assert tiff.tile_size == (16, 16)
    assert tiff.nodata == -99.0
    assert tiff.epsg == 32723
    assert tiff.transform == (ORIGIN_X, 0.5, ORIGIN_Y, -0.5)
    assert tiff.band_names == NAMES
    for band in range(3):
        np.testing.assert_array_equal(tiff.window(band, 0, 0, 32, 32), data[band])
        # A window that crosses the tile edges at 16 and 32 px.
        np.testing.assert_array_equal(tiff.window(band, 10, 12, 22, 20), data[band, 12:20, 10:22])


def test_window_partly_outside_the_image_is_nodata():
    data = samples(1, 32, 32, "float32")
    tiff = GeoTiff(CountingReader(write_tiff(data, predictor=3)))

    out = tiff.window(0, -4, -3, 36, 34)

    assert out.shape == (37, 40)
    np.testing.assert_array_equal(out[3:35, 4:36], data[0])
    outside = np.ones(out.shape, dtype=bool)
    outside[3:35, 4:36] = False
    assert np.all(out[outside] == -99.0)


def test_float_without_nodata_fills_outside_with_nan():
    data = samples(1, 20, 20, "float32")
    tiff = GeoTiff(CountingReader(write_tiff(data, predictor=3, nodata=None)))

    out = tiff.window(0, -2, 0, 4, 2)

    assert tiff.nodata is None
    assert np.isnan(out[:, :2]).all()
    np.testing.assert_array_equal(out[:, 2:], data[0, 0:2, 0:4])


def test_chunky_uint8_with_horizontal_predictor_and_no_band_names():
    data = samples(3, 20, 25, "uint8")
    tiff = GeoTiff(CountingReader(write_tiff(data, planar=1, predictor=2)))

    assert tiff.dtype == np.uint8
    assert tiff.band_names == []
    for band in range(3):
        np.testing.assert_array_equal(tiff.window(band, 0, 0, 25, 20), data[band])
        np.testing.assert_array_equal(tiff.window(band, 9, 15, 20, 19), data[band, 15:19, 9:20])


def test_bigtiff_reads_the_same_windows():
    data = samples(3, 32, 32, "float32")
    tiff = GeoTiff(CountingReader(write_tiff(data, predictor=3, big=True, names=NAMES)))

    assert tiff.band_names == NAMES
    assert tiff.transform == (ORIGIN_X, 0.5, ORIGIN_Y, -0.5)
    np.testing.assert_array_equal(tiff.window(2, 0, 0, 32, 32), data[2])


@pytest.mark.parametrize(("dtype", "predictor"), [("u2", 2), ("f4", 3)])
def test_big_endian_reads_the_same_windows(dtype, predictor):
    data = samples(2, 25, 37, dtype)
    tiff = GeoTiff(CountingReader(write_tiff(data, endian=">", predictor=predictor)))

    for band in range(2):
        np.testing.assert_array_equal(tiff.window(band, 0, 0, 37, 25), data[band])


@pytest.mark.parametrize(
    ("dtype", "predictor"),
    [("u1", 2), ("u2", 2), ("i2", 2), ("i4", 2), ("f4", 3), ("f8", 3), ("u1", 1), ("f4", 1)],
)
@pytest.mark.parametrize("planar", [1, 2])
@pytest.mark.parametrize("compression", [1, 8, 5])
def test_every_layout_round_trips(dtype, predictor, planar, compression):
    data = samples(3, 25, 37, dtype)  # edge tiles are part tiles
    blob = write_tiff(data, planar=planar, predictor=predictor, compression=compression)

    tiff = GeoTiff(CountingReader(blob))

    assert tiff.dtype == np.dtype(dtype)
    for band in range(3):
        np.testing.assert_array_equal(tiff.window(band, 0, 0, 37, 25), data[band])
        np.testing.assert_array_equal(tiff.window(band, 10, 12, 22, 20), data[band, 12:20, 10:22])


def test_a_window_reads_only_the_tiles_it_overlaps():
    data = samples(3, 64, 64, "float32")
    reader = CountingReader(write_tiff(data, predictor=3))
    tiff = GeoTiff(reader)
    reader.requests.clear()

    # One tile of one band: a single read of that tile's bytes.
    tiff.window(1, 0, 0, 8, 8)

    assert len(reader.requests) == 1


def test_the_parsed_ifd_and_recent_tiles_are_not_read_again():
    data = samples(1, 128, 128, "u1")
    reader = CountingReader(write_tiff(data, planar=1, predictor=1, compression=1))
    tiff = GeoTiff(reader)

    # 64 tiles, and the cache keeps 32: the last one read is still there, the first is not.
    tiff.window(0, 0, 0, 128, 128)
    reader.requests.clear()
    tiff.window(0, 112, 112, 128, 128)
    assert reader.requests == []

    tiff.window(0, 0, 0, 16, 16)
    assert len(reader.requests) == 1


def test_open_reads_a_local_file(tmp_path: Path):
    data = samples(3, 32, 32, "float32")
    path = tmp_path / "tile.tif"
    path.write_bytes(write_tiff(data, predictor=3))

    tiff = GeoTiff.open(path)

    np.testing.assert_array_equal(tiff.window(2, 5, 5, 30, 30), data[2, 5:30, 5:30])


def test_a_band_outside_the_image_is_refused():
    tiff = GeoTiff(CountingReader(write_tiff(samples(3, 16, 16, "u1"), predictor=2)))

    with pytest.raises(ValueError, match="band 3"):
        tiff.window(3, 0, 0, 4, 4)


# -- what the reader refuses ------------------------------------------------------------


def test_strips_are_refused_with_a_clear_message():
    blob = write_tiff(samples(1, 32, 32, "float32"), predictor=3, strips=True)

    with pytest.raises(ValueError, match="strips"):
        GeoTiff(CountingReader(blob))


def test_an_unknown_compression_is_refused():
    blob = write_tiff(samples(1, 32, 32, "float32"), predictor=3, compression=7)

    with pytest.raises(ValueError, match="compression 7"):
        GeoTiff(CountingReader(blob))


def test_the_floating_point_predictor_needs_floats():
    blob = write_tiff(samples(1, 16, 16, "u1"), predictor=3)

    with pytest.raises(ValueError, match="floating-point predictor"):
        GeoTiff(CountingReader(blob))


def test_the_horizontal_predictor_needs_integers():
    blob = write_tiff(samples(1, 16, 16, "f4"), predictor=2)

    with pytest.raises(ValueError, match="horizontal predictor"):
        GeoTiff(CountingReader(blob))


def test_a_file_that_is_not_tiff_is_refused():
    with pytest.raises(ValueError, match="not a TIFF"):
        GeoTiff(CountingReader(b"PK\x03\x04" + b"\0" * 32))


# -- LZW -------------------------------------------------------------------------------


def test_lzw_decodes_the_tiff_spec_example():
    # "-----A---B" as TIFF 6.0 codes it: clear, 45, 258, 258, 65, 259, 66, end.
    codes = [(256, 9), (45, 9), (258, 9), (258, 9), (65, 9), (259, 9), (66, 9), (257, 9)]

    assert _lzw_decode(_pack_codes(codes)) == b"-----A---B"


def test_lzw_round_trip_crosses_the_10_and_11_bit_widths():
    rng = np.random.default_rng(3)
    data = bytes(rng.integers(0, 4, 6000, dtype=np.uint8))  # 4 symbols: long code table

    assert _lzw_decode(lzw_encode(data)) == data


def test_neighbouring_tiles_are_read_in_one_request():
    """Over HTTP each request costs a round trip: a window's tiles, side by side in the
    file, come in one."""
    data = samples(1, 128, 128, "u1")
    reader = CountingReader(write_tiff(data, planar=1, predictor=1, compression=8))
    tiff = GeoTiff(reader)
    reader.requests.clear()
    np.testing.assert_array_equal(tiff.window(0, 0, 0, 128, 128), data[0])
    assert len(reader.requests) == 1


def test_the_header_and_its_directories_are_read_in_a_block_or_two():
    """The directories are parsed a few bytes at a time; over HTTP each would be a round trip."""
    data = samples(3, 128, 128, "float32")
    reader = CountingReader(write_tiff(data, predictor=3))
    GeoTiff(reader, pixel=1.0)
    assert len(reader.requests) <= 2