Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
14.2 KB
Raw Blame
"""3D-GloBFP: download, index and read, with no network.

The grid is a small shapefile written here, zipped, and served from memory through
a mock transport, so the real download, unpack, index and read code run end to end.
"""

from __future__ import annotations

import io
import struct
import zipfile

import httpx
import numpy as np
import pytest

from azimut.engine.buildings import globfp, globfp_grids, model

LAT, LON = 12.83, 43.18
TILE = model.tile_of(LAT, LON, model.ZOOM)
_z, _x, _y = TILE
WEST, SOUTH, EAST, NORTH = model.tile_bounds(*TILE)
EAST_TILE = model.tile_of((SOUTH + NORTH) / 2, EAST + (EAST - WEST) / 2, model.ZOOM)
EAST_WEST, _s, EAST_EAST, _n = model.tile_bounds(*EAST_TILE)
MID_LAT = (SOUTH + NORTH) / 2


def square(cx: float, cy: float, half: float, clockwise: bool = True) -> list[tuple[float, float]]:
    ring = [(cx - half, cy - half), (cx + half, cy - half), (cx + half, cy + half), (cx - half, cy + half)]
    return ring if not clockwise else list(reversed(ring))


def _polygon(rings: list[list[tuple[float, float]]]) -> bytes:
    points: list[tuple[float, float]] = []
    parts: list[int] = []
    for ring in rings:
        parts.append(len(points))
        points.extend(list(ring) + [ring[0]])
    xs = [p[0] for p in points]
    ys = [p[1] for p in points]
    flat = [c for p in points for c in p]
    return (
        struct.pack("<i", 5)
        + struct.pack("<4d", min(xs), min(ys), max(xs), max(ys))
        + struct.pack("<ii", len(rings), len(points))
        + struct.pack(f"<{len(parts)}i", *parts)
        + struct.pack(f"<{len(flat)}d", *flat)
    )


def _file_header(length: int, bbox: tuple[float, float, float, float]) -> bytes:
    return (
        struct.pack(">i", 9994)
        + b"\0" * 20
        + struct.pack(">i", length // 2)
        + struct.pack("<ii", 1000, 5)
        + struct.pack("<4d", *bbox)
        + b"\0" * 32
    )


def _dbf(heights: list[float]) -> bytes:
    fields = [(b"FID_1", b"N", 10, 0), (b"Height", b"N", 12, 2)]
    header_len = 32 + 32 * len(fields) + 1
    record_len = 1 + sum(size for _n, _t, size, _d in fields)
    out = struct.pack("<BBBBIHH", 3, 126, 1, 1, len(heights), header_len, record_len) + b"\0" * 20
    for name, kind, size, dec in fields:
        out += name.ljust(11, b"\0") + kind + b"\0" * 4 + bytes([size, dec]) + b"\0" * 14
    out += b"\r"
    for i, height in enumerate(heights):
        out += b" " + str(i + 1).rjust(10).encode() + f"{height:.2f}".rjust(12).encode()
    return out


def shapefile(polygons: list[list[list[tuple[float, float]]]], heights: list[float]) -> dict[str, bytes]:
    """The .shp, .shx and .dbf of one shapefile, one record per polygon."""
    body = b""
    index = b""
    offset = 100
    points = [p for poly in polygons for ring in poly for p in ring]
    for number, poly in enumerate(polygons, 1):
        content = _polygon(poly)
        body += struct.pack(">ii", number, len(content) // 2) + content
        index += struct.pack(">ii", offset // 2, len(content) // 2)
        offset += 8 + len(content)
    bbox = (
        min(p[0] for p in points), min(p[1] for p in points),
        max(p[0] for p in points), max(p[1] for p in points),
    )
    return {
        ".shp": _file_header(100 + len(body), bbox) + body,
        ".shx": _file_header(100 + len(index), bbox) + index,
        ".dbf": _dbf(heights),
    }


# Dotted, like the real cells ("...13.75_DJ_ER_YM"): a suffix taken with
# `with_suffix` would cut such a name in the wrong place.
DOTTED = "cell_1563_42.5_12.5_43.75_13.75"


def zipped(members: dict[str, bytes], stem: str = DOTTED) -> bytes:
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
        for ext, data in members.items():
            archive.writestr(f"{stem}{ext}", data)
    return buffer.getvalue()


# One cell of the test grid: a building in the tile, one in the tile to the east,
# one across their border, a courtyard, two parts, a zero height, and a shed.
def _cell_polygons() -> tuple[list[list[list[tuple[float, float]]]], list[float]]:
    inside = (WEST + 0.004, SOUTH + 0.004)
    polygons = [
        [square(WEST + 0.002, MID_LAT, 0.0002)],  # 0: in the tile, 12.5 m
        [square(EAST_WEST + 0.005, MID_LAT, 0.0002)],  # 1: only in the east tile, 30 m
        [square(EAST, MID_LAT, 0.0002)],  # 2: across the border, 8 m
        [  # 3: a courtyard, outer clockwise, hole counter-clockwise
            square(inside[0], inside[1], 0.0005),
            square(inside[0], inside[1], 0.0002, clockwise=False),
        ],
        [  # 4: two separate clockwise parts in one record
            square(WEST + 0.012, SOUTH + 0.002, 0.0005),
            square(WEST + 0.012, SOUTH + 0.012, 0.0005),
        ],
        [square(WEST + 0.016, SOUTH + 0.016, 0.0002)],  # 5: height 0, unknown
        [square(WEST + 0.006, SOUTH + 0.010, 0.000005)],  # 6: about 1 m², dropped
    ]
    heights = [12.5, 30.0, 8.0, 20.0, 5.0, 0.0, 15.0]
    return polygons, heights


def _grid(size: int, file_id: int = 1, name: str = "cell.zip") -> globfp.Grid:
    return globfp.Grid(
        name=name, west=WEST - 0.01, south=SOUTH - 0.01, east=EAST_EAST + 0.01, north=NORTH + 0.01,
        file_id=file_id, size=size,
    )


@pytest.fixture()
def cell(monkeypatch, tmp_workspace):
    """The test grid's zip, registered as the one grid the app knows of."""
    polygons, heights = _cell_polygons()
    body = zipped(shapefile(polygons, heights))
    grid = _grid(len(body))
    monkeypatch.setattr(globfp_grids, "GRIDS", ((grid.name, grid.west, grid.south, grid.east, grid.north, grid.file_id, grid.size),))
    return grid, body


def _serve(monkeypatch, body: bytes | None = None, status: int = 200, error: Exception | None = None):
    """Route the download's GET to memory. Returns the list of requested URLs."""
    asked: list[str] = []

    def handler(request: httpx.Request) -> httpx.Response:
        asked.append(str(request.url))
        if error is not None:
            raise error
        if status != 200 or body is None:
            return httpx.Response(status)
        return httpx.Response(200, content=body, headers={"content-length": str(len(body))})

    def make_client() -> httpx.Client:
        return httpx.Client(transport=httpx.MockTransport(handler), follow_redirects=True)

    monkeypatch.setattr(globfp, "_client", make_client)
    return asked


def _summary(buildings: list[model.Building]) -> list[tuple[float | None, str, int]]:
    return sorted(((b.height, b.kind, len(b.rings)) for b in buildings), key=repr)


def _root():
    return model.cache_root() / globfp.SOURCE


def test_download_then_read_returns_the_buildings_of_each_tile(monkeypatch, cell):
    grid, body = cell
    asked = _serve(monkeypatch, body)

    globfp.download(grid)

    assert asked == [grid.url]
    assert globfp.is_ready(grid)
    here = globfp.read_tile(*TILE)
    assert _summary(here) == sorted(
        [
            (5.0, "estimated", 1),
            (5.0, "estimated", 1),
            (8.0, "estimated", 1),
            (12.5, "estimated", 1),
            (20.0, "estimated", 2),
            (None, "unknown", 1),
        ],
        key=repr,
    )
    assert {b.source for b in here} == {"globfp"}
    assert {b.date for b in here} == {"2020"}
    assert {b.min_height for b in here} == {0.0}
    there = globfp.read_tile(*EAST_TILE)
    assert _summary(there) == sorted([(8.0, "estimated", 1), (30.0, "estimated", 1)], key=repr)


def test_a_tile_with_no_buildings_reads_empty(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    far = model.tile_of(LAT, LON + 0.1, model.ZOOM)
    assert globfp.read_tile(*far) == []


def test_a_building_across_two_tiles_comes_back_in_both(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    straddler = [b for b in globfp.read_tile(*TILE) if b.height == 8.0]
    beyond = [b for b in globfp.read_tile(*EAST_TILE) if b.height == 8.0]
    assert len(straddler) == 1 and len(beyond) == 1
    assert straddler[0].rings == beyond[0].rings


def test_a_courtyard_is_a_hole_of_its_outer_ring(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    (courtyard,) = [b for b in globfp.read_tile(*TILE) if b.height == 20.0]
    assert len(courtyard.rings) == 2
    outer, hole = courtyard.rings
    lat0 = outer[0][1]
    assert model.ring_area(outer, lat0) < 0 < model.ring_area(hole, lat0)  # clockwise, counter-clockwise
    assert abs(model.ring_area(outer, lat0)) > abs(model.ring_area(hole, lat0))
    assert len(outer) == 4 and len(hole) == 4
    assert outer[0] != outer[-1]  # rings come out open


def test_two_separate_parts_are_two_buildings(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    parts = [b for b in globfp.read_tile(*TILE) if b.height == 5.0]
    assert len(parts) == 2
    assert all(len(b.rings) == 1 for b in parts)
    assert parts[0].rings != parts[1].rings


def test_a_zero_height_is_unknown_not_zero(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    unknown = [b for b in globfp.read_tile(*TILE) if b.kind == "unknown"]
    assert len(unknown) == 1
    assert unknown[0].height is None


def test_a_shed_under_the_minimum_area_is_dropped(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    assert all(b.height != 15.0 for b in globfp.read_tile(*TILE))


def test_progress_reports_the_bytes_of_the_zip(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    seen: list[tuple[int, int]] = []

    globfp.download(grid, progress=lambda done, total: seen.append((done, total)))

    assert seen
    assert seen[-1] == (len(body), len(body))
    assert [done for done, _total in seen] == sorted(done for done, _total in seen)


def test_a_cancelled_download_leaves_nothing(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)

    globfp.download(grid, cancelled=lambda: True)

    assert not globfp.is_ready(grid)
    assert globfp.read_tile(*TILE) == []
    incoming = _root() / "incoming"
    assert not incoming.exists() or list(incoming.iterdir()) == []
    assert not (_root() / grid.folder_name).exists()
    leftovers = [p for p in _root().iterdir() if p.name != "incoming"] if _root().exists() else []
    assert leftovers == []


@pytest.mark.parametrize(
    "answer",
    [
        {"status": 500},
        {"error": httpx.ConnectError("refused")},
    ],
    ids=["http-error", "no-connection"],
)
def test_a_failed_download_is_unavailable_and_leaves_nothing(monkeypatch, cell, answer):
    grid, _body = cell
    _serve(monkeypatch, **answer)

    with pytest.raises(model.BuildingsUnavailable):
        globfp.download(grid)

    assert not globfp.is_ready(grid)
    assert not (_root() / grid.folder_name).exists()
    assert list((_root() / "incoming").iterdir()) == []


def test_a_download_that_is_not_a_zip_is_unavailable(monkeypatch, cell):
    grid, _body = cell
    _serve(monkeypatch, b"<html>not a grid</html>")

    with pytest.raises(model.BuildingsUnavailable):
        globfp.download(grid)
    assert not globfp.is_ready(grid)


def test_a_zip_without_a_complete_shapefile_is_unavailable(monkeypatch, cell):
    grid, _body = cell
    shp_only = zipped({".shp": shapefile(*_cell_polygons())[".shp"]})
    _serve(monkeypatch, shp_only)

    with pytest.raises(model.BuildingsUnavailable):
        globfp.download(grid)
    assert not globfp.is_ready(grid)


def test_read_never_touches_the_network(monkeypatch, cell):
    grid, body = cell

    def no_client() -> httpx.Client:
        raise AssertionError("read_tile must not reach the network")

    monkeypatch.setattr(globfp, "_client", no_client)
    assert globfp.read_tile(*TILE) == []  # not downloaded yet

    _serve(monkeypatch, body)
    globfp.download(grid)
    monkeypatch.setattr(globfp, "_client", no_client)
    assert len(globfp.read_tile(*TILE)) == 6


def test_needed_lists_the_grids_not_on_disk(monkeypatch, cell):
    grid, body = cell
    assert [g.name for g in globfp.needed(*TILE)] == [grid.name]

    _serve(monkeypatch, body)
    globfp.download(grid)
    assert globfp.needed(*TILE) == []


def test_grids_for_tile_follows_the_grid_bounds(monkeypatch):
    rows = (
        ("in.zip", WEST - 0.001, SOUTH - 0.001, WEST + 0.001, SOUTH + 0.001, 1, 10),
        ("beside.zip", EAST, SOUTH, EAST + 1.0, NORTH, 2, 10),  # shares an edge only
        ("away.zip", 0.0, 0.0, 1.0, 1.0, 3, 10),
        ("over.zip", WEST - 1.0, SOUTH - 1.0, EAST + 1.0, NORTH + 1.0, 4, 10),
    )
    monkeypatch.setattr(globfp_grids, "GRIDS", rows)

    names = [g.name for g in globfp.grids_for_tile(*TILE)]
    assert names == ["in.zip", "over.zip"]
    assert globfp.grids_for_tile(*TILE)[0].url == "https://ndownloader.figshare.com/files/1"


def test_redownload_replaces_the_folder(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)
    first = {b.height for b in globfp.read_tile(*TILE)}

    other = shapefile([[square(WEST + 0.003, MID_LAT, 0.0002)]], [77.0])
    _serve(monkeypatch, zipped(other))
    globfp.download(grid)

    assert {b.height for b in globfp.read_tile(*TILE)} == {77.0}
    assert 12.5 in first
    leftovers = [p.name for p in _root().iterdir() if ".old-" in p.name or p.name.endswith(".tmp")]
    assert leftovers == []


def test_the_generated_grid_index_is_well_formed():
    grids = globfp_grids.GRIDS
    assert grids
    names = [row[0] for row in grids]
    assert names == sorted(names)
    assert len({row[5] for row in grids}) == len(grids)
    for name, west, south, east, north, file_id, size in grids:
        assert name.endswith(".zip")
        assert -180 <= west < east <= 180
        assert -90 <= south < north <= 90
        assert file_id > 0 and size > 0


def test_read_tile_refuses_another_zoom(tmp_workspace):
    with pytest.raises(ValueError):
        globfp.read_tile(13, _x, _y)


def test_the_index_is_sorted_by_tile_code(monkeypatch, cell):
    grid, body = cell
    _serve(monkeypatch, body)
    globfp.download(grid)

    with np.load(_root() / grid.folder_name / "index.npz") as data:
        codes = data["codes"]
        assert codes.dtype == np.int64
        assert np.all(np.diff(codes) >= 0)
        assert int(codes.max()) < 2**28  # x and y both below 2**14