Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
9.3 KB
Raw Blame
"""Rebuild the grid index of 3D-GloBFP: src/azimut/engine/buildings/globfp_grids.py.

3D-GloBFP (2024, ESSD 16:5357, DOI 10.5281/zenodo.15487037) is
distributed as one zipped shapefile per grid cell, over ten figshare articles.
The app needs to know, before it downloads anything, which grid covers which
part of the world. Most file names carry the bounds (`gridID_west_south_east_
north_regions.zip`). The Chinese and US files are named by province or state,
so their bounds are read from the .shp header instead: the zip is read through
HTTP Range requests, the central directory and then the start of the .shp
member, so no whole file is ever downloaded here.

A file whose bounds cannot be read is logged and left out; the summary says how
many were left out, so the maintainer can see it.

Run it once from the repo root, when the dataset's file list changes:

    uv run python scripts/build_globfp_index.py
"""

from __future__ import annotations

import datetime
import json
import logging
import math
import os
import re
import struct
import sys
import zlib
from pathlib import Path

import httpx

ARTICLES = (
    28879733, 28881749, 28882700, 28889813, 28890593,
    28891631, 28903454, 28903853, 28904453, 28906499,
)
FILES_API = "https://api.figshare.com/v2/articles/{article}/files?page_size=1000&page={page}"
PAGE_SIZE = 1000
DOI = "10.5281/zenodo.15487037"
LICENCE = "CC BY 4.0"
OUT = Path(__file__).resolve().parent.parent / "src/azimut/engine/buildings/globfp_grids.py"
NAME_RE = re.compile(r"^(\d+)_(-?[\d.]+)_(-?[\d.]+)_(-?[\d.]+)_(-?[\d.]+)_.+\.zip$")
ZIP_EOCD = 0x06054B50
ZIP_CENTRAL = 0x02014B50
ZIP_LOCAL = 0x04034B50
SHP_CODE = 9994
# The bounds are rounded outwards, to the micro-degree, so the rounding never
# drops a tile at a grid's edge.
DECIMALS = 6
TIMEOUT = httpx.Timeout(60.0)

log = logging.getLogger("build_globfp_index")
Entry = tuple[str, float, float, float, float, int, int]


def list_files(client: httpx.Client, article: int) -> list[dict]:
    """Every file of one figshare article, one page at a time."""
    files: list[dict] = []
    page = 1
    while True:
        response = client.get(FILES_API.format(article=article, page=page))
        response.raise_for_status()
        batch = response.json()
        files.extend(batch)
        if len(batch) < PAGE_SIZE:
            return files
        page += 1


def _range(client: httpx.Client, url: str, start: int, end: int) -> bytes:
    """Bytes start..end (inclusive) of a file; the host must answer with a range."""
    response = client.get(url, headers={"Range": f"bytes={start}-{end}"})
    if response.status_code != 206:
        raise RuntimeError(f"no range support (HTTP {response.status_code})")
    return response.content


def _central_directory(client: httpx.Client, url: str, size: int) -> list[tuple[str, int, int, int]]:
    """(name, method, compressed size, local header offset) of every member of a zip."""
    tail_len = min(size, 65_557)  # the end record plus the longest possible comment
    tail = _range(client, url, size - tail_len, size - 1)
    at = tail.rfind(struct.pack("<I", ZIP_EOCD))
    if at < 0 or at + 22 > len(tail):
        raise ValueError("no end-of-central-directory record")
    _sig, _disk, _disk_cd, _on_disk, _total, cd_size, cd_offset, _comment = struct.unpack_from(
        "<IHHHHIIH", tail, at
    )
    if cd_offset == 0xFFFFFFFF:
        raise ValueError("zip64 archives are not read here")
    cd = _range(client, url, cd_offset, cd_offset + cd_size - 1)
    members: list[tuple[str, int, int, int]] = []
    pos = 0
    while pos + 46 <= len(cd) and struct.unpack_from("<I", cd, pos)[0] == ZIP_CENTRAL:
        fields = struct.unpack_from("<IHHHHHHIIIHHHHHII", cd, pos)
        method, csize, name_len, extra_len, comment_len, local_offset = (
            fields[4], fields[8], fields[10], fields[11], fields[12], fields[16],
        )
        name = cd[pos + 46 : pos + 46 + name_len].decode("utf-8", "replace")
        members.append((name, method, csize, local_offset))
        pos += 46 + name_len + extra_len + comment_len
    return members


def _shp_head(client: httpx.Client, url: str, member: tuple[str, int, int, int]) -> bytes:
    """The first 100 bytes of one .shp member, inflated if it is deflated."""
    _name, method, csize, local_offset = member
    local = _range(client, url, local_offset, local_offset + 29)
    if struct.unpack_from("<I", local, 0)[0] != ZIP_LOCAL:
        raise ValueError("bad local file header")
    name_len, extra_len = struct.unpack_from("<HH", local, 26)
    start = local_offset + 30 + name_len + extra_len
    if method == 0:
        return _range(client, url, start, start + 99)
    if method != 8:
        raise ValueError(f"unsupported compression method {method}")
    inflater = zlib.decompressobj(-15)
    out = b""
    pos = start
    while len(out) < 100 and pos < start + csize:
        chunk = _range(client, url, pos, min(pos + 4095, start + csize - 1))
        if not chunk:
            break
        out += inflater.decompress(chunk)
        pos += len(chunk)
    if len(out) < 100:
        raise ValueError("the .shp header is cut short")
    return out[:100]


def header_bounds(client: httpx.Client, url: str, size: int) -> tuple[float, float, float, float]:
    """The union of the bounding boxes in every .shp member's header."""
    shps = [m for m in _central_directory(client, url, size) if m[0].lower().endswith(".shp")]
    if not shps:
        raise ValueError("no .shp member")
    boxes = []
    for member in shps:
        head = _shp_head(client, url, member)
        if struct.unpack_from(">i", head, 0)[0] != SHP_CODE:
            raise ValueError(f"{member[0]} is not a shapefile")
        boxes.append(struct.unpack_from("<4d", head, 36))  # xmin, ymin, xmax, ymax
    west = min(b[0] for b in boxes)
    south = min(b[1] for b in boxes)
    east = max(b[2] for b in boxes)
    north = max(b[3] for b in boxes)
    if not (-180 <= west < east <= 180 and -90 <= south < north <= 90):
        raise ValueError(f"implausible bounds {west, south, east, north}")
    return west, south, east, north


def _outward(value: float, up: bool) -> float:
    scale = 10**DECIMALS
    return (math.ceil(value * scale) if up else math.floor(value * scale)) / scale


def build(client: httpx.Client) -> tuple[list[Entry], int, list[str]]:
    """The index entries, how many were bound by header, and the names left out."""
    entries: dict[str, Entry] = {}
    from_header = 0
    failed: list[str] = []
    for article in ARTICLES:
        for file in list_files(client, article):
            name, file_id, size = file["name"], int(file["id"]), int(file["size"])
            if name in entries and entries[name][6] >= size:
                log.warning("duplicate file name %s in article %s: smaller copy skipped", name, article)
                continue
            if name in entries:
                # The same name twice: the larger copy is the whole cell, the smaller one a part.
                log.warning("duplicate file name %s in article %s: larger copy kept", name, article)
            match = NAME_RE.match(name)
            if match:
                west, south, east, north = (float(g) for g in match.groups()[1:])
            else:
                try:
                    west, south, east, north = header_bounds(client, file["download_url"], size)
                except (httpx.HTTPError, RuntimeError, ValueError, struct.error, zlib.error) as exc:
                    log.error("left out %s: %s", name, exc)
                    failed.append(name)
                    continue
                from_header += 1
            entries[name] = (
                name,
                _outward(west, up=False), _outward(south, up=False),
                _outward(east, up=True), _outward(north, up=True),
                file_id, size,
            )
    return sorted(entries.values()), from_header, sorted(failed)


def render(entries: list[Entry], generated: str) -> str:
    lines = [
        '"""3D-GloBFP grid index: generated, do not edit by hand.',
        "",
        "Written by scripts/build_globfp_index.py on " + generated + " from the ten figshare",
        "articles of the dataset (2024, DOI " + DOI + ").",
        "Licence: " + LICENCE + ". Each entry is (name, west, south, east, north, file_id,",
        "size_bytes), sorted by name; the bounds are WGS84 degrees.",
        '"""',
        "",
        "GRIDS: tuple[tuple[str, float, float, float, float, int, int], ...] = (",
    ]
    for name, west, south, east, north, file_id, size in entries:
        lines.append(
            f"    ({json.dumps(name)}, {west!r}, {south!r}, {east!r}, {north!r}, {file_id}, {size}),"
        )
    lines.append(")")
    return "\n".join(lines) + "\n"


def main() -> int:
    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
    with httpx.Client(timeout=TIMEOUT, follow_redirects=True) as client:
        entries, from_header, failed = build(client)
    generated = datetime.date.today().isoformat()
    part = OUT.with_name(OUT.name + ".part")
    part.write_text(render(entries, generated), encoding="utf-8", newline="\n")
    os.replace(part, OUT)
    print(f"entries:        {len(entries)}")
    print(f"bound by header: {from_header}")
    print(f"failed:         {len(failed)}")
    for name in failed:
        print(f"  left out: {name}")
    print(f"wrote {OUT}")
    return 0


if __name__ == "__main__":
    sys.exit(main())