The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.
JavaScript
"""The buildings service: what it fetches, when, what it keeps and what it says."""
from __future__ import annotations
import pytest
from azimut.engine.buildings import globfp, google25d, ign, model, nl3dbag, nrw, nyc, osm, overture, service, wire
LAT, LON = 45.7578, 4.8320
KEY = model.tile_of(LAT, LON)
GRID = globfp.Grid("9999_4.0_45.0_5.25_46.25_FR.zip", 4.0, 45.0, 5.25, 46.25, 1, 51_000_000)
def house(lat: float, lon: float, *, height: float | None, kind: str, source: str, date: str = "") -> model.Building:
d = 0.00006 # about 5 to 7 m
ring = ((lon - d, lat - d), (lon + d, lat - d), (lon + d, lat + d), (lon - d, lat + d))
return model.Building(rings=(ring,), height=height, kind=kind, source=source, date=date) # type: ignore[arg-type]
class Sources:
"""Stand-ins for the three sources, counting what is asked of them."""
def __init__(self, monkeypatch, *, ign_covers=True, osm_fails=False, grid_ready=False, traced=None):
self.calls: list[tuple[str, model.Key]] = []
self.traced = traced or []
self.grid_ready = grid_ready
self.downloads = 0
monkeypatch.setattr(ign, "covers", lambda z, x, y: ign_covers)
# the Netherlands and Google's rasters have nothing here
monkeypatch.setattr(nl3dbag, "covers", lambda z, x, y: False)
monkeypatch.setattr(nrw, "covers", lambda z, x, y: False)
monkeypatch.setattr(nyc, "covers", lambda z, x, y: False)
monkeypatch.setattr(overture, "fetch_tile", self._overture)
monkeypatch.setattr(google25d, "cached_rasters", lambda z, x, y: [])
monkeypatch.setattr(ign, "fetch_tile", self._ign)
monkeypatch.setattr(osm, "fetch_tile", self._osm_failing if osm_fails else self._osm)
monkeypatch.setattr(globfp, "grids_for_tile", lambda z, x, y: [GRID])
monkeypatch.setattr(globfp, "is_ready", lambda grid: self.grid_ready)
monkeypatch.setattr(globfp, "needed", lambda z, x, y: [] if self.grid_ready else [GRID])
monkeypatch.setattr(globfp, "all_grids", lambda: [GRID])
monkeypatch.setattr(globfp, "download", self._download)
monkeypatch.setattr(globfp, "read_tile", self._globfp)
def _here(self, z: int, x: int, y: int) -> bool:
return (z, x, y) == KEY
def _ign(self, z, x, y):
self.calls.append(("ign", (z, x, y)))
return [house(LAT, LON, height=18.6, kind="measured", source="ign", date="2022-08-10")] if self._here(z, x, y) else []
def _osm(self, z, x, y):
self.calls.append(("osm", (z, x, y)))
return [house(LAT, LON, height=None, kind="unknown", source="osm", date="2026-10-04")] if self._here(z, x, y) else []
def _overture(self, z, x, y):
self.calls.append(("overture", (z, x, y)))
return list(self.traced) if self._here(z, x, y) else []
def _osm_failing(self, z, x, y):
self.calls.append(("osm", (z, x, y)))
raise model.BuildingsUnavailable("OpenStreetMap buildings could not be read just now (timeout)")
def _globfp(self, z, x, y):
self.calls.append(("globfp", (z, x, y)))
return [house(LAT, LON, height=9.5, kind="estimated", source="globfp", date="2020")] if self._here(z, x, y) else []
def _download(self, grid, progress=None, cancelled=None):
self.downloads += 1
if progress:
progress(grid.size, grid.size)
self.grid_ready = True
@pytest.fixture(autouse=True)
def fresh(tmp_workspace):
service.forget()
yield
service.wait_idle()
service.forget()
def settled(sources=None, radius=500.0):
"""Ask until every tile is in, as the view's polling would."""
for _ in range(50):
answer = service.status(LAT, LON, radius, sources)
service.wait_idle()
if not answer["pending"] and not answer["working"] and not any(d["done"] < d["total"] for d in answer["downloads"]):
return service.status(LAT, LON, radius, sources)
raise AssertionError("the area never settled")
def test_the_tiles_round_an_eye_are_nearest_first_and_bounded():
keys = service.tiles_around(LAT, LON, 2000)
assert keys[0] == KEY
assert 4 <= len(keys) <= 16
assert len(service.tiles_around(LAT, LON, 1e9)) <= service.MAX_TILES
assert service.tiles_around(LAT, LON, 0) == [KEY]
def test_the_tiles_round_an_eye_go_on_across_the_antimeridian():
n = 2**model.ZOOM
# Taveuni, Fiji: the 180th meridian runs a kilometre from the eye, either way round
for lon in (179.99, -179.99):
keys = service.tiles_around(-16.8, lon, 2000)
assert {x for _z, x, _y in keys} >= {0, n - 1}
assert all(0 <= x < n and 0 <= y < n for _z, x, y in keys)
assert service.tiles_around(-16.8, 179.99, 100) == [model.tile_of(-16.8, 179.99)]
def test_nothing_is_asked_until_the_area_is(monkeypatch):
sources = Sources(monkeypatch)
assert sources.calls == [] and sources.downloads == 0
def test_the_area_fills_from_the_quick_sources_and_says_where_each_height_comes_from(monkeypatch):
Sources(monkeypatch)
answer = settled()
assert answer["counts"]["measured"] == 1
assert answer["footprints"] == {"ign": 1}
ign_source = next(s for s in answer["sources"] if s["id"] == "ign")
assert ign_source["licence"] == "Licence Ouverte 2.0" and ign_source["newest"] == "2022-08-10"
# 3D-GloBFP waits for the analyst, with its size in front of them
assert [n for n in answer["needs"] if n["source"] == "globfp"] == [
{"grid": GRID.name, "size": GRID.size, "source": "globfp", "label": "3D-GloBFP"}
]
def test_a_source_is_read_once_then_from_the_workspace(monkeypatch):
sources = Sources(monkeypatch)
settled()
asked = len(sources.calls)
service.forget()
settled()
assert len(sources.calls) == asked
def test_a_failing_source_is_said_and_the_tile_is_drawn_from_the_others(monkeypatch):
Sources(monkeypatch, ign_covers=False, osm_fails=True, grid_ready=True)
answer = settled()
assert answer["failed"] == [{"source": "osm", "reason": "OpenStreetMap buildings could not be read just now (timeout)"}]
assert answer["retry_in"] > 0
assert answer["counts"]["estimated"] == 1
assert answer["footprints"] == {"globfp": 1}
def test_a_failed_tile_is_not_asked_again_at_once(monkeypatch):
sources = Sources(monkeypatch, ign_covers=False, osm_fails=True, grid_ready=True)
settled()
asked = sum(1 for source, _ in sources.calls if source == "osm")
settled()
assert sum(1 for source, _ in sources.calls if source == "osm") == asked
def test_a_grid_downloads_only_once_the_analyst_agrees_and_the_tile_is_drawn_again(monkeypatch):
sources = Sources(monkeypatch, ign_covers=False)
before = settled()
assert sources.downloads == 0
assert before["counts"] == {"measured": 0, "estimated": 0, "range": 0, "unknown": 1}
version = next(t["version"] for t in before["tiles"] if t["key"] == service.key_text(KEY))
assert service.allow([GRID.name, "not-a-grid.zip"]) == [GRID.name]
after = settled()
assert sources.downloads == 1
assert [n for n in after["needs"] if n["source"] == "globfp"] == [] and after["downloads"] == []
# the OSM house now carries 3D-GloBFP's estimate, and its tile a new version
assert after["counts"]["estimated"] == 1 and after["counts"]["unknown"] == 0
assert next(t["version"] for t in after["tiles"] if t["key"] == service.key_text(KEY)) != version
def test_a_failed_download_is_said_and_may_be_asked_again(monkeypatch):
sources = Sources(monkeypatch, ign_covers=False)
def refuse(grid, progress=None, cancelled=None):
raise model.BuildingsUnavailable("3D-GloBFP answered HTTP 503 for the grid")
monkeypatch.setattr(globfp, "download", refuse)
service.allow([GRID.name])
service.wait_idle()
answer = service.status(LAT, LON, 500)
assert answer["downloads"][0]["error"] == "3D-GloBFP answered HTTP 503 for the grid"
assert [n for n in answer["needs"] if n["source"] == "globfp"] == [
{"grid": GRID.name, "size": GRID.size, "source": "globfp", "label": "3D-GloBFP"}
]
monkeypatch.setattr(globfp, "download", sources._download)
assert service.allow([GRID.name]) == [GRID.name]
assert settled()["counts"]["estimated"] == 1
def test_ready_tiles_come_as_bytes_the_view_reads_and_others_as_nothing(monkeypatch):
Sources(monkeypatch)
settled()
far = (model.ZOOM, KEY[1] + 40, KEY[2])
got = service.tile_bytes([KEY, far])
buildings = wire.decode(got[KEY], *KEY)
assert [(b.source, b.kind) for b in buildings] == [("ign", "measured")]
assert buildings[0].height == pytest.approx(18.6, abs=1e-5) # sent as a 32-bit float
# the far tile was only asked now: it is not in yet
assert got[far] is None
def test_a_switched_off_source_is_neither_read_nor_drawn(monkeypatch):
sources = Sources(monkeypatch)
answer = settled(sources=["osm"])
assert {source for source, _ in sources.calls} == {"osm"}
assert answer["needs"] == [] and answer["footprints"] == {"osm": 1}
def test_the_building_under_a_point_says_whose_footprint_and_whose_height(monkeypatch):
Sources(monkeypatch, ign_covers=False, grid_ready=True)
settled()
found = service.building_at(LAT, LON)
assert found is not None
assert (found["source"], found["height_source"], found["height"], found["kind"]) == ("osm", "globfp", 9.5, "estimated")
assert (found["date"], found["height_date"]) == ("2026-10-04", "2020")
assert service.building_at(LAT + 0.001, LON) is None
def test_clearing_forgets_every_tile_kept(monkeypatch):
sources = Sources(monkeypatch)
settled()
assert service.clear() > 0
asked = len(sources.calls)
settled()
assert len(sources.calls) > asked
def test_the_sweep_holds_the_tile_files_under_their_budget(monkeypatch):
Sources(monkeypatch)
settled(radius=2000)
files = list((model.cache_root() / "tiles").rglob("*.json.gz"))
assert len(files) > 4
freed = service.sweep(tiles_budget=1)
assert freed > 0
assert len(list((model.cache_root() / "tiles").rglob("*.json.gz"))) < len(files)
def test_where_the_ign_has_surveyed_the_tile_no_grid_is_offered(monkeypatch):
sources = Sources(monkeypatch)
answer = settled()
# the IGN covers and fills the eye's tile: only the empty tiles round it may ask for the grid
assert [n for n in answer["needs"] if n["source"] == "globfp"] == [
{"grid": GRID.name, "size": GRID.size, "source": "globfp", "label": "3D-GloBFP"}
]
assert ("globfp", KEY) not in sources.calls
lone = settled(radius=100)
assert lone["needs"] == []
def test_a_point_read_off_a_wall_finds_its_building(monkeypatch):
Sources(monkeypatch, ign_covers=False, grid_ready=True)
settled()
# the house is about 6.5 m from its middle to its east wall: 1 m past the wall still finds it
kx, _ky = model.metres_frame(LAT)
assert service.building_at(LAT, LON + 7.5 / kx) is not None
assert service.building_at(LAT, LON + 15.0 / kx) is None
RASTER = google25d.Raster("v1/geotiffs/00824_2023_06_30/tile_a.tif", 32631, 0, 0, 1, 1)
def googled(monkeypatch):
"""Google's rasters over the eye's tile: looked up in the background, read on a yes."""
known: dict = {}
read: dict = {}
monkeypatch.setattr(google25d, "cached_rasters", lambda z, x, y: known.get((z, x, y)))
def resolve(z, x, y):
known[(z, x, y)] = [RASTER] if (z, x, y) == KEY else []
return known[(z, x, y)]
def fetch(z, x, y):
read[(z, x, y)] = 1.0
return 1
monkeypatch.setattr(google25d, "resolve_tile", resolve)
monkeypatch.setattr(google25d, "fetch_tile", fetch)
monkeypatch.setattr(google25d, "has_tile", lambda z, x, y: read.get((z, x, y)))
monkeypatch.setattr(google25d, "sampler", lambda z, x, y: (lambda buildings: [14.5] * len(buildings)))
return read
def test_google_heights_wait_for_a_yes_then_lend_their_estimate(monkeypatch):
Sources(monkeypatch, ign_covers=False)
read = googled(monkeypatch)
before = settled()
assert [n for n in before["needs"] if n["source"] == "google"] == [
{"grid": "google", "size": google25d.TILE_BYTES, "source": "google", "label": "Open Buildings 2.5D (Google)"}
]
assert before["counts"]["unknown"] == 1 and not read
assert service.allow(["google"]) == ["google"]
after = settled()
assert after["counts"]["estimated"] == 1
found = service.building_at(LAT, LON)
assert (found["height"], found["height_source"], found["height_date"]) == (14.5, "google", "2023")
def test_the_netherlands_survey_draws_its_own_footprints(monkeypatch):
sources = Sources(monkeypatch, ign_covers=False)
monkeypatch.setattr(nl3dbag, "covers", lambda z, x, y: True)
def bag(z, x, y):
sources.calls.append(("nl3dbag", (z, x, y)))
return [house(LAT, LON, height=21.0, kind="measured", source="nl3dbag")] if (z, x, y) == KEY else []
monkeypatch.setattr(nl3dbag, "fetch_tile", bag)
answer = settled()
assert answer["footprints"].get("nl3dbag") == 1
assert service.building_at(LAT, LON)["height"] == 21.0
def test_overture_waits_for_a_yes_then_draws_the_footprints_osm_lacks(monkeypatch):
street = [house(LAT, LON + i * 0.0003, height=None, kind="unknown", source="overture", date="2023-05-01")
for i in range(6)]
sources = Sources(monkeypatch, ign_covers=False, traced=street)
before = settled()
assert not any(name == "overture" for name, _key in sources.calls)
(asked,) = [n for n in before["needs"] if n["source"] == "overture"]
assert asked["label"] == "Overture Maps" and asked["size"] % overture.TILE_BYTES == 0
assert before["footprints"] == {"osm": 1}
assert service.allow(["overture"]) == ["overture"]
after = settled()
assert ("overture", KEY) in sources.calls
assert after["footprints"] == {"overture": 6}
assert not [n for n in after["needs"] if n["source"] == "overture"]
used = next(s for s in after["sources"] if s["id"] == "overture")
assert used["newest"] == "2023-05-01"
def test_overture_is_not_offered_where_a_survey_has_the_tile(monkeypatch):
sources = Sources(monkeypatch)
assert all(n["source"] != "overture" for n in settled(radius=100)["needs"])
assert not any(name == "overture" for name, _key in sources.calls)
def test_a_tile_shows_its_last_buildings_while_the_next_merge_runs(monkeypatch):
import threading
Sources(monkeypatch, ign_covers=False)
first = settled()
shown = next(t for t in first["tiles"] if t["key"] == service.key_text(KEY))
release = threading.Event()
real = service.merge.merge_tile
def slow(*args, **kwargs):
release.wait(5)
return real(*args, **kwargs)
monkeypatch.setattr(service.merge, "merge_tile", slow)
service.allow([GRID.name])
service.wait_idle(timeout=0.5) # the grid is in, the new merge waits
answer = service.status(LAT, LON, 500)
now = next(t for t in answer["tiles"] if t["key"] == service.key_text(KEY))
# answered at once, with the buildings it had, and saying more is coming
assert now == shown and answer["working"] > 0 and answer["counts"]["unknown"] == 1
release.set()
assert settled()["counts"]["estimated"] == 1
def test_a_source_moving_from_waiting_to_downloading_sends_nothing_again(monkeypatch):
import threading
Sources(monkeypatch, ign_covers=False)
before = {t["key"]: t["version"] for t in settled()["tiles"]}
release = threading.Event()
monkeypatch.setattr(overture, "fetch_tile", lambda z, x, y: release.wait(5) and [])
service.allow(["overture"])
answer = service.status(LAT, LON, 500)
# Overture is being read: the buildings are the same, so is the version
assert {t["key"]: t["version"] for t in answer["tiles"]} == before
assert answer["working"] > 0
release.set()