Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
14.8 KB
Raw Blame
"""Named summits from OpenFreeMap's tiles: the rings, the cache, the waits, the route."""

import json
import os
import re
import threading
import time

import numpy as np
import pytest

from azimut.engine import mvt, peaks, terrain, tiles
from fakerelief import Relief, offset

EYE = (46.55, 7.9)
TEMPLATE = "https://tiles.openfreemap.org/planet/20261004_test/{z}/{x}/{y}.pbf"
_TILE = re.compile(r"/(\d+)/(\d+)/(\d+)\.pbf$")


@pytest.fixture(autouse=True)
def fresh_tiles():
    """No test inherits another's tiles in flight, failures or build address."""
    peaks.wait_idle()
    peaks.forget()
    yield
    peaks.wait_idle()
    peaks.forget()


def _summit(name, lat, lon, ele=None, kind="peak", **tags):
    return {"name": name, "class": kind, "lat": lat, "lon": lon,
            **({"ele": ele} if ele is not None else {}), **tags}


def _tile(z, x, y, summits):
    """A tile as OpenFreeMap writes one: a layer to step over, then the summits,
    with those just past its edge in its buffer as a tile server keeps them."""
    place = mvt.Layer("place")
    place.points(np.array([10]), np.array([10]), {"name": "Grindelwald", "class": "village"})
    layer = mvt.Layer(peaks.LAYER)
    for summit in summits:
        fx, fy = tiles.project(summit["lat"], summit["lon"], z)
        px, py = round((fx - x) * mvt.EXTENT), round((fy - y) * mvt.EXTENT)
        if -64 <= px < mvt.EXTENT + 64 and -64 <= py < mvt.EXTENT + 64:
            tags = {k: v for k, v in summit.items() if k not in ("lat", "lon")}
            layer.points(np.array([px]), np.array([py]), tags)
    return mvt.encode([place, layer])


def _source(monkeypatch, summits, template=TEMPLATE, tile_status=200, refuse=False):
    """Answer the TileJSON and every tile from these summits, listing what was asked."""
    asked = []

    def get(url):
        asked.append(url)
        if url == peaks.TILEJSON:
            return 200, json.dumps({"tiles": [template]}).encode()
        if refuse:
            raise peaks.PeaksUnavailable("offline")
        if tile_status != 200:
            return tile_status, b""
        z, x, y = (int(part) for part in _TILE.search(url).groups())
        return 200, _tile(z, x, y, summits)

    monkeypatch.setattr(peaks, "_get", get)
    return asked


def _tiles_asked(asked):
    return [url for url in asked if url != peaks.TILEJSON]


def _settled(lat, lon, radius, **kwargs):
    """What a view reads once every tile asked for has come in."""
    peaks.known_around(lat, lon, radius, **kwargs)
    assert peaks.wait_idle()
    return peaks.known_around(lat, lon, radius)


def test_tiles_are_read_deeper_near_the_eye_and_nearest_first():
    plan = peaks.tiles_around(*EYE, 150_000)
    zooms = [key[0] for key, _inner, _outer in plan]
    assert set(zooms) == {12, 11, 10, 9}
    first = plan[0][0]
    assert first == (12, *(int(c) for c in tiles.project(*EYE, 12)))
    # the bands meet end to end, out to the radius
    bands = sorted({(inner, outer) for _key, inner, outer in plan})
    assert bands == [(0.0, 15_000.0), (15_000.0, 40_000.0), (40_000.0, 100_000.0), (100_000.0, 150_000.0)]
    # each band is read at its own zoom
    assert {key[0]: (inner, outer) for key, inner, outer in plan} == {
        12: bands[0], 11: bands[1], 10: bands[2], 9: bands[3],
    }


def test_a_view_across_the_date_line_reads_tiles_on_both_sides():
    keys = {key for key, _inner, _outer in peaks.tiles_around(10.0, 179.99, 20_000)}
    xs = {x for z, x, _y in keys if z == 12}
    assert 0 in xs and (1 << 12) - 1 in xs


def test_the_widest_view_stops_at_the_nearer_tiles(monkeypatch):
    monkeypatch.setattr(peaks, "MAX_TILES", 40)
    peaks.tiles_around.cache_clear()
    plan = peaks.tiles_around(69.6, 20.2, 200_000)
    assert len(plan) == 40
    assert {key[0] for key, _inner, _outer in plan} <= {12, 11}


def test_each_summit_is_named_once_by_the_ring_it_stands_in(tmp_workspace, monkeypatch):
    spots = {"Near": 5_000.0, "Middle": 30_000.0, "Far": 70_000.0, "Farthest": 130_000.0}
    summits = [_summit(name, *offset(*EYE, 45.0, metres), 2000) for name, metres in spots.items()]
    _source(monkeypatch, summits)
    known = _settled(*EYE, 150_000)
    assert known.pending == known.failed == 0
    # every ring's tiles hold all four, yet each is named once
    assert sorted(p.name for p in known.peaks) == sorted(spots)
    for peak in known.peaks:
        assert terrain.distance(*EYE, peak.lat, peak.lon) == pytest.approx(spots[peak.name], abs=40)


def test_summits_are_read_once_then_from_the_workspace(tmp_workspace, monkeypatch):
    asked = _source(monkeypatch, [_summit("Eiger", 46.5776, 8.0053, 3967)])
    first = _settled(*EYE, 20_000)
    count = len(_tiles_asked(asked))
    second = peaks.known_around(*EYE, 20_000)
    assert count == len(peaks.tiles_around(*EYE, 20_000))
    assert len(_tiles_asked(asked)) == count
    assert asked.count(peaks.TILEJSON) == 1
    assert first.pending == second.pending == 0
    assert [p.name for p in second.peaks] == ["Eiger"]
    key = (12, *(int(c) for c in tiles.project(46.5776, 8.0053, 12)))
    kept = json.loads(peaks._tile_path(key).read_text(encoding="utf-8"))
    assert kept[0]["name"] == "Eiger" and kept[0]["ele"] == 3967


def test_only_named_summits_and_volcanoes_are_kept(tmp_workspace, monkeypatch):
    lat, lon = offset(*EYE, 120.0, 4_000)
    _source(monkeypatch, [
        _summit("Jabal Ahmam", lat, lon, 2400, **{"name:en": "Ahmam"}),
        _summit("Etna", *offset(*EYE, 200.0, 6_000), 3357, kind="volcano"),
        _summit("Hahnenmoospass", *offset(*EYE, 300.0, 3_000), 1950, kind="saddle"),
        _summit("", *offset(*EYE, 10.0, 2_000), 1500),
        _summit("Too high", *offset(*EYE, 60.0, 7_000), 12_000),
    ])
    named = {p.name: p for p in _settled(*EYE, 10_000).peaks}
    assert sorted(named) == ["Etna", "Jabal Ahmam", "Too high"]
    assert named["Jabal Ahmam"].name_en == "Ahmam"
    assert named["Etna"].name_en is None
    assert named["Too high"].ele is None  # the terrain gives it one
    assert named["Jabal Ahmam"].lat == pytest.approx(lat, abs=1e-4)


def test_the_view_never_waits_on_the_tiles(tmp_workspace, monkeypatch):
    gate = threading.Event()

    def slow(url):
        if url == peaks.TILEJSON:
            return 200, json.dumps({"tiles": [TEMPLATE]}).encode()
        gate.wait(5)
        return 200, b""

    monkeypatch.setattr(peaks, "_get", slow)
    known = peaks.known_around(*EYE, 60_000)
    assert known.peaks == [] and known.pending == len(peaks.tiles_around(*EYE, 60_000))
    # asked again while the first are out: nothing is asked twice
    again = peaks.known_around(*EYE, 60_000)
    assert again.pending == known.pending
    gate.set()
    # the tiles come in before the workspace goes, or a thread writes into a
    # folder being torn down
    assert peaks.wait_idle()


def test_the_eye_s_own_tile_is_asked_first(tmp_workspace, monkeypatch):
    asked = _source(monkeypatch, [])
    monkeypatch.setattr(peaks, "PARALLEL", 1)
    monkeypatch.setattr(peaks, "_pool", None)
    _settled(46.9, 7.1, 80_000)
    x, y = (int(c) for c in tiles.project(46.9, 7.1, 12))
    assert _tiles_asked(asked)[0] == TEMPLATE.format(z=12, x=x, y=y)


def test_a_build_that_is_gone_sends_the_view_to_the_new_one(tmp_workspace, monkeypatch):
    newer = TEMPLATE.replace("20261004_test", "20261011_test")
    asked = []

    def get(url):
        asked.append(url)
        if url == peaks.TILEJSON:
            return 200, json.dumps({"tiles": [newer if peaks._template else TEMPLATE]}).encode()
        if "20261004_test" in url:
            return 404, b""
        z, x, y = (int(part) for part in _TILE.search(url).groups())
        return 200, _tile(z, x, y, [_summit("Eiger", 46.5776, 8.0053, 3967)])

    monkeypatch.setattr(peaks, "_get", get)
    monkeypatch.setattr(peaks, "TEMPLATE_FRESH", 0.0)
    known = _settled(*EYE, 20_000)
    assert [p.name for p in known.peaks] == ["Eiger"]
    assert asked.count(peaks.TILEJSON) >= 2
    assert peaks._template[0] == newer


def test_a_tile_the_current_build_lacks_is_an_empty_one(tmp_workspace, monkeypatch):
    asked = _source(monkeypatch, [], tile_status=404)
    known = _settled(*EYE, 1_000)
    assert known.failed == 0 and known.peaks == []
    # the build was read a moment ago, so it is not read again for each tile
    assert asked.count(peaks.TILEJSON) == 1


def test_an_unreachable_host_fails_every_tile_at_once(tmp_workspace, monkeypatch):
    asked = []

    def get(url):
        asked.append(url)
        raise peaks.PeaksUnavailable("offline")

    monkeypatch.setattr(peaks, "_get", get)
    known = _settled(*EYE, 20_000)
    assert known.failed == len(peaks.tiles_around(*EYE, 20_000)) > 1
    assert asked == [peaks.TILEJSON]


def test_a_tile_address_off_openfreemap_is_never_asked(tmp_workspace, monkeypatch):
    asked = _source(monkeypatch, [], template="https://example.com/{z}/{x}/{y}.pbf")
    known = _settled(*EYE, 1_000)
    assert known.failed == len(peaks.tiles_around(*EYE, 1_000))
    assert _tiles_asked(asked) == []


def test_a_tile_that_could_not_be_read_is_left_longer_each_time(tmp_workspace, monkeypatch):
    asked = _source(monkeypatch, [], refuse=True)
    known = _settled(0.5, -30.5, 1_000)
    count = len(peaks.tiles_around(0.5, -30.5, 1_000))
    assert known.failed == count and known.pending == 0 and known.peaks == []
    assert known.retry_in == pytest.approx(peaks.RETRY_AFTER, abs=1)
    tried = len(_tiles_asked(asked))
    assert peaks.known_around(0.5, -30.5, 1_000).failed == count
    assert len(_tiles_asked(asked)) == tried  # not asked again so soon
    # once the wait is over the view's next question asks again, and a second
    # failure waits twice as long
    for key in list(peaks._failed):
        peaks._failed[key] = (time.monotonic() - 1, peaks._failed[key][1])
    again = _settled(0.5, -30.5, 1_000)
    assert len(_tiles_asked(asked)) == 2 * tried
    assert again.retry_in == pytest.approx(2 * peaks.RETRY_AFTER, abs=1)
    assert all(wait == 2 * peaks.RETRY_AFTER for _due, wait in peaks._failed.values())


def test_trying_again_asks_at_once(tmp_workspace, monkeypatch):
    _source(monkeypatch, [], refuse=True)
    assert _settled(0.5, -30.5, 1_000).failed
    asked = _source(monkeypatch, [_summit("Seamount", *offset(0.5, -30.5, 0.0, 500), 10)])
    assert peaks.known_around(0.5, -30.5, 1_000).failed  # still waiting
    assert _tiles_asked(asked) == []
    known = _settled(0.5, -30.5, 1_000, retry=True)
    assert known.failed == 0 and [p.name for p in known.peaks] == ["Seamount"]


def test_a_stale_tile_is_asked_again(tmp_workspace, monkeypatch):
    asked = _source(monkeypatch, [])
    _settled(0.5, -30.5, 1_000)
    _settled(0.5, -30.5, 1_000)
    tried = len(_tiles_asked(asked))
    assert tried == len(peaks.tiles_around(0.5, -30.5, 1_000))
    old = time.time() - (peaks.TTL_DAYS + 1) * 86400
    for path in peaks.cache_dir().rglob("*.json"):
        os.utime(path, (old, old))
    _settled(0.5, -30.5, 1_000)
    assert len(_tiles_asked(asked)) == 2 * tried


def test_the_route_places_each_summit_where_the_eye_sees_it(client, monkeypatch):
    north = offset(*EYE, 0.0, 10_000.0)
    east = offset(*EYE, 90.0, 20_000.0)
    _source(monkeypatch, [_summit("North top", *north, 2500), _summit("East top", *east)])
    monkeypatch.setattr(terrain, "tile", Relief(peaks=(), plain=1200.0))
    terrain.forget_decoded()
    params = {"lat": EYE[0], "lon": EYE[1], "altitude": 1000, "far": 50_000}
    early = client.get("/api/horizon/peaks", params=params).json()
    assert early["pending"] > 0  # answered at once, with what is known so far
    assert peaks.wait_idle()
    body = client.get("/api/horizon/peaks", params=params).json()
    assert body["pending"] == 0 and body["failed"] == 0 and body["retry_in"] == 0
    named = {row["name"]: row for row in body["peaks"]}
    assert named["North top"]["azimuth"] == pytest.approx(0.0, abs=0.01)
    assert named["North top"]["distance"] == pytest.approx(10_000, rel=1e-3)
    # 1500 m above the eye at 10 km, less the Earth's drop: just under 8.5°
    assert 8.3 < named["North top"]["angle"] < 8.55
    # the tile gave the east summit no height, so the terrain did
    assert named["East top"]["ele"] == pytest.approx(1200.0, abs=0.5)
    assert named["East top"]["ele_from_osm"] is False
    assert "OpenFreeMap" in body["credits"][0]["attribution"]
    assert "© OpenStreetMap contributors" in body["credits"][0]["attribution"]


def test_the_route_says_what_could_not_be_read_and_tries_again_when_told(client, monkeypatch):
    _source(monkeypatch, [], refuse=True)
    params = {"lat": EYE[0], "lon": EYE[1], "altitude": 1000, "far": 20_000}
    client.get("/api/horizon/peaks", params=params)
    assert peaks.wait_idle()
    body = client.get("/api/horizon/peaks", params=params).json()
    assert body["peaks"] == []
    assert body["failed"] == len(peaks.tiles_around(*EYE, 20_000))
    assert 0 < body["retry_in"] <= peaks.RETRY_AFTER
    _source(monkeypatch, [])
    again = client.get("/api/horizon/peaks", params={**params, "retry": "true"}).json()
    assert again["failed"] == 0 and again["pending"] == body["failed"]
    assert peaks.wait_idle()


# -- the raw tile, for the buildings source -----------------------------------------

def test_a_vector_tile_comes_back_with_the_build_that_served_it(tmp_workspace, monkeypatch):
    body = _tile(12, 2100, 1400, [_summit("Eiger", 46.5776, 8.0053, 3967)])

    def get(url):
        if url == peaks.TILEJSON:
            return 200, json.dumps({"tiles": [TEMPLATE]}).encode()
        return 200, body

    monkeypatch.setattr(peaks, "_get", get)
    assert peaks.fetch_vector_tile(12, 2100, 1400) == (body, TEMPLATE)


def test_a_tile_the_build_lacks_is_none_and_a_newer_build_is_tried(tmp_workspace, monkeypatch):
    newer = TEMPLATE.replace("20261004_test", "20261011_test")
    body = _tile(12, 2100, 1400, [])
    monkeypatch.setattr(peaks, "TEMPLATE_FRESH", 0.0)

    def get(url):
        if url == peaks.TILEJSON:
            return 200, json.dumps({"tiles": [newer if peaks._template else TEMPLATE]}).encode()
        if "20261004_test" in url:
            return 404, b""
        return 200, body

    monkeypatch.setattr(peaks, "_get", get)
    assert peaks.fetch_vector_tile(12, 2100, 1400) == (body, newer)


def test_a_tile_missing_from_every_build_is_none(tmp_workspace, monkeypatch):
    _source(monkeypatch, [], tile_status=404)
    assert peaks.fetch_vector_tile(12, 2100, 1400) == (None, TEMPLATE)


def test_another_status_on_a_vector_tile_is_unavailable(tmp_workspace, monkeypatch):
    _source(monkeypatch, [], tile_status=500)
    with pytest.raises(peaks.PeaksUnavailable, match="500"):
        peaks.fetch_vector_tile(12, 2100, 1400)


def test_a_build_names_its_day_from_its_tile_address():
    assert peaks.build_date("https://tiles.openfreemap.org/planet/20261004_113936_pt/{z}/{x}/{y}.pbf") == "2026-10-04"
    # a build address that is not the weekly one says nothing
    assert peaks.build_date(TEMPLATE) == ""
    assert peaks.build_date("https://tiles.openfreemap.org/planet/20261340_113936_pt/{z}/{x}/{y}.pbf") == ""