Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
9.8 KB
Raw Blame
"""The vector tile encoder, read back by a reader written from the specification."""

from __future__ import annotations

import gzip
import struct

import numpy as np
import pytest

from azimut.engine import mvt
from mvtread import _fields, _packed, geometry, read, ring_area


def test_many_varints_are_written_as_each_one_would_be():
    values = [0, 1, 127, 128, 300, 16383, 16384, 2**21, 2**35, 2**63 - 1]
    assert mvt._varints(np.array(values, dtype=np.uint64)) == b"".join(mvt._varint(v) for v in values)
    assert mvt._varints(np.array([], dtype=np.uint64)) == b""


def test_a_step_back_is_zigzagged_to_a_small_number():
    steps = np.array([0, -1, 1, -2, 2, -4096, 4096])
    assert mvt._zigzag(steps).tolist() == [0, 1, 2, 3, 4, 8191, 8192]


def test_points_come_back_where_they_were_put():
    xs, ys = np.array([10, 4000, 4000, 0]), np.array([20, 5, 4095, 0])
    commands = _packed(mvt.points_geometry(xs, ys))
    # one MoveTo for all of them
    assert commands[0] == 1 | 4 << 3
    assert geometry(commands) == [[(10, 20)], [(4000, 5)], [(4000, 4095)], [(0, 0)]]


def test_rectangles_come_back_as_outer_rings():
    x0, y0 = np.array([0, -50]), np.array([0, 100])
    x1, y1 = np.array([10, 4200]), np.array([30, 180])
    parts = geometry(_packed(mvt.rectangles_geometry(x0, y0, x1, y1)))
    assert parts == [[(0, 0), (10, 0), (10, 30), (0, 30)], [(-50, 100), (4200, 100), (4200, 180), (-50, 180)]]
    # clockwise on screen, which is what makes each one filled rather than a hole
    assert all(ring_area(ring) > 0 for ring in parts)


def test_a_layer_says_its_name_extent_and_version_and_shares_its_values():
    layer = mvt.Layer("marks")
    layer.points(np.array([1]), np.array([2]), {"recent": 1, "side": 375})
    layer.points(np.array([3]), np.array([4]), {"recent": 1, "side": 400})
    tile = read(mvt.encode([layer]))
    marks = tile["marks"]
    assert (marks["extent"], marks["version"]) == (mvt.EXTENT, 2)
    assert [f["properties"] for f in marks["features"]] == [{"recent": 1, "side": 375}, {"recent": 1, "side": 400}]
    # "recent: 1" is written once for both features
    assert marks["keys"] == ["recent", "side"] and marks["values"] == [1, 375, 400]


def test_every_kind_of_value_reads_back_as_itself():
    layer = mvt.Layer("kinds")
    properties = {"count": 7, "below": -3, "ratio": 0.25, "name": "fire", "on": True}
    layer.points(np.array([0]), np.array([0]), properties)
    [feature] = read(mvt.encode([layer]))["kinds"]["features"]
    assert feature["properties"] == properties
    # a bool is not the int 1, even when both are in one layer
    both = mvt.Layer("both")
    both.points(np.array([0]), np.array([0]), {"a": 1})
    both.points(np.array([0]), np.array([0]), {"a": True})
    assert [f["properties"]["a"] for f in read(mvt.encode([both]))["both"]["features"]] == [1, True]


def test_a_layer_with_nothing_in_it_is_left_out():
    empty, full = mvt.Layer("empty"), mvt.Layer("full")
    empty.points(np.array([], dtype=np.int64), np.array([], dtype=np.int64), {"recent": 0})
    full.points(np.array([5]), np.array([6]), {"recent": 0})
    assert set(read(mvt.encode([empty, full]))) == {"full"}
    assert mvt.encode([empty]) == b""
    # and a tile is layers and nothing else
    assert [number for number, _ in _fields(mvt.encode([full]))] == [3]


# -- reading someone else's tile -------------------------------------------------


def _peaks_tile() -> bytes:
    roads, summits = mvt.Layer("transportation"), mvt.Layer("mountain_peak")
    roads.rectangles(np.array([0]), np.array([0]), np.array([10]), np.array([10]), {"class": "road"})
    summits.points(np.array([1024]), np.array([2048]), {"name": "Eiger", "ele": 3967})
    # past the right edge, in the buffer a tile server keeps
    summits.points(np.array([mvt.EXTENT + 20]), np.array([-20]), {"name": "Mönch"})
    summits.rectangles(np.array([1]), np.array([1]), np.array([2]), np.array([2]), {"name": "a ridge"})
    return mvt.encode([roads, summits])


def test_the_points_of_one_layer_are_read_and_the_rest_stepped_over():
    points = mvt.read_points(_peaks_tile(), "mountain_peak")
    assert points == [
        (0.25, 0.5, {"name": "Eiger", "ele": 3967}),
        ((mvt.EXTENT + 20) / mvt.EXTENT, -20 / mvt.EXTENT, {"name": "Mönch"}),
    ]
    assert mvt.read_points(_peaks_tile(), "place") == []
    # as a tile server stores it
    assert mvt.read_points(gzip.compress(_peaks_tile()), "mountain_peak") == points


def test_a_layer_s_own_extent_is_kept():
    layer = mvt.Layer("mountain_peak")
    layer.points(np.array([128]), np.array([384]), {"name": "Eiger"})
    body = layer.encode()
    extent = mvt._varint(5 << 3) + mvt._varint(mvt.EXTENT)
    assert body.endswith(extent)
    small = mvt._field(3, body[: -len(extent)] + mvt._varint(5 << 3) + mvt._varint(512))
    assert mvt.read_points(small, "mountain_peak") == [(0.25, 0.75, {"name": "Eiger"})]


def test_every_kind_of_value_a_tile_server_writes_is_read():
    layer = mvt.Layer("kinds")
    properties = {"count": 7, "below": -3, "ratio": 0.25, "name": "fire", "on": True}
    layer.points(np.array([0]), np.array([0]), properties)
    [(_x, _y, read_back)] = mvt.read_points(mvt.encode([layer]), "kinds")
    assert read_back == properties
    # a float and a plain int64, which this encoder never writes
    assert mvt._read_value(memoryview(mvt._varint(2 << 3 | 5) + struct.pack("<f", 0.5))) == 0.5
    assert mvt._read_value(memoryview(mvt._varint(4 << 3) + mvt._varint(2**64 - 2))) == -2


def test_a_tile_cut_short_or_not_a_tile_is_refused():
    tile = _peaks_tile()
    with pytest.raises(ValueError):
        mvt.read_points(tile[:-3], "mountain_peak")
    with pytest.raises(ValueError):
        mvt.read_points(b"\x1f\x8b not gzip at all", "mountain_peak")


# -- reading polygons -----------------------------------------------------------

SQUARE = [(0, 0), (100, 0), (100, 100), (0, 100)]  # clockwise on screen: an exterior ring
HOLE = [(30, 30), (30, 70), (70, 70), (70, 30)]  # counter-clockwise: a hole in it


def _zigzag(d: int) -> int:
    return (d << 1) ^ (d >> 63)


def _polygon_geometry(*rings: list[tuple[int, int]]) -> bytes:
    """Rings written by hand as the specification has them: MoveTo the first point,
    LineTo the rest, ClosePath. The cursor carries from one ring to the next."""
    commands: list[int] = []
    cx = cy = 0
    for ring in rings:
        for index, (px, py) in enumerate(ring):
            if index == 0:
                commands.append(mvt._command(1, 1))
            elif index == 1:
                commands.append(mvt._command(2, len(ring) - 1))
            commands += [_zigzag(px - cx), _zigzag(py - cy)]
            cx, cy = px, py
        commands.append(mvt._command(7, 1))
    return b"".join(mvt._varint(c) for c in commands)


def _layer(name: str, *features: tuple[int, bytes, dict]) -> mvt.Layer:
    layer = mvt.Layer(name)
    layer.features.extend(features)
    return layer


def _fractions(points: list[tuple[int, int]]) -> list[tuple[float, float]]:
    return [(x / mvt.EXTENT, y / mvt.EXTENT) for x, y in points]


def test_one_square_is_one_polygon_with_its_open_ring():
    layer = _layer("buildings", (3, _polygon_geometry(SQUARE), {"render_height": 12}))
    assert mvt.read_polygons(mvt.encode([layer]), "buildings") == [
        ([_fractions(SQUARE)], {"render_height": 12}),
    ]


def test_a_square_with_a_hole_is_one_polygon_of_two_rings():
    layer = _layer("buildings", (3, _polygon_geometry(SQUARE, HOLE), {"render_height": 12}))
    assert mvt.read_polygons(mvt.encode([layer]), "buildings") == [
        ([_fractions(SQUARE), _fractions(HOLE)], {"render_height": 12}),
    ]


def test_two_polygons_in_one_feature_are_two_entries_with_its_properties():
    shifted = [(x + 200, y) for x, y in SQUARE]
    layer = _layer("buildings", (3, _polygon_geometry(SQUARE, shifted), {"render_height": 9}))
    assert mvt.read_polygons(mvt.encode([layer]), "buildings") == [
        ([_fractions(SQUARE)], {"render_height": 9}),
        ([_fractions(shifted)], {"render_height": 9}),
    ]


def test_a_hole_before_any_exterior_and_a_zero_area_ring_are_dropped():
    flat = [(0, 0), (50, 50), (100, 100)]
    geometry = _polygon_geometry(HOLE, flat, SQUARE)
    layer = _layer("buildings", (3, geometry, {}))
    assert mvt.read_polygons(mvt.encode([layer]), "buildings") == [([_fractions(SQUARE)], {})]


def test_other_layers_and_points_are_ignored():
    roads = _layer("roads", (3, _polygon_geometry(SQUARE), {"class": "road"}))
    point = mvt.points_geometry(np.array([5]), np.array([5]))
    beside = [(x + 200, y) for x, y in SQUARE]
    buildings = _layer("buildings", (1, point, {"name": "a point"}), (3, _polygon_geometry(beside), {"render_height": 3}))
    tile = mvt.encode([roads, buildings])
    assert mvt.read_polygons(tile, "buildings") == [([_fractions(beside)], {"render_height": 3})]
    assert mvt.read_polygons(tile, "roads") == [([_fractions(SQUARE)], {"class": "road"})]
    assert mvt.read_polygons(tile, "water") == []


def test_a_gzipped_tile_is_opened_first():
    layer = _layer("buildings", (3, _polygon_geometry(SQUARE), {"render_height": 12}))
    tile = mvt.encode([layer])
    assert mvt.read_polygons(gzip.compress(tile), "buildings") == mvt.read_polygons(tile, "buildings")


def test_a_polygon_tile_cut_short_or_not_a_tile_is_refused():
    layer = _layer("buildings", (3, _polygon_geometry(SQUARE, HOLE), {"render_height": 12}))
    tile = mvt.encode([layer])
    with pytest.raises(ValueError):
        mvt.read_polygons(tile[:-3], "buildings")
    with pytest.raises(ValueError):
        mvt.read_polygons(b"\x1f\x8b not gzip at all", "buildings")


def test_a_point_in_the_buffer_comes_out_below_zero():
    beyond = [(-20, 0), (100, 0), (100, 100), (-20, 100)]
    layer = _layer("buildings", (3, _polygon_geometry(beyond), {}))
    [(rings, _props)] = mvt.read_polygons(mvt.encode([layer]), "buildings")
    assert rings[0][0] == (-20 / mvt.EXTENT, 0.0)
    assert rings[0][0][0] < 0