Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Remove osmium dependency

Belluxxb1387f82ae3c
diff --git a/README.md b/README.md
index 55ff788..275cdef 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ This was born as a way to let LLMs quickly gather information from the physical
## Install
-Requires Python 3.11+. Pillow and the Python [osmium](https://osmcode.org/pyosmium/) package are installed automatically with Aleph. No API keys required.
+Requires Python 3.11+. Pillow is the only dependency and is installed automatically with Aleph. OSM processing uses the bundled Python script and standard library. No API keys required.
```sh
python3 -m venv .venv
@@ -54,7 +54,13 @@ Here is an overview of the structure of the output dir after a capture:
- `map.osm`: OpenStreetMap roads, buildings, water, and other features
- `terrain.tif`: Elevation as a GeoTIFF (original tiles in `terrain/`)
-OSM data comes from [Geofabrik](https://download.geofabrik.de/). Aleph downloads the smallest regional file covering your selection, then extracts the selected rectangle locally with Python's `osmium` package. The first download can be hundreds of megabytes even for a small selection. Regional files are reused from `$XDG_CACHE_HOME/aleph/geofabrik` (usually `~/.cache/aleph/geofabrik`); use `--refresh` on CLI requests to download the latest snapshot. Map manifests record the source and snapshot date. Geofabrik omits contributor names, user IDs, and changeset IDs.
+OSM data comes from [Geofabrik](https://download.geofabrik.de/). Aleph downloads the smallest regional file covering your selection, then extracts the selected rectangle locally with its bundled [PBF reader](src/pbf.py). The first download can be hundreds of megabytes even for a small selection. Regional files are reused from `$XDG_CACHE_HOME/aleph/geofabrik` (usually `~/.cache/aleph/geofabrik`); use `--refresh` on CLI requests to download the latest snapshot. Map manifests record the source and snapshot date. Geofabrik omits contributor names, user IDs, and changeset IDs.
+
+The extractor also runs independently, with no third-party packages:
+
+```sh
+python3 src/pbf.py region.osm.pbf 41.88 12.48 41.90 12.51 -o map.osm
+```
## From the terminal (for scripts and agents)
diff --git a/docs/cli.md b/docs/cli.md
index 1089293..52854ce 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -52,7 +52,7 @@ For example, `--stops 10` requests ten positions along the street. Choose a view
If Aleph finds several places or street branches, choose from the returned options with `--match ID` or `--route N`, respectively.
> [!TIP]
-> Quick requests start immediately and cache imagery/geocoder responses for 24 hours. OSM roads, nearby POIs, and maps use Geofabrik regional downloads, processed locally with the Python `osmium` dependency. These regional files are retained until explicitly refreshed; the first request may download hundreds of megabytes. Use `--refresh` to fetch fresh responses and regional files, or `--cache-dir PATH` to change the cache location (default: `$XDG_CACHE_HOME/aleph` or `~/.cache/aleph`).
+> Quick requests start immediately and cache imagery/geocoder responses for 24 hours. OSM roads, nearby POIs, and maps use Geofabrik regional downloads, processed locally by Aleph's standard-library PBF reader. These regional files are retained until explicitly refreshed; the first request may download hundreds of megabytes. Use `--refresh` to fetch fresh responses and regional files, or `--cache-dir PATH` to change the cache location (default: `$XDG_CACHE_HOME/aleph` or `~/.cache/aleph`).
## Capture an area
diff --git a/docs/osm-benchmark.json b/docs/osm-benchmark.json
index 5950da4..7072b3b 100644
--- a/docs/osm-benchmark.json
+++ b/docs/osm-benchmark.json
@@ -329,5 +329,65 @@
"missing_way_nodes": 0
}
]
+ },
+ "stdlib_pbf": {
+ "python_site_packages_enabled": false,
+ "workers": 8,
+ "regional_file_cached": true,
+ "network_requests": 0,
+ "tests_passed": 51,
+ "results": [
+ {
+ "area_km2": 10,
+ "bounds": [
+ 41.875980496522295,
+ 12.473098695094054,
+ 41.904419503477705,
+ 12.511301304905947
+ ],
+ "map_seconds": 15.08,
+ "map_bytes": 39695725,
+ "osm_data_at": "2026-09-20T20:22:06Z",
+ "counts": {
+ "node": 177525,
+ "way": 32787,
+ "relation": 3219
+ },
+ "matches_native": true,
+ "missing_way_nodes": 0,
+ "previous_map_seconds": 22.49,
+ "normalized_opl_sha256": "6d15bc46365c66559a3f61001616771714517778534fb6afb4d47664ec5fcbb6"
+ },
+ {
+ "area_km2": 1000,
+ "bounds": [
+ 41.74800496522297,
+ 12.301186950940533,
+ 42.03239503477703,
+ 12.683213049059468
+ ],
+ "map_seconds": 23.95,
+ "map_bytes": 560701743,
+ "osm_data_at": "2026-09-20T20:22:06Z",
+ "counts": {
+ "node": 3172834,
+ "way": 469155,
+ "relation": 21004
+ },
+ "matches_native": true,
+ "missing_way_nodes": 0,
+ "previous_map_seconds": 40.54,
+ "normalized_opl_sha256": "a868eaa29e95cc6cb375925e3eeb0becfd6ebabf1fe96b42fd78dd6175d2c2c6"
+ }
+ ],
+ "query_checks": {
+ "highway_ways": 12412,
+ "road_geometry_matches": true,
+ "nearby_ids_match": true,
+ "connected_street_ways": 11
+ },
+ "cancellation_seconds": 0.86,
+ "validation_method": "One final SHA-256 comparison of canonical OPL output against the native baseline, after performance iteration.",
+ "validation_seconds": 23.16
}
}
diff --git a/docs/osm-benchmark.md b/docs/osm-benchmark.md
index 2842f4c..a375395 100644
--- a/docs/osm-benchmark.md
+++ b/docs/osm-benchmark.md
@@ -1,16 +1,41 @@
# OSM provider benchmark — 21 September 2026
-Aleph now uses the Python `osmium` dependency for Geofabrik extraction and local road geometry. The native Osmium CLI measurements below describe the earlier implementation, which has been replaced. No Overpass service is used by either Geofabrik implementation.
+Aleph now uses its bundled standard-library [PBF reader](../src/pbf.py) for Geofabrik extraction and local road geometry. The Python `osmium` and native Osmium CLI measurements below describe earlier implementations. None of the Geofabrik implementations uses Overpass.
## Method
-Both selections are centered on the Colosseum in Rome, at 41.8902° N, 12.4922° E. They are squares approximately 3.162 km and 31.623 km wide, with spherical areas of 10.000 and 999.999 km². Tests ran on macOS 15.7.1 ARM64 with Python 3.12.12. The original CLI comparison used Osmium 1.19.1; the current Python implementation uses `osmium` 4.3.1.
+Both selections are centered on the Colosseum in Rome, at 41.8902° N, 12.4922° E. They are squares approximately 3.162 km and 31.623 km wide, with spherical areas of 10.000 and 999.999 km². Tests ran on macOS 15.7.1 ARM64 with Python 3.12.12 and eight available CPUs. The earlier implementations used Osmium CLI 1.19.1 and Python `osmium` 4.3.1.
The baseline is commit `38aa50015fe347a71bcd1c0906376a8a46e3779d`. Its road and map queries, memory budgets, HTTP retries, and three-server fallback were unchanged. Each network operation had an outer 600-second benchmark deadline. A deadline result means no usable response arrived within the test budget, not that the client exhausted every possible retry. Imagery and terrain downloads were excluded.
These are single live trials, so network timings depend on server load. Local files were available in the filesystem cache. Software installation time is excluded.
-## Current Python Osmium results
+## Current standard-library extractor
+
+`src/pbf.py` is a standalone script using only Python's standard library. It decodes packed PBF arrays in bulk with standard-library byte operations and integer arithmetic, processes blocks across CPU cores, and decodes strings only when selected objects reference them. XML ways are formatted in workers; selected coordinates and relation records are reused within the operation. No derived files or indexes are cached on disk. It supports the sorted snapshot PBFs used by Geofabrik, including raw/zlib blobs, ordinary/dense nodes, complete ways, parent relations, and recursive POI references. It does not implement Osmium's unrelated commands, history processing, or other optional compression formats.
+
+These exports ran with `python -S`, so no site packages were available, using the same cached 383.6 MB central Italy file:
+
+| Area | Optimized script | Initial script | Python `osmium` | Native CLI | New XML size |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| 10 km² | 15.08 s | 22.49 s | 161.65 s | 3.39 s | 39.7 MB |
+| 1,000 km² | 23.95 s | 40.54 s | 173.88 s | 3.66 s | 560.7 MB |
+
+The optimized script takes 33% and 41% less time than the initial bundled script, respectively, and is about 10.7× and 7.3× faster than the earlier Python `osmium` implementation. The native CLI remains faster. Timings include parsing the PBF index, selecting objects, completing ways, and writing XML, and exclude downloads and subsequent validation. Capture estimates now allow 30 seconds for map extraction based on this region.
+
+After performance iteration, one final comparison normalized both exports and the native baseline to OPL with the native CLI and compared SHA-256 hashes. Both match, including object IDs, coordinates, tags, metadata, way references and relation members. The matching native baseline has zero missing way nodes. The native CLI was used only as a validation oracle; extraction still runs with no third-party packages. The real 10 km² road query matches all 12,412 ways, and the Colosseum nearby results and Via del Corso's 11 connected ways match as well. A final live cancellation check joined all workers in 0.86 seconds and preserved the previous output. XML sizes differ from the CLI output because formatting differs.
+
+The test suite has 51 passing tests with `osmium` uninstalled. Tests cover bulk integer decoding, both node encodings, UTF-8/XML escaping, metadata, non-default coordinate offsets/granularity, split packed fields, malformed input, nested relations, cancellation, atomic output, and standalone execution without site packages.
+
+To run the script independently:
+
+```sh
+python3 src/pbf.py centro.osm.pbf \
+ 41.875980496522295 12.473098695094054 41.904419503477705 12.511301304905947 \
+ -o rome-10.osm
+```
+
+## Earlier Python Osmium results
The Python implementation installs through `pip install .` and uses no external Osmium executable. It reads PBF files directly, preserves complete ways and parent relations, and completes nested POI relations. Roads and nearby places are read directly into Python without intermediate XML files.
@@ -21,7 +46,7 @@ Both exports reused the same cached central Italy regional file, with network ac
| 10 km² | 161.65 s | 41.3 MB |
| 1,000 km² | 173.88 s | 571.4 MB |
-These single-trial timings exclude downloading the regional file and subsequent validation. Python scans the entire regional file to select the rectangle, so the regional file's size matters even for small selections. Capture estimates now allow about three minutes for local map extraction, based on this region.
+These single-trial timings exclude downloading the regional file and subsequent validation. This implementation scanned the entire regional file through Python OSM objects to select the rectangle, so the regional file's size mattered even for small selections.
Both exports have identical node, way, and relation IDs to the earlier CLI exports, and both pass way-reference validation through Python Osmium. The 10 km² export's 12,412 highway ways match the earlier road geometry exactly. Nearby results around the Colosseum and Via del Corso's 11 connected ways also matched when read from the new export. All 44 tests pass with the native tool absent from `PATH`, including cancellation, nested relations, offline reuse, and preserving previous files on failure.
diff --git a/pyproject.toml b/pyproject.toml
index 8b1b033..5ca7dc8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ version = "0.1.0"
description = "Capture Street View photos, satellite imagery, and OpenStreetMap data."
readme = "README.md"
requires-python = ">=3.11"
-dependencies = ["Pillow>=10", "osmium>=4.3,<5"]
+dependencies = ["Pillow>=10"]
[project.scripts]
alephgeo = "aleph:main"
diff --git a/requirements.txt b/requirements.txt
index 2a75295..a40d05d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
Pillow>=10
-osmium>=4.3,<5
diff --git a/src/capture.py b/src/capture.py
index 6bacf12..ccbe767 100644
--- a/src/capture.py
+++ b/src/capture.py
@@ -140,8 +140,8 @@ def estimate(run):
2026-09-19 17:26:41 UTC: 66 Street View photos in 36 s,
including panorama metadata. 2026-09-19 17:09:18 UTC:
1,020 satellite tiles in 374 s and 9 terrain tiles in 10 s.
- 2026-09-21: Python Osmium extracted 10 km² from the 384 MB central Italy
- region in 162 s; allow about 3 minutes per map. Region size affects this cost.
+ 2026-09-21: the bundled PBF extractor took 15–24 s for 10–1,000 km² from
+ the 384 MB central Italy region. Region size affects this cost.
Planning, initial regional downloads and final exports are excluded.
"""
counts = dict(streetview_photos=0, streetview_stops=0, satellite_tiles=0,
@@ -168,7 +168,7 @@ def estimate(run):
+ counts["satellite_tiles"] + counts["terrain_tiles"])
download_seconds = (counts["streetview_photos"] * 0.55
+ counts["satellite_tiles"] * 0.37
- + counts["terrain_tiles"] * 1.1 + counts["osm_maps"] * 180)
+ + counts["terrain_tiles"] * 1.1 + counts["osm_maps"] * 30)
delay_seconds = max(0, requests - 1) * run["options"]["delay"]
return dict(counts, seconds=math.ceil(download_seconds + delay_seconds))
diff --git a/src/osm.py b/src/osm.py
index 5808027..d950596 100644
--- a/src/osm.py
+++ b/src/osm.py
@@ -1,19 +1,17 @@
-"""Download Geofabrik regions and select local OSM data with Osmium."""
+"""Download Geofabrik regions and read them with the bundled PBF extractor."""
import hashlib
import json
from collections import defaultdict
-from contextlib import contextmanager, nullcontext
+from contextlib import nullcontext
from itertools import pairwise
from xml.etree import ElementTree as ET
-import osmium
-
from .common import APP_AGENT, RequestError, atomic_path, write_bytes
from .geo import clip, extent, ring_contains, signed_area
+from .pbf import PBF, POI_KEYS
INDEX = "https://download.geofabrik.de/index-v1.json"
-POI_KEYS = ("amenity", "tourism", "shop", "leisure", "historic", "office")
def polygons(geometry):
@@ -62,53 +60,6 @@ def objects(path, cancel=lambda: None):
raise ValueError("OSM file contains invalid XML.") from error
-def read_data(objects, *, centers=False):
- """Copy selected Osmium objects, retaining node IDs for road topology."""
- nodes, items, boxes, relations = {}, [], {}, []
- kinds = {"n": "node", "w": "way", "r": "relation"}
- for obj in objects:
- identity, kind = obj.id, kinds[obj.type_str()]
- if kind == "node":
- lat, lon = obj.lat, obj.lon
- nodes[identity] = (lat, lon)
- if not centers:
- continue
- boxes[("node", identity)] = (lat, lon, lat, lon)
- tags = dict(obj.tags)
- item = dict(type=kind, id=identity, tags=tags)
- if kind == "way":
- refs = [n.ref for n in obj.nodes]
- if any(ref not in nodes for ref in refs):
- raise OSError("The Geofabrik extract contains incomplete way geometry.")
- points = [nodes[ref] for ref in refs]
- item.update(nodes=refs, points=points)
- if centers and points:
- boxes[("way", identity)] = extent(points)
- elif kind == "relation":
- item["members"] = [(kinds[m.type], m.ref) for m in obj.members]
- relations.append(item)
- if tags or kind != "node":
- items.append(item)
- pending = relations
- while pending:
- unresolved = []
- for item in pending:
- members = [boxes[ref] for ref in item["members"] if ref in boxes]
- if not members or len(members) != len(item["members"]):
- unresolved.append(item)
- continue
- boxes[("relation", item["id"])] = (min(b[0] for b in members), min(b[1] for b in members),
- max(b[2] for b in members), max(b[3] for b in members))
- if len(unresolved) == len(pending):
- break
- pending = unresolved
- for item in items:
- box = boxes.get((item["type"], item["id"]))
- if box:
- item["center"] = ((box[0] + box[2]) / 2, (box[1] + box[3]) / 2)
- return items
-
-
class Source:
def __init__(self, client):
self.client = client
@@ -116,22 +67,6 @@ class Source:
self.index = None
self.regions = {}
- @contextmanager
- def processing(self):
- self.client.check_cancel()
- try:
- yield
- except RuntimeError as error:
- raise OSError(f"Osmium failed: {error}") from error
- self.client.check_cancel()
-
- def scan(self, source, *filters, entities=osmium.osm.OBJECT):
- with self.processing(), osmium.io.Reader(source, entities) as reader:
- for index, obj in enumerate(osmium.OsmFileIterator(reader, *filters)):
- if index % 8192 == 0:
- self.client.check_cancel()
- yield obj
-
def region(self, area, progress):
if self.index is None:
path = self.directory / "index.json"
@@ -162,77 +97,40 @@ class Source:
progress(label)
self.client.get(url, destination=local, timeout=180, user_agent=APP_AGENT,
progress=lambda done, total: progress(label, done, total))
- with self.processing(), osmium.io.Reader(osmium.io.File(local, "pbf"), osmium.osm.NOTHING) as reader:
- stamp = reader.header().get("osmosis_replication_timestamp")
+ stamp = PBF(local, self.client.check_cancel).timestamp
self.regions[url] = path, dict(source_url=url, region=props["name"], osm_data_at=stamp or None)
return self.regions[url]
- def select(self, source, area):
- """Select the rectangle, complete its ways, and retain parent relations."""
- south, west, north, east = area
- box = osmium.osm.Box(west, south, east, north)
- spatial, selected = osmium.IdTracker(), osmium.IdTracker()
- parents, pending = defaultdict(list), []
- for obj in self.scan(source):
- if obj.is_node():
- if box.contains(obj.location):
- spatial.add_node(obj.id)
- selected.add_node(obj.id)
- elif obj.is_way():
- if spatial.contains_any_references(obj):
- spatial.add_way(obj.id)
- selected.add_way(obj.id)
- selected.add_references(obj)
- else:
- if spatial.contains_any_references(obj):
- pending.append(obj.id)
- for member in obj.members:
- if member.type == "r":
- parents[member.ref].append(obj.id)
- while pending:
- identity = pending.pop()
- if identity not in selected.relation_ids():
- selected.add_relation(identity)
- pending.extend(parents[identity])
- return selected
-
def extract(self, source, area, output):
- selected = self.select(source, area)
- south, west, north, east = area
- header = osmium.io.Header()
- header.add_box(osmium.osm.Box(west, south, east, north))
- with self.processing(), osmium.SimpleWriter(osmium.io.File(output, "osm"), header=header, overwrite=True) as writer:
- for obj in self.scan(source, selected.id_filter()):
- writer.add(obj)
+ PBF(source, self.client.check_cancel).export(area, output)
def export(self, area, output, progress):
source, metadata = self.region(area, progress)
progress("Extracting OSM map")
- with atomic_path(output) as temporary:
- self.extract(source, area, temporary)
+ self.extract(source, area, output)
return metadata
def data(self, area, *, poi=False, progress=lambda *args: None):
source, metadata = self.region(area, progress)
progress("Reading nearby places" if poi else "Reading roads")
- local = self.select(source, area)
- selected = osmium.IdTracker()
- entities = osmium.osm.OBJECT if poi else osmium.osm.WAY
+ pbf = PBF(source, self.client.check_cancel)
+ local = pbf.select(area)
+ selected = (set(), set(), set())
keys = POI_KEYS if poi else ("highway",)
- add = {"n": selected.add_node, "w": selected.add_way, "r": selected.add_relation}
- for obj in self.scan(source, local.id_filter(), osmium.filter.KeyFilter(*keys), entities=entities):
- add[obj.type_str()](obj.id)
- selected.add_references(obj)
- if poi and len(selected.relation_ids()):
- # Complete only selected POIs, including nested relations outside the rectangle.
- with self.processing():
- while True:
- self.client.check_cancel()
- count = len(selected.relation_ids())
- selected.complete_backward_references(source, relation_depth=1)
- if len(selected.relation_ids()) == count:
- break
- return read_data(self.scan(source, selected.id_filter()), centers=poi), metadata
+ if poi:
+ for node in pbf.nodes(local[0], "objects"):
+ if any(key in node["tags"] for key in keys):
+ selected[0].add(node["id"])
+ for relation in pbf.relations(local[2]):
+ if any(key in relation["tags"] for key in keys):
+ selected[2].add(relation["id"])
+ for way in pbf.ways(local[1]):
+ if any(key in way["tags"] for key in keys):
+ selected[1].add(way["id"])
+ selected[0].update(way["nodes"])
+ if selected[2]:
+ pbf.complete(selected)
+ return pbf.read(selected, centers=poi), metadata
def street(self, place, progress=lambda *args: None):
identity = int(place["id"].split("/")[1])
@@ -241,17 +139,17 @@ class Source:
while True:
source, _ = self.region(area, progress)
progress("Reading named street")
- tags = next((dict(w.tags) for w in self.scan(source, osmium.filter.IdFilter([identity]),
- entities=osmium.osm.WAY)), {})
+ pbf = PBF(source, self.client.check_cancel)
+ seed = next(pbf.ways({identity}), None)
+ tags = {} if seed is None else seed["tags"]
if not tags.get("highway") or not tags.get("name"):
raise RequestError("place_not_found", "The selected street is absent or unnamed in the Geofabrik snapshot.")
name = tags["name"]
- selected = osmium.IdTracker()
- for way in self.scan(source, osmium.filter.TagFilter(("name", name)), osmium.filter.KeyFilter("highway"),
- entities=osmium.osm.WAY):
- selected.add_way(way.id)
- selected.add_references(way)
- ways = {w["id"]: w for w in read_data(self.scan(source, selected.id_filter()))}
+ nodes, identities = set(), set()
+ for way in pbf.ways(name=name):
+ identities.add(way["id"])
+ nodes.update(way["nodes"])
+ ways = {w["id"]: w for w in pbf.read((nodes, identities, set()))}
by_node = defaultdict(list)
for way in ways.values():
for node in way["nodes"]:
diff --git a/src/pbf.py b/src/pbf.py
new file mode 100644
index 0000000..a26085e
--- /dev/null
+++ b/src/pbf.py
@@ -0,0 +1,815 @@
+"""Standalone, standard-library reader/extractor for sorted OSM snapshot PBFs.
+
+Run: python src/pbf.py REGION.osm.pbf S W N E -o map.osm
+Schema: https://github.com/openstreetmap/OSM-binary/tree/master/osmpbf
+Supports raw/zlib blobs, ordinary/dense nodes, ways, relations and OSM metadata.
+"""
+
+import argparse
+import bisect
+import math
+import os
+import re
+import struct
+import sys
+import tempfile
+import zlib
+from array import array
+from collections import defaultdict, deque
+from concurrent.futures import ProcessPoolExecutor, TimeoutError
+from datetime import datetime, timezone
+from functools import lru_cache
+from itertools import accumulate, chain
+from multiprocessing import get_context
+from pathlib import Path
+from xml.sax.saxutils import quoteattr
+
+MAX_BLOB = 32 * 1024 * 1024
+KINDS = {1: "node", 2: "node", 3: "way", 4: "relation"}
+MEMBERS = ("node", "way", "relation")
+POI_KEYS = ("amenity", "tourism", "shop", "leisure", "historic", "office")
+SIGNED_SMALL = tuple((v >> 1) ^ -(v & 1) for v in range(128))
+SIGNED_BYTES = bytes(((v >> 1) ^ -(v & 1)) & 255 for v in range(256))
+MULTIBYTE = re.compile(rb"[\x80-\xff]")
+VARINTS = re.compile(rb"[\x80-\xff]+[\x00-\x7f]")
+SHORT_RUN = re.compile(rb"[\x00-\x7f]{32}")
+LONG_VARINT = re.compile(rb"[\x80-\xff]{4}")
+TAG_ROWS = re.compile(rb"(?<![\x80-\xff])\x00")
+quoted = lru_cache(maxsize=8192)(quoteattr)
+_SPATIAL = None
+
+
+def varint(data, offset):
+ value = shift = 0
+ while offset < len(data) and shift < 70:
+ byte = data[offset]
+ offset += 1
+ value |= (byte & 127) << shift
+ if byte < 128:
+ if value >= 1 << 64:
+ break
+ return value, offset
+ shift += 7
+ raise ValueError("Invalid PBF varint.")
+
+
+def fields(data):
+ offset = 0
+ while offset < len(data):
+ key = data[offset]
+ offset += 1
+ if key >= 128:
+ key, offset = varint(data, offset - 1)
+ number, wire = key >> 3, key & 7
+ if not number:
+ raise ValueError("Invalid PBF field number.")
+ if wire == 0:
+ value, offset = varint(data, offset)
+ elif wire in (1, 2, 5):
+ if wire == 2:
+ if offset >= len(data):
+ raise ValueError("Truncated PBF length.")
+ size = data[offset]
+ offset += 1
+ if size >= 128:
+ size, offset = varint(data, offset - 1)
+ else:
+ size = 8 if wire == 1 else 4
+ end = offset + size
+ if end > len(data):
+ raise ValueError("Truncated PBF field.")
+ value, offset = data[offset:end], end
+ else:
+ raise ValueError("Unsupported PBF wire type.")
+ yield number, value
+
+
+@lru_cache(maxsize=16)
+def lanes(count):
+ """One bit at the start of each 32-bit lane of a Python integer."""
+ return int.from_bytes(b"\x01\0\0\0" * count, "little")
+
+
+def unpack_bulk(data, signed):
+ """Decode short varints together using C-backed bytes and big integers.
+
+ Each input byte becomes a 32-bit lane. Two masked shifts combine up to
+ four base-128 digits. Mark continuation lanes with 0x80, delete them in
+ C, then compact the remaining digits into ordinary 32-bit integers.
+ """
+ count = len(data)
+ one = lanes(count)
+ low = one * 127
+ raw = int.from_bytes(data.decode("latin1").encode("utf-32le"), "little")
+ more = (raw >> 7) & one
+ value = raw & low
+ mask = more * 0xffffffff
+ value |= ((value >> 32) << 7) & mask
+ mask &= mask >> 32
+ value |= ((value >> 64) << 14) & mask
+ drop = (more << 32) & one
+ value = ((value & low) | ((value & (low << 7)) << 1)
+ | ((value & (low << 14)) << 2) | ((value & (low << 21)) << 3))
+ value = (value & ((one ^ drop) * 0xffffffff)) | (drop * 0x80808080)
+ packed = value.to_bytes(count * 4, "little").translate(None, b"\x80")
+ one = lanes(len(packed) // 4)
+ low = one * 127
+ value = int.from_bytes(packed, "little")
+ value = ((value & low) | ((value & (low << 8)) >> 1)
+ | ((value & (low << 16)) >> 2) | ((value & (low << 24)) >> 3))
+ if signed:
+ value = ((value >> 1) & (one * 0x7fffffff)) ^ ((value & one) * 0xffffffff)
+ result = array("i" if signed else "I", value.to_bytes(len(packed), "little"))
+ if sys.byteorder != "little":
+ result.byteswap()
+ return result
+
+
+def unpack(data, signed=False):
+ """Decode packed integers, with a C-level fast path for single-byte values."""
+ if not MULTIBYTE.search(data):
+ return array("b", data.translate(SIGNED_BYTES)) if signed else data
+ if len(data) > 512 and SHORT_RUN.search(data):
+ values, offset = [], 0
+ for match in VARINTS.finditer(data):
+ tail = data[offset:match.start()]
+ values.extend(array("b", tail.translate(SIGNED_BYTES)) if signed else tail)
+ value, _ = varint(match[0], 0)
+ values.append(zigzag(value) if signed else value)
+ offset = match.end()
+ tail = data[offset:]
+ if MULTIBYTE.search(tail):
+ raise ValueError("Truncated PBF packed integer.")
+ values.extend(array("b", tail.translate(SIGNED_BYTES)) if signed else tail)
+ return values
+ if len(data) > 512:
+ first, offset = varint(data, 0)
+ tail = data[offset:]
+ if not LONG_VARINT.search(tail):
+ if tail and tail[-1] >= 128:
+ raise ValueError("Truncated PBF packed integer.")
+ values = [zigzag(first) if signed else first]
+ values.extend(unpack_bulk(tail, signed))
+ return values
+ values = []
+ packed = iter(data)
+ small = SIGNED_SMALL if signed else range(128)
+ try:
+ for byte in packed:
+ if byte < 128:
+ values.append(small[byte])
+ continue
+ value = byte & 127
+ byte = next(packed)
+ value |= (byte & 127) << 7
+ if byte >= 128:
+ byte = next(packed)
+ value |= (byte & 127) << 14
+ shift = 21
+ while byte >= 128:
+ byte = next(packed)
+ value |= (byte & 127) << shift
+ shift += 7
+ if shift > 70:
+ raise ValueError("Invalid PBF packed integer.")
+ if value >= 1 << 64:
+ raise ValueError("PBF integer overflow.")
+ values.append((value >> 1) ^ -(value & 1) if signed else value)
+ except StopIteration:
+ raise ValueError("Truncated PBF packed integer.") from None
+ return values
+
+
+def zigzag(value):
+ return (value >> 1) ^ -(value & 1)
+
+
+def signed64(value):
+ return value - (1 << 64) if value >= 1 << 63 else value
+
+
+def inflate(data):
+ blob = dict(fields(data))
+ if 1 in blob:
+ raw = blob[1]
+ elif 3 in blob:
+ decoder = zlib.decompressobj()
+ try:
+ raw = decoder.decompress(blob[3], MAX_BLOB + 1)
+ except zlib.error as error:
+ raise ValueError("Invalid PBF zlib data.") from error
+ if not decoder.eof or decoder.unused_data:
+ raise ValueError("Truncated or oversized PBF zlib blob.")
+ else:
+ raise ValueError("PBF compression must be raw or zlib.")
+ if len(raw) > MAX_BLOB or (2 in blob and len(raw) != blob[2]):
+ raise ValueError("Invalid PBF blob size.")
+ return raw
+
+
+def block(path, entry):
+ with open(path, "rb") as stream:
+ stream.seek(entry[0])
+ data = stream.read(entry[1])
+ if len(data) != entry[1]:
+ raise ValueError("Truncated PBF blob.")
+ values, groups = {}, []
+ for key, value in fields(inflate(data)):
+ if key == 2:
+ groups.append(value)
+ else:
+ values[key] = value
+ if 1 not in values:
+ raise ValueError("PBF block has no string table.")
+ if not 0 < values.get(17, 100) < 1 << 31 or not 0 < values.get(18, 1000) < 1 << 31:
+ raise ValueError("Invalid PBF coordinate or date granularity.")
+ return values, groups
+
+
+def columns(message):
+ values = {}
+ for key, value in fields(message):
+ # Packed repeated fields may occur in more than one segment.
+ if key in values and isinstance(value, bytes):
+ values[key] += value
+ else:
+ values[key] = value
+ return values
+
+
+def dense(message):
+ values = columns(message)
+ ids = list(accumulate(unpack(values.get(1, b""), True)))
+ lats = list(accumulate(unpack(values.get(8, b""), True)))
+ lons = list(accumulate(unpack(values.get(9, b""), True)))
+ if not len(ids) == len(lats) == len(lons):
+ raise ValueError("Mismatched dense PBF coordinate columns.")
+ return values, ids, lats, lons
+
+
+def scan_nodes(task):
+ path, entry, area, retain = task
+ values, groups = block(path, entry)
+ gran = values.get(17, 100)
+ lat_offset, lon_offset = signed64(values.get(19, 0)), signed64(values.get(20, 0))
+ if gran <= 0:
+ raise ValueError("Invalid PBF coordinate granularity.")
+ if area is not None:
+ south, west, north, east = area
+ south, north = math.ceil((south * 1e9 - lat_offset) / gran), math.floor((north * 1e9 - lat_offset) / gran)
+ west, east = math.ceil((west * 1e9 - lon_offset) / gran), math.floor((east * 1e9 - lon_offset) / gran)
+ selected, first, last, kinds = array("q"), None, None, 0
+ coordinates = (array("q"), array("q")) if retain else None
+ for group in groups:
+ for kind, message in fields(group):
+ if kind not in KINDS:
+ raise ValueError("Unsupported OSM primitive in PBF snapshot.")
+ kinds |= 1 << kind
+ if kind == 2:
+ if area is None:
+ ids = list(accumulate(unpack(columns(message).get(1, b""), True)))
+ else:
+ _, ids, lats, lons = dense(message)
+ elif kind == 1:
+ node = dict(fields(message))
+ ids, lats, lons = [zigzag(node[1])], [zigzag(node[8])], [zigzag(node[9])]
+ else:
+ continue
+ if ids:
+ if any(a >= b for a, b in zip(ids, ids[1:])) or (last is not None and ids[0] <= last):
+ raise ValueError("PBF nodes must be sorted by unique ID.")
+ first = ids[0] if first is None else first
+ last = ids[-1]
+ if area is not None:
+ if retain:
+ hits = [i for i, (lat, lon) in enumerate(zip(lats, lons))
+ if south <= lat <= north and west <= lon <= east]
+ selected.extend(ids[i] for i in hits)
+ coordinates[0].extend(lats[i] for i in hits)
+ coordinates[1].extend(lons[i] for i in hits)
+ else:
+ selected.extend(identity for identity, lat, lon in zip(ids, lats, lons)
+ if south <= lat <= north and west <= lon <= east)
+ return kinds, first, last, selected, coordinates
+
+
+def spatial_nodes(ids):
+ global _SPATIAL
+ _SPATIAL = set(ids)
+
+
+def way_references(message):
+ """Skip tags and metadata without allocating a dictionary for every way."""
+ offset, end = 0, len(message)
+ refs = b""
+ while offset < end:
+ key = message[offset]
+ offset += 1
+ if key >= 128 or key & 7 not in (0, 2):
+ return columns(message).get(8, b"")
+ if key & 7 == 0:
+ while offset < end and message[offset] >= 128:
+ offset += 1
+ offset += 1
+ else:
+ size = message[offset]
+ offset += 1
+ if size >= 128:
+ size, offset = varint(message, offset - 1)
+ stop = offset + size
+ if stop > end:
+ raise ValueError("Truncated PBF way.")
+ if key == 66:
+ refs += message[offset:stop]
+ offset = stop
+ return refs
+
+
+def references(data):
+ if not data:
+ return iter(())
+ first, offset = varint(data, 0)
+ return accumulate(chain((zigzag(first),), unpack(data[offset:], True)))
+
+
+def scan_ways(task):
+ path, entry, xml = task
+ values, groups = block(path, entry)
+ selected, identities, nodes = [], array("q"), set()
+ strings = None
+ for group in groups:
+ for kind, message in fields(group):
+ if kind == 3:
+ packed = way_references(message)
+ if not _SPATIAL.isdisjoint(references(packed)):
+ row = columns(message)
+ refs = list(references(packed))
+ if strings is None:
+ strings = StringTable(values[1])
+ item = dict(type="way", id=row[1], nodes=refs, tags=tags(row, strings),
+ attrs=info(dict(fields(row.get(4, b""))), strings, values.get(18, 1000)))
+ if xml:
+ identities.append(row[1])
+ nodes.update(refs)
+ selected.append(xml_object(item))
+ else:
+ selected.append(item)
+ return (identities, array("q", nodes), "".join(selected).encode()) if xml else selected
+
+
+class StringTable:
+ """Decode UTF-8 only for strings referenced by selected objects."""
+
+ def __init__(self, data):
+ self.values = [value for key, value in fields(data) if key == 1]
+
+ def __getitem__(self, index):
+ try:
+ if index < 0:
+ raise IndexError(index)
+ value = self.values[index]
+ if isinstance(value, bytes):
+ value = self.values[index] = value.decode("utf-8")
+ return value
+ except IndexError as error:
+ raise ValueError("PBF string index is out of range.") from error
+
+
+def tags(values, strings):
+ keys, vals = unpack(values.get(2, b"")), unpack(values.get(3, b""))
+ if len(keys) != len(vals):
+ raise ValueError("Mismatched PBF tag columns.")
+ return {strings[k]: strings[v] for k, v in zip(keys, vals)}
+
+
+@lru_cache(maxsize=65536)
+def timestamp(value, granularity=1000):
+ return datetime.fromtimestamp(value * granularity / 1000, timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def info(values, strings, date_gran):
+ attrs = {}
+ for key, value in values.items():
+ if key == 1:
+ attrs["version"] = value
+ elif key == 2:
+ attrs["timestamp"] = timestamp(value, date_gran)
+ elif key == 3 and value:
+ attrs["changeset"] = value
+ elif key == 4 and value:
+ attrs["uid"] = value
+ elif key == 5 and value:
+ attrs["user"] = strings[value]
+ elif key == 6:
+ attrs["visible"] = "true" if value else "false"
+ return attrs
+
+
+def xml_object(item):
+ kind = item["type"]
+ # IDs, coordinates and metadata values are generated numbers/ISO dates.
+ # Only user names, tags and member roles need XML escaping.
+ attributes = f'id="{item["id"]}"' + "".join(
+ f' {key}={quoted(str(value))}' if key == "user" else f' {key}="{value}"'
+ for key, value in item.get("attrs", {}).items())
+ children = []
+ if kind == "node":
+ lat, lon = (f"{item[key]:.9f}".rstrip("0").rstrip(".") for key in ("lat", "lon"))
+ attributes += f' lat="{lat}" lon="{lon}"'
+ elif kind == "way":
+ children = [f'<nd ref="{ref}"/>' for ref in item["nodes"]]
+ else:
+ children = [f'<member type="{kind}" ref="{ref}" role={quoted(role)}/>'
+ for kind, ref, role in item["members"]]
+ children.extend(f'<tag k={quoted(k)} v={quoted(v)}/>' for k, v in item["tags"].items())
+ return (f"<{kind} {attributes}>" + "".join(children) + f"</{kind}>\n" if children
+ else f"<{kind} {attributes}/>\n")
+
+
+def read_nodes(task):
+ path, entry, wanted, mode, coordinates = task
+ known = dict(zip(wanted, zip(*coordinates))) if coordinates else None
+ wanted = set(wanted)
+ values, groups = block(path, entry)
+ strings = StringTable(values[1]) if mode != "points" else []
+ gran, date_gran = values.get(17, 100), values.get(18, 1000)
+ lat_offset, lon_offset = signed64(values.get(19, 0)), signed64(values.get(20, 0))
+ output = []
+ for group in groups:
+ for kind, message in fields(group):
+ if kind not in (1, 2):
+ continue
+ if kind == 1:
+ node = dict(fields(message))
+ identity = zigzag(node[1])
+ if identity not in wanted:
+ continue
+ rows = [(identity, zigzag(node[8]), zigzag(node[9]),
+ tags(node, strings) if mode != "points" else {},
+ info(dict(fields(node.get(4, b""))), strings, date_gran) if mode == "xml" else {})]
+ else:
+ if known is None:
+ node, ids, lats, lons = dense(message)
+ else:
+ node = columns(message)
+ ids = list(accumulate(unpack(node.get(1, b""), True)))
+ lats, lons = [0] * len(ids), [0] * len(ids)
+ if len(wanted) * 8 < len(ids):
+ selected = sorted(i for identity in wanted
+ if (i := bisect.bisect_left(ids, identity)) < len(ids) and ids[i] == identity)
+ else:
+ selected = [i for i, identity in enumerate(ids) if identity in wanted]
+ if not selected:
+ continue
+ if known is not None:
+ for i in selected:
+ lats[i], lons[i] = known[ids[i]]
+ tag_rows, metadata = {}, {}
+ if mode != "points":
+ data = node.get(10, b"")
+ if data:
+ rows = TAG_ROWS.split(data)
+ if len(rows) != len(ids) + 1 or rows[-1]:
+ raise ValueError("Mismatched dense PBF tag rows.")
+ for i in selected:
+ if rows[i]:
+ packed = unpack(rows[i])
+ if len(packed) % 2:
+ raise ValueError("Truncated dense PBF tags.")
+ tag_rows[i] = {strings[k]: strings[v] for k, v in zip(packed[::2], packed[1::2])}
+ if mode == "xml":
+ for key, data in columns(node.get(5, b"")).items():
+ if data and key in (2, 3, 4, 5):
+ first, offset = varint(data, 0)
+ if not data[offset:].strip(b"\0"):
+ metadata[key] = [zigzag(first)] * (len(data) - offset + 1)
+ else:
+ metadata[key] = list(accumulate(unpack(data, True)))
+ else:
+ metadata[key] = unpack(data)
+ if len(metadata[key]) != len(ids):
+ raise ValueError("Mismatched dense PBF metadata columns.")
+ if mode == "xml":
+ extra = {k: v for k, v in metadata.items() if k not in (1, 2)}
+ simple = 1 in metadata and 2 in metadata and all(v.count(v[0]) == len(v) for v in extra.values())
+ suffix = "".join(f' {k}={quoted(str(v))}' for k, v in
+ info({k: v[0] for k, v in extra.items()}, strings, date_gran).items()) if simple else ""
+ precision = 7 if gran % 100 == lat_offset % 100 == lon_offset % 100 == 0 else 9
+ for i in selected:
+ lat, lon = (lat_offset + gran * lats[i]) / 1e9, (lon_offset + gran * lons[i]) / 1e9
+ if not -90 <= lat <= 90 or not -180 <= lon <= 180:
+ raise ValueError("Invalid PBF node coordinates.")
+ if simple:
+ attrs = f' version="{metadata[1][i]}" timestamp="{timestamp(metadata[2][i], date_gran)}"{suffix}'
+ else:
+ attrs = "".join(f' {k}={quoted(str(v))}' for k, v in
+ info({k: v[i] for k, v in metadata.items()}, strings, date_gran).items())
+ head = f'<node id="{ids[i]}"{attrs} lat="{lat:.{precision}f}" lon="{lon:.{precision}f}"'
+ node_tags = tag_rows.get(i)
+ output.append(head + ">" + "".join(f'<tag k={quoted(k)} v={quoted(v)}/>'
+ for k, v in node_tags.items()) + "</node>\n" if node_tags else head + "/>\n")
+ continue
+ rows = ((ids[i], lats[i], lons[i], tag_rows.get(i, {}), {}) for i in selected)
+ for identity, lat, lon, node_tags, attrs in rows:
+ lat, lon = (lat_offset + gran * lat) / 1e9, (lon_offset + gran * lon) / 1e9
+ if not -90 <= lat <= 90 or not -180 <= lon <= 180:
+ raise ValueError("Invalid PBF node coordinates.")
+ if mode == "points":
+ output.append((identity, lat, lon))
+ else:
+ item = dict(type="node", id=identity, lat=lat, lon=lon, tags=node_tags, attrs=attrs)
+ output.append(xml_object(item) if mode == "xml" else item)
+ return "".join(output).encode("utf-8") if mode == "xml" else output
+
+
+class PBF:
+ """Read a Geofabrik snapshot; all indexing is temporary and in memory."""
+
+ def __init__(self, path, cancel=lambda: None):
+ self.path, self.cancel = Path(path), cancel
+ self.entries = []
+ self.selected_ways = {}
+ self.selected_relations = {}
+ self.selected_coordinates = {}
+ self.timestamp = None
+ seen_header = False
+ with self.path.open("rb") as stream:
+ size = self.path.stat().st_size
+ while stream.tell() < size:
+ cancel()
+ length = stream.read(4)
+ if len(length) != 4 or not 0 < (length := struct.unpack(">I", length)[0]) < 65536:
+ raise ValueError("Invalid PBF block header.")
+ header = dict(fields(stream.read(length)))
+ count = header.get(3, 0)
+ if not 0 < count <= MAX_BLOB or stream.tell() + count > size:
+ raise ValueError("Truncated or oversized PBF block.")
+ if header.get(1) == b"OSMHeader" and not seen_header:
+ seen_header = True
+ for key, value in fields(inflate(stream.read(count))):
+ if key == 4 and value not in (b"OsmSchema-V0.6", b"DenseNodes"):
+ raise ValueError(f"Unsupported required PBF feature: {value!r}")
+ if key == 32:
+ self.timestamp = timestamp(value)
+ elif header.get(1) == b"OSMData" and seen_header:
+ self.entries.append([stream.tell(), count, 0, None, None])
+ stream.seek(count, 1)
+ else:
+ raise ValueError("Unsupported PBF block type or order.")
+ if not seen_header:
+ raise ValueError("PBF snapshot contains no header.")
+
+ def parallel(self, function, tasks, initializer=None, initargs=()):
+ workers = min(8, os.cpu_count() or 1) if self.path.stat().st_size > 16 * 1024 * 1024 else 1
+ if workers == 1:
+ if initializer is not None:
+ initializer(*initargs)
+ for task in tasks:
+ self.cancel()
+ yield function(task)
+ return
+ pool = ProcessPoolExecutor(workers, mp_context=get_context("spawn"), initializer=initializer, initargs=initargs)
+ pending, tasks = deque(), iter(tasks)
+ try:
+ for _ in range(workers * 8):
+ if (task := next(tasks, None)) is not None:
+ pending.append(pool.submit(function, task))
+ while pending:
+ self.cancel()
+ try:
+ result = pending[0].result(timeout=0.2)
+ except TimeoutError:
+ continue
+ pending.popleft()
+ if (task := next(tasks, None)) is not None:
+ pending.append(pool.submit(function, task))
+ yield result
+ finally:
+ for future in pending:
+ future.cancel()
+ pool.shutdown(wait=True, cancel_futures=True)
+
+ def raw(self, kinds, wanted=None):
+ if wanted is not None and not wanted:
+ return
+ for entry in self.entries:
+ self.cancel()
+ if entry[2] and not any(entry[2] & (1 << k) for k in kinds):
+ continue
+ values, groups = block(self.path, entry)
+ strings = None
+ for group in groups:
+ for kind, message in fields(group):
+ entry[2] |= 1 << kind
+ if kind not in kinds:
+ continue
+ # Geofabrik writes ID first. Skip other fields for unselected objects.
+ if wanted is not None and message[:1] == b"\x08" and varint(message, 1)[0] not in wanted:
+ continue
+ row = columns(message)
+ if wanted is not None and row[1] not in wanted:
+ continue
+ if strings is None:
+ strings = StringTable(values[1])
+ yield kind, row, strings, values.get(18, 1000)
+
+ def ways(self, wanted=None, name=None):
+ if wanted is not None and wanted <= self.selected_ways.keys():
+ for identity in sorted(wanted):
+ yield self.selected_ways[identity]
+ return
+ for _, row, strings, date_gran in self.raw((3,), wanted):
+ properties = tags(row, strings)
+ if name is not None and (properties.get("name") != name or not properties.get("highway")):
+ continue
+ way = dict(type="way", id=row[1], tags=properties,
+ nodes=list(accumulate(unpack(row.get(8, b""), True))),
+ attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))
+ if wanted is not None or name is not None:
+ self.selected_ways[way["id"]] = way
+ yield way
+
+ def relations(self, wanted=None):
+ rows = ((4, *self.selected_relations[i]) for i in sorted(wanted)) if (
+ wanted is not None and wanted <= self.selected_relations.keys()) else self.raw((4,), wanted)
+ for _, row, strings, date_gran in rows:
+ refs = list(accumulate(unpack(row.get(9, b""), True)))
+ roles, kinds = unpack(row.get(8, b"")), unpack(row.get(10, b""))
+ if not len(refs) == len(roles) == len(kinds):
+ raise ValueError("Mismatched PBF relation member columns.")
+ if any(kind > 2 for kind in kinds):
+ raise ValueError("Invalid PBF relation member type.")
+ yield dict(type="relation", id=row[1], tags=tags(row, strings),
+ members=[(MEMBERS[kind], ref, strings[role]) for kind, ref, role in zip(kinds, refs, roles)],
+ attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))
+
+ def select(self, area, xml=False):
+ self.way_xml = []
+ self.selected_coordinates.clear()
+ spatial, nodes, ways, relations = set(), set(), set(), set()
+ tasks = ((self.path, entry, area, xml) for entry in self.entries)
+ last = None
+ for entry, (kinds, first, end, selected, coordinates) in zip(self.entries, self.parallel(scan_nodes, tasks)):
+ entry[2:] = kinds, first, end
+ if first is not None:
+ if last is not None and first <= last:
+ raise ValueError("PBF node blocks must be sorted by unique ID.")
+ last = end
+ spatial.update(selected)
+ if coordinates is not None and selected:
+ self.selected_coordinates[entry[0]] = selected, coordinates
+ nodes.update(spatial)
+ tasks = ((self.path, entry, xml) for entry in self.entries if entry[2] & 8)
+ for selected in self.parallel(scan_ways, tasks, spatial_nodes, (array("q", spatial),)):
+ if xml:
+ identities, refs, chunk = selected
+ ways.update(identities)
+ nodes.update(refs)
+ self.way_xml.append(chunk)
+ continue
+ for way in selected:
+ ways.add(way["id"])
+ nodes.update(way["nodes"])
+ self.selected_ways[way["id"]] = way
+ parents, pending = defaultdict(list), []
+ rows = {}
+ for _, row, strings, date_gran in self.raw((4,)):
+ identity = row[1]
+ rows[identity] = row, strings, date_gran
+ refs = accumulate(unpack(row.get(9, b""), True))
+ for kind, ref in zip(unpack(row.get(10, b"")), refs):
+ if kind == 2:
+ parents[ref].append(identity)
+ elif ref in (spatial if kind == 0 else ways):
+ pending.append(identity)
+ while pending:
+ identity = pending.pop()
+ if identity not in relations:
+ relations.add(identity)
+ pending.extend(parents[identity])
+ self.selected_relations = {i: rows[i] for i in relations}
+ return nodes, ways, relations
+
+ def nodes(self, wanted, mode="points"):
+ ordered = sorted(wanted)
+ if not ordered:
+ return
+ unknown = [entry for entry in self.entries if not entry[2] or (entry[2] & 6 and entry[3] is None)]
+ for entry, (kinds, first, last, _, _) in zip(
+ unknown, self.parallel(scan_nodes, ((self.path, e, None, False) for e in unknown))):
+ entry[2:] = kinds, first, last
+ tasks = []
+ for entry in self.entries:
+ if entry[3] is not None:
+ ids = ordered[bisect.bisect_left(ordered, entry[3]):bisect.bisect_right(ordered, entry[4])]
+ if ids:
+ saved = self.selected_coordinates.get(entry[0])
+ coordinates = saved[1] if saved is not None and saved[0] == array("q", ids) else None
+ tasks.append((self.path, entry, ids, mode, coordinates))
+ for rows in self.parallel(read_nodes, tasks):
+ self.cancel()
+ if mode == "xml":
+ yield rows
+ else:
+ yield from rows
+
+ def complete(self, selected):
+ nodes, ways, relations = selected
+ pending = set(relations)
+ while pending:
+ unseen = set()
+ for relation in self.relations(pending):
+ for kind, ref, _ in relation["members"]:
+ if kind == "node":
+ nodes.add(ref)
+ elif kind == "way":
+ ways.add(ref)
+ elif ref not in relations:
+ unseen.add(ref)
+ relations.update(unseen)
+ pending = unseen
+ for way in self.ways(ways):
+ nodes.update(way["nodes"])
+
+ def read(self, selected, centers=False):
+ wanted_nodes, wanted_ways, wanted_relations = selected
+ points, items, boxes = {}, [], {}
+ for node in self.nodes(wanted_nodes, "objects" if centers else "points"):
+ identity, lat, lon = (node["id"], node["lat"], node["lon"]) if centers else node
+ points[identity] = lat, lon
+ if centers:
+ boxes[("node", identity)] = lat, lon, lat, lon
+ if node["tags"]:
+ items.append(dict(type="node", id=identity, tags=node["tags"], center=(lat, lon)))
+ for way in self.ways(wanted_ways):
+ if any(ref not in points for ref in way["nodes"]):
+ raise ValueError("PBF extract contains incomplete way geometry.")
+ geometry = [points[ref] for ref in way["nodes"]]
+ items.append(dict(type="way", id=way["id"], tags=way["tags"], nodes=way["nodes"], points=geometry))
+ if centers and geometry:
+ lats, lons = zip(*geometry)
+ boxes[("way", way["id"])] = min(lats), min(lons), max(lats), max(lons)
+ pending = [dict(type="relation", id=r["id"], tags=r["tags"], members=[m[:2] for m in r["members"]])
+ for r in self.relations(wanted_relations)]
+ items.extend(pending)
+ while centers and pending:
+ unresolved = []
+ for item in pending:
+ members = [boxes[ref] for ref in item["members"] if ref in boxes]
+ if not members or len(members) != len(item["members"]):
+ unresolved.append(item)
+ continue
+ boxes[("relation", item["id"])] = (min(b[0] for b in members), min(b[1] for b in members),
+ max(b[2] for b in members), max(b[3] for b in members))
+ if len(unresolved) == len(pending):
+ break
+ pending = unresolved
+ for item in items:
+ if (box := boxes.get((item["type"], item["id"]))) is not None:
+ item["center"] = (box[0] + box[2]) / 2, (box[1] + box[3]) / 2
+ return items
+
+ def export(self, area, output):
+ selected = self.select(area, xml=True)
+ output = Path(output)
+ output.parent.mkdir(parents=True, exist_ok=True)
+ fd, name = tempfile.mkstemp(prefix=f".{output.name}-", dir=output.parent)
+ try:
+ with os.fdopen(fd, "wb") as stream:
+ south, west, north, east = area
+ stream.write((f'<?xml version="1.0" encoding="UTF-8"?>\n<osm version="0.6" generator="Aleph">\n'
+ f'<bounds minlat="{south}" minlon="{west}" maxlat="{north}" maxlon="{east}"/>\n').encode())
+ written = 0
+ for chunk in self.nodes(selected[0], "xml"):
+ stream.write(chunk)
+ written += chunk.count(b"<node ")
+ if written != len(selected[0]):
+ raise ValueError("PBF extract contains incomplete way geometry.")
+ for chunk in self.way_xml:
+ stream.write(chunk)
+ for item in self.relations(selected[2]):
+ stream.write(xml_object(item).encode("utf-8"))
+ stream.write(b"</osm>\n")
+ self.cancel()
+ os.replace(name, output)
+ finally:
+ Path(name).unlink(missing_ok=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("source", type=Path)
+ parser.add_argument("bbox", nargs=4, type=float, metavar="COORD")
+ parser.add_argument("-o", "--output", type=Path, required=True)
+ args = parser.parse_args()
+ south, west, north, east = args.bbox
+ if not all(math.isfinite(v) for v in args.bbox) or not (-90 <= south < north <= 90 and -180 <= west < east <= 180):
+ parser.error("Use a nonempty bounding box in south west north east order.")
+ try:
+ PBF(args.source).export(args.bbox, args.output)
+ except (OSError, ValueError, zlib.error, KeyError, IndexError) as error:
+ parser.exit(1, f"PBF extraction failed: {error}\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/data/dense.osm.pbf b/tests/data/dense.osm.pbf
new file mode 100644
index 0000000..3a8ac2e
Binary files /dev/null and b/tests/data/dense.osm.pbf differ
diff --git a/tests/data/nodes.osm.pbf b/tests/data/nodes.osm.pbf
new file mode 100644
index 0000000..36dbc25
Binary files /dev/null and b/tests/data/nodes.osm.pbf differ
diff --git a/tests/test_osm.py b/tests/test_osm.py
index df5490d..e77bf90 100644
--- a/tests/test_osm.py
+++ b/tests/test_osm.py
@@ -6,8 +6,6 @@ import unittest
from unittest.mock import Mock, patch
from xml.etree import ElementTree as ET
-import osmium
-
from src.common import Client
from src.osm import Source, covers
@@ -39,27 +37,7 @@ class ExtractTests(unittest.TestCase):
index = dict(features=[dict(type="Feature", properties=dict(name="Test region", urls=dict(pbf=url)),
geometry=polygon([[-2, -2], [2, -2], [2, 2], [-2, 2], [-2, -2]]))])
(self.source.directory / "index.json").write_text(json.dumps(index))
- xml = self.folder / "fixture.osm"
- xml.write_text('''<osm version="0.6">
- <node id="1" lat="0" lon="0"><tag k="amenity" v="cafe"/></node>
- <node id="2" lat="0" lon="0.001"/>
- <node id="3" lat="0" lon="0.002"/>
- <node id="4" lat="0.01" lon="0.01"/>
- <node id="5" lat="0.01" lon="0.011"/>
- <node id="6" lat="0" lon="-0.003"/>
- <way id="10"><nd ref="1"/><nd ref="2"/><tag k="highway" v="residential"/><tag k="name" v="Main Street"/></way>
- <way id="11"><nd ref="2"/><nd ref="3"/><tag k="highway" v="residential"/><tag k="name" v="Main Street"/></way>
- <way id="12"><nd ref="4"/><nd ref="5"/><tag k="highway" v="residential"/><tag k="name" v="Main Street"/></way>
- <way id="13"><nd ref="1"/><nd ref="6"/></way>
- <relation id="19"><member type="relation" ref="20" role=""/>
- <tag k="tourism" v="attraction"/><tag k="name" v="Museum complex"/></relation>
- <relation id="20"><member type="way" ref="10" role=""/><member type="way" ref="11" role=""/>
- <tag k="tourism" v="attraction"/><tag k="name" v="Museum"/></relation>
- </osm>''')
- header = osmium.io.Header()
- header.set("osmosis_replication_timestamp", "2026-09-20T20:00:00Z")
- with osmium.SimpleWriter(self.path, header=header) as writer:
- osmium.apply(xml, writer)
+ self.path.write_bytes((Path(__file__).parent / "data" / "dense.osm.pbf").read_bytes())
self.area = (-.0001, -.0001, .0001, .0005)
def test_export_completes_ways_and_preserves_partial_relation_members(self):
@@ -105,7 +83,7 @@ class ExtractTests(unittest.TestCase):
Path(kwargs["destination"]).write_bytes(b"not a PBF")
self.client.get.side_effect = download
- with self.assertRaises(OSError):
+ with self.assertRaises(ValueError):
self.source.region(self.area, lambda *args: None)
self.assertEqual(self.path.read_bytes(), original)
diff --git a/tests/test_pbf.py b/tests/test_pbf.py
new file mode 100644
index 0000000..32fd2a7
--- /dev/null
+++ b/tests/test_pbf.py
@@ -0,0 +1,123 @@
+"""PBF wire-format boundaries and extraction checked without external packages."""
+
+import random
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+import zlib
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+from src.pbf import PBF, unpack
+
+
+def integer(value):
+ result = bytearray()
+ while value >= 128:
+ result.append((value & 127) | 128)
+ value >>= 7
+ return bytes(result) + bytes([value])
+
+
+def field(number, value):
+ if isinstance(value, bytes):
+ return integer(number * 8 + 2) + integer(len(value)) + value
+ return integer(number * 8) + integer(value)
+
+
+def packed(values, signed=False):
+ return b"".join(integer((v << 1) ^ (v >> 63) if signed else v) for v in values)
+
+
+def blob(kind, payload, compressed=False):
+ data = field(2, len(payload)) + field(3, zlib.compress(payload)) if compressed else field(1, payload)
+ header = field(1, kind) + field(3, len(data))
+ return struct.pack(">I", len(header)) + header + data
+
+
+def snapshot(group, *, options=b"", features=(b"OsmSchema-V0.6", b"DenseNodes")):
+ header = b"".join(field(4, feature) for feature in features)
+ data = field(1, field(1, b"")) + field(2, group) + options
+ return blob(b"OSMHeader", header) + blob(b"OSMData", data, True)
+
+
+class PBFTests(unittest.TestCase):
+ def setUp(self):
+ self.folder = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ self.path = self.folder / "region.osm.pbf"
+ self.output = self.folder / "map.osm"
+ self.area = (-.0001, -.0001, .0001, .0005)
+
+ def test_dense_and_normal_nodes_preserve_tags_metadata_and_complete_ways(self):
+ results = []
+ for name in ("dense.osm.pbf", "nodes.osm.pbf"):
+ source = PBF(Path(__file__).parent / "data" / name)
+ source.export(self.area, self.output)
+ root = ET.parse(self.output).getroot()
+ self.assertEqual({n.attrib["id"] for n in root.findall("node")}, {"1", "2", "6"})
+ node = root.find("node")
+ self.assertEqual(node.attrib["version"], "3")
+ self.assertEqual(node.attrib["timestamp"], "2026-09-19T12:34:56Z")
+ self.assertEqual(node.find("tag[@k='name']").attrib["v"], 'Caffè & "Tea" <Terrace>')
+ results.append(source.read(source.select(self.area), centers=True))
+ self.assertEqual(*results)
+
+ def test_nondefault_offsets_granularity_and_split_packed_fields(self):
+ ids = field(1, packed([10], True)) + field(1, packed([1], True))
+ dense = ids + field(8, packed([-10, 20], True)) + field(9, packed([5, -10], True))
+ options = field(17, 10_000_000) + field(19, (1 << 64) - 1_000_000_000) + field(20, 2_000_000_000)
+ self.path.write_bytes(snapshot(field(2, dense), options=options))
+ source = PBF(self.path)
+ selected = source.select((-2, 1, 0, 3))
+ self.assertEqual(selected, ({10, 11}, set(), set()))
+ self.assertEqual(list(source.nodes(selected[0])), [(10, -1.1, 2.05), (11, -.9, 1.95)])
+
+ def test_integer_boundaries_and_malformed_varints(self):
+ values = [0, 1, 127, 128, 16384, 1 << 32, (1 << 64) - 1]
+ self.assertEqual(list(unpack(packed(values))), values)
+ signed = [0, -1, 63, -64, 64, -65, 1 << 40, -(1 << 63)]
+ self.assertEqual(list(unpack(packed(signed, True), True)), signed)
+ for data in (b"\x80", b"\x80" * 10, b"\xff" * 9 + b"\x02"):
+ with self.subTest(data=data), self.assertRaises(ValueError):
+ unpack(data)
+
+ def test_bulk_integer_decoding_and_lane_boundaries(self):
+ rng = random.Random(17)
+ for signed in (False, True):
+ limit = 1 << (27 if signed else 28)
+ values = [rng.randrange(-limit if signed else 0, limit) for _ in range(2000)]
+ values[0] = -(1 << 63) if signed else (1 << 64) - 1
+ values[1:9] = [0, 1, 63, 64, 127, 128, 16383, 16384]
+ if signed:
+ values[9:14] = [-1, -64, -65, -8192, -8193]
+ data = packed(values, signed)
+ self.assertEqual(list(unpack(data, signed)), values)
+ with self.assertRaises(ValueError):
+ unpack(data + b"\x80", signed)
+ values = [1 << 40] + [1] * 1000 + [1 << 30, 0, 3]
+ self.assertEqual(list(unpack(packed(values, True), True)), values)
+
+ def test_rejects_unknown_features_and_truncated_files(self):
+ for data in (b"", b"\x00\x00", snapshot(b"", features=(b"HistoricalInformation",)), snapshot(b"")[:-1]):
+ self.path.write_bytes(data)
+ with self.subTest(data=data[:20]), self.assertRaises(ValueError):
+ PBF(self.path)
+
+ def test_missing_way_node_preserves_previous_output(self):
+ node = field(1, 2) + field(8, 0) + field(9, 0)
+ way = field(1, 10) + field(8, packed([1, 998], True))
+ self.path.write_bytes(snapshot(field(1, node) + field(3, way)))
+ self.output.write_bytes(b"previous")
+ with self.assertRaisesRegex(ValueError, "incomplete way"):
+ PBF(self.path).export(self.area, self.output)
+ self.assertEqual(self.output.read_bytes(), b"previous")
+ self.assertEqual(list(self.folder.glob(".map.osm-*")), [])
+
+ def test_standalone_script_runs_without_site_packages(self):
+ source = Path(__file__).parent / "data" / "dense.osm.pbf"
+ script = Path(__file__).resolve().parents[1] / "src" / "pbf.py"
+ subprocess.run([sys.executable, "-S", str(script), str(source), *map(str, self.area),
+ "-o", str(self.output)], check=True, capture_output=True)
+ self.assertEqual(len(ET.parse(self.output).getroot().findall("way")), 2)