Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Use numpy dependency for speedups
Belluxx82386041444f
diff --git a/src/mesh.py b/src/mesh.pyindex 095858f..dbbd265 100644--- a/src/mesh.py+++ b/src/mesh.py@@ -11,13 +11,12 @@ import jsonimport mathimport shutilimport struct-import sysimport tempfile-from array import array-from itertools import accumulate++import numpy as npfrom .common import atomic_path, fetch-from .pbf import fields, unpack+from .pbf import fields, varintsBASE = "https://kh.google.com/rt/earth/"LEAF, NODATA, USE_IMAGERY_EPOCH = 4, 8, 16@@ -25,7 +24,6 @@ JPEG = 1LOWEST, HIGHEST = -1000, 9000 # Heights searched for nodes, in meters from the sphere.GLB_LIMIT = 2**32TILES_PER_MESH = 64 # Unreal's Nanite allows 64 materials per mesh; Godot allows 256 surfaces.-BITS = [bytes((byte >> i) & 1 for i in range(8)) for byte in range(256)]def first(data):@@ -175,24 +173,19 @@ def check(data):raise ValueError("Unexpected 3D mesh node data.")-def words(low, high):- """Little-endian 16-bit values from separate low and high byte planes."""- joined = bytearray(2 * len(low))- joined[0::2], joined[1::2] = low, high- values = array("H", joined)- if sys.byteorder != "little":- values.byteswap()- return values--def counted(data, name):"""Packed integers preceded by their count."""- values = unpack(data)- if not values or len(values) != values[0] + 1:+ values = varints(data).astype(np.int64)+ if not values.size or values.size != values[0] + 1:raise ValueError(f"Unexpected 3D mesh {name}.")return values[1:]+def fitted(values, size):+ """Truncate or zero-pad to size."""+ return np.pad(values[:size], (0, max(0, size - len(values))))++def decode(data, mask, axes, radius, inside):"""Yield (positions, texture coordinates, triangles, JPEG) per mesh, in local meters.@@ -227,48 +220,49 @@ def decode(data, mask, axes, radius, inside):if 1 not in texture or texture.get(2, JPEG) != JPEG:raise ValueError("Unexpected 3D mesh texture.")- xs, ys, zs = ([v & 255 for v in accumulate(vertices[i * count:(i + 1) * count])] for i in range(3))- east = [a0 * x + a1 * y + a2 * z + at for x, y, z in zip(xs, ys, zs)]- north = [b0 * x + b1 * y + b2 * z + bt for x, y, z in zip(xs, ys, zs)]- up = [c0 * x + c1 * y + c2 * z + ct for x, y, z in zip(xs, ys, zs)]+ xs, ys, zs = np.cumsum(np.frombuffer(vertices, np.uint8).reshape(3, count), axis=1, dtype=np.uint8).astype(float)+ east = a0 * xs + a1 * ys + a2 * zs + at+ north = b0 * xs + b1 * ys + b2 * zs + bt+ up = c0 * xs + c1 * ys + c2 * zs + ctu_mod, v_mod = 1 + uv[0] + (uv[1] << 8), 1 + uv[2] + (uv[3] << 8)- planes = [uv[4 + i * count:4 + (i + 1) * count] for i in range(4)]+ # Four byte planes: low u, low v, high u, high v.+ planes = np.frombuffer(uv, np.uint8, offset=4).reshape(4, count).astype(np.int64)if len(offsets) == 16:u_offset, v_offset, u_scale, v_scale = struct.unpack("<4f", offsets)else:u_offset, v_offset, u_scale, v_scale = 0.5, 0.5, 1 / u_mod, 1 / v_mod- us = [(u % u_mod + u_offset) * u_scale for u in accumulate(words(planes[0], planes[2]))]- vs = [(v % v_mod + v_offset) * v_scale for v in accumulate(words(planes[1], planes[3]))]+ us = (np.cumsum(planes[0] | planes[2] << 8) % u_mod + u_offset) * u_scale+ vs = (np.cumsum(planes[1] | planes[3] << 8) % v_mod + v_offset) * v_scale# A zero index adds the next new vertex; others count back from the newest.- strip, zeros = [], 0- for value in counted(mesh.get(3, b""), "indices"):- strip.append(zeros - value)- if not value:- zeros += 1- if zeros > count or (strip and min(strip) < 0):+ values = counted(mesh.get(3, b""), "indices")+ new = values == 0+ strip = np.cumsum(new) - new - values+ if new.sum() > count or (strip.size and strip.min() < 0):raise ValueError("Unexpected 3D mesh indices.")# Runs cycle through eight octants per layer; trailing positions belong to no drawn layer.runs = counted(mesh.get(8, b""), "octants")- keep = b"".join((b"\1" if j < 24 and not mask >> (j & 7) & 1 else b"\0") * size- for j, size in enumerate(runs)).ljust(len(strip), b"\0")- skirts = b"".join(BITS[byte] for byte in mesh.get(13, b"")).ljust(len(strip), b"\0")- triangles = []- for t in range(len(strip) - 2):- if not keep[t + 2] or skirts[t]:- continue- a, b, c = strip[t], strip[t + 1], strip[t + 2]- if a != b != c != a and inside((east[a] + east[b] + east[c]) / 3, (north[a] + north[b] + north[c]) / 3,- (up[a] + up[b] + up[c]) / 3):- triangles += (a, c, b) if t & 1 else (a, b, c)- if not triangles:+ layers = np.arange(runs.size)+ keep = fitted(np.repeat((layers < 24) & ~(mask >> (layers & 7) & 1).astype(bool), runs), strip.size)+ skirts = fitted(np.unpackbits(np.frombuffer(mesh.get(13, b""), np.uint8), bitorder="little"), strip.size)+ a, b, c = strip[:-2], strip[1:-1], strip[2:]+ drawn = (keep[2:].astype(bool) & ~skirts[:-2].astype(bool) & (a != b) & (b != c) & (c != a)+ & inside((east[a] + east[b] + east[c]) / 3, (north[a] + north[b] + north[c]) / 3,+ (up[a] + up[b] + up[c]) / 3))+ # Odd triangles in a strip are wound the other way.+ odd = np.arange(a.size) % 2 == 1+ triangles = np.stack((a, np.where(odd, c, b), np.where(odd, b, c)), axis=1)[drawn].ravel()+ if not triangles.size:continue- used = list(dict.fromkeys(triangles))- index = {v: i for i, v in enumerate(used)}+ # Vertices in order of first use.+ unique, first_use = np.unique(triangles, return_index=True)+ used = unique[np.argsort(first_use)]+ index = np.empty(count, np.int64)+ index[used] = np.arange(used.size)# glTF is Y-up: x east, y up, z south.- positions = array("f", [value for v in used for value in (east[v], up[v], -north[v])])- texcoords = array("f", [value for v in used for value in (us[v], vs[v])])- indices = array("H" if len(used) <= 65536 else "I", [index[v] for v in triangles])+ positions = np.stack((east[used], up[used], -north[used]), axis=1).astype(np.float32)+ texcoords = np.stack((us[used], vs[used]), axis=1).astype(np.float32)+ indices = index[triangles].astype(np.uint16 if used.size <= 65536 else np.uint32)yield positions, texcoords, indices, texture[1]@@ -282,8 +276,9 @@ def crop(area, axes, radius):def inside(e, n, u):r = radius + ux, y, z = ex * e + nx * n + ux * r, ey * e + ny * n + uy * r, ez * e + nz * n + uz * r- length = math.sqrt(x * x + y * y + z * z)- return low * length <= z <= high * length and x * west_x + y * west_y >= 0 >= x * east_x + y * east_y+ length = np.sqrt(x * x + y * y + z * z)+ return ((low * length <= z) & (z <= high * length)+ & (x * west_x + y * west_y >= 0) & (x * east_x + y * east_y <= 0))return inside@@ -311,10 +306,8 @@ def export(path, folder, stage, area, progress):progress("Building mesh.glb", 0, len(saved))with tempfile.TemporaryFile(dir=folder) as spool:def view(data, target=None):- if sys.byteorder != "little" and isinstance(data, array):- data = array(data.typecode, data)- data.byteswap()- data = bytes(data)+ if isinstance(data, np.ndarray):+ data = data.astype(data.dtype.newbyteorder("<")).tobytes()gltf["bufferViews"].append(dict(buffer=0, byteOffset=spool.tell(), byteLength=len(data),**({"target": target} if target else {})))spool.write(data + b"\0" * (-len(data) % 4))@@ -331,12 +324,11 @@ def export(path, folder, stage, area, progress):i, k = len(gltf["materials"]), len(gltf["accessors"])name = node[0] if not part else f"{node[0]}-{part}"gltf["accessors"] += [- dict(bufferView=view(positions, 34962), componentType=5126, count=len(positions) // 3,- type="VEC3", min=[min(positions[j::3]) for j in range(3)],- max=[max(positions[j::3]) for j in range(3)]),- dict(bufferView=view(texcoords, 34962), componentType=5126, count=len(texcoords) // 2,+ dict(bufferView=view(positions, 34962), componentType=5126, count=len(positions),+ type="VEC3", min=positions.min(axis=0).tolist(), max=positions.max(axis=0).tolist()),+ dict(bufferView=view(texcoords, 34962), componentType=5126, count=len(texcoords),type="VEC2"),- dict(bufferView=view(indices, 34963), componentType=5123 if indices.typecode == "H" else 5125,+ dict(bufferView=view(indices, 34963), componentType=5123 if indices.dtype == np.uint16 else 5125,count=len(indices), type="SCALAR"),]gltf["images"].append(dict(bufferView=view(jpeg), mimeType="image/jpeg"))@@ -361,15 +353,12 @@ def export(path, folder, stage, area, progress):for accessor in positions:span = gltf["bufferViews"][accessor["bufferView"]]spool.seek(span["byteOffset"])- values = array("f", spool.read(span["byteLength"]))- if sys.byteorder != "little":- values.byteswap()- values[1::3] = array("f", [y - base for y in values[1::3]])- accessor["min"][1], accessor["max"][1] = min(values[1::3]), max(values[1::3])- if sys.byteorder != "little":- values.byteswap()+ values = np.frombuffer(spool.read(span["byteLength"]), "<f4").reshape(-1, 3).copy()+ # Subtract in double precision, then round once to float32.+ values[:, 1] = values[:, 1].astype(float) - base+ accessor["min"][1], accessor["max"][1] = float(values[:, 1].min()), float(values[:, 1].max())spool.seek(span["byteOffset"])- spool.write(bytes(values))+ spool.write(values.tobytes())spool.seek(0, 2)gltf["asset"]["extras"] = dict(latitude=lat, longitude=lon, base=base, radius=radius, bounds=list(area),diff --git a/src/pbf.py b/src/pbf.pyindex e5bd7bd..13a5c34 100644--- a/src/pbf.py+++ b/src/pbf.py@@ -1,4 +1,4 @@-"""Standard-library reader/extractor for sorted OSM snapshot PBFs.+"""Reader/extractor for sorted OSM snapshot PBFs.Schema: https://github.com/openstreetmap/OSM-binary/tree/master/osmpbfSupports raw/zlib blobs, ordinary/dense nodes, ways, relations and OSM metadata.@@ -9,7 +9,6 @@ import mathimport osimport reimport struct-import sysimport zlibfrom array import arrayfrom collections import defaultdict, deque@@ -21,16 +20,14 @@ from multiprocessing import get_contextfrom pathlib import Pathfrom xml.sax.saxutils import quoteattr+import numpy as np+from .common import atomic_pathMAX_BLOB = 32 * 1024 * 1024MEMBERS = ("node", "way", "relation")-SIGNED_SMALL = tuple((v >> 1) ^ -(v & 1) for v in range(128))SIGNED_BYTES = bytes(((v >> 1) ^ -(v & 1)) & 255 for v in range(256))-MULTIBYTE = re.compile(rb"[\x80-\xff]")-VARINTS = re.compile(rb"[\x80-\xff]+[\x00-\x7f]")-SHORT_RUN = re.compile(rb"[\x00-\x7f]{32}")-LONG_VARINT = re.compile(rb"[\x80-\xff]{4}")+NUMPY_BYTES = 256 # Shorter packed fields decode faster in Python than through numpy's per-call overhead.TAG_ROWS = re.compile(rb"(?<![\x80-\xff])\x00")quoted = lru_cache(maxsize=8192)(quoteattr)_SPATIAL = None@@ -81,99 +78,44 @@ def fields(data):yield number, value-@lru_cache(maxsize=16)-def lanes(count):- """One bit at the start of each 32-bit lane of a Python integer."""- return int.from_bytes(b"\x01\0\0\0" * count, "little")---def unpack_bulk(data, signed):- """Decode short varints together using C-backed bytes and big integers.-- Each input byte becomes a 32-bit lane. Two masked shifts combine up to- four base-128 digits. Mark continuation lanes with 0x80, delete them in- C, then compact the remaining digits into ordinary 32-bit integers.- """- count = len(data)- one = lanes(count)- low = one * 127- raw = int.from_bytes(data.decode("latin1").encode("utf-32le"), "little")- more = (raw >> 7) & one- value = raw & low- mask = more * 0xffffffff- value |= ((value >> 32) << 7) & mask- mask &= mask >> 32- value |= ((value >> 64) << 14) & mask- drop = (more << 32) & one- value = ((value & low) | ((value & (low << 7)) << 1)- | ((value & (low << 14)) << 2) | ((value & (low << 21)) << 3))- value = (value & ((one ^ drop) * 0xffffffff)) | (drop * 0x80808080)- packed = value.to_bytes(count * 4, "little").translate(None, b"\x80")- one = lanes(len(packed) // 4)- low = one * 127- value = int.from_bytes(packed, "little")- value = ((value & low) | ((value & (low << 8)) >> 1)- | ((value & (low << 16)) >> 2) | ((value & (low << 24)) >> 3))+def varints(data, signed=False):+ """Packed varints as a uint64 array, or int64 when zigzag-encoded."""+ raw = np.frombuffer(data, np.uint8)+ if not raw.size:+ return np.zeros(0, np.int64 if signed else np.uint64)+ if raw[-1] >= 128:+ raise ValueError("Truncated PBF packed integer.")+ ends = np.flatnonzero(raw < 128)+ starts = np.concatenate(([0], ends[:-1] + 1))+ lengths = ends - starts + 1+ # Ten bytes hold 64 bits only when the last carries a single bit.+ if lengths.max() > 10 or (raw[ends[lengths == 10]] > 1).any():+ raise ValueError("PBF integer overflow.")+ shifts = (np.arange(raw.size) - np.repeat(starts, lengths)).astype(np.uint64) * np.uint64(7)+ values = np.bitwise_or.reduceat((raw & 127).astype(np.uint64) << shifts, starts)if signed:- value = ((value >> 1) & (one * 0x7fffffff)) ^ ((value & one) * 0xffffffff)- result = array("i" if signed else "I", value.to_bytes(len(packed), "little"))- if sys.byteorder != "little":- result.byteswap()- return result+ return (values >> np.uint64(1)).astype(np.int64) ^ -(values & np.uint64(1)).astype(np.int64)+ return valuesdef unpack(data, signed=False):- """Decode packed integers, with a C-level fast path for single-byte values."""- if not MULTIBYTE.search(data):+ """Packed integers as a sequence of Python ints."""+ if len(data) >= NUMPY_BYTES:+ return varints(data, signed).tolist()+ if data.isascii():return array("b", data.translate(SIGNED_BYTES)) if signed else data- if len(data) > 512 and SHORT_RUN.search(data):- values, offset = [], 0- for match in VARINTS.finditer(data):- tail = data[offset:match.start()]- values.extend(array("b", tail.translate(SIGNED_BYTES)) if signed else tail)- value, _ = varint(match[0], 0)- values.append(zigzag(value) if signed else value)- offset = match.end()- tail = data[offset:]- if MULTIBYTE.search(tail):- raise ValueError("Truncated PBF packed integer.")- values.extend(array("b", tail.translate(SIGNED_BYTES)) if signed else tail)- return values- if len(data) > 512:- first, offset = varint(data, 0)- tail = data[offset:]- if not LONG_VARINT.search(tail):- if tail and tail[-1] >= 128:- raise ValueError("Truncated PBF packed integer.")- values = [zigzag(first) if signed else first]- values.extend(unpack_bulk(tail, signed))- return values- values = []- packed = iter(data)- small = SIGNED_SMALL if signed else range(128)- try:- for byte in packed:- if byte < 128:- values.append(small[byte])- continue- value = byte & 127- byte = next(packed)- value |= (byte & 127) << 7- if byte >= 128:- byte = next(packed)- value |= (byte & 127) << 14- shift = 21- while byte >= 128:- byte = next(packed)- value |= (byte & 127) << shift- shift += 7- if shift > 70:- raise ValueError("Invalid PBF packed integer.")- if value >= 1 << 64:- raise ValueError("PBF integer overflow.")+ values, value, shift = [], 0, 0+ for byte in data:+ value |= (byte & 127) << shift+ if byte < 128:+ if value >= 1 << 64:+ raise ValueError("PBF integer overflow.")values.append((value >> 1) ^ -(value & 1) if signed else value)- except StopIteration:- raise ValueError("Truncated PBF packed integer.") from None+ value = shift = 0+ elif (shift := shift + 7) > 63:+ raise ValueError("Invalid PBF packed integer.")+ if shift:+ raise ValueError("Truncated PBF packed integer.")return values@@ -185,12 +127,13 @@ def signed64(value):return value - (1 << 64) if value >= 1 << 63 else value+def column(data):+ """A delta-coded column as an int64 array."""+ return np.cumsum(varints(data, True))++def deltas(data):- """Separate the first absolute value so small deltas use the fast path."""- if not data:- return iter(())- first, offset = varint(data, 0)- return accumulate(chain((zigzag(first),), unpack(data[offset:], True)))+ return column(data).tolist() if len(data) >= NUMPY_BYTES else list(accumulate(unpack(data, True)))def inflate(data):@@ -244,9 +187,9 @@ def columns(message):def dense(message):values = columns(message)- ids = list(deltas(values.get(1, b"")))- lats = list(deltas(values.get(8, b"")))- lons = list(deltas(values.get(9, b"")))+ ids = deltas(values.get(1, b""))+ lats = deltas(values.get(8, b""))+ lons = deltas(values.get(9, b""))if not len(ids) == len(lats) == len(lons):raise ValueError("Mismatched dense PBF coordinate columns.")return values, ids, lats, lons@@ -261,44 +204,40 @@ def scan_nodes(task):south, west, north, east = areasouth, north = math.ceil((south * 1e9 - lat_offset) / gran), math.floor((north * 1e9 - lat_offset) / gran)west, east = math.ceil((west * 1e9 - lon_offset) / gran), math.floor((east * 1e9 - lon_offset) / gran)- selected, first, last, kinds = array("q"), None, None, 0- coordinates = (array("q"), array("q")) if retain else None+ hits, first, last, kinds = [], None, None, 0for group in groups:for kind, message in fields(group):if kind not in (1, 2, 3, 4):raise ValueError("Unsupported OSM primitive in PBF snapshot.")kinds |= 1 << kindif kind == 2:- if area is None:- ids = list(deltas(columns(message).get(1, b"")))- else:- _, ids, lats, lons = dense(message)+ row = columns(message)+ ids = column(row.get(1, b""))+ if area is not None:+ lats, lons = column(row.get(8, b"")), column(row.get(9, b""))+ if not ids.size == lats.size == lons.size:+ raise ValueError("Mismatched dense PBF coordinate columns.")elif kind == 1:node = dict(fields(message))- ids, lats, lons = [zigzag(node[1])], [zigzag(node[8])], [zigzag(node[9])]+ ids, lats, lons = (np.array([zigzag(node[key])]) for key in (1, 8, 9))else:continue- if ids:- if any(a >= b for a, b in zip(ids, ids[1:])) or (last is not None and ids[0] <= last):+ if ids.size:+ if (np.diff(ids) <= 0).any() or (last is not None and ids[0] <= last):raise ValueError("PBF nodes must be sorted by unique ID.")- first = ids[0] if first is None else first- last = ids[-1]+ first = int(ids[0]) if first is None else first+ last = int(ids[-1])if area is not None:- if retain:- hits = [i for i, (lat, lon) in enumerate(zip(lats, lons))- if south <= lat <= north and west <= lon <= east]- selected.extend(ids[i] for i in hits)- coordinates[0].extend(lats[i] for i in hits)- coordinates[1].extend(lons[i] for i in hits)- else:- selected.extend(identity for identity, lat, lon in zip(ids, lats, lons)- if south <= lat <= north and west <= lon <= east)- return kinds, first, last, selected, coordinates+ inside = (lats >= south) & (lats <= north) & (lons >= west) & (lons <= east)+ hits.append((ids[inside], lats[inside], lons[inside]))+ selected, lats, lons = (np.concatenate(parts) for parts in zip(*hits)) if hits else [np.zeros(0, np.int64)] * 3+ coordinates = (array("q", lats.tobytes()), array("q", lons.tobytes())) if retain else None+ return kinds, first, last, array("q", selected.tobytes()), coordinatesdef spatial_nodes(ids):global _SPATIAL- _SPATIAL = set(ids)+ _SPATIAL = np.sort(np.array(ids, np.int64))def way_references(message):@@ -328,38 +267,56 @@ def way_references(message):return refs+def touching(messages):+ """Which ways reference a spatial node, decoding all of a block's references at once."""+ refs = [way_references(message) for message in messages]+ data = b"".join(refs)+ if not data:+ return np.zeros(len(messages), bool)+ # The way of each reference, from where its last byte falls among the ways' byte ranges.+ owners = np.searchsorted(np.cumsum([len(r) for r in refs]), np.flatnonzero(np.frombuffer(data, np.uint8) < 128),+ side="right")+ totals = np.cumsum(varints(data, True))+ # Deltas restart in each way: subtract the running total reached before its first reference.+ firsts = np.searchsorted(owners, owners, side="left")+ nodes = totals - np.where(firsts > 0, totals[firsts - 1], 0)+ found = np.minimum(np.searchsorted(_SPATIAL, nodes), _SPATIAL.size - 1)+ hit = np.zeros(len(messages), bool)+ if _SPATIAL.size:+ hit[owners[_SPATIAL[found] == nodes]] = True+ return hit++def scan_ways(task):"""Select spatial ways or requested member IDs and record the block's ID range."""path, entry, xml, wanted = taskvalues, groups = block(path, entry)+ messages = [message for group in groups for kind, message in fields(group) if kind == 3]+ if not messages:+ raise ValueError("PBF way block contains no ways.")++ def identity(message):+ # Geofabrik writes ID first.+ return varint(message, 1)[0] if message[:1] == b"\x08" else columns(message)[1]++ if wanted is not None:+ chosen = [message for message in messages if identity(message) in wanted]+ else:+ chosen = [message for message, hit in zip(messages, touching(messages)) if hit]selected, identities, nodes = [], array("q"), set()- first = last = None- strings = None- for group in groups:- for kind, message in fields(group):- if kind == 3:- if first is None:- first = message- last = message- if wanted is not None:- identity = varint(message, 1)[0] if message[:1] == b"\x08" else columns(message)[1]- if identity not in wanted:- continue- packed = way_references(message)- if wanted is not None or not _SPATIAL.isdisjoint(deltas(packed)):- row = columns(message)- refs = list(deltas(packed))- if strings is None:- strings = StringTable(values[1])- item = dict(type="way", id=row[1], nodes=refs, tags=tags(row, strings),- attrs=info(dict(fields(row.get(4, b""))), strings, values.get(18, 1000)))- if xml:- identities.append(row[1])- nodes.update(refs)- selected.append(xml_object(item))- else:- selected.append(item)- bounds = tuple(varint(m, 1)[0] if m[:1] == b"\x08" else columns(m)[1] for m in (first, last))+ strings = StringTable(values[1]) if chosen else None+ for message in chosen:+ row = columns(message)+ refs = deltas(row.get(8, b""))+ item = dict(type="way", id=row[1], nodes=refs, tags=tags(row, strings),+ attrs=info(dict(fields(row.get(4, b""))), strings, values.get(18, 1000)))+ if xml:+ identities.append(row[1])+ nodes.update(refs)+ selected.append(xml_object(item))+ else:+ selected.append(item)+ bounds = identity(messages[0]), identity(messages[-1])result = (identities, array("q", nodes), "".join(selected).encode()) if xml else selectedreturn bounds, result@@ -462,7 +419,7 @@ def read_nodes(task):node, ids, lats, lons = dense(message)else:node = columns(message)- ids = list(deltas(node.get(1, b"")))+ ids = deltas(node.get(1, b""))lats, lons = [0] * len(ids), [0] * len(ids)if len(wanted) * 8 < len(ids):selected = sorted(i for identity in wanted@@ -494,7 +451,7 @@ def read_nodes(task):if not data[offset:].strip(b"\0"):metadata[key] = [zigzag(first)] * (len(data) - offset + 1)else:- metadata[key] = list(deltas(data))+ metadata[key] = deltas(data)else:metadata[key] = unpack(data)if len(metadata[key]) != len(ids):@@ -631,7 +588,7 @@ class PBF:if name is not None and (properties.get("name") != name or not properties.get("highway")):continueway = dict(type="way", id=row[1], tags=properties,- nodes=list(deltas(row.get(8, b""))),+ nodes=deltas(row.get(8, b"")),attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))if wanted is not None or name is not None:self.selected_ways[way["id"]] = way@@ -641,7 +598,7 @@ class PBF:rows = ((4, *self.selected_relations[i]) for i in sorted(wanted)) if (wanted is not None and wanted <= self.selected_relations.keys()) else self.raw((4,), wanted)for _, row, strings, date_gran in rows:- refs = list(deltas(row.get(9, b"")))+ refs = deltas(row.get(9, b""))roles, kinds = unpack(row.get(8, b"")), unpack(row.get(10, b""))if not len(refs) == len(roles) == len(kinds):raise ValueError("Mismatched PBF relation member columns.")@@ -688,7 +645,7 @@ class PBF:for _, row, strings, date_gran in self.raw((4,)):identity = row[1]rows[identity] = row, strings, date_gran- refs = list(deltas(row.get(9, b"")))+ refs = deltas(row.get(9, b""))kinds = unpack(row.get(10, b""))matched = Falsefor kind, ref in zip(kinds, refs):diff --git a/tests/test_pbf.py b/tests/test_pbf.pyindex e202c76..5387c64 100644--- a/tests/test_pbf.py+++ b/tests/test_pbf.py@@ -36,9 +36,8 @@ def block(kind, payload):class PBFTests(unittest.TestCase):- def test_packed_integer_fast_paths_match_independent_encoder(self):- # Exercise byte translation, mixed runs, bulk lanes, and the long-varint- # fallback on both sides of the 512-byte optimization threshold.+ def test_packed_integers_match_independent_encoder(self):+ # Short fields decode in Python, long ones with numpy; cover both sides of the threshold.cases = {"single_byte": list(range(64)) * 10,"short": [0, 63, 64, 127, 128, 8191, 8192, 2**20, 2**27 - 1],@@ -54,7 +53,7 @@ class PBFTests(unittest.TestCase):self.assertEqual(list(pbf.unpack(uint(2**64 - 1))), [2**64 - 1])self.assertEqual(list(pbf.unpack(packed([-2**63, 2**63 - 1], True), True)), [-2**63, 2**63 - 1])- def test_truncated_or_overflowing_integers_fail_in_every_fast_path(self):+ def test_truncated_or_overflowing_integers_fail_in_both_decoders(self):prefixes = [b"", b"\x00" * 600, packed([128, 8192] * 200), packed([2**40] * 100)]for prefix in prefixes:for suffix in (b"\x80", uint(2**64)):