Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Speedup name lookups
Belluxx71f1bfdd8427
diff --git a/src/pbf.py b/src/pbf.pyindex 13a5c34..ef8e68b 100644--- a/src/pbf.py+++ b/src/pbf.py@@ -287,6 +287,55 @@ def touching(messages):return hit+def identity(message):+ # Geofabrik writes ID first.+ return varint(message, 1)[0] if message[:1] == b"\x08" else columns(message)[1]+++def way_item(row, strings, date_gran):+ return dict(type="way", id=row[1], tags=tags(row, strings), nodes=deltas(row.get(8, b"")),+ attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))+++def relation_item(row, strings, date_gran):+ refs = deltas(row.get(9, b""))+ roles, kinds = unpack(row.get(8, b"")), unpack(row.get(10, b""))+ if not len(refs) == len(roles) == len(kinds):+ raise ValueError("Mismatched PBF relation member columns.")+ if any(kind > 2 for kind in kinds):+ raise ValueError("Invalid PBF relation member type.")+ return dict(type="relation", id=row[1], tags=tags(row, strings),+ members=[(MEMBERS[kind], ref, strings[role]) for kind, ref, role in zip(kinds, refs, roles)],+ attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))+++def scan_objects(task):+ """A block's ways or relations: all of them, those with wanted IDs, or highways with a name.++ Also returns the block's object kinds and the ID range of the requested kind.+ """+ path, entry, kind, wanted, name = task+ values, groups = block(path, entry)+ kinds, items, first, last = 0, [], None, None+ strings = StringTable(values[1])+ # Objects can only carry a name present in their block's string table.+ absent = name is not None and name.encode() not in strings.values+ wanted = None if wanted is None else set(wanted)+ for group in groups:+ for number, message in fields(group):+ kinds |= 1 << number+ if number != kind:+ continue+ current = identity(message)+ first, last = current if first is None else first, current+ if absent or (wanted is not None and current not in wanted):+ continue+ item = (way_item if kind == 3 else relation_item)(columns(message), strings, values.get(18, 1000))+ if name is None or (item["tags"].get("name") == name and item["tags"].get("highway")):+ items.append(item)+ return kinds, (first, last), items++def scan_ways(task):"""Select spatial ways or requested member IDs and record the block's ID range."""path, entry, xml, wanted = task@@ -295,10 +344,6 @@ def scan_ways(task):if not messages:raise ValueError("PBF way block contains no ways.")- def identity(message):- # Geofabrik writes ID first.- return varint(message, 1)[0] if message[:1] == b"\x08" else columns(message)[1]-if wanted is not None:chosen = [message for message in messages if identity(message) in wanted]else:@@ -306,13 +351,10 @@ def scan_ways(task):selected, identities, nodes = [], array("q"), set()strings = StringTable(values[1]) if chosen else Nonefor message in chosen:- row = columns(message)- refs = deltas(row.get(8, b""))- item = dict(type="way", id=row[1], nodes=refs, tags=tags(row, strings),- attrs=info(dict(fields(row.get(4, b""))), strings, values.get(18, 1000)))+ item = way_item(columns(message), strings, values.get(18, 1000))if xml:- identities.append(row[1])- nodes.update(refs)+ identities.append(item["id"])+ nodes.update(item["nodes"])selected.append(xml_object(item))else:selected.append(item)@@ -496,6 +538,7 @@ class PBF:self.selected_ways = {}self.selected_relations = {}self.selected_coordinates = {}+ self.spans = {} # Block offset → first and last way or relation ID.self.timestamp = Noneseen_header = Falsewith self.path.open("rb") as stream:@@ -554,59 +597,47 @@ class PBF:future.cancel()pool.shutdown(wait=True, cancel_futures=True)- def raw(self, kinds, wanted=None):- if wanted is not None and not wanted:- return+ def objects(self, kind, wanted=None, name=None):+ """Ways (3) or relations (4) in file order, decoded in parallel; see scan_objects."""+ if wanted is not None:+ if not wanted:+ return+ wanted = sorted(wanted)+ tasks = []for entry in self.entries:- self.cancel()- if entry[2] and not any(entry[2] & (1 << k) for k in kinds):+ if entry[2] and not entry[2] & 1 << kind:continue- values, groups = block(self.path, entry)- strings = None- for group in groups:- for kind, message in fields(group):- entry[2] |= 1 << kind- if kind not in kinds:- continue- # Geofabrik writes ID first. Skip other fields for unselected objects.- if wanted is not None and message[:1] == b"\x08" and varint(message, 1)[0] not in wanted:- continue- row = columns(message)- if wanted is not None and row[1] not in wanted:- continue- if strings is None:- strings = StringTable(values[1])- yield kind, row, strings, values.get(18, 1000)+ ids = wanted+ if wanted is not None and entry[0] in self.spans:+ # Blocks are sorted by ID, so send each only the IDs within its range.+ first, last = self.spans[entry[0]]+ ids = wanted[bisect.bisect_left(wanted, first):bisect.bisect_right(wanted, last)]+ if not ids:+ continue+ tasks.append((entry, (self.path, entry, kind, None if ids is None else array("q", ids), name)))+ results = self.parallel(scan_objects, (task for _, task in tasks))+ for (entry, _), (kinds, span, items) in zip(tasks, results):+ entry[2] |= kinds+ if span[0] is not None:+ self.spans[entry[0]] = span+ yield from itemsdef ways(self, wanted=None, name=None):if wanted is not None and wanted <= self.selected_ways.keys():for identity in sorted(wanted):yield self.selected_ways[identity]return- for _, row, strings, date_gran in self.raw((3,), wanted):- properties = tags(row, strings)- if name is not None and (properties.get("name") != name or not properties.get("highway")):- continue- way = dict(type="way", id=row[1], tags=properties,- nodes=deltas(row.get(8, b"")),- attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))+ for way in self.objects(3, wanted, name):if wanted is not None or name is not None:self.selected_ways[way["id"]] = wayyield waydef relations(self, wanted=None):- rows = ((4, *self.selected_relations[i]) for i in sorted(wanted)) if (- wanted is not None and wanted <= self.selected_relations.keys()) else self.raw((4,), wanted)- for _, row, strings, date_gran in rows:- refs = deltas(row.get(9, b""))- roles, kinds = unpack(row.get(8, b"")), unpack(row.get(10, b""))- if not len(refs) == len(roles) == len(kinds):- raise ValueError("Mismatched PBF relation member columns.")- if any(kind > 2 for kind in kinds):- raise ValueError("Invalid PBF relation member type.")- yield dict(type="relation", id=row[1], tags=tags(row, strings),- members=[(MEMBERS[kind], ref, strings[role]) for kind, ref, role in zip(kinds, refs, roles)],- attrs=info(dict(fields(row.get(4, b""))), strings, date_gran))+ if wanted is not None and wanted <= self.selected_relations.keys():+ for identity in sorted(wanted):+ yield self.selected_relations[identity]+ return+ yield from self.objects(4, wanted)def select(self, area, xml=False):self.way_xml = {}@@ -629,6 +660,7 @@ class PBF:way_blocks = []results = self.parallel(scan_ways, tasks, spatial_nodes, (array("q", spatial),))for entry, (bounds, selected) in zip(entries, results, strict=True):+ self.spans[entry[0]] = boundsif xml:identities, refs, chunk = selectedways.update(identities)@@ -642,24 +674,22 @@ class PBF:self.selected_ways[way["id"]] = wayparents, pending = defaultdict(list), []rows, extra_ways = {}, set()- for _, row, strings, date_gran in self.raw((4,)):- identity = row[1]- rows[identity] = row, strings, date_gran- refs = deltas(row.get(9, b""))- kinds = unpack(row.get(10, b""))+ for relation in self.relations():+ identity = relation["id"]+ rows[identity] = relationmatched = False- for kind, ref in zip(kinds, refs):- if kind == 2:+ for kind, ref, _ in relation["members"]:+ if kind == "relation":parents[ref].append(identity)- elif ref in (spatial if kind == 0 else ways):+ elif ref in (spatial if kind == "node" else ways):matched = Trueif matched:pending.append(identity)- if xml and tags(row, strings).get("type") == "multipolygon":- for kind, ref in zip(kinds, refs):- if kind == 0:+ if xml and relation["tags"].get("type") == "multipolygon":+ for kind, ref, _ in relation["members"]:+ if kind == "node":nodes.add(ref)- elif kind == 1:+ elif kind == "way":extra_ways.add(ref)while pending:identity = pending.pop()