Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Add parallel workers for all major data sources
Belluxx62089d15be64
diff --git a/src/capture.py b/src/capture.pyindex ea8366b..1ad7fbe 100644--- a/src/capture.py+++ b/src/capture.py@@ -11,7 +11,7 @@ from pathlib import Pathfrom PIL import Imagefrom . import mesh, satellite, streetview, terrain-from .common import MissingImagery, contained, now, number, open_image, positive, write_bytes, write_json+from .common import MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_jsonfrom .geo import bounds, collection, coordinate, feature, grid, pixel, tile_ring, tilesSOURCES = ("streetview", "satellite", "osm", "mesh")@@ -23,7 +23,7 @@ DEFAULT_OPTIONS = dict()# Average connection speed / hardware timings, just to give an idea to the user, not precise-REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.27, satellite_tile=0.25, terrain_tile=1.39,+REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.02, satellite_tile=0.02, terrain_tile=0.13,mesh_node=0.021)SPHERE_TILES = (1, 2, 8, 32, 128, 512)OSM_EXPORT_SECONDS = 10@@ -208,15 +208,13 @@ def capture_streetview(stage, run, folder, client, progress):def capture_satellite(stage, run, folder, client, progress):- progress("Satellite", len(stage["results"]), total(stage))- for i, tile in enumerate(tiles(stage["grid"])):- if i < len(stage["results"]):- continue- address = f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}"+ done = len(stage["results"])+ progress("Satellite", done, total(stage))+ remaining = list(tiles(stage["grid"]))[done:]+ addresses = [f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}" for tile in remaining]+ for tile, address, data in zip(remaining, addresses, fetch(client, addresses, missing_ok=True)):extension = run["options"]["satellite_format"]- try:- data = client.get(address, missing_ok=True)- except MissingImagery:+ if data is None:# Persist the gap so resume and offline export keep it transparent.extension = "png"with BytesIO() as buffer, Image.new("RGBA", (256, 256)) as image:@@ -233,8 +231,9 @@ def capture_satellite(stage, run, folder, client, progress):f"_x{tile['x']}_y{tile['y']}.{extension}")write_bytes(folder / name, data)stage["results"].append(dict(tile, filename=name, source_url=address, captured_at=now()))+ done += 1yield- progress("Satellite", i + 1, total(stage))+ progress("Satellite", done, total(stage))def capture_osm(stage, run, folder, client, progress):@@ -244,25 +243,26 @@ def capture_osm(stage, run, folder, client, progress):def capture_terrain(stage, run, folder, client, progress):- progress("Downloading terrain tiles", len(stage["results"]), total(stage))- for i, tile in enumerate(tiles(stage["grid"])):- if i < len(stage["results"]):- continue- address = f"https://elevation-tiles-prod.s3.amazonaws.com/geotiff/{tile['zoom']}/{tile['x']}/{tile['y']}.tif"- data = client.get(address)+ done = len(stage["results"])+ progress("Downloading terrain tiles", done, total(stage))+ remaining = list(tiles(stage["grid"]))[done:]+ addresses = [f"https://elevation-tiles-prod.s3.amazonaws.com/geotiff/{tile['zoom']}/{tile['x']}/{tile['y']}.tif"+ for tile in remaining]+ for tile, address, data in zip(remaining, addresses, fetch(client, addresses)):terrain.validate(terrain.TIFF(BytesIO(data)), tile)name = f"terrain/tiles/terrain_z{tile['zoom']}_x{tile['x']}_y{tile['y']}.tif"write_bytes(folder / name, data)stage["results"].append(dict(tile, filename=name, source_url=address, captured_at=now()))+ done += 1yield- progress("Downloading terrain tiles", i + 1, total(stage))+ progress("Downloading terrain tiles", done, total(stage))def capture_mesh(stage, run, folder, client, progress):done = len(stage["results"])progress("3D mesh", done, total(stage))nodes = stage["nodes"][done:]- for node, data in zip(nodes, mesh.fetch(client, map(mesh.address, nodes))):+ for node, data in zip(nodes, fetch(client, map(mesh.address, nodes), missing_ok=True)):if data is None:stage["results"].append(dict(status="skipped", reason="The 3D mesh node is not available."))else:diff --git a/src/common.py b/src/common.pyindex 475bc06..780c6b3 100644--- a/src/common.py+++ b/src/common.py@@ -7,7 +7,10 @@ import osimport shutilimport sysimport tempfile+import threadingimport time+from collections import deque+from concurrent.futures import ThreadPoolExecutorfrom contextlib import contextmanagerfrom datetime import datetime, timezonefrom functools import cached_property@@ -21,6 +24,7 @@ from PIL import ImageDEFAULT_CACHE = Path(os.environ.get("XDG_CACHE_HOME", str(Path.home() / ".cache"))) / "aleph"APP_AGENT = "Aleph/0.1"+WORKERS = 16BROWSER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.0 Safari/605.1.15"@@ -165,6 +169,7 @@ class CachedClient(Client):"""Cache successful quick-query responses for one week."""hits = misses = 0+ lock = threading.Lock()def get(self, address, **kwargs):if kwargs.get("destination") is not None:@@ -172,14 +177,42 @@ class CachedClient(Client):key = hashlib.sha256(address.encode()).hexdigest()path = self.cache_dir / keyif not self.refresh and path.is_file() and time.time() - path.stat().st_mtime < 7 * 24 * 60 * 60:- self.hits += 1+ with self.lock:+ self.hits += 1return path.read_bytes()result = super().get(address, **kwargs)write_bytes(path, result)- self.misses += 1+ with self.lock:+ self.misses += 1return result+def fetch(client, addresses, *, missing_ok=False):+ """Yield responses in order with a few requests in flight; with missing_ok, None marks missing data."""+ def get(address):+ try:+ return client.get(address, missing_ok=missing_ok)+ except MissingImagery:+ return None++ # Keep requests sequential when the user asks for pauses between them.+ if client.delay:+ yield from map(get, addresses)+ return+ pending = deque()+ with ThreadPoolExecutor(WORKERS) as pool:+ try:+ for address in addresses:+ pending.append(pool.submit(get, address))+ if len(pending) > 2 * WORKERS:+ yield pending.popleft().result()+ while pending:+ yield pending.popleft().result()+ finally:+ for future in pending:+ future.cancel()++class Progress:def __init__(self):self.stream = sys.stderrdiff --git a/src/mesh.py b/src/mesh.pyindex 685eacd..352205a 100644--- a/src/mesh.py+++ b/src/mesh.py@@ -14,49 +14,20 @@ import structimport sysimport tempfilefrom array import array-from collections import deque-from concurrent.futures import ThreadPoolExecutorfrom itertools import accumulate-from .common import MissingImagery, atomic_path+from .common import atomic_path, fetchfrom .pbf import fields, unpackBASE = "https://kh.google.com/rt/earth/"LEAF, NODATA, USE_IMAGERY_EPOCH = 4, 8, 16JPEG = 1-WORKERS = 16LOWEST, HIGHEST = -1000, 9000 # Heights searched for nodes, in meters from the sphere.GLB_LIMIT = 2**32TILES_PER_MESH = 64 # Unreal's Nanite allows 64 materials per mesh; Godot allows 256 surfaces.BITS = [bytes((byte >> i) & 1 for i in range(8)) for byte in range(256)]-def fetch(client, addresses):- """Yield responses in order with a few requests in flight; None marks missing data."""- def get(address):- try:- return client.get(address, missing_ok=True)- except MissingImagery:- return None-- # Keep requests sequential when the user asks for pauses between them.- if client.delay:- yield from map(get, addresses)- return- pending = deque()- with ThreadPoolExecutor(WORKERS) as pool:- try:- for address in addresses:- pending.append(pool.submit(get, address))- if len(pending) > 2 * WORKERS:- yield pending.popleft().result()- while pending:- yield pending.popleft().result()- finally:- for future in pending:- future.cancel()--def first(data):"""Protobuf fields keeping the first value of repeated ones."""values = {}@@ -161,7 +132,7 @@ def plan(client, area, level, progress, *, allow_empty=False):while pending:batch, pending = pending, []addresses = [f"{BASE}BulkMetadata/pb=!1m2!1s{path}!2u{epoch}" for path, epoch in batch]- for (head, epoch), data in zip(batch, fetch(client, addresses)):+ for (head, epoch), data in zip(batch, fetch(client, addresses, missing_ok=True)):done += 1progress("Finding 3D mesh nodes", done)if data is None:diff --git a/src/streetview.py b/src/streetview.pyindex d359ae7..3a94463 100644--- a/src/streetview.py+++ b/src/streetview.py@@ -10,7 +10,7 @@ from pathlib import Pathfrom PIL import Image-from .common import MissingImagery, number, open_image, save_image, url, write_bytes+from .common import MissingImagery, fetch, number, open_image, save_image, url, write_bytesfrom .geo import Line, RoadIndex, clip, distance, grid, inside, tilesEXCLUDED = re.compile(r"^(construction|proposed|planned|abandoned|razed|demolished)$")@@ -67,10 +67,11 @@ def coverage(client, area, progress):count = coverage_grid["rows"] * coverage_grid["columns"]found = {}progress("Finding panoramas", 0, count)- for i, tile in enumerate(tiles(coverage_grid), 1):- address = url("https://www.google.com/maps/photometa/ac/v1",- pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")- for view in parse_coverage(client.get(address)):+ addresses = (url("https://www.google.com/maps/photometa/ac/v1",+ pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")+ for tile in tiles(coverage_grid))+ for i, data in enumerate(fetch(client, addresses), 1):+ for view in parse_coverage(data):found[view["pano_id"]] = viewprogress("Finding panoramas", i, count)return list(found.values())@@ -153,19 +154,21 @@ def save_sphere(client, metadata, zoom, target):columns, rows = math.ceil(width / tw), math.ceil(height / th)target = Path(target)cache = target.parent / ".sphere-tiles" / metadata["pano_id"] / f"{zoom}-{width}x{height}-{tw}x{th}"+ positions = [(x, y) for y in range(rows) for x in range(columns)]+ missing = [(x, y) for x, y in positions if not (cache / f"{x}-{y}.tile").is_file()]+ addresses = (url("https://streetviewpixels-pa.googleapis.com/v1/tile", cb_client="maps_sv.tactile",+ panoid=metadata["pano_id"], zoom=zoom, x=x, y=y) for x, y in missing)with Image.new("RGB", (width, height)) as sphere:- for y in range(rows):- for x in range(columns):- client.check_cancel()- path = cache / f"{x}-{y}.tile"- cached = path.is_file()- data = path.read_bytes() if cached else client.get(- url("https://streetviewpixels-pa.googleapis.com/v1/tile", cb_client="maps_sv.tactile",- panoid=metadata["pano_id"], zoom=zoom, x=x, y=y), missing_ok=True)- with open_image(data, (tw, th)) as tile:- if not cached:- write_bytes(path, data)- sphere.paste(tile, (x * tw, y * th))+ for (x, y), data in zip(missing, fetch(client, addresses, missing_ok=True)):+ if data is None:+ raise MissingImagery("The requested image is not available.")+ with open_image(data, (tw, th)) as tile:+ write_bytes(cache / f"{x}-{y}.tile", data)+ sphere.paste(tile, (x * tw, y * th))+ for x, y in set(positions) - set(missing):+ client.check_cancel()+ with open_image((cache / f"{x}-{y}.tile").read_bytes(), (tw, th)) as tile:+ sphere.paste(tile, (x * tw, y * th))client.check_cancel()save_image(sphere, target, quality=90)shutil.rmtree(cache)