Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Add parallel workers for all major data sources

Belluxx62089d15be64
diff --git a/src/capture.py b/src/capture.py
index ea8366b..1ad7fbe 100644
--- a/src/capture.py
+++ b/src/capture.py
@@ -11,7 +11,7 @@ from pathlib import Path
from PIL import Image
from . import mesh, satellite, streetview, terrain
-from .common import MissingImagery, contained, now, number, open_image, positive, write_bytes, write_json
+from .common import MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_json
from .geo import bounds, collection, coordinate, feature, grid, pixel, tile_ring, tiles
SOURCES = ("streetview", "satellite", "osm", "mesh")
@@ -23,7 +23,7 @@ DEFAULT_OPTIONS = dict(
)
# Average connection speed / hardware timings, just to give an idea to the user, not precise
-REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.27, satellite_tile=0.25, terrain_tile=1.39,
+REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.02, satellite_tile=0.02, terrain_tile=0.13,
mesh_node=0.021)
SPHERE_TILES = (1, 2, 8, 32, 128, 512)
OSM_EXPORT_SECONDS = 10
@@ -208,15 +208,13 @@ def capture_streetview(stage, run, folder, client, progress):
def capture_satellite(stage, run, folder, client, progress):
- progress("Satellite", len(stage["results"]), total(stage))
- for i, tile in enumerate(tiles(stage["grid"])):
- if i < len(stage["results"]):
- continue
- address = f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}"
+ done = len(stage["results"])
+ progress("Satellite", done, total(stage))
+ remaining = list(tiles(stage["grid"]))[done:]
+ addresses = [f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}" for tile in remaining]
+ for tile, address, data in zip(remaining, addresses, fetch(client, addresses, missing_ok=True)):
extension = run["options"]["satellite_format"]
- try:
- data = client.get(address, missing_ok=True)
- except MissingImagery:
+ if data is None:
# Persist the gap so resume and offline export keep it transparent.
extension = "png"
with BytesIO() as buffer, Image.new("RGBA", (256, 256)) as image:
@@ -233,8 +231,9 @@ def capture_satellite(stage, run, folder, client, progress):
f"_x{tile['x']}_y{tile['y']}.{extension}")
write_bytes(folder / name, data)
stage["results"].append(dict(tile, filename=name, source_url=address, captured_at=now()))
+ done += 1
yield
- progress("Satellite", i + 1, total(stage))
+ progress("Satellite", done, total(stage))
def capture_osm(stage, run, folder, client, progress):
@@ -244,25 +243,26 @@ def capture_osm(stage, run, folder, client, progress):
def capture_terrain(stage, run, folder, client, progress):
- progress("Downloading terrain tiles", len(stage["results"]), total(stage))
- for i, tile in enumerate(tiles(stage["grid"])):
- if i < len(stage["results"]):
- continue
- address = f"https://elevation-tiles-prod.s3.amazonaws.com/geotiff/{tile['zoom']}/{tile['x']}/{tile['y']}.tif"
- data = client.get(address)
+ done = len(stage["results"])
+ progress("Downloading terrain tiles", done, total(stage))
+ remaining = list(tiles(stage["grid"]))[done:]
+ addresses = [f"https://elevation-tiles-prod.s3.amazonaws.com/geotiff/{tile['zoom']}/{tile['x']}/{tile['y']}.tif"
+ for tile in remaining]
+ for tile, address, data in zip(remaining, addresses, fetch(client, addresses)):
terrain.validate(terrain.TIFF(BytesIO(data)), tile)
name = f"terrain/tiles/terrain_z{tile['zoom']}_x{tile['x']}_y{tile['y']}.tif"
write_bytes(folder / name, data)
stage["results"].append(dict(tile, filename=name, source_url=address, captured_at=now()))
+ done += 1
yield
- progress("Downloading terrain tiles", i + 1, total(stage))
+ progress("Downloading terrain tiles", done, total(stage))
def capture_mesh(stage, run, folder, client, progress):
done = len(stage["results"])
progress("3D mesh", done, total(stage))
nodes = stage["nodes"][done:]
- for node, data in zip(nodes, mesh.fetch(client, map(mesh.address, nodes))):
+ for node, data in zip(nodes, fetch(client, map(mesh.address, nodes), missing_ok=True)):
if data is None:
stage["results"].append(dict(status="skipped", reason="The 3D mesh node is not available."))
else:
diff --git a/src/common.py b/src/common.py
index 475bc06..780c6b3 100644
--- a/src/common.py
+++ b/src/common.py
@@ -7,7 +7,10 @@ import os
import shutil
import sys
import tempfile
+import threading
import time
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from datetime import datetime, timezone
from functools import cached_property
@@ -21,6 +24,7 @@ from PIL import Image
DEFAULT_CACHE = Path(os.environ.get("XDG_CACHE_HOME", str(Path.home() / ".cache"))) / "aleph"
APP_AGENT = "Aleph/0.1"
+WORKERS = 16
BROWSER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.0 Safari/605.1.15"
@@ -165,6 +169,7 @@ class CachedClient(Client):
"""Cache successful quick-query responses for one week."""
hits = misses = 0
+ lock = threading.Lock()
def get(self, address, **kwargs):
if kwargs.get("destination") is not None:
@@ -172,14 +177,42 @@ class CachedClient(Client):
key = hashlib.sha256(address.encode()).hexdigest()
path = self.cache_dir / key
if not self.refresh and path.is_file() and time.time() - path.stat().st_mtime < 7 * 24 * 60 * 60:
- self.hits += 1
+ with self.lock:
+ self.hits += 1
return path.read_bytes()
result = super().get(address, **kwargs)
write_bytes(path, result)
- self.misses += 1
+ with self.lock:
+ self.misses += 1
return result
+def fetch(client, addresses, *, missing_ok=False):
+ """Yield responses in order with a few requests in flight; with missing_ok, None marks missing data."""
+ def get(address):
+ try:
+ return client.get(address, missing_ok=missing_ok)
+ except MissingImagery:
+ return None
+
+ # Keep requests sequential when the user asks for pauses between them.
+ if client.delay:
+ yield from map(get, addresses)
+ return
+ pending = deque()
+ with ThreadPoolExecutor(WORKERS) as pool:
+ try:
+ for address in addresses:
+ pending.append(pool.submit(get, address))
+ if len(pending) > 2 * WORKERS:
+ yield pending.popleft().result()
+ while pending:
+ yield pending.popleft().result()
+ finally:
+ for future in pending:
+ future.cancel()
+
+
class Progress:
def __init__(self):
self.stream = sys.stderr
diff --git a/src/mesh.py b/src/mesh.py
index 685eacd..352205a 100644
--- a/src/mesh.py
+++ b/src/mesh.py
@@ -14,49 +14,20 @@ import struct
import sys
import tempfile
from array import array
-from collections import deque
-from concurrent.futures import ThreadPoolExecutor
from itertools import accumulate
-from .common import MissingImagery, atomic_path
+from .common import atomic_path, fetch
from .pbf import fields, unpack
BASE = "https://kh.google.com/rt/earth/"
LEAF, NODATA, USE_IMAGERY_EPOCH = 4, 8, 16
JPEG = 1
-WORKERS = 16
LOWEST, HIGHEST = -1000, 9000 # Heights searched for nodes, in meters from the sphere.
GLB_LIMIT = 2**32
TILES_PER_MESH = 64 # Unreal's Nanite allows 64 materials per mesh; Godot allows 256 surfaces.
BITS = [bytes((byte >> i) & 1 for i in range(8)) for byte in range(256)]
-def fetch(client, addresses):
- """Yield responses in order with a few requests in flight; None marks missing data."""
- def get(address):
- try:
- return client.get(address, missing_ok=True)
- except MissingImagery:
- return None
-
- # Keep requests sequential when the user asks for pauses between them.
- if client.delay:
- yield from map(get, addresses)
- return
- pending = deque()
- with ThreadPoolExecutor(WORKERS) as pool:
- try:
- for address in addresses:
- pending.append(pool.submit(get, address))
- if len(pending) > 2 * WORKERS:
- yield pending.popleft().result()
- while pending:
- yield pending.popleft().result()
- finally:
- for future in pending:
- future.cancel()
-
-
def first(data):
"""Protobuf fields keeping the first value of repeated ones."""
values = {}
@@ -161,7 +132,7 @@ def plan(client, area, level, progress, *, allow_empty=False):
while pending:
batch, pending = pending, []
addresses = [f"{BASE}BulkMetadata/pb=!1m2!1s{path}!2u{epoch}" for path, epoch in batch]
- for (head, epoch), data in zip(batch, fetch(client, addresses)):
+ for (head, epoch), data in zip(batch, fetch(client, addresses, missing_ok=True)):
done += 1
progress("Finding 3D mesh nodes", done)
if data is None:
diff --git a/src/streetview.py b/src/streetview.py
index d359ae7..3a94463 100644
--- a/src/streetview.py
+++ b/src/streetview.py
@@ -10,7 +10,7 @@ from pathlib import Path
from PIL import Image
-from .common import MissingImagery, number, open_image, save_image, url, write_bytes
+from .common import MissingImagery, fetch, number, open_image, save_image, url, write_bytes
from .geo import Line, RoadIndex, clip, distance, grid, inside, tiles
EXCLUDED = re.compile(r"^(construction|proposed|planned|abandoned|razed|demolished)$")
@@ -67,10 +67,11 @@ def coverage(client, area, progress):
count = coverage_grid["rows"] * coverage_grid["columns"]
found = {}
progress("Finding panoramas", 0, count)
- for i, tile in enumerate(tiles(coverage_grid), 1):
- address = url("https://www.google.com/maps/photometa/ac/v1",
- pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")
- for view in parse_coverage(client.get(address)):
+ addresses = (url("https://www.google.com/maps/photometa/ac/v1",
+ pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")
+ for tile in tiles(coverage_grid))
+ for i, data in enumerate(fetch(client, addresses), 1):
+ for view in parse_coverage(data):
found[view["pano_id"]] = view
progress("Finding panoramas", i, count)
return list(found.values())
@@ -153,19 +154,21 @@ def save_sphere(client, metadata, zoom, target):
columns, rows = math.ceil(width / tw), math.ceil(height / th)
target = Path(target)
cache = target.parent / ".sphere-tiles" / metadata["pano_id"] / f"{zoom}-{width}x{height}-{tw}x{th}"
+ positions = [(x, y) for y in range(rows) for x in range(columns)]
+ missing = [(x, y) for x, y in positions if not (cache / f"{x}-{y}.tile").is_file()]
+ addresses = (url("https://streetviewpixels-pa.googleapis.com/v1/tile", cb_client="maps_sv.tactile",
+ panoid=metadata["pano_id"], zoom=zoom, x=x, y=y) for x, y in missing)
with Image.new("RGB", (width, height)) as sphere:
- for y in range(rows):
- for x in range(columns):
- client.check_cancel()
- path = cache / f"{x}-{y}.tile"
- cached = path.is_file()
- data = path.read_bytes() if cached else client.get(
- url("https://streetviewpixels-pa.googleapis.com/v1/tile", cb_client="maps_sv.tactile",
- panoid=metadata["pano_id"], zoom=zoom, x=x, y=y), missing_ok=True)
- with open_image(data, (tw, th)) as tile:
- if not cached:
- write_bytes(path, data)
- sphere.paste(tile, (x * tw, y * th))
+ for (x, y), data in zip(missing, fetch(client, addresses, missing_ok=True)):
+ if data is None:
+ raise MissingImagery("The requested image is not available.")
+ with open_image(data, (tw, th)) as tile:
+ write_bytes(cache / f"{x}-{y}.tile", data)
+ sphere.paste(tile, (x * tw, y * th))
+ for x, y in set(positions) - set(missing):
+ client.check_cancel()
+ with open_image((cache / f"{x}-{y}.tile").read_bytes(), (tw, th)) as tile:
+ sphere.paste(tile, (x * tw, y * th))
client.check_cancel()
save_image(sphere, target, quality=90)
shutil.rmtree(cache)