Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Add n workers params
Belluxx1726364e4789
diff --git a/aleph.py b/aleph.pyindex 4a8af8e..341a655 100644--- a/aleph.py+++ b/aleph.py@@ -102,14 +102,14 @@ def query(args, progress):if args.pano_id and args.radius is not None:raise ValueError("--radius applies to coordinates or a place name.")keys += ("pano_id", "street", "route", "reverse", "stops", "step", "view", "heading",- "look_at", "pitch", "fov", "radius", "streetview_format", "full_sphere", "sphere_zoom")+ "look_at", "pitch", "fov", "radius", "streetview_format", "full_sphere", "sphere_zoom", "workers")operation = quick.street_photoselse:if args.size is not None and (args.bbox is not None or args.tile is not None):raise ValueError("--size applies to coordinates or a place name.")if args.tile is not None and args.zoom is not None:raise ValueError("--tile already includes its zoom; omit --zoom.")- keys += ("bbox", "tile", "size", "zoom", "satellite_format")+ keys += ("bbox", "tile", "size", "zoom", "satellite_format", "workers")operation = quick.satellitevalues = {key: getattr(args, key) for key in keys if getattr(args, key) is not None}result = operation(client, args.output, progress, endpoint=args.geocoder, **values)diff --git a/src/assets/app.js b/src/assets/app.jsindex 72030fe..b42c40a 100644--- a/src/assets/app.js+++ b/src/assets/app.js@@ -948,7 +948,8 @@ async function planCapture(event) {const options = {include: data.getAll("include"), depth: data.get("depth"),full_sphere: data.get("full_sphere") === "true",streetview_format: data.get("streetview_format"), satellite_format: data.get("satellite_format")};- for (const key of ["step", "fov", "delay", "satellite_zoom", "terrain_zoom", "sphere_zoom", "mesh_level"]) options[key] = Number(data.get(key));+ for (const key of ["step", "fov", "delay", "satellite_zoom", "terrain_zoom", "sphere_zoom", "mesh_level",+ "streetview_workers", "satellite_workers", "terrain_workers", "mesh_workers"]) options[key] = Number(data.get(key));const bounds = [...state.draftBounds];state.submitting = true;setDrawMode(false);diff --git a/src/assets/index.html b/src/assets/index.htmlindex 9aaa389..7bdb933 100644--- a/src/assets/index.html+++ b/src/assets/index.html@@ -136,6 +136,7 @@<h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="m12 3 9 5-9 5-9-5 9-5Zm-9 9 9 5 9-5M3 16l9 5 9-5"/></svg>Satellite</h3><div class="settings-grid"><label>Tile format<select name="satellite_format"><option value="jpg">JPEG</option><option value="png">PNG</option></select></label>+ <label>Parallel requests<input name="satellite_workers" type="number" min="1" max="64" step="1" required></label></div></div><div class="settings-group" data-source="streetview">@@ -143,11 +144,19 @@<div class="settings-grid"><label>Road coverage<select name="depth"><option value="main">Main roads</option><option value="roads">All roads</option><option value="all">Roads + paths</option></select></label><label>Photo format<select name="streetview_format"><option value="jpg">JPEG</option><option value="png">PNG</option></select></label>+ <label>Parallel requests<input name="streetview_workers" type="number" min="1" max="64" step="1" required></label></div></div><div class="settings-group" data-source="osm"><h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="m2 20 7-14 4 7 3-5 6 12H2Zm4-8 3 2 3-3M13 13l3 3 3-2"/></svg>Map & terrain</h3>- <div class="settings-grid"><label>Terrain zoom<input name="terrain_zoom" type="number" min="1" max="14" required></label></div>+ <div class="settings-grid">+ <label>Terrain zoom<input name="terrain_zoom" type="number" min="1" max="14" required></label>+ <label>Parallel requests<input name="terrain_workers" type="number" min="1" max="64" step="1" required></label>+ </div>+ </div>+ <div class="settings-group" data-source="mesh">+ <h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="m12 2 9 5v10l-9 5-9-5V7l9-5Zm0 10 9-5M12 12 3 7m9 5v10"/></svg>3D mesh</h3>+ <div class="settings-grid"><label>Parallel requests<input name="mesh_workers" type="number" min="1" max="64" step="1" required></label></div></div><div class="settings-group"><h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="M4 4v16M12 4v16M20 4v16M1 8h6m2 8h6m2-6h6"/></svg>General</h3>diff --git a/src/capture.py b/src/capture.pyindex 1ad7fbe..a9c49a9 100644--- a/src/capture.py+++ b/src/capture.py@@ -11,7 +11,7 @@ from pathlib import Pathfrom PIL import Imagefrom . import mesh, satellite, streetview, terrain-from .common import MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_json+from .common import WORKERS, MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_jsonfrom .geo import bounds, collection, coordinate, feature, grid, pixel, tile_ring, tilesSOURCES = ("streetview", "satellite", "osm", "mesh")@@ -20,11 +20,15 @@ DEFAULT_OPTIONS = dict(include=["streetview", "satellite", "osm"], step=30, fov=75, depth="roads", streetview_format="jpg",full_sphere=False, sphere_zoom=3,delay=0, satellite_zoom=18, satellite_format="jpg", terrain_zoom=14, mesh_level=21,+ streetview_workers=WORKERS, satellite_workers=WORKERS, terrain_workers=WORKERS, mesh_workers=WORKERS,)-# Average connection speed / hardware timings, just to give an idea to the user, not precise-REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.02, satellite_tile=0.02, terrain_tile=0.13,- mesh_node=0.021)+# Average sequential request timings, just to give an idea to the user, not precise+REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.27, satellite_tile=0.25, terrain_tile=1.39,+ mesh_node=0.34)+# Options choosing how many requests of each kind run in parallel.+REQUEST_WORKERS = dict(sphere_tile="streetview_workers", satellite_tile="satellite_workers",+ terrain_tile="terrain_workers", mesh_node="mesh_workers")SPHERE_TILES = (1, 2, 8, 32, 128, 512)OSM_EXPORT_SECONDS = 10@@ -77,7 +81,8 @@ def settings(values=None):positive(options["step"], "step")number(options["delay"], "delay", 0)for key, low, high in (("fov", 5, 175), ("sphere_zoom", 0, 5), ("satellite_zoom", 1, 21), ("terrain_zoom", 1, 14),- ("mesh_level", 1, 22)):+ ("mesh_level", 1, 22), ("streetview_workers", 1, 64), ("satellite_workers", 1, 64),+ ("terrain_workers", 1, 64), ("mesh_workers", 1, 64)):options[key] = number(options[key], key, low, high, whole=True)if type(options["full_sphere"]) is not bool:raise ValueError("full_sphere must be true or false.")@@ -115,7 +120,8 @@ def plan(client, area, options, progress):stages.append(dict(mode="osm", results=[], notes=OSM_NOTES))stages.append(dict(mode="terrain", grid=grid(area, options["terrain_zoom"]), results=[], notes=TERRAIN_NOTES))if "mesh" in options["include"]:- stage = mesh.plan(client, area, options["mesh_level"], progress, allow_empty=len(options["include"]) > 1)+ stage = mesh.plan(client, area, options["mesh_level"], options["mesh_workers"], progress,+ allow_empty=len(options["include"]) > 1)stages.append(dict(stage, notes=MESH_NOTES))return dict(format="aleph-python", version=FORMAT_VERSION, bounds=area, options=options,started_at=now(), state="planned", stages=stages)@@ -158,9 +164,15 @@ def estimate(run):counts["mesh_nodes"] = requests["mesh_node"] = remainingelse:counts["terrain_tiles"] = requests["terrain_tile"] = remaining- seconds = sum(count * REQUEST_SECONDS[kind] for kind, count in requests.items())+ options = run["options"]++ def workers(kind):+ # Requests run one at a time when the user asks for pauses between them.+ return 1 if options["delay"] or kind not in REQUEST_WORKERS else options[REQUEST_WORKERS[kind]]++ seconds = sum(count * REQUEST_SECONDS[kind] / workers(kind) for kind, count in requests.items())seconds += counts["osm_maps"] * OSM_EXPORT_SECONDS- seconds += max(0, sum(requests.values()) - 1) * run["options"]["delay"]+ seconds += max(0, sum(requests.values()) - 1) * options["delay"]return dict(counts, seconds=math.ceil(seconds))@@ -191,7 +203,8 @@ def capture_streetview(stage, run, folder, client, progress):if sphere:del photo["heading"] # A sphere has no single perspective camera heading.photo["filename"] = f"streetview/photos/{index + 1:06d}_sphere.{options['streetview_format']}"- photo.update(streetview.save_sphere(client, metadata, options["sphere_zoom"], folder / photo["filename"]))+ photo.update(streetview.save_sphere(client, metadata, options["sphere_zoom"], folder / photo["filename"],+ options["streetview_workers"]))else:side = "left" if index % 2 == 0 else "right"photo.update(side=side, heading=(sample["heading"] + streetview.VIEW_OFFSETS[side]) % 360,@@ -212,7 +225,8 @@ def capture_satellite(stage, run, folder, client, progress):progress("Satellite", done, total(stage))remaining = list(tiles(stage["grid"]))[done:]addresses = [f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}" for tile in remaining]- for tile, address, data in zip(remaining, addresses, fetch(client, addresses, missing_ok=True)):+ downloads = fetch(client, addresses, run["options"]["satellite_workers"], missing_ok=True)+ for tile, address, data in zip(remaining, addresses, downloads):extension = run["options"]["satellite_format"]if data is None:# Persist the gap so resume and offline export keep it transparent.@@ -248,7 +262,7 @@ def capture_terrain(stage, run, folder, client, progress):remaining = list(tiles(stage["grid"]))[done:]addresses = [f"https://elevation-tiles-prod.s3.amazonaws.com/geotiff/{tile['zoom']}/{tile['x']}/{tile['y']}.tif"for tile in remaining]- for tile, address, data in zip(remaining, addresses, fetch(client, addresses)):+ for tile, address, data in zip(remaining, addresses, fetch(client, addresses, run["options"]["terrain_workers"])):terrain.validate(terrain.TIFF(BytesIO(data)), tile)name = f"terrain/tiles/terrain_z{tile['zoom']}_x{tile['x']}_y{tile['y']}.tif"write_bytes(folder / name, data)@@ -262,7 +276,7 @@ def capture_mesh(stage, run, folder, client, progress):done = len(stage["results"])progress("3D mesh", done, total(stage))nodes = stage["nodes"][done:]- for node, data in zip(nodes, fetch(client, map(mesh.address, nodes), missing_ok=True)):+ for node, data in zip(nodes, fetch(client, map(mesh.address, nodes), run["options"]["mesh_workers"], missing_ok=True)):if data is None:stage["results"].append(dict(status="skipped", reason="The 3D mesh node is not available."))else:diff --git a/src/cli.py b/src/cli.pyindex 73172fd..d5e29d1 100644--- a/src/cli.py+++ b/src/cli.py@@ -5,7 +5,7 @@ import osfrom pathlib import Pathfrom . import capture, places-from .common import DEFAULT_CACHE, RequestError+from .common import DEFAULT_CACHE, WORKERS, RequestErrorclass Parser(argparse.ArgumentParser):@@ -54,6 +54,10 @@ def _network_options(command):_cache_options(command)+def _workers_option(command, flag, requests):+ command.add_argument(flag, type=int, metavar="N", help=f"{requests}, 1–64 (default: {WORKERS}); --delay makes them sequential")++def _sphere_options(command):command.add_argument("--full-sphere", action="store_true", help="save one full 360° panorama per stop; camera direction and fov do not apply")command.add_argument("--sphere-zoom", type=int, choices=range(6), default=3,@@ -113,6 +117,7 @@ def _streetview_command(commands):_sphere_options(street)_network_options(street)+ _workers_option(street, "--workers", "parallel coverage and sphere tile requests")street.add_argument("--streetview-format", choices=("jpg", "png"), default="jpg")_output_options(street)@@ -128,6 +133,7 @@ def _satellite_command(commands):satellite.add_argument("--zoom", type=int, help="satellite zoom, 1–21 (default: 19)")_network_options(satellite)+ _workers_option(satellite, "--workers", "parallel tile requests")satellite.add_argument("--tile-format", dest="satellite_format", choices=("jpg", "png"), default="jpg", help="saved tile format; merged PNG and COG are always produced (default: jpg)")_output_options(satellite)@@ -164,6 +170,10 @@ def _capture_command(commands):# Requests, cache, and output shared by all capture sources.area.add_argument("--delay", type=float, help="pause between requests in seconds (default: 0)")+ _workers_option(area, "--streetview-workers", "parallel Street View coverage and sphere tile requests")+ _workers_option(area, "--satellite-workers", "parallel satellite tile requests")+ _workers_option(area, "--terrain-workers", "parallel terrain tile requests")+ _workers_option(area, "--mesh-workers", "parallel 3D mesh requests")_cache_options(area)_output_options(area)area.set_defaults(**capture.DEFAULT_OPTIONS)diff --git a/src/common.py b/src/common.pyindex 780c6b3..9b8649c 100644--- a/src/common.py+++ b/src/common.py@@ -187,8 +187,8 @@ class CachedClient(Client):return result-def fetch(client, addresses, *, missing_ok=False):- """Yield responses in order with a few requests in flight; with missing_ok, None marks missing data."""+def fetch(client, addresses, workers, *, missing_ok=False):+ """Yield responses in order with up to workers requests in flight; with missing_ok, None marks missing data."""def get(address):try:return client.get(address, missing_ok=missing_ok)@@ -196,15 +196,15 @@ def fetch(client, addresses, *, missing_ok=False):return None# Keep requests sequential when the user asks for pauses between them.- if client.delay:+ if client.delay or workers == 1:yield from map(get, addresses)returnpending = deque()- with ThreadPoolExecutor(WORKERS) as pool:+ with ThreadPoolExecutor(workers) as pool:try:for address in addresses:pending.append(pool.submit(get, address))- if len(pending) > 2 * WORKERS:+ if len(pending) > 2 * workers:yield pending.popleft().result()while pending:yield pending.popleft().result()diff --git a/src/mesh.py b/src/mesh.pyindex 352205a..095858f 100644--- a/src/mesh.py+++ b/src/mesh.py@@ -120,7 +120,7 @@ def bulk(data, head, epoch):yield path, packed, node.get(2, epoch), node.get(5, epoch), node.get(7, imagery), box-def plan(client, area, level, progress, *, allow_empty=False):+def plan(client, area, level, workers, progress, *, allow_empty=False):"""Nodes with data meeting the rectangle down to level, and the octants their children refine."""planet = dict(fields(client.get(BASE + "PlanetoidMetadata")))root = dict(fields(planet[1]))@@ -132,7 +132,7 @@ def plan(client, area, level, progress, *, allow_empty=False):while pending:batch, pending = pending, []addresses = [f"{BASE}BulkMetadata/pb=!1m2!1s{path}!2u{epoch}" for path, epoch in batch]- for (head, epoch), data in zip(batch, fetch(client, addresses, missing_ok=True)):+ for (head, epoch), data in zip(batch, fetch(client, addresses, workers, missing_ok=True)):done += 1progress("Finding 3D mesh nodes", done)if data is None:diff --git a/src/quick.py b/src/quick.pyindex e9595a6..27fe804 100644--- a/src/quick.py+++ b/src/quick.py@@ -4,12 +4,12 @@ import tempfilefrom pathlib import Pathfrom . import capture, places, routes, streetview-from .common import MissingImagery, RequestError, now, number, positive, write_json+from .common import WORKERS, MissingImagery, RequestError, now, number, positive, write_jsonfrom .geo import around, bearing, bounds, coordinate, distance, extent, point-def coverage(client, area, progress):+def coverage(client, area, progress, workers):try:- return streetview.coverage(client, area, progress)+ return streetview.coverage(client, area, progress, workers)except (OSError, ValueError) as error:raise RequestError("provider_unavailable", f"Street View coverage request failed: {error}") from error@@ -21,12 +21,12 @@ def location(client, *, at=None, place=None, match=None, best_match=False, endporeturn (selected["lat"], selected["lon"]), selected-def street_samples(client, sections, stops, step, progress):+def street_samples(client, sections, stops, step, progress, workers):"""Match panoramas to target positions along every selected street section."""south, west, north, east = extent([p for section in sections for p in section["points"]])lower, upper = around((south, west), 80), around((north, east), 80)area = bounds((lower[0], lower[1], upper[2], upper[3]))- views = coverage(client, area, progress)+ views = coverage(client, area, progress, workers)ways, _ = client.maps.data(area, progress=progress)context = streetview.roads(ways, area, "all")samples, gaps, requested = [], [], 0@@ -47,8 +47,9 @@ def street_samples(client, sections, stops, step, progress):def street_photos(client, output, progress, *, at=None, place=None, pano_id=None, street=None, match=None,best_match=False, endpoint=places.GEOCODER, route=None, reverse=False, stops=None, step=None,view="forward", heading=0, look_at=None, pitch=0, fov=75, radius=50, streetview_format="jpg",- full_sphere=False, sphere_zoom=3):+ full_sphere=False, sphere_zoom=3, workers=WORKERS):number(sphere_zoom, "sphere_zoom", 0, 5, whole=True)+ number(workers, "workers", 1, 64, whole=True)streetview.angle(heading, "heading")streetview.angle(pitch, "pitch")fov = number(fov, "fov", 5, 175, whole=True)@@ -64,7 +65,7 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=Noneif street is not None:selected_place = places.choose(client, street, match=match, best_match=best_match, street=True, endpoint=endpoint)sections = routes.resolve(client, selected_place, route=route, reverse=reverse, progress=progress)- samples, gaps, requested_stops = street_samples(client, sections, stops, step, progress)+ samples, gaps, requested_stops = street_samples(client, sections, stops, step, progress, workers)elif pano_id:if not streetview.PANO_ID.fullmatch(pano_id):raise ValueError("Invalid panorama ID.")@@ -72,7 +73,7 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=Noneelse:requested, selected_place = location(client, at=at, place=place, match=match,best_match=best_match, endpoint=endpoint)- views = coverage(client, around(requested, radius * 2), progress)+ views = coverage(client, around(requested, radius * 2), progress, workers)nearest = min(views, key=lambda v: (distance(requested, (v["lat"], v["lon"])), v["pano_id"]), default=None)samples = []if nearest and distance(requested, (nearest["lat"], nearest["lon"])) <= radius:@@ -102,7 +103,7 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=Nonephoto = {**sample, **metadata}if "heading" in photo:photo["road_heading"] = photo.pop("heading")- photo.update(streetview.save_sphere(client, metadata, sphere_zoom, path), path=str(path))+ photo.update(streetview.save_sphere(client, metadata, sphere_zoom, path, workers), path=str(path))result["photos"].append(photo)else:for direction in directions if street else (None,):@@ -140,7 +141,8 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=Nonedef satellite(client, output, progress, *, at=None, place=None, match=None, best_match=False,- endpoint=places.GEOCODER, bbox=None, tile=None, size=200, zoom=19, satellite_format="jpg"):+ endpoint=places.GEOCODER, bbox=None, tile=None, size=200, zoom=19, satellite_format="jpg",+ workers=WORKERS):if tile is not None:zoom, x, y = tilenumber(zoom, "zoom", 1, 21, whole=True)@@ -158,7 +160,7 @@ def satellite(client, output, progress, *, at=None, place=None, match=None, bestbest_match=best_match, endpoint=endpoint)area = around(center, size)run = capture.plan(client, area, dict(include=["satellite"], satellite_zoom=zoom,- satellite_format=satellite_format), progress)+ satellite_format=satellite_format, satellite_workers=workers), progress)folder = capture.create_folder(run, output)try:capture.download(run, folder, client, progress)diff --git a/src/streetview.py b/src/streetview.pyindex 3a94463..94ba885 100644--- a/src/streetview.py+++ b/src/streetview.py@@ -61,7 +61,7 @@ def parse_coverage(data):raise ValueError("Unreadable Google panorama coverage; the endpoint may have changed.") from error-def coverage(client, area, progress):+def coverage(client, area, progress, workers):"""Every Google panorama listed in the zoom-17 tiles that cover the area."""coverage_grid = grid(area, 17)count = coverage_grid["rows"] * coverage_grid["columns"]@@ -70,7 +70,7 @@ def coverage(client, area, progress):addresses = (url("https://www.google.com/maps/photometa/ac/v1",pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")for tile in tiles(coverage_grid))- for i, data in enumerate(fetch(client, addresses), 1):+ for i, data in enumerate(fetch(client, addresses, workers), 1):for view in parse_coverage(data):found[view["pano_id"]] = viewprogress("Finding panoramas", i, count)@@ -146,7 +146,7 @@ def save_photo(client, pano_id, heading, fov, pitch, path):return dict(width=PHOTO_SIZE[0], height=PHOTO_SIZE[1], source_url=address)-def save_sphere(client, metadata, zoom, target):+def save_sphere(client, metadata, zoom, target, workers):"""Assemble native panorama tiles; retain partial tiles for retry/resume."""zoom = min(zoom, len(metadata["image_sizes"]) - 1)width, height = metadata["image_sizes"][zoom]@@ -159,7 +159,7 @@ def save_sphere(client, metadata, zoom, target):addresses = (url("https://streetviewpixels-pa.googleapis.com/v1/tile", cb_client="maps_sv.tactile",panoid=metadata["pano_id"], zoom=zoom, x=x, y=y) for x, y in missing)with Image.new("RGB", (width, height)) as sphere:- for (x, y), data in zip(missing, fetch(client, addresses, missing_ok=True)):+ for (x, y), data in zip(missing, fetch(client, addresses, workers, missing_ok=True)):if data is None:raise MissingImagery("The requested image is not available.")with open_image(data, (tw, th)) as tile:@@ -284,7 +284,8 @@ def plan(client, area, options, progress, *, allow_empty=False):parts = roads(ways, area, options["depth"])if not parts and not allow_empty:raise ValueError("No mapped roads at this path depth. Choose another area or depth.")- found = [view for view in coverage(client, area, progress) if inside((view["lat"], view["lon"]), area)]+ found = [view for view in coverage(client, area, progress, options["streetview_workers"])+ if inside((view["lat"], view["lon"]), area)]progress("Indexing roads")lines = [Line(part["points"]) for part in parts]index = RoadIndex(lines)