Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Add n workers params

Belluxx1726364e4789
diff --git a/aleph.py b/aleph.py
index 4a8af8e..341a655 100644
--- a/aleph.py
+++ b/aleph.py
@@ -102,14 +102,14 @@ def query(args, progress):
if args.pano_id and args.radius is not None:
raise ValueError("--radius applies to coordinates or a place name.")
keys += ("pano_id", "street", "route", "reverse", "stops", "step", "view", "heading",
- "look_at", "pitch", "fov", "radius", "streetview_format", "full_sphere", "sphere_zoom")
+ "look_at", "pitch", "fov", "radius", "streetview_format", "full_sphere", "sphere_zoom", "workers")
operation = quick.street_photos
else:
if args.size is not None and (args.bbox is not None or args.tile is not None):
raise ValueError("--size applies to coordinates or a place name.")
if args.tile is not None and args.zoom is not None:
raise ValueError("--tile already includes its zoom; omit --zoom.")
- keys += ("bbox", "tile", "size", "zoom", "satellite_format")
+ keys += ("bbox", "tile", "size", "zoom", "satellite_format", "workers")
operation = quick.satellite
values = {key: getattr(args, key) for key in keys if getattr(args, key) is not None}
result = operation(client, args.output, progress, endpoint=args.geocoder, **values)
diff --git a/src/assets/app.js b/src/assets/app.js
index 72030fe..b42c40a 100644
--- a/src/assets/app.js
+++ b/src/assets/app.js
@@ -948,7 +948,8 @@ async function planCapture(event) {
const options = {include: data.getAll("include"), depth: data.get("depth"),
full_sphere: data.get("full_sphere") === "true",
streetview_format: data.get("streetview_format"), satellite_format: data.get("satellite_format")};
- for (const key of ["step", "fov", "delay", "satellite_zoom", "terrain_zoom", "sphere_zoom", "mesh_level"]) options[key] = Number(data.get(key));
+ for (const key of ["step", "fov", "delay", "satellite_zoom", "terrain_zoom", "sphere_zoom", "mesh_level",
+ "streetview_workers", "satellite_workers", "terrain_workers", "mesh_workers"]) options[key] = Number(data.get(key));
const bounds = [...state.draftBounds];
state.submitting = true;
setDrawMode(false);
diff --git a/src/assets/index.html b/src/assets/index.html
index 9aaa389..7bdb933 100644
--- a/src/assets/index.html
+++ b/src/assets/index.html
@@ -136,6 +136,7 @@
<h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="m12 3 9 5-9 5-9-5 9-5Zm-9 9 9 5 9-5M3 16l9 5 9-5"/></svg>Satellite</h3>
<div class="settings-grid">
<label>Tile format<select name="satellite_format"><option value="jpg">JPEG</option><option value="png">PNG</option></select></label>
+ <label>Parallel requests<input name="satellite_workers" type="number" min="1" max="64" step="1" required></label>
</div>
</div>
<div class="settings-group" data-source="streetview">
@@ -143,11 +144,19 @@
<div class="settings-grid">
<label>Road coverage<select name="depth"><option value="main">Main roads</option><option value="roads">All roads</option><option value="all">Roads + paths</option></select></label>
<label>Photo format<select name="streetview_format"><option value="jpg">JPEG</option><option value="png">PNG</option></select></label>
+ <label>Parallel requests<input name="streetview_workers" type="number" min="1" max="64" step="1" required></label>
</div>
</div>
<div class="settings-group" data-source="osm">
<h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="m2 20 7-14 4 7 3-5 6 12H2Zm4-8 3 2 3-3M13 13l3 3 3-2"/></svg>Map & terrain</h3>
- <div class="settings-grid"><label>Terrain zoom<input name="terrain_zoom" type="number" min="1" max="14" required></label></div>
+ <div class="settings-grid">
+ <label>Terrain zoom<input name="terrain_zoom" type="number" min="1" max="14" required></label>
+ <label>Parallel requests<input name="terrain_workers" type="number" min="1" max="64" step="1" required></label>
+ </div>
+ </div>
+ <div class="settings-group" data-source="mesh">
+ <h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="m12 2 9 5v10l-9 5-9-5V7l9-5Zm0 10 9-5M12 12 3 7m9 5v10"/></svg>3D mesh</h3>
+ <div class="settings-grid"><label>Parallel requests<input name="mesh_workers" type="number" min="1" max="64" step="1" required></label></div>
</div>
<div class="settings-group">
<h3 class="settings-divider"><svg viewBox="0 0 24 24" fill="none" aria-hidden="true"><path d="M4 4v16M12 4v16M20 4v16M1 8h6m2 8h6m2-6h6"/></svg>General</h3>
diff --git a/src/capture.py b/src/capture.py
index 1ad7fbe..a9c49a9 100644
--- a/src/capture.py
+++ b/src/capture.py
@@ -11,7 +11,7 @@ from pathlib import Path
from PIL import Image
from . import mesh, satellite, streetview, terrain
-from .common import MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_json
+from .common import WORKERS, MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_json
from .geo import bounds, collection, coordinate, feature, grid, pixel, tile_ring, tiles
SOURCES = ("streetview", "satellite", "osm", "mesh")
@@ -20,11 +20,15 @@ DEFAULT_OPTIONS = dict(
include=["streetview", "satellite", "osm"], step=30, fov=75, depth="roads", streetview_format="jpg",
full_sphere=False, sphere_zoom=3,
delay=0, satellite_zoom=18, satellite_format="jpg", terrain_zoom=14, mesh_level=21,
+ streetview_workers=WORKERS, satellite_workers=WORKERS, terrain_workers=WORKERS, mesh_workers=WORKERS,
)
-# Average connection speed / hardware timings, just to give an idea to the user, not precise
-REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.02, satellite_tile=0.02, terrain_tile=0.13,
- mesh_node=0.021)
+# Average sequential request timings, just to give an idea to the user, not precise
+REQUEST_SECONDS = dict(metadata=0.17, photo=0.46, sphere_tile=0.27, satellite_tile=0.25, terrain_tile=1.39,
+ mesh_node=0.34)
+# Options choosing how many requests of each kind run in parallel.
+REQUEST_WORKERS = dict(sphere_tile="streetview_workers", satellite_tile="satellite_workers",
+ terrain_tile="terrain_workers", mesh_node="mesh_workers")
SPHERE_TILES = (1, 2, 8, 32, 128, 512)
OSM_EXPORT_SECONDS = 10
@@ -77,7 +81,8 @@ def settings(values=None):
positive(options["step"], "step")
number(options["delay"], "delay", 0)
for key, low, high in (("fov", 5, 175), ("sphere_zoom", 0, 5), ("satellite_zoom", 1, 21), ("terrain_zoom", 1, 14),
- ("mesh_level", 1, 22)):
+ ("mesh_level", 1, 22), ("streetview_workers", 1, 64), ("satellite_workers", 1, 64),
+ ("terrain_workers", 1, 64), ("mesh_workers", 1, 64)):
options[key] = number(options[key], key, low, high, whole=True)
if type(options["full_sphere"]) is not bool:
raise ValueError("full_sphere must be true or false.")
@@ -115,7 +120,8 @@ def plan(client, area, options, progress):
stages.append(dict(mode="osm", results=[], notes=OSM_NOTES))
stages.append(dict(mode="terrain", grid=grid(area, options["terrain_zoom"]), results=[], notes=TERRAIN_NOTES))
if "mesh" in options["include"]:
- stage = mesh.plan(client, area, options["mesh_level"], progress, allow_empty=len(options["include"]) > 1)
+ stage = mesh.plan(client, area, options["mesh_level"], options["mesh_workers"], progress,
+ allow_empty=len(options["include"]) > 1)
stages.append(dict(stage, notes=MESH_NOTES))
return dict(format="aleph-python", version=FORMAT_VERSION, bounds=area, options=options,
started_at=now(), state="planned", stages=stages)
@@ -158,9 +164,15 @@ def estimate(run):
counts["mesh_nodes"] = requests["mesh_node"] = remaining
else:
counts["terrain_tiles"] = requests["terrain_tile"] = remaining
- seconds = sum(count * REQUEST_SECONDS[kind] for kind, count in requests.items())
+ options = run["options"]
+
+ def workers(kind):
+ # Requests run one at a time when the user asks for pauses between them.
+ return 1 if options["delay"] or kind not in REQUEST_WORKERS else options[REQUEST_WORKERS[kind]]
+
+ seconds = sum(count * REQUEST_SECONDS[kind] / workers(kind) for kind, count in requests.items())
seconds += counts["osm_maps"] * OSM_EXPORT_SECONDS
- seconds += max(0, sum(requests.values()) - 1) * run["options"]["delay"]
+ seconds += max(0, sum(requests.values()) - 1) * options["delay"]
return dict(counts, seconds=math.ceil(seconds))
@@ -191,7 +203,8 @@ def capture_streetview(stage, run, folder, client, progress):
if sphere:
del photo["heading"] # A sphere has no single perspective camera heading.
photo["filename"] = f"streetview/photos/{index + 1:06d}_sphere.{options['streetview_format']}"
- photo.update(streetview.save_sphere(client, metadata, options["sphere_zoom"], folder / photo["filename"]))
+ photo.update(streetview.save_sphere(client, metadata, options["sphere_zoom"], folder / photo["filename"],
+ options["streetview_workers"]))
else:
side = "left" if index % 2 == 0 else "right"
photo.update(side=side, heading=(sample["heading"] + streetview.VIEW_OFFSETS[side]) % 360,
@@ -212,7 +225,8 @@ def capture_satellite(stage, run, folder, client, progress):
progress("Satellite", done, total(stage))
remaining = list(tiles(stage["grid"]))[done:]
addresses = [f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}" for tile in remaining]
- for tile, address, data in zip(remaining, addresses, fetch(client, addresses, missing_ok=True)):
+ downloads = fetch(client, addresses, run["options"]["satellite_workers"], missing_ok=True)
+ for tile, address, data in zip(remaining, addresses, downloads):
extension = run["options"]["satellite_format"]
if data is None:
# Persist the gap so resume and offline export keep it transparent.
@@ -248,7 +262,7 @@ def capture_terrain(stage, run, folder, client, progress):
remaining = list(tiles(stage["grid"]))[done:]
addresses = [f"https://elevation-tiles-prod.s3.amazonaws.com/geotiff/{tile['zoom']}/{tile['x']}/{tile['y']}.tif"
for tile in remaining]
- for tile, address, data in zip(remaining, addresses, fetch(client, addresses)):
+ for tile, address, data in zip(remaining, addresses, fetch(client, addresses, run["options"]["terrain_workers"])):
terrain.validate(terrain.TIFF(BytesIO(data)), tile)
name = f"terrain/tiles/terrain_z{tile['zoom']}_x{tile['x']}_y{tile['y']}.tif"
write_bytes(folder / name, data)
@@ -262,7 +276,7 @@ def capture_mesh(stage, run, folder, client, progress):
done = len(stage["results"])
progress("3D mesh", done, total(stage))
nodes = stage["nodes"][done:]
- for node, data in zip(nodes, fetch(client, map(mesh.address, nodes), missing_ok=True)):
+ for node, data in zip(nodes, fetch(client, map(mesh.address, nodes), run["options"]["mesh_workers"], missing_ok=True)):
if data is None:
stage["results"].append(dict(status="skipped", reason="The 3D mesh node is not available."))
else:
diff --git a/src/cli.py b/src/cli.py
index 73172fd..d5e29d1 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -5,7 +5,7 @@ import os
from pathlib import Path
from . import capture, places
-from .common import DEFAULT_CACHE, RequestError
+from .common import DEFAULT_CACHE, WORKERS, RequestError
class Parser(argparse.ArgumentParser):
@@ -54,6 +54,10 @@ def _network_options(command):
_cache_options(command)
+def _workers_option(command, flag, requests):
+ command.add_argument(flag, type=int, metavar="N", help=f"{requests}, 1–64 (default: {WORKERS}); --delay makes them sequential")
+
+
def _sphere_options(command):
command.add_argument("--full-sphere", action="store_true", help="save one full 360° panorama per stop; camera direction and fov do not apply")
command.add_argument("--sphere-zoom", type=int, choices=range(6), default=3,
@@ -113,6 +117,7 @@ def _streetview_command(commands):
_sphere_options(street)
_network_options(street)
+ _workers_option(street, "--workers", "parallel coverage and sphere tile requests")
street.add_argument("--streetview-format", choices=("jpg", "png"), default="jpg")
_output_options(street)
@@ -128,6 +133,7 @@ def _satellite_command(commands):
satellite.add_argument("--zoom", type=int, help="satellite zoom, 1–21 (default: 19)")
_network_options(satellite)
+ _workers_option(satellite, "--workers", "parallel tile requests")
satellite.add_argument("--tile-format", dest="satellite_format", choices=("jpg", "png"), default="jpg", help="saved tile format; merged PNG and COG are always produced (default: jpg)")
_output_options(satellite)
@@ -164,6 +170,10 @@ def _capture_command(commands):
# Requests, cache, and output shared by all capture sources.
area.add_argument("--delay", type=float, help="pause between requests in seconds (default: 0)")
+ _workers_option(area, "--streetview-workers", "parallel Street View coverage and sphere tile requests")
+ _workers_option(area, "--satellite-workers", "parallel satellite tile requests")
+ _workers_option(area, "--terrain-workers", "parallel terrain tile requests")
+ _workers_option(area, "--mesh-workers", "parallel 3D mesh requests")
_cache_options(area)
_output_options(area)
area.set_defaults(**capture.DEFAULT_OPTIONS)
diff --git a/src/common.py b/src/common.py
index 780c6b3..9b8649c 100644
--- a/src/common.py
+++ b/src/common.py
@@ -187,8 +187,8 @@ class CachedClient(Client):
return result
-def fetch(client, addresses, *, missing_ok=False):
- """Yield responses in order with a few requests in flight; with missing_ok, None marks missing data."""
+def fetch(client, addresses, workers, *, missing_ok=False):
+ """Yield responses in order with up to workers requests in flight; with missing_ok, None marks missing data."""
def get(address):
try:
return client.get(address, missing_ok=missing_ok)
@@ -196,15 +196,15 @@ def fetch(client, addresses, *, missing_ok=False):
return None
# Keep requests sequential when the user asks for pauses between them.
- if client.delay:
+ if client.delay or workers == 1:
yield from map(get, addresses)
return
pending = deque()
- with ThreadPoolExecutor(WORKERS) as pool:
+ with ThreadPoolExecutor(workers) as pool:
try:
for address in addresses:
pending.append(pool.submit(get, address))
- if len(pending) > 2 * WORKERS:
+ if len(pending) > 2 * workers:
yield pending.popleft().result()
while pending:
yield pending.popleft().result()
diff --git a/src/mesh.py b/src/mesh.py
index 352205a..095858f 100644
--- a/src/mesh.py
+++ b/src/mesh.py
@@ -120,7 +120,7 @@ def bulk(data, head, epoch):
yield path, packed, node.get(2, epoch), node.get(5, epoch), node.get(7, imagery), box
-def plan(client, area, level, progress, *, allow_empty=False):
+def plan(client, area, level, workers, progress, *, allow_empty=False):
"""Nodes with data meeting the rectangle down to level, and the octants their children refine."""
planet = dict(fields(client.get(BASE + "PlanetoidMetadata")))
root = dict(fields(planet[1]))
@@ -132,7 +132,7 @@ def plan(client, area, level, progress, *, allow_empty=False):
while pending:
batch, pending = pending, []
addresses = [f"{BASE}BulkMetadata/pb=!1m2!1s{path}!2u{epoch}" for path, epoch in batch]
- for (head, epoch), data in zip(batch, fetch(client, addresses, missing_ok=True)):
+ for (head, epoch), data in zip(batch, fetch(client, addresses, workers, missing_ok=True)):
done += 1
progress("Finding 3D mesh nodes", done)
if data is None:
diff --git a/src/quick.py b/src/quick.py
index e9595a6..27fe804 100644
--- a/src/quick.py
+++ b/src/quick.py
@@ -4,12 +4,12 @@ import tempfile
from pathlib import Path
from . import capture, places, routes, streetview
-from .common import MissingImagery, RequestError, now, number, positive, write_json
+from .common import WORKERS, MissingImagery, RequestError, now, number, positive, write_json
from .geo import around, bearing, bounds, coordinate, distance, extent, point
-def coverage(client, area, progress):
+def coverage(client, area, progress, workers):
try:
- return streetview.coverage(client, area, progress)
+ return streetview.coverage(client, area, progress, workers)
except (OSError, ValueError) as error:
raise RequestError("provider_unavailable", f"Street View coverage request failed: {error}") from error
@@ -21,12 +21,12 @@ def location(client, *, at=None, place=None, match=None, best_match=False, endpo
return (selected["lat"], selected["lon"]), selected
-def street_samples(client, sections, stops, step, progress):
+def street_samples(client, sections, stops, step, progress, workers):
"""Match panoramas to target positions along every selected street section."""
south, west, north, east = extent([p for section in sections for p in section["points"]])
lower, upper = around((south, west), 80), around((north, east), 80)
area = bounds((lower[0], lower[1], upper[2], upper[3]))
- views = coverage(client, area, progress)
+ views = coverage(client, area, progress, workers)
ways, _ = client.maps.data(area, progress=progress)
context = streetview.roads(ways, area, "all")
samples, gaps, requested = [], [], 0
@@ -47,8 +47,9 @@ def street_samples(client, sections, stops, step, progress):
def street_photos(client, output, progress, *, at=None, place=None, pano_id=None, street=None, match=None,
best_match=False, endpoint=places.GEOCODER, route=None, reverse=False, stops=None, step=None,
view="forward", heading=0, look_at=None, pitch=0, fov=75, radius=50, streetview_format="jpg",
- full_sphere=False, sphere_zoom=3):
+ full_sphere=False, sphere_zoom=3, workers=WORKERS):
number(sphere_zoom, "sphere_zoom", 0, 5, whole=True)
+ number(workers, "workers", 1, 64, whole=True)
streetview.angle(heading, "heading")
streetview.angle(pitch, "pitch")
fov = number(fov, "fov", 5, 175, whole=True)
@@ -64,7 +65,7 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=None
if street is not None:
selected_place = places.choose(client, street, match=match, best_match=best_match, street=True, endpoint=endpoint)
sections = routes.resolve(client, selected_place, route=route, reverse=reverse, progress=progress)
- samples, gaps, requested_stops = street_samples(client, sections, stops, step, progress)
+ samples, gaps, requested_stops = street_samples(client, sections, stops, step, progress, workers)
elif pano_id:
if not streetview.PANO_ID.fullmatch(pano_id):
raise ValueError("Invalid panorama ID.")
@@ -72,7 +73,7 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=None
else:
requested, selected_place = location(client, at=at, place=place, match=match,
best_match=best_match, endpoint=endpoint)
- views = coverage(client, around(requested, radius * 2), progress)
+ views = coverage(client, around(requested, radius * 2), progress, workers)
nearest = min(views, key=lambda v: (distance(requested, (v["lat"], v["lon"])), v["pano_id"]), default=None)
samples = []
if nearest and distance(requested, (nearest["lat"], nearest["lon"])) <= radius:
@@ -102,7 +103,7 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=None
photo = {**sample, **metadata}
if "heading" in photo:
photo["road_heading"] = photo.pop("heading")
- photo.update(streetview.save_sphere(client, metadata, sphere_zoom, path), path=str(path))
+ photo.update(streetview.save_sphere(client, metadata, sphere_zoom, path, workers), path=str(path))
result["photos"].append(photo)
else:
for direction in directions if street else (None,):
@@ -140,7 +141,8 @@ def street_photos(client, output, progress, *, at=None, place=None, pano_id=None
def satellite(client, output, progress, *, at=None, place=None, match=None, best_match=False,
- endpoint=places.GEOCODER, bbox=None, tile=None, size=200, zoom=19, satellite_format="jpg"):
+ endpoint=places.GEOCODER, bbox=None, tile=None, size=200, zoom=19, satellite_format="jpg",
+ workers=WORKERS):
if tile is not None:
zoom, x, y = tile
number(zoom, "zoom", 1, 21, whole=True)
@@ -158,7 +160,7 @@ def satellite(client, output, progress, *, at=None, place=None, match=None, best
best_match=best_match, endpoint=endpoint)
area = around(center, size)
run = capture.plan(client, area, dict(include=["satellite"], satellite_zoom=zoom,
- satellite_format=satellite_format), progress)
+ satellite_format=satellite_format, satellite_workers=workers), progress)
folder = capture.create_folder(run, output)
try:
capture.download(run, folder, client, progress)
diff --git a/src/streetview.py b/src/streetview.py
index 3a94463..94ba885 100644
--- a/src/streetview.py
+++ b/src/streetview.py
@@ -61,7 +61,7 @@ def parse_coverage(data):
raise ValueError("Unreadable Google panorama coverage; the endpoint may have changed.") from error
-def coverage(client, area, progress):
+def coverage(client, area, progress, workers):
"""Every Google panorama listed in the zoom-17 tiles that cover the area."""
coverage_grid = grid(area, 17)
count = coverage_grid["rows"] * coverage_grid["columns"]
@@ -70,7 +70,7 @@ def coverage(client, area, progress):
addresses = (url("https://www.google.com/maps/photometa/ac/v1",
pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")
for tile in tiles(coverage_grid))
- for i, data in enumerate(fetch(client, addresses), 1):
+ for i, data in enumerate(fetch(client, addresses, workers), 1):
for view in parse_coverage(data):
found[view["pano_id"]] = view
progress("Finding panoramas", i, count)
@@ -146,7 +146,7 @@ def save_photo(client, pano_id, heading, fov, pitch, path):
return dict(width=PHOTO_SIZE[0], height=PHOTO_SIZE[1], source_url=address)
-def save_sphere(client, metadata, zoom, target):
+def save_sphere(client, metadata, zoom, target, workers):
"""Assemble native panorama tiles; retain partial tiles for retry/resume."""
zoom = min(zoom, len(metadata["image_sizes"]) - 1)
width, height = metadata["image_sizes"][zoom]
@@ -159,7 +159,7 @@ def save_sphere(client, metadata, zoom, target):
addresses = (url("https://streetviewpixels-pa.googleapis.com/v1/tile", cb_client="maps_sv.tactile",
panoid=metadata["pano_id"], zoom=zoom, x=x, y=y) for x, y in missing)
with Image.new("RGB", (width, height)) as sphere:
- for (x, y), data in zip(missing, fetch(client, addresses, missing_ok=True)):
+ for (x, y), data in zip(missing, fetch(client, addresses, workers, missing_ok=True)):
if data is None:
raise MissingImagery("The requested image is not available.")
with open_image(data, (tw, th)) as tile:
@@ -284,7 +284,8 @@ def plan(client, area, options, progress, *, allow_empty=False):
parts = roads(ways, area, options["depth"])
if not parts and not allow_empty:
raise ValueError("No mapped roads at this path depth. Choose another area or depth.")
- found = [view for view in coverage(client, area, progress) if inside((view["lat"], view["lon"]), area)]
+ found = [view for view in coverage(client, area, progress, options["streetview_workers"])
+ if inside((view["lat"], view["lon"]), area)]
progress("Indexing roads")
lines = [Line(part["points"]) for part in parts]
index = RoadIndex(lines)