Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Separate terrain and OSM
Belluxx21075d0029d3
diff --git a/docs/cli.md b/docs/cli.mdindex e58342d..a9f4e0f 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -81,7 +81,7 @@ alephgeo capture create --bbox 41.8895 12.4910 41.8910 12.4940 -o capturesThe command shows an estimate and asks before downloading. Add `--yes` to skip the prompt, or `--plan` to save a plan for later.-Use `--sources` to select `streetview`, `satellite`, or `osm` (including terrain), or a combination. These three are the default.+Use `--sources` to select `streetview`, `satellite`, `osm`, or `terrain`, or a combination. These four are the default.Add `mesh` for Google Earth's textured 3D model of the area, saved as `mesh.glb`:diff --git a/src/capture.py b/src/capture.pyindex 2eddd4d..baf275b 100644--- a/src/capture.py+++ b/src/capture.py@@ -14,10 +14,10 @@ from . import mesh, satellite, streetview, terrainfrom .common import WORKERS, MissingImagery, contained, fetch, now, number, open_image, positive, write_bytes, write_jsonfrom .geo import MERCATOR_RADIUS, bounds, collection, coordinate, feature, grid, pixel, tile_ring, tiles-SOURCES = ("streetview", "satellite", "osm", "mesh")-FORMAT_VERSION = 4+SOURCES = ("streetview", "satellite", "osm", "terrain", "mesh")+FORMAT_VERSION = 5DEFAULT_OPTIONS = dict(- include=["streetview", "satellite", "osm"], step=30, fov=75, depth="roads", streetview_format="jpg",+ include=["streetview", "satellite", "osm", "terrain"], step=30, fov=75, depth="roads", streetview_format="jpg",full_sphere=False, sphere_zoom=3,delay=0, satellite_zoom=18, satellite_format="jpg", terrain_zoom=14, mesh_level=21,streetview_workers=WORKERS, satellite_workers=WORKERS, terrain_workers=WORKERS, mesh_workers=WORKERS,@@ -76,7 +76,7 @@ def settings(values=None):options = {**DEFAULT_OPTIONS, **values}include = options["include"]if not isinstance(include, list) or not include or any(source not in SOURCES for source in include):- raise ValueError("Choose one or more sources: streetview, satellite, osm, mesh.")+ raise ValueError("Choose one or more sources: streetview, satellite, osm, terrain, mesh.")options["include"] = [source for source in SOURCES if source in include]positive(options["step"], "step")number(options["delay"], "delay", 0)@@ -94,11 +94,6 @@ def settings(values=None):return options-def modes(include):- """Stage modes for the selected sources; osm also downloads terrain."""- return [mode for source in include for mode in (("osm", "terrain") if source == "osm" else (source,))]--def create_folder(run, parent):"""Persist an already prepared capture without repeating planning requests."""parent = Path(parent).expanduser().resolve()@@ -118,6 +113,7 @@ def plan(client, area, options, progress):stages.append(dict(mode="satellite", grid=grid(area, options["satellite_zoom"]), results=[]))if "osm" in options["include"]:stages.append(dict(mode="osm", results=[], notes=OSM_NOTES))+ if "terrain" in options["include"]:stages.append(dict(mode="terrain", grid=grid(area, options["terrain_zoom"]), results=[], notes=TERRAIN_NOTES))if "mesh" in options["include"]:stage = mesh.plan(client, area, options["mesh_level"], options["mesh_workers"], progress,@@ -323,7 +319,7 @@ def export(run, folder, progress, *, rebuild=True):def save_preview(run, folder):"""Write the README, planned street paths, and a north-up plan.svg per imagery stage."""- if run["options"]["include"] == ["osm"]:+ if set(run["options"]["include"]) <= {"osm", "terrain"}:returnwrite_bytes(folder / "README.txt", README)area = run["bounds"]@@ -395,7 +391,7 @@ def download(run, folder, client, progress):except (OSError, ValueError) as error:if stage["mode"] != "osm":raise- # Keep terrain usable if the map fails; report the error after saving it.+ # Keep the other sources going if the map fails; report the error after saving it.errors.append(error)write_json(folder / "manifest.json", run)except (Exception, KeyboardInterrupt) as error:@@ -421,7 +417,7 @@ def load(folder, *, check_files=True):raise ValueError("Unsupported capture format. Start a new capture with this version of Aleph.")bounds(run["bounds"], minimum=0)options = settings(run["options"])- if [s["mode"] for s in run["stages"]] != modes(options["include"]):+ if [s["mode"] for s in run["stages"]] != options["include"]:raise ValueError("Invalid saved stages.")for stage in run["stages"]:if stage["mode"] in ("satellite", "terrain"):diff --git a/src/cli.py b/src/cli.pyindex 16d61be..c3a822a 100644--- a/src/cli.py+++ b/src/cli.py@@ -143,7 +143,7 @@ def _capture_command(commands):actions = captures.add_subparsers(dest="action", required=True)area = actions.add_parser("create", help="Plan and download an area with multiple sources")area.add_argument("--bbox", required=True, nargs=4, type=float, metavar=("S", "W", "N", "E"))- area.add_argument("--sources", dest="include", nargs="+", choices=capture.SOURCES, help="sources to download (default: streetview satellite osm; osm includes terrain)")+ area.add_argument("--sources", dest="include", nargs="+", choices=capture.SOURCES, help="sources to download (default: streetview satellite osm terrain)")# Planning and confirmation.planning = area.add_mutually_exclusive_group()diff --git a/src/dashboard.py b/src/dashboard.pyindex 9e8fb07..5de3ec1 100644--- a/src/dashboard.py+++ b/src/dashboard.py@@ -332,8 +332,8 @@ class Dashboard:case "POST", ["api", "estimate"]:area, options = area_and_options(body)result = {}- for mode, source in (("satellite", "satellite"), ("terrain", "osm")):- if source in options["include"]:+ for mode in ("satellite", "terrain"):+ if mode in options["include"]:stage = dict(mode=mode, grid=grid(area, options[f"{mode}_zoom"]), results=[])result[mode] = dict(imagery(area, stage, 512 if mode == "terrain" else 256),seconds=capture.estimate(dict(stages=[stage], options=options))["seconds"])diff --git a/src/web/app.js b/src/web/app.jsindex 7bdc994..2b77e97 100644--- a/src/web/app.js+++ b/src/web/app.js@@ -113,6 +113,7 @@ const state = {drag: null,preview: null,planError: null,+ watched: null, // Id of the job whose end this page still has to handle.captureId: null,capture: null,photos: [],@@ -426,10 +427,10 @@ function estimate() {estimateTimer = setTimeout(async () => {const count = ++estimateCount;const {include, satellite_zoom, terrain_zoom, satellite_workers, terrain_workers, delay} = readForm();- const outputs = {satellite: $("[data-source=satellite] .estimate"), osm: $("[data-source=osm] .estimate")};+ const outputs = {satellite: $("[data-source=satellite] .estimate"), terrain: $("[data-source=terrain] .estimate")};const details = {satellite: $("[data-detail=satellite]"), terrain: $("[data-detail=terrain]")};let result = {};- if (state.area && (include.includes("satellite") || include.includes("osm"))) {+ if (state.area && (include.includes("satellite") || include.includes("terrain"))) {try {result = await api("/api/estimate", {bounds: state.area, options:{include, satellite_zoom, terrain_zoom, satellite_workers, terrain_workers, delay}});@@ -442,7 +443,7 @@ function estimate() {outputs.satellite.textContent = satellite ? `${plural(satellite.tiles, "tile")} · ~${duration(satellite.seconds)}` : "";details.satellite.textContent = satellite? `${resolution(satellite.meters_per_pixel)} · ${number(satellite.width)} × ${number(satellite.height)} px` : "";- outputs.osm.textContent = terrain ? `${plural(terrain.tiles, "terrain tile")}` : "";+ outputs.terrain.textContent = terrain ? `${plural(terrain.tiles, "tile")} · ~${duration(terrain.seconds)}` : "";details.terrain.textContent = terrain ? `${resolution(terrain.meters_per_pixel)} elevation` : "";}, 120);}@@ -564,6 +565,7 @@ async function previewPlan(event) {setData("plan", EMPTY);try {state.job = await api("/api/plan", {bounds: state.area, options: readForm()});+ state.watched = state.job.id;} catch (failure) {error.textContent = failure.message;error.hidden = false;@@ -773,9 +775,11 @@ async function poll() {pollTimer = setTimeout(poll, 3000);return;}- const previous = state.job;state.job = job;- const finished = job && job.state !== "running" && previous?.id === job.id && previous.state === "running";+ if (job?.state === "running") state.watched = job.id;+ // Small plans can end before the first poll, so match the job this page started rather than a state change.+ const finished = !!job && job.state !== "running" && state.watched === job.id;+ if (finished) state.watched = null;if (job?.kind === "plan" && finished) {if (job.state === "done") {try {@@ -829,7 +833,7 @@ function route() {estimate();if (!state.area) setDrawing(true);} else if (view === "preview") {- if (!state.preview && !running("plan")) {+ if (!state.preview && !state.planError && !running("plan") && state.watched !== state.job?.id) {location.replace("#/new");return;}@@ -879,6 +883,7 @@ function bind() {$("#start").disabled = true;try {state.job = await api("/api/capture", {plan: state.preview.plan});+ state.watched = state.job.id;state.preview = null;location.hash = `#/capture/${encodeURIComponent(state.job.capture)}`;poll();@@ -912,6 +917,7 @@ function bind() {$("#resume").addEventListener("click", async () => {try {state.job = await api(capturePath(state.capture.id, "/resume"), {});+ state.watched = state.job.id;renderCapture();poll();} catch (error) {diff --git a/src/web/index.html b/src/web/index.htmlindex a63293c..932f48b 100644--- a/src/web/index.html+++ b/src/web/index.html@@ -115,10 +115,16 @@</fieldset><fieldset class="source" data-source="osm">- <label class="source-head"><input type="checkbox" name="include" value="osm"><i data-icon="osm"></i><span>OSM map + terrain</span><output class="estimate"></output></label>+ <label class="source-head"><input type="checkbox" name="include" value="osm"><i data-icon="osm"></i><span>OSM map</span><output class="estimate"></output></label><div class="fields">- <p class="detail">OpenStreetMap data clipped to the area, and elevation tiles.</p>- <label class="field slider"><span>Terrain zoom</span><span class="range"><input type="range" name="terrain_zoom" min="8" max="14" step="1"><b class="knob"></b><output class="tag"></output></span></label>+ <p class="detail">OpenStreetMap data clipped to the area. The first capture in a region downloads its Geofabrik file, which can be hundreds of MB.</p>+ </div>+ </fieldset>++ <fieldset class="source" data-source="terrain">+ <label class="source-head"><input type="checkbox" name="include" value="terrain"><i data-icon="terrain"></i><span>Terrain</span><output class="estimate"></output></label>+ <div class="fields">+ <label class="field slider"><span>Zoom</span><span class="range"><input type="range" name="terrain_zoom" min="8" max="14" step="1"><b class="knob"></b><output class="tag"></output></span></label><p class="detail" data-detail="terrain"></p><label class="field"><span>Parallel requests</span><input type="number" name="terrain_workers" min="1" max="64" step="1"></label></div>