Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Add 3D view to dashboard
Belluxx34ab6503afd8
diff --git a/README.md b/README.mdindex 1fc958f..c6ea9d3 100644--- a/README.md+++ b/README.md@@ -45,7 +45,7 @@ alephgeo --helpalephgeo dashboard -o captures```-The dashboard opens in your browser at `http://127.0.0.1:8100`. Draw a rectangle on the map, choose the sources and their settings, check the preview and time estimate, then start the capture. Open any capture to show or hide each layer. Captures are saved in `captures`.+The dashboard opens in your browser at `http://127.0.0.1:8100`. Draw a rectangle on the map, choose the sources and their settings, check the preview and time estimate, then start the capture. Open any capture to show or hide each layer, or to explore its 3D mesh or terrain relief in 3D. Captures are saved in `captures`.The basemap comes from [OpenFreeMap](https://openfreemap.org) and needs an internet connection.diff --git a/src/dashboard.py b/src/dashboard.pyindex d321305..7fa5a25 100644--- a/src/dashboard.py+++ b/src/dashboard.py@@ -281,10 +281,13 @@ class Dashboard:cached = self.osm[identity] = signature, datareturn cached[1], "application/geo+json", 0- def tile(self, identity, mode, z, x, y):+ def tile(self, identity, mode, z, x, y, dem=False):folder, stage, _ = self.stage(identity, mode)+ # 3D terrain reloads as a whole, so it waits for every tile instead of growing with the download.+ if dem and len(stage["results"]) < capture.total(stage):+ raise FileNotFoundError("Terrain is still downloading.")z = int(z)- tile = self.tiles.get(folder, stage, z, int(x), int(y.split(".")[0]))+ tile = self.tiles.get(folder, stage, z, int(x), int(y.split(".")[0]), dem)if tile is None:raise FileNotFoundError("No saved tile here.")# Saved patches never change. Built tiles are not cached, so rendering changes show up at once.@@ -331,6 +334,8 @@ class Dashboard:return self.tile(identity, "satellite", z, x, y)case "GET", ["api", "captures", identity, "hillshade", z, x, y]:return self.tile(identity, "terrain", z, x, y)+ case "GET", ["api", "captures", identity, "dem", z, x, y]:+ return self.tile(identity, "terrain", z, x, y, dem=True)case "GET", ["api", "captures", identity, "files", *path] if path:return self.file(identity, path)case "POST", ["api", "estimate"]:diff --git a/src/layers.py b/src/layers.pyindex 7e4dc07..7a9c709 100644--- a/src/layers.py+++ b/src/layers.py@@ -81,9 +81,9 @@ def heights(path):if source.value(259) != 1:data = zlib.decompress(data)if source.float and predictor == 3:- # Rows hold byte differences across four planes, most significant first.- planes = np.cumsum(np.frombuffer(data, np.uint8).reshape(size, 4, size), axis=2, dtype=np.uint8)- block = planes.transpose(0, 2, 1).copy().view(">f4")+ # Rows hold four byte planes, most significant first, with differences running on across planes.+ rows = np.cumsum(np.frombuffer(data, np.uint8).reshape(size, 4 * size), axis=1, dtype=np.uint8)+ block = rows.reshape(size, 4, size).transpose(0, 2, 1).copy().view(">f4")elif source.float:block = np.frombuffer(data, order + "f4")else:@@ -125,6 +125,13 @@ def hillshade(center, neighbors, meters_per_pixel, exaggeration):return Image.fromarray(np.dstack((tone, tone, tone, np.where(inner, alpha, 0).astype(np.uint8))), "RGBA")+def terrarium(image, base):+ """Heights above base as Terrarium RGB for MapLibre terrain, which has no transparency: missing heights sit at base."""+ values = np.asarray(image, np.float64)+ values = np.clip(np.where(values > NODATA / 2, values - base, 0) + 32768, 0, 65535)+ return Image.fromarray(np.dstack((values // 256, values % 256, values % 1 * 256)).astype(np.uint8), "RGB")++def png(image):with BytesIO() as buffer:image.save(buffer, format="PNG", compress_level=1)@@ -140,9 +147,10 @@ class Tiles:self.lock = threading.Lock()# One build per layer at a time: parallel builds of overlapping overviews would repeat the same decoding.self.builds = defaultdict(threading.Lock)+ self.bases = {} # Folder → (saved count, lowest height).- def get(self, folder, stage, z, x, y):- """Return (bytes, content type), or None where nothing is saved yet."""+ def get(self, folder, stage, z, x, y, dem=False):+ """Return (bytes, content type), or None where nothing is saved yet; dem asks for terrain heights, not shading."""grid = stage["grid"]if not lowest(stage) <= z <= grid["zoom"] or not (0 <= x < 2**z and 0 <= y < 2**z):return None@@ -157,8 +165,12 @@ class Tiles:build = self.builds[folder, stage["mode"]]with build:image = self.image(folder, stage, z, x, y)+ base = self.base(folder, stage) if dem and image is not None else Noneif image is None:return None+ if dem:+ with terrarium(image, base) as encoded:+ return png(encoded), "image/png"if stage["mode"] == "terrain":# Building neighbors would cost as much as the tile itself; edges are extrapolated without them.neighbors = [self.cached(folder, stage, z, x + dx, y + dy) for dx, dy in NEIGHBORS]@@ -169,6 +181,18 @@ class Tiles:return png(shaded), "image/png"return png(image), "image/png"+ def base(self, folder, stage):+ """Lowest saved height, so terrain rises from the flat map around it; callers hold the build lock."""+ version = len(stage["results"])+ if self.bases.get(folder, (None,))[0] != version:+ lowest = np.inf+ for result in stage["results"]:+ with heights(contained(folder, result["filename"])) as image:+ values = np.asarray(image)+ lowest = min(lowest, values.min(initial=np.inf, where=values > NODATA / 2))+ self.bases[folder] = version, 0.0 if lowest == np.inf else float(lowest)+ return self.bases[folder][1]+def cached(self, folder, stage, z, x, y):version = saved(stage["grid"], len(stage["results"]), z, x, y)with self.lock:diff --git a/src/web/app.js b/src/web/app.jsindex 4c78095..33489ae 100644--- a/src/web/app.js+++ b/src/web/app.js@@ -51,6 +51,8 @@ const SHORTER = {streetview: "wider Street View spacing", satellite: "a lower samesh: "lower 3D mesh detail"};// Progress phases that a capture stage row already shows.const DOWNLOADS = new Set(["Street View", "Satellite", "Downloading terrain tiles", "3D mesh"]);+const LARGE_MESH = 250e6; // Bytes of mesh.glb that ask first: its textures can need several GB of graphics memory.+const PITCH = 60; // Degrees the map tilts to for 3D layers.// Helpers@@ -159,9 +161,12 @@ const state = {search: [],toggles: {photos: true, route: true, satellite: true, terrain: true, buildings: true, roads: true, water: true,outline: true, basemap: true, ...store.get("toggles", {})},+ solid: null, // 3D layer shown in a capture: "mesh", "relief", or null.+ meshLoading: false,};let map;+let navigation;let basemapLayers = [];// Raster source and layer ids per mode, oldest first; a refresh loads on top until it replaces the older one.const rasters = {satellite: [], terrain: []};@@ -368,9 +373,126 @@ function clearCaptureLayers() {for (const ids of Object.values(rasters)) ids.splice(0).forEach(removeRaster);for (const source of ["osm", "photos", "selected", "bounds", "plan"]) setData(source, EMPTY);Object.assign(state, {photos: [], photoCount: 0, photoAgain: false, live: {}, selected: null});+ if (state.solid) {+ state.solid = null;+ syncSolid();+ map.jumpTo({pitch: 0, bearing: 0}); // The next view fits its bounds, which would keep a half-eased tilt.+ }+ if (map.getSource("dem")) map.removeSource("dem");$("#viewer").hidden = true;}+// 3D++let three = null; // three.js and its glTF loader, imported on first use.++function loadThree() {+ three ??= Promise.all([import("three"), import("three/addons/loaders/GLTFLoader.js")]).catch(error => {+ three = null; // Try again next time, such as once back online.+ throw error;+ });+ return three;+}++// A custom layer drawing mesh.glb, in meters east (x), up (y) and south (z) of the capture's center.+async function meshLayer(capture) {+ const [THREE, {GLTFLoader}] = await loadThree();+ const {scene} = await new GLTFLoader().loadAsync(absolute(capturePath(capture.id, "/files/mesh.glb")));+ // Rest the ground along its edges on the flat map, where they meet: the lowest point at y = 0 can be a pit,+ // such as an arena floor or an underpass. A low share of edge heights skips roofs cut by the edges.+ const box = new THREE.Box3().setFromObject(scene), edges = [];+ scene.traverse(({geometry}) => {+ const position = geometry?.attributes.position;+ for (let i = 0; i < (position?.count ?? 0); i++) {+ const x = position.getX(i), z = position.getZ(i);+ if (Math.min(x - box.min.x, box.max.x - x, z - box.min.z, box.max.z - z) < 10) edges.push(position.getY(i));+ }+ });+ edges.sort((a, b) => a - b);+ const ground = edges[Math.floor(edges.length / 10)] ?? 0;+ const [s, w, n, e] = capture.bounds;+ const origin = maplibregl.MercatorCoordinate.fromLngLat([(w + e) / 2, (s + n) / 2], -ground);+ const scale = origin.meterInMercatorCoordinateUnits();+ // Mercator runs x east, y south and z up: turn y-up into z-up, then mirror y so glTF's south stays south.+ const place = new THREE.Matrix4().makeTranslation(origin.x, origin.y, origin.z)+ .scale(new THREE.Vector3(scale, -scale, scale)).multiply(new THREE.Matrix4().makeRotationX(Math.PI / 2));+ const camera = new THREE.Camera();+ let renderer;+ return {+ id: "mesh", type: "custom", renderingMode: "3d",+ onAdd(map, gl) {+ renderer = new THREE.WebGLRenderer({canvas: map.getCanvas(), context: gl});+ renderer.autoClear = false; // Draw over the map.+ },+ render(gl, {defaultProjectionData}) {+ camera.projectionMatrix.fromArray(defaultProjectionData.mainMatrix).multiply(place);+ renderer.resetState();+ renderer.render(scene, camera);+ },+ onRemove() {+ scene.traverse(object => {+ object.geometry?.dispose();+ object.material?.map?.dispose();+ object.material?.dispose();+ });+ renderer.dispose();+ },+ };+}++// 3D layers tilt the map and let it rotate; otherwise it stays flat and north up.+function tilt(on) {+ if (on === map.dragRotate.isEnabled()) return;+ for (const handler of [map.dragRotate, map.touchPitch]) on ? handler.enable() : handler.disable();+ for (const handler of [map.touchZoomRotate, map.keyboard]) on ? handler.enableRotation() : handler.disableRotation();+ map.removeControl(navigation);+ navigation = new maplibregl.NavigationControl({showCompass: on, visualizePitch: true});+ map.addControl(navigation, "top-right");+ map.easeTo(on ? {pitch: PITCH} : {pitch: 0, bearing: 0});+}++async function setSolid(kind, on) {+ const bytes = state.capture.files["mesh.glb"];+ if (on && kind === "mesh" && bytes > LARGE_MESH && !await confirmMesh(bytes)) return renderLayers();+ state.solid = on ? kind : null;+ renderLayers();+ syncSolid();+}++// Show the mesh or the terrain relief, one at a time since the mesh has its own ground.+async function syncSolid() {+ const {solid, capture} = state;+ if (solid !== "relief" && map.getTerrain()) map.setTerrain(null);+ if (solid !== "mesh" && map.getLayer("mesh")) map.removeLayer("mesh");+ tilt(!!solid);+ if (solid === "relief") {+ const [s, w, n, e] = capture.bounds, {minzoom, zoom} = capture.stages.terrain;+ if (!map.getSource("dem")) {+ map.addSource("dem", {type: "raster-dem", encoding: "terrarium", tileSize: 512, bounds: [w, s, e, n], minzoom,+ maxzoom: zoom, tiles: [absolute(capturePath(capture.id, "/dem/{z}/{x}/{y}"))]});+ }+ map.setTerrain({source: "dem"});+ }+ if (solid !== "mesh" || map.getLayer("mesh")) return;+ state.meshLoading = true;+ renderLayers();+ const current = () => state.capture?.id === capture.id && state.solid === "mesh";+ try {+ const layer = await meshLayer(capture);+ if (current() && !map.getLayer("mesh")) map.addLayer(layer);+ } catch (error) {+ if (current()) {+ state.solid = null;+ syncSolid();+ $("#capture-error").textContent = `Could not show the 3D mesh: ${error.message}`;+ $("#capture-error").hidden = false;+ }+ } finally {+ state.meshLoading = false;+ renderLayers();+ }+}+// Area drawingfunction setArea(area) {@@ -680,17 +802,30 @@ function confirmCapture(preview) {// Time per source, why it is slow and what helps; resolves true to go ahead.function confirmSlow(subject, times, text, anyway) {- const dialog = $("#slow");const pace = speed(Object.values(times).reduce((sum, seconds) => sum + seconds, 0));- $("h2", dialog).innerHTML = `${icon(pace.icon)}${subject} will take ${pace.name}`;- $("h2", dialog).classList.toggle("alert", !!pace.alert);- $("ul", dialog).innerHTML = Object.entries(times).map(([mode, seconds]) => {+ const rows = Object.entries(times).map(([mode, seconds]) => {const {icon: name, alert} = speed(seconds);return `<li>${icon(mode)}<span>${NAMES[mode]}</span><span class="estimate ${alert ? "alert" : ""}">${icon(name)}${speed(seconds).name}</span></li>`;}).join("");+ return ask(`${icon(pace.icon)}${subject} will take ${pace.name}`, !!pace.alert, rows, text, anyway, "Change settings");+}++function confirmMesh(bytes) {+ return ask(`${icon("warning")}mesh.glb is ${megabytes(bytes)}`, true, "", "Its textures may need several GB of "+ + "graphics memory, which can crash this tab. Blender handles large meshes better.", "Show anyway", "Cancel");+}++// Resolves true to go ahead.+function ask(title, alert, rows, text, anyway, back) {+ const dialog = $("#ask");+ $("h2", dialog).innerHTML = title;+ $("h2", dialog).classList.toggle("alert", alert);+ $("ul", dialog).innerHTML = rows;+ $("ul", dialog).hidden = !rows;$("p", dialog).textContent = text;$("[value=go]", dialog).textContent = anyway;+ $("[value='']", dialog).textContent = back;dialog.returnValue = "";dialog.showModal();return new Promise(resolve => dialog.addEventListener("close", () => resolve(dialog.returnValue === "go"), {once: true}));@@ -818,9 +953,11 @@ function renderLayers() {const c = state.capture;if (!c) return;const s = c.stages;- const toggle = (key, name, value = "", disabled = false) => `<label class="toggle">- <input type="checkbox" data-toggle="${key}" ${state.toggles[key] ? "checked" : ""} ${disabled ? "disabled" : ""}>+ const row = (attribute, checked, name, value, disabled) => `<label class="toggle">+ <input type="checkbox" ${attribute} ${checked ? "checked" : ""} ${disabled ? "disabled" : ""}><span>${name}</span><span class="value">${esc(value)}</span></label>`;+ const toggle = (key, name, value = "", disabled = false) => row(`data-toggle="${key}"`, state.toggles[key], name, value, disabled);+ const solid = (kind, name, value, disabled) => row(`data-solid="${kind}"`, state.solid === kind, name, value, disabled);const groups = [];if (s.streetview) {const skipped = s.streetview.skipped ? ` · ${number(s.streetview.skipped)} skipped` : "";@@ -836,11 +973,15 @@ function renderLayers() {groups.push(["osm", toggle("buildings", "Buildings", note, !saved) + toggle("roads", "Roads and paths", note, !saved)+ toggle("water", "Water and land", note, !saved)]);}- if (s.terrain) groups.push(["terrain", toggle("terrain", "Hillshade", `zoom ${s.terrain.zoom} · ${tileCount("terrain")}`)]);+ if (s.terrain) {+ const saved = (state.live.terrain ?? s.terrain.done) >= s.terrain.total;+ groups.push(["terrain", toggle("terrain", "Hillshade", `zoom ${s.terrain.zoom} · ${tileCount("terrain")}`)+ + solid("relief", "3D relief", saved ? "" : "after download", !saved)]);+ }if (s.mesh) {const glb = c.files["mesh.glb"];- groups.push(["mesh", `<p class="toggle"><span>mesh.glb</span><span class="value">${- glb ? megabytes(glb) : `${number(s.mesh.done)} / ${plural(s.mesh.total, "node")}`}</span></p>`]);+ groups.push(["mesh", solid("mesh", "Show in 3D", state.meshLoading ? "Loading…" : glb ? megabytes(glb)+ : `${number(s.mesh.done)} / ${plural(s.mesh.total, "node")}`, !glb)]);}groups.push(["map", toggle("outline", "Capture outline") + toggle("basemap", "Basemap")]);$("#layers").innerHTML = groups.map(([mode, rows]) =>@@ -1030,7 +1171,8 @@ function bind() {$("#captures").addEventListener("mouseleave", () => hoverCapture(null));$("#layers").addEventListener("change", event => {- const key = event.target.dataset.toggle;+ const {toggle: key, solid} = event.target.dataset;+ if (solid) return setSolid(solid, event.target.checked);if (!key) return;state.toggles[key] = event.target.checked;store.set("toggles", state.toggles);@@ -1143,9 +1285,11 @@ async function init() {const view = store.get("view", {center: [12.49, 41.89], zoom: 2});map = new maplibregl.Map({container: "map", style: await basemap(), center: view.center, zoom: view.zoom,- attributionControl: {compact: true}, dragRotate: false, pitchWithRotate: false});+ attributionControl: {compact: true}, dragRotate: false, touchPitch: false});map.touchZoomRotate.disableRotation();- map.addControl(new maplibregl.NavigationControl({showCompass: false}), "top-right");+ map.keyboard.disableRotation();+ navigation = new maplibregl.NavigationControl({showCompass: false});+ map.addControl(navigation, "top-right");map.addControl(new maplibregl.ScaleControl({unit: "metric"}), "bottom-left");await map.once("load");diff --git a/src/web/index.html b/src/web/index.htmlindex 2a39509..c850bab 100644--- a/src/web/index.html+++ b/src/web/index.html@@ -7,6 +7,11 @@<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Ccircle cx='8' cy='8' r='7.5' fill='%23F8F1E5' stroke='%2334302B'/%3E%3Cpath d='M6 1c-1 3 2 5 1 7s2 4 3 7' stroke='%23FF4938' stroke-width='2' fill='none'/%3E%3C/svg%3E"><link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/maplibre-gl.css"><link rel="stylesheet" href="/style.css">+ <!-- three.js draws 3D meshes; app.js imports it on first use. -->+ <script type="importmap">+ {"imports": {"three": "https://unpkg.com/[email protected]/build/three.module.js",+ "three/addons/": "https://unpkg.com/[email protected]/examples/jsm/"}}+ </script><script defer src="https://unpkg.com/[email protected]/dist/maplibre-gl.js"></script><script defer src="/app.js"></script></head>@@ -210,14 +215,14 @@</section></aside>- <dialog id="slow">+ <dialog id="ask"><form method="dialog"><h2></h2><ul class="stages compact"></ul><p class="detail"></p><div class="buttons"><button value="go" class="subtle"></button>- <button value="" class="primary" autofocus>Change settings</button>+ <button value="" class="primary" autofocus></button></div></form></dialog>diff --git a/tests/test_imagery.py b/tests/test_imagery.pyindex 1422f96..8b6895e 100644--- a/tests/test_imagery.py+++ b/tests/test_imagery.py@@ -85,7 +85,8 @@ def float_tile(order, predictor, x=8):encoded = bytearray()for y in range(512):row = raw[y * 2048:(y + 1) * 2048]- planes = b"".join(row[i::4] for i in range(4))+ # Planes start from the most significant byte, whatever the file's byte order.+ planes = b"".join(row[i::4] for i in ((0, 1, 2, 3) if order == ">" else (3, 2, 1, 0)))encoded.extend(bytes([planes[0]]) + bytes((b - a) & 255 for a, b in zip(planes, planes[1:])))data = zlib.compress(encoded)else:@@ -141,14 +142,15 @@ def int16_tile(order, x=8):return terrain.header(tags, order) + b"".join(blocks), heights-def decode_float_block(data, predictor):+def decode_float_block(data, predictor, order):raw = zlib.decompress(data)if predictor != 3:return rawdecoded = bytearray()for y in range(256):planes = bytes(v & 255 for v in accumulate(raw[y * 1024:(y + 1) * 1024]))- decoded.extend(v for pixel in zip(*(planes[i * 256:(i + 1) * 256] for i in range(4))) for v in pixel)+ planes = [planes[i * 256:(i + 1) * 256] for i in range(4)]+ decoded.extend(v for pixel in zip(*(planes if order == ">" else planes[::-1])) for v in pixel)return bytes(decoded)@@ -176,8 +178,13 @@ class TerrainTests(unittest.TestCase):left = column % 2 * 1024expected = b"".join(original[y * 2048 + left:y * 2048 + left + 1024]for y in range(row * 256, (row + 1) * 256))- self.assertEqual(decode_float_block(block, predictor), expected)+ self.assertEqual(decode_float_block(block, predictor, order), expected)self.assertEqual(len(merged.values(324)), 8)+ # The dashboard decodes saved tiles itself; prediction differences running across planes spiked heights.+ for x, raw in zip((8, 9), originals):+ with layers.heights(folder / f"{x}.tif") as decoded:+ self.assertEqual(decoded.tobytes(), struct.pack(f"={512 * 512}f",+ *struct.unpack(f"{order}{512 * 512}f", raw)))def test_truncated_or_misplaced_terrain_is_rejected(self):data, _ = float_tile("<", 3)