Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Fix terrain shading and decode
Belluxxb142666c9e49
diff --git a/README.md b/README.mdindex 0dc9321..1fc958f 100644--- a/README.md+++ b/README.md@@ -23,7 +23,7 @@ This was born as a way to let LLMs quickly gather information from the physical## Install-Requires Python 3.11+. Pillow is the only dependency. No API keys are required.+Requires Python 3.11+. The only dependencies are Pillow and numpy. No API keys are required.```shgit clone https://github.com/Belluxx/Aleph.gitdiff --git a/pyproject.toml b/pyproject.tomlindex 313e0be..dfa1c19 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -8,7 +8,7 @@ version = "0.1.0"description = "Capture Street View photos, satellite imagery, and OpenStreetMap data."readme = "README.md"requires-python = ">=3.11"-dependencies = ["Pillow>=10"]+dependencies = ["Pillow>=10", "numpy>=1.25"][project.scripts]alephgeo = "aleph:main"diff --git a/src/dashboard.py b/src/dashboard.pyindex 5de3ec1..80f2a98 100644--- a/src/dashboard.py+++ b/src/dashboard.py@@ -123,7 +123,7 @@ def summary(identity, run, live):if stage["mode"] in ("streetview", "mesh"):info["skipped"] = sum(result.get("status") == "skipped" for result in results[:count])if "grid" in stage:- info["zoom"] = stage["grid"]["zoom"]+ info.update(zoom=stage["grid"]["zoom"], minzoom=layers.lowest(stage))state = "interrupted" if run["state"] == "running" and not live else run["state"]return dict(id=identity, bounds=run["bounds"], started_at=run["started_at"], finished_at=run.get("finished_at"),state=state, error=run.get("error"), options=run["options"], stages=stages)@@ -282,11 +282,12 @@ class Dashboard:def tile(self, identity, mode, z, x, y):folder, stage, _ = self.stage(identity, mode)- tile = self.tiles.get(folder, stage, int(z), int(x), int(y.split(".")[0]))+ z = int(z)+ tile = self.tiles.get(folder, stage, z, int(x), int(y.split(".")[0]))if tile is None:raise FileNotFoundError("No saved tile here.")- # Tile URLs carry the saved count, so a response never changes.- return *tile, 86400+ # Saved patches never change. Built tiles are not cached, so rendering changes show up at once.+ return *tile, 86400 if mode == "satellite" and z == stage["grid"]["zoom"] else 0def file(self, identity, path):target = contained(self.folder(identity), "/".join(path))@@ -303,7 +304,7 @@ class Dashboard:case "GET", [("app.js" | "style.css") as name]:return (WEB / name).read_bytes(), mimetypes.guess_type(name)[0], 0case "GET", ["api", "config"]:- return dict(root=str(self.root), defaults=capture.DEFAULT_OPTIONS, overviews=layers.OVERVIEWS)+ return dict(root=str(self.root), defaults=capture.DEFAULT_OPTIONS)case "GET", ["api", "job"]:return self.job_state()case "GET", ["api", "preview"]:@@ -325,8 +326,10 @@ class Dashboard:return self.photos(identity, int(query.get("after", ["0"])[0]))case "GET", ["api", "captures", identity, "osm"]:return self.osm_layer(identity)- case "GET", ["api", "captures", identity, ("satellite" | "terrain") as mode, z, x, y]:- return self.tile(identity, mode, z, x, y)+ case "GET", ["api", "captures", identity, "satellite", z, x, y]:+ return self.tile(identity, "satellite", z, x, y)+ case "GET", ["api", "captures", identity, "hillshade", z, x, y]:+ return self.tile(identity, "terrain", z, x, y)case "GET", ["api", "captures", identity, "files", *path] if path:return self.file(identity, path)case "POST", ["api", "estimate"]:diff --git a/src/layers.py b/src/layers.pyindex 0b19279..7e4dc07 100644--- a/src/layers.py+++ b/src/layers.py@@ -1,23 +1,28 @@"""Browser-ready views of saved captures: imagery and terrain tiles, and OSM as GeoJSON."""import math-import struct-import sysimport threadingimport zlibfrom collections import OrderedDict, defaultdictfrom io import BytesIO-from itertools import accumulatefrom xml.etree import ElementTree as ET+import numpy as npfrom PIL import Imagefrom . import terrainfrom .common import contained-from .geo import collection, feature, ring_contains, signed_area+from .geo import MERCATOR_RADIUS, collection, coordinate, feature, ring_contains, signed_areaCACHE_BYTES = 384 * 2**20-OVERVIEWS = 6 # Zoom levels built below a grid's own zoom.+OVERVIEWS = dict(satellite=6, terrain=8) # Most zoom levels built below a grid's own zoom.+# Missing heights. Overviews average it in, which keeps those pixels far below any real height.+NODATA = -1e6+NEIGHBORS = ((-1, 0), (1, 0), (0, -1), (0, 1)) # West, east, north, south.+# Opacity per unit of brightness change from flat ground, and its maximum. Shadows span a much wider range.+SHADOW = 160, 120+HIGHLIGHT = 400, 120+RELIEF = 4 # Overview levels per doubling of slope exaggeration.AREA_TAGS = {"building", "landuse", "amenity", "leisure", "boundary", "place", "shop",@@ -57,61 +62,67 @@ def saved(grid, count, z, x, y):return row * columns + end if row >= top else 0-def unpredict(data, width):- """Undo the TIFF floating point predictor, giving big-endian Float32 rows."""- rows = bytearray()- for y in range(0, len(data), width * 4):- # Each row holds byte differences across four planes, most significant first.- planes = bytes(v & 255 for v in accumulate(data[y:y + width * 4]))- row = bytearray(width * 4)- for plane in range(4):- row[plane::4] = planes[plane * width:(plane + 1) * width]- rows += row- return bytes(rows)---def undifference(data, width, order):- """Undo the TIFF horizontal predictor for 16-bit samples."""- fmt = f"{order}{width}H"- return b"".join(struct.pack(fmt, *(v & 0xFFFF for v in accumulate(struct.unpack(fmt, data[y:y + width * 2]))))- for y in range(0, len(data), width * 2))+def lowest(stage):+ """Lowest zoom with tiles: a few levels past where the grid fits in one tile, within the overview limit."""+ grid = stage["grid"]+ levels = math.ceil(math.log2(max(grid["columns"], grid["rows"]))) + 3+ return max(0, grid["zoom"] - min(levels, OVERVIEWS[stage["mode"]]))def heights(path):"""Decode one saved 512-pixel Int16 or Float32 terrain tile without libtiff."""with path.open("rb") as stream:source = terrain.TIFF(stream)- size = source.value(322)+ size, order, predictor = source.value(322), source.order, source.value(317, 1)per_row = 512 // size- image = Image.new("F", (512, 512))+ values = np.empty((512, 512), np.float32)for i, (offset, length) in enumerate(zip(source.values(324), source.values(325))):data = source.read(offset, length)if source.value(259) != 1:data = zlib.decompress(data)- big = source.order == ">"- if source.value(317, 1) == 3:- data, big = unpredict(data, size), True- elif source.value(317, 1) == 2:- data = undifference(data, size, source.order)- mode, raw_mode = ("F", "F;32BF" if big else "F;32F") if source.float else ("I", "I;16BS" if big else "I;16S")- with (- Image.frombytes(mode, (size, size), data, "raw", raw_mode) as block,- block.convert("F") as values,- ):- image.paste(values, (i % per_row * size, i // per_row * size))+ if source.float and predictor == 3:+ # Rows hold byte differences across four planes, most significant first.+ planes = np.cumsum(np.frombuffer(data, np.uint8).reshape(size, 4, size), axis=2, dtype=np.uint8)+ block = planes.transpose(0, 2, 1).copy().view(">f4")+ elif source.float:+ block = np.frombuffer(data, order + "f4")+ else:+ block = np.frombuffer(data, order + "u2").reshape(size, size)+ if predictor == 2:+ block = np.cumsum(block, axis=1, dtype=np.uint16)+ block = block.astype(np.int16)+ y, x = divmod(i, per_row)+ values[y * size:(y + 1) * size, x * size:(x + 1) * size] = block.reshape(size, size)nodata = float(b"".join(source.values(42113)).rstrip(b"\0"))- # Terrarium has no NoData value. Unknown heights use sea level.- image.putdata([v if math.isfinite(v) and v != nodata else 0- for (v,) in struct.iter_unpack("=f", image.tobytes())])- return image---def terrarium(image):- """Encode heights as RGB, where meters = R * 256 + G + B / 256 - 32768."""- with image.point(lambda h: h * 256 + 32768 * 256) as scaled, scaled.convert("I") as values:- # Native 32-bit integers hold the three channels below an unused high byte.- raw = "BGRX" if sys.byteorder == "little" else "XRGB"- return Image.frombytes("RGB", image.size, values.tobytes(), "raw", raw)+ values[~np.isfinite(values) | (values == nodata)] = NODATA+ return Image.fromarray(values, "F")+++def hillshade(center, neighbors, meters_per_pixel, exaggeration):+ """Shadows and highlights lit from the northwest, transparent where heights are missing."""+ center = np.asarray(center, np.float32)+ # Without a neighbor, continue the slope at the edge so adjacent tiles still meet smoothly.+ padded = np.pad(center, 1, mode="reflect", reflect_type="odd")+ # Neighbors' facing edges, where known, keep slopes continuous across tiles.+ for (dx, dy), neighbor in zip(NEIGHBORS, neighbors):+ if neighbor is None:+ continue+ neighbor = np.asarray(neighbor, np.float32)+ target, edge = {(-1, 0): (np.s_[1:-1, 0], np.s_[:, -1]), (1, 0): (np.s_[1:-1, -1], np.s_[:, 0]),+ (0, -1): (np.s_[0, 1:-1], np.s_[-1, :]), (0, 1): (np.s_[-1, 1:-1], np.s_[0, :])}[dx, dy]+ padded[target] = np.where(neighbor[edge] > NODATA / 2, neighbor[edge], padded[target])+ scale = exaggeration / (2 * meters_per_pixel)+ east = (padded[1:-1, 2:] - padded[1:-1, :-2]) * scale+ south = (padded[2:, 1:-1] - padded[:-2, 1:-1]) * scale+ # Light comes from x west, y north, 45° up: (-0.5, -0.5, 0.7071) with x east and y south.+ shade = (east * 0.5 + south * 0.5 + 0.7071) / np.sqrt(east * east + south * south + 1) - 0.7071+ (dark, darkest), (light, lightest) = SHADOW, HIGHLIGHT+ alpha = np.minimum(np.maximum(-shade, 0) * dark, darkest) + np.minimum(np.maximum(shade, 0) * light, lightest)+ # Slopes next to a missing height are unreliable, so the known area shrinks by a pixel inside the tile.+ known = np.pad(center > NODATA / 2, 1, mode="edge")+ inner = np.logical_and.reduce([known[y:y + center.shape[0], x:x + center.shape[1]] for y in range(3) for x in range(3)])+ tone = np.where(shade > 0, 255, 0).astype(np.uint8)+ return Image.fromarray(np.dstack((tone, tone, tone, np.where(inner, alpha, 0).astype(np.uint8))), "RGBA")def png(image):@@ -127,11 +138,13 @@ class Tiles:self.cache = OrderedDict()self.used = 0self.lock = threading.Lock()+ # One build per layer at a time: parallel builds of overlapping overviews would repeat the same decoding.+ self.builds = defaultdict(threading.Lock)def get(self, folder, stage, z, x, y):"""Return (bytes, content type), or None where nothing is saved yet."""grid = stage["grid"]- if not grid["zoom"] - OVERVIEWS <= z <= grid["zoom"] or not (0 <= x < 2**z and 0 <= y < 2**z):+ if not lowest(stage) <= z <= grid["zoom"] or not (0 <= x < 2**z and 0 <= y < 2**z):return Noneif stage["mode"] == "satellite" and z == grid["zoom"]:# Saved patches are already browser images.@@ -140,25 +153,39 @@ class Tiles:return Nonepath = contained(folder, stage["results"][version - 1]["filename"])return path.read_bytes(), "image/png" if path.suffix == ".png" else "image/jpeg"- image = self.image(folder, stage, z, x, y)+ with self.lock:+ build = self.builds[folder, stage["mode"]]+ with build:+ image = self.image(folder, stage, z, x, y)if image is None:return Noneif stage["mode"] == "terrain":- with terrarium(image) as rgb:- return png(rgb), "image/png"+ # Building neighbors would cost as much as the tile itself; edges are extrapolated without them.+ neighbors = [self.cached(folder, stage, z, x + dx, y + dy) for dx, dy in NEIGHBORS]+ latitude = math.radians(coordinate((x + 0.5) * 256, (y + 0.5) * 256, z)[0])+ meters = 2 * math.pi * MERCATOR_RADIUS * math.cos(latitude) / (512 * 2**z)+ # Zoomed out, averaged heights flatten slopes; exaggerate them to keep relief visible.+ with hillshade(image, neighbors, meters, 2 ** ((grid["zoom"] - z) / RELIEF)) as shaded:+ return png(shaded), "image/png"return png(image), "image/png"+ def cached(self, folder, stage, z, x, y):+ version = saved(stage["grid"], len(stage["results"]), z, x, y)+ with self.lock:+ entry = self.cache.get((folder, stage["mode"], z, x, y))+ if entry and version and entry[0] == version:+ self.cache.move_to_end((folder, stage["mode"], z, x, y))+ return entry[1]+ return None+def image(self, folder, stage, z, x, y):grid, results = stage["grid"], stage["results"]version = saved(grid, len(results), z, x, y)if not version:return Nonekey = folder, stage["mode"], z, x, y- with self.lock:- entry = self.cache.get(key)- if entry and entry[0] == version:- self.cache.move_to_end(key)- return entry[1]+ if (image := self.cached(folder, stage, z, x, y)) is not None:+ return imageelevation = stage["mode"] == "terrain"if z == grid["zoom"]:path = contained(folder, results[version - 1]["filename"])@@ -170,22 +197,13 @@ class Tiles:else:size = 512 if elevation else 256half = size // 2- image = Image.new("F" if elevation else "RGBA", (size, size))- missing, means = [], []+ image = Image.new("F", (size, size), NODATA) if elevation else Image.new("RGBA", (size, size))for dy in range(2):for dx in range(2):child = self.image(folder, stage, z + 1, 2 * x + dx, 2 * y + dy)- if child is None:- missing.append((dx * half, dy * half))- continue- with child.resize((half, half), Image.Resampling.BOX) as smaller:- image.paste(smaller, (dx * half, dy * half))- if elevation:- with smaller.resize((1, 1), Image.Resampling.BOX) as mean:- means.append(mean.getpixel((0, 0)))- # Sea level outside the grid would draw cliffs at its edges; use the tile's mean height.- for left, top in missing if means else ():- image.paste(sum(means) / len(means), (left, top, left + half, top + half))+ if child is not None:+ with child.resize((half, half), Image.Resampling.BOX) as smaller:+ image.paste(smaller, (dx * half, dy * half))with self.lock:previous = self.cache.pop(key, None)if previous:@@ -193,7 +211,9 @@ class Tiles:self.cache[key] = version, imageself.used += image.width * image.height * 4while self.used > CACHE_BYTES:- _, (_, dropped) = self.cache.popitem(last=False)+ # Finer tiles are cheapest to rebuild, so the least recently used of them go first.+ victim = max(self.cache, key=lambda cached: cached[2])+ _, dropped = self.cache.pop(victim)self.used -= dropped.width * dropped.height * 4return imagediff --git a/src/web/app.js b/src/web/app.jsindex 2b77e97..35b52a3 100644--- a/src/web/app.js+++ b/src/web/app.js@@ -288,19 +288,19 @@ function setData(source, data) {// Rasters are added once a capture has saved tiles; their URLs carry the saved count.function setRaster(mode, info) {- const tiles = [absolute(capturePath(state.capture.id, `/${mode}/{z}/{x}/{y}?v=${info.done}`))];+ const path = mode === "terrain" ? "hillshade" : mode;+ const tiles = [absolute(capturePath(state.capture.id, `/${path}/{z}/{x}/{y}?v=${info.done}`))];const source = map.getSource(mode);if (source) return source.setTiles(tiles);const [s, w, n, e] = state.capture.bounds;- const spec = {tiles, bounds: [w, s, e, n], minzoom: Math.max(0, info.zoom - state.config.overviews), maxzoom: info.zoom};+ const spec = {tiles, bounds: [w, s, e, n], minzoom: info.minzoom, maxzoom: info.zoom};if (mode === "satellite") {map.addSource(mode, {type: "raster", tileSize: 256, ...spec});addLayer({id: "satellite", type: "raster", source: mode, paint: {"raster-fade-duration": 0}}, "osm-land");} else {- map.addSource(mode, {type: "raster-dem", encoding: "terrarium", tileSize: 512, ...spec});- addLayer({id: "hillshade", type: "hillshade", source: mode, paint: {- "hillshade-shadow-color": "--hill-shadow", "hillshade-highlight-color": "--hill-light",- "hillshade-accent-color": "--hill-shadow", "hillshade-exaggeration": 0.55}}, "osm-land");+ // The server shades terrain itself, so tiles stay transparent where heights are missing.+ map.addSource(mode, {type: "raster", tileSize: 512, ...spec});+ addLayer({id: "hillshade", type: "raster", source: mode, paint: {"raster-fade-duration": 0}}, "osm-land");}syncMap();}@@ -620,7 +620,7 @@ function refresh(stages, force = false) {if (!force && info.done < info.total && Date.now() - (live[`${mode}At`] || 0) < 2500) continue;live[mode] = info.done;live[`${mode}At`] = Date.now();- setRaster(mode, {...info, zoom: info.zoom ?? state.capture.stages[mode].zoom});+ setRaster(mode, {...state.capture.stages[mode], ...info});}if (stages.osm?.done && !live.osm) {live.osm = true;diff --git a/src/web/style.css b/src/web/style.cssindex 7b15abc..103b62e 100644--- a/src/web/style.css+++ b/src/web/style.css@@ -17,8 +17,6 @@--map-rail: #d3c8b5;--map-boundary: #c9bda9;--map-label: #8a7f70;- --hill-shadow: #34302b;- --hill-light: #ffffff;--sans: ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif;--mono: ui-monospace, "SF Mono", Menlo, Consolas, monospace;@@ -44,8 +42,6 @@--map-rail: #3a3832;--map-boundary: #4a463f;--map-label: #8f877a;- --hill-shadow: #000000;- --hill-light: #f3eee3;color-scheme: dark;}}diff --git a/tests/test_imagery.py b/tests/test_imagery.pyindex 593e479..1422f96 100644--- a/tests/test_imagery.py+++ b/tests/test_imagery.py@@ -198,9 +198,16 @@ class TerrainTests(unittest.TestCase):with (folder / "terrain.tif").open("rb") as stream:merged = terrain.TIFF(stream)self.assertEqual((merged.value(258), merged.value(317)), (16, 2))- # The dashboard decodes saved tiles itself and encodes them as Terrarium RGB.- with layers.heights(folder / "8.tif") as decoded, layers.terrarium(decoded) as rgb:+ # The dashboard decodes saved tiles itself to shade them.+ with layers.heights(folder / "8.tif") as decoded:self.assertEqual(list(struct.unpack(f"={512 * 512}f", decoded.tobytes())), heights)- pixels = rgb.tobytes()- self.assertEqual([pixels[i] * 256 + pixels[i + 1] + pixels[i + 2] / 256 - 32768- for i in range(0, len(pixels), 3)], heights)++ def test_hillshade_is_transparent_without_heights(self):+ # Overviews mix saved and missing heights; shading the missing ones drew straight lines on the map.+ center = Image.new("F", (8, 8))+ center.putdata([layers.NODATA if x < 4 else x * 10 for _ in range(8) for x in range(8)])+ with layers.hillshade(center, [None] * 4, 1, 1) as shaded, shaded.getchannel("A") as alpha:+ self.assertTrue(all(alpha.getpixel((x, y)) == 0 for x in range(5) for y in range(8)))+ self.assertTrue(all(alpha.getpixel((x, y)) > 0 for x in (5, 6) for y in range(1, 7)))+ with layers.hillshade(Image.new("F", (8, 8), 300), [None] * 4, 1, 1) as flat:+ self.assertEqual(flat.getchannel("A").getextrema(), (0, 0))