Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Fix terrain shading and decode

Belluxxb142666c9e49
diff --git a/README.md b/README.md
index 0dc9321..1fc958f 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ This was born as a way to let LLMs quickly gather information from the physical
## Install
-Requires Python 3.11+. Pillow is the only dependency. No API keys are required.
+Requires Python 3.11+. The only dependencies are Pillow and numpy. No API keys are required.
```sh
git clone https://github.com/Belluxx/Aleph.git
diff --git a/pyproject.toml b/pyproject.toml
index 313e0be..dfa1c19 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ version = "0.1.0"
description = "Capture Street View photos, satellite imagery, and OpenStreetMap data."
readme = "README.md"
requires-python = ">=3.11"
-dependencies = ["Pillow>=10"]
+dependencies = ["Pillow>=10", "numpy>=1.25"]
[project.scripts]
alephgeo = "aleph:main"
diff --git a/src/dashboard.py b/src/dashboard.py
index 5de3ec1..80f2a98 100644
--- a/src/dashboard.py
+++ b/src/dashboard.py
@@ -123,7 +123,7 @@ def summary(identity, run, live):
if stage["mode"] in ("streetview", "mesh"):
info["skipped"] = sum(result.get("status") == "skipped" for result in results[:count])
if "grid" in stage:
- info["zoom"] = stage["grid"]["zoom"]
+ info.update(zoom=stage["grid"]["zoom"], minzoom=layers.lowest(stage))
state = "interrupted" if run["state"] == "running" and not live else run["state"]
return dict(id=identity, bounds=run["bounds"], started_at=run["started_at"], finished_at=run.get("finished_at"),
state=state, error=run.get("error"), options=run["options"], stages=stages)
@@ -282,11 +282,12 @@ class Dashboard:
def tile(self, identity, mode, z, x, y):
folder, stage, _ = self.stage(identity, mode)
- tile = self.tiles.get(folder, stage, int(z), int(x), int(y.split(".")[0]))
+ z = int(z)
+ tile = self.tiles.get(folder, stage, z, int(x), int(y.split(".")[0]))
if tile is None:
raise FileNotFoundError("No saved tile here.")
- # Tile URLs carry the saved count, so a response never changes.
- return *tile, 86400
+ # Saved patches never change. Built tiles are not cached, so rendering changes show up at once.
+ return *tile, 86400 if mode == "satellite" and z == stage["grid"]["zoom"] else 0
def file(self, identity, path):
target = contained(self.folder(identity), "/".join(path))
@@ -303,7 +304,7 @@ class Dashboard:
case "GET", [("app.js" | "style.css") as name]:
return (WEB / name).read_bytes(), mimetypes.guess_type(name)[0], 0
case "GET", ["api", "config"]:
- return dict(root=str(self.root), defaults=capture.DEFAULT_OPTIONS, overviews=layers.OVERVIEWS)
+ return dict(root=str(self.root), defaults=capture.DEFAULT_OPTIONS)
case "GET", ["api", "job"]:
return self.job_state()
case "GET", ["api", "preview"]:
@@ -325,8 +326,10 @@ class Dashboard:
return self.photos(identity, int(query.get("after", ["0"])[0]))
case "GET", ["api", "captures", identity, "osm"]:
return self.osm_layer(identity)
- case "GET", ["api", "captures", identity, ("satellite" | "terrain") as mode, z, x, y]:
- return self.tile(identity, mode, z, x, y)
+ case "GET", ["api", "captures", identity, "satellite", z, x, y]:
+ return self.tile(identity, "satellite", z, x, y)
+ case "GET", ["api", "captures", identity, "hillshade", z, x, y]:
+ return self.tile(identity, "terrain", z, x, y)
case "GET", ["api", "captures", identity, "files", *path] if path:
return self.file(identity, path)
case "POST", ["api", "estimate"]:
diff --git a/src/layers.py b/src/layers.py
index 0b19279..7e4dc07 100644
--- a/src/layers.py
+++ b/src/layers.py
@@ -1,23 +1,28 @@
"""Browser-ready views of saved captures: imagery and terrain tiles, and OSM as GeoJSON."""
import math
-import struct
-import sys
import threading
import zlib
from collections import OrderedDict, defaultdict
from io import BytesIO
-from itertools import accumulate
from xml.etree import ElementTree as ET
+import numpy as np
from PIL import Image
from . import terrain
from .common import contained
-from .geo import collection, feature, ring_contains, signed_area
+from .geo import MERCATOR_RADIUS, collection, coordinate, feature, ring_contains, signed_area
CACHE_BYTES = 384 * 2**20
-OVERVIEWS = 6 # Zoom levels built below a grid's own zoom.
+OVERVIEWS = dict(satellite=6, terrain=8) # Most zoom levels built below a grid's own zoom.
+# Missing heights. Overviews average it in, which keeps those pixels far below any real height.
+NODATA = -1e6
+NEIGHBORS = ((-1, 0), (1, 0), (0, -1), (0, 1)) # West, east, north, south.
+# Opacity per unit of brightness change from flat ground, and its maximum. Shadows span a much wider range.
+SHADOW = 160, 120
+HIGHLIGHT = 400, 120
+RELIEF = 4 # Overview levels per doubling of slope exaggeration.
AREA_TAGS = {
"building", "landuse", "amenity", "leisure", "boundary", "place", "shop",
@@ -57,61 +62,67 @@ def saved(grid, count, z, x, y):
return row * columns + end if row >= top else 0
-def unpredict(data, width):
- """Undo the TIFF floating point predictor, giving big-endian Float32 rows."""
- rows = bytearray()
- for y in range(0, len(data), width * 4):
- # Each row holds byte differences across four planes, most significant first.
- planes = bytes(v & 255 for v in accumulate(data[y:y + width * 4]))
- row = bytearray(width * 4)
- for plane in range(4):
- row[plane::4] = planes[plane * width:(plane + 1) * width]
- rows += row
- return bytes(rows)
-
-
-def undifference(data, width, order):
- """Undo the TIFF horizontal predictor for 16-bit samples."""
- fmt = f"{order}{width}H"
- return b"".join(struct.pack(fmt, *(v & 0xFFFF for v in accumulate(struct.unpack(fmt, data[y:y + width * 2]))))
- for y in range(0, len(data), width * 2))
+def lowest(stage):
+ """Lowest zoom with tiles: a few levels past where the grid fits in one tile, within the overview limit."""
+ grid = stage["grid"]
+ levels = math.ceil(math.log2(max(grid["columns"], grid["rows"]))) + 3
+ return max(0, grid["zoom"] - min(levels, OVERVIEWS[stage["mode"]]))
def heights(path):
"""Decode one saved 512-pixel Int16 or Float32 terrain tile without libtiff."""
with path.open("rb") as stream:
source = terrain.TIFF(stream)
- size = source.value(322)
+ size, order, predictor = source.value(322), source.order, source.value(317, 1)
per_row = 512 // size
- image = Image.new("F", (512, 512))
+ values = np.empty((512, 512), np.float32)
for i, (offset, length) in enumerate(zip(source.values(324), source.values(325))):
data = source.read(offset, length)
if source.value(259) != 1:
data = zlib.decompress(data)
- big = source.order == ">"
- if source.value(317, 1) == 3:
- data, big = unpredict(data, size), True
- elif source.value(317, 1) == 2:
- data = undifference(data, size, source.order)
- mode, raw_mode = ("F", "F;32BF" if big else "F;32F") if source.float else ("I", "I;16BS" if big else "I;16S")
- with (
- Image.frombytes(mode, (size, size), data, "raw", raw_mode) as block,
- block.convert("F") as values,
- ):
- image.paste(values, (i % per_row * size, i // per_row * size))
+ if source.float and predictor == 3:
+ # Rows hold byte differences across four planes, most significant first.
+ planes = np.cumsum(np.frombuffer(data, np.uint8).reshape(size, 4, size), axis=2, dtype=np.uint8)
+ block = planes.transpose(0, 2, 1).copy().view(">f4")
+ elif source.float:
+ block = np.frombuffer(data, order + "f4")
+ else:
+ block = np.frombuffer(data, order + "u2").reshape(size, size)
+ if predictor == 2:
+ block = np.cumsum(block, axis=1, dtype=np.uint16)
+ block = block.astype(np.int16)
+ y, x = divmod(i, per_row)
+ values[y * size:(y + 1) * size, x * size:(x + 1) * size] = block.reshape(size, size)
nodata = float(b"".join(source.values(42113)).rstrip(b"\0"))
- # Terrarium has no NoData value. Unknown heights use sea level.
- image.putdata([v if math.isfinite(v) and v != nodata else 0
- for (v,) in struct.iter_unpack("=f", image.tobytes())])
- return image
-
-
-def terrarium(image):
- """Encode heights as RGB, where meters = R * 256 + G + B / 256 - 32768."""
- with image.point(lambda h: h * 256 + 32768 * 256) as scaled, scaled.convert("I") as values:
- # Native 32-bit integers hold the three channels below an unused high byte.
- raw = "BGRX" if sys.byteorder == "little" else "XRGB"
- return Image.frombytes("RGB", image.size, values.tobytes(), "raw", raw)
+ values[~np.isfinite(values) | (values == nodata)] = NODATA
+ return Image.fromarray(values, "F")
+
+
+def hillshade(center, neighbors, meters_per_pixel, exaggeration):
+ """Shadows and highlights lit from the northwest, transparent where heights are missing."""
+ center = np.asarray(center, np.float32)
+ # Without a neighbor, continue the slope at the edge so adjacent tiles still meet smoothly.
+ padded = np.pad(center, 1, mode="reflect", reflect_type="odd")
+ # Neighbors' facing edges, where known, keep slopes continuous across tiles.
+ for (dx, dy), neighbor in zip(NEIGHBORS, neighbors):
+ if neighbor is None:
+ continue
+ neighbor = np.asarray(neighbor, np.float32)
+ target, edge = {(-1, 0): (np.s_[1:-1, 0], np.s_[:, -1]), (1, 0): (np.s_[1:-1, -1], np.s_[:, 0]),
+ (0, -1): (np.s_[0, 1:-1], np.s_[-1, :]), (0, 1): (np.s_[-1, 1:-1], np.s_[0, :])}[dx, dy]
+ padded[target] = np.where(neighbor[edge] > NODATA / 2, neighbor[edge], padded[target])
+ scale = exaggeration / (2 * meters_per_pixel)
+ east = (padded[1:-1, 2:] - padded[1:-1, :-2]) * scale
+ south = (padded[2:, 1:-1] - padded[:-2, 1:-1]) * scale
+ # Light comes from x west, y north, 45° up: (-0.5, -0.5, 0.7071) with x east and y south.
+ shade = (east * 0.5 + south * 0.5 + 0.7071) / np.sqrt(east * east + south * south + 1) - 0.7071
+ (dark, darkest), (light, lightest) = SHADOW, HIGHLIGHT
+ alpha = np.minimum(np.maximum(-shade, 0) * dark, darkest) + np.minimum(np.maximum(shade, 0) * light, lightest)
+ # Slopes next to a missing height are unreliable, so the known area shrinks by a pixel inside the tile.
+ known = np.pad(center > NODATA / 2, 1, mode="edge")
+ inner = np.logical_and.reduce([known[y:y + center.shape[0], x:x + center.shape[1]] for y in range(3) for x in range(3)])
+ tone = np.where(shade > 0, 255, 0).astype(np.uint8)
+ return Image.fromarray(np.dstack((tone, tone, tone, np.where(inner, alpha, 0).astype(np.uint8))), "RGBA")
def png(image):
@@ -127,11 +138,13 @@ class Tiles:
self.cache = OrderedDict()
self.used = 0
self.lock = threading.Lock()
+ # One build per layer at a time: parallel builds of overlapping overviews would repeat the same decoding.
+ self.builds = defaultdict(threading.Lock)
def get(self, folder, stage, z, x, y):
"""Return (bytes, content type), or None where nothing is saved yet."""
grid = stage["grid"]
- if not grid["zoom"] - OVERVIEWS <= z <= grid["zoom"] or not (0 <= x < 2**z and 0 <= y < 2**z):
+ if not lowest(stage) <= z <= grid["zoom"] or not (0 <= x < 2**z and 0 <= y < 2**z):
return None
if stage["mode"] == "satellite" and z == grid["zoom"]:
# Saved patches are already browser images.
@@ -140,25 +153,39 @@ class Tiles:
return None
path = contained(folder, stage["results"][version - 1]["filename"])
return path.read_bytes(), "image/png" if path.suffix == ".png" else "image/jpeg"
- image = self.image(folder, stage, z, x, y)
+ with self.lock:
+ build = self.builds[folder, stage["mode"]]
+ with build:
+ image = self.image(folder, stage, z, x, y)
if image is None:
return None
if stage["mode"] == "terrain":
- with terrarium(image) as rgb:
- return png(rgb), "image/png"
+ # Building neighbors would cost as much as the tile itself; edges are extrapolated without them.
+ neighbors = [self.cached(folder, stage, z, x + dx, y + dy) for dx, dy in NEIGHBORS]
+ latitude = math.radians(coordinate((x + 0.5) * 256, (y + 0.5) * 256, z)[0])
+ meters = 2 * math.pi * MERCATOR_RADIUS * math.cos(latitude) / (512 * 2**z)
+ # Zoomed out, averaged heights flatten slopes; exaggerate them to keep relief visible.
+ with hillshade(image, neighbors, meters, 2 ** ((grid["zoom"] - z) / RELIEF)) as shaded:
+ return png(shaded), "image/png"
return png(image), "image/png"
+ def cached(self, folder, stage, z, x, y):
+ version = saved(stage["grid"], len(stage["results"]), z, x, y)
+ with self.lock:
+ entry = self.cache.get((folder, stage["mode"], z, x, y))
+ if entry and version and entry[0] == version:
+ self.cache.move_to_end((folder, stage["mode"], z, x, y))
+ return entry[1]
+ return None
+
def image(self, folder, stage, z, x, y):
grid, results = stage["grid"], stage["results"]
version = saved(grid, len(results), z, x, y)
if not version:
return None
key = folder, stage["mode"], z, x, y
- with self.lock:
- entry = self.cache.get(key)
- if entry and entry[0] == version:
- self.cache.move_to_end(key)
- return entry[1]
+ if (image := self.cached(folder, stage, z, x, y)) is not None:
+ return image
elevation = stage["mode"] == "terrain"
if z == grid["zoom"]:
path = contained(folder, results[version - 1]["filename"])
@@ -170,22 +197,13 @@ class Tiles:
else:
size = 512 if elevation else 256
half = size // 2
- image = Image.new("F" if elevation else "RGBA", (size, size))
- missing, means = [], []
+ image = Image.new("F", (size, size), NODATA) if elevation else Image.new("RGBA", (size, size))
for dy in range(2):
for dx in range(2):
child = self.image(folder, stage, z + 1, 2 * x + dx, 2 * y + dy)
- if child is None:
- missing.append((dx * half, dy * half))
- continue
- with child.resize((half, half), Image.Resampling.BOX) as smaller:
- image.paste(smaller, (dx * half, dy * half))
- if elevation:
- with smaller.resize((1, 1), Image.Resampling.BOX) as mean:
- means.append(mean.getpixel((0, 0)))
- # Sea level outside the grid would draw cliffs at its edges; use the tile's mean height.
- for left, top in missing if means else ():
- image.paste(sum(means) / len(means), (left, top, left + half, top + half))
+ if child is not None:
+ with child.resize((half, half), Image.Resampling.BOX) as smaller:
+ image.paste(smaller, (dx * half, dy * half))
with self.lock:
previous = self.cache.pop(key, None)
if previous:
@@ -193,7 +211,9 @@ class Tiles:
self.cache[key] = version, image
self.used += image.width * image.height * 4
while self.used > CACHE_BYTES:
- _, (_, dropped) = self.cache.popitem(last=False)
+ # Finer tiles are cheapest to rebuild, so the least recently used of them go first.
+ victim = max(self.cache, key=lambda cached: cached[2])
+ _, dropped = self.cache.pop(victim)
self.used -= dropped.width * dropped.height * 4
return image
diff --git a/src/web/app.js b/src/web/app.js
index 2b77e97..35b52a3 100644
--- a/src/web/app.js
+++ b/src/web/app.js
@@ -288,19 +288,19 @@ function setData(source, data) {
// Rasters are added once a capture has saved tiles; their URLs carry the saved count.
function setRaster(mode, info) {
- const tiles = [absolute(capturePath(state.capture.id, `/${mode}/{z}/{x}/{y}?v=${info.done}`))];
+ const path = mode === "terrain" ? "hillshade" : mode;
+ const tiles = [absolute(capturePath(state.capture.id, `/${path}/{z}/{x}/{y}?v=${info.done}`))];
const source = map.getSource(mode);
if (source) return source.setTiles(tiles);
const [s, w, n, e] = state.capture.bounds;
- const spec = {tiles, bounds: [w, s, e, n], minzoom: Math.max(0, info.zoom - state.config.overviews), maxzoom: info.zoom};
+ const spec = {tiles, bounds: [w, s, e, n], minzoom: info.minzoom, maxzoom: info.zoom};
if (mode === "satellite") {
map.addSource(mode, {type: "raster", tileSize: 256, ...spec});
addLayer({id: "satellite", type: "raster", source: mode, paint: {"raster-fade-duration": 0}}, "osm-land");
} else {
- map.addSource(mode, {type: "raster-dem", encoding: "terrarium", tileSize: 512, ...spec});
- addLayer({id: "hillshade", type: "hillshade", source: mode, paint: {
- "hillshade-shadow-color": "--hill-shadow", "hillshade-highlight-color": "--hill-light",
- "hillshade-accent-color": "--hill-shadow", "hillshade-exaggeration": 0.55}}, "osm-land");
+ // The server shades terrain itself, so tiles stay transparent where heights are missing.
+ map.addSource(mode, {type: "raster", tileSize: 512, ...spec});
+ addLayer({id: "hillshade", type: "raster", source: mode, paint: {"raster-fade-duration": 0}}, "osm-land");
}
syncMap();
}
@@ -620,7 +620,7 @@ function refresh(stages, force = false) {
if (!force && info.done < info.total && Date.now() - (live[`${mode}At`] || 0) < 2500) continue;
live[mode] = info.done;
live[`${mode}At`] = Date.now();
- setRaster(mode, {...info, zoom: info.zoom ?? state.capture.stages[mode].zoom});
+ setRaster(mode, {...state.capture.stages[mode], ...info});
}
if (stages.osm?.done && !live.osm) {
live.osm = true;
diff --git a/src/web/style.css b/src/web/style.css
index 7b15abc..103b62e 100644
--- a/src/web/style.css
+++ b/src/web/style.css
@@ -17,8 +17,6 @@
--map-rail: #d3c8b5;
--map-boundary: #c9bda9;
--map-label: #8a7f70;
- --hill-shadow: #34302b;
- --hill-light: #ffffff;
--sans: ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif;
--mono: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
@@ -44,8 +42,6 @@
--map-rail: #3a3832;
--map-boundary: #4a463f;
--map-label: #8f877a;
- --hill-shadow: #000000;
- --hill-light: #f3eee3;
color-scheme: dark;
}
}
diff --git a/tests/test_imagery.py b/tests/test_imagery.py
index 593e479..1422f96 100644
--- a/tests/test_imagery.py
+++ b/tests/test_imagery.py
@@ -198,9 +198,16 @@ class TerrainTests(unittest.TestCase):
with (folder / "terrain.tif").open("rb") as stream:
merged = terrain.TIFF(stream)
self.assertEqual((merged.value(258), merged.value(317)), (16, 2))
- # The dashboard decodes saved tiles itself and encodes them as Terrarium RGB.
- with layers.heights(folder / "8.tif") as decoded, layers.terrarium(decoded) as rgb:
+ # The dashboard decodes saved tiles itself to shade them.
+ with layers.heights(folder / "8.tif") as decoded:
self.assertEqual(list(struct.unpack(f"={512 * 512}f", decoded.tobytes())), heights)
- pixels = rgb.tobytes()
- self.assertEqual([pixels[i] * 256 + pixels[i + 1] + pixels[i + 2] / 256 - 32768
- for i in range(0, len(pixels), 3)], heights)
+
+ def test_hillshade_is_transparent_without_heights(self):
+ # Overviews mix saved and missing heights; shading the missing ones drew straight lines on the map.
+ center = Image.new("F", (8, 8))
+ center.putdata([layers.NODATA if x < 4 else x * 10 for _ in range(8) for x in range(8)])
+ with layers.hillshade(center, [None] * 4, 1, 1) as shaded, shaded.getchannel("A") as alpha:
+ self.assertTrue(all(alpha.getpixel((x, y)) == 0 for x in range(5) for y in range(8)))
+ self.assertTrue(all(alpha.getpixel((x, y)) > 0 for x in (5, 6) for y in range(1, 7)))
+ with layers.hillshade(Image.new("F", (8, 8), 300), [None] * 4, 1, 1) as flat:
+ self.assertEqual(flat.getchannel("A").getextrema(), (0, 0))