Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Support Int16 terrain tiles

Belluxx8e4b93d561ae
diff --git a/src/capture.py b/src/capture.py
index ef61c00..ce33cdc 100644
--- a/src/capture.py
+++ b/src/capture.py
@@ -32,7 +32,7 @@ OSM_NOTES = {
"selection": "Selected ways are complete. Selected multipolygons include all available outer boundaries and holes, but members missing from the regional file remain unresolved. Other relations may also be incomplete. Objects may extend outside the rectangle; crossings without inside nodes and enclosing polygons may be absent.",
}
TERRAIN_NOTES = {
- "terrain": "Float32 heights in meters, EPSG:3857, without resampling. Full edge tiles extend beyond the rectangle. Tiles are checkpointed individually; terrain.tif is built after all terrain tiles are saved.",
+ "terrain": "Heights in meters (Int16 up to zoom 12, Float32 from zoom 13), EPSG:3857, without resampling. Full edge tiles extend beyond the rectangle. Tiles are checkpointed individually; terrain.tif is built after all terrain tiles are saved.",
"quality": "Terrain resolution, dates, accuracy and vertical reference vary by source. Higher zoom does not guarantee more detail.",
}
@@ -47,7 +47,7 @@ satellite.tif is a lossless RGBA Cloud Optimized GeoTIFF in EPSG:3857.
It is north-up, cropped to enclosing pixels, with internal overviews.
satellite.png contains the same full-resolution pixels for easy viewing.
Transparent pixels mark missing imagery or unfinished downloads.
-terrain.tif is Float32 meters in EPSG:3857, without resampling.
+terrain.tif holds meters in EPSG:3857 (Int16 up to zoom 12, else Float32).
Keep terrain/tiles/ for resume and offline export.
Filenames in the manifest and GeoJSON are relative to this run folder.
See manifest.json for settings, sources, imagery dates, and coverage notes.
diff --git a/src/dashboard_tiles.py b/src/dashboard_tiles.py
index 00c53f6..8d3a503 100644
--- a/src/dashboard_tiles.py
+++ b/src/dashboard_tiles.py
@@ -25,8 +25,15 @@ def unpredict(data, width):
return bytes(rows)
+def undifference(data, width, order):
+ """Undo the TIFF horizontal predictor for 16-bit samples."""
+ fmt = f"{order}{width}H"
+ return b"".join(struct.pack(fmt, *(v & 0xFFFF for v in accumulate(struct.unpack(fmt, data[y:y + width * 2]))))
+ for y in range(0, len(data), width * 2))
+
+
def terrain_tile(path):
- """Decode one saved 512-pixel terrain tile without libtiff."""
+ """Decode one saved 512-pixel Int16 or Float32 terrain tile without libtiff."""
with path.open("rb") as stream:
source = terrain.TIFF(stream)
size = source.value(322)
@@ -36,12 +43,17 @@ def terrain_tile(path):
data = source.read(offset, length)
if source.value(259) != 1:
data = zlib.decompress(data)
+ big = source.order == ">"
if source.value(317, 1) == 3:
- data, raw_mode = unpredict(data, size), "F;32BF"
- else:
- raw_mode = "F;32F" if source.order == "<" else "F;32BF"
- with Image.frombytes("F", (size, size), data, "raw", raw_mode) as block:
- image.paste(block, (i % per_row * size, i // per_row * size))
+ data, big = unpredict(data, size), True
+ elif source.value(317, 1) == 2:
+ data = undifference(data, size, source.order)
+ mode, raw_mode = ("F", "F;32BF" if big else "F;32F") if source.float else ("I", "I;16BS" if big else "I;16S")
+ with (
+ Image.frombytes(mode, (size, size), data, "raw", raw_mode) as block,
+ block.convert("F") as heights,
+ ):
+ image.paste(heights, (i % per_row * size, i // per_row * size))
nodata = float(b"".join(source.values(42113)).rstrip(b"\0"))
# RGB elevation has no NoData representation. Unknown heights use sea level.
image.putdata([v if math.isfinite(v) and v != nodata else 0
diff --git a/src/terrain.py b/src/terrain.py
index 904d39f..a923011 100644
--- a/src/terrain.py
+++ b/src/terrain.py
@@ -1,4 +1,7 @@
-"""Join Float32 GeoTIFFs, copying 256-pixel blocks and splitting larger ones."""
+"""Join terrain GeoTIFFs, copying 256-pixel blocks and splitting larger ones.
+
+Sources use Int16 heights up to zoom 12 and Float32 from zoom 13.
+"""
import math
import struct
@@ -34,12 +37,14 @@ class TIFF:
size = struct.calcsize(self.order + TYPES[kind]) * length
raw = entry[8 : 8 + size] if size <= 4 else self.read(location, size)
self.tags[code] = kind, length, raw
- required = {258: 32, 339: 3, 277: 1, 262: 1}
- if any(self.value(key) != value for key, value in required.items()) or self.value(259) not in (1, 8, 32946):
- raise ValueError("Expected tiled Float32 terrain.")
- if (self.value(322), self.value(323)) not in ((256, 256), (512, 512)):
- raise ValueError("Expected 256- or 512-pixel terrain blocks.")
- if self.value(274, 1) != 1 or self.value(284, 1) != 1 or self.value(317, 1) not in (1, 3):
+ self.float = (self.value(258), self.value(339)) == (32, 3)
+ if ((not self.float and (self.value(258), self.value(339)) != (16, 2))
+ or self.value(277) != 1 or self.value(262) != 1 or self.value(259) not in (1, 8, 32946)):
+ raise ValueError("Expected tiled Float32 or Int16 terrain.")
+ # Only Float32 sources use 512-pixel blocks, which blocks() splits.
+ if (self.value(322), self.value(323)) not in ((256, 256), (512, 512) if self.float else (256, 256)):
+ raise ValueError("Unexpected terrain block size.")
+ if self.value(274, 1) != 1 or self.value(284, 1) != 1 or self.value(317, 1) not in ((1, 3) if self.float else (1, 2)):
raise ValueError("Unsupported terrain layout.")
if any(key not in self.tags for key in (33550, 33922, 34735, 42113, 324, 325)):
raise ValueError("Missing terrain georeferencing or blocks.")
diff --git a/tests/test_imagery.py b/tests/test_imagery.py
index b018c87..2b8c48e 100644
--- a/tests/test_imagery.py
+++ b/tests/test_imagery.py
@@ -12,7 +12,7 @@ from unittest.mock import patch
from PIL import Image
-from src import geo, satellite, terrain
+from src import dashboard_tiles, geo, satellite, terrain
def quiet(*args):
@@ -110,6 +110,37 @@ def float_tile(order, predictor, x=8):
return terrain.header(tags, order) + data, raw
+def int16_tile(order, x=8):
+ """Synthetic 512px Int16 source as used up to zoom 12: four 256px blocks, predictor 2."""
+ heights = [(i * 37 + x * 11) % 9000 - 500 for i in range(512 * 512)]
+ blocks = []
+ for by, bx in ((0, 0), (0, 1), (1, 0), (1, 1)):
+ encoded = b""
+ for y in range(by * 256, by * 256 + 256):
+ row = heights[y * 512 + bx * 256:y * 512 + bx * 256 + 256]
+ encoded += struct.pack(f"{order}256H", *((b - a) & 0xFFFF for a, b in zip([0] + row, row)))
+ blocks.append(zlib.compress(encoded))
+ scale = 2 * math.pi * 6_378_137 / 16 / 512
+ tags = {}
+
+ def tag(code, kind, values):
+ tags[code] = kind, len(values), struct.pack(order + terrain.TYPES[kind] * len(values), *values)
+
+ for code, value in {256: 512, 257: 512, 258: 16, 259: 8, 262: 1, 277: 1,
+ 317: 2, 322: 256, 323: 256, 339: 2}.items():
+ tag(code, 3, [value])
+ tag(33550, 12, [scale, scale, 0])
+ tag(33922, 12, [0, 0, 0, -math.pi * 6_378_137 + x * 512 * scale,
+ math.pi * 6_378_137 - 5 * 512 * scale, 0])
+ tag(34735, 3, [1, 1, 0, 2, 1025, 0, 1, 1, 3072, 0, 1, 3857])
+ tag(42113, 2, [bytes([v]) for v in b"-32768\0"])
+ tag(324, 4, [0] * 4)
+ tag(325, 4, [len(block) for block in blocks])
+ cursor = len(terrain.header(tags, order))
+ tag(324, 4, list(accumulate([cursor] + [len(block) for block in blocks[:-1]])))
+ return terrain.header(tags, order) + b"".join(blocks), heights
+
+
def decode_float_block(data, predictor):
raw = zlib.decompress(data)
if predictor != 3:
@@ -153,3 +184,19 @@ class TerrainTests(unittest.TestCase):
for payload, x in ((data[:-1], 8), (data, 9)):
with self.subTest(truncated=len(payload) < len(data), x=x), self.assertRaises(ValueError):
terrain.validate(terrain.TIFF(BytesIO(payload)), dict(x=x, y=5, zoom=4))
+
+ def test_int16_terrain_merges_and_decodes(self):
+ # Sources up to zoom 12 are Int16 with horizontal differencing, not Float32.
+ for order in ("<", ">"):
+ with self.subTest(order=order), TemporaryDirectory() as directory:
+ folder = Path(directory)
+ data, heights = int16_tile(order)
+ (folder / "8.tif").write_bytes(data)
+ terrain.validate(terrain.TIFF(BytesIO(data)), dict(x=8, y=5, zoom=4))
+ terrain.merge(folder / "terrain.tif", folder, [dict(x=8, y=5, zoom=4, filename="8.tif")],
+ dict(columns=1, rows=1, x0=8, y0=5, zoom=4), quiet)
+ with (folder / "terrain.tif").open("rb") as stream:
+ merged = terrain.TIFF(stream)
+ self.assertEqual((merged.value(258), merged.value(317)), (16, 2))
+ decoded = dashboard_tiles.terrain_tile(folder / "8.tif").tobytes()
+ self.assertEqual(list(struct.unpack(f"={512 * 512}f", decoded)), heights)