Give satellite imagery, terrain data, buildings, streetview photos to your LLM
"""Check exported pixels, placement, and the custom TIFF binary layouts."""
import math
import struct
import unittest
import zlib
from io import BytesIO
from itertools import accumulate
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from PIL import Image
from alephgeo import geo, layers, satellite, terrain
def quiet(*args):
pass
class SatelliteTests(unittest.TestCase):
def test_tile_aligned_bounds_do_not_add_extra_pixels_or_tiles(self):
for zoom, x, y in ((1, 0, 0), (1, 1, 1), (17, 23456, 43210), (21, 2097151, 2097151)):
with self.subTest(zoom=zoom, x=x, y=y):
north, west = geo.coordinate(x * 256, y * 256, zoom)
south, east = geo.coordinate((x + 1) * 256, (y + 1) * 256, zoom)
self.assertEqual(geo.grid(geo.corners((south, west, north, east)), zoom), dict(
zoom=zoom, left=x * 256, top=y * 256, width=256, height=256,
x0=x, y0=y, columns=1, rows=1, spans=[[0, 0, 1, 0]], count=1))
north, west = geo.coordinate(1024 - 0.01, 1280 - 0.01, 21)
south, east = geo.coordinate(1280 + 0.01, 1536 + 0.01, 21)
self.assertEqual(geo.grid(geo.corners((south, west, north, east)), 21), dict(
zoom=21, left=1023, top=1279, width=258, height=258,
x0=3, y0=4, columns=3, rows=3, spans=[[0, 0, 3, 0], [1, 0, 3, 3], [2, 0, 3, 6]], count=9))
def test_saved_tiles_follow_spans_with_gaps(self):
# Row 0 holds columns 0–2, row 1 none, and row 2 columns 0 and 2: tiles 0 to 4 in download order.
grid = dict(zoom=2, x0=0, y0=0, columns=3, rows=3, spans=[[0, 0, 3, 0], [2, 0, 1, 3], [2, 2, 3, 4]], count=5)
for count, z, x, y, expected in [(5, 2, 2, 2, 5), (4, 2, 2, 2, 0), (5, 2, 1, 2, 0), (5, 2, 1, 1, 0),
(5, 1, 0, 0, 2), (5, 1, 0, 1, 4), (3, 1, 0, 1, 0), (5, 1, 1, 1, 5),
(4, 0, 0, 0, 4), (2, 0, 0, 0, 2), (0, 0, 0, 0, 0)]:
with self.subTest(count=count, tile=(z, x, y)):
self.assertEqual(layers.saved(grid, count, z, x, y), expected)
def test_partial_mosaic_crops_pixels_and_writes_georeferenced_overview(self):
grid = dict(zoom=4, left=1027, top=1287, width=301, height=259,
x0=4, y0=5, columns=2, rows=2, spans=[[0, 0, 2, 0], [1, 0, 2, 2]], count=4)
(north, west), (south, east) = geo.coordinate(1027, 1287, 4), geo.coordinate(1328, 1546, 4)
area = geo.corners((south, west, north, east))
with TemporaryDirectory() as directory, Image.new("RGBA", (512, 512)) as canvas:
folder = Path(directory)
results = []
for i in range(3): # Fourth tile has not been downloaded yet.
row, column = divmod(i, 2)
with Image.new("RGBA", (256, 256)) as image:
image.putdata([(x, y, i * 60, 200) for y in range(256) for x in range(256)])
image.save(folder / f"{i}.png")
canvas.paste(image, (column * 256, row * 256))
results.append(dict(x=4 + column, y=5 + row, zoom=4, filename=f"{i}.png"))
stage = dict(grid=grid, results=results)
with canvas.crop((3, 7, 304, 266)) as expected:
# Force BigTIFF using a small file; never allocate gigabytes.
for limit, magic in ((2**32, b"II*\0"), (1, b"II+\0")):
with self.subTest(format=magic), patch("alephgeo.satellite.TIFF_LIMIT", limit):
satellite.merge(stage, folder, quiet, area)
self.assertEqual((folder / "satellite.tif").read_bytes()[:4], magic)
with Image.open(folder / "satellite.png") as png:
self.assertEqual(png.size, expected.size)
self.assertEqual(png.tobytes(), expected.tobytes())
with Image.open(folder / "satellite.tif") as tif:
self.assertEqual(tif.size, expected.size)
self.assertEqual(tif.tobytes(), expected.tobytes())
self.assertEqual(tif.n_frames, 2)
scale = 2 * math.pi * 6_378_137 / 4096
self.assertEqual(tif.tag_v2[33550], (scale, scale, 0))
self.assertEqual(tif.tag_v2[33922], (
0, 0, 0, 1027 * scale - math.pi * 6_378_137,
math.pi * 6_378_137 - 1287 * scale, 0))
self.assertIn(3857, tif.tag_v2[34735])
full_resolution_start = min(tif.tag_v2[324])
tif.seek(1)
self.assertEqual(tif.size, (151, 130))
self.assertEqual(tif.tag_v2[254], 1)
self.assertLess(max(tif.tag_v2[324]), full_resolution_start)
with expected.resize((151, 130), Image.Resampling.BOX) as overview:
self.assertEqual(tif.tobytes(), overview.tobytes())
def float_tile(order, predictor, x=8):
"""Synthetic 512px source; predictor oracle is independent of block splitting."""
raw = b"".join(struct.pack(order + "f", ((i * 257 + x * 131) % 10000 - 5000) / 37)
for i in range(512 * 512))
if predictor == 3:
encoded = bytearray()
for y in range(512):
row = raw[y * 2048:(y + 1) * 2048]
# Planes start from the most significant byte, whatever the file's byte order.
planes = b"".join(row[i::4] for i in ((0, 1, 2, 3) if order == ">" else (3, 2, 1, 0)))
encoded.extend(bytes([planes[0]]) + bytes((b - a) & 255 for a, b in zip(planes, planes[1:])))
data = zlib.compress(encoded)
else:
data = zlib.compress(raw)
scale = 2 * math.pi * 6_378_137 / 16 / 512
tags = {}
def tag(code, kind, values):
tags[code] = kind, len(values), struct.pack(order + terrain.TYPES[kind] * len(values), *values)
for code, value in {256: 512, 257: 512, 258: 32, 259: 8, 262: 1, 277: 1,
317: predictor, 322: 512, 323: 512, 339: 3}.items():
tag(code, 3, [value])
tag(33550, 12, [scale, scale, 0])
tag(33922, 12, [0, 0, 0, -math.pi * 6_378_137 + x * 512 * scale,
math.pi * 6_378_137 - 5 * 512 * scale, 0])
tag(34735, 3, [1, 1, 0, 2, 1025, 0, 1, 1, 3072, 0, 1, 3857])
tag(42113, 2, [bytes([v]) for v in b"-9999\0"])
tag(324, 4, [0])
tag(325, 4, [len(data)])
tag(324, 4, [len(terrain.header(tags, order))])
return terrain.header(tags, order) + data, raw
def int16_tile(order, x=8):
"""Synthetic 512px Int16 source as used up to zoom 12: four 256px blocks, predictor 2."""
heights = [(i * 37 + x * 11) % 9000 - 500 for i in range(512 * 512)]
blocks = []
for by, bx in ((0, 0), (0, 1), (1, 0), (1, 1)):
encoded = b""
for y in range(by * 256, by * 256 + 256):
row = heights[y * 512 + bx * 256:y * 512 + bx * 256 + 256]
encoded += struct.pack(f"{order}256H", *((b - a) & 0xFFFF for a, b in zip([0] + row, row)))
blocks.append(zlib.compress(encoded))
scale = 2 * math.pi * 6_378_137 / 16 / 512
tags = {}
def tag(code, kind, values):
tags[code] = kind, len(values), struct.pack(order + terrain.TYPES[kind] * len(values), *values)
for code, value in {256: 512, 257: 512, 258: 16, 259: 8, 262: 1, 277: 1,
317: 2, 322: 256, 323: 256, 339: 2}.items():
tag(code, 3, [value])
tag(33550, 12, [scale, scale, 0])
tag(33922, 12, [0, 0, 0, -math.pi * 6_378_137 + x * 512 * scale,
math.pi * 6_378_137 - 5 * 512 * scale, 0])
tag(34735, 3, [1, 1, 0, 2, 1025, 0, 1, 1, 3072, 0, 1, 3857])
tag(42113, 2, [bytes([v]) for v in b"-32768\0"])
tag(324, 4, [0] * 4)
tag(325, 4, [len(block) for block in blocks])
cursor = len(terrain.header(tags, order))
tag(324, 4, list(accumulate([cursor] + [len(block) for block in blocks[:-1]])))
return terrain.header(tags, order) + b"".join(blocks), heights
def decode_float_block(data, predictor, order):
raw = zlib.decompress(data)
if predictor != 3:
return raw
decoded = bytearray()
for y in range(256):
planes = bytes(v & 255 for v in accumulate(raw[y * 1024:(y + 1) * 1024]))
planes = [planes[i * 256:(i + 1) * 256] for i in range(4)]
decoded.extend(v for pixel in zip(*(planes if order == ">" else planes[::-1])) for v in pixel)
return bytes(decoded)
class TerrainTests(unittest.TestCase):
def test_merge_preserves_float_pixels_when_splitting_predictor_blocks(self):
for order, predictor in (("<", 1), ("<", 3), (">", 3)):
with self.subTest(order=order, predictor=predictor), TemporaryDirectory() as directory:
folder = Path(directory)
results, originals = [], []
for x in (8, 9):
data, raw = float_tile(order, predictor, x)
(folder / f"{x}.tif").write_bytes(data)
originals.append(raw)
results.append(dict(x=x, y=5, zoom=4, filename=f"{x}.tif"))
terrain.merge(folder / "terrain.tif", folder, results,
dict(columns=2, rows=1, x0=8, y0=5, zoom=4), quiet)
with (folder / "terrain.tif").open("rb") as stream:
merged = terrain.TIFF(stream)
self.assertEqual((merged.value(256), merged.value(257)), (1024, 512))
self.assertEqual(merged.order, order)
self.assertEqual(merged.value(317), predictor)
for i, block in enumerate(merged.blocks()):
row, column = divmod(i, 4)
original = originals[column // 2]
left = column % 2 * 1024
expected = b"".join(original[y * 2048 + left:y * 2048 + left + 1024]
for y in range(row * 256, (row + 1) * 256))
self.assertEqual(decode_float_block(block, predictor, order), expected)
self.assertEqual(len(merged.values(324)), 8)
# The dashboard decodes saved tiles itself; prediction differences running across planes spiked heights.
for x, raw in zip((8, 9), originals):
with layers.heights(folder / f"{x}.tif") as decoded:
self.assertEqual(decoded.tobytes(), struct.pack(f"={512 * 512}f",
*struct.unpack(f"{order}{512 * 512}f", raw)))
def test_truncated_or_misplaced_terrain_is_rejected(self):
data, _ = float_tile("<", 3)
for payload, x in ((data[:-1], 8), (data, 9)):
with self.subTest(truncated=len(payload) < len(data), x=x), self.assertRaises(ValueError):
terrain.validate(terrain.TIFF(BytesIO(payload)), dict(x=x, y=5, zoom=4))
def test_int16_terrain_merges_and_decodes(self):
# Sources up to zoom 12 are Int16 with horizontal differencing, not Float32.
for order in ("<", ">"):
with self.subTest(order=order), TemporaryDirectory() as directory:
folder = Path(directory)
data, heights = int16_tile(order)
(folder / "8.tif").write_bytes(data)
terrain.validate(terrain.TIFF(BytesIO(data)), dict(x=8, y=5, zoom=4))
terrain.merge(folder / "terrain.tif", folder, [dict(x=8, y=5, zoom=4, filename="8.tif")],
dict(columns=1, rows=1, x0=8, y0=5, zoom=4), quiet)
with (folder / "terrain.tif").open("rb") as stream:
merged = terrain.TIFF(stream)
self.assertEqual((merged.value(258), merged.value(317)), (16, 2))
# The dashboard decodes saved tiles itself to shade them.
with layers.heights(folder / "8.tif") as decoded:
self.assertEqual(list(struct.unpack(f"={512 * 512}f", decoded.tobytes())), heights)
def test_hillshade_is_transparent_without_heights(self):
# Overviews mix saved and missing heights; shading the missing ones drew straight lines on the map.
center = Image.new("F", (8, 8))
center.putdata([layers.NODATA if x < 4 else x * 10 for _ in range(8) for x in range(8)])
with layers.hillshade(center, [None] * 4, 1, 1) as shaded, shaded.getchannel("A") as alpha:
self.assertTrue(all(alpha.getpixel((x, y)) == 0 for x in range(5) for y in range(8)))
self.assertTrue(all(alpha.getpixel((x, y)) > 0 for x in (5, 6) for y in range(1, 7)))
with layers.hillshade(Image.new("F", (8, 8), 300), [None] * 4, 1, 1) as flat:
self.assertEqual(flat.getchannel("A").getextrema(), (0, 0))