Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Handle missing satellite imagery

Belluxx4b27e239b3ed
diff --git a/src/capture.py b/src/capture.py
index f55bb91..a327831 100644
--- a/src/capture.py
+++ b/src/capture.py
@@ -264,7 +264,13 @@ def capture_satellite(stage, folder, client, progress):
if i < len(stage["results"]):
continue
address = f"https://mt1.google.com/vt/lyrs=s&x={tile['x']}&y={tile['y']}&z={tile['zoom']}"
- data = client.get(address)
+ try:
+ data = client.get(address, missing_ok=True)
+ except MissingImagery:
+ # Persist the gap so resume and offline export keep it transparent.
+ with BytesIO() as buffer, Image.new("RGBA", (256, 256)) as image:
+ image.save(buffer, format="PNG")
+ data = buffer.getvalue()
with open_image(data, (256, 256)) as image:
extension = "jpg" if image.format == "JPEG" else "png"
name = (
diff --git a/src/common.py b/src/common.py
index ea56b39..1759cd8 100644
--- a/src/common.py
+++ b/src/common.py
@@ -91,13 +91,11 @@ class Client:
self.request_log("POST" if data is not None else "GET", address, attempt, error)
if isinstance(error, HTTPError):
error.close()
+ if missing_ok and error.code == 404:
+ raise MissingImagery("The requested image is not available.") from error
if attempt < 3:
continue
if isinstance(error, HTTPError):
- if missing_ok and error.code == 404:
- raise MissingImagery(
- "The panorama image is no longer available."
- ) from error
raise OSError(
f"{urlsplit(address).hostname}: HTTP {error.code}. Resume to retry."
) from error
diff --git a/tests/test_capture.py b/tests/test_capture.py
index 240cbba..1ddc29f 100644
--- a/tests/test_capture.py
+++ b/tests/test_capture.py
@@ -1,8 +1,10 @@
import json
import tempfile
import unittest
+from io import BytesIO
from pathlib import Path
from unittest.mock import Mock, patch
+from urllib.error import HTTPError
from urllib.parse import parse_qs, urlsplit
from PIL import Image
@@ -137,6 +139,33 @@ class CaptureRecoveryTests(unittest.TestCase):
self.assertEqual([mosaic.getpixel(point) for point in ((0, 0), (255, 0), (0, 255), (255, 255))],
[(*color, 255) for color in colors])
+ def test_satellite_404_leaves_transparent_tile_without_retrying_on_resume(self):
+ north, west = geo.coordinate(384, 384, 2)
+ south, east = geo.coordinate(640, 640, 2)
+ run = capture.plan(self.client, [south, west, north, east],
+ dict(include=["satellite"], satellite_zoom=2), self.progress)
+ missing = HTTPError("https://mt1.google.com/vt/", 404, "Not Found", {}, BytesIO())
+ responses = [BytesIO(image_bytes((256, 256), "red")), missing,
+ BytesIO(image_bytes((256, 256), "blue")),
+ BytesIO(image_bytes((256, 256), "green"))]
+ with patch("src.common.urlopen", side_effect=responses) as get:
+ capture.download(run, self.folder, Client(), self.progress)
+ self.assertEqual(get.call_count, 4)
+ self.assertTrue(missing.fp.closed)
+ saved = capture.load(self.folder)
+ self.assertEqual(saved["state"], "complete")
+ original = (self.folder / "satellite.png").read_bytes()
+ with Image.open(self.folder / "satellite.png") as mosaic:
+ self.assertEqual(mosaic.size, (256, 256))
+ self.assertEqual(mosaic.getpixel((0, 0)), (255, 0, 0, 255))
+ self.assertEqual(mosaic.crop((128, 0, 256, 128)).getchannel("A").getextrema(), (0, 0))
+ self.assertEqual(mosaic.getpixel((0, 255)), (0, 0, 255, 255))
+ self.assertEqual(mosaic.getpixel((255, 255)), (0, 128, 0, 255))
+ self.client.get.reset_mock()
+ capture.download(saved, self.folder, self.client, self.progress)
+ self.client.get.assert_not_called()
+ self.assertEqual((self.folder / "satellite.png").read_bytes(), original)
+
def test_osm_failure_still_saves_terrain_and_resume_fetches_only_map(self):
run = capture.plan(self.client, [1, 1, 2, 2],
dict(include=["osm"], terrain_zoom=1), self.progress)
diff --git a/tests/test_common.py b/tests/test_common.py
index 7891ba0..8a274d6 100644
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -65,18 +65,19 @@ class NetworkTests(unittest.TestCase):
self.assertEqual(self.urlopen.call_count, 4)
self.assertEqual(self.sleeps, [3, 3, 4])
- def test_only_opted_in_404_becomes_missing_imagery_after_retries(self):
+ def test_only_opted_in_404_becomes_missing_imagery_without_retries(self):
for code, missing_ok, expected in ((404, True, common.MissingImagery), (404, False, OSError), (500, True, OSError)):
with self.subTest(code=code, missing_ok=missing_ok):
- errors = [HTTPError("https://example.test/tile", code, "unavailable", {}, BytesIO()) for _ in range(4)]
+ attempts = 1 if code == 404 and missing_ok else 4
+ errors = [HTTPError("https://example.test/tile", code, "unavailable", {}, BytesIO()) for _ in range(attempts)]
self.urlopen.reset_mock(side_effect=True)
self.urlopen.side_effect = errors
self.sleeps.clear()
with self.assertRaises(expected) as raised:
common.Client().get("https://example.test/tile", missing_ok=missing_ok)
self.assertIs(raised.exception.__cause__, errors[-1])
- self.assertEqual(self.urlopen.call_count, 4)
- self.assertEqual(self.sleeps, [1, 2, 4])
+ self.assertEqual(self.urlopen.call_count, attempts)
+ self.assertEqual(self.sleeps, [] if attempts == 1 else [1, 2, 4])
self.assertTrue(all(error.fp.closed for error in errors))
def test_cancellation_during_backoff_stops_before_another_request(self):