Skip to content

armgit / Aleph

Public Imported from GitHub
0

Give satellite imagery, terrain data, buildings, streetview photos to your LLM

Fix occasional garbage response from google

Belluxx58175488fb58
diff --git a/src/common.py b/src/common.py
index 3b51fe8..2824a55 100644
--- a/src/common.py
+++ b/src/common.py
@@ -119,7 +119,8 @@ class Client:
self.cancel()
def get(self, address, *, missing_ok=False, timeout=60, user_agent=BROWSER_AGENT,
- destination=None, progress=None):
+ destination=None, progress=None, check=None):
+ """Retry failed requests; check raises ValueError on a malformed response to retry it too."""
for attempt in range(4):
backoff = 2 ** (attempt - 1) if attempt else 0
until = time.monotonic() + max(backoff, self.delay - (time.monotonic() - self.finished))
@@ -147,8 +148,12 @@ class Client:
raise OSError("Incomplete download; retrying.")
self.check_cancel()
result = Path(destination)
+ if check:
+ check(result)
self.check_cancel()
return result
+ except MissingImagery:
+ raise
except Exception as error:
if self.request_log:
self.request_log("GET", address, attempt, error)
@@ -187,11 +192,11 @@ class CachedClient(Client):
return result
-def fetch(client, addresses, workers, *, missing_ok=False):
+def fetch(client, addresses, workers, *, missing_ok=False, check=None):
"""Yield responses in order with up to workers requests in flight; with missing_ok, None marks missing data."""
def get(address):
try:
- return client.get(address, missing_ok=missing_ok)
+ return client.get(address, missing_ok=missing_ok, check=check)
except MissingImagery:
return None
diff --git a/src/streetview.py b/src/streetview.py
index 94ba885..0d741e1 100644
--- a/src/streetview.py
+++ b/src/streetview.py
@@ -70,7 +70,7 @@ def coverage(client, area, progress, workers):
addresses = (url("https://www.google.com/maps/photometa/ac/v1",
pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")
for tile in tiles(coverage_grid))
- for i, data in enumerate(fetch(client, addresses, workers), 1):
+ for i, data in enumerate(fetch(client, addresses, workers, check=parse_coverage), 1):
for view in parse_coverage(data):
found[view["pano_id"]] = view
progress("Finding panoramas", i, count)
@@ -117,7 +117,11 @@ def metadata_url(pano_id):
def fetch_metadata(client, sample, *, full_sphere=False):
"""Refuse a panorama whose identity or position changed since it was found."""
- metadata = parse_metadata(client.get(metadata_url(sample["pano_id"])), full_sphere=full_sphere)
+ def parse(data):
+ return parse_metadata(data, full_sphere=full_sphere)
+
+ # Google occasionally answers with a malformed response; checking it inside get retries it.
+ metadata = parse(client.get(metadata_url(sample["pano_id"]), check=parse))
if metadata["pano_id"] != sample["pano_id"] or (
"lat" in sample and distance((sample["lat"], sample["lon"]), (metadata["lat"], metadata["lon"])) > 1):
raise ValueError("The panorama identity or position changed. Plan again or rerun with --refresh.")
diff --git a/tests/test_common.py b/tests/test_common.py
index 2ffe8d4..a629dac 100644
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -67,6 +67,19 @@ class DownloadTests(unittest.TestCase):
self.assertEqual(target.read_bytes(), b"previous snapshot")
self.assertEqual(list(Path(directory).iterdir()), [target])
+ def test_malformed_response_is_retried_until_it_checks_out(self):
+ def check(data):
+ if data != b"good":
+ raise ValueError("Unreadable response.")
+
+ with (
+ patch("src.common.urlopen", side_effect=[response(b"garbage", 7), response(b"good", 4)]) as get,
+ patch("src.common.time.sleep"),
+ patch("src.common.time.monotonic", side_effect=count(step=10)),
+ ):
+ self.assertEqual(Client().get("https://example.test/meta", check=check), b"good")
+ self.assertEqual(get.call_count, 2)
+
def test_containment_rejects_traversal_and_symlink_escape(self):
with TemporaryDirectory() as directory:
root = Path(directory) / "capture"