Give satellite imagery, terrain data, buildings, streetview photos to your LLM
Fix occasional garbage response from google
Belluxx58175488fb58
diff --git a/src/common.py b/src/common.pyindex 3b51fe8..2824a55 100644--- a/src/common.py+++ b/src/common.py@@ -119,7 +119,8 @@ class Client:self.cancel()def get(self, address, *, missing_ok=False, timeout=60, user_agent=BROWSER_AGENT,- destination=None, progress=None):+ destination=None, progress=None, check=None):+ """Retry failed requests; check raises ValueError on a malformed response to retry it too."""for attempt in range(4):backoff = 2 ** (attempt - 1) if attempt else 0until = time.monotonic() + max(backoff, self.delay - (time.monotonic() - self.finished))@@ -147,8 +148,12 @@ class Client:raise OSError("Incomplete download; retrying.")self.check_cancel()result = Path(destination)+ if check:+ check(result)self.check_cancel()return result+ except MissingImagery:+ raiseexcept Exception as error:if self.request_log:self.request_log("GET", address, attempt, error)@@ -187,11 +192,11 @@ class CachedClient(Client):return result-def fetch(client, addresses, workers, *, missing_ok=False):+def fetch(client, addresses, workers, *, missing_ok=False, check=None):"""Yield responses in order with up to workers requests in flight; with missing_ok, None marks missing data."""def get(address):try:- return client.get(address, missing_ok=missing_ok)+ return client.get(address, missing_ok=missing_ok, check=check)except MissingImagery:return Nonediff --git a/src/streetview.py b/src/streetview.pyindex 94ba885..0d741e1 100644--- a/src/streetview.py+++ b/src/streetview.py@@ -70,7 +70,7 @@ def coverage(client, area, progress, workers):addresses = (url("https://www.google.com/maps/photometa/ac/v1",pb=f"!1m1!1smaps_sv.tactile!6m3!1i{tile['x']}!2i{tile['y']}!3i17!8b1")for tile in tiles(coverage_grid))- for i, data in enumerate(fetch(client, addresses, workers), 1):+ for i, data in enumerate(fetch(client, addresses, workers, check=parse_coverage), 1):for view in parse_coverage(data):found[view["pano_id"]] = viewprogress("Finding panoramas", i, count)@@ -117,7 +117,11 @@ def metadata_url(pano_id):def fetch_metadata(client, sample, *, full_sphere=False):"""Refuse a panorama whose identity or position changed since it was found."""- metadata = parse_metadata(client.get(metadata_url(sample["pano_id"])), full_sphere=full_sphere)+ def parse(data):+ return parse_metadata(data, full_sphere=full_sphere)++ # Google occasionally answers with a malformed response; checking it inside get retries it.+ metadata = parse(client.get(metadata_url(sample["pano_id"]), check=parse))if metadata["pano_id"] != sample["pano_id"] or ("lat" in sample and distance((sample["lat"], sample["lon"]), (metadata["lat"], metadata["lon"])) > 1):raise ValueError("The panorama identity or position changed. Plan again or rerun with --refresh.")diff --git a/tests/test_common.py b/tests/test_common.pyindex 2ffe8d4..a629dac 100644--- a/tests/test_common.py+++ b/tests/test_common.py@@ -67,6 +67,19 @@ class DownloadTests(unittest.TestCase):self.assertEqual(target.read_bytes(), b"previous snapshot")self.assertEqual(list(Path(directory).iterdir()), [target])+ def test_malformed_response_is_retried_until_it_checks_out(self):+ def check(data):+ if data != b"good":+ raise ValueError("Unreadable response.")++ with (+ patch("src.common.urlopen", side_effect=[response(b"garbage", 7), response(b"good", 4)]) as get,+ patch("src.common.time.sleep"),+ patch("src.common.time.monotonic", side_effect=count(step=10)),+ ):+ self.assertEqual(Client().get("https://example.test/meta", check=check), b"good")+ self.assertEqual(get.call_count, 2)+def test_containment_rejects_traversal_and_symlink_escape(self):with TemporaryDirectory() as directory:root = Path(directory) / "capture"