Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Desktop release: a dropped download is tried again (#3111)

lingrd06d76c3a4018
diff --git a/scripts/tests/test_desktop_verification.py b/scripts/tests/test_desktop_verification.py
index 7e197206c..2c46735b6 100644
--- a/scripts/tests/test_desktop_verification.py
+++ b/scripts/tests/test_desktop_verification.py
@@ -22,6 +22,7 @@ class DesktopVerificationTests(unittest.TestCase):
self.active = 0
self.peak = 0
self.delay = 0.03
+ self.drops_left = 0
self.lock = threading.Lock()
test = self
@@ -36,6 +37,15 @@ class DesktopVerificationTests(unittest.TestCase):
elif self.path == "/missing":
self.send_error(404)
return
+ elif self.path == "/flaky":
+ # The connection dropped with no answer, as Harness-macos.zip's did in 1.3.65's publish.
+ with test.lock:
+ test.drops_left -= 1
+ drop = test.drops_left >= 0
+ if drop:
+ self.close_connection = True
+ return
+ payload = test.payload
else:
with test.lock:
test.active += 1
@@ -100,6 +110,33 @@ class DesktopVerificationTests(unittest.TestCase):
self.assertEqual(len(self.requests), 1)
self.manifest[verify.KEYS[0]] = original
+ def flaky_first_artifact(self, drops):
+ self.manifest[verify.KEYS[0]]["url"] = f"{self.base}/flaky"
+ self.drops_left = drops
+ return self.run_verification()
+
+ def test_a_dropped_connection_is_tried_again_and_passes(self):
+ receipt = self.flaky_first_artifact(drops=2)
+ self.assertEqual(receipt["status"], "passed")
+ self.assertEqual(sum(path == "/flaky" for path, _ in self.requests), 3)
+
+ def test_a_connection_that_keeps_dropping_fails_naming_the_download(self):
+ receipt = self.flaky_first_artifact(drops=99)
+ self.assertEqual(receipt["status"], "failed")
+ self.assertEqual(sum(path == "/flaky" for path, _ in self.requests), 3)
+ self.assertIn("download failed", receipt["artifacts"][0]["error"])
+
+ def test_wrong_bytes_and_a_refused_download_are_not_tried_again(self):
+ self.manifest[verify.KEYS[0]]["sha256"] = "0" * 64
+ key = f"/{verify.KEYS[0]}"
+ receipt = self.run_verification()
+ self.assertEqual(receipt["artifacts"][0]["error"], "SHA-256 mismatch")
+ self.assertEqual(sum(path == key for path, _ in self.requests), 1)
+ self.requests.clear()
+ self.manifest[verify.KEYS[0]]["url"] = f"{self.base}/missing"
+ self.assertEqual(self.run_verification()["status"], "failed")
+ self.assertEqual(sum(path == "/missing" for path, _ in self.requests), 1)
+
def test_hash_size_and_http_failures_are_not_reported_as_passed(self):
for patch in ({"sha256": "0" * 64}, {"size": len(self.payload) + 1}, {"size": len(self.payload) - 1}, {"url": self.base + "/missing"}):
with self.subTest(patch=patch):
diff --git a/scripts/verify-desktop-release.py b/scripts/verify-desktop-release.py
index 97ad188de..1dfe1db0e 100644
--- a/scripts/verify-desktop-release.py
+++ b/scripts/verify-desktop-release.py
@@ -25,6 +25,12 @@ KEYS = (
)
MANIFEST = "https://storage.googleapis.com/s3-autonomous-upgrade-3/harness/desktop/metadata.json"
USER_AGENT = "Dart/3.13 (dart:io)"
+# curl's exit codes for a connection lost, not an answer: could not connect, partial file, TLS or
+# handshake reset, empty reply, send error, receive error. A reset on one of nine downloads failed
+# 1.3.65's publish after its manifest was already live.
+CONNECTION_DROPPED = frozenset({7, 18, 35, 52, 55, 56})
+DOWNLOAD_TRIES = 3
+RETRY_PAUSE_SECONDS = 2
def utc_now():
@@ -77,6 +83,35 @@ def entries_for_version(manifest, version, keys=KEYS):
return entries
+class ConnectionDropped(ValueError):
+ """The network dropped the download. Nothing was said about the file, so it is worth another try."""
+
+
+def download(entry, timeout):
+ """The file's size and digest. A timeout is not retried: it would break the deadline this check promises."""
+ digest = hashlib.sha256()
+ size = 0
+ with subprocess.Popen(
+ curl_command(entry["url"], timeout) + ["--max-filesize", str(entry["size"])],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ ) as process:
+ try:
+ for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""):
+ size += len(chunk)
+ digest.update(chunk)
+ error = process.stderr.read().decode(errors="replace").strip()
+ code = process.wait()
+ if code in CONNECTION_DROPPED:
+ raise ConnectionDropped(f"download failed: {error}")
+ if code != 0:
+ raise ValueError(f"download failed: {error}")
+ finally:
+ if process.poll() is None:
+ process.kill()
+ process.wait()
+ return size, digest
+
+
def verify_artifact(item, timeout):
key, entry = item
started = time.monotonic()
@@ -85,21 +120,14 @@ def verify_artifact(item, timeout):
digest = hashlib.sha256()
size = 0
try:
- with subprocess.Popen(
- curl_command(entry["url"], timeout) + ["--max-filesize", str(entry["size"])],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- ) as process:
+ for attempt in range(1, DOWNLOAD_TRIES + 1):
try:
- for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""):
- size += len(chunk)
- digest.update(chunk)
- error = process.stderr.read().decode(errors="replace").strip()
- if process.wait() != 0:
- raise ValueError(f"download failed: {error}")
- finally:
- if process.poll() is None:
- process.kill()
- process.wait()
+ size, digest = download(entry, timeout)
+ break
+ except ConnectionDropped:
+ if attempt == DOWNLOAD_TRIES:
+ raise
+ time.sleep(attempt * RETRY_PAUSE_SECONDS)
if size != entry["size"]:
raise ValueError(f"size mismatch: expected {entry['size']}, received {size}")
if digest.hexdigest() != entry["sha256"]: