Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

8.2 KB
Raw Blame
"""Exercise the real HTTP/download/hash contract, including deadlines and concurrency."""
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import importlib.util
import json
from pathlib import Path
import tempfile
import threading
import time
import unittest

SCRIPT = Path(__file__).resolve().parents[1] / "verify-desktop-release.py"
spec = importlib.util.spec_from_file_location("verify_desktop", SCRIPT)
verify = importlib.util.module_from_spec(spec)
spec.loader.exec_module(verify)


class DesktopVerificationTests(unittest.TestCase):
    def setUp(self):
        self.payload = b"release artifact fixture\0" * 100
        self.requests = []
        self.active = 0
        self.peak = 0
        self.delay = 0.03
        self.drops_left = 0
        self.lock = threading.Lock()
        test = self

        class Handler(BaseHTTPRequestHandler):
            def log_message(self, *_):
                pass

            def do_GET(self):
                test.requests.append((self.path, self.headers.get("User-Agent")))
                if self.path == "/metadata.json":
                    payload = json.dumps(test.manifest).encode()
                elif self.path == "/missing":
                    self.send_error(404)
                    return
                elif self.path == "/flaky":
                    # The connection dropped with no answer, as Harness-macos.zip's did in 1.3.65's publish.
                    with test.lock:
                        test.drops_left -= 1
                        drop = test.drops_left >= 0
                    if drop:
                        self.close_connection = True
                        return
                    payload = test.payload
                else:
                    with test.lock:
                        test.active += 1
                        test.peak = max(test.peak, test.active)
                    time.sleep(test.delay)
                    with test.lock:
                        test.active -= 1
                    payload = test.payload
                self.send_response(200)
                self.send_header("Content-Length", str(len(payload)))
                self.end_headers()
                try:
                    self.wfile.write(payload)
                except (BrokenPipeError, ConnectionResetError):
                    pass

        self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
        self.thread.start()
        self.base = f"http://127.0.0.1:{self.server.server_port}"
        self.manifest = {
            key: dict(version="1.2.3", url=f"{self.base}/{key}",
                      size=len(self.payload), sha256=hashlib.sha256(self.payload).hexdigest())
            for key in verify.KEYS
        }

    def tearDown(self):
        self.server.shutdown()
        self.server.server_close()
        self.thread.join()

    def run_verification(self, **kwargs):
        return verify.verify("1.2.3", self.base + "/metadata.json", **kwargs)

    def test_all_nine_full_downloads_with_updater_agent_and_bounded_parallelism(self):
        receipt = self.run_verification(jobs=2)
        self.assertEqual(receipt["status"], "passed", receipt)
        self.assertEqual(len(receipt["artifacts"]), 9)
        self.assertEqual(self.peak, 2)
        self.assertEqual(len(self.requests), 10)
        self.assertTrue(all(agent == verify.USER_AGENT for _, agent in self.requests))
        self.assertTrue(all(item["actual_size"] == len(self.payload) for item in receipt["artifacts"]))

    def test_missing_or_mixed_version_fails_before_artifact_downloads(self):
        for change in ("missing", "version"):
            with self.subTest(change=change):
                original = self.manifest.pop(verify.KEYS[0])
                if change == "version":
                    self.manifest[verify.KEYS[0]] = dict(original, version="1.2.2")
                self.requests.clear()
                self.assertEqual(self.run_verification()["status"], "failed")
                self.assertEqual(len(self.requests), 1)
                self.manifest[verify.KEYS[0]] = original

    def test_bad_schema_fails_before_artifact_downloads(self):
        for patch in ({"size": True}, {"size": 0}, {"sha256": "bad"}, {"url": "file:///etc/passwd"}, {"url": "http://example.com/download"}):
            with self.subTest(patch=patch):
                original = self.manifest[verify.KEYS[0]]
                self.manifest[verify.KEYS[0]] = dict(original, **patch)
                self.requests.clear()
                self.assertEqual(self.run_verification()["status"], "failed")
                self.assertEqual(len(self.requests), 1)
                self.manifest[verify.KEYS[0]] = original

    def flaky_first_artifact(self, drops):
        self.manifest[verify.KEYS[0]]["url"] = f"{self.base}/flaky"
        self.drops_left = drops
        return self.run_verification()

    def test_a_dropped_connection_is_tried_again_and_passes(self):
        receipt = self.flaky_first_artifact(drops=2)
        self.assertEqual(receipt["status"], "passed")
        self.assertEqual(sum(path == "/flaky" for path, _ in self.requests), 3)

    def test_a_connection_that_keeps_dropping_fails_naming_the_download(self):
        receipt = self.flaky_first_artifact(drops=99)
        self.assertEqual(receipt["status"], "failed")
        self.assertEqual(sum(path == "/flaky" for path, _ in self.requests), 3)
        self.assertIn("download failed", receipt["artifacts"][0]["error"])

    def test_wrong_bytes_and_a_refused_download_are_not_tried_again(self):
        self.manifest[verify.KEYS[0]]["sha256"] = "0" * 64
        key = f"/{verify.KEYS[0]}"
        receipt = self.run_verification()
        self.assertEqual(receipt["artifacts"][0]["error"], "SHA-256 mismatch")
        self.assertEqual(sum(path == key for path, _ in self.requests), 1)
        self.requests.clear()
        self.manifest[verify.KEYS[0]]["url"] = f"{self.base}/missing"
        self.assertEqual(self.run_verification()["status"], "failed")
        self.assertEqual(sum(path == "/missing" for path, _ in self.requests), 1)

    def test_hash_size_and_http_failures_are_not_reported_as_passed(self):
        for patch in ({"sha256": "0" * 64}, {"size": len(self.payload) + 1}, {"size": len(self.payload) - 1}, {"url": self.base + "/missing"}):
            with self.subTest(patch=patch):
                original = self.manifest[verify.KEYS[0]]
                self.manifest[verify.KEYS[0]] = dict(original, **patch)
                receipt = self.run_verification()
                self.assertEqual(receipt["status"], "failed")
                self.assertEqual(receipt["artifacts"][0]["status"], "failed")
                self.assertEqual(sum(item["status"] == "passed" for item in receipt["artifacts"]), 8)
                self.manifest[verify.KEYS[0]] = original

    def test_transfer_deadline_fails_in_bounded_time(self):
        self.delay = 1.5
        start = time.monotonic()
        # One wave, as with six downloads: two waves of nine sat at the 1.3 s bound on a loaded Mac.
        receipt = self.run_verification(timeout=0.1, jobs=len(verify.KEYS))
        self.assertEqual(receipt["status"], "failed")
        self.assertLess(time.monotonic() - start, 1.3)

    def test_cli_preserves_failed_receipt_and_exit_status(self):
        self.manifest.pop(verify.KEYS[0])
        with tempfile.TemporaryDirectory() as folder:
            receipt = Path(folder) / "receipt.json"
            code = verify.main(["1.2.3", "--manifest-url", self.base + "/metadata.json", "--receipt", str(receipt)])
            self.assertEqual(code, 1)
            self.assertEqual(json.loads(receipt.read_text())["status"], "failed")

    def test_internal_macos_scope_keeps_all_four_and_release_still_requires_nine(self):
        macos = tuple(key for key in verify.KEYS if key.startswith("desktop-macos"))
        manifest = {key: self.manifest[key] for key in macos}
        self.assertEqual(len(verify.entries_for_version(manifest, "1.2.3", keys=macos)), 4)
        with self.assertRaises(ValueError):
            verify.entries_for_version(manifest, "1.2.3")
        manifest.pop(macos[-1])
        with self.assertRaises(ValueError):
            verify.entries_for_version(manifest, "1.2.3", keys=macos)


if __name__ == "__main__":
    unittest.main()