Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(tools): hash-verified qpack installers for Hugging Face and static mirrors
Léon Avic Simmonse547494ebf67
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.mdindex 01a7a2ce..673e5d5f 100644--- a/THIRD_PARTY_NOTICES.md+++ b/THIRD_PARTY_NOTICES.md@@ -59,3 +59,13 @@ text.manifest-last completion policy from Swiftlet's `StreamingInstaller.swift` atcommit `86246618ba2af30334227e09ff84a6a7182c2a40`. It is a newtransport-neutral Python implementation and does not copy Swift source code.++`c/tools/qpack_http_install.py` is a new Python HTTP frontend over that policy.+It interoperates with Hugging Face-hosted Swiftlet qpack repositories but does+not copy Swiftlet or huggingface_hub source code.++`c/tools/qpack_mirror_install.py` and the static-mirror support in+`c/tools/qpack_http_install.py` interoperate with the `hashes.json` schema+emitted by Swiftlet's `scripts/verify_container.py`. They are new Python+implementations with stricter path, transport, credential, resume, and digest+validation; no Swiftlet source code is copied.diff --git a/c/tests/fixtures/swiftlet_qpack_manifest.json b/c/tests/fixtures/swiftlet_qpack_manifest.jsonnew file mode 100644index 00000000..0f881dbe--- /dev/null+++ b/c/tests/fixtures/swiftlet_qpack_manifest.json@@ -0,0 +1,57 @@+{+ "files" : {+ "chat_template.jinja" : 7764,+ "config.json" : 23591,+ "model.safetensors" : 1389579123,+ "packed_experts/layer_00.bin" : 452984832,+ "packed_experts/layer_01.bin" : 452984832,+ "packed_experts/layer_02.bin" : 452984832,+ "packed_experts/layer_03.bin" : 452984832,+ "packed_experts/layer_04.bin" : 452984832,+ "packed_experts/layer_05.bin" : 452984832,+ "packed_experts/layer_06.bin" : 452984832,+ "packed_experts/layer_07.bin" : 452984832,+ "packed_experts/layer_08.bin" : 452984832,+ "packed_experts/layer_09.bin" : 452984832,+ "packed_experts/layer_10.bin" : 452984832,+ "packed_experts/layer_11.bin" : 452984832,+ "packed_experts/layer_12.bin" : 452984832,+ "packed_experts/layer_13.bin" : 452984832,+ "packed_experts/layer_14.bin" : 452984832,+ "packed_experts/layer_15.bin" : 452984832,+ "packed_experts/layer_16.bin" : 452984832,+ "packed_experts/layer_17.bin" : 452984832,+ "packed_experts/layer_18.bin" : 452984832,+ "packed_experts/layer_19.bin" : 452984832,+ "packed_experts/layer_20.bin" : 452984832,+ "packed_experts/layer_21.bin" : 452984832,+ "packed_experts/layer_22.bin" : 452984832,+ "packed_experts/layer_23.bin" : 452984832,+ "packed_experts/layer_24.bin" : 452984832,+ "packed_experts/layer_25.bin" : 452984832,+ "packed_experts/layer_26.bin" : 452984832,+ "packed_experts/layer_27.bin" : 452984832,+ "packed_experts/layer_28.bin" : 452984832,+ "packed_experts/layer_29.bin" : 452984832,+ "packed_experts/layer_30.bin" : 452984832,+ "packed_experts/layer_31.bin" : 452984832,+ "packed_experts/layer_32.bin" : 452984832,+ "packed_experts/layer_33.bin" : 452984832,+ "packed_experts/layer_34.bin" : 452984832,+ "packed_experts/layer_35.bin" : 452984832,+ "packed_experts/layer_36.bin" : 452984832,+ "packed_experts/layer_37.bin" : 452984832,+ "packed_experts/layer_38.bin" : 452984832,+ "packed_experts/layer_39.bin" : 452984832,+ "packed_experts/layout.json" : 2048,+ "tokenizer.json" : 19989343,+ "tokenizer_config.json" : 1139,+ "vocab.json" : 6722759+ },+ "magic" : "QPACK",+ "modelName" : "qwen3_5_moe_text",+ "quantBits" : 4,+ "quantGroupSize" : 64,+ "sourceCheckpoint" : "qwen3.6-35b-mlx4bit",+ "version" : 1+}diff --git a/c/tests/test_qpack_http_install.py b/c/tests/test_qpack_http_install.pynew file mode 100644index 00000000..3f347531--- /dev/null+++ b/c/tests/test_qpack_http_install.py@@ -0,0 +1,532 @@+import hashlib+import http.client+import json+import tempfile+import unittest+from pathlib import Path+from urllib.request import Request++from tools.qpack_http_install import (+ STATE_NAME,+ QpackHTTPError,+ SafeRedirectHandler,+ _check_adapter_namespace,+ _declared_auxiliary_names,+ install_hf_qpack,+)+from tools.qpack_install_policy import parse_manifest+++COMMIT = "0123456789abcdef0123456789abcdef01234567"+++class Headers:+ def __init__(self, values=()):+ self.values = list(values)++ def get_all(self, name, default=None):+ matches = [value for key, value in self.values+ if key.lower() == name.lower()]+ return matches if matches else (default or [])++ def items(self):+ return iter(self.values)+++class Response:+ def __init__(self, status, body=b"", headers=()):+ self.status = status+ self.body = body+ self.headers = Headers(headers)+ self.offset = 0+ self.closed = False++ def read(self, size=-1):+ if size is None or size < 0:+ size = len(self.body) - self.offset+ chunk = self.body[self.offset:self.offset + size]+ self.offset += len(chunk)+ return chunk++ def close(self):+ self.closed = True+++class IncompleteResponse(Response):+ def read(self, _size=-1):+ if self.offset == 0:+ self.offset = len(self.body)+ raise http.client.IncompleteRead(self.body)+ return b""+++class ScriptedTransport:+ def __init__(self, *responses):+ self.responses = list(responses)+ self.calls = []++ def request(self, method, url, headers, timeout):+ self.calls.append({+ "method": method,+ "url": url,+ "headers": dict(headers),+ "timeout": timeout,+ })+ if not self.responses:+ raise AssertionError(f"unexpected request: {method} {url}")+ response = self.responses.pop(0)+ if callable(response):+ return response(self.calls[-1])+ return response++ def assert_empty(self):+ if self.responses:+ raise AssertionError(f"{len(self.responses)} scripted responses unused")+++class QpackHTTPInstallTest(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name) / "tiny.qpack"+ self.files = {+ "model.safetensors": b"weights",+ "packed_experts/layout.json": b"layout",+ }+ self.config = b'{"model_type":"qwen3_next"}'+ self.manifest = self.make_manifest(+ {name: len(value) for name, value in self.files.items()})++ @staticmethod+ def make_manifest(files):+ return json.dumps({+ "magic": "QPACK",+ "version": 1,+ "modelName": "tiny-qwen",+ "sourceCheckpoint": f"owner/model@{COMMIT}",+ "quantBits": 4,+ "quantGroupSize": 32,+ "files": files,+ }, sort_keys=True, separators=(",", ":")).encode()++ @staticmethod+ def response(body, *, status=200, etag=None, checksum=False,+ content_range=None, extra=()):+ headers = [("Content-Length", str(len(body))),+ ("X-Repo-Commit", COMMIT)]+ if etag is not None:+ headers.append(("ETag", etag))+ if checksum:+ headers.append(("X-Checksum-Sha256",+ hashlib.sha256(body).hexdigest()))+ if content_range is not None:+ headers.append(("Content-Range", content_range))+ headers.extend(extra)+ return Response(status, body, headers)++ def manifest_response(self):+ return self.response(self.manifest, etag='"manifest"', checksum=True)++ def config_response(self, body=None, status=200):+ body = self.config if body is None else body+ return self.response(body, status=status, etag='"config"', checksum=True)++ def artifact_response(self, name, *, offset=0, status=None, etag=None,+ body=None, extra=()):+ value = self.files[name]+ body = value[offset:] if body is None else body+ status = status if status is not None else (206 if offset else 200)+ content_range = None+ if status == 206:+ content_range = f"bytes {offset}-{len(value) - 1}/{len(value)}"+ checksum = hashlib.sha256(value).hexdigest()+ headers = [("Content-Length", str(len(body))),+ ("X-Repo-Commit", COMMIT),+ ("ETag", etag or f'"{name}"'),+ ("X-Checksum-Sha256", checksum)]+ if content_range is not None:+ headers.append(("Content-Range", content_range))+ headers.extend(extra)+ return Response(status, body, headers)++ def success_responses(self, include_config=True):+ responses = [self.manifest_response()]+ if include_config:+ responses.append(self.config_response())+ responses.extend(self.artifact_response(name)+ for name in sorted(self.files))+ return responses++ def install(self, transport, *, root=None, revision=COMMIT, retries=0,+ token=None, optional_aux=()):+ return install_hf_qpack(+ "owner/model", revision, root or self.root,+ endpoint="https://hub.example", token=token,+ transport=transport, retries=retries, timeout=7,+ chunk_size=3, optional_aux=optional_aux,+ sleep=lambda _seconds: None,+ log=lambda _message: None)++ def test_resolves_revision_then_uses_only_commit_pinned_urls(self):+ api = Response(200, json.dumps({"sha": COMMIT}).encode(),+ [("Content-Length", str(len(json.dumps({"sha": COMMIT}))))])+ transport = ScriptedTransport(api, *self.success_responses())++ result = self.install(transport, revision="main", token="secret-token")++ self.assertEqual(result.commit, COMMIT)+ self.assertEqual(result.downloaded_files, 3)+ self.assertEqual(result.downloaded_bytes,+ len(self.config) + sum(map(len, self.files.values())))+ self.assertTrue((self.root / "manifest.json").is_file())+ self.assertEqual((self.root / "config.json").read_bytes(), self.config)+ for name, value in self.files.items():+ self.assertEqual((self.root / name).read_bytes(), value)+ self.assertIn("/api/models/owner/model/revision/main",+ transport.calls[0]["url"])+ for call in transport.calls[1:]:+ self.assertIn(f"/resolve/{COMMIT}/", call["url"])+ self.assertEqual(call["headers"]["Authorization"],+ "Bearer secret-token")+ state = (self.root / STATE_NAME).read_text(encoding="utf-8")+ self.assertIn(f"owner/model@{COMMIT}", state)+ self.assertNotIn("secret-token", state)+ transport.assert_empty()++ def test_short_body_retries_with_range_and_if_range(self):+ first = self.artifact_response(+ "model.safetensors", body=self.files["model.safetensors"][:3])+ # The declared response length is the full object, but EOF arrives early.+ first.headers = Headers([+ ("Content-Length", str(len(self.files["model.safetensors"]))),+ ("X-Repo-Commit", COMMIT),+ ("ETag", '"model.safetensors"'),+ ("X-Checksum-Sha256",+ hashlib.sha256(self.files["model.safetensors"]).hexdigest()),+ ])+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(), first,+ self.artifact_response("model.safetensors", offset=3),+ self.artifact_response("packed_experts/layout.json"))++ result = self.install(transport, retries=1)++ model_calls = [call for call in transport.calls+ if call["url"].endswith("/model.safetensors")]+ self.assertEqual(model_calls[0]["headers"]["Range"], "bytes=0-6")+ self.assertNotIn("If-Range", model_calls[0]["headers"])+ self.assertEqual(model_calls[1]["headers"]["Range"], "bytes=3-6")+ self.assertEqual(model_calls[1]["headers"]["If-Range"],+ '"model.safetensors"')+ self.assertEqual(result.resumed_files, 1)+ self.assertEqual((self.root / "model.safetensors").read_bytes(),+ self.files["model.safetensors"])+ transport.assert_empty()++ def test_incomplete_read_partial_bytes_are_resumed(self):+ value = self.files["model.safetensors"]+ first = IncompleteResponse(200, value[:3], [+ ("Content-Length", str(len(value))),+ ("ETag", '"model.safetensors"'),+ ("X-Checksum-Sha256", hashlib.sha256(value).hexdigest()),+ ])+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(), first,+ self.artifact_response("model.safetensors", offset=3),+ self.artifact_response("packed_experts/layout.json"))++ self.install(transport, retries=1)++ model_calls = [call for call in transport.calls+ if call["url"].endswith("/model.safetensors")]+ self.assertEqual(model_calls[1]["headers"]["Range"], "bytes=3-6")+ self.assertEqual((self.root / "model.safetensors").read_bytes(), value)++ def test_resume_rejects_http_200_and_keeps_existing_prefix(self):+ first = self.artifact_response(+ "model.safetensors", body=self.files["model.safetensors"][:3])+ first.headers = Headers([+ ("Content-Length", str(len(self.files["model.safetensors"]))),+ ("ETag", '"model.safetensors"'),+ ])+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(), first)+ with self.assertRaisesRegex(QpackHTTPError, "short HTTP body"):+ self.install(transport)+ state = json.loads((self.root / STATE_NAME).read_text())+ self.assertEqual(state["files"]["model.safetensors"]["etag"],+ '"model.safetensors"')++ resumed = self.artifact_response(+ "model.safetensors", offset=3, status=200)+ transport = ScriptedTransport(self.manifest_response(), resumed)+ with self.assertRaisesRegex(QpackHTTPError, "did not return HTTP 206"):+ self.install(transport)+ self.assertEqual((self.root / "model.safetensors.part").read_bytes(),+ self.files["model.safetensors"][:3])++ def test_resume_rejects_changed_etag_before_appending(self):+ first = self.artifact_response(+ "model.safetensors", body=self.files["model.safetensors"][:3])+ first.headers = Headers([+ ("Content-Length", str(len(self.files["model.safetensors"]))),+ ("ETag", '"original"'),+ ])+ with self.assertRaises(QpackHTTPError):+ self.install(ScriptedTransport(+ self.manifest_response(), self.config_response(), first))++ changed = self.artifact_response(+ "model.safetensors", offset=3, etag='"changed"')+ with self.assertRaisesRegex(QpackHTTPError, "ETag changed"):+ self.install(ScriptedTransport(self.manifest_response(), changed))+ self.assertEqual((self.root / "model.safetensors.part").read_bytes(),+ self.files["model.safetensors"][:3])++ def test_checksum_mismatch_resets_partial_and_never_publishes_manifest(self):+ response = self.artifact_response("model.safetensors")+ response.headers = Headers([+ ("Content-Length", str(len(self.files["model.safetensors"]))),+ ("X-Checksum-Sha256", "f" * 64),+ ])+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(), response)++ with self.assertRaisesRegex(QpackHTTPError, "SHA-256 mismatch"):+ self.install(transport)++ self.assertEqual((self.root / "model.safetensors.part").stat().st_size, 0)+ self.assertFalse((self.root / "manifest.json").exists())++ def test_range_and_encoding_protocol_errors_fail_closed(self):+ cases = {+ "missing-range": Response(206, self.files["model.safetensors"], [+ ("Content-Length", "7"),+ ]),+ "wrong-range": Response(206, self.files["model.safetensors"], [+ ("Content-Length", "7"), ("Content-Range", "bytes 1-6/7"),+ ]),+ "range-on-200": Response(200, self.files["model.safetensors"], [+ ("Content-Length", "7"), ("Content-Range", "bytes 0-6/7"),+ ]),+ "duplicate-length": Response(200, self.files["model.safetensors"], [+ ("Content-Length", "7"), ("Content-Length", "7"),+ ]),+ "compressed": Response(200, self.files["model.safetensors"], [+ ("Content-Length", "7"), ("Content-Encoding", "gzip"),+ ]),+ }+ for label, response in cases.items():+ with self.subTest(label=label):+ root = Path(self.temporary.name) / f"{label}.qpack"+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(), response)+ with self.assertRaises(QpackHTTPError):+ self.install(transport, root=root)+ self.assertFalse((root / "manifest.json").exists())++ def test_required_config_must_exist_and_be_valid_before_weights(self):+ cases = [+ self.config_response(body=b"", status=404),+ self.config_response(body=b"not-json"),+ self.config_response(body=b"[]"),+ ]+ for index, response in enumerate(cases):+ with self.subTest(index=index):+ root = Path(self.temporary.name) / f"config-{index}.qpack"+ transport = ScriptedTransport(self.manifest_response(), response)+ with self.assertRaises(QpackHTTPError):+ self.install(transport, root=root)+ self.assertEqual(len(transport.calls), 2)+ self.assertFalse((root / "manifest.json").exists())++ def test_manifest_declared_auxiliaries_use_the_resumable_artifact_path(self):+ tokenizer = b'{"version":"1.0"}'+ files = dict(self.files)+ files["config.json"] = self.config+ files["tokenizer.json"] = tokenizer+ raw = self.make_manifest({name: len(value) for name, value in files.items()})+ responses = [self.response(raw)]+ for name in ["config.json", "tokenizer.json", "model.safetensors",+ "packed_experts/layout.json"]:+ value = files[name]+ responses.append(Response(200, value, [+ ("Content-Length", str(len(value))),+ ("ETag", f'"{name}"'),+ ("X-Checksum-Sha256", hashlib.sha256(value).hexdigest()),+ ]))+ transport = ScriptedTransport(*responses)++ self.install(transport)++ self.assertEqual((self.root / "config.json").read_bytes(), self.config)+ self.assertEqual((self.root / "tokenizer.json").read_bytes(), tokenizer)+ artifact_calls = [call["url"].rsplit("/", 1)[-1]+ for call in transport.calls[1:]]+ self.assertEqual(artifact_calls[0], "config.json")+ transport.assert_empty()++ def test_optional_auxiliary_generator_is_not_consumed_by_validation(self):+ tokenizer = b'{"version":"1.0"}'+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(),+ self.response(tokenizer, checksum=True),+ *(self.artifact_response(name) for name in sorted(self.files)))++ self.install(+ transport,+ optional_aux=(name for name in ("tokenizer.json",)))++ self.assertEqual((self.root / "tokenizer.json").read_bytes(), tokenizer)+ self.assertTrue(transport.calls[2]["url"].endswith("/tokenizer.json"))+ transport.assert_empty()++ def test_real_swiftlet_manifest_declared_auxiliaries_are_accepted(self):+ fixture = Path(__file__).parent / "fixtures" / "swiftlet_qpack_manifest.json"+ plan = parse_manifest(fixture.read_bytes(), f"hf:owner/model@{COMMIT}")+ _check_adapter_namespace(plan)+ self.assertEqual(+ _declared_auxiliary_names(plan),+ {"chat_template.jinja", "config.json", "tokenizer.json",+ "tokenizer_config.json", "vocab.json"})+ self.assertEqual(len(plan.files), 47)++ def test_hugging_face_manifest_may_declare_hashes_json_as_an_artifact(self):+ files = dict(self.files)+ files["hashes.json"] = b'{"files":{}}'+ raw = self.make_manifest({name: len(value) for name, value in files.items()})+ responses = [self.response(raw), self.config_response()]+ for name in sorted(files):+ value = files[name]+ responses.append(Response(200, value, [+ ("Content-Length", str(len(value))),+ ("ETag", f'"{name}"'),+ ("X-Checksum-Sha256", hashlib.sha256(value).hexdigest()),+ ]))++ self.install(ScriptedTransport(*responses))++ self.assertEqual((self.root / "hashes.json").read_bytes(), files["hashes.json"])++ def test_manifest_cannot_claim_adapter_state_or_sidecar_namespace(self):+ for name in (STATE_NAME, "Config.json", "tokenizer.json/child",+ ".tokenizer.json.part"):+ with self.subTest(name=name):+ files = {"packed_experts/layout.json": 6, name: 1}+ raw = self.make_manifest(files)+ transport = ScriptedTransport(self.response(raw))+ with self.assertRaisesRegex(QpackHTTPError, "collides"):+ self.install(transport,+ root=Path(self.temporary.name) / name.replace("/", "_"))+ self.assertEqual(len(transport.calls), 1)++ def test_completed_install_is_offline_checked_against_validator_state(self):+ transport = ScriptedTransport(*self.success_responses())+ self.install(transport)++ transport = ScriptedTransport(self.manifest_response())+ result = self.install(transport)+ self.assertTrue(result.already_complete)+ self.assertEqual(result.downloaded_files, 0)+ self.assertEqual(len(transport.calls), 1)++ (self.root / "model.safetensors").write_bytes(b"WEIGHTS")+ with self.assertRaisesRegex(QpackHTTPError, "artifact changed"):+ self.install(ScriptedTransport(self.manifest_response()))++ def test_generic_digest_on_resumed_range_is_not_a_full_object_checksum(self):+ value = self.files["model.safetensors"]+ first = self.artifact_response("model.safetensors", body=value[:3])+ first.headers = Headers([+ ("Content-Length", str(len(value))),+ ("ETag", '"model.safetensors"'),+ ("Digest", "sha-256=:range-one:"),+ ])+ second = self.artifact_response("model.safetensors", offset=3)+ second.headers = Headers([+ ("Content-Length", str(len(value) - 3)),+ ("Content-Range", f"bytes 3-{len(value) - 1}/{len(value)}"),+ ("ETag", '"model.safetensors"'),+ ("Digest", "sha-256=:range-two:"),+ ])+ transport = ScriptedTransport(+ self.manifest_response(), self.config_response(), first, second,+ self.artifact_response("packed_experts/layout.json"))+ self.install(transport, retries=1)+ state = json.loads((self.root / STATE_NAME).read_text())+ self.assertIsNone(state["files"]["model.safetensors"]["sha256"])+ self.assertEqual(state["files"]["model.safetensors"]["local_sha256"],+ hashlib.sha256(value).hexdigest())++ def test_lost_http_state_rejects_an_already_committed_artifact(self):+ short_layout = self.artifact_response(+ "packed_experts/layout.json",+ body=self.files["packed_experts/layout.json"][:2])+ short_layout.headers = Headers([+ ("Content-Length", str(len(self.files["packed_experts/layout.json"]))),+ ("ETag", '"layout"'),+ ])+ with self.assertRaisesRegex(QpackHTTPError, "short HTTP body"):+ self.install(ScriptedTransport(+ self.manifest_response(), self.config_response(),+ self.artifact_response("model.safetensors"), short_layout))+ self.assertTrue((self.root / "model.safetensors").is_file())+ (self.root / STATE_NAME).unlink()++ with self.assertRaisesRegex(QpackHTTPError, "no validator"):+ self.install(ScriptedTransport(+ self.manifest_response(), self.config_response()))+ self.assertFalse((self.root / "manifest.json").exists())++ def test_revision_resolution_retries_transient_status(self):+ api_body = json.dumps({"sha": COMMIT}).encode()+ retry = Response(503, b"", [("Content-Length", "0")])+ api = Response(200, api_body,+ [("Content-Length", str(len(api_body)))])+ transport = ScriptedTransport(retry, api, *self.success_responses())+ self.install(transport, revision="main", retries=1)+ self.assertEqual(len([call for call in transport.calls+ if "/api/models/" in call["url"]]), 2)++ def test_endpoint_must_be_an_https_origin(self):+ for endpoint in ("http://hub.example", "https://hub.example/path",+ "https://[email protected]"):+ with self.subTest(endpoint=endpoint), self.assertRaisesRegex(+ QpackHTTPError, "HTTPS origin"):+ install_hf_qpack(+ "owner/model", COMMIT, self.root, endpoint=endpoint,+ transport=ScriptedTransport(), optional_aux=())+++class SafeRedirectHandlerTest(unittest.TestCase):+ def setUp(self):+ self.handler = SafeRedirectHandler()++ @staticmethod+ def request():+ return Request("https://hub.example/object",+ headers={"Authorization": "Bearer secret"})++ def test_cross_origin_redirect_strips_authorization(self):+ redirected = self.handler.redirect_request(+ self.request(), None, 302, "Found", {},+ "https://cdn.example/object")+ self.assertIsNone(redirected.get_header("Authorization"))++ def test_same_origin_redirect_keeps_authorization(self):+ redirected = self.handler.redirect_request(+ self.request(), None, 302, "Found", {},+ "https://hub.example/elsewhere")+ self.assertEqual(redirected.get_header("Authorization"), "Bearer secret")++ def test_redirect_rejects_downgrade_and_userinfo(self):+ for url in ("http://hub.example/object",+ "https://[email protected]/object"):+ with self.subTest(url=url), self.assertRaisesRegex(+ QpackHTTPError, "unsafe redirect"):+ self.handler.redirect_request(+ self.request(), None, 302, "Found", {}, url)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_qpack_mirror_install.py b/c/tests/test_qpack_mirror_install.pynew file mode 100644index 00000000..2ab85867--- /dev/null+++ b/c/tests/test_qpack_mirror_install.py@@ -0,0 +1,422 @@+import hashlib+import json+import os+import tempfile+import unittest+from pathlib import Path+from unittest import mock+from urllib.request import Request++from tools.qpack_http_install import (+ HASHES_NAME,+ QpackHTTPError,+ SafeRedirectHandler,+ install_mirror_qpack,+ parse_hashes_index,+)+from tools.qpack_install_policy import QpackInstallError+++class Headers:+ def __init__(self, values=()):+ self.values = list(values)++ def get_all(self, name, default=None):+ matches = [value for key, value in self.values+ if key.lower() == name.lower()]+ return matches if matches else (default or [])++ def items(self):+ return iter(self.values)+++class Response:+ def __init__(self, status, body=b"", headers=()):+ self.status = status+ self.body = body+ self.headers = Headers(headers)+ self.offset = 0++ def read(self, size=-1):+ if size is None or size < 0:+ size = len(self.body) - self.offset+ chunk = self.body[self.offset:self.offset + size]+ self.offset += len(chunk)+ return chunk++ def close(self):+ pass+++class ScriptedTransport:+ def __init__(self, *responses):+ self.responses = list(responses)+ self.calls = []++ def request(self, method, url, headers, timeout):+ call = {+ "method": method,+ "url": url,+ "headers": dict(headers),+ "timeout": timeout,+ }+ self.calls.append(call)+ if not self.responses:+ raise AssertionError(f"unexpected request: {method} {url}")+ response = self.responses.pop(0)+ return response(call) if callable(response) else response++ def assert_empty(self):+ if self.responses:+ raise AssertionError(f"{len(self.responses)} scripted responses unused")+++class QpackMirrorInstallTest(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name) / "tiny.qpack"+ self.base_url = "https://mirror.example/models/tiny"+ self.config = b'{"model_type":"qwen3_next"}'+ self.files = {+ "model.safetensors": b"weights",+ "packed_experts/layout.json": b"layout",+ }+ self.manifest = json.dumps({+ "magic": "QPACK",+ "version": 1,+ "modelName": "tiny-qwen",+ "sourceCheckpoint": "owner/model@static",+ "quantBits": 4,+ "quantGroupSize": 32,+ "files": {name: len(value) for name, value in self.files.items()},+ }, sort_keys=True, separators=(",", ":")).encode()++ @staticmethod+ def response(body, *, status=200, etag=None, content_length=None,+ content_range=None, checksum=None):+ headers = [("Content-Length", str(+ len(body) if content_length is None else content_length))]+ if etag is not None:+ headers.append(("ETag", etag))+ if content_range is not None:+ headers.append(("Content-Range", content_range))+ if checksum is not None:+ headers.append(("X-Checksum-Sha256", checksum))+ return Response(status, body, headers)++ def hash_map(self, *, extra=None, omit=(), replacements=None):+ values = {+ "manifest.json": self.manifest,+ "config.json": self.config,+ **self.files,+ **(extra or {}),+ }+ hashes = {+ name: hashlib.sha256(value).hexdigest()+ for name, value in values.items() if name not in omit+ }+ hashes.update(replacements or {})+ return hashes++ def hashes_raw(self, **kwargs):+ return json.dumps(+ {"files": self.hash_map(**kwargs)},+ sort_keys=True, separators=(",", ":")).encode()++ def artifact_response(self, name, *, offset=0, status=None, etag=None,+ body=None):+ value = self.files[name]+ body = value[offset:] if body is None else body+ status = status if status is not None else (206 if offset else 200)+ content_range = None+ if status == 206:+ content_range = f"bytes {offset}-{len(value) - 1}/{len(value)}"+ return self.response(+ body, status=status, etag=etag,+ content_length=len(body), content_range=content_range)++ def success_responses(self, *, hashes_raw=None, optional=()):+ hashes_raw = hashes_raw or self.hashes_raw(+ extra=dict(optional))+ responses = [+ self.response(hashes_raw, etag='"hashes"'),+ self.response(self.manifest, etag='"manifest"'),+ self.response(self.config, etag='"config"'),+ ]+ responses.extend(self.response(value) for _name, value in optional)+ responses.extend(self.artifact_response(name) for name in sorted(self.files))+ return responses++ def install(self, transport, *, root=None, **kwargs):+ return install_mirror_qpack(+ self.base_url, root or self.root, transport=transport,+ retries=kwargs.pop("retries", 0), timeout=7, chunk_size=3,+ sleep=lambda _seconds: None, log=lambda _message: None, **kwargs)++ def test_installs_legacy_index_and_persists_exact_raw_hashes(self):+ hashes_raw = self.hashes_raw()+ transport = ScriptedTransport(*self.success_responses(hashes_raw=hashes_raw))++ with mock.patch.dict(os.environ, {"HF_TOKEN": "must-not-leak"}):+ result = self.install(transport)++ self.assertIsNone(result.commit)+ self.assertEqual(+ result.source_id,+ f"mirror:{self.base_url}@sha256:"+ f"{hashlib.sha256(hashes_raw).hexdigest()}")+ self.assertEqual(result.downloaded_files, 3)+ self.assertEqual((self.root / "manifest.json").read_bytes(), self.manifest)+ self.assertEqual((self.root / "config.json").read_bytes(), self.config)+ self.assertEqual((self.root / HASHES_NAME).read_bytes(), hashes_raw)+ self.assertTrue(all("Authorization" not in call["headers"]+ for call in transport.calls))+ self.assertEqual(+ [call["url"].rsplit("/", 1)[-1] for call in transport.calls[:3]],+ ["hashes.json", "manifest.json", "config.json"])+ self.assertEqual(transport.calls[0]["headers"]["Cache-Control"],+ "no-cache")+ transport.assert_empty()++ def test_v1_and_legacy_have_the_same_semantic_identity(self):+ hashes = self.hash_map()+ legacy = json.dumps({"files": hashes}, indent=2).encode()+ v1 = json.dumps({+ "files": {+ name: {"sha256": digest, "size": len({+ "manifest.json": self.manifest,+ "config.json": self.config,+ **self.files,+ }[name])}+ for name, digest in reversed(list(hashes.items()))+ },+ "schema": "qpack.hashes.v1",+ }, separators=(",", ":")).encode()++ legacy_entries, legacy_id = parse_hashes_index(legacy)+ v1_entries, v1_id = parse_hashes_index(v1)++ self.assertEqual(legacy_id, v1_id)+ self.assertEqual(+ {name: entry.sha256 for name, entry in legacy_entries.items()},+ {name: entry.sha256 for name, entry in v1_entries.items()})++ def test_v1_size_must_match_the_manifest_before_install_starts(self):+ values = {+ "manifest.json": self.manifest,+ "config.json": self.config,+ **self.files,+ }+ index = {+ name: {+ "size": len(value) + int(name == "model.safetensors"),+ "sha256": hashlib.sha256(value).hexdigest(),+ }+ for name, value in values.items()+ }+ hashes_raw = json.dumps({+ "schema": "qpack.hashes.v1",+ "files": index,+ }).encode()+ transport = ScriptedTransport(+ self.response(hashes_raw), self.response(self.manifest))++ with self.assertRaisesRegex(QpackHTTPError, "size mismatch"):+ self.install(transport)++ self.assertFalse(self.root.exists())++ def test_raw_hash_pin_and_manifest_hash_fail_before_destination_creation(self):+ hashes_raw = self.hashes_raw()+ with self.assertRaisesRegex(QpackHTTPError, "pinned SHA-256"):+ self.install(+ ScriptedTransport(self.response(hashes_raw)),+ hashes_sha256="f" * 64)+ self.assertFalse(self.root.exists())++ wrong = self.hashes_raw(replacements={"manifest.json": "f" * 64})+ with self.assertRaisesRegex(QpackHTTPError, "SHA-256 mismatch"):+ self.install(ScriptedTransport(+ self.response(wrong), self.response(self.manifest)))+ self.assertFalse(self.root.exists())++ def test_missing_hash_entry_stops_before_large_transfers(self):+ hashes_raw = self.hashes_raw(omit=("config.json",))+ transport = ScriptedTransport(+ self.response(hashes_raw), self.response(self.manifest))++ with self.assertRaisesRegex(QpackHTTPError, "does not cover"):+ self.install(transport)++ self.assertEqual(len(transport.calls), 2)+ self.assertFalse(self.root.exists())++ def test_extra_hashed_sidecars_are_validated_but_not_downloaded(self):+ hashes_raw = self.hashes_raw(extra={+ "README.md": b"notes",+ "licenses/NOTICE.txt": b"notice",+ })+ transport = ScriptedTransport(+ *self.success_responses(hashes_raw=hashes_raw))++ self.install(transport)++ requested = {call["url"].removeprefix(self.base_url + "/")+ for call in transport.calls}+ self.assertNotIn("README.md", requested)+ self.assertNotIn("licenses/NOTICE.txt", requested)++ def test_resume_without_etag_uses_exact_range_and_full_hash(self):+ value = self.files["model.safetensors"]+ first = self.artifact_response(+ "model.safetensors", body=value[:3])+ first.headers = Headers([("Content-Length", str(len(value)))])+ with self.assertRaisesRegex(QpackHTTPError, "short HTTP body"):+ self.install(ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest),+ self.response(self.config), first))++ transport = ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest),+ self.artifact_response("model.safetensors", offset=3),+ self.artifact_response("packed_experts/layout.json"))+ result = self.install(transport)++ model_call = transport.calls[2]+ self.assertEqual(model_call["headers"]["Range"], "bytes=3-6")+ self.assertNotIn("If-Match", model_call["headers"])+ self.assertEqual(result.resumed_files, 1)+ self.assertEqual((self.root / "model.safetensors").read_bytes(), value)++ def test_failed_if_match_restarts_without_appending(self):+ value = self.files["model.safetensors"]+ first = self.artifact_response(+ "model.safetensors", body=value[:3], etag='"old"')+ first.headers = Headers([+ ("Content-Length", str(len(value))),+ ("ETag", '"old"'),+ ])+ with self.assertRaises(QpackHTTPError):+ self.install(ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest),+ self.response(self.config), first))++ transport = ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest),+ self.response(b"", status=412),+ self.artifact_response("model.safetensors", etag='"new"'),+ self.artifact_response("packed_experts/layout.json"))+ self.install(transport)++ self.assertEqual(transport.calls[2]["headers"]["If-Match"], '"old"')+ self.assertNotIn("If-Match", transport.calls[3]["headers"])+ self.assertEqual(transport.calls[3]["headers"]["Range"], "bytes=0-6")+ self.assertEqual((self.root / "model.safetensors").read_bytes(), value)++ def test_changed_snapshot_cannot_resume_existing_journal(self):+ value = self.files["model.safetensors"]+ first = self.artifact_response("model.safetensors", body=value[:3])+ first.headers = Headers([("Content-Length", str(len(value)))])+ with self.assertRaises(QpackHTTPError):+ self.install(ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest),+ self.response(self.config), first))+ old_hashes = (self.root / HASHES_NAME).read_bytes()++ changed = self.hashes_raw(replacements={+ "model.safetensors": hashlib.sha256(b"changed").hexdigest(),+ })+ transport = ScriptedTransport(+ self.response(changed), self.response(self.manifest))+ with self.assertRaisesRegex(QpackInstallError, "another source"):+ self.install(transport)+ self.assertEqual((self.root / HASHES_NAME).read_bytes(), old_hashes)+ self.assertEqual(len(transport.calls), 2)++ def test_file_hash_mismatch_truncates_and_never_publishes_manifest(self):+ wrong = self.hashes_raw(replacements={+ "model.safetensors": hashlib.sha256(b"different").hexdigest(),+ })+ transport = ScriptedTransport(+ self.response(wrong), self.response(self.manifest),+ self.response(self.config), self.artifact_response("model.safetensors"))++ with self.assertRaisesRegex(QpackHTTPError, "SHA-256 mismatch"):+ self.install(transport)++ self.assertEqual((self.root / "model.safetensors.part").stat().st_size, 0)+ self.assertFalse((self.root / "manifest.json").exists())++ def test_listed_optional_auxiliary_is_required_and_verified(self):+ tokenizer = b'{"version":"1.0"}'+ hashes_raw = self.hashes_raw(extra={"tokenizer.json": tokenizer})+ transport = ScriptedTransport(*self.success_responses(+ hashes_raw=hashes_raw, optional=(("tokenizer.json", tokenizer),)))++ self.install(transport)++ self.assertEqual((self.root / "tokenizer.json").read_bytes(), tokenizer)+ self.assertTrue(transport.calls[3]["url"].endswith("/tokenizer.json"))++ root = Path(self.temporary.name) / "missing-optional.qpack"+ transport = ScriptedTransport(+ self.response(hashes_raw), self.response(self.manifest),+ self.response(self.config), self.response(b"", status=404))+ with self.assertRaisesRegex(QpackHTTPError, "HTTP 404"):+ self.install(transport, root=root)++ def test_completed_mirror_install_is_rehashed_without_object_requests(self):+ self.install(ScriptedTransport(*self.success_responses()))+ transport = ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest))++ result = self.install(transport)++ self.assertTrue(result.already_complete)+ self.assertEqual(result.downloaded_files, 0)+ self.assertEqual(len(transport.calls), 2)++ (self.root / "model.safetensors").write_bytes(b"WEIGHTS")+ with self.assertRaisesRegex(QpackHTTPError, "artifact changed"):+ self.install(ScriptedTransport(+ self.response(self.hashes_raw()), self.response(self.manifest)))++ def test_hashes_parser_and_base_url_fail_closed(self):+ digest = "a" * 64+ invalid_indexes = [+ b'{"files":{"manifest.json":"' + digest.encode()+ + b'","manifest.json":"' + digest.encode() + b'"}}',+ json.dumps({"files": {"../manifest.json": digest}}).encode(),+ json.dumps({"files": {"hashes.json": digest}}).encode(),+ json.dumps({"files": {"manifest.json": "bad"}}).encode(),+ json.dumps({"schema": "future", "files": {"manifest.json": digest}}).encode(),+ ]+ for raw in invalid_indexes:+ with self.subTest(raw=raw), self.assertRaises(QpackHTTPError):+ parse_hashes_index(raw)++ for base_url in (+ "http://mirror.example/qpack",+ "https://[email protected]/qpack",+ "https://mirror.example/a/../qpack",+ "https://mirror.example/qpack?token=secret"):+ with self.subTest(base_url=base_url), self.assertRaises(QpackHTTPError):+ install_mirror_qpack(+ base_url, self.root, transport=ScriptedTransport())++ def test_cross_origin_redirect_strips_all_explicit_secrets(self):+ handler = SafeRedirectHandler()+ request = Request("https://mirror.example/object", headers={+ "Authorization": "Bearer secret",+ "Cookie": "session=secret",+ "X-Api-Key": "secret",+ })+ redirected = handler.redirect_request(+ request, None, 302, "Found", {}, "https://cdn.example/object")+ for name in ("Authorization", "Cookie", "X-Api-Key"):+ self.assertIsNone(redirected.get_header(name))+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/README.md b/c/tools/README.mdindex 24d84cec..ffc53c0b 100644--- a/c/tools/README.md+++ b/c/tools/README.md@@ -11,6 +11,27 @@ not runtime dependencies of the C engine.transport checksum or ETag. It covers manifest-declared container artifacts,not required runtime auxiliaries such as `config.json`. Directory-fsync crashdurability is POSIX-only.+- `qpack_http_install.py`: dependency-free Hugging Face frontend for that+ policy. It resolves a branch or tag to an immutable commit before opening the+ install session, streams manifest-declared files directly into `.part`+ files, resumes only exact HTTP ranges, and publishes `manifest.json` last.+ `config.json` is required; manifest-declared tokenizer/configuration files+ use the same resumable artifact path, while undeclared optional files are+ copied as verified sidecars when present. Strong ETags, documented server+ SHA-256 values, and a locally computed full-file SHA-256 are persisted in+ `.qpack-http.json` and checked on resume. `HF_TOKEN` is sent only to the Hub+ origin and is stripped on cross-origin HTTPS redirects.+- `qpack_mirror_install.py`: static HTTPS/R2 frontend over the same transfer+ engine. A strict `hashes.json` is required and fetched before the manifest;+ it must cover `manifest.json`, `config.json`, and every manifest artifact.+ The legacy Swiftlet `{"files":{"path":"sha256"}}` schema and the sized+ `qpack.hashes.v1` schema are accepted. The normalized mirror URL plus the+ digest of the exact raw index bytes bind resume state; indexed runtime files receive+ authoritative full-object SHA-256 values, while other valid indexed+ sidecars are ignored. The exact raw index is persisted beside the installed+ container. Mirror installs never inherit+ `HF_TOKEN`; an explicit bearer token can be read from a named environment+ variable and is stripped on cross-origin redirects.- `convert_fmt4_to_fmt2.py`: fmt=4 (grouped int4) -> fmt=2 (per-row int4)re-quant of a GLM-5.2 container, for Metal-backend compatibility(see `docs/METAL-M1ULTRA-FMT2-REPORT.md`)@@ -35,8 +56,30 @@ Run them from `c/`, for example:```shpython3 tools/convert_fp8_to_int4.py --selftestpython3 tools/make_glm_bench_model.py --output /tmp/colibri-bench+python3 tools/qpack_http_install.py OWNER/REPO \+ --revision main --output /path/to/model.qpack+python3 tools/qpack_mirror_install.py \+ https://models.example/qwen/model.qpack \+ --output /path/to/model.qpack \+ --hashes-sha256 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef```+An interrupted qpack install can be restarted with the same command. Keep the+source repository and revision unchanged. The installer rejects HTTP range or+validator inconsistencies instead of appending uncertain bytes. It requires+`Content-Length` for large artifacts and never stages the original BF16/FP8+checkpoint, but it does not convert a raw checkpoint into qpack.++Static mirrors must use HTTPS and publish objects first, `manifest.json` next,+and `hashes.json` last as the remote completion marker. Without+`--hashes-sha256`, the first valid TLS response is trust on first use: the+URL-and-index identity prevents a later snapshot from resuming into those+partials, but it cannot authenticate a mirror that was already compromised on+the first request. Prefer a versioned mirror prefix and an out-of-band index+digest for production distribution. The installer uses exact byte ranges and+`If-Match` when a strong ETag is available; final SHA-256 verification remains+the authority.+`make_glm_oracle.py` also produces the quantized routed-expert fixtures for thefmt=6 (E8/IQ3, rotation-bearing) and fmt=4 (grouped int4, no-rotation control)parity gate (#3/#7). Only the routed experts are quantized; shared/dense/attndiff --git a/c/tools/qpack_http_install.py b/c/tools/qpack_http_install.pynew file mode 100644index 00000000..07b602ba--- /dev/null+++ b/c/tools/qpack_http_install.py@@ -0,0 +1,1176 @@+#!/usr/bin/env python3+"""Install a commit-pinned Hugging Face qpack without staging raw weights."""++from __future__ import annotations++import argparse+import errno+import hashlib+import http.client+import json+import os+import re+import stat+import sys+import time+from contextlib import closing+from dataclasses import dataclass+from pathlib import Path+from urllib.error import HTTPError, URLError+from urllib.parse import quote, unquote, urljoin, urlsplit+from urllib.request import HTTPRedirectHandler, Request, build_opener++try:+ from tools.qpack_install_policy import (+ MAX_FILE_SIZE,+ MAX_FILES,+ MAX_MANIFEST,+ QpackFile,+ QpackInstallError,+ _decode_json,+ _validate_name,+ begin_install,+ commit_file,+ parse_manifest,+ publish_manifest,+ resume_decision,+ validate_delivery,+ )+except ModuleNotFoundError: # Direct execution from c/tools.+ from qpack_install_policy import (+ MAX_FILE_SIZE,+ MAX_FILES,+ MAX_MANIFEST,+ QpackFile,+ QpackInstallError,+ _decode_json,+ _validate_name,+ begin_install,+ commit_file,+ parse_manifest,+ publish_manifest,+ resume_decision,+ validate_delivery,+ )+++STATE_SCHEMA = "colibri.qpack-http.v1"+STATE_NAME = ".qpack-http.json"+STATE_TEMP_NAME = "..qpack-http.json.part"+HASHES_NAME = "hashes.json"+CHUNK_SIZE = 4 * 1024 * 1024+MAX_AUX_SIZE = 128 * 1024 * 1024+MAX_API_SIZE = 1024 * 1024+MAX_HASHES_SIZE = 16 * 1024 * 1024+MAX_HASH_ENTRIES = MAX_FILES + 16+RETRYABLE_STATUS = {429, 500, 502, 503, 504}+COMMIT_RE = re.compile(r"^[0-9a-fA-F]{40}$")+CONTENT_RANGE_RE = re.compile(r"^bytes ([0-9]+)-([0-9]+)/([0-9]+)$")+SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")+REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")++REQUIRED_AUX = ("config.json",)+OPTIONAL_AUX = (+ "tokenizer.json",+ "tokenizer_config.json",+ "vocab.json",+ "merges.txt",+ "chat_template.jinja",+ "generation_config.json",+ "special_tokens_map.json",+ "added_tokens.json",+)+++class QpackHTTPError(RuntimeError):+ pass+++class RetryableTransferError(QpackHTTPError):+ pass+++class HTTPStatusFailure(QpackHTTPError):+ def __init__(self, url: str, status: int):+ super().__init__(f"HTTP {status} for {_safe_url(url)}")+ self.url = url+ self.status = status+++@dataclass(frozen=True)+class ResolvedSource:+ repo: str+ commit: str | None+ endpoint: str+ base_url: str+ source_id: str+ headers: tuple[tuple[str, str], ...]+ repo_commit: str | None = None+ resume_header: str = "If-Range"+ restart_changed_resume: bool = False++ def url(self, name: str) -> str:+ encoded = "/".join(quote(component, safe="") for component in name.split("/"))+ return f"{self.base_url}/{encoded}"+++@dataclass(frozen=True)+class InstallResult:+ output_dir: Path+ commit: str | None+ downloaded_files: int+ downloaded_bytes: int+ resumed_files: int+ already_complete: bool+ source_id: str = ""+++@dataclass(frozen=True)+class HashEntry:+ sha256: str+ size: int | None = None+++def _safe_url(url: str) -> str:+ split = urlsplit(url)+ host = split.hostname or ""+ port = f":{split.port}" if split.port else ""+ return f"{split.scheme}://{host}{port}{split.path}"+++def _origin(url: str):+ split = urlsplit(url)+ port = split.port+ if port is None:+ port = 443 if split.scheme == "https" else 80+ return split.scheme.lower(), (split.hostname or "").lower(), port+++class SafeRedirectHandler(HTTPRedirectHandler):+ """Keep credentials on-origin and never follow a downgrade or userinfo URL."""++ SENSITIVE_HEADERS = (+ "Authorization",+ "Cookie",+ "Proxy-Authorization",+ "X-Api-Key",+ )++ def redirect_request(self, request, fp, code, msg, headers, new_url):+ target = urljoin(request.full_url, new_url)+ split = urlsplit(target)+ if split.scheme.lower() != "https" or split.username or split.password:+ raise QpackHTTPError(f"unsafe redirect to {_safe_url(target)}")+ redirected = super().redirect_request(request, fp, code, msg, headers, target)+ if redirected is not None and _origin(request.full_url) != _origin(target):+ for name in self.SENSITIVE_HEADERS:+ redirected.remove_header(name)+ for existing in list(redirected.unredirected_hdrs):+ if existing.lower() == name.lower():+ redirected.unredirected_hdrs.pop(existing, None)+ return redirected+++class URLLibTransport:+ def __init__(self):+ self.opener = build_opener(SafeRedirectHandler())++ def request(self, method: str, url: str, headers, timeout: float):+ request = Request(url, headers=dict(headers), method=method)+ try:+ return self.opener.open(request, timeout=timeout)+ except HTTPError as error:+ return error+ except URLError as error:+ raise RetryableTransferError(+ f"request failed for {_safe_url(url)}: {error.reason}") from error+++def _header_values(headers, name: str):+ if hasattr(headers, "get_all"):+ return list(headers.get_all(name, []))+ values = []+ for key, value in headers.items():+ if key.lower() != name.lower():+ continue+ if isinstance(value, (list, tuple)):+ values.extend(str(item) for item in value)+ else:+ values.append(str(value))+ return values+++def _single_header(headers, name: str, required=False):+ values = _header_values(headers, name)+ if len(values) > 1:+ raise QpackHTTPError(f"duplicate {name} response header")+ if required and not values:+ raise QpackHTTPError(f"missing {name} response header")+ return values[0].strip() if values else None+++def _status(response) -> int:+ value = getattr(response, "status", None)+ if value is None and hasattr(response, "getcode"):+ value = response.getcode()+ return int(value or 0)+++def _strong_etag(headers):+ value = _single_header(headers, "ETag")+ if not value or value[:2].lower() == "w/":+ return None+ if len(value) < 2 or value[0] != '"' or value[-1] != '"':+ return None+ return value+++def _checksum_sha256(headers):+ direct = _single_header(headers, "X-Checksum-Sha256")+ if not direct:+ return None+ direct = direct.strip('"')+ if not SHA256_RE.fullmatch(direct):+ raise QpackHTTPError("invalid X-Checksum-Sha256 response header")+ return direct.lower()+++def _validate_mirror_base_url(base_url: str) -> str:+ if not isinstance(base_url, str) or not base_url or len(base_url) > 8192:+ raise QpackHTTPError("mirror base URL is invalid")+ try:+ split = urlsplit(base_url)+ _ = split.port+ except ValueError as error:+ raise QpackHTTPError("mirror base URL is invalid") from error+ if (split.scheme.lower() != "https" or not split.hostname+ or split.username or split.password or split.query or split.fragment):+ raise QpackHTTPError(+ "mirror base URL must be HTTPS without userinfo, query, or fragment")+ try:+ decoded_path = unquote(split.path, errors="strict")+ except UnicodeError as error:+ raise QpackHTTPError("mirror base URL path is invalid") from error+ components = decoded_path.split("/")+ if ("\\" in decoded_path or any(ord(character) < 32 for character in decoded_path)+ or any(part in (".", "..") for part in components)+ or any(part == "" for part in components[1:-1])):+ raise QpackHTTPError("mirror base URL path is unsafe")+ return base_url.rstrip("/")+++def _hash_entry_name(name: str) -> str:+ if name == "manifest.json":+ return name+ try:+ return _validate_name(name)+ except QpackInstallError as error:+ raise QpackHTTPError(str(error)) from error+++def _semantic_hashes_digest(entries: dict[str, HashEntry]) -> str:+ digest = hashlib.sha256()+ for name in sorted(entries):+ encoded = name.encode("ascii")+ digest.update(len(encoded).to_bytes(4, "big"))+ digest.update(encoded)+ digest.update(bytes.fromhex(entries[name].sha256))+ return digest.hexdigest()+++def parse_hashes_index(raw: bytes) -> tuple[dict[str, HashEntry], str]:+ """Strictly parse Swiftlet's legacy index or qpack.hashes.v1."""+ try:+ value = _decode_json(raw, HASHES_NAME, MAX_HASHES_SIZE)+ except QpackInstallError as error:+ raise QpackHTTPError(str(error)) from error+ if not isinstance(value, dict):+ raise QpackHTTPError("hashes.json must be an object")+ schema = value.get("schema")+ if schema is None:+ if set(value) != {"files"}:+ raise QpackHTTPError("legacy hashes.json has unsupported fields")+ elif schema == "qpack.hashes.v1":+ if set(value) != {"schema", "files"}:+ raise QpackHTTPError("qpack.hashes.v1 has unsupported fields")+ else:+ raise QpackHTTPError("unsupported hashes.json schema")+ files = value.get("files")+ if (not isinstance(files, dict) or not files+ or len(files) > MAX_HASH_ENTRIES):+ raise QpackHTTPError("hashes.json files table is invalid")++ entries = {}+ normalized_names = set()+ for raw_name, raw_entry in files.items():+ name = _hash_entry_name(raw_name)+ if name == HASHES_NAME:+ raise QpackHTTPError("hashes.json cannot list itself")+ normalized = name.casefold()+ if normalized in normalized_names:+ raise QpackHTTPError(f"case-colliding hashes.json path: {name}")+ normalized_names.add(normalized)+ if schema is None:+ sha256 = raw_entry+ size = None+ else:+ if not isinstance(raw_entry, dict) or set(raw_entry) != {"size", "sha256"}:+ raise QpackHTTPError(f"invalid qpack.hashes.v1 entry: {name}")+ sha256 = raw_entry.get("sha256")+ size = raw_entry.get("size")+ if (isinstance(size, bool) or not isinstance(size, int)+ or size < 0 or size > MAX_FILE_SIZE):+ raise QpackHTTPError(f"invalid hashes.json size for {name}")+ if not isinstance(sha256, str) or not SHA256_RE.fullmatch(sha256):+ raise QpackHTTPError(f"invalid hashes.json SHA-256 for {name}")+ entries[name] = HashEntry(sha256.lower(), size)++ sorted_names = sorted(normalized_names)+ occupied = set(sorted_names)+ for name in sorted_names:+ components = name.split("/")+ for count in range(1, len(components)):+ if "/".join(components[:count]) in occupied:+ raise QpackHTTPError("hashes.json has a file/parent path collision")+ return entries, _semantic_hashes_digest(entries)+++def _read_bounded(response, maximum: int, description: str) -> bytes:+ data = response.read(maximum + 1)+ if len(data) > maximum:+ raise QpackHTTPError(f"{description} exceeds {maximum} bytes")+ return data+++def _fsync_directory(path: Path) -> None:+ if os.name == "nt":+ return+ descriptor = os.open(path, os.O_RDONLY)+ try:+ try:+ os.fsync(descriptor)+ except OSError as error:+ if error.errno not in (errno.EBADF, errno.EINVAL, errno.ENOTSUP):+ raise+ finally:+ os.close(descriptor)+++def _atomic_bytes(path: Path, payload: bytes) -> None:+ temporary = path.with_name(f".{path.name}.part")+ if path.is_symlink() or temporary.is_symlink():+ raise QpackHTTPError(f"refusing symlink auxiliary path: {path}")+ temporary.unlink(missing_ok=True)+ try:+ with temporary.open("xb") as stream:+ stream.write(payload)+ stream.flush()+ os.fsync(stream.fileno())+ os.replace(temporary, path)+ _fsync_directory(path.parent)+ except Exception:+ temporary.unlink(missing_ok=True)+ raise+++def _atomic_json(path: Path, payload) -> None:+ raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + b"\n"+ _atomic_bytes(path, raw)+++def _state_path(root: Path) -> Path:+ return root / STATE_NAME+++def _new_state(session, manifest_etag):+ return {+ "schema": STATE_SCHEMA,+ "source_id": session.plan.source_id,+ "manifest_sha256": session.plan.manifest_sha256,+ "manifest_etag": manifest_etag,+ "files": {},+ "aux": {},+ }+++def _load_state(session, manifest_etag, require_existing=False):+ path = _state_path(session.root)+ if path.is_symlink():+ raise QpackHTTPError("qpack HTTP state is a symlink")+ if not path.exists():+ if require_existing:+ raise QpackHTTPError("completed HTTP install has no validator state")+ state = _new_state(session, manifest_etag)+ _atomic_json(path, state)+ return state+ try:+ state = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, UnicodeError, json.JSONDecodeError) as error:+ raise QpackHTTPError(f"invalid qpack HTTP state: {error}") from error+ expected = (STATE_SCHEMA, session.plan.source_id, session.plan.manifest_sha256)+ actual = (state.get("schema"), state.get("source_id"),+ state.get("manifest_sha256")) if isinstance(state, dict) else ()+ if actual != expected or not isinstance(state.get("files"), dict) \+ or not isinstance(state.get("aux"), dict):+ raise QpackHTTPError("qpack HTTP state belongs to another source or manifest")+ stored_etag = state.get("manifest_etag")+ if stored_etag and manifest_etag and stored_etag != manifest_etag:+ raise QpackHTTPError("qpack manifest ETag changed for an immutable source")+ if stored_etag is None and manifest_etag is not None:+ state["manifest_etag"] = manifest_etag+ _atomic_json(path, state)+ return state+++def _save_state(session, state) -> None:+ _atomic_json(_state_path(session.root), state)+++def _declared_auxiliary_names(plan):+ declared = {file.name for file in plan.files}+ return {name for name in (*REQUIRED_AUX, *OPTIONAL_AUX) if name in declared}+++def _check_adapter_namespace(plan, reserve_hashes=False) -> None:+ reserved = {+ STATE_NAME.casefold(),+ STATE_TEMP_NAME.casefold(),+ }+ if reserve_hashes:+ reserved.update((+ HASHES_NAME.casefold(),+ f".{HASHES_NAME}.part".casefold(),+ ))+ declared_aux = _declared_auxiliary_names(plan)+ for name in (*REQUIRED_AUX, *OPTIONAL_AUX):+ if name in declared_aux:+ continue+ reserved.add(name.casefold())+ reserved.add((f".{name}.part").casefold())+ for file in plan.files:+ if file.name in declared_aux and file.size > MAX_AUX_SIZE:+ raise QpackHTTPError(+ f"qpack auxiliary file is too large: {file.name}")+ name = file.name.casefold()+ if (name in reserved+ or any(name.startswith(path + "/") for path in reserved)+ or any(path.startswith(name + "/") for path in reserved)):+ raise QpackHTTPError(+ f"qpack manifest collides with HTTP installer path: {file.name}")+++def _request_headers(source: ResolvedSource, extra=()):+ values = dict(source.headers)+ values["Accept-Encoding"] = "identity"+ values["User-Agent"] = "colibri-qpack-install/1.0"+ values.update(dict(extra))+ return values+++def _response_metadata(response, source: ResolvedSource, expected_size=None):+ headers = response.headers+ encoding = _single_header(headers, "Content-Encoding")+ if encoding and encoding.lower() != "identity":+ raise QpackHTTPError(f"unsupported content encoding: {encoding}")+ repo_commit = _single_header(headers, "X-Repo-Commit")+ if (repo_commit and source.repo_commit is not None+ and repo_commit.lower() != source.repo_commit):+ raise QpackHTTPError("response belongs to a different repository commit")+ linked_size = _single_header(headers, "X-Linked-Size")+ if linked_size is not None and expected_size is not None:+ try:+ linked_size_value = int(linked_size)+ except ValueError as error:+ raise QpackHTTPError("invalid X-Linked-Size response header") from error+ if linked_size_value != expected_size:+ raise QpackHTTPError("response linked size differs from qpack manifest")+ return _strong_etag(headers), _checksum_sha256(headers)+++def _resolve_source(repo: str, revision: str, endpoint: str, token: str | None,+ transport, timeout: float, retries: int, sleep) -> ResolvedSource:+ if (not REPO_RE.fullmatch(repo)+ or any(part in (".", "..") for part in repo.split("/"))):+ raise QpackHTTPError("repository must be OWNER/REPO")+ if not isinstance(revision, str) or not revision or len(revision) > 4096:+ raise QpackHTTPError("revision must be a non-empty commit or ref")+ endpoint = endpoint.rstrip("/")+ split = urlsplit(endpoint)+ if split.scheme != "https" or not split.hostname or split.username or split.password \+ or split.path not in ("", "/") or split.query or split.fragment:+ raise QpackHTTPError("Hugging Face endpoint must be an HTTPS origin")+ headers = {}+ if token:+ headers["Authorization"] = f"Bearer {token}"+ headers["Accept-Encoding"] = "identity"+ headers["User-Agent"] = "colibri-qpack-install/1.0"+ if COMMIT_RE.fullmatch(revision):+ commit = revision.lower()+ else:+ api = (f"{endpoint}/api/models/{quote(repo, safe='/')}/revision/"+ f"{quote(revision, safe='')}?expand=sha")+ for attempt in range(retries + 1):+ try:+ response = transport.request("GET", api, headers, timeout)+ with closing(response):+ status = _status(response)+ if status in RETRYABLE_STATUS:+ raise HTTPStatusFailure(api, status)+ if status != 200:+ raise HTTPStatusFailure(api, status)+ try:+ raw = _read_bounded(+ response, MAX_API_SIZE,+ "Hugging Face revision metadata")+ except (OSError, http.client.IncompleteRead) as error:+ raise RetryableTransferError(+ "interrupted Hugging Face revision metadata") from error+ break+ except (RetryableTransferError, HTTPStatusFailure) as error:+ retryable = (isinstance(error, RetryableTransferError)+ or error.status in RETRYABLE_STATUS)+ if not retryable or attempt >= retries:+ raise+ sleep(min(30, 2 ** attempt))+ try:+ value = json.loads(raw)+ commit = value["sha"].lower()+ except (UnicodeError, json.JSONDecodeError, KeyError, AttributeError) as error:+ raise QpackHTTPError("Hugging Face revision metadata has no commit SHA") from error+ if not COMMIT_RE.fullmatch(commit):+ raise QpackHTTPError("Hugging Face returned an invalid commit SHA")+ encoded_repo = "/".join(quote(part, safe="") for part in repo.split("/"))+ base_url = f"{endpoint}/{encoded_repo}/resolve/{commit}"+ source_id = f"hf:{endpoint}/models/{repo}@{commit}"+ return ResolvedSource(repo, commit, endpoint, base_url, source_id,+ tuple(headers.items()), repo_commit=commit)+++def _mirror_source(base_url: str, source_id: str,+ authorization: str | None) -> ResolvedSource:+ base_url = _validate_mirror_base_url(base_url)+ headers = ()+ if authorization is not None:+ if (not isinstance(authorization, str) or not authorization.strip()+ or "\r" in authorization or "\n" in authorization):+ raise QpackHTTPError("mirror authorization value is invalid")+ headers = (("Authorization", authorization),)+ split = urlsplit(base_url)+ origin = f"{split.scheme}://{split.netloc}"+ return ResolvedSource(+ repo="", commit=None, endpoint=origin, base_url=base_url,+ source_id=source_id, headers=headers, resume_header="If-Match",+ restart_changed_resume=True)+++def _fetch_small(source, name, transport, timeout, maximum, required,+ retries, sleep, expected_checksum=None, expected_size=None,+ request_headers=()):+ url = source.url(name)+ for attempt in range(retries + 1):+ try:+ response = transport.request(+ "GET", url, _request_headers(source, request_headers), timeout)+ with closing(response):+ status = _status(response)+ if status == 404 and not required:+ return None, None+ if status in RETRYABLE_STATUS:+ raise HTTPStatusFailure(url, status)+ if status != 200:+ raise HTTPStatusFailure(url, status)+ etag, checksum = _response_metadata(response, source)+ if (expected_checksum is not None and checksum is not None+ and checksum != expected_checksum):+ raise QpackHTTPError(+ f"server SHA-256 disagrees with hashes.json for {name}")+ try:+ raw = _read_bounded(response, maximum, name)+ except (OSError, http.client.IncompleteRead) as error:+ raise RetryableTransferError(+ f"interrupted HTTP body for {name}") from error+ content_length = _single_header(response.headers, "Content-Length")+ if content_length is not None:+ try:+ expected = int(content_length)+ except ValueError as error:+ raise QpackHTTPError(+ "invalid Content-Length response header") from error+ if expected != len(raw):+ raise RetryableTransferError(f"short HTTP body for {name}")+ if expected_size is not None and len(raw) != expected_size:+ raise QpackHTTPError(f"hashes.json size mismatch for {name}")+ digest = hashlib.sha256(raw).hexdigest()+ if expected_checksum is not None and digest != expected_checksum:+ raise QpackHTTPError(f"SHA-256 mismatch for {name}")+ if checksum and digest != checksum:+ raise QpackHTTPError(f"SHA-256 mismatch for {name}")+ return raw, etag+ except (RetryableTransferError, HTTPStatusFailure) as error:+ retryable = (isinstance(error, RetryableTransferError)+ or error.status in RETRYABLE_STATUS)+ if not retryable or attempt >= retries:+ raise+ sleep(min(30, 2 ** attempt))+ raise QpackHTTPError(f"retry budget exhausted for {name}")+++def _verify_or_fetch_aux(session, state, source, name, required, transport,+ timeout, retries, sleep, completed,+ expected_checksum=None, expected_size=None):+ target = session.root / name+ recorded = state["aux"].get(name)+ if recorded is not None:+ if (not isinstance(recorded, dict)+ or not isinstance(recorded.get("sha256"), str)+ or not SHA256_RE.fullmatch(recorded["sha256"])):+ raise QpackHTTPError(f"invalid validator state for {name}")+ if (expected_checksum is not None+ and recorded["sha256"].lower() != expected_checksum):+ raise QpackHTTPError(f"validator state disagrees with hashes.json for {name}")+ if target.is_symlink() or not target.is_file():+ raise QpackHTTPError(f"validated auxiliary file is missing: {name}")+ raw = target.read_bytes()+ if expected_size is not None and len(raw) != expected_size:+ raise QpackHTTPError(f"validated auxiliary file has wrong size: {name}")+ if hashlib.sha256(raw).hexdigest() != recorded["sha256"].lower():+ raise QpackHTTPError(f"validated auxiliary file changed: {name}")+ return False, 0+ if completed:+ if required:+ raise QpackHTTPError(f"completed qpack has no validator for {name}")+ return False, 0+ raw, etag = _fetch_small(source, name, transport, timeout, MAX_AUX_SIZE,+ required, retries, sleep, expected_checksum,+ expected_size)+ if raw is None:+ return False, 0+ if name == "config.json":+ try:+ config = json.loads(raw)+ except (UnicodeError, json.JSONDecodeError) as error:+ raise QpackHTTPError("config.json is not valid JSON") from error+ if not isinstance(config, dict) or not config:+ raise QpackHTTPError("config.json must be a non-empty object")+ _atomic_bytes(target, raw)+ state["aux"][name] = {+ "sha256": expected_checksum or hashlib.sha256(raw).hexdigest(),+ "size": len(raw),+ "etag": etag,+ }+ _save_state(session, state)+ return True, len(raw)+++def _verify_or_store_prefetched_aux(session, state, name, raw, completed):+ target = session.root / name+ digest = hashlib.sha256(raw).hexdigest()+ recorded = state["aux"].get(name)+ if recorded is not None:+ if (not isinstance(recorded, dict) or recorded.get("sha256") != digest+ or recorded.get("size") != len(raw)):+ raise QpackHTTPError(f"validator state changed for {name}")+ if target.is_symlink() or not target.is_file() \+ or target.read_bytes() != raw:+ raise QpackHTTPError(f"validated auxiliary file changed: {name}")+ return+ if completed:+ raise QpackHTTPError(f"completed qpack has no validator for {name}")+ _atomic_bytes(target, raw)+ state["aux"][name] = {+ "sha256": digest,+ "size": len(raw),+ "etag": None,+ }+ _save_state(session, state)+++def _validate_config(path: Path) -> None:+ try:+ raw = path.read_bytes()+ if not raw or len(raw) > MAX_AUX_SIZE:+ raise QpackHTTPError("config.json has an invalid size")+ config = json.loads(raw)+ except (OSError, UnicodeError, json.JSONDecodeError) as error:+ raise QpackHTTPError("config.json is not valid JSON") from error+ if not isinstance(config, dict) or not config:+ raise QpackHTTPError("config.json must be a non-empty object")+++def _open_partial(decision, file: QpackFile):+ flags = os.O_RDWR | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)+ if decision.action == "restart":+ flags |= os.O_CREAT | os.O_TRUNC+ descriptor = os.open(decision.partial_path, flags, 0o644)+ metadata = os.fstat(descriptor)+ if not stat.S_ISREG(metadata.st_mode):+ os.close(descriptor)+ raise QpackHTTPError(f"qpack partial is not a regular file: {file.name}")+ expected = 0 if decision.action == "restart" else decision.offset+ if metadata.st_size != expected:+ os.close(descriptor)+ raise QpackHTTPError(f"qpack partial changed while opening: {file.name}")+ return descriptor+++def _write_all(descriptor, data: bytes) -> None:+ offset = 0+ while offset < len(data):+ written = os.write(descriptor, data[offset:])+ if written <= 0:+ raise QpackHTTPError("short write to qpack partial")+ offset += written+++def _bind_response_validator(session, state, file, etag, checksum,+ expected_checksum=None):+ if (expected_checksum is not None and checksum is not None+ and checksum != expected_checksum):+ raise QpackHTTPError(+ f"server SHA-256 disagrees with hashes.json for {file.name}")+ authoritative_checksum = expected_checksum or checksum+ record = state["files"].get(file.name)+ if record is None:+ record = {+ "size": file.size,+ "etag": etag,+ "sha256": authoritative_checksum,+ "local_sha256": None,+ }+ state["files"][file.name] = record+ _save_state(session, state)+ return record+ if not isinstance(record, dict) or record.get("size") != file.size:+ raise QpackHTTPError(f"invalid HTTP validator state for {file.name}")+ stored_etag = record.get("etag")+ stored_checksum = record.get("sha256")+ if stored_etag is not None and etag != stored_etag:+ raise QpackHTTPError(f"strong ETag changed or disappeared for {file.name}")+ if expected_checksum is not None:+ if stored_checksum != expected_checksum:+ raise QpackHTTPError(+ f"validator state disagrees with hashes.json for {file.name}")+ elif stored_checksum is not None and checksum != stored_checksum:+ raise QpackHTTPError(+ f"SHA-256 validator changed or disappeared for {file.name}")+ changed = False+ if stored_etag is None and etag is not None:+ record["etag"] = etag+ changed = True+ if stored_checksum is None and authoritative_checksum is not None:+ record["sha256"] = authoritative_checksum+ changed = True+ if changed:+ _save_state(session, state)+ return record+++def _verify_completed_file_state(session, state, file, chunk_size,+ expected_checksum=None) -> None:+ record = state["files"].get(file.name)+ if not isinstance(record, dict) or record.get("size") != file.size:+ raise QpackHTTPError(f"completed qpack has no validator for {file.name}")+ etag = record.get("etag")+ checksum = record.get("sha256")+ local_checksum = record.get("local_sha256")+ if etag is not None and (+ not isinstance(etag, str) or _strong_etag({"ETag": etag}) != etag):+ raise QpackHTTPError(f"invalid stored ETag for {file.name}")+ if checksum is not None and (+ not isinstance(checksum, str) or not SHA256_RE.fullmatch(checksum)):+ raise QpackHTTPError(f"invalid stored SHA-256 for {file.name}")+ if expected_checksum is not None and checksum != expected_checksum:+ raise QpackHTTPError(+ f"validator state disagrees with hashes.json for {file.name}")+ if (not isinstance(local_checksum, str)+ or not SHA256_RE.fullmatch(local_checksum)):+ raise QpackHTTPError(f"completed qpack has no local digest for {file.name}")+ digest = hashlib.sha256()+ with (session.root / file.name).open("rb") as stream:+ while True:+ chunk = stream.read(chunk_size)+ if not chunk:+ break+ digest.update(chunk)+ actual = digest.hexdigest()+ if checksum is not None and actual != checksum.lower():+ raise QpackHTTPError(f"validated qpack artifact changed: {file.name}")+ if actual != local_checksum.lower():+ raise QpackHTTPError(f"validated qpack artifact changed: {file.name}")+++def _validate_range_headers(response, file, offset):+ status = _status(response)+ remaining = file.size - offset+ if offset > 0 and status != 206:+ raise QpackHTTPError(f"resumed request for {file.name} did not return HTTP 206")+ if offset == 0 and status not in (200, 206):+ raise HTTPStatusFailure(file.name, status)+ content_length = _single_header(response.headers, "Content-Length", required=True)+ try:+ content_length = int(content_length)+ except ValueError as error:+ raise QpackHTTPError("invalid Content-Length response header") from error+ if content_length != remaining:+ raise QpackHTTPError(f"Content-Length mismatch for {file.name}")+ content_range = _single_header(response.headers, "Content-Range",+ required=status == 206)+ if status == 200:+ if content_range is not None:+ raise QpackHTTPError("HTTP 200 response unexpectedly has Content-Range")+ return remaining+ match = CONTENT_RANGE_RE.fullmatch(content_range or "")+ if match is None:+ raise QpackHTTPError(f"invalid Content-Range for {file.name}")+ start, end, total = map(int, match.groups())+ if start != offset or end != file.size - 1 or total != file.size:+ raise QpackHTTPError(f"Content-Range mismatch for {file.name}")+ if end - start + 1 != remaining:+ raise QpackHTTPError(f"Content-Range length mismatch for {file.name}")+ return remaining+++def _download_file(session, state, source, file, transport, timeout,+ retries, sleep, chunk_size, expected_checksum=None):+ decision = resume_decision(session, file)+ if decision.action == "complete":+ _verify_completed_file_state(+ session, state, file, chunk_size, expected_checksum)+ return False, 0, False+ resumed = decision.action == "resume" and decision.offset > 0+ restarted_from_zero = False+ descriptor = _open_partial(decision, file)+ try:+ cursor = decision.offset if decision.action == "resume" else 0+ attempts = 0+ while cursor < file.size:+ record = state["files"].get(file.name)+ if record is not None and not isinstance(record, dict):+ raise QpackHTTPError(+ f"invalid HTTP validator state for {file.name}")+ record = record or {}+ headers = {"Range": f"bytes={cursor}-{file.size - 1}"}+ if cursor > 0 and record.get("etag"):+ headers[source.resume_header] = record["etag"]+ url = source.url(file.name)+ try:+ response = transport.request(+ "GET", url, _request_headers(source, headers.items()), timeout)+ with closing(response):+ status = _status(response)+ if status in RETRYABLE_STATUS:+ raise HTTPStatusFailure(url, status)+ if (source.restart_changed_resume and cursor > 0+ and status in (200, 412)):+ os.ftruncate(descriptor, 0)+ os.fsync(descriptor)+ record["etag"] = None+ record["local_sha256"] = None+ _save_state(session, state)+ cursor = 0+ restarted_from_zero = True+ continue+ response_bytes = _validate_range_headers(response, file, cursor)+ etag, checksum = _response_metadata(response, source, file.size)+ if (source.restart_changed_resume and cursor > 0+ and record.get("etag") is not None+ and etag != record["etag"]):+ os.ftruncate(descriptor, 0)+ os.fsync(descriptor)+ record["etag"] = None+ record["local_sha256"] = None+ _save_state(session, state)+ cursor = 0+ restarted_from_zero = True+ continue+ _bind_response_validator(+ session, state, file, etag, checksum,+ expected_checksum)+ os.lseek(descriptor, cursor, os.SEEK_SET)+ delivered = 0+ request_offset = cursor+ while True:+ try:+ chunk = response.read(chunk_size)+ except http.client.IncompleteRead as error:+ chunk = error.partial+ if chunk:+ if delivered + len(chunk) > response_bytes:+ os.ftruncate(descriptor, request_offset)+ os.fsync(descriptor)+ raise QpackHTTPError(+ f"HTTP body overrun for {file.name}")+ _write_all(descriptor, chunk)+ delivered += len(chunk)+ raise RetryableTransferError(+ f"interrupted HTTP body for {file.name}") from error+ except OSError as error:+ raise RetryableTransferError(+ f"interrupted HTTP body for {file.name}") from error+ if not chunk:+ break+ if delivered + len(chunk) > response_bytes:+ os.ftruncate(descriptor, request_offset)+ os.fsync(descriptor)+ raise QpackHTTPError(f"HTTP body overrun for {file.name}")+ validate_delivery(file, cursor + delivered,+ cursor + delivered, len(chunk))+ _write_all(descriptor, chunk)+ delivered += len(chunk)+ os.fsync(descriptor)+ if delivered != response_bytes:+ cursor += delivered+ raise RetryableTransferError(+ f"short HTTP body for {file.name}")+ cursor += delivered+ validate_delivery(file, request_offset, request_offset,+ delivered, complete=cursor == file.size)+ attempts = 0+ except (RetryableTransferError, HTTPStatusFailure) as error:+ retryable = (isinstance(error, RetryableTransferError)+ or error.status in RETRYABLE_STATUS)+ if not retryable or attempts >= retries:+ raise+ attempts += 1+ sleep(min(30, 2 ** (attempts - 1)))+ cursor = os.fstat(descriptor).st_size+ resumed = resumed or cursor > 0+ record = state["files"].get(file.name)+ if not isinstance(record, dict):+ raise QpackHTTPError(f"invalid HTTP validator state for {file.name}")+ checksum = record.get("sha256")+ os.lseek(descriptor, 0, os.SEEK_SET)+ digest = hashlib.sha256()+ while True:+ chunk = os.read(descriptor, chunk_size)+ if not chunk:+ break+ digest.update(chunk)+ local_checksum = digest.hexdigest()+ if checksum and local_checksum != checksum:+ os.ftruncate(descriptor, 0)+ os.fsync(descriptor)+ raise QpackHTTPError(f"SHA-256 mismatch for {file.name}")+ stored_local = record.get("local_sha256")+ if stored_local is not None and stored_local != local_checksum:+ os.ftruncate(descriptor, 0)+ os.fsync(descriptor)+ raise QpackHTTPError(f"local SHA-256 changed for {file.name}")+ if stored_local is None:+ record["local_sha256"] = local_checksum+ _save_state(session, state)+ finally:+ os.close(descriptor)+ commit_file(session, file)+ downloaded = file.size if restarted_from_zero else file.size - decision.offset+ return True, downloaded, resumed+++def _validate_hash_coverage(plan, manifest_raw, entries) -> None:+ required = {"manifest.json", "config.json", *(file.name for file in plan.files)}+ missing = sorted(required - entries.keys())+ if missing:+ raise QpackHTTPError(+ f"hashes.json does not cover required file: {missing[0]}")+ manifest_size = entries["manifest.json"].size+ if manifest_size is not None and manifest_size != len(manifest_raw):+ raise QpackHTTPError("hashes.json size mismatch for manifest.json")+ for file in plan.files:+ indexed_size = entries[file.name].size+ if indexed_size is not None and indexed_size != file.size:+ raise QpackHTTPError(f"hashes.json size mismatch for {file.name}")+++def _install_resolved_qpack(source, output_dir, manifest_raw, manifest_etag,+ transport, retries, timeout, chunk_size,+ optional_aux, sleep, log, expected_hashes=None,+ persisted_hashes=None) -> InstallResult:+ try:+ plan = parse_manifest(manifest_raw, source.source_id)+ except QpackInstallError as error:+ raise QpackHTTPError(str(error)) from error+ _check_adapter_namespace(plan, reserve_hashes=persisted_hashes is not None)+ optional_aux = tuple(optional_aux)+ for name in optional_aux:+ if name not in OPTIONAL_AUX:+ raise QpackHTTPError(f"unsupported optional auxiliary path: {name}")+ if expected_hashes is not None:+ _validate_hash_coverage(plan, manifest_raw, expected_hashes)+ declared_aux = _declared_auxiliary_names(plan)+ files_by_name = {file.name: file for file in plan.files}++ downloaded_files = 0+ downloaded_bytes = 0+ resumed_files = 0+ with begin_install(output_dir, plan) as session:+ state = _load_state(session, manifest_etag,+ require_existing=session.complete)+ if persisted_hashes is not None:+ _verify_or_store_prefetched_aux(+ session, state, HASHES_NAME, persisted_hashes,+ session.complete)+ if "config.json" in declared_aux and not session.complete:+ changed, count, resumed = _download_file(+ session, state, source, files_by_name["config.json"],+ transport, timeout, retries, sleep, chunk_size,+ expected_hashes["config.json"].sha256+ if expected_hashes is not None else None)+ downloaded_files += int(changed)+ downloaded_bytes += count+ resumed_files += int(resumed)+ elif "config.json" not in declared_aux:+ changed, count = _verify_or_fetch_aux(+ session, state, source, "config.json", True, transport, timeout,+ retries, sleep, session.complete,+ expected_hashes["config.json"].sha256+ if expected_hashes is not None else None,+ expected_hashes["config.json"].size+ if expected_hashes is not None else None)+ downloaded_files += int(changed)+ downloaded_bytes += count+ _validate_config(session.root / "config.json")+ if not session.complete:+ for name in sorted(declared_aux - {"config.json"}):+ changed, count, resumed = _download_file(+ session, state, source, files_by_name[name], transport,+ timeout, retries, sleep, chunk_size,+ expected_hashes[name].sha256+ if expected_hashes is not None else None)+ downloaded_files += int(changed)+ downloaded_bytes += count+ resumed_files += int(resumed)+ for name in optional_aux:+ if name in declared_aux:+ continue+ changed, count = _verify_or_fetch_aux(+ session, state, source, name, expected_hashes is not None,+ transport, timeout, retries, sleep, session.complete,+ expected_hashes[name].sha256+ if expected_hashes is not None else None,+ expected_hashes[name].size+ if expected_hashes is not None else None)+ downloaded_files += int(changed)+ downloaded_bytes += count+ if session.complete:+ for file in plan.files:+ _verify_completed_file_state(+ session, state, file, chunk_size,+ expected_hashes[file.name].sha256+ if expected_hashes is not None else None)+ log(f"qpack already complete at {session.root}")+ return InstallResult(session.root, source.commit, downloaded_files,+ downloaded_bytes, 0, True, source.source_id)++ for file in plan.files:+ if file.name in declared_aux:+ continue+ changed, count, resumed = _download_file(+ session, state, source, file, transport, timeout,+ retries, sleep, chunk_size,+ expected_hashes[file.name].sha256+ if expected_hashes is not None else None)+ downloaded_files += int(changed)+ downloaded_bytes += count+ resumed_files += int(resumed)+ if changed:+ log(f"installed {file.name} ({file.size} bytes)")+ publish_manifest(session, manifest_raw)+ log(f"qpack complete at {session.root}")+ return InstallResult(session.root, source.commit, downloaded_files,+ downloaded_bytes, resumed_files, False,+ source.source_id)+++def install_hf_qpack(repo: str, revision: str, output_dir,+ *, endpoint="https://huggingface.co", token=None,+ transport=None, retries=5, timeout=120.0,+ chunk_size=CHUNK_SIZE, optional_aux=OPTIONAL_AUX,+ sleep=time.sleep, log=print) -> InstallResult:+ if retries < 0 or timeout <= 0 or chunk_size <= 0:+ raise QpackHTTPError("retry, timeout, and chunk-size values are invalid")+ transport = transport or URLLibTransport()+ source = _resolve_source(repo, revision, endpoint, token,+ transport, timeout, retries, sleep)+ manifest_raw, manifest_etag = _fetch_small(+ source, "manifest.json", transport, timeout, MAX_MANIFEST,+ True, retries, sleep)+ return _install_resolved_qpack(+ source, output_dir, manifest_raw, manifest_etag, transport, retries,+ timeout, chunk_size, optional_aux, sleep, log)+++def install_mirror_qpack(base_url: str, output_dir, *, hashes_sha256=None,+ authorization=None, transport=None, retries=5,+ timeout=120.0, chunk_size=CHUNK_SIZE,+ sleep=time.sleep, log=print) -> InstallResult:+ """Install a static HTTPS qpack whose hashes.json is the completion marker."""+ if retries < 0 or timeout <= 0 or chunk_size <= 0:+ raise QpackHTTPError("retry, timeout, and chunk-size values are invalid")+ if hashes_sha256 is not None:+ if not isinstance(hashes_sha256, str) or not SHA256_RE.fullmatch(hashes_sha256):+ raise QpackHTTPError("hashes_sha256 must be a 64-digit SHA-256")+ hashes_sha256 = hashes_sha256.lower()+ transport = transport or URLLibTransport()+ bootstrap = _mirror_source(base_url, "mirror:bootstrap", authorization)+ hashes_raw, _ = _fetch_small(+ bootstrap, HASHES_NAME, transport, timeout, MAX_HASHES_SIZE, True,+ retries, sleep, request_headers=(("Cache-Control", "no-cache"),))+ raw_digest = hashlib.sha256(hashes_raw).hexdigest()+ if hashes_sha256 is not None and raw_digest != hashes_sha256:+ raise QpackHTTPError("hashes.json does not match the pinned SHA-256")+ expected_hashes, _ = parse_hashes_index(hashes_raw)+ source = _mirror_source(+ bootstrap.base_url,+ f"mirror:{bootstrap.base_url}@sha256:{raw_digest}", authorization)+ manifest_entry = expected_hashes.get("manifest.json")+ if manifest_entry is None:+ raise QpackHTTPError("hashes.json does not cover required file: manifest.json")+ manifest_raw, _ = _fetch_small(+ source, "manifest.json", transport, timeout, MAX_MANIFEST, True,+ retries, sleep, manifest_entry.sha256, manifest_entry.size,+ (("Cache-Control", "no-cache"),))+ mirror_aux = tuple(+ name for name in OPTIONAL_AUX if name in expected_hashes)+ return _install_resolved_qpack(+ source, output_dir, manifest_raw, None, transport, retries, timeout,+ chunk_size, mirror_aux, sleep, log, expected_hashes,+ hashes_raw)+++def _parse_args(argv=None):+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("repo", help="Hugging Face repository as OWNER/REPO")+ parser.add_argument("--revision", required=True,+ help="commit SHA or revision resolved to a commit before download")+ parser.add_argument("--output", required=True, type=Path,+ help="destination .qpack directory")+ parser.add_argument("--endpoint", default="https://huggingface.co",+ help=argparse.SUPPRESS)+ parser.add_argument("--retries", type=int, default=5)+ parser.add_argument("--timeout", type=float, default=120.0)+ return parser.parse_args(argv)+++def main(argv=None):+ args = _parse_args(argv)+ try:+ result = install_hf_qpack(+ args.repo, args.revision, args.output, endpoint=args.endpoint,+ token=os.environ.get("HF_TOKEN"), retries=args.retries,+ timeout=args.timeout)+ except (OSError, QpackInstallError, QpackHTTPError) as error:+ print(f"qpack install failed: {error}", file=sys.stderr)+ return 1+ print(json.dumps({+ "output": str(result.output_dir),+ "commit": result.commit,+ "downloaded_files": result.downloaded_files,+ "downloaded_bytes": result.downloaded_bytes,+ "resumed_files": result.resumed_files,+ "already_complete": result.already_complete,+ }, sort_keys=True))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/c/tools/qpack_mirror_install.py b/c/tools/qpack_mirror_install.pynew file mode 100644index 00000000..3dcd125f--- /dev/null+++ b/c/tools/qpack_mirror_install.py@@ -0,0 +1,71 @@+#!/usr/bin/env python3+"""Install a hash-indexed qpack from a static HTTPS mirror."""++from __future__ import annotations++import argparse+import json+import os+import re+import sys+from pathlib import Path++try:+ from tools.qpack_http_install import QpackHTTPError, install_mirror_qpack+ from tools.qpack_install_policy import QpackInstallError+except ModuleNotFoundError: # Direct execution from c/tools.+ from qpack_http_install import QpackHTTPError, install_mirror_qpack+ from qpack_install_policy import QpackInstallError+++ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")+++def _parse_args(argv=None):+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("base_url", help="HTTPS prefix containing hashes.json")+ parser.add_argument("--output", required=True, type=Path,+ help="destination .qpack directory")+ parser.add_argument("--hashes-sha256",+ help="optional out-of-band SHA-256 pin for hashes.json")+ parser.add_argument(+ "--bearer-token-env", metavar="NAME",+ help="read an optional mirror bearer token from environment variable NAME")+ parser.add_argument("--retries", type=int, default=5)+ parser.add_argument("--timeout", type=float, default=120.0)+ return parser.parse_args(argv)+++def main(argv=None):+ args = _parse_args(argv)+ authorization = None+ try:+ if args.bearer_token_env is not None:+ if not ENV_NAME_RE.fullmatch(args.bearer_token_env):+ raise QpackHTTPError("bearer token environment name is invalid")+ token = os.environ.get(args.bearer_token_env)+ if not token:+ raise QpackHTTPError(+ f"bearer token environment variable is empty: "+ f"{args.bearer_token_env}")+ authorization = f"Bearer {token}"+ result = install_mirror_qpack(+ args.base_url, args.output, hashes_sha256=args.hashes_sha256,+ authorization=authorization, retries=args.retries,+ timeout=args.timeout)+ except (OSError, QpackInstallError, QpackHTTPError) as error:+ print(f"qpack mirror install failed: {error}", file=sys.stderr)+ return 1+ print(json.dumps({+ "output": str(result.output_dir),+ "source_id": result.source_id,+ "downloaded_files": result.downloaded_files,+ "downloaded_bytes": result.downloaded_bytes,+ "resumed_files": result.resumed_files,+ "already_complete": result.already_complete,+ }, sort_keys=True))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())