Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(tools): hash-verified qpack installers for Hugging Face and static mirrors

Léon Avic Simmonse547494ebf67
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index 01a7a2ce..673e5d5f 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -59,3 +59,13 @@ text.
manifest-last completion policy from Swiftlet's `StreamingInstaller.swift` at
commit `86246618ba2af30334227e09ff84a6a7182c2a40`. It is a new
transport-neutral Python implementation and does not copy Swift source code.
+
+`c/tools/qpack_http_install.py` is a new Python HTTP frontend over that policy.
+It interoperates with Hugging Face-hosted Swiftlet qpack repositories but does
+not copy Swiftlet or huggingface_hub source code.
+
+`c/tools/qpack_mirror_install.py` and the static-mirror support in
+`c/tools/qpack_http_install.py` interoperate with the `hashes.json` schema
+emitted by Swiftlet's `scripts/verify_container.py`. They are new Python
+implementations with stricter path, transport, credential, resume, and digest
+validation; no Swiftlet source code is copied.
diff --git a/c/tests/fixtures/swiftlet_qpack_manifest.json b/c/tests/fixtures/swiftlet_qpack_manifest.json
new file mode 100644
index 00000000..0f881dbe
--- /dev/null
+++ b/c/tests/fixtures/swiftlet_qpack_manifest.json
@@ -0,0 +1,57 @@
+{
+ "files" : {
+ "chat_template.jinja" : 7764,
+ "config.json" : 23591,
+ "model.safetensors" : 1389579123,
+ "packed_experts/layer_00.bin" : 452984832,
+ "packed_experts/layer_01.bin" : 452984832,
+ "packed_experts/layer_02.bin" : 452984832,
+ "packed_experts/layer_03.bin" : 452984832,
+ "packed_experts/layer_04.bin" : 452984832,
+ "packed_experts/layer_05.bin" : 452984832,
+ "packed_experts/layer_06.bin" : 452984832,
+ "packed_experts/layer_07.bin" : 452984832,
+ "packed_experts/layer_08.bin" : 452984832,
+ "packed_experts/layer_09.bin" : 452984832,
+ "packed_experts/layer_10.bin" : 452984832,
+ "packed_experts/layer_11.bin" : 452984832,
+ "packed_experts/layer_12.bin" : 452984832,
+ "packed_experts/layer_13.bin" : 452984832,
+ "packed_experts/layer_14.bin" : 452984832,
+ "packed_experts/layer_15.bin" : 452984832,
+ "packed_experts/layer_16.bin" : 452984832,
+ "packed_experts/layer_17.bin" : 452984832,
+ "packed_experts/layer_18.bin" : 452984832,
+ "packed_experts/layer_19.bin" : 452984832,
+ "packed_experts/layer_20.bin" : 452984832,
+ "packed_experts/layer_21.bin" : 452984832,
+ "packed_experts/layer_22.bin" : 452984832,
+ "packed_experts/layer_23.bin" : 452984832,
+ "packed_experts/layer_24.bin" : 452984832,
+ "packed_experts/layer_25.bin" : 452984832,
+ "packed_experts/layer_26.bin" : 452984832,
+ "packed_experts/layer_27.bin" : 452984832,
+ "packed_experts/layer_28.bin" : 452984832,
+ "packed_experts/layer_29.bin" : 452984832,
+ "packed_experts/layer_30.bin" : 452984832,
+ "packed_experts/layer_31.bin" : 452984832,
+ "packed_experts/layer_32.bin" : 452984832,
+ "packed_experts/layer_33.bin" : 452984832,
+ "packed_experts/layer_34.bin" : 452984832,
+ "packed_experts/layer_35.bin" : 452984832,
+ "packed_experts/layer_36.bin" : 452984832,
+ "packed_experts/layer_37.bin" : 452984832,
+ "packed_experts/layer_38.bin" : 452984832,
+ "packed_experts/layer_39.bin" : 452984832,
+ "packed_experts/layout.json" : 2048,
+ "tokenizer.json" : 19989343,
+ "tokenizer_config.json" : 1139,
+ "vocab.json" : 6722759
+ },
+ "magic" : "QPACK",
+ "modelName" : "qwen3_5_moe_text",
+ "quantBits" : 4,
+ "quantGroupSize" : 64,
+ "sourceCheckpoint" : "qwen3.6-35b-mlx4bit",
+ "version" : 1
+}
diff --git a/c/tests/test_qpack_http_install.py b/c/tests/test_qpack_http_install.py
new file mode 100644
index 00000000..3f347531
--- /dev/null
+++ b/c/tests/test_qpack_http_install.py
@@ -0,0 +1,532 @@
+import hashlib
+import http.client
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from urllib.request import Request
+
+from tools.qpack_http_install import (
+ STATE_NAME,
+ QpackHTTPError,
+ SafeRedirectHandler,
+ _check_adapter_namespace,
+ _declared_auxiliary_names,
+ install_hf_qpack,
+)
+from tools.qpack_install_policy import parse_manifest
+
+
+COMMIT = "0123456789abcdef0123456789abcdef01234567"
+
+
+class Headers:
+ def __init__(self, values=()):
+ self.values = list(values)
+
+ def get_all(self, name, default=None):
+ matches = [value for key, value in self.values
+ if key.lower() == name.lower()]
+ return matches if matches else (default or [])
+
+ def items(self):
+ return iter(self.values)
+
+
+class Response:
+ def __init__(self, status, body=b"", headers=()):
+ self.status = status
+ self.body = body
+ self.headers = Headers(headers)
+ self.offset = 0
+ self.closed = False
+
+ def read(self, size=-1):
+ if size is None or size < 0:
+ size = len(self.body) - self.offset
+ chunk = self.body[self.offset:self.offset + size]
+ self.offset += len(chunk)
+ return chunk
+
+ def close(self):
+ self.closed = True
+
+
+class IncompleteResponse(Response):
+ def read(self, _size=-1):
+ if self.offset == 0:
+ self.offset = len(self.body)
+ raise http.client.IncompleteRead(self.body)
+ return b""
+
+
+class ScriptedTransport:
+ def __init__(self, *responses):
+ self.responses = list(responses)
+ self.calls = []
+
+ def request(self, method, url, headers, timeout):
+ self.calls.append({
+ "method": method,
+ "url": url,
+ "headers": dict(headers),
+ "timeout": timeout,
+ })
+ if not self.responses:
+ raise AssertionError(f"unexpected request: {method} {url}")
+ response = self.responses.pop(0)
+ if callable(response):
+ return response(self.calls[-1])
+ return response
+
+ def assert_empty(self):
+ if self.responses:
+ raise AssertionError(f"{len(self.responses)} scripted responses unused")
+
+
+class QpackHTTPInstallTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name) / "tiny.qpack"
+ self.files = {
+ "model.safetensors": b"weights",
+ "packed_experts/layout.json": b"layout",
+ }
+ self.config = b'{"model_type":"qwen3_next"}'
+ self.manifest = self.make_manifest(
+ {name: len(value) for name, value in self.files.items()})
+
+ @staticmethod
+ def make_manifest(files):
+ return json.dumps({
+ "magic": "QPACK",
+ "version": 1,
+ "modelName": "tiny-qwen",
+ "sourceCheckpoint": f"owner/model@{COMMIT}",
+ "quantBits": 4,
+ "quantGroupSize": 32,
+ "files": files,
+ }, sort_keys=True, separators=(",", ":")).encode()
+
+ @staticmethod
+ def response(body, *, status=200, etag=None, checksum=False,
+ content_range=None, extra=()):
+ headers = [("Content-Length", str(len(body))),
+ ("X-Repo-Commit", COMMIT)]
+ if etag is not None:
+ headers.append(("ETag", etag))
+ if checksum:
+ headers.append(("X-Checksum-Sha256",
+ hashlib.sha256(body).hexdigest()))
+ if content_range is not None:
+ headers.append(("Content-Range", content_range))
+ headers.extend(extra)
+ return Response(status, body, headers)
+
+ def manifest_response(self):
+ return self.response(self.manifest, etag='"manifest"', checksum=True)
+
+ def config_response(self, body=None, status=200):
+ body = self.config if body is None else body
+ return self.response(body, status=status, etag='"config"', checksum=True)
+
+ def artifact_response(self, name, *, offset=0, status=None, etag=None,
+ body=None, extra=()):
+ value = self.files[name]
+ body = value[offset:] if body is None else body
+ status = status if status is not None else (206 if offset else 200)
+ content_range = None
+ if status == 206:
+ content_range = f"bytes {offset}-{len(value) - 1}/{len(value)}"
+ checksum = hashlib.sha256(value).hexdigest()
+ headers = [("Content-Length", str(len(body))),
+ ("X-Repo-Commit", COMMIT),
+ ("ETag", etag or f'"{name}"'),
+ ("X-Checksum-Sha256", checksum)]
+ if content_range is not None:
+ headers.append(("Content-Range", content_range))
+ headers.extend(extra)
+ return Response(status, body, headers)
+
+ def success_responses(self, include_config=True):
+ responses = [self.manifest_response()]
+ if include_config:
+ responses.append(self.config_response())
+ responses.extend(self.artifact_response(name)
+ for name in sorted(self.files))
+ return responses
+
+ def install(self, transport, *, root=None, revision=COMMIT, retries=0,
+ token=None, optional_aux=()):
+ return install_hf_qpack(
+ "owner/model", revision, root or self.root,
+ endpoint="https://hub.example", token=token,
+ transport=transport, retries=retries, timeout=7,
+ chunk_size=3, optional_aux=optional_aux,
+ sleep=lambda _seconds: None,
+ log=lambda _message: None)
+
+ def test_resolves_revision_then_uses_only_commit_pinned_urls(self):
+ api = Response(200, json.dumps({"sha": COMMIT}).encode(),
+ [("Content-Length", str(len(json.dumps({"sha": COMMIT}))))])
+ transport = ScriptedTransport(api, *self.success_responses())
+
+ result = self.install(transport, revision="main", token="secret-token")
+
+ self.assertEqual(result.commit, COMMIT)
+ self.assertEqual(result.downloaded_files, 3)
+ self.assertEqual(result.downloaded_bytes,
+ len(self.config) + sum(map(len, self.files.values())))
+ self.assertTrue((self.root / "manifest.json").is_file())
+ self.assertEqual((self.root / "config.json").read_bytes(), self.config)
+ for name, value in self.files.items():
+ self.assertEqual((self.root / name).read_bytes(), value)
+ self.assertIn("/api/models/owner/model/revision/main",
+ transport.calls[0]["url"])
+ for call in transport.calls[1:]:
+ self.assertIn(f"/resolve/{COMMIT}/", call["url"])
+ self.assertEqual(call["headers"]["Authorization"],
+ "Bearer secret-token")
+ state = (self.root / STATE_NAME).read_text(encoding="utf-8")
+ self.assertIn(f"owner/model@{COMMIT}", state)
+ self.assertNotIn("secret-token", state)
+ transport.assert_empty()
+
+ def test_short_body_retries_with_range_and_if_range(self):
+ first = self.artifact_response(
+ "model.safetensors", body=self.files["model.safetensors"][:3])
+ # The declared response length is the full object, but EOF arrives early.
+ first.headers = Headers([
+ ("Content-Length", str(len(self.files["model.safetensors"]))),
+ ("X-Repo-Commit", COMMIT),
+ ("ETag", '"model.safetensors"'),
+ ("X-Checksum-Sha256",
+ hashlib.sha256(self.files["model.safetensors"]).hexdigest()),
+ ])
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(), first,
+ self.artifact_response("model.safetensors", offset=3),
+ self.artifact_response("packed_experts/layout.json"))
+
+ result = self.install(transport, retries=1)
+
+ model_calls = [call for call in transport.calls
+ if call["url"].endswith("/model.safetensors")]
+ self.assertEqual(model_calls[0]["headers"]["Range"], "bytes=0-6")
+ self.assertNotIn("If-Range", model_calls[0]["headers"])
+ self.assertEqual(model_calls[1]["headers"]["Range"], "bytes=3-6")
+ self.assertEqual(model_calls[1]["headers"]["If-Range"],
+ '"model.safetensors"')
+ self.assertEqual(result.resumed_files, 1)
+ self.assertEqual((self.root / "model.safetensors").read_bytes(),
+ self.files["model.safetensors"])
+ transport.assert_empty()
+
+ def test_incomplete_read_partial_bytes_are_resumed(self):
+ value = self.files["model.safetensors"]
+ first = IncompleteResponse(200, value[:3], [
+ ("Content-Length", str(len(value))),
+ ("ETag", '"model.safetensors"'),
+ ("X-Checksum-Sha256", hashlib.sha256(value).hexdigest()),
+ ])
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(), first,
+ self.artifact_response("model.safetensors", offset=3),
+ self.artifact_response("packed_experts/layout.json"))
+
+ self.install(transport, retries=1)
+
+ model_calls = [call for call in transport.calls
+ if call["url"].endswith("/model.safetensors")]
+ self.assertEqual(model_calls[1]["headers"]["Range"], "bytes=3-6")
+ self.assertEqual((self.root / "model.safetensors").read_bytes(), value)
+
+ def test_resume_rejects_http_200_and_keeps_existing_prefix(self):
+ first = self.artifact_response(
+ "model.safetensors", body=self.files["model.safetensors"][:3])
+ first.headers = Headers([
+ ("Content-Length", str(len(self.files["model.safetensors"]))),
+ ("ETag", '"model.safetensors"'),
+ ])
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(), first)
+ with self.assertRaisesRegex(QpackHTTPError, "short HTTP body"):
+ self.install(transport)
+ state = json.loads((self.root / STATE_NAME).read_text())
+ self.assertEqual(state["files"]["model.safetensors"]["etag"],
+ '"model.safetensors"')
+
+ resumed = self.artifact_response(
+ "model.safetensors", offset=3, status=200)
+ transport = ScriptedTransport(self.manifest_response(), resumed)
+ with self.assertRaisesRegex(QpackHTTPError, "did not return HTTP 206"):
+ self.install(transport)
+ self.assertEqual((self.root / "model.safetensors.part").read_bytes(),
+ self.files["model.safetensors"][:3])
+
+ def test_resume_rejects_changed_etag_before_appending(self):
+ first = self.artifact_response(
+ "model.safetensors", body=self.files["model.safetensors"][:3])
+ first.headers = Headers([
+ ("Content-Length", str(len(self.files["model.safetensors"]))),
+ ("ETag", '"original"'),
+ ])
+ with self.assertRaises(QpackHTTPError):
+ self.install(ScriptedTransport(
+ self.manifest_response(), self.config_response(), first))
+
+ changed = self.artifact_response(
+ "model.safetensors", offset=3, etag='"changed"')
+ with self.assertRaisesRegex(QpackHTTPError, "ETag changed"):
+ self.install(ScriptedTransport(self.manifest_response(), changed))
+ self.assertEqual((self.root / "model.safetensors.part").read_bytes(),
+ self.files["model.safetensors"][:3])
+
+ def test_checksum_mismatch_resets_partial_and_never_publishes_manifest(self):
+ response = self.artifact_response("model.safetensors")
+ response.headers = Headers([
+ ("Content-Length", str(len(self.files["model.safetensors"]))),
+ ("X-Checksum-Sha256", "f" * 64),
+ ])
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(), response)
+
+ with self.assertRaisesRegex(QpackHTTPError, "SHA-256 mismatch"):
+ self.install(transport)
+
+ self.assertEqual((self.root / "model.safetensors.part").stat().st_size, 0)
+ self.assertFalse((self.root / "manifest.json").exists())
+
+ def test_range_and_encoding_protocol_errors_fail_closed(self):
+ cases = {
+ "missing-range": Response(206, self.files["model.safetensors"], [
+ ("Content-Length", "7"),
+ ]),
+ "wrong-range": Response(206, self.files["model.safetensors"], [
+ ("Content-Length", "7"), ("Content-Range", "bytes 1-6/7"),
+ ]),
+ "range-on-200": Response(200, self.files["model.safetensors"], [
+ ("Content-Length", "7"), ("Content-Range", "bytes 0-6/7"),
+ ]),
+ "duplicate-length": Response(200, self.files["model.safetensors"], [
+ ("Content-Length", "7"), ("Content-Length", "7"),
+ ]),
+ "compressed": Response(200, self.files["model.safetensors"], [
+ ("Content-Length", "7"), ("Content-Encoding", "gzip"),
+ ]),
+ }
+ for label, response in cases.items():
+ with self.subTest(label=label):
+ root = Path(self.temporary.name) / f"{label}.qpack"
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(), response)
+ with self.assertRaises(QpackHTTPError):
+ self.install(transport, root=root)
+ self.assertFalse((root / "manifest.json").exists())
+
+ def test_required_config_must_exist_and_be_valid_before_weights(self):
+ cases = [
+ self.config_response(body=b"", status=404),
+ self.config_response(body=b"not-json"),
+ self.config_response(body=b"[]"),
+ ]
+ for index, response in enumerate(cases):
+ with self.subTest(index=index):
+ root = Path(self.temporary.name) / f"config-{index}.qpack"
+ transport = ScriptedTransport(self.manifest_response(), response)
+ with self.assertRaises(QpackHTTPError):
+ self.install(transport, root=root)
+ self.assertEqual(len(transport.calls), 2)
+ self.assertFalse((root / "manifest.json").exists())
+
+ def test_manifest_declared_auxiliaries_use_the_resumable_artifact_path(self):
+ tokenizer = b'{"version":"1.0"}'
+ files = dict(self.files)
+ files["config.json"] = self.config
+ files["tokenizer.json"] = tokenizer
+ raw = self.make_manifest({name: len(value) for name, value in files.items()})
+ responses = [self.response(raw)]
+ for name in ["config.json", "tokenizer.json", "model.safetensors",
+ "packed_experts/layout.json"]:
+ value = files[name]
+ responses.append(Response(200, value, [
+ ("Content-Length", str(len(value))),
+ ("ETag", f'"{name}"'),
+ ("X-Checksum-Sha256", hashlib.sha256(value).hexdigest()),
+ ]))
+ transport = ScriptedTransport(*responses)
+
+ self.install(transport)
+
+ self.assertEqual((self.root / "config.json").read_bytes(), self.config)
+ self.assertEqual((self.root / "tokenizer.json").read_bytes(), tokenizer)
+ artifact_calls = [call["url"].rsplit("/", 1)[-1]
+ for call in transport.calls[1:]]
+ self.assertEqual(artifact_calls[0], "config.json")
+ transport.assert_empty()
+
+ def test_optional_auxiliary_generator_is_not_consumed_by_validation(self):
+ tokenizer = b'{"version":"1.0"}'
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(),
+ self.response(tokenizer, checksum=True),
+ *(self.artifact_response(name) for name in sorted(self.files)))
+
+ self.install(
+ transport,
+ optional_aux=(name for name in ("tokenizer.json",)))
+
+ self.assertEqual((self.root / "tokenizer.json").read_bytes(), tokenizer)
+ self.assertTrue(transport.calls[2]["url"].endswith("/tokenizer.json"))
+ transport.assert_empty()
+
+ def test_real_swiftlet_manifest_declared_auxiliaries_are_accepted(self):
+ fixture = Path(__file__).parent / "fixtures" / "swiftlet_qpack_manifest.json"
+ plan = parse_manifest(fixture.read_bytes(), f"hf:owner/model@{COMMIT}")
+ _check_adapter_namespace(plan)
+ self.assertEqual(
+ _declared_auxiliary_names(plan),
+ {"chat_template.jinja", "config.json", "tokenizer.json",
+ "tokenizer_config.json", "vocab.json"})
+ self.assertEqual(len(plan.files), 47)
+
+ def test_hugging_face_manifest_may_declare_hashes_json_as_an_artifact(self):
+ files = dict(self.files)
+ files["hashes.json"] = b'{"files":{}}'
+ raw = self.make_manifest({name: len(value) for name, value in files.items()})
+ responses = [self.response(raw), self.config_response()]
+ for name in sorted(files):
+ value = files[name]
+ responses.append(Response(200, value, [
+ ("Content-Length", str(len(value))),
+ ("ETag", f'"{name}"'),
+ ("X-Checksum-Sha256", hashlib.sha256(value).hexdigest()),
+ ]))
+
+ self.install(ScriptedTransport(*responses))
+
+ self.assertEqual((self.root / "hashes.json").read_bytes(), files["hashes.json"])
+
+ def test_manifest_cannot_claim_adapter_state_or_sidecar_namespace(self):
+ for name in (STATE_NAME, "Config.json", "tokenizer.json/child",
+ ".tokenizer.json.part"):
+ with self.subTest(name=name):
+ files = {"packed_experts/layout.json": 6, name: 1}
+ raw = self.make_manifest(files)
+ transport = ScriptedTransport(self.response(raw))
+ with self.assertRaisesRegex(QpackHTTPError, "collides"):
+ self.install(transport,
+ root=Path(self.temporary.name) / name.replace("/", "_"))
+ self.assertEqual(len(transport.calls), 1)
+
+ def test_completed_install_is_offline_checked_against_validator_state(self):
+ transport = ScriptedTransport(*self.success_responses())
+ self.install(transport)
+
+ transport = ScriptedTransport(self.manifest_response())
+ result = self.install(transport)
+ self.assertTrue(result.already_complete)
+ self.assertEqual(result.downloaded_files, 0)
+ self.assertEqual(len(transport.calls), 1)
+
+ (self.root / "model.safetensors").write_bytes(b"WEIGHTS")
+ with self.assertRaisesRegex(QpackHTTPError, "artifact changed"):
+ self.install(ScriptedTransport(self.manifest_response()))
+
+ def test_generic_digest_on_resumed_range_is_not_a_full_object_checksum(self):
+ value = self.files["model.safetensors"]
+ first = self.artifact_response("model.safetensors", body=value[:3])
+ first.headers = Headers([
+ ("Content-Length", str(len(value))),
+ ("ETag", '"model.safetensors"'),
+ ("Digest", "sha-256=:range-one:"),
+ ])
+ second = self.artifact_response("model.safetensors", offset=3)
+ second.headers = Headers([
+ ("Content-Length", str(len(value) - 3)),
+ ("Content-Range", f"bytes 3-{len(value) - 1}/{len(value)}"),
+ ("ETag", '"model.safetensors"'),
+ ("Digest", "sha-256=:range-two:"),
+ ])
+ transport = ScriptedTransport(
+ self.manifest_response(), self.config_response(), first, second,
+ self.artifact_response("packed_experts/layout.json"))
+ self.install(transport, retries=1)
+ state = json.loads((self.root / STATE_NAME).read_text())
+ self.assertIsNone(state["files"]["model.safetensors"]["sha256"])
+ self.assertEqual(state["files"]["model.safetensors"]["local_sha256"],
+ hashlib.sha256(value).hexdigest())
+
+ def test_lost_http_state_rejects_an_already_committed_artifact(self):
+ short_layout = self.artifact_response(
+ "packed_experts/layout.json",
+ body=self.files["packed_experts/layout.json"][:2])
+ short_layout.headers = Headers([
+ ("Content-Length", str(len(self.files["packed_experts/layout.json"]))),
+ ("ETag", '"layout"'),
+ ])
+ with self.assertRaisesRegex(QpackHTTPError, "short HTTP body"):
+ self.install(ScriptedTransport(
+ self.manifest_response(), self.config_response(),
+ self.artifact_response("model.safetensors"), short_layout))
+ self.assertTrue((self.root / "model.safetensors").is_file())
+ (self.root / STATE_NAME).unlink()
+
+ with self.assertRaisesRegex(QpackHTTPError, "no validator"):
+ self.install(ScriptedTransport(
+ self.manifest_response(), self.config_response()))
+ self.assertFalse((self.root / "manifest.json").exists())
+
+ def test_revision_resolution_retries_transient_status(self):
+ api_body = json.dumps({"sha": COMMIT}).encode()
+ retry = Response(503, b"", [("Content-Length", "0")])
+ api = Response(200, api_body,
+ [("Content-Length", str(len(api_body)))])
+ transport = ScriptedTransport(retry, api, *self.success_responses())
+ self.install(transport, revision="main", retries=1)
+ self.assertEqual(len([call for call in transport.calls
+ if "/api/models/" in call["url"]]), 2)
+
+ def test_endpoint_must_be_an_https_origin(self):
+ for endpoint in ("http://hub.example", "https://hub.example/path",
+ "https://[email protected]"):
+ with self.subTest(endpoint=endpoint), self.assertRaisesRegex(
+ QpackHTTPError, "HTTPS origin"):
+ install_hf_qpack(
+ "owner/model", COMMIT, self.root, endpoint=endpoint,
+ transport=ScriptedTransport(), optional_aux=())
+
+
+class SafeRedirectHandlerTest(unittest.TestCase):
+ def setUp(self):
+ self.handler = SafeRedirectHandler()
+
+ @staticmethod
+ def request():
+ return Request("https://hub.example/object",
+ headers={"Authorization": "Bearer secret"})
+
+ def test_cross_origin_redirect_strips_authorization(self):
+ redirected = self.handler.redirect_request(
+ self.request(), None, 302, "Found", {},
+ "https://cdn.example/object")
+ self.assertIsNone(redirected.get_header("Authorization"))
+
+ def test_same_origin_redirect_keeps_authorization(self):
+ redirected = self.handler.redirect_request(
+ self.request(), None, 302, "Found", {},
+ "https://hub.example/elsewhere")
+ self.assertEqual(redirected.get_header("Authorization"), "Bearer secret")
+
+ def test_redirect_rejects_downgrade_and_userinfo(self):
+ for url in ("http://hub.example/object",
+ "https://[email protected]/object"):
+ with self.subTest(url=url), self.assertRaisesRegex(
+ QpackHTTPError, "unsafe redirect"):
+ self.handler.redirect_request(
+ self.request(), None, 302, "Found", {}, url)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_qpack_mirror_install.py b/c/tests/test_qpack_mirror_install.py
new file mode 100644
index 00000000..2ab85867
--- /dev/null
+++ b/c/tests/test_qpack_mirror_install.py
@@ -0,0 +1,422 @@
+import hashlib
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+from urllib.request import Request
+
+from tools.qpack_http_install import (
+ HASHES_NAME,
+ QpackHTTPError,
+ SafeRedirectHandler,
+ install_mirror_qpack,
+ parse_hashes_index,
+)
+from tools.qpack_install_policy import QpackInstallError
+
+
+class Headers:
+ def __init__(self, values=()):
+ self.values = list(values)
+
+ def get_all(self, name, default=None):
+ matches = [value for key, value in self.values
+ if key.lower() == name.lower()]
+ return matches if matches else (default or [])
+
+ def items(self):
+ return iter(self.values)
+
+
+class Response:
+ def __init__(self, status, body=b"", headers=()):
+ self.status = status
+ self.body = body
+ self.headers = Headers(headers)
+ self.offset = 0
+
+ def read(self, size=-1):
+ if size is None or size < 0:
+ size = len(self.body) - self.offset
+ chunk = self.body[self.offset:self.offset + size]
+ self.offset += len(chunk)
+ return chunk
+
+ def close(self):
+ pass
+
+
+class ScriptedTransport:
+ def __init__(self, *responses):
+ self.responses = list(responses)
+ self.calls = []
+
+ def request(self, method, url, headers, timeout):
+ call = {
+ "method": method,
+ "url": url,
+ "headers": dict(headers),
+ "timeout": timeout,
+ }
+ self.calls.append(call)
+ if not self.responses:
+ raise AssertionError(f"unexpected request: {method} {url}")
+ response = self.responses.pop(0)
+ return response(call) if callable(response) else response
+
+ def assert_empty(self):
+ if self.responses:
+ raise AssertionError(f"{len(self.responses)} scripted responses unused")
+
+
+class QpackMirrorInstallTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name) / "tiny.qpack"
+ self.base_url = "https://mirror.example/models/tiny"
+ self.config = b'{"model_type":"qwen3_next"}'
+ self.files = {
+ "model.safetensors": b"weights",
+ "packed_experts/layout.json": b"layout",
+ }
+ self.manifest = json.dumps({
+ "magic": "QPACK",
+ "version": 1,
+ "modelName": "tiny-qwen",
+ "sourceCheckpoint": "owner/model@static",
+ "quantBits": 4,
+ "quantGroupSize": 32,
+ "files": {name: len(value) for name, value in self.files.items()},
+ }, sort_keys=True, separators=(",", ":")).encode()
+
+ @staticmethod
+ def response(body, *, status=200, etag=None, content_length=None,
+ content_range=None, checksum=None):
+ headers = [("Content-Length", str(
+ len(body) if content_length is None else content_length))]
+ if etag is not None:
+ headers.append(("ETag", etag))
+ if content_range is not None:
+ headers.append(("Content-Range", content_range))
+ if checksum is not None:
+ headers.append(("X-Checksum-Sha256", checksum))
+ return Response(status, body, headers)
+
+ def hash_map(self, *, extra=None, omit=(), replacements=None):
+ values = {
+ "manifest.json": self.manifest,
+ "config.json": self.config,
+ **self.files,
+ **(extra or {}),
+ }
+ hashes = {
+ name: hashlib.sha256(value).hexdigest()
+ for name, value in values.items() if name not in omit
+ }
+ hashes.update(replacements or {})
+ return hashes
+
+ def hashes_raw(self, **kwargs):
+ return json.dumps(
+ {"files": self.hash_map(**kwargs)},
+ sort_keys=True, separators=(",", ":")).encode()
+
+ def artifact_response(self, name, *, offset=0, status=None, etag=None,
+ body=None):
+ value = self.files[name]
+ body = value[offset:] if body is None else body
+ status = status if status is not None else (206 if offset else 200)
+ content_range = None
+ if status == 206:
+ content_range = f"bytes {offset}-{len(value) - 1}/{len(value)}"
+ return self.response(
+ body, status=status, etag=etag,
+ content_length=len(body), content_range=content_range)
+
+ def success_responses(self, *, hashes_raw=None, optional=()):
+ hashes_raw = hashes_raw or self.hashes_raw(
+ extra=dict(optional))
+ responses = [
+ self.response(hashes_raw, etag='"hashes"'),
+ self.response(self.manifest, etag='"manifest"'),
+ self.response(self.config, etag='"config"'),
+ ]
+ responses.extend(self.response(value) for _name, value in optional)
+ responses.extend(self.artifact_response(name) for name in sorted(self.files))
+ return responses
+
+ def install(self, transport, *, root=None, **kwargs):
+ return install_mirror_qpack(
+ self.base_url, root or self.root, transport=transport,
+ retries=kwargs.pop("retries", 0), timeout=7, chunk_size=3,
+ sleep=lambda _seconds: None, log=lambda _message: None, **kwargs)
+
+ def test_installs_legacy_index_and_persists_exact_raw_hashes(self):
+ hashes_raw = self.hashes_raw()
+ transport = ScriptedTransport(*self.success_responses(hashes_raw=hashes_raw))
+
+ with mock.patch.dict(os.environ, {"HF_TOKEN": "must-not-leak"}):
+ result = self.install(transport)
+
+ self.assertIsNone(result.commit)
+ self.assertEqual(
+ result.source_id,
+ f"mirror:{self.base_url}@sha256:"
+ f"{hashlib.sha256(hashes_raw).hexdigest()}")
+ self.assertEqual(result.downloaded_files, 3)
+ self.assertEqual((self.root / "manifest.json").read_bytes(), self.manifest)
+ self.assertEqual((self.root / "config.json").read_bytes(), self.config)
+ self.assertEqual((self.root / HASHES_NAME).read_bytes(), hashes_raw)
+ self.assertTrue(all("Authorization" not in call["headers"]
+ for call in transport.calls))
+ self.assertEqual(
+ [call["url"].rsplit("/", 1)[-1] for call in transport.calls[:3]],
+ ["hashes.json", "manifest.json", "config.json"])
+ self.assertEqual(transport.calls[0]["headers"]["Cache-Control"],
+ "no-cache")
+ transport.assert_empty()
+
+ def test_v1_and_legacy_have_the_same_semantic_identity(self):
+ hashes = self.hash_map()
+ legacy = json.dumps({"files": hashes}, indent=2).encode()
+ v1 = json.dumps({
+ "files": {
+ name: {"sha256": digest, "size": len({
+ "manifest.json": self.manifest,
+ "config.json": self.config,
+ **self.files,
+ }[name])}
+ for name, digest in reversed(list(hashes.items()))
+ },
+ "schema": "qpack.hashes.v1",
+ }, separators=(",", ":")).encode()
+
+ legacy_entries, legacy_id = parse_hashes_index(legacy)
+ v1_entries, v1_id = parse_hashes_index(v1)
+
+ self.assertEqual(legacy_id, v1_id)
+ self.assertEqual(
+ {name: entry.sha256 for name, entry in legacy_entries.items()},
+ {name: entry.sha256 for name, entry in v1_entries.items()})
+
+ def test_v1_size_must_match_the_manifest_before_install_starts(self):
+ values = {
+ "manifest.json": self.manifest,
+ "config.json": self.config,
+ **self.files,
+ }
+ index = {
+ name: {
+ "size": len(value) + int(name == "model.safetensors"),
+ "sha256": hashlib.sha256(value).hexdigest(),
+ }
+ for name, value in values.items()
+ }
+ hashes_raw = json.dumps({
+ "schema": "qpack.hashes.v1",
+ "files": index,
+ }).encode()
+ transport = ScriptedTransport(
+ self.response(hashes_raw), self.response(self.manifest))
+
+ with self.assertRaisesRegex(QpackHTTPError, "size mismatch"):
+ self.install(transport)
+
+ self.assertFalse(self.root.exists())
+
+ def test_raw_hash_pin_and_manifest_hash_fail_before_destination_creation(self):
+ hashes_raw = self.hashes_raw()
+ with self.assertRaisesRegex(QpackHTTPError, "pinned SHA-256"):
+ self.install(
+ ScriptedTransport(self.response(hashes_raw)),
+ hashes_sha256="f" * 64)
+ self.assertFalse(self.root.exists())
+
+ wrong = self.hashes_raw(replacements={"manifest.json": "f" * 64})
+ with self.assertRaisesRegex(QpackHTTPError, "SHA-256 mismatch"):
+ self.install(ScriptedTransport(
+ self.response(wrong), self.response(self.manifest)))
+ self.assertFalse(self.root.exists())
+
+ def test_missing_hash_entry_stops_before_large_transfers(self):
+ hashes_raw = self.hashes_raw(omit=("config.json",))
+ transport = ScriptedTransport(
+ self.response(hashes_raw), self.response(self.manifest))
+
+ with self.assertRaisesRegex(QpackHTTPError, "does not cover"):
+ self.install(transport)
+
+ self.assertEqual(len(transport.calls), 2)
+ self.assertFalse(self.root.exists())
+
+ def test_extra_hashed_sidecars_are_validated_but_not_downloaded(self):
+ hashes_raw = self.hashes_raw(extra={
+ "README.md": b"notes",
+ "licenses/NOTICE.txt": b"notice",
+ })
+ transport = ScriptedTransport(
+ *self.success_responses(hashes_raw=hashes_raw))
+
+ self.install(transport)
+
+ requested = {call["url"].removeprefix(self.base_url + "/")
+ for call in transport.calls}
+ self.assertNotIn("README.md", requested)
+ self.assertNotIn("licenses/NOTICE.txt", requested)
+
+ def test_resume_without_etag_uses_exact_range_and_full_hash(self):
+ value = self.files["model.safetensors"]
+ first = self.artifact_response(
+ "model.safetensors", body=value[:3])
+ first.headers = Headers([("Content-Length", str(len(value)))])
+ with self.assertRaisesRegex(QpackHTTPError, "short HTTP body"):
+ self.install(ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest),
+ self.response(self.config), first))
+
+ transport = ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest),
+ self.artifact_response("model.safetensors", offset=3),
+ self.artifact_response("packed_experts/layout.json"))
+ result = self.install(transport)
+
+ model_call = transport.calls[2]
+ self.assertEqual(model_call["headers"]["Range"], "bytes=3-6")
+ self.assertNotIn("If-Match", model_call["headers"])
+ self.assertEqual(result.resumed_files, 1)
+ self.assertEqual((self.root / "model.safetensors").read_bytes(), value)
+
+ def test_failed_if_match_restarts_without_appending(self):
+ value = self.files["model.safetensors"]
+ first = self.artifact_response(
+ "model.safetensors", body=value[:3], etag='"old"')
+ first.headers = Headers([
+ ("Content-Length", str(len(value))),
+ ("ETag", '"old"'),
+ ])
+ with self.assertRaises(QpackHTTPError):
+ self.install(ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest),
+ self.response(self.config), first))
+
+ transport = ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest),
+ self.response(b"", status=412),
+ self.artifact_response("model.safetensors", etag='"new"'),
+ self.artifact_response("packed_experts/layout.json"))
+ self.install(transport)
+
+ self.assertEqual(transport.calls[2]["headers"]["If-Match"], '"old"')
+ self.assertNotIn("If-Match", transport.calls[3]["headers"])
+ self.assertEqual(transport.calls[3]["headers"]["Range"], "bytes=0-6")
+ self.assertEqual((self.root / "model.safetensors").read_bytes(), value)
+
+ def test_changed_snapshot_cannot_resume_existing_journal(self):
+ value = self.files["model.safetensors"]
+ first = self.artifact_response("model.safetensors", body=value[:3])
+ first.headers = Headers([("Content-Length", str(len(value)))])
+ with self.assertRaises(QpackHTTPError):
+ self.install(ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest),
+ self.response(self.config), first))
+ old_hashes = (self.root / HASHES_NAME).read_bytes()
+
+ changed = self.hashes_raw(replacements={
+ "model.safetensors": hashlib.sha256(b"changed").hexdigest(),
+ })
+ transport = ScriptedTransport(
+ self.response(changed), self.response(self.manifest))
+ with self.assertRaisesRegex(QpackInstallError, "another source"):
+ self.install(transport)
+ self.assertEqual((self.root / HASHES_NAME).read_bytes(), old_hashes)
+ self.assertEqual(len(transport.calls), 2)
+
+ def test_file_hash_mismatch_truncates_and_never_publishes_manifest(self):
+ wrong = self.hashes_raw(replacements={
+ "model.safetensors": hashlib.sha256(b"different").hexdigest(),
+ })
+ transport = ScriptedTransport(
+ self.response(wrong), self.response(self.manifest),
+ self.response(self.config), self.artifact_response("model.safetensors"))
+
+ with self.assertRaisesRegex(QpackHTTPError, "SHA-256 mismatch"):
+ self.install(transport)
+
+ self.assertEqual((self.root / "model.safetensors.part").stat().st_size, 0)
+ self.assertFalse((self.root / "manifest.json").exists())
+
+ def test_listed_optional_auxiliary_is_required_and_verified(self):
+ tokenizer = b'{"version":"1.0"}'
+ hashes_raw = self.hashes_raw(extra={"tokenizer.json": tokenizer})
+ transport = ScriptedTransport(*self.success_responses(
+ hashes_raw=hashes_raw, optional=(("tokenizer.json", tokenizer),)))
+
+ self.install(transport)
+
+ self.assertEqual((self.root / "tokenizer.json").read_bytes(), tokenizer)
+ self.assertTrue(transport.calls[3]["url"].endswith("/tokenizer.json"))
+
+ root = Path(self.temporary.name) / "missing-optional.qpack"
+ transport = ScriptedTransport(
+ self.response(hashes_raw), self.response(self.manifest),
+ self.response(self.config), self.response(b"", status=404))
+ with self.assertRaisesRegex(QpackHTTPError, "HTTP 404"):
+ self.install(transport, root=root)
+
+ def test_completed_mirror_install_is_rehashed_without_object_requests(self):
+ self.install(ScriptedTransport(*self.success_responses()))
+ transport = ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest))
+
+ result = self.install(transport)
+
+ self.assertTrue(result.already_complete)
+ self.assertEqual(result.downloaded_files, 0)
+ self.assertEqual(len(transport.calls), 2)
+
+ (self.root / "model.safetensors").write_bytes(b"WEIGHTS")
+ with self.assertRaisesRegex(QpackHTTPError, "artifact changed"):
+ self.install(ScriptedTransport(
+ self.response(self.hashes_raw()), self.response(self.manifest)))
+
+ def test_hashes_parser_and_base_url_fail_closed(self):
+ digest = "a" * 64
+ invalid_indexes = [
+ b'{"files":{"manifest.json":"' + digest.encode()
+ + b'","manifest.json":"' + digest.encode() + b'"}}',
+ json.dumps({"files": {"../manifest.json": digest}}).encode(),
+ json.dumps({"files": {"hashes.json": digest}}).encode(),
+ json.dumps({"files": {"manifest.json": "bad"}}).encode(),
+ json.dumps({"schema": "future", "files": {"manifest.json": digest}}).encode(),
+ ]
+ for raw in invalid_indexes:
+ with self.subTest(raw=raw), self.assertRaises(QpackHTTPError):
+ parse_hashes_index(raw)
+
+ for base_url in (
+ "http://mirror.example/qpack",
+ "https://[email protected]/qpack",
+ "https://mirror.example/a/../qpack",
+ "https://mirror.example/qpack?token=secret"):
+ with self.subTest(base_url=base_url), self.assertRaises(QpackHTTPError):
+ install_mirror_qpack(
+ base_url, self.root, transport=ScriptedTransport())
+
+ def test_cross_origin_redirect_strips_all_explicit_secrets(self):
+ handler = SafeRedirectHandler()
+ request = Request("https://mirror.example/object", headers={
+ "Authorization": "Bearer secret",
+ "Cookie": "session=secret",
+ "X-Api-Key": "secret",
+ })
+ redirected = handler.redirect_request(
+ request, None, 302, "Found", {}, "https://cdn.example/object")
+ for name in ("Authorization", "Cookie", "X-Api-Key"):
+ self.assertIsNone(redirected.get_header(name))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/README.md b/c/tools/README.md
index 24d84cec..ffc53c0b 100644
--- a/c/tools/README.md
+++ b/c/tools/README.md
@@ -11,6 +11,27 @@ not runtime dependencies of the C engine.
transport checksum or ETag. It covers manifest-declared container artifacts,
not required runtime auxiliaries such as `config.json`. Directory-fsync crash
durability is POSIX-only.
+- `qpack_http_install.py`: dependency-free Hugging Face frontend for that
+ policy. It resolves a branch or tag to an immutable commit before opening the
+ install session, streams manifest-declared files directly into `.part`
+ files, resumes only exact HTTP ranges, and publishes `manifest.json` last.
+ `config.json` is required; manifest-declared tokenizer/configuration files
+ use the same resumable artifact path, while undeclared optional files are
+ copied as verified sidecars when present. Strong ETags, documented server
+ SHA-256 values, and a locally computed full-file SHA-256 are persisted in
+ `.qpack-http.json` and checked on resume. `HF_TOKEN` is sent only to the Hub
+ origin and is stripped on cross-origin HTTPS redirects.
+- `qpack_mirror_install.py`: static HTTPS/R2 frontend over the same transfer
+ engine. A strict `hashes.json` is required and fetched before the manifest;
+ it must cover `manifest.json`, `config.json`, and every manifest artifact.
+ The legacy Swiftlet `{"files":{"path":"sha256"}}` schema and the sized
+ `qpack.hashes.v1` schema are accepted. The normalized mirror URL plus the
+ digest of the exact raw index bytes bind resume state; indexed runtime files receive
+ authoritative full-object SHA-256 values, while other valid indexed
+ sidecars are ignored. The exact raw index is persisted beside the installed
+ container. Mirror installs never inherit
+ `HF_TOKEN`; an explicit bearer token can be read from a named environment
+ variable and is stripped on cross-origin redirects.
- `convert_fmt4_to_fmt2.py`: fmt=4 (grouped int4) -> fmt=2 (per-row int4)
re-quant of a GLM-5.2 container, for Metal-backend compatibility
(see `docs/METAL-M1ULTRA-FMT2-REPORT.md`)
@@ -35,8 +56,30 @@ Run them from `c/`, for example:
```sh
python3 tools/convert_fp8_to_int4.py --selftest
python3 tools/make_glm_bench_model.py --output /tmp/colibri-bench
+python3 tools/qpack_http_install.py OWNER/REPO \
+ --revision main --output /path/to/model.qpack
+python3 tools/qpack_mirror_install.py \
+ https://models.example/qwen/model.qpack \
+ --output /path/to/model.qpack \
+ --hashes-sha256 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
```
+An interrupted qpack install can be restarted with the same command. Keep the
+source repository and revision unchanged. The installer rejects HTTP range or
+validator inconsistencies instead of appending uncertain bytes. It requires
+`Content-Length` for large artifacts and never stages the original BF16/FP8
+checkpoint, but it does not convert a raw checkpoint into qpack.
+
+Static mirrors must use HTTPS and publish objects first, `manifest.json` next,
+and `hashes.json` last as the remote completion marker. Without
+`--hashes-sha256`, the first valid TLS response is trust on first use: the
+URL-and-index identity prevents a later snapshot from resuming into those
+partials, but it cannot authenticate a mirror that was already compromised on
+the first request. Prefer a versioned mirror prefix and an out-of-band index
+digest for production distribution. The installer uses exact byte ranges and
+`If-Match` when a strong ETag is available; final SHA-256 verification remains
+the authority.
+
`make_glm_oracle.py` also produces the quantized routed-expert fixtures for the
fmt=6 (E8/IQ3, rotation-bearing) and fmt=4 (grouped int4, no-rotation control)
parity gate (#3/#7). Only the routed experts are quantized; shared/dense/attn
diff --git a/c/tools/qpack_http_install.py b/c/tools/qpack_http_install.py
new file mode 100644
index 00000000..07b602ba
--- /dev/null
+++ b/c/tools/qpack_http_install.py
@@ -0,0 +1,1176 @@
+#!/usr/bin/env python3
+"""Install a commit-pinned Hugging Face qpack without staging raw weights."""
+
+from __future__ import annotations
+
+import argparse
+import errno
+import hashlib
+import http.client
+import json
+import os
+import re
+import stat
+import sys
+import time
+from contextlib import closing
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote, unquote, urljoin, urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+try:
+ from tools.qpack_install_policy import (
+ MAX_FILE_SIZE,
+ MAX_FILES,
+ MAX_MANIFEST,
+ QpackFile,
+ QpackInstallError,
+ _decode_json,
+ _validate_name,
+ begin_install,
+ commit_file,
+ parse_manifest,
+ publish_manifest,
+ resume_decision,
+ validate_delivery,
+ )
+except ModuleNotFoundError: # Direct execution from c/tools.
+ from qpack_install_policy import (
+ MAX_FILE_SIZE,
+ MAX_FILES,
+ MAX_MANIFEST,
+ QpackFile,
+ QpackInstallError,
+ _decode_json,
+ _validate_name,
+ begin_install,
+ commit_file,
+ parse_manifest,
+ publish_manifest,
+ resume_decision,
+ validate_delivery,
+ )
+
+
+STATE_SCHEMA = "colibri.qpack-http.v1"
+STATE_NAME = ".qpack-http.json"
+STATE_TEMP_NAME = "..qpack-http.json.part"
+HASHES_NAME = "hashes.json"
+CHUNK_SIZE = 4 * 1024 * 1024
+MAX_AUX_SIZE = 128 * 1024 * 1024
+MAX_API_SIZE = 1024 * 1024
+MAX_HASHES_SIZE = 16 * 1024 * 1024
+MAX_HASH_ENTRIES = MAX_FILES + 16
+RETRYABLE_STATUS = {429, 500, 502, 503, 504}
+COMMIT_RE = re.compile(r"^[0-9a-fA-F]{40}$")
+CONTENT_RANGE_RE = re.compile(r"^bytes ([0-9]+)-([0-9]+)/([0-9]+)$")
+SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
+REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+
+REQUIRED_AUX = ("config.json",)
+OPTIONAL_AUX = (
+ "tokenizer.json",
+ "tokenizer_config.json",
+ "vocab.json",
+ "merges.txt",
+ "chat_template.jinja",
+ "generation_config.json",
+ "special_tokens_map.json",
+ "added_tokens.json",
+)
+
+
+class QpackHTTPError(RuntimeError):
+ pass
+
+
+class RetryableTransferError(QpackHTTPError):
+ pass
+
+
+class HTTPStatusFailure(QpackHTTPError):
+ def __init__(self, url: str, status: int):
+ super().__init__(f"HTTP {status} for {_safe_url(url)}")
+ self.url = url
+ self.status = status
+
+
+@dataclass(frozen=True)
+class ResolvedSource:
+ repo: str
+ commit: str | None
+ endpoint: str
+ base_url: str
+ source_id: str
+ headers: tuple[tuple[str, str], ...]
+ repo_commit: str | None = None
+ resume_header: str = "If-Range"
+ restart_changed_resume: bool = False
+
+ def url(self, name: str) -> str:
+ encoded = "/".join(quote(component, safe="") for component in name.split("/"))
+ return f"{self.base_url}/{encoded}"
+
+
+@dataclass(frozen=True)
+class InstallResult:
+ output_dir: Path
+ commit: str | None
+ downloaded_files: int
+ downloaded_bytes: int
+ resumed_files: int
+ already_complete: bool
+ source_id: str = ""
+
+
+@dataclass(frozen=True)
+class HashEntry:
+ sha256: str
+ size: int | None = None
+
+
+def _safe_url(url: str) -> str:
+ split = urlsplit(url)
+ host = split.hostname or ""
+ port = f":{split.port}" if split.port else ""
+ return f"{split.scheme}://{host}{port}{split.path}"
+
+
+def _origin(url: str):
+ split = urlsplit(url)
+ port = split.port
+ if port is None:
+ port = 443 if split.scheme == "https" else 80
+ return split.scheme.lower(), (split.hostname or "").lower(), port
+
+
+class SafeRedirectHandler(HTTPRedirectHandler):
+ """Keep credentials on-origin and never follow a downgrade or userinfo URL."""
+
+ SENSITIVE_HEADERS = (
+ "Authorization",
+ "Cookie",
+ "Proxy-Authorization",
+ "X-Api-Key",
+ )
+
+ def redirect_request(self, request, fp, code, msg, headers, new_url):
+ target = urljoin(request.full_url, new_url)
+ split = urlsplit(target)
+ if split.scheme.lower() != "https" or split.username or split.password:
+ raise QpackHTTPError(f"unsafe redirect to {_safe_url(target)}")
+ redirected = super().redirect_request(request, fp, code, msg, headers, target)
+ if redirected is not None and _origin(request.full_url) != _origin(target):
+ for name in self.SENSITIVE_HEADERS:
+ redirected.remove_header(name)
+ for existing in list(redirected.unredirected_hdrs):
+ if existing.lower() == name.lower():
+ redirected.unredirected_hdrs.pop(existing, None)
+ return redirected
+
+
+class URLLibTransport:
+ def __init__(self):
+ self.opener = build_opener(SafeRedirectHandler())
+
+ def request(self, method: str, url: str, headers, timeout: float):
+ request = Request(url, headers=dict(headers), method=method)
+ try:
+ return self.opener.open(request, timeout=timeout)
+ except HTTPError as error:
+ return error
+ except URLError as error:
+ raise RetryableTransferError(
+ f"request failed for {_safe_url(url)}: {error.reason}") from error
+
+
+def _header_values(headers, name: str):
+ if hasattr(headers, "get_all"):
+ return list(headers.get_all(name, []))
+ values = []
+ for key, value in headers.items():
+ if key.lower() != name.lower():
+ continue
+ if isinstance(value, (list, tuple)):
+ values.extend(str(item) for item in value)
+ else:
+ values.append(str(value))
+ return values
+
+
+def _single_header(headers, name: str, required=False):
+ values = _header_values(headers, name)
+ if len(values) > 1:
+ raise QpackHTTPError(f"duplicate {name} response header")
+ if required and not values:
+ raise QpackHTTPError(f"missing {name} response header")
+ return values[0].strip() if values else None
+
+
+def _status(response) -> int:
+ value = getattr(response, "status", None)
+ if value is None and hasattr(response, "getcode"):
+ value = response.getcode()
+ return int(value or 0)
+
+
+def _strong_etag(headers):
+ value = _single_header(headers, "ETag")
+ if not value or value[:2].lower() == "w/":
+ return None
+ if len(value) < 2 or value[0] != '"' or value[-1] != '"':
+ return None
+ return value
+
+
+def _checksum_sha256(headers):
+ direct = _single_header(headers, "X-Checksum-Sha256")
+ if not direct:
+ return None
+ direct = direct.strip('"')
+ if not SHA256_RE.fullmatch(direct):
+ raise QpackHTTPError("invalid X-Checksum-Sha256 response header")
+ return direct.lower()
+
+
+def _validate_mirror_base_url(base_url: str) -> str:
+ if not isinstance(base_url, str) or not base_url or len(base_url) > 8192:
+ raise QpackHTTPError("mirror base URL is invalid")
+ try:
+ split = urlsplit(base_url)
+ _ = split.port
+ except ValueError as error:
+ raise QpackHTTPError("mirror base URL is invalid") from error
+ if (split.scheme.lower() != "https" or not split.hostname
+ or split.username or split.password or split.query or split.fragment):
+ raise QpackHTTPError(
+ "mirror base URL must be HTTPS without userinfo, query, or fragment")
+ try:
+ decoded_path = unquote(split.path, errors="strict")
+ except UnicodeError as error:
+ raise QpackHTTPError("mirror base URL path is invalid") from error
+ components = decoded_path.split("/")
+ if ("\\" in decoded_path or any(ord(character) < 32 for character in decoded_path)
+ or any(part in (".", "..") for part in components)
+ or any(part == "" for part in components[1:-1])):
+ raise QpackHTTPError("mirror base URL path is unsafe")
+ return base_url.rstrip("/")
+
+
+def _hash_entry_name(name: str) -> str:
+ if name == "manifest.json":
+ return name
+ try:
+ return _validate_name(name)
+ except QpackInstallError as error:
+ raise QpackHTTPError(str(error)) from error
+
+
+def _semantic_hashes_digest(entries: dict[str, HashEntry]) -> str:
+ digest = hashlib.sha256()
+ for name in sorted(entries):
+ encoded = name.encode("ascii")
+ digest.update(len(encoded).to_bytes(4, "big"))
+ digest.update(encoded)
+ digest.update(bytes.fromhex(entries[name].sha256))
+ return digest.hexdigest()
+
+
+def parse_hashes_index(raw: bytes) -> tuple[dict[str, HashEntry], str]:
+ """Strictly parse Swiftlet's legacy index or qpack.hashes.v1."""
+ try:
+ value = _decode_json(raw, HASHES_NAME, MAX_HASHES_SIZE)
+ except QpackInstallError as error:
+ raise QpackHTTPError(str(error)) from error
+ if not isinstance(value, dict):
+ raise QpackHTTPError("hashes.json must be an object")
+ schema = value.get("schema")
+ if schema is None:
+ if set(value) != {"files"}:
+ raise QpackHTTPError("legacy hashes.json has unsupported fields")
+ elif schema == "qpack.hashes.v1":
+ if set(value) != {"schema", "files"}:
+ raise QpackHTTPError("qpack.hashes.v1 has unsupported fields")
+ else:
+ raise QpackHTTPError("unsupported hashes.json schema")
+ files = value.get("files")
+ if (not isinstance(files, dict) or not files
+ or len(files) > MAX_HASH_ENTRIES):
+ raise QpackHTTPError("hashes.json files table is invalid")
+
+ entries = {}
+ normalized_names = set()
+ for raw_name, raw_entry in files.items():
+ name = _hash_entry_name(raw_name)
+ if name == HASHES_NAME:
+ raise QpackHTTPError("hashes.json cannot list itself")
+ normalized = name.casefold()
+ if normalized in normalized_names:
+ raise QpackHTTPError(f"case-colliding hashes.json path: {name}")
+ normalized_names.add(normalized)
+ if schema is None:
+ sha256 = raw_entry
+ size = None
+ else:
+ if not isinstance(raw_entry, dict) or set(raw_entry) != {"size", "sha256"}:
+ raise QpackHTTPError(f"invalid qpack.hashes.v1 entry: {name}")
+ sha256 = raw_entry.get("sha256")
+ size = raw_entry.get("size")
+ if (isinstance(size, bool) or not isinstance(size, int)
+ or size < 0 or size > MAX_FILE_SIZE):
+ raise QpackHTTPError(f"invalid hashes.json size for {name}")
+ if not isinstance(sha256, str) or not SHA256_RE.fullmatch(sha256):
+ raise QpackHTTPError(f"invalid hashes.json SHA-256 for {name}")
+ entries[name] = HashEntry(sha256.lower(), size)
+
+ sorted_names = sorted(normalized_names)
+ occupied = set(sorted_names)
+ for name in sorted_names:
+ components = name.split("/")
+ for count in range(1, len(components)):
+ if "/".join(components[:count]) in occupied:
+ raise QpackHTTPError("hashes.json has a file/parent path collision")
+ return entries, _semantic_hashes_digest(entries)
+
+
+def _read_bounded(response, maximum: int, description: str) -> bytes:
+ data = response.read(maximum + 1)
+ if len(data) > maximum:
+ raise QpackHTTPError(f"{description} exceeds {maximum} bytes")
+ return data
+
+
+def _fsync_directory(path: Path) -> None:
+ if os.name == "nt":
+ return
+ descriptor = os.open(path, os.O_RDONLY)
+ try:
+ try:
+ os.fsync(descriptor)
+ except OSError as error:
+ if error.errno not in (errno.EBADF, errno.EINVAL, errno.ENOTSUP):
+ raise
+ finally:
+ os.close(descriptor)
+
+
+def _atomic_bytes(path: Path, payload: bytes) -> None:
+ temporary = path.with_name(f".{path.name}.part")
+ if path.is_symlink() or temporary.is_symlink():
+ raise QpackHTTPError(f"refusing symlink auxiliary path: {path}")
+ temporary.unlink(missing_ok=True)
+ try:
+ with temporary.open("xb") as stream:
+ stream.write(payload)
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(temporary, path)
+ _fsync_directory(path.parent)
+ except Exception:
+ temporary.unlink(missing_ok=True)
+ raise
+
+
+def _atomic_json(path: Path, payload) -> None:
+ raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + b"\n"
+ _atomic_bytes(path, raw)
+
+
+def _state_path(root: Path) -> Path:
+ return root / STATE_NAME
+
+
+def _new_state(session, manifest_etag):
+ return {
+ "schema": STATE_SCHEMA,
+ "source_id": session.plan.source_id,
+ "manifest_sha256": session.plan.manifest_sha256,
+ "manifest_etag": manifest_etag,
+ "files": {},
+ "aux": {},
+ }
+
+
+def _load_state(session, manifest_etag, require_existing=False):
+ path = _state_path(session.root)
+ if path.is_symlink():
+ raise QpackHTTPError("qpack HTTP state is a symlink")
+ if not path.exists():
+ if require_existing:
+ raise QpackHTTPError("completed HTTP install has no validator state")
+ state = _new_state(session, manifest_etag)
+ _atomic_json(path, state)
+ return state
+ try:
+ state = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, UnicodeError, json.JSONDecodeError) as error:
+ raise QpackHTTPError(f"invalid qpack HTTP state: {error}") from error
+ expected = (STATE_SCHEMA, session.plan.source_id, session.plan.manifest_sha256)
+ actual = (state.get("schema"), state.get("source_id"),
+ state.get("manifest_sha256")) if isinstance(state, dict) else ()
+ if actual != expected or not isinstance(state.get("files"), dict) \
+ or not isinstance(state.get("aux"), dict):
+ raise QpackHTTPError("qpack HTTP state belongs to another source or manifest")
+ stored_etag = state.get("manifest_etag")
+ if stored_etag and manifest_etag and stored_etag != manifest_etag:
+ raise QpackHTTPError("qpack manifest ETag changed for an immutable source")
+ if stored_etag is None and manifest_etag is not None:
+ state["manifest_etag"] = manifest_etag
+ _atomic_json(path, state)
+ return state
+
+
+def _save_state(session, state) -> None:
+ _atomic_json(_state_path(session.root), state)
+
+
+def _declared_auxiliary_names(plan):
+ declared = {file.name for file in plan.files}
+ return {name for name in (*REQUIRED_AUX, *OPTIONAL_AUX) if name in declared}
+
+
+def _check_adapter_namespace(plan, reserve_hashes=False) -> None:
+ reserved = {
+ STATE_NAME.casefold(),
+ STATE_TEMP_NAME.casefold(),
+ }
+ if reserve_hashes:
+ reserved.update((
+ HASHES_NAME.casefold(),
+ f".{HASHES_NAME}.part".casefold(),
+ ))
+ declared_aux = _declared_auxiliary_names(plan)
+ for name in (*REQUIRED_AUX, *OPTIONAL_AUX):
+ if name in declared_aux:
+ continue
+ reserved.add(name.casefold())
+ reserved.add((f".{name}.part").casefold())
+ for file in plan.files:
+ if file.name in declared_aux and file.size > MAX_AUX_SIZE:
+ raise QpackHTTPError(
+ f"qpack auxiliary file is too large: {file.name}")
+ name = file.name.casefold()
+ if (name in reserved
+ or any(name.startswith(path + "/") for path in reserved)
+ or any(path.startswith(name + "/") for path in reserved)):
+ raise QpackHTTPError(
+ f"qpack manifest collides with HTTP installer path: {file.name}")
+
+
+def _request_headers(source: ResolvedSource, extra=()):
+ values = dict(source.headers)
+ values["Accept-Encoding"] = "identity"
+ values["User-Agent"] = "colibri-qpack-install/1.0"
+ values.update(dict(extra))
+ return values
+
+
+def _response_metadata(response, source: ResolvedSource, expected_size=None):
+ headers = response.headers
+ encoding = _single_header(headers, "Content-Encoding")
+ if encoding and encoding.lower() != "identity":
+ raise QpackHTTPError(f"unsupported content encoding: {encoding}")
+ repo_commit = _single_header(headers, "X-Repo-Commit")
+ if (repo_commit and source.repo_commit is not None
+ and repo_commit.lower() != source.repo_commit):
+ raise QpackHTTPError("response belongs to a different repository commit")
+ linked_size = _single_header(headers, "X-Linked-Size")
+ if linked_size is not None and expected_size is not None:
+ try:
+ linked_size_value = int(linked_size)
+ except ValueError as error:
+ raise QpackHTTPError("invalid X-Linked-Size response header") from error
+ if linked_size_value != expected_size:
+ raise QpackHTTPError("response linked size differs from qpack manifest")
+ return _strong_etag(headers), _checksum_sha256(headers)
+
+
+def _resolve_source(repo: str, revision: str, endpoint: str, token: str | None,
+ transport, timeout: float, retries: int, sleep) -> ResolvedSource:
+ if (not REPO_RE.fullmatch(repo)
+ or any(part in (".", "..") for part in repo.split("/"))):
+ raise QpackHTTPError("repository must be OWNER/REPO")
+ if not isinstance(revision, str) or not revision or len(revision) > 4096:
+ raise QpackHTTPError("revision must be a non-empty commit or ref")
+ endpoint = endpoint.rstrip("/")
+ split = urlsplit(endpoint)
+ if split.scheme != "https" or not split.hostname or split.username or split.password \
+ or split.path not in ("", "/") or split.query or split.fragment:
+ raise QpackHTTPError("Hugging Face endpoint must be an HTTPS origin")
+ headers = {}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ headers["Accept-Encoding"] = "identity"
+ headers["User-Agent"] = "colibri-qpack-install/1.0"
+ if COMMIT_RE.fullmatch(revision):
+ commit = revision.lower()
+ else:
+ api = (f"{endpoint}/api/models/{quote(repo, safe='/')}/revision/"
+ f"{quote(revision, safe='')}?expand=sha")
+ for attempt in range(retries + 1):
+ try:
+ response = transport.request("GET", api, headers, timeout)
+ with closing(response):
+ status = _status(response)
+ if status in RETRYABLE_STATUS:
+ raise HTTPStatusFailure(api, status)
+ if status != 200:
+ raise HTTPStatusFailure(api, status)
+ try:
+ raw = _read_bounded(
+ response, MAX_API_SIZE,
+ "Hugging Face revision metadata")
+ except (OSError, http.client.IncompleteRead) as error:
+ raise RetryableTransferError(
+ "interrupted Hugging Face revision metadata") from error
+ break
+ except (RetryableTransferError, HTTPStatusFailure) as error:
+ retryable = (isinstance(error, RetryableTransferError)
+ or error.status in RETRYABLE_STATUS)
+ if not retryable or attempt >= retries:
+ raise
+ sleep(min(30, 2 ** attempt))
+ try:
+ value = json.loads(raw)
+ commit = value["sha"].lower()
+ except (UnicodeError, json.JSONDecodeError, KeyError, AttributeError) as error:
+ raise QpackHTTPError("Hugging Face revision metadata has no commit SHA") from error
+ if not COMMIT_RE.fullmatch(commit):
+ raise QpackHTTPError("Hugging Face returned an invalid commit SHA")
+ encoded_repo = "/".join(quote(part, safe="") for part in repo.split("/"))
+ base_url = f"{endpoint}/{encoded_repo}/resolve/{commit}"
+ source_id = f"hf:{endpoint}/models/{repo}@{commit}"
+ return ResolvedSource(repo, commit, endpoint, base_url, source_id,
+ tuple(headers.items()), repo_commit=commit)
+
+
+def _mirror_source(base_url: str, source_id: str,
+ authorization: str | None) -> ResolvedSource:
+ base_url = _validate_mirror_base_url(base_url)
+ headers = ()
+ if authorization is not None:
+ if (not isinstance(authorization, str) or not authorization.strip()
+ or "\r" in authorization or "\n" in authorization):
+ raise QpackHTTPError("mirror authorization value is invalid")
+ headers = (("Authorization", authorization),)
+ split = urlsplit(base_url)
+ origin = f"{split.scheme}://{split.netloc}"
+ return ResolvedSource(
+ repo="", commit=None, endpoint=origin, base_url=base_url,
+ source_id=source_id, headers=headers, resume_header="If-Match",
+ restart_changed_resume=True)
+
+
+def _fetch_small(source, name, transport, timeout, maximum, required,
+ retries, sleep, expected_checksum=None, expected_size=None,
+ request_headers=()):
+ url = source.url(name)
+ for attempt in range(retries + 1):
+ try:
+ response = transport.request(
+ "GET", url, _request_headers(source, request_headers), timeout)
+ with closing(response):
+ status = _status(response)
+ if status == 404 and not required:
+ return None, None
+ if status in RETRYABLE_STATUS:
+ raise HTTPStatusFailure(url, status)
+ if status != 200:
+ raise HTTPStatusFailure(url, status)
+ etag, checksum = _response_metadata(response, source)
+ if (expected_checksum is not None and checksum is not None
+ and checksum != expected_checksum):
+ raise QpackHTTPError(
+ f"server SHA-256 disagrees with hashes.json for {name}")
+ try:
+ raw = _read_bounded(response, maximum, name)
+ except (OSError, http.client.IncompleteRead) as error:
+ raise RetryableTransferError(
+ f"interrupted HTTP body for {name}") from error
+ content_length = _single_header(response.headers, "Content-Length")
+ if content_length is not None:
+ try:
+ expected = int(content_length)
+ except ValueError as error:
+ raise QpackHTTPError(
+ "invalid Content-Length response header") from error
+ if expected != len(raw):
+ raise RetryableTransferError(f"short HTTP body for {name}")
+ if expected_size is not None and len(raw) != expected_size:
+ raise QpackHTTPError(f"hashes.json size mismatch for {name}")
+ digest = hashlib.sha256(raw).hexdigest()
+ if expected_checksum is not None and digest != expected_checksum:
+ raise QpackHTTPError(f"SHA-256 mismatch for {name}")
+ if checksum and digest != checksum:
+ raise QpackHTTPError(f"SHA-256 mismatch for {name}")
+ return raw, etag
+ except (RetryableTransferError, HTTPStatusFailure) as error:
+ retryable = (isinstance(error, RetryableTransferError)
+ or error.status in RETRYABLE_STATUS)
+ if not retryable or attempt >= retries:
+ raise
+ sleep(min(30, 2 ** attempt))
+ raise QpackHTTPError(f"retry budget exhausted for {name}")
+
+
+def _verify_or_fetch_aux(session, state, source, name, required, transport,
+ timeout, retries, sleep, completed,
+ expected_checksum=None, expected_size=None):
+ target = session.root / name
+ recorded = state["aux"].get(name)
+ if recorded is not None:
+ if (not isinstance(recorded, dict)
+ or not isinstance(recorded.get("sha256"), str)
+ or not SHA256_RE.fullmatch(recorded["sha256"])):
+ raise QpackHTTPError(f"invalid validator state for {name}")
+ if (expected_checksum is not None
+ and recorded["sha256"].lower() != expected_checksum):
+ raise QpackHTTPError(f"validator state disagrees with hashes.json for {name}")
+ if target.is_symlink() or not target.is_file():
+ raise QpackHTTPError(f"validated auxiliary file is missing: {name}")
+ raw = target.read_bytes()
+ if expected_size is not None and len(raw) != expected_size:
+ raise QpackHTTPError(f"validated auxiliary file has wrong size: {name}")
+ if hashlib.sha256(raw).hexdigest() != recorded["sha256"].lower():
+ raise QpackHTTPError(f"validated auxiliary file changed: {name}")
+ return False, 0
+ if completed:
+ if required:
+ raise QpackHTTPError(f"completed qpack has no validator for {name}")
+ return False, 0
+ raw, etag = _fetch_small(source, name, transport, timeout, MAX_AUX_SIZE,
+ required, retries, sleep, expected_checksum,
+ expected_size)
+ if raw is None:
+ return False, 0
+ if name == "config.json":
+ try:
+ config = json.loads(raw)
+ except (UnicodeError, json.JSONDecodeError) as error:
+ raise QpackHTTPError("config.json is not valid JSON") from error
+ if not isinstance(config, dict) or not config:
+ raise QpackHTTPError("config.json must be a non-empty object")
+ _atomic_bytes(target, raw)
+ state["aux"][name] = {
+ "sha256": expected_checksum or hashlib.sha256(raw).hexdigest(),
+ "size": len(raw),
+ "etag": etag,
+ }
+ _save_state(session, state)
+ return True, len(raw)
+
+
+def _verify_or_store_prefetched_aux(session, state, name, raw, completed):
+ target = session.root / name
+ digest = hashlib.sha256(raw).hexdigest()
+ recorded = state["aux"].get(name)
+ if recorded is not None:
+ if (not isinstance(recorded, dict) or recorded.get("sha256") != digest
+ or recorded.get("size") != len(raw)):
+ raise QpackHTTPError(f"validator state changed for {name}")
+ if target.is_symlink() or not target.is_file() \
+ or target.read_bytes() != raw:
+ raise QpackHTTPError(f"validated auxiliary file changed: {name}")
+ return
+ if completed:
+ raise QpackHTTPError(f"completed qpack has no validator for {name}")
+ _atomic_bytes(target, raw)
+ state["aux"][name] = {
+ "sha256": digest,
+ "size": len(raw),
+ "etag": None,
+ }
+ _save_state(session, state)
+
+
+def _validate_config(path: Path) -> None:
+ try:
+ raw = path.read_bytes()
+ if not raw or len(raw) > MAX_AUX_SIZE:
+ raise QpackHTTPError("config.json has an invalid size")
+ config = json.loads(raw)
+ except (OSError, UnicodeError, json.JSONDecodeError) as error:
+ raise QpackHTTPError("config.json is not valid JSON") from error
+ if not isinstance(config, dict) or not config:
+ raise QpackHTTPError("config.json must be a non-empty object")
+
+
+def _open_partial(decision, file: QpackFile):
+ flags = os.O_RDWR | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
+ if decision.action == "restart":
+ flags |= os.O_CREAT | os.O_TRUNC
+ descriptor = os.open(decision.partial_path, flags, 0o644)
+ metadata = os.fstat(descriptor)
+ if not stat.S_ISREG(metadata.st_mode):
+ os.close(descriptor)
+ raise QpackHTTPError(f"qpack partial is not a regular file: {file.name}")
+ expected = 0 if decision.action == "restart" else decision.offset
+ if metadata.st_size != expected:
+ os.close(descriptor)
+ raise QpackHTTPError(f"qpack partial changed while opening: {file.name}")
+ return descriptor
+
+
+def _write_all(descriptor, data: bytes) -> None:
+ offset = 0
+ while offset < len(data):
+ written = os.write(descriptor, data[offset:])
+ if written <= 0:
+ raise QpackHTTPError("short write to qpack partial")
+ offset += written
+
+
+def _bind_response_validator(session, state, file, etag, checksum,
+ expected_checksum=None):
+ if (expected_checksum is not None and checksum is not None
+ and checksum != expected_checksum):
+ raise QpackHTTPError(
+ f"server SHA-256 disagrees with hashes.json for {file.name}")
+ authoritative_checksum = expected_checksum or checksum
+ record = state["files"].get(file.name)
+ if record is None:
+ record = {
+ "size": file.size,
+ "etag": etag,
+ "sha256": authoritative_checksum,
+ "local_sha256": None,
+ }
+ state["files"][file.name] = record
+ _save_state(session, state)
+ return record
+ if not isinstance(record, dict) or record.get("size") != file.size:
+ raise QpackHTTPError(f"invalid HTTP validator state for {file.name}")
+ stored_etag = record.get("etag")
+ stored_checksum = record.get("sha256")
+ if stored_etag is not None and etag != stored_etag:
+ raise QpackHTTPError(f"strong ETag changed or disappeared for {file.name}")
+ if expected_checksum is not None:
+ if stored_checksum != expected_checksum:
+ raise QpackHTTPError(
+ f"validator state disagrees with hashes.json for {file.name}")
+ elif stored_checksum is not None and checksum != stored_checksum:
+ raise QpackHTTPError(
+ f"SHA-256 validator changed or disappeared for {file.name}")
+ changed = False
+ if stored_etag is None and etag is not None:
+ record["etag"] = etag
+ changed = True
+ if stored_checksum is None and authoritative_checksum is not None:
+ record["sha256"] = authoritative_checksum
+ changed = True
+ if changed:
+ _save_state(session, state)
+ return record
+
+
+def _verify_completed_file_state(session, state, file, chunk_size,
+ expected_checksum=None) -> None:
+ record = state["files"].get(file.name)
+ if not isinstance(record, dict) or record.get("size") != file.size:
+ raise QpackHTTPError(f"completed qpack has no validator for {file.name}")
+ etag = record.get("etag")
+ checksum = record.get("sha256")
+ local_checksum = record.get("local_sha256")
+ if etag is not None and (
+ not isinstance(etag, str) or _strong_etag({"ETag": etag}) != etag):
+ raise QpackHTTPError(f"invalid stored ETag for {file.name}")
+ if checksum is not None and (
+ not isinstance(checksum, str) or not SHA256_RE.fullmatch(checksum)):
+ raise QpackHTTPError(f"invalid stored SHA-256 for {file.name}")
+ if expected_checksum is not None and checksum != expected_checksum:
+ raise QpackHTTPError(
+ f"validator state disagrees with hashes.json for {file.name}")
+ if (not isinstance(local_checksum, str)
+ or not SHA256_RE.fullmatch(local_checksum)):
+ raise QpackHTTPError(f"completed qpack has no local digest for {file.name}")
+ digest = hashlib.sha256()
+ with (session.root / file.name).open("rb") as stream:
+ while True:
+ chunk = stream.read(chunk_size)
+ if not chunk:
+ break
+ digest.update(chunk)
+ actual = digest.hexdigest()
+ if checksum is not None and actual != checksum.lower():
+ raise QpackHTTPError(f"validated qpack artifact changed: {file.name}")
+ if actual != local_checksum.lower():
+ raise QpackHTTPError(f"validated qpack artifact changed: {file.name}")
+
+
+def _validate_range_headers(response, file, offset):
+ status = _status(response)
+ remaining = file.size - offset
+ if offset > 0 and status != 206:
+ raise QpackHTTPError(f"resumed request for {file.name} did not return HTTP 206")
+ if offset == 0 and status not in (200, 206):
+ raise HTTPStatusFailure(file.name, status)
+ content_length = _single_header(response.headers, "Content-Length", required=True)
+ try:
+ content_length = int(content_length)
+ except ValueError as error:
+ raise QpackHTTPError("invalid Content-Length response header") from error
+ if content_length != remaining:
+ raise QpackHTTPError(f"Content-Length mismatch for {file.name}")
+ content_range = _single_header(response.headers, "Content-Range",
+ required=status == 206)
+ if status == 200:
+ if content_range is not None:
+ raise QpackHTTPError("HTTP 200 response unexpectedly has Content-Range")
+ return remaining
+ match = CONTENT_RANGE_RE.fullmatch(content_range or "")
+ if match is None:
+ raise QpackHTTPError(f"invalid Content-Range for {file.name}")
+ start, end, total = map(int, match.groups())
+ if start != offset or end != file.size - 1 or total != file.size:
+ raise QpackHTTPError(f"Content-Range mismatch for {file.name}")
+ if end - start + 1 != remaining:
+ raise QpackHTTPError(f"Content-Range length mismatch for {file.name}")
+ return remaining
+
+
+def _download_file(session, state, source, file, transport, timeout,
+ retries, sleep, chunk_size, expected_checksum=None):
+ decision = resume_decision(session, file)
+ if decision.action == "complete":
+ _verify_completed_file_state(
+ session, state, file, chunk_size, expected_checksum)
+ return False, 0, False
+ resumed = decision.action == "resume" and decision.offset > 0
+ restarted_from_zero = False
+ descriptor = _open_partial(decision, file)
+ try:
+ cursor = decision.offset if decision.action == "resume" else 0
+ attempts = 0
+ while cursor < file.size:
+ record = state["files"].get(file.name)
+ if record is not None and not isinstance(record, dict):
+ raise QpackHTTPError(
+ f"invalid HTTP validator state for {file.name}")
+ record = record or {}
+ headers = {"Range": f"bytes={cursor}-{file.size - 1}"}
+ if cursor > 0 and record.get("etag"):
+ headers[source.resume_header] = record["etag"]
+ url = source.url(file.name)
+ try:
+ response = transport.request(
+ "GET", url, _request_headers(source, headers.items()), timeout)
+ with closing(response):
+ status = _status(response)
+ if status in RETRYABLE_STATUS:
+ raise HTTPStatusFailure(url, status)
+ if (source.restart_changed_resume and cursor > 0
+ and status in (200, 412)):
+ os.ftruncate(descriptor, 0)
+ os.fsync(descriptor)
+ record["etag"] = None
+ record["local_sha256"] = None
+ _save_state(session, state)
+ cursor = 0
+ restarted_from_zero = True
+ continue
+ response_bytes = _validate_range_headers(response, file, cursor)
+ etag, checksum = _response_metadata(response, source, file.size)
+ if (source.restart_changed_resume and cursor > 0
+ and record.get("etag") is not None
+ and etag != record["etag"]):
+ os.ftruncate(descriptor, 0)
+ os.fsync(descriptor)
+ record["etag"] = None
+ record["local_sha256"] = None
+ _save_state(session, state)
+ cursor = 0
+ restarted_from_zero = True
+ continue
+ _bind_response_validator(
+ session, state, file, etag, checksum,
+ expected_checksum)
+ os.lseek(descriptor, cursor, os.SEEK_SET)
+ delivered = 0
+ request_offset = cursor
+ while True:
+ try:
+ chunk = response.read(chunk_size)
+ except http.client.IncompleteRead as error:
+ chunk = error.partial
+ if chunk:
+ if delivered + len(chunk) > response_bytes:
+ os.ftruncate(descriptor, request_offset)
+ os.fsync(descriptor)
+ raise QpackHTTPError(
+ f"HTTP body overrun for {file.name}")
+ _write_all(descriptor, chunk)
+ delivered += len(chunk)
+ raise RetryableTransferError(
+ f"interrupted HTTP body for {file.name}") from error
+ except OSError as error:
+ raise RetryableTransferError(
+ f"interrupted HTTP body for {file.name}") from error
+ if not chunk:
+ break
+ if delivered + len(chunk) > response_bytes:
+ os.ftruncate(descriptor, request_offset)
+ os.fsync(descriptor)
+ raise QpackHTTPError(f"HTTP body overrun for {file.name}")
+ validate_delivery(file, cursor + delivered,
+ cursor + delivered, len(chunk))
+ _write_all(descriptor, chunk)
+ delivered += len(chunk)
+ os.fsync(descriptor)
+ if delivered != response_bytes:
+ cursor += delivered
+ raise RetryableTransferError(
+ f"short HTTP body for {file.name}")
+ cursor += delivered
+ validate_delivery(file, request_offset, request_offset,
+ delivered, complete=cursor == file.size)
+ attempts = 0
+ except (RetryableTransferError, HTTPStatusFailure) as error:
+ retryable = (isinstance(error, RetryableTransferError)
+ or error.status in RETRYABLE_STATUS)
+ if not retryable or attempts >= retries:
+ raise
+ attempts += 1
+ sleep(min(30, 2 ** (attempts - 1)))
+ cursor = os.fstat(descriptor).st_size
+ resumed = resumed or cursor > 0
+ record = state["files"].get(file.name)
+ if not isinstance(record, dict):
+ raise QpackHTTPError(f"invalid HTTP validator state for {file.name}")
+ checksum = record.get("sha256")
+ os.lseek(descriptor, 0, os.SEEK_SET)
+ digest = hashlib.sha256()
+ while True:
+ chunk = os.read(descriptor, chunk_size)
+ if not chunk:
+ break
+ digest.update(chunk)
+ local_checksum = digest.hexdigest()
+ if checksum and local_checksum != checksum:
+ os.ftruncate(descriptor, 0)
+ os.fsync(descriptor)
+ raise QpackHTTPError(f"SHA-256 mismatch for {file.name}")
+ stored_local = record.get("local_sha256")
+ if stored_local is not None and stored_local != local_checksum:
+ os.ftruncate(descriptor, 0)
+ os.fsync(descriptor)
+ raise QpackHTTPError(f"local SHA-256 changed for {file.name}")
+ if stored_local is None:
+ record["local_sha256"] = local_checksum
+ _save_state(session, state)
+ finally:
+ os.close(descriptor)
+ commit_file(session, file)
+ downloaded = file.size if restarted_from_zero else file.size - decision.offset
+ return True, downloaded, resumed
+
+
+def _validate_hash_coverage(plan, manifest_raw, entries) -> None:
+ required = {"manifest.json", "config.json", *(file.name for file in plan.files)}
+ missing = sorted(required - entries.keys())
+ if missing:
+ raise QpackHTTPError(
+ f"hashes.json does not cover required file: {missing[0]}")
+ manifest_size = entries["manifest.json"].size
+ if manifest_size is not None and manifest_size != len(manifest_raw):
+ raise QpackHTTPError("hashes.json size mismatch for manifest.json")
+ for file in plan.files:
+ indexed_size = entries[file.name].size
+ if indexed_size is not None and indexed_size != file.size:
+ raise QpackHTTPError(f"hashes.json size mismatch for {file.name}")
+
+
+def _install_resolved_qpack(source, output_dir, manifest_raw, manifest_etag,
+ transport, retries, timeout, chunk_size,
+ optional_aux, sleep, log, expected_hashes=None,
+ persisted_hashes=None) -> InstallResult:
+ try:
+ plan = parse_manifest(manifest_raw, source.source_id)
+ except QpackInstallError as error:
+ raise QpackHTTPError(str(error)) from error
+ _check_adapter_namespace(plan, reserve_hashes=persisted_hashes is not None)
+ optional_aux = tuple(optional_aux)
+ for name in optional_aux:
+ if name not in OPTIONAL_AUX:
+ raise QpackHTTPError(f"unsupported optional auxiliary path: {name}")
+ if expected_hashes is not None:
+ _validate_hash_coverage(plan, manifest_raw, expected_hashes)
+ declared_aux = _declared_auxiliary_names(plan)
+ files_by_name = {file.name: file for file in plan.files}
+
+ downloaded_files = 0
+ downloaded_bytes = 0
+ resumed_files = 0
+ with begin_install(output_dir, plan) as session:
+ state = _load_state(session, manifest_etag,
+ require_existing=session.complete)
+ if persisted_hashes is not None:
+ _verify_or_store_prefetched_aux(
+ session, state, HASHES_NAME, persisted_hashes,
+ session.complete)
+ if "config.json" in declared_aux and not session.complete:
+ changed, count, resumed = _download_file(
+ session, state, source, files_by_name["config.json"],
+ transport, timeout, retries, sleep, chunk_size,
+ expected_hashes["config.json"].sha256
+ if expected_hashes is not None else None)
+ downloaded_files += int(changed)
+ downloaded_bytes += count
+ resumed_files += int(resumed)
+ elif "config.json" not in declared_aux:
+ changed, count = _verify_or_fetch_aux(
+ session, state, source, "config.json", True, transport, timeout,
+ retries, sleep, session.complete,
+ expected_hashes["config.json"].sha256
+ if expected_hashes is not None else None,
+ expected_hashes["config.json"].size
+ if expected_hashes is not None else None)
+ downloaded_files += int(changed)
+ downloaded_bytes += count
+ _validate_config(session.root / "config.json")
+ if not session.complete:
+ for name in sorted(declared_aux - {"config.json"}):
+ changed, count, resumed = _download_file(
+ session, state, source, files_by_name[name], transport,
+ timeout, retries, sleep, chunk_size,
+ expected_hashes[name].sha256
+ if expected_hashes is not None else None)
+ downloaded_files += int(changed)
+ downloaded_bytes += count
+ resumed_files += int(resumed)
+ for name in optional_aux:
+ if name in declared_aux:
+ continue
+ changed, count = _verify_or_fetch_aux(
+ session, state, source, name, expected_hashes is not None,
+ transport, timeout, retries, sleep, session.complete,
+ expected_hashes[name].sha256
+ if expected_hashes is not None else None,
+ expected_hashes[name].size
+ if expected_hashes is not None else None)
+ downloaded_files += int(changed)
+ downloaded_bytes += count
+ if session.complete:
+ for file in plan.files:
+ _verify_completed_file_state(
+ session, state, file, chunk_size,
+ expected_hashes[file.name].sha256
+ if expected_hashes is not None else None)
+ log(f"qpack already complete at {session.root}")
+ return InstallResult(session.root, source.commit, downloaded_files,
+ downloaded_bytes, 0, True, source.source_id)
+
+ for file in plan.files:
+ if file.name in declared_aux:
+ continue
+ changed, count, resumed = _download_file(
+ session, state, source, file, transport, timeout,
+ retries, sleep, chunk_size,
+ expected_hashes[file.name].sha256
+ if expected_hashes is not None else None)
+ downloaded_files += int(changed)
+ downloaded_bytes += count
+ resumed_files += int(resumed)
+ if changed:
+ log(f"installed {file.name} ({file.size} bytes)")
+ publish_manifest(session, manifest_raw)
+ log(f"qpack complete at {session.root}")
+ return InstallResult(session.root, source.commit, downloaded_files,
+ downloaded_bytes, resumed_files, False,
+ source.source_id)
+
+
+def install_hf_qpack(repo: str, revision: str, output_dir,
+ *, endpoint="https://huggingface.co", token=None,
+ transport=None, retries=5, timeout=120.0,
+ chunk_size=CHUNK_SIZE, optional_aux=OPTIONAL_AUX,
+ sleep=time.sleep, log=print) -> InstallResult:
+ if retries < 0 or timeout <= 0 or chunk_size <= 0:
+ raise QpackHTTPError("retry, timeout, and chunk-size values are invalid")
+ transport = transport or URLLibTransport()
+ source = _resolve_source(repo, revision, endpoint, token,
+ transport, timeout, retries, sleep)
+ manifest_raw, manifest_etag = _fetch_small(
+ source, "manifest.json", transport, timeout, MAX_MANIFEST,
+ True, retries, sleep)
+ return _install_resolved_qpack(
+ source, output_dir, manifest_raw, manifest_etag, transport, retries,
+ timeout, chunk_size, optional_aux, sleep, log)
+
+
+def install_mirror_qpack(base_url: str, output_dir, *, hashes_sha256=None,
+ authorization=None, transport=None, retries=5,
+ timeout=120.0, chunk_size=CHUNK_SIZE,
+ sleep=time.sleep, log=print) -> InstallResult:
+ """Install a static HTTPS qpack whose hashes.json is the completion marker."""
+ if retries < 0 or timeout <= 0 or chunk_size <= 0:
+ raise QpackHTTPError("retry, timeout, and chunk-size values are invalid")
+ if hashes_sha256 is not None:
+ if not isinstance(hashes_sha256, str) or not SHA256_RE.fullmatch(hashes_sha256):
+ raise QpackHTTPError("hashes_sha256 must be a 64-digit SHA-256")
+ hashes_sha256 = hashes_sha256.lower()
+ transport = transport or URLLibTransport()
+ bootstrap = _mirror_source(base_url, "mirror:bootstrap", authorization)
+ hashes_raw, _ = _fetch_small(
+ bootstrap, HASHES_NAME, transport, timeout, MAX_HASHES_SIZE, True,
+ retries, sleep, request_headers=(("Cache-Control", "no-cache"),))
+ raw_digest = hashlib.sha256(hashes_raw).hexdigest()
+ if hashes_sha256 is not None and raw_digest != hashes_sha256:
+ raise QpackHTTPError("hashes.json does not match the pinned SHA-256")
+ expected_hashes, _ = parse_hashes_index(hashes_raw)
+ source = _mirror_source(
+ bootstrap.base_url,
+ f"mirror:{bootstrap.base_url}@sha256:{raw_digest}", authorization)
+ manifest_entry = expected_hashes.get("manifest.json")
+ if manifest_entry is None:
+ raise QpackHTTPError("hashes.json does not cover required file: manifest.json")
+ manifest_raw, _ = _fetch_small(
+ source, "manifest.json", transport, timeout, MAX_MANIFEST, True,
+ retries, sleep, manifest_entry.sha256, manifest_entry.size,
+ (("Cache-Control", "no-cache"),))
+ mirror_aux = tuple(
+ name for name in OPTIONAL_AUX if name in expected_hashes)
+ return _install_resolved_qpack(
+ source, output_dir, manifest_raw, None, transport, retries, timeout,
+ chunk_size, mirror_aux, sleep, log, expected_hashes,
+ hashes_raw)
+
+
+def _parse_args(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("repo", help="Hugging Face repository as OWNER/REPO")
+ parser.add_argument("--revision", required=True,
+ help="commit SHA or revision resolved to a commit before download")
+ parser.add_argument("--output", required=True, type=Path,
+ help="destination .qpack directory")
+ parser.add_argument("--endpoint", default="https://huggingface.co",
+ help=argparse.SUPPRESS)
+ parser.add_argument("--retries", type=int, default=5)
+ parser.add_argument("--timeout", type=float, default=120.0)
+ return parser.parse_args(argv)
+
+
+def main(argv=None):
+ args = _parse_args(argv)
+ try:
+ result = install_hf_qpack(
+ args.repo, args.revision, args.output, endpoint=args.endpoint,
+ token=os.environ.get("HF_TOKEN"), retries=args.retries,
+ timeout=args.timeout)
+ except (OSError, QpackInstallError, QpackHTTPError) as error:
+ print(f"qpack install failed: {error}", file=sys.stderr)
+ return 1
+ print(json.dumps({
+ "output": str(result.output_dir),
+ "commit": result.commit,
+ "downloaded_files": result.downloaded_files,
+ "downloaded_bytes": result.downloaded_bytes,
+ "resumed_files": result.resumed_files,
+ "already_complete": result.already_complete,
+ }, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/c/tools/qpack_mirror_install.py b/c/tools/qpack_mirror_install.py
new file mode 100644
index 00000000..3dcd125f
--- /dev/null
+++ b/c/tools/qpack_mirror_install.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Install a hash-indexed qpack from a static HTTPS mirror."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+try:
+ from tools.qpack_http_install import QpackHTTPError, install_mirror_qpack
+ from tools.qpack_install_policy import QpackInstallError
+except ModuleNotFoundError: # Direct execution from c/tools.
+ from qpack_http_install import QpackHTTPError, install_mirror_qpack
+ from qpack_install_policy import QpackInstallError
+
+
+ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def _parse_args(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("base_url", help="HTTPS prefix containing hashes.json")
+ parser.add_argument("--output", required=True, type=Path,
+ help="destination .qpack directory")
+ parser.add_argument("--hashes-sha256",
+ help="optional out-of-band SHA-256 pin for hashes.json")
+ parser.add_argument(
+ "--bearer-token-env", metavar="NAME",
+ help="read an optional mirror bearer token from environment variable NAME")
+ parser.add_argument("--retries", type=int, default=5)
+ parser.add_argument("--timeout", type=float, default=120.0)
+ return parser.parse_args(argv)
+
+
+def main(argv=None):
+ args = _parse_args(argv)
+ authorization = None
+ try:
+ if args.bearer_token_env is not None:
+ if not ENV_NAME_RE.fullmatch(args.bearer_token_env):
+ raise QpackHTTPError("bearer token environment name is invalid")
+ token = os.environ.get(args.bearer_token_env)
+ if not token:
+ raise QpackHTTPError(
+ f"bearer token environment variable is empty: "
+ f"{args.bearer_token_env}")
+ authorization = f"Bearer {token}"
+ result = install_mirror_qpack(
+ args.base_url, args.output, hashes_sha256=args.hashes_sha256,
+ authorization=authorization, retries=args.retries,
+ timeout=args.timeout)
+ except (OSError, QpackInstallError, QpackHTTPError) as error:
+ print(f"qpack mirror install failed: {error}", file=sys.stderr)
+ return 1
+ print(json.dumps({
+ "output": str(result.output_dir),
+ "source_id": result.source_id,
+ "downloaded_files": result.downloaded_files,
+ "downloaded_bytes": result.downloaded_bytes,
+ "resumed_files": result.resumed_files,
+ "already_complete": result.already_complete,
+ }, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())