Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(tools): resumable qpack install policy

Léon Avic Simmons40f466f3560b
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index ca473c78..01a7a2ce 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -48,3 +48,14 @@ What that checkout contains, and the licences that apply when you build with it:
- NVIDIA CUTLASS / CuTe (BSD-3-Clause), the `third-party/cutlass` submodule of
that commit @ f3fde58372d33e9a5650ba7b80fc48b3b49d40c8. License text:
`c/third_party/deepgemm/third-party/cutlass/LICENSE.txt` after the fetch.
+
+## Swiftlet qpack installers (`c/tools/qpack_*install*.py`)
+
+Swiftlet (https://github.com/leonickson1/Swiftlet) is licensed under Apache
+License 2.0. This repository's `LICENSE` contains the applicable Apache 2.0
+text.
+
+`c/tools/qpack_install_policy.py` adapts the source-bound resume and
+manifest-last completion policy from Swiftlet's `StreamingInstaller.swift` at
+commit `86246618ba2af30334227e09ff84a6a7182c2a40`. It is a new
+transport-neutral Python implementation and does not copy Swift source code.
diff --git a/c/tests/test_qpack_install_policy.py b/c/tests/test_qpack_install_policy.py
new file mode 100644
index 00000000..dff628c7
--- /dev/null
+++ b/c/tests/test_qpack_install_policy.py
@@ -0,0 +1,308 @@
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from tools.qpack_install_policy import (
+ JOURNAL,
+ MANIFEST,
+ MAX_FILE_SIZE,
+ QpackFile,
+ QpackInstallError,
+ begin_install,
+ commit_file,
+ parse_manifest,
+ publish_manifest,
+ resume_decision,
+ validate_delivery,
+)
+
+
+class QpackInstallPolicyTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name) / "model.qpack"
+ self.raw = self.manifest({
+ "model.safetensors": 7,
+ "packed_experts/layout.json": 5,
+ })
+ self.plan = parse_manifest(self.raw, "hf:owner/model@0123456789abcdef")
+ self.sessions = []
+ self.addCleanup(self.close_sessions)
+
+ def close_sessions(self):
+ for session in reversed(self.sessions):
+ session.close()
+
+ def begin(self, plan=None):
+ session = begin_install(self.root, plan or self.plan)
+ self.sessions.append(session)
+ return session
+
+ @staticmethod
+ def manifest(files, **extra):
+ value = {
+ "magic": "QPACK",
+ "version": 1,
+ "modelName": "tiny-qwen",
+ "sourceCheckpoint": "owner/model@0123456789abcdef",
+ "quantBits": 4,
+ "quantGroupSize": 64,
+ "files": files,
+ }
+ value.update(extra)
+ return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
+
+ def file(self, name):
+ return next(file for file in self.plan.files if file.name == name)
+
+ def write_partial(self, session, file, data):
+ decision = resume_decision(session, file)
+ decision.partial_path.write_bytes(data)
+ return decision.partial_path
+
+ def commit_all(self, session):
+ for file in self.plan.files:
+ self.write_partial(session, self.file(file.name),
+ bytes([file.size]) * file.size)
+ commit_file(session, file)
+
+ def test_manifest_is_sorted_and_bound_to_exact_bytes_and_source(self):
+ reordered = self.raw + b"\n"
+ other = parse_manifest(reordered, self.plan.source_id)
+ self.assertEqual([file.name for file in other.files],
+ ["model.safetensors", "packed_experts/layout.json"])
+ self.assertNotEqual(other.manifest_sha256, self.plan.manifest_sha256)
+ self.assertNotEqual(parse_manifest(self.raw, "local:fixture"), self.plan)
+
+ def test_manifest_rejects_duplicate_keys(self):
+ raw = (b'{"magic":"QPACK","version":1,"files":'
+ b'{"packed_experts/layout.json":5,'
+ b'"packed_experts/layout.json":6}}')
+ with self.assertRaisesRegex(QpackInstallError, "duplicate JSON key"):
+ parse_manifest(raw, "local:fixture")
+
+ def test_manifest_rejects_nonfinite_numbers_and_escaped_nuls(self):
+ raw = self.raw.replace(b'"quantBits":4', b'"quantBits":NaN')
+ with self.assertRaisesRegex(QpackInstallError, "non-finite"):
+ parse_manifest(raw, "local:fixture")
+ value = json.loads(self.raw)
+ value["modelName"] = "tiny\x00qwen"
+ with self.assertRaisesRegex(QpackInstallError, "embedded NUL"):
+ parse_manifest(json.dumps(value).encode(), "local:fixture")
+
+ def test_manifest_rejects_unsafe_and_reserved_paths(self):
+ invalid = [
+ "../escape", "/absolute", "a//b", "a/./b", "a/../b",
+ "a\\b", "manifest.json", ".qpack-install.json",
+ ".qpack-install.lock", ".manifest.json.part", "nul\x00name",
+ "C:/escape", "C:escape", "packed_experts/CON",
+ "packed_experts/aux.txt", "packed_experts/trailing. ",
+ "packed_experts/file:stream", "packed_experts/a?.bin",
+ "packed_experts/caf\u00e9.bin",
+ ]
+ for name in invalid:
+ files = {"packed_experts/layout.json": 5, name: 1}
+ with self.subTest(name=name), self.assertRaises(QpackInstallError):
+ parse_manifest(self.manifest(files), "local:fixture")
+
+ def test_manifest_rejects_invalid_sizes_and_missing_layout(self):
+ for size in (True, False, 0, -1, 1.5, MAX_FILE_SIZE + 1):
+ with self.subTest(size=size), self.assertRaises(QpackInstallError):
+ parse_manifest(self.manifest({
+ "packed_experts/layout.json": size,
+ }), "local:fixture")
+ with self.assertRaisesRegex(QpackInstallError, "layout.json"):
+ parse_manifest(self.manifest({"model.safetensors": 7}), "local:fixture")
+ with self.assertRaisesRegex(QpackInstallError, "source_id"):
+ parse_manifest(self.raw, "")
+
+ def test_manifest_matches_swiftlet_required_fields_and_file_limit(self):
+ for field in ("modelName", "sourceCheckpoint"):
+ value = json.loads(self.raw)
+ value.pop(field)
+ with self.subTest(field=field), self.assertRaisesRegex(
+ QpackInstallError, "supported QPACK"):
+ parse_manifest(json.dumps(value).encode(), "local:fixture")
+ for field in ("quantBits", "quantGroupSize"):
+ value = json.loads(self.raw)
+ value[field] = True
+ with self.subTest(field=field), self.assertRaisesRegex(
+ QpackInstallError, field):
+ parse_manifest(json.dumps(value).encode(), "local:fixture")
+ too_many = {"packed_experts/layout.json": 5}
+ too_many.update({f"packed_experts/file-{index}.bin": 1
+ for index in range(8192)})
+ with self.assertRaisesRegex(QpackInstallError, "files table"):
+ parse_manifest(self.manifest(too_many), "local:fixture")
+
+ def test_manifest_rejects_plan_wide_path_collisions(self):
+ collisions = [
+ {"packed_experts/layout.json": 5, "foo": 1, "foo.part": 1},
+ {"packed_experts/layout.json": 5, "foo": 1, "foo/bar": 1},
+ {"packed_experts/layout.json": 5, "Foo": 1, "foo": 1},
+ ]
+ for files in collisions:
+ with self.subTest(files=files), self.assertRaisesRegex(
+ QpackInstallError, "namespace collision"):
+ parse_manifest(self.manifest(files), "local:fixture")
+
+ def test_begin_binds_partials_and_rejects_unbound_or_mismatched_work(self):
+ self.root.mkdir()
+ (self.root / "model.safetensors.part").write_bytes(b"old")
+ with self.assertRaisesRegex(QpackInstallError, "unbound"):
+ begin_install(self.root, self.plan)
+ (self.root / "model.safetensors.part").unlink()
+
+ session = self.begin()
+ with self.assertRaisesRegex(QpackInstallError, "locked"):
+ begin_install(self.root, self.plan)
+ session.close()
+ other_raw = self.manifest({
+ "model.safetensors": 8,
+ "packed_experts/layout.json": 5,
+ })
+ other = parse_manifest(other_raw, self.plan.source_id)
+ with self.assertRaisesRegex(QpackInstallError, "another source or manifest"):
+ begin_install(self.root, other)
+
+ def test_malformed_journal_and_closed_session_fail_closed(self):
+ session = self.begin()
+ session.close()
+ (self.root / JOURNAL).write_text('{"schema":"wrong"}', encoding="utf-8")
+ with self.assertRaisesRegex(QpackInstallError, "journal is invalid"):
+ begin_install(self.root, self.plan)
+ with self.assertRaisesRegex(QpackInstallError, "closed or invalid"):
+ resume_decision(session, self.plan.files[0])
+
+ def test_resume_decisions_cover_missing_partial_and_committed_files(self):
+ session = self.begin()
+ file = self.file("model.safetensors")
+ decision = resume_decision(session, file)
+ self.assertEqual((decision.action, decision.offset), ("restart", 0))
+
+ decision.partial_path.write_bytes(b"abc")
+ decision = resume_decision(session, file)
+ self.assertEqual((decision.action, decision.offset), ("resume", 3))
+
+ decision.partial_path.write_bytes(b"too-large")
+ decision = resume_decision(session, file)
+ self.assertEqual((decision.action, decision.offset), ("restart", 0))
+
+ decision.partial_path.write_bytes(b"1234567")
+ commit_file(session, file)
+ decision = resume_decision(session, file)
+ self.assertEqual((decision.action, decision.offset), ("complete", 7))
+
+ def test_relative_root_remains_stable_after_working_directory_change(self):
+ original = Path.cwd()
+ self.addCleanup(os.chdir, original)
+ os.chdir(self.temporary.name)
+ session = begin_install(Path("relative") / "model.qpack", self.plan)
+ self.sessions.append(session)
+ os.chdir(original)
+ decision = resume_decision(session, self.plan.files[0])
+ self.assertTrue(decision.partial_path.is_absolute())
+ self.assertEqual(decision.offset, 0)
+
+ def test_delivery_requires_exact_offsets_bounds_and_final_length(self):
+ file = QpackFile("model.safetensors", 10)
+ self.assertEqual(validate_delivery(file, 3, 3, 4), 7)
+ self.assertEqual(validate_delivery(file, 3, 3, 7, complete=True), 10)
+ invalid = [
+ (-1, 0, 1, False),
+ (3, 0, 7, True),
+ (3, 3, 8, False),
+ (3, 3, 6, True),
+ ]
+ for requested, delivered, size, complete in invalid:
+ with self.subTest(values=(requested, delivered, size, complete)), \
+ self.assertRaises(QpackInstallError):
+ validate_delivery(file, requested, delivered, size,
+ complete=complete)
+
+ def test_incomplete_commit_never_publishes_a_final_path(self):
+ session = self.begin()
+ file = self.file("model.safetensors")
+ partial = self.write_partial(session, file, b"short")
+ with self.assertRaisesRegex(QpackInstallError, "not complete"):
+ commit_file(session, file)
+ self.assertTrue(partial.exists())
+ self.assertFalse(self.root.joinpath(file.name).exists())
+
+ def test_manifest_is_published_last_and_completion_is_idempotent(self):
+ session = self.begin()
+ first = self.plan.files[0]
+ self.write_partial(session, first, bytes([first.size]) * first.size)
+ commit_file(session, first)
+ with self.assertRaisesRegex(QpackInstallError, "all files"):
+ publish_manifest(session, self.raw)
+ self.assertFalse((self.root / MANIFEST).exists())
+
+ remaining = self.plan.files[1]
+ self.write_partial(session, remaining,
+ bytes([remaining.size]) * remaining.size)
+ commit_file(session, remaining)
+ publish_manifest(session, self.raw)
+ self.assertEqual((self.root / MANIFEST).read_bytes(), self.raw)
+ self.assertFalse((self.root / JOURNAL).exists())
+
+ session.close()
+ complete_session = self.begin()
+ self.assertTrue(complete_session.complete)
+ self.assertFalse((self.root / JOURNAL).exists())
+
+ def test_crash_boundaries_remain_recoverable(self):
+ session = self.begin()
+ first = self.plan.files[0]
+ self.write_partial(session, first, bytes([first.size]) * first.size)
+ self.assertEqual(resume_decision(session, first).offset, first.size)
+ session.close()
+ session = self.begin()
+ commit_file(session, first)
+ self.assertEqual(resume_decision(session, first).action, "complete")
+
+ for file in self.plan.files[1:]:
+ self.write_partial(session, file, bytes([file.size]) * file.size)
+ commit_file(session, file)
+ stale_manifest_temp = self.root / ".manifest.json.part"
+ stale_manifest_temp.write_bytes(b"interrupted")
+ publish_manifest(session, self.raw)
+ self.assertEqual((self.root / MANIFEST).read_bytes(), self.raw)
+ self.assertFalse(stale_manifest_temp.exists())
+
+ def test_existing_manifest_cleans_a_stale_matching_journal(self):
+ session = self.begin()
+ self.commit_all(session)
+ (self.root / MANIFEST).write_bytes(self.raw)
+ self.assertTrue((self.root / JOURNAL).exists())
+ session.close()
+ complete_session = self.begin()
+ self.assertTrue(complete_session.complete)
+ self.assertFalse((self.root / JOURNAL).exists())
+
+ @unittest.skipUnless(hasattr(os, "symlink"), "symlinks unsupported")
+ def test_manifest_symlink_cannot_mark_install_complete(self):
+ session = self.begin()
+ self.commit_all(session)
+ external = Path(self.temporary.name) / "external-manifest.json"
+ external.write_bytes(self.raw)
+ os.symlink(external, self.root / MANIFEST)
+ with self.assertRaisesRegex(QpackInstallError, "symlink"):
+ publish_manifest(session, self.raw)
+ self.assertTrue((self.root / JOURNAL).exists())
+
+ @unittest.skipUnless(hasattr(os, "symlink"), "symlinks unsupported")
+ def test_symlinked_artifact_parent_is_rejected(self):
+ outside = Path(self.temporary.name) / "outside"
+ outside.mkdir()
+ self.root.mkdir()
+ os.symlink(outside, self.root / "packed_experts", target_is_directory=True)
+ with self.assertRaisesRegex(QpackInstallError, "symlink"):
+ begin_install(self.root, self.plan)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/README.md b/c/tools/README.md
index 7a7135e8..24d84cec 100644
--- a/c/tools/README.md
+++ b/c/tools/README.md
@@ -4,6 +4,13 @@ These scripts support model preparation and offline engineering work. They are
not runtime dependencies of the C engine.
- `convert_fp8_to_int4.py`, `download_glm52.py`: model preparation
+- `qpack_install_policy.py`: source-bound resume decisions and manifest-last
+ publication primitives for future qpack download/repack frontends. Hold its
+ install session for the full transfer. Qpack v1 has sizes but no content
+ hashes, so frontends must also enforce immutable revisions and any available
+ transport checksum or ETag. It covers manifest-declared container artifacts,
+ not required runtime auxiliaries such as `config.json`. Directory-fsync crash
+ durability is POSIX-only.
- `convert_fmt4_to_fmt2.py`: fmt=4 (grouped int4) -> fmt=2 (per-row int4)
re-quant of a GLM-5.2 container, for Metal-backend compatibility
(see `docs/METAL-M1ULTRA-FMT2-REPORT.md`)
diff --git a/c/tools/qpack_install_policy.py b/c/tools/qpack_install_policy.py
new file mode 100644
index 00000000..e5941e09
--- /dev/null
+++ b/c/tools/qpack_install_policy.py
@@ -0,0 +1,513 @@
+#!/usr/bin/env python3
+"""Resumable qpack policy with POSIX-durable manifest-last publication.
+
+Qpack v1 records file sizes but no content digests. Callers must use an
+immutable ``source_id`` and validate any ETag or checksum supplied by the
+transport; this module prevents cross-revision resume, not silent media damage.
+"""
+
+from __future__ import annotations
+
+import errno
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Literal
+
+
+SCHEMA = "colibri.qpack-install.v1"
+JOURNAL = ".qpack-install.json"
+LOCK = ".qpack-install.lock"
+MANIFEST = "manifest.json"
+MAX_MANIFEST = 16 * 1024 * 1024
+MAX_JOURNAL = 32 * 1024 * 1024
+MAX_FILE_SIZE = (1 << 53) - 1
+MAX_FILES = 8192
+WINDOWS_RESERVED = {
+ "con", "prn", "aux", "nul",
+ *(f"com{number}" for number in range(1, 10)),
+ *(f"lpt{number}" for number in range(1, 10)),
+}
+
+
+class QpackInstallError(RuntimeError):
+ pass
+
+
+@dataclass(frozen=True, order=True)
+class QpackFile:
+ name: str
+ size: int
+
+
+@dataclass(frozen=True)
+class QpackInstallPlan:
+ source_id: str
+ manifest_sha256: str
+ files: tuple[QpackFile, ...]
+
+
+@dataclass(frozen=True)
+class ResumeDecision:
+ action: Literal["complete", "resume", "restart"]
+ offset: int
+ partial_path: Path
+
+
+@dataclass
+class QpackInstallSession:
+ root: Path
+ plan: QpackInstallPlan
+ _lock_stream: object
+ complete: bool = False
+ closed: bool = False
+
+ def close(self) -> None:
+ if self.closed:
+ return
+ _unlock(self._lock_stream)
+ self._lock_stream.close()
+ self.closed = True
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, _type, _value, _traceback):
+ self.close()
+
+
+def _object_without_duplicates(pairs):
+ value = {}
+ for key, item in pairs:
+ if key in value:
+ raise QpackInstallError(f"duplicate JSON key: {key}")
+ value[key] = item
+ return value
+
+
+def _reject_constant(value):
+ raise QpackInstallError(f"non-finite JSON number: {value}")
+
+
+def _has_nul(value) -> bool:
+ if isinstance(value, str):
+ return "\x00" in value
+ if isinstance(value, list):
+ return any(_has_nul(item) for item in value)
+ if isinstance(value, dict):
+ return any(_has_nul(key) or _has_nul(item) for key, item in value.items())
+ return False
+
+
+def _decode_json(raw: bytes, description: str, maximum=MAX_MANIFEST):
+ if not isinstance(raw, (bytes, bytearray)) or not raw or len(raw) > maximum:
+ raise QpackInstallError(f"{description} has an invalid size")
+ try:
+ text = raw.decode("utf-8")
+ value = json.loads(text, object_pairs_hook=_object_without_duplicates,
+ parse_constant=_reject_constant)
+ if _has_nul(value):
+ raise QpackInstallError(f"{description} contains an embedded NUL")
+ return value
+ except (UnicodeDecodeError, json.JSONDecodeError, RecursionError) as error:
+ raise QpackInstallError(f"invalid {description}: {error}") from error
+
+
+def _validate_name(name: str) -> str:
+ if (not isinstance(name, str) or not name or not name.isascii()
+ or "\\" in name or "\x00" in name
+ or len(name.encode("utf-8")) > 4096):
+ raise QpackInstallError("qpack file name is invalid")
+ components = name.split("/")
+ path = PurePosixPath(name)
+ if (path.is_absolute() or any(part in ("", ".", "..") for part in components)
+ or path.as_posix() != name):
+ raise QpackInstallError(f"unsafe qpack file name: {name}")
+ for component in components:
+ if (component[-1] in (" ", ".")
+ or len(component.encode("utf-8")) > 255
+ or any(ord(character) < 32 or character in '<>:"|?*'
+ for character in component)
+ or component.split(".", 1)[0].casefold() in WINDOWS_RESERVED):
+ raise QpackInstallError(f"Windows-unsafe qpack file name: {name}")
+ reserved = {
+ MANIFEST, JOURNAL, LOCK, f".{MANIFEST}.part",
+ f".{JOURNAL}.part",
+ }
+ if name.casefold() in {value.casefold() for value in reserved}:
+ raise QpackInstallError(f"reserved qpack file name: {name}")
+ return name
+
+
+def _validate_file(file: QpackFile) -> None:
+ _validate_name(file.name)
+ if (isinstance(file.size, bool) or not isinstance(file.size, int)
+ or file.size <= 0 or file.size > MAX_FILE_SIZE):
+ raise QpackInstallError(f"invalid qpack file size for {file.name}")
+
+
+def parse_manifest(raw: bytes, source_id: str) -> QpackInstallPlan:
+ """Parse a qpack v1 manifest and bind it to an immutable source identity."""
+ if (not isinstance(source_id, str) or not source_id.strip()
+ or "\x00" in source_id or len(source_id.encode("utf-8")) > 4096):
+ raise QpackInstallError("source_id must identify an immutable source revision")
+ manifest = _decode_json(raw, "qpack manifest")
+ if not isinstance(manifest, dict):
+ raise QpackInstallError("qpack manifest must be an object")
+ version = manifest.get("version")
+ if (manifest.get("magic") != "QPACK" or isinstance(version, bool)
+ or version != 1
+ or not isinstance(manifest.get("modelName"), str)
+ or not manifest["modelName"]
+ or not isinstance(manifest.get("sourceCheckpoint"), str)
+ or not manifest["sourceCheckpoint"]):
+ raise QpackInstallError("manifest is not a supported QPACK v1 container")
+ files = manifest.get("files")
+ if not isinstance(files, dict) or not files or len(files) > MAX_FILES:
+ raise QpackInstallError("qpack manifest files table is invalid")
+ for field in ("quantBits", "quantGroupSize"):
+ value = manifest.get(field)
+ if (value is not None and (isinstance(value, bool)
+ or not isinstance(value, int)
+ or value < 0 or value > (1 << 31) - 1)):
+ raise QpackInstallError(f"manifest {field} is invalid")
+
+ planned = []
+ for name, size in files.items():
+ name = _validate_name(name)
+ file = QpackFile(name, size)
+ _validate_file(file)
+ planned.append(file)
+ if not any(file.name == "packed_experts/layout.json" for file in planned):
+ raise QpackInstallError("qpack manifest does not declare packed_experts/layout.json")
+ _validate_namespace(planned)
+ return QpackInstallPlan(
+ source_id=source_id,
+ manifest_sha256=hashlib.sha256(raw).hexdigest(),
+ files=tuple(sorted(planned)),
+ )
+
+
+def _validate_namespace(files) -> None:
+ paths = [MANIFEST, JOURNAL, LOCK, f".{MANIFEST}.part", f".{JOURNAL}.part"]
+ for file in files:
+ paths.extend((file.name, file.name + ".part"))
+ normalized = sorted(path.casefold() for path in paths)
+ for previous, path in zip(normalized, normalized[1:]):
+ if path == previous:
+ raise QpackInstallError(f"qpack file namespace collision: {path}")
+ occupied = set(normalized)
+ for path in normalized:
+ components = path.split("/")
+ for count in range(1, len(components)):
+ parent = "/".join(components[:count])
+ if parent in occupied:
+ raise QpackInstallError(
+ f"qpack file/parent namespace collision: {parent}")
+
+
+def _fsync_directory(path: Path) -> None:
+ if os.name == "nt":
+ return
+ descriptor = os.open(path, os.O_RDONLY)
+ try:
+ try:
+ os.fsync(descriptor)
+ except OSError as error:
+ if error.errno not in (errno.EBADF, errno.EINVAL, errno.ENOTSUP):
+ raise
+ finally:
+ os.close(descriptor)
+
+
+def _atomic_bytes(path: Path, payload: bytes) -> None:
+ temporary = path.with_name(f".{path.name}.part")
+ if temporary.is_symlink():
+ raise QpackInstallError(f"refusing symlink temporary path: {temporary}")
+ temporary.unlink(missing_ok=True)
+ try:
+ with temporary.open("xb") as stream:
+ stream.write(payload)
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(temporary, path)
+ _fsync_directory(path.parent)
+ except Exception:
+ temporary.unlink(missing_ok=True)
+ raise
+
+
+def _atomic_json(path: Path, payload) -> None:
+ raw = json.dumps(payload, sort_keys=True, separators=(",", ":"),
+ ensure_ascii=False).encode("utf-8") + b"\n"
+ if len(raw) > MAX_JOURNAL:
+ raise QpackInstallError("qpack install journal exceeds its size limit")
+ _atomic_bytes(path, raw)
+
+
+def _journal_payload(plan: QpackInstallPlan):
+ return {
+ "schema": SCHEMA,
+ "source_id": plan.source_id,
+ "manifest_sha256": plan.manifest_sha256,
+ "files": [{"name": file.name, "size": file.size} for file in plan.files],
+ }
+
+
+def _read_json_file(path: Path, description: str):
+ try:
+ if path.is_symlink():
+ raise QpackInstallError(f"refusing symlink {description}: {path}")
+ return _decode_json(path.read_bytes(), description, MAX_JOURNAL)
+ except OSError as error:
+ raise QpackInstallError(f"cannot read {description} {path}: {error}") from error
+
+
+def _root(path) -> Path:
+ root = Path(path).expanduser().absolute()
+ if root.is_symlink():
+ raise QpackInstallError(f"qpack install root is a symlink: {root}")
+ root.mkdir(parents=True, exist_ok=True)
+ if not root.is_dir():
+ raise QpackInstallError(f"qpack install root is not a directory: {root}")
+ return root
+
+
+def _lock(root: Path):
+ path = root / LOCK
+ if path.is_symlink():
+ raise QpackInstallError(f"qpack install lock is a symlink: {path}")
+ stream = path.open("a+b")
+ try:
+ if os.name == "nt":
+ import msvcrt
+ stream.seek(0, os.SEEK_END)
+ if stream.tell() == 0:
+ stream.write(b"\0")
+ stream.flush()
+ stream.seek(0)
+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+ else:
+ import fcntl
+ fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ return stream
+ except (OSError, BlockingIOError) as error:
+ stream.close()
+ raise QpackInstallError(f"qpack install is locked: {root}") from error
+
+
+def _unlock(stream) -> None:
+ try:
+ if os.name == "nt":
+ import msvcrt
+ stream.seek(0)
+ msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+ fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
+ except OSError:
+ pass
+
+
+def _paths(root: Path, file: QpackFile, create_parent: bool = False):
+ _validate_file(file)
+ parent = root
+ components = file.name.split("/")
+ for component in components[:-1]:
+ parent = parent / component
+ if parent.is_symlink():
+ raise QpackInstallError(f"qpack path crosses a symlink: {parent}")
+ if parent.exists() and not parent.is_dir():
+ raise QpackInstallError(f"qpack parent is not a directory: {parent}")
+ if create_parent and not parent.exists():
+ parent.mkdir()
+ _fsync_directory(parent.parent)
+ target = parent / components[-1]
+ partial = target.with_name(target.name + ".part")
+ if target.is_symlink() or partial.is_symlink():
+ raise QpackInstallError(f"qpack artifact path is a symlink: {target}")
+ return target, partial
+
+
+def _matching_journal(root: Path, plan: QpackInstallPlan | None = None):
+ path = root / JOURNAL
+ if not path.is_file():
+ raise QpackInstallError("qpack install journal is missing")
+ journal = _read_json_file(path, "qpack install journal")
+ if not isinstance(journal, dict) or journal.get("schema") != SCHEMA:
+ raise QpackInstallError("qpack install journal is invalid")
+ if plan is not None and journal != _journal_payload(plan):
+ raise QpackInstallError("qpack install journal belongs to another source or manifest")
+ return journal
+
+
+def _journal_declares(root: Path, file: QpackFile) -> None:
+ journal = _matching_journal(root)
+ entry = {"name": file.name, "size": file.size}
+ if entry not in journal.get("files", []):
+ raise QpackInstallError(f"qpack install journal does not declare {file.name}")
+
+
+def _all_files_exact(root: Path, plan: QpackInstallPlan) -> bool:
+ for file in plan.files:
+ target, partial = _paths(root, file)
+ if (partial.exists() or not target.is_file()
+ or target.stat().st_size != file.size):
+ return False
+ return True
+
+
+def begin_install(root, plan: QpackInstallPlan) -> QpackInstallSession:
+ """Lock the root and create or verify its source-bound install journal."""
+ _validate_plan(plan)
+ root = _root(root)
+ lock_stream = _lock(root)
+ session = QpackInstallSession(root, plan, lock_stream)
+ manifest_path = root / MANIFEST
+ journal_path = root / JOURNAL
+ try:
+ if manifest_path.is_symlink():
+ raise QpackInstallError("existing qpack manifest is a symlink")
+ if manifest_path.exists():
+ if not manifest_path.is_file():
+ raise QpackInstallError("existing qpack manifest is not a regular file")
+ digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
+ if digest != plan.manifest_sha256 or not _all_files_exact(root, plan):
+ raise QpackInstallError("install root contains a different or incomplete qpack")
+ journal_path.unlink(missing_ok=True)
+ _fsync_directory(root)
+ session.complete = True
+ return session
+
+ if journal_path.is_symlink():
+ raise QpackInstallError("qpack install journal is a symlink")
+ if journal_path.exists():
+ _matching_journal(root, plan)
+ return session
+
+ for file in plan.files:
+ target, partial = _paths(root, file)
+ if target.exists() or partial.exists():
+ raise QpackInstallError(
+ f"unbound qpack artifact exists before journal creation: {file.name}")
+ _atomic_json(journal_path, _journal_payload(plan))
+ return session
+ except Exception:
+ session.close()
+ raise
+
+
+def _validate_plan(plan: QpackInstallPlan) -> None:
+ if (not isinstance(plan, QpackInstallPlan)
+ or not isinstance(plan.source_id, str) or not plan.source_id.strip()
+ or "\x00" in plan.source_id
+ or len(plan.source_id.encode("utf-8")) > 4096
+ or not isinstance(plan.manifest_sha256, str)
+ or len(plan.manifest_sha256) != 64):
+ raise QpackInstallError("qpack install plan identity is invalid")
+ try:
+ int(plan.manifest_sha256, 16)
+ except ValueError as error:
+ raise QpackInstallError("qpack manifest digest is invalid") from error
+ if (not isinstance(plan.files, tuple) or not plan.files
+ or len(plan.files) > MAX_FILES):
+ raise QpackInstallError("qpack install plan files are invalid")
+ for file in plan.files:
+ _validate_file(file)
+ _validate_namespace(plan.files)
+
+
+def _require_session(session: QpackInstallSession) -> None:
+ if not isinstance(session, QpackInstallSession) or session.closed:
+ raise QpackInstallError("qpack install session is closed or invalid")
+ if session.complete:
+ raise QpackInstallError("qpack install is already complete")
+ _matching_journal(session.root, session.plan)
+
+
+def resume_decision(session: QpackInstallSession,
+ file: QpackFile) -> ResumeDecision:
+ _require_session(session)
+ _journal_declares(session.root, file)
+ target, partial = _paths(session.root, file, create_parent=True)
+ if target.exists():
+ if not target.is_file() or target.stat().st_size != file.size:
+ raise QpackInstallError(f"committed qpack artifact has wrong size: {file.name}")
+ if partial.exists():
+ raise QpackInstallError(f"committed and partial qpack artifacts both exist: {file.name}")
+ return ResumeDecision("complete", file.size, partial)
+ if not partial.exists():
+ return ResumeDecision("restart", 0, partial)
+ if not partial.is_file():
+ raise QpackInstallError(f"qpack partial is not a regular file: {partial}")
+ size = partial.stat().st_size
+ if size > file.size:
+ return ResumeDecision("restart", 0, partial)
+ return ResumeDecision("resume", size, partial)
+
+
+def validate_delivery(file: QpackFile, requested_offset: int,
+ delivered_offset: int, delivered_size: int,
+ *, complete: bool = False) -> int:
+ """Validate the byte range a transport claims it delivered."""
+ _validate_file(file)
+ values = (requested_offset, delivered_offset, delivered_size)
+ if any(isinstance(value, bool) or not isinstance(value, int) for value in values):
+ raise QpackInstallError("delivery offsets and size must be integers")
+ if requested_offset < 0 or requested_offset > file.size:
+ raise QpackInstallError("requested qpack offset is out of bounds")
+ if delivered_offset != requested_offset:
+ raise QpackInstallError("transport did not honor the requested qpack range")
+ if delivered_size < 0 or delivered_offset + delivered_size > file.size:
+ raise QpackInstallError("delivered qpack range is out of bounds")
+ end = delivered_offset + delivered_size
+ if complete and end != file.size:
+ raise QpackInstallError("transport reported a short final qpack delivery")
+ return end
+
+
+def commit_file(session: QpackInstallSession, file: QpackFile) -> None:
+ """Durably publish one exact-size partial artifact."""
+ _require_session(session)
+ _journal_declares(session.root, file)
+ target, partial = _paths(session.root, file, create_parent=True)
+ if target.exists():
+ if target.is_file() and target.stat().st_size == file.size and not partial.exists():
+ return
+ raise QpackInstallError(f"cannot commit over existing qpack artifact: {file.name}")
+ if not partial.is_file() or partial.stat().st_size != file.size:
+ raise QpackInstallError(f"qpack partial is not complete: {file.name}")
+ descriptor = os.open(partial, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ try:
+ os.fsync(descriptor)
+ finally:
+ os.close(descriptor)
+ os.replace(partial, target)
+ _fsync_directory(target.parent)
+
+
+def publish_manifest(session: QpackInstallSession, raw: bytes) -> None:
+ """Publish manifest.json last, after every declared artifact is durable."""
+ _require_session(session)
+ root = session.root
+ plan = session.plan
+ parsed = parse_manifest(raw, session.plan.source_id)
+ if parsed != session.plan:
+ raise QpackInstallError("manifest bytes do not match the qpack install plan")
+ if not _all_files_exact(root, plan):
+ raise QpackInstallError("cannot publish qpack manifest before all files are complete")
+ manifest_path = root / MANIFEST
+ if manifest_path.is_symlink():
+ raise QpackInstallError("refusing symlink qpack manifest")
+ if manifest_path.exists():
+ if (not manifest_path.is_file()
+ or hashlib.sha256(manifest_path.read_bytes()).hexdigest()
+ != plan.manifest_sha256):
+ raise QpackInstallError("refusing to replace a different qpack manifest")
+ else:
+ _atomic_bytes(manifest_path, raw)
+ (root / JOURNAL).unlink(missing_ok=True)
+ _fsync_directory(root)
+ session.complete = True