Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(tools): resumable qpack install policy
Léon Avic Simmons40f466f3560b
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.mdindex ca473c78..01a7a2ce 100644--- a/THIRD_PARTY_NOTICES.md+++ b/THIRD_PARTY_NOTICES.md@@ -48,3 +48,14 @@ What that checkout contains, and the licences that apply when you build with it:- NVIDIA CUTLASS / CuTe (BSD-3-Clause), the `third-party/cutlass` submodule ofthat commit @ f3fde58372d33e9a5650ba7b80fc48b3b49d40c8. License text:`c/third_party/deepgemm/third-party/cutlass/LICENSE.txt` after the fetch.++## Swiftlet qpack installers (`c/tools/qpack_*install*.py`)++Swiftlet (https://github.com/leonickson1/Swiftlet) is licensed under Apache+License 2.0. This repository's `LICENSE` contains the applicable Apache 2.0+text.++`c/tools/qpack_install_policy.py` adapts the source-bound resume and+manifest-last completion policy from Swiftlet's `StreamingInstaller.swift` at+commit `86246618ba2af30334227e09ff84a6a7182c2a40`. It is a new+transport-neutral Python implementation and does not copy Swift source code.diff --git a/c/tests/test_qpack_install_policy.py b/c/tests/test_qpack_install_policy.pynew file mode 100644index 00000000..dff628c7--- /dev/null+++ b/c/tests/test_qpack_install_policy.py@@ -0,0 +1,308 @@+import json+import os+import tempfile+import unittest+from pathlib import Path++from tools.qpack_install_policy import (+ JOURNAL,+ MANIFEST,+ MAX_FILE_SIZE,+ QpackFile,+ QpackInstallError,+ begin_install,+ commit_file,+ parse_manifest,+ publish_manifest,+ resume_decision,+ validate_delivery,+)+++class QpackInstallPolicyTest(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name) / "model.qpack"+ self.raw = self.manifest({+ "model.safetensors": 7,+ "packed_experts/layout.json": 5,+ })+ self.plan = parse_manifest(self.raw, "hf:owner/model@0123456789abcdef")+ self.sessions = []+ self.addCleanup(self.close_sessions)++ def close_sessions(self):+ for session in reversed(self.sessions):+ session.close()++ def begin(self, plan=None):+ session = begin_install(self.root, plan or self.plan)+ self.sessions.append(session)+ return session++ @staticmethod+ def manifest(files, **extra):+ value = {+ "magic": "QPACK",+ "version": 1,+ "modelName": "tiny-qwen",+ "sourceCheckpoint": "owner/model@0123456789abcdef",+ "quantBits": 4,+ "quantGroupSize": 64,+ "files": files,+ }+ value.update(extra)+ return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()++ def file(self, name):+ return next(file for file in self.plan.files if file.name == name)++ def write_partial(self, session, file, data):+ decision = resume_decision(session, file)+ decision.partial_path.write_bytes(data)+ return decision.partial_path++ def commit_all(self, session):+ for file in self.plan.files:+ self.write_partial(session, self.file(file.name),+ bytes([file.size]) * file.size)+ commit_file(session, file)++ def test_manifest_is_sorted_and_bound_to_exact_bytes_and_source(self):+ reordered = self.raw + b"\n"+ other = parse_manifest(reordered, self.plan.source_id)+ self.assertEqual([file.name for file in other.files],+ ["model.safetensors", "packed_experts/layout.json"])+ self.assertNotEqual(other.manifest_sha256, self.plan.manifest_sha256)+ self.assertNotEqual(parse_manifest(self.raw, "local:fixture"), self.plan)++ def test_manifest_rejects_duplicate_keys(self):+ raw = (b'{"magic":"QPACK","version":1,"files":'+ b'{"packed_experts/layout.json":5,'+ b'"packed_experts/layout.json":6}}')+ with self.assertRaisesRegex(QpackInstallError, "duplicate JSON key"):+ parse_manifest(raw, "local:fixture")++ def test_manifest_rejects_nonfinite_numbers_and_escaped_nuls(self):+ raw = self.raw.replace(b'"quantBits":4', b'"quantBits":NaN')+ with self.assertRaisesRegex(QpackInstallError, "non-finite"):+ parse_manifest(raw, "local:fixture")+ value = json.loads(self.raw)+ value["modelName"] = "tiny\x00qwen"+ with self.assertRaisesRegex(QpackInstallError, "embedded NUL"):+ parse_manifest(json.dumps(value).encode(), "local:fixture")++ def test_manifest_rejects_unsafe_and_reserved_paths(self):+ invalid = [+ "../escape", "/absolute", "a//b", "a/./b", "a/../b",+ "a\\b", "manifest.json", ".qpack-install.json",+ ".qpack-install.lock", ".manifest.json.part", "nul\x00name",+ "C:/escape", "C:escape", "packed_experts/CON",+ "packed_experts/aux.txt", "packed_experts/trailing. ",+ "packed_experts/file:stream", "packed_experts/a?.bin",+ "packed_experts/caf\u00e9.bin",+ ]+ for name in invalid:+ files = {"packed_experts/layout.json": 5, name: 1}+ with self.subTest(name=name), self.assertRaises(QpackInstallError):+ parse_manifest(self.manifest(files), "local:fixture")++ def test_manifest_rejects_invalid_sizes_and_missing_layout(self):+ for size in (True, False, 0, -1, 1.5, MAX_FILE_SIZE + 1):+ with self.subTest(size=size), self.assertRaises(QpackInstallError):+ parse_manifest(self.manifest({+ "packed_experts/layout.json": size,+ }), "local:fixture")+ with self.assertRaisesRegex(QpackInstallError, "layout.json"):+ parse_manifest(self.manifest({"model.safetensors": 7}), "local:fixture")+ with self.assertRaisesRegex(QpackInstallError, "source_id"):+ parse_manifest(self.raw, "")++ def test_manifest_matches_swiftlet_required_fields_and_file_limit(self):+ for field in ("modelName", "sourceCheckpoint"):+ value = json.loads(self.raw)+ value.pop(field)+ with self.subTest(field=field), self.assertRaisesRegex(+ QpackInstallError, "supported QPACK"):+ parse_manifest(json.dumps(value).encode(), "local:fixture")+ for field in ("quantBits", "quantGroupSize"):+ value = json.loads(self.raw)+ value[field] = True+ with self.subTest(field=field), self.assertRaisesRegex(+ QpackInstallError, field):+ parse_manifest(json.dumps(value).encode(), "local:fixture")+ too_many = {"packed_experts/layout.json": 5}+ too_many.update({f"packed_experts/file-{index}.bin": 1+ for index in range(8192)})+ with self.assertRaisesRegex(QpackInstallError, "files table"):+ parse_manifest(self.manifest(too_many), "local:fixture")++ def test_manifest_rejects_plan_wide_path_collisions(self):+ collisions = [+ {"packed_experts/layout.json": 5, "foo": 1, "foo.part": 1},+ {"packed_experts/layout.json": 5, "foo": 1, "foo/bar": 1},+ {"packed_experts/layout.json": 5, "Foo": 1, "foo": 1},+ ]+ for files in collisions:+ with self.subTest(files=files), self.assertRaisesRegex(+ QpackInstallError, "namespace collision"):+ parse_manifest(self.manifest(files), "local:fixture")++ def test_begin_binds_partials_and_rejects_unbound_or_mismatched_work(self):+ self.root.mkdir()+ (self.root / "model.safetensors.part").write_bytes(b"old")+ with self.assertRaisesRegex(QpackInstallError, "unbound"):+ begin_install(self.root, self.plan)+ (self.root / "model.safetensors.part").unlink()++ session = self.begin()+ with self.assertRaisesRegex(QpackInstallError, "locked"):+ begin_install(self.root, self.plan)+ session.close()+ other_raw = self.manifest({+ "model.safetensors": 8,+ "packed_experts/layout.json": 5,+ })+ other = parse_manifest(other_raw, self.plan.source_id)+ with self.assertRaisesRegex(QpackInstallError, "another source or manifest"):+ begin_install(self.root, other)++ def test_malformed_journal_and_closed_session_fail_closed(self):+ session = self.begin()+ session.close()+ (self.root / JOURNAL).write_text('{"schema":"wrong"}', encoding="utf-8")+ with self.assertRaisesRegex(QpackInstallError, "journal is invalid"):+ begin_install(self.root, self.plan)+ with self.assertRaisesRegex(QpackInstallError, "closed or invalid"):+ resume_decision(session, self.plan.files[0])++ def test_resume_decisions_cover_missing_partial_and_committed_files(self):+ session = self.begin()+ file = self.file("model.safetensors")+ decision = resume_decision(session, file)+ self.assertEqual((decision.action, decision.offset), ("restart", 0))++ decision.partial_path.write_bytes(b"abc")+ decision = resume_decision(session, file)+ self.assertEqual((decision.action, decision.offset), ("resume", 3))++ decision.partial_path.write_bytes(b"too-large")+ decision = resume_decision(session, file)+ self.assertEqual((decision.action, decision.offset), ("restart", 0))++ decision.partial_path.write_bytes(b"1234567")+ commit_file(session, file)+ decision = resume_decision(session, file)+ self.assertEqual((decision.action, decision.offset), ("complete", 7))++ def test_relative_root_remains_stable_after_working_directory_change(self):+ original = Path.cwd()+ self.addCleanup(os.chdir, original)+ os.chdir(self.temporary.name)+ session = begin_install(Path("relative") / "model.qpack", self.plan)+ self.sessions.append(session)+ os.chdir(original)+ decision = resume_decision(session, self.plan.files[0])+ self.assertTrue(decision.partial_path.is_absolute())+ self.assertEqual(decision.offset, 0)++ def test_delivery_requires_exact_offsets_bounds_and_final_length(self):+ file = QpackFile("model.safetensors", 10)+ self.assertEqual(validate_delivery(file, 3, 3, 4), 7)+ self.assertEqual(validate_delivery(file, 3, 3, 7, complete=True), 10)+ invalid = [+ (-1, 0, 1, False),+ (3, 0, 7, True),+ (3, 3, 8, False),+ (3, 3, 6, True),+ ]+ for requested, delivered, size, complete in invalid:+ with self.subTest(values=(requested, delivered, size, complete)), \+ self.assertRaises(QpackInstallError):+ validate_delivery(file, requested, delivered, size,+ complete=complete)++ def test_incomplete_commit_never_publishes_a_final_path(self):+ session = self.begin()+ file = self.file("model.safetensors")+ partial = self.write_partial(session, file, b"short")+ with self.assertRaisesRegex(QpackInstallError, "not complete"):+ commit_file(session, file)+ self.assertTrue(partial.exists())+ self.assertFalse(self.root.joinpath(file.name).exists())++ def test_manifest_is_published_last_and_completion_is_idempotent(self):+ session = self.begin()+ first = self.plan.files[0]+ self.write_partial(session, first, bytes([first.size]) * first.size)+ commit_file(session, first)+ with self.assertRaisesRegex(QpackInstallError, "all files"):+ publish_manifest(session, self.raw)+ self.assertFalse((self.root / MANIFEST).exists())++ remaining = self.plan.files[1]+ self.write_partial(session, remaining,+ bytes([remaining.size]) * remaining.size)+ commit_file(session, remaining)+ publish_manifest(session, self.raw)+ self.assertEqual((self.root / MANIFEST).read_bytes(), self.raw)+ self.assertFalse((self.root / JOURNAL).exists())++ session.close()+ complete_session = self.begin()+ self.assertTrue(complete_session.complete)+ self.assertFalse((self.root / JOURNAL).exists())++ def test_crash_boundaries_remain_recoverable(self):+ session = self.begin()+ first = self.plan.files[0]+ self.write_partial(session, first, bytes([first.size]) * first.size)+ self.assertEqual(resume_decision(session, first).offset, first.size)+ session.close()+ session = self.begin()+ commit_file(session, first)+ self.assertEqual(resume_decision(session, first).action, "complete")++ for file in self.plan.files[1:]:+ self.write_partial(session, file, bytes([file.size]) * file.size)+ commit_file(session, file)+ stale_manifest_temp = self.root / ".manifest.json.part"+ stale_manifest_temp.write_bytes(b"interrupted")+ publish_manifest(session, self.raw)+ self.assertEqual((self.root / MANIFEST).read_bytes(), self.raw)+ self.assertFalse(stale_manifest_temp.exists())++ def test_existing_manifest_cleans_a_stale_matching_journal(self):+ session = self.begin()+ self.commit_all(session)+ (self.root / MANIFEST).write_bytes(self.raw)+ self.assertTrue((self.root / JOURNAL).exists())+ session.close()+ complete_session = self.begin()+ self.assertTrue(complete_session.complete)+ self.assertFalse((self.root / JOURNAL).exists())++ @unittest.skipUnless(hasattr(os, "symlink"), "symlinks unsupported")+ def test_manifest_symlink_cannot_mark_install_complete(self):+ session = self.begin()+ self.commit_all(session)+ external = Path(self.temporary.name) / "external-manifest.json"+ external.write_bytes(self.raw)+ os.symlink(external, self.root / MANIFEST)+ with self.assertRaisesRegex(QpackInstallError, "symlink"):+ publish_manifest(session, self.raw)+ self.assertTrue((self.root / JOURNAL).exists())++ @unittest.skipUnless(hasattr(os, "symlink"), "symlinks unsupported")+ def test_symlinked_artifact_parent_is_rejected(self):+ outside = Path(self.temporary.name) / "outside"+ outside.mkdir()+ self.root.mkdir()+ os.symlink(outside, self.root / "packed_experts", target_is_directory=True)+ with self.assertRaisesRegex(QpackInstallError, "symlink"):+ begin_install(self.root, self.plan)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/README.md b/c/tools/README.mdindex 7a7135e8..24d84cec 100644--- a/c/tools/README.md+++ b/c/tools/README.md@@ -4,6 +4,13 @@ These scripts support model preparation and offline engineering work. They arenot runtime dependencies of the C engine.- `convert_fp8_to_int4.py`, `download_glm52.py`: model preparation+- `qpack_install_policy.py`: source-bound resume decisions and manifest-last+ publication primitives for future qpack download/repack frontends. Hold its+ install session for the full transfer. Qpack v1 has sizes but no content+ hashes, so frontends must also enforce immutable revisions and any available+ transport checksum or ETag. It covers manifest-declared container artifacts,+ not required runtime auxiliaries such as `config.json`. Directory-fsync crash+ durability is POSIX-only.- `convert_fmt4_to_fmt2.py`: fmt=4 (grouped int4) -> fmt=2 (per-row int4)re-quant of a GLM-5.2 container, for Metal-backend compatibility(see `docs/METAL-M1ULTRA-FMT2-REPORT.md`)diff --git a/c/tools/qpack_install_policy.py b/c/tools/qpack_install_policy.pynew file mode 100644index 00000000..e5941e09--- /dev/null+++ b/c/tools/qpack_install_policy.py@@ -0,0 +1,513 @@+#!/usr/bin/env python3+"""Resumable qpack policy with POSIX-durable manifest-last publication.++Qpack v1 records file sizes but no content digests. Callers must use an+immutable ``source_id`` and validate any ETag or checksum supplied by the+transport; this module prevents cross-revision resume, not silent media damage.+"""++from __future__ import annotations++import errno+import hashlib+import json+import os+from dataclasses import dataclass+from pathlib import Path, PurePosixPath+from typing import Literal+++SCHEMA = "colibri.qpack-install.v1"+JOURNAL = ".qpack-install.json"+LOCK = ".qpack-install.lock"+MANIFEST = "manifest.json"+MAX_MANIFEST = 16 * 1024 * 1024+MAX_JOURNAL = 32 * 1024 * 1024+MAX_FILE_SIZE = (1 << 53) - 1+MAX_FILES = 8192+WINDOWS_RESERVED = {+ "con", "prn", "aux", "nul",+ *(f"com{number}" for number in range(1, 10)),+ *(f"lpt{number}" for number in range(1, 10)),+}+++class QpackInstallError(RuntimeError):+ pass+++@dataclass(frozen=True, order=True)+class QpackFile:+ name: str+ size: int+++@dataclass(frozen=True)+class QpackInstallPlan:+ source_id: str+ manifest_sha256: str+ files: tuple[QpackFile, ...]+++@dataclass(frozen=True)+class ResumeDecision:+ action: Literal["complete", "resume", "restart"]+ offset: int+ partial_path: Path+++@dataclass+class QpackInstallSession:+ root: Path+ plan: QpackInstallPlan+ _lock_stream: object+ complete: bool = False+ closed: bool = False++ def close(self) -> None:+ if self.closed:+ return+ _unlock(self._lock_stream)+ self._lock_stream.close()+ self.closed = True++ def __enter__(self):+ return self++ def __exit__(self, _type, _value, _traceback):+ self.close()+++def _object_without_duplicates(pairs):+ value = {}+ for key, item in pairs:+ if key in value:+ raise QpackInstallError(f"duplicate JSON key: {key}")+ value[key] = item+ return value+++def _reject_constant(value):+ raise QpackInstallError(f"non-finite JSON number: {value}")+++def _has_nul(value) -> bool:+ if isinstance(value, str):+ return "\x00" in value+ if isinstance(value, list):+ return any(_has_nul(item) for item in value)+ if isinstance(value, dict):+ return any(_has_nul(key) or _has_nul(item) for key, item in value.items())+ return False+++def _decode_json(raw: bytes, description: str, maximum=MAX_MANIFEST):+ if not isinstance(raw, (bytes, bytearray)) or not raw or len(raw) > maximum:+ raise QpackInstallError(f"{description} has an invalid size")+ try:+ text = raw.decode("utf-8")+ value = json.loads(text, object_pairs_hook=_object_without_duplicates,+ parse_constant=_reject_constant)+ if _has_nul(value):+ raise QpackInstallError(f"{description} contains an embedded NUL")+ return value+ except (UnicodeDecodeError, json.JSONDecodeError, RecursionError) as error:+ raise QpackInstallError(f"invalid {description}: {error}") from error+++def _validate_name(name: str) -> str:+ if (not isinstance(name, str) or not name or not name.isascii()+ or "\\" in name or "\x00" in name+ or len(name.encode("utf-8")) > 4096):+ raise QpackInstallError("qpack file name is invalid")+ components = name.split("/")+ path = PurePosixPath(name)+ if (path.is_absolute() or any(part in ("", ".", "..") for part in components)+ or path.as_posix() != name):+ raise QpackInstallError(f"unsafe qpack file name: {name}")+ for component in components:+ if (component[-1] in (" ", ".")+ or len(component.encode("utf-8")) > 255+ or any(ord(character) < 32 or character in '<>:"|?*'+ for character in component)+ or component.split(".", 1)[0].casefold() in WINDOWS_RESERVED):+ raise QpackInstallError(f"Windows-unsafe qpack file name: {name}")+ reserved = {+ MANIFEST, JOURNAL, LOCK, f".{MANIFEST}.part",+ f".{JOURNAL}.part",+ }+ if name.casefold() in {value.casefold() for value in reserved}:+ raise QpackInstallError(f"reserved qpack file name: {name}")+ return name+++def _validate_file(file: QpackFile) -> None:+ _validate_name(file.name)+ if (isinstance(file.size, bool) or not isinstance(file.size, int)+ or file.size <= 0 or file.size > MAX_FILE_SIZE):+ raise QpackInstallError(f"invalid qpack file size for {file.name}")+++def parse_manifest(raw: bytes, source_id: str) -> QpackInstallPlan:+ """Parse a qpack v1 manifest and bind it to an immutable source identity."""+ if (not isinstance(source_id, str) or not source_id.strip()+ or "\x00" in source_id or len(source_id.encode("utf-8")) > 4096):+ raise QpackInstallError("source_id must identify an immutable source revision")+ manifest = _decode_json(raw, "qpack manifest")+ if not isinstance(manifest, dict):+ raise QpackInstallError("qpack manifest must be an object")+ version = manifest.get("version")+ if (manifest.get("magic") != "QPACK" or isinstance(version, bool)+ or version != 1+ or not isinstance(manifest.get("modelName"), str)+ or not manifest["modelName"]+ or not isinstance(manifest.get("sourceCheckpoint"), str)+ or not manifest["sourceCheckpoint"]):+ raise QpackInstallError("manifest is not a supported QPACK v1 container")+ files = manifest.get("files")+ if not isinstance(files, dict) or not files or len(files) > MAX_FILES:+ raise QpackInstallError("qpack manifest files table is invalid")+ for field in ("quantBits", "quantGroupSize"):+ value = manifest.get(field)+ if (value is not None and (isinstance(value, bool)+ or not isinstance(value, int)+ or value < 0 or value > (1 << 31) - 1)):+ raise QpackInstallError(f"manifest {field} is invalid")++ planned = []+ for name, size in files.items():+ name = _validate_name(name)+ file = QpackFile(name, size)+ _validate_file(file)+ planned.append(file)+ if not any(file.name == "packed_experts/layout.json" for file in planned):+ raise QpackInstallError("qpack manifest does not declare packed_experts/layout.json")+ _validate_namespace(planned)+ return QpackInstallPlan(+ source_id=source_id,+ manifest_sha256=hashlib.sha256(raw).hexdigest(),+ files=tuple(sorted(planned)),+ )+++def _validate_namespace(files) -> None:+ paths = [MANIFEST, JOURNAL, LOCK, f".{MANIFEST}.part", f".{JOURNAL}.part"]+ for file in files:+ paths.extend((file.name, file.name + ".part"))+ normalized = sorted(path.casefold() for path in paths)+ for previous, path in zip(normalized, normalized[1:]):+ if path == previous:+ raise QpackInstallError(f"qpack file namespace collision: {path}")+ occupied = set(normalized)+ for path in normalized:+ components = path.split("/")+ for count in range(1, len(components)):+ parent = "/".join(components[:count])+ if parent in occupied:+ raise QpackInstallError(+ f"qpack file/parent namespace collision: {parent}")+++def _fsync_directory(path: Path) -> None:+ if os.name == "nt":+ return+ descriptor = os.open(path, os.O_RDONLY)+ try:+ try:+ os.fsync(descriptor)+ except OSError as error:+ if error.errno not in (errno.EBADF, errno.EINVAL, errno.ENOTSUP):+ raise+ finally:+ os.close(descriptor)+++def _atomic_bytes(path: Path, payload: bytes) -> None:+ temporary = path.with_name(f".{path.name}.part")+ if temporary.is_symlink():+ raise QpackInstallError(f"refusing symlink temporary path: {temporary}")+ temporary.unlink(missing_ok=True)+ try:+ with temporary.open("xb") as stream:+ stream.write(payload)+ stream.flush()+ os.fsync(stream.fileno())+ os.replace(temporary, path)+ _fsync_directory(path.parent)+ except Exception:+ temporary.unlink(missing_ok=True)+ raise+++def _atomic_json(path: Path, payload) -> None:+ raw = json.dumps(payload, sort_keys=True, separators=(",", ":"),+ ensure_ascii=False).encode("utf-8") + b"\n"+ if len(raw) > MAX_JOURNAL:+ raise QpackInstallError("qpack install journal exceeds its size limit")+ _atomic_bytes(path, raw)+++def _journal_payload(plan: QpackInstallPlan):+ return {+ "schema": SCHEMA,+ "source_id": plan.source_id,+ "manifest_sha256": plan.manifest_sha256,+ "files": [{"name": file.name, "size": file.size} for file in plan.files],+ }+++def _read_json_file(path: Path, description: str):+ try:+ if path.is_symlink():+ raise QpackInstallError(f"refusing symlink {description}: {path}")+ return _decode_json(path.read_bytes(), description, MAX_JOURNAL)+ except OSError as error:+ raise QpackInstallError(f"cannot read {description} {path}: {error}") from error+++def _root(path) -> Path:+ root = Path(path).expanduser().absolute()+ if root.is_symlink():+ raise QpackInstallError(f"qpack install root is a symlink: {root}")+ root.mkdir(parents=True, exist_ok=True)+ if not root.is_dir():+ raise QpackInstallError(f"qpack install root is not a directory: {root}")+ return root+++def _lock(root: Path):+ path = root / LOCK+ if path.is_symlink():+ raise QpackInstallError(f"qpack install lock is a symlink: {path}")+ stream = path.open("a+b")+ try:+ if os.name == "nt":+ import msvcrt+ stream.seek(0, os.SEEK_END)+ if stream.tell() == 0:+ stream.write(b"\0")+ stream.flush()+ stream.seek(0)+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)+ else:+ import fcntl+ fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)+ return stream+ except (OSError, BlockingIOError) as error:+ stream.close()+ raise QpackInstallError(f"qpack install is locked: {root}") from error+++def _unlock(stream) -> None:+ try:+ if os.name == "nt":+ import msvcrt+ stream.seek(0)+ msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)+ else:+ import fcntl+ fcntl.flock(stream.fileno(), fcntl.LOCK_UN)+ except OSError:+ pass+++def _paths(root: Path, file: QpackFile, create_parent: bool = False):+ _validate_file(file)+ parent = root+ components = file.name.split("/")+ for component in components[:-1]:+ parent = parent / component+ if parent.is_symlink():+ raise QpackInstallError(f"qpack path crosses a symlink: {parent}")+ if parent.exists() and not parent.is_dir():+ raise QpackInstallError(f"qpack parent is not a directory: {parent}")+ if create_parent and not parent.exists():+ parent.mkdir()+ _fsync_directory(parent.parent)+ target = parent / components[-1]+ partial = target.with_name(target.name + ".part")+ if target.is_symlink() or partial.is_symlink():+ raise QpackInstallError(f"qpack artifact path is a symlink: {target}")+ return target, partial+++def _matching_journal(root: Path, plan: QpackInstallPlan | None = None):+ path = root / JOURNAL+ if not path.is_file():+ raise QpackInstallError("qpack install journal is missing")+ journal = _read_json_file(path, "qpack install journal")+ if not isinstance(journal, dict) or journal.get("schema") != SCHEMA:+ raise QpackInstallError("qpack install journal is invalid")+ if plan is not None and journal != _journal_payload(plan):+ raise QpackInstallError("qpack install journal belongs to another source or manifest")+ return journal+++def _journal_declares(root: Path, file: QpackFile) -> None:+ journal = _matching_journal(root)+ entry = {"name": file.name, "size": file.size}+ if entry not in journal.get("files", []):+ raise QpackInstallError(f"qpack install journal does not declare {file.name}")+++def _all_files_exact(root: Path, plan: QpackInstallPlan) -> bool:+ for file in plan.files:+ target, partial = _paths(root, file)+ if (partial.exists() or not target.is_file()+ or target.stat().st_size != file.size):+ return False+ return True+++def begin_install(root, plan: QpackInstallPlan) -> QpackInstallSession:+ """Lock the root and create or verify its source-bound install journal."""+ _validate_plan(plan)+ root = _root(root)+ lock_stream = _lock(root)+ session = QpackInstallSession(root, plan, lock_stream)+ manifest_path = root / MANIFEST+ journal_path = root / JOURNAL+ try:+ if manifest_path.is_symlink():+ raise QpackInstallError("existing qpack manifest is a symlink")+ if manifest_path.exists():+ if not manifest_path.is_file():+ raise QpackInstallError("existing qpack manifest is not a regular file")+ digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest()+ if digest != plan.manifest_sha256 or not _all_files_exact(root, plan):+ raise QpackInstallError("install root contains a different or incomplete qpack")+ journal_path.unlink(missing_ok=True)+ _fsync_directory(root)+ session.complete = True+ return session++ if journal_path.is_symlink():+ raise QpackInstallError("qpack install journal is a symlink")+ if journal_path.exists():+ _matching_journal(root, plan)+ return session++ for file in plan.files:+ target, partial = _paths(root, file)+ if target.exists() or partial.exists():+ raise QpackInstallError(+ f"unbound qpack artifact exists before journal creation: {file.name}")+ _atomic_json(journal_path, _journal_payload(plan))+ return session+ except Exception:+ session.close()+ raise+++def _validate_plan(plan: QpackInstallPlan) -> None:+ if (not isinstance(plan, QpackInstallPlan)+ or not isinstance(plan.source_id, str) or not plan.source_id.strip()+ or "\x00" in plan.source_id+ or len(plan.source_id.encode("utf-8")) > 4096+ or not isinstance(plan.manifest_sha256, str)+ or len(plan.manifest_sha256) != 64):+ raise QpackInstallError("qpack install plan identity is invalid")+ try:+ int(plan.manifest_sha256, 16)+ except ValueError as error:+ raise QpackInstallError("qpack manifest digest is invalid") from error+ if (not isinstance(plan.files, tuple) or not plan.files+ or len(plan.files) > MAX_FILES):+ raise QpackInstallError("qpack install plan files are invalid")+ for file in plan.files:+ _validate_file(file)+ _validate_namespace(plan.files)+++def _require_session(session: QpackInstallSession) -> None:+ if not isinstance(session, QpackInstallSession) or session.closed:+ raise QpackInstallError("qpack install session is closed or invalid")+ if session.complete:+ raise QpackInstallError("qpack install is already complete")+ _matching_journal(session.root, session.plan)+++def resume_decision(session: QpackInstallSession,+ file: QpackFile) -> ResumeDecision:+ _require_session(session)+ _journal_declares(session.root, file)+ target, partial = _paths(session.root, file, create_parent=True)+ if target.exists():+ if not target.is_file() or target.stat().st_size != file.size:+ raise QpackInstallError(f"committed qpack artifact has wrong size: {file.name}")+ if partial.exists():+ raise QpackInstallError(f"committed and partial qpack artifacts both exist: {file.name}")+ return ResumeDecision("complete", file.size, partial)+ if not partial.exists():+ return ResumeDecision("restart", 0, partial)+ if not partial.is_file():+ raise QpackInstallError(f"qpack partial is not a regular file: {partial}")+ size = partial.stat().st_size+ if size > file.size:+ return ResumeDecision("restart", 0, partial)+ return ResumeDecision("resume", size, partial)+++def validate_delivery(file: QpackFile, requested_offset: int,+ delivered_offset: int, delivered_size: int,+ *, complete: bool = False) -> int:+ """Validate the byte range a transport claims it delivered."""+ _validate_file(file)+ values = (requested_offset, delivered_offset, delivered_size)+ if any(isinstance(value, bool) or not isinstance(value, int) for value in values):+ raise QpackInstallError("delivery offsets and size must be integers")+ if requested_offset < 0 or requested_offset > file.size:+ raise QpackInstallError("requested qpack offset is out of bounds")+ if delivered_offset != requested_offset:+ raise QpackInstallError("transport did not honor the requested qpack range")+ if delivered_size < 0 or delivered_offset + delivered_size > file.size:+ raise QpackInstallError("delivered qpack range is out of bounds")+ end = delivered_offset + delivered_size+ if complete and end != file.size:+ raise QpackInstallError("transport reported a short final qpack delivery")+ return end+++def commit_file(session: QpackInstallSession, file: QpackFile) -> None:+ """Durably publish one exact-size partial artifact."""+ _require_session(session)+ _journal_declares(session.root, file)+ target, partial = _paths(session.root, file, create_parent=True)+ if target.exists():+ if target.is_file() and target.stat().st_size == file.size and not partial.exists():+ return+ raise QpackInstallError(f"cannot commit over existing qpack artifact: {file.name}")+ if not partial.is_file() or partial.stat().st_size != file.size:+ raise QpackInstallError(f"qpack partial is not complete: {file.name}")+ descriptor = os.open(partial, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))+ try:+ os.fsync(descriptor)+ finally:+ os.close(descriptor)+ os.replace(partial, target)+ _fsync_directory(target.parent)+++def publish_manifest(session: QpackInstallSession, raw: bytes) -> None:+ """Publish manifest.json last, after every declared artifact is durable."""+ _require_session(session)+ root = session.root+ plan = session.plan+ parsed = parse_manifest(raw, session.plan.source_id)+ if parsed != session.plan:+ raise QpackInstallError("manifest bytes do not match the qpack install plan")+ if not _all_files_exact(root, plan):+ raise QpackInstallError("cannot publish qpack manifest before all files are complete")+ manifest_path = root / MANIFEST+ if manifest_path.is_symlink():+ raise QpackInstallError("refusing symlink qpack manifest")+ if manifest_path.exists():+ if (not manifest_path.is_file()+ or hashlib.sha256(manifest_path.read_bytes()).hexdigest()+ != plan.manifest_sha256):+ raise QpackInstallError("refusing to replace a different qpack manifest")+ else:+ _atomic_bytes(manifest_path, raw)+ (root / JOURNAL).unlink(missing_ok=True)+ _fsync_directory(root)+ session.complete = True