Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat: add learned partial mirror planner
Bill Herald0324d1081fbc
diff --git a/README.md b/README.mdindex 86aad79c..3b9bbc5e 100644--- a/README.md+++ b/README.md@@ -135,6 +135,26 @@ the hottest ones automatically — colibrì literally gets faster the more you uit. On multi-socket hosts, `COLI_NUMA=1` interleaves the resident weights acrossmemory controllers ([#82](https://github.com/JustVugg/colibri/issues/82)).+For a second drive that cannot hold the whole model, Colibri can rank a partial+mirror from the expert history it already learns. Run a few representative+prompts first so `.coli_usage` reflects the workload, then plan, stage, and+verify the mirror:++```bash+./c/coli mirror plan --model /fast/glm52_i4 --mirror /second/glm52_i4 \+ --budget-gib 200 --reserve-gib 20+./c/coli mirror stage --model /fast/glm52_i4 --mirror /second/glm52_i4 \+ --budget-gib 200 --reserve-gib 20+./c/coli mirror verify --model /fast/glm52_i4 --mirror /second/glm52_i4+```++The planner reads safetensors headers directly, follows split-model directories+from `COLI_MODEL_DIRS`, and prioritizes shards that can serve the hottest routed+experts. Staging never changes the primary model: it copies through temporary+files, preserves the requested free-space reserve, verifies every shard with+SHA-256, never deletes an existing mirror shard, and atomically publishes a+receipt only after the selected mirror is ready.+### Never wait for the disk twiceMisses are expensive, so the engine spends most of its cleverness avoiding anddiff --git a/c/coli b/c/coliindex 5864e689..764dd1b0 100755--- a/c/coli+++ b/c/coli@@ -8,6 +8,7 @@ Run GLM-5.2 (744B) locally on CPU with roughly 15-26 GB of RAM.coli run "prompt" one-shot generationcoli info model, RAM, disk, and configuration statuscoli plan Disk / RAM / VRAM resource plan+ coli mirror Plan, stage, or verify a learned partial mirrorcoli doctor installation and execution-plan diagnosticscoli bench [task...] quality benchmarks (MMLU/HellaSwag/...)coli convert convert GLM-5.2-FP8 to int4, one shard at a time@@ -949,6 +950,27 @@ def cmd_convert(a):print(f" {C.dim}[2/2] int8 MTP head (speculative drafts){C.r}")sys.exit(subprocess.call(mtp_cmd+["--mtp"]))++def cmd_mirror(a):+ if not a.mirror:+ sys.exit("mirror path required: pass --mirror or set COLI_MODEL_MIRROR")+ command = [sys.executable, os.path.join(TOOLS, "mirror_plan.py"), a.action,+ "--model", a.model, "--mirror", a.mirror]+ source_dirs = list(a.source_dir)+ configured = os.environ.get("COLI_MODEL_DIRS", "")+ if configured:+ source_dirs.extend(part.strip() for part in re.split(r"[;,]", configured)+ if part.strip())+ for directory in source_dirs:+ command += ["--source-dir", directory]+ if a.usage:+ command += ["--usage", a.usage]+ if a.action != "verify":+ command += ["--budget-gib", str(a.budget_gib),+ "--reserve-gib", str(a.reserve_gib)]+ return subprocess.call(command)++def main():common=argparse.ArgumentParser(add_help=False)common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile)@@ -969,6 +991,14 @@ def main():sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])pp=sub.add_parser("plan",parents=[common])pp.add_argument("--json",action="store_true")+ pm=sub.add_parser("mirror", parents=[common],+ help="plan, stage, or verify a usage-ranked partial model mirror")+ pm.add_argument("action", choices=("plan", "stage", "verify"))+ pm.add_argument("--mirror", default=os.environ.get("COLI_MODEL_MIRROR"))+ pm.add_argument("--source-dir", action="append", default=[])+ pm.add_argument("--usage")+ pm.add_argument("--budget-gib", type=float, default=0)+ pm.add_argument("--reserve-gib", type=float, default=10)pd=sub.add_parser("doctor",parents=[common])pd.add_argument("--json",action="store_true",help="emit a versioned JSON report")pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")@@ -1012,7 +1042,8 @@ def main():pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0)pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)")a=ap.parse_args()- handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,+ handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"mirror":cmd_mirror,+ "doctor":cmd_doctor,"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench,"convert":cmd_convert,"web":cmd_web}.get(a.cmd)if handler: sys.exit(handler(a) or 0)diff --git a/c/tests/test_mirror_plan.py b/c/tests/test_mirror_plan.pynew file mode 100644index 00000000..4da76712--- /dev/null+++ b/c/tests/test_mirror_plan.py@@ -0,0 +1,148 @@+import json+import struct+import tempfile+import unittest+from pathlib import Path+from unittest import mock++from tools.mirror_plan import (RECEIPT, MirrorError, create_plan, discover_shards,+ stage_mirror, usage_counts, verify_mirror)+++class MirrorPlannerTest(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.root = Path(self.temporary.name)+ self.model = self.root / "model"+ self.mirror = self.root / "mirror"+ self.split = self.root / "split"+ self.model.mkdir()+ self.split.mkdir()+ self.usage = self.model / ".coli_usage"++ def tearDown(self):+ self.temporary.cleanup()++ @staticmethod+ def write_shard(directory, name, tensors):+ offset = 0+ header = {}+ payload = bytearray()+ for tensor, size in tensors:+ header[tensor] = {"dtype": "U8", "shape": [size],+ "data_offsets": [offset, offset + size]}+ payload.extend(bytes([len(header) % 251]) * size)+ offset += size+ encoded = json.dumps(header, separators=(",", ":")).encode()+ path = directory / name+ path.write_bytes(struct.pack("<Q", len(encoded)) + encoded + payload)+ return path++ def test_usage_parser_ignores_malformed_rows_and_accumulates(self):+ self.usage.write_text("0 1 7\ninvalid\n0 1 5\n-1 2 9\n1 2 0\n", encoding="utf-8")+ self.assertEqual(usage_counts(self.usage), {(0, 1): 12})++ def test_plan_activates_hot_gate_shard_before_companion_shard(self):+ gate = self.write_shard(self.model, "hot-gate.safetensors", [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 20),+ ])+ companion = self.write_shard(self.model, "hot-down.safetensors", [+ ("model.layers.0.mlp.experts.0.down_proj.weight", 60),+ ])+ self.write_shard(self.model, "cold-gate.safetensors", [+ ("model.layers.0.mlp.experts.1.gate_proj.weight", 20),+ ])+ self.usage.write_text("0 0 100\n0 1 1\n", encoding="utf-8")+ budget = gate.stat().st_size + companion.stat().st_size++ plan, selected = create_plan(self.model, self.mirror, [], self.usage, budget, 0)++ self.assertTrue(plan["admitted"])+ self.assertEqual([item["name"] for item in selected],+ ["hot-gate.safetensors", "hot-down.safetensors"])+ self.assertEqual(selected[0]["activated_experts"], 1)+ self.assertEqual(selected[1]["activated_experts"], 0)++ def test_plan_requires_learned_usage_instead_of_guessing(self):+ shard = self.write_shard(self.model, "model.safetensors", [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 8),+ ])+ plan, selected = create_plan(+ self.model, self.mirror, [], self.usage, shard.stat().st_size, 0)+ self.assertFalse(plan["admitted"])+ self.assertEqual(plan["reason"], "usage_history_missing")+ self.assertEqual(selected, [])++ def test_split_directories_are_searched_and_basenames_are_deduplicated(self):+ primary = self.write_shard(self.model, "same.safetensors", [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 8),+ ])+ self.write_shard(self.split, "same.safetensors", [+ ("model.layers.0.mlp.experts.1.gate_proj.weight", 16),+ ])+ extra = self.write_shard(self.split, "extra.safetensors", [+ ("model.layers.0.mlp.experts.2.gate_proj.weight", 12),+ ])+ _directories, candidates = discover_shards(self.model, [self.split])+ by_name = {item["name"]: item for item in candidates}+ self.assertEqual(by_name["same.safetensors"]["source"], primary)+ self.assertEqual(by_name["extra.safetensors"]["source"], extra)++ def test_stage_is_atomic_resumable_and_sha256_verified(self):+ source = self.write_shard(self.model, "hot.safetensors", [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 32),+ ])+ self.usage.write_text("0 0 25\n", encoding="utf-8")+ budget = source.stat().st_size++ result = stage_mirror(self.model, self.mirror, [], self.usage, budget, 0)++ self.assertTrue(result["ready"])+ target = self.mirror / source.name+ self.assertEqual(target.read_bytes(), source.read_bytes())+ self.assertTrue((self.mirror / RECEIPT).is_file())+ first_mtime = target.stat().st_mtime_ns++ repeated = stage_mirror(self.model, self.mirror, [], self.usage, budget, 0)+ self.assertTrue(repeated["ready"])+ self.assertEqual(repeated["plan"]["remaining_copy_bytes"], 0)+ self.assertEqual(target.stat().st_mtime_ns, first_mtime)++ content = target.read_bytes()+ target.write_bytes(bytes([content[0] ^ 1]) + content[1:])+ verification = verify_mirror(self.mirror)+ self.assertFalse(verification["ready"])+ self.assertEqual(verification["failures"], ["hot.safetensors (sha256)"])++ def test_reserve_preflight_writes_no_shard_or_receipt(self):+ source = self.write_shard(self.model, "hot.safetensors", [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 32),+ ])+ self.usage.write_text("0 0 25\n", encoding="utf-8")+ disk = mock.Mock(free=source.stat().st_size - 1)+ with mock.patch("tools.mirror_plan.shutil.disk_usage", return_value=disk):+ result = stage_mirror(+ self.model, self.mirror, [], self.usage, source.stat().st_size, 0)+ self.assertFalse(result["admitted"])+ self.assertEqual(result["reason"], "free_space_reserve")+ self.assertFalse((self.mirror / source.name).exists())+ self.assertFalse((self.mirror / RECEIPT).exists())++ def test_verify_rejects_receipt_path_traversal(self):+ self.mirror.mkdir()+ receipt = {"schema": "colibri.partial-mirror.v1", "files": [{+ "name": "../outside.safetensors", "size": 1, "sha256": "0" * 64,+ }]}+ (self.mirror / RECEIPT).write_text(json.dumps(receipt), encoding="utf-8")+ result = verify_mirror(self.mirror)+ self.assertFalse(result["ready"])+ self.assertEqual(result["failures"], ["invalid_receipt_entry"])++ def test_invalid_safetensors_header_fails_closed(self):+ (self.model / "bad.safetensors").write_bytes(struct.pack("<Q", 999) + b"{}")+ with self.assertRaises(MirrorError):+ discover_shards(self.model, [])+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/mirror_plan.py b/c/tools/mirror_plan.pynew file mode 100644index 00000000..0e488979--- /dev/null+++ b/c/tools/mirror_plan.py@@ -0,0 +1,436 @@+#!/usr/bin/env python3+"""Plan, stage, and verify a usage-ranked partial model mirror."""++import argparse+import errno+import hashlib+import json+import math+import os+import re+import shutil+import struct+import sys+from datetime import datetime, timezone+from fractions import Fraction+from pathlib import Path+++SCHEMA = "colibri.partial-mirror.v1"+RECEIPT = ".colibri-mirror.json"+GIB = 1024 ** 3+MAX_HEADER = 256 * 1024 * 1024+COPY_CHUNK = 16 * 1024 * 1024+EXPERT_KEY = re.compile(+ r"(?:^|\.)layers\.(\d+)\.mlp\.experts\.(\d+)\."+ r"(gate_proj|up_proj|down_proj)\.weight(?:\.qs)?$"+)+SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")+++class MirrorError(RuntimeError):+ pass+++def load_json(path):+ try:+ value = json.loads(path.read_text(encoding="utf-8-sig"))+ except (OSError, UnicodeError, json.JSONDecodeError) as error:+ raise MirrorError(f"Cannot read JSON from {path}: {error}") from error+ if not isinstance(value, dict):+ raise MirrorError(f"Expected a JSON object in {path}")+ return value+++def sha256(path):+ digest = hashlib.sha256()+ with path.open("rb") as stream:+ for chunk in iter(lambda: stream.read(COPY_CHUNK), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def usage_counts(path):+ counts = {}+ if not path.is_file():+ return counts+ try:+ lines = path.read_text(encoding="utf-8", errors="replace").splitlines()+ except OSError as error:+ raise MirrorError(f"Cannot read usage history {path}: {error}") from error+ for line in lines:+ fields = line.split()+ if len(fields) != 3:+ continue+ try:+ layer, expert, count = map(int, fields)+ except ValueError:+ continue+ if layer >= 0 and expert >= 0 and count > 0:+ key = (layer, expert)+ counts[key] = counts.get(key, 0) + count+ return counts+++def shard_metadata(path):+ size = path.stat().st_size+ if size < 8:+ raise MirrorError(f"Safetensors shard is truncated: {path}")+ with path.open("rb") as stream:+ prefix = stream.read(8)+ header_size = struct.unpack("<Q", prefix)[0]+ if not 0 < header_size <= MAX_HEADER or header_size > size - 8:+ raise MirrorError(f"Invalid safetensors header length in {path}")+ try:+ header = json.loads(stream.read(header_size))+ except (UnicodeError, json.JSONDecodeError) as error:+ raise MirrorError(f"Invalid safetensors header in {path}: {error}") from error+ if not isinstance(header, dict):+ raise MirrorError(f"Safetensors header is not an object: {path}")++ expert_bytes = {}+ gate_experts = set()+ for name, metadata in header.items():+ if name == "__metadata__" or not isinstance(name, str) or not isinstance(metadata, dict):+ continue+ match = EXPERT_KEY.search(name)+ offsets = metadata.get("data_offsets")+ if match is None or not isinstance(offsets, list) or len(offsets) != 2:+ continue+ start, end = offsets+ if (isinstance(start, bool) or isinstance(end, bool) or+ not isinstance(start, int) or not isinstance(end, int) or+ start < 0 or end < start or end > size - 8 - header_size):+ raise MirrorError(f"Invalid tensor offsets for {name} in {path}")+ expert = (int(match.group(1)), int(match.group(2)))+ expert_bytes[expert] = expert_bytes.get(expert, 0) + end - start+ if match.group(3) == "gate_proj" and name.endswith(".weight"):+ gate_experts.add(expert)+ return size, expert_bytes, gate_experts+++def discover_shards(model, source_dirs):+ directories = []+ for directory in (model, *source_dirs):+ resolved = Path(directory).expanduser().resolve()+ if resolved not in directories:+ directories.append(resolved)+ candidates = []+ seen = set()+ for directory in directories:+ if not directory.is_dir():+ raise MirrorError(f"Model shard directory does not exist: {directory}")+ for path in sorted(directory.glob("*.safetensors")):+ if path.name in seen or not path.is_file():+ continue+ seen.add(path.name)+ size, expert_bytes, gate_experts = shard_metadata(path)+ candidates.append({+ "name": path.name,+ "size": size,+ "source": path,+ "expert_bytes": expert_bytes,+ "gate_experts": gate_experts,+ })+ if not candidates:+ raise MirrorError("No safetensors shards were found in the model source directories")+ return directories, candidates+++def select_shards(candidates, counts, budget):+ # The engine decides an expert's replica from its gate_proj shard. A shard+ # containing only up/down weights cannot serve that expert from the mirror+ # until its gate shard is present, so score companions only after activation.+ # The byte-weighted usage score estimates I/O moved off the primary drive.+ selected = []+ activated = set()+ remaining = list(candidates)+ used = 0+ while True:+ choices = []+ for item in remaining:+ if used + item["size"] > budget:+ continue+ eligible = activated | item["gate_experts"]+ benefit = sum(counts.get(expert, 0) * byte_count+ for expert, byte_count in item["expert_bytes"].items()+ if expert in eligible)+ if benefit:+ choices.append((item, benefit))+ if not choices:+ break+ item, benefit = min(+ choices,+ key=lambda choice: (+ -Fraction(choice[1], choice[0]["size"]),+ -choice[1],+ choice[0]["size"],+ choice[0]["name"],+ ),+ )+ newly_activated = item["gate_experts"] - activated+ selected.append({**item, "weighted_expert_bytes": benefit,+ "activated_experts": len(newly_activated)})+ activated.update(item["gate_experts"])+ remaining.remove(item)+ used += item["size"]+ return selected+++def existing_parent(path):+ candidate = path+ while not candidate.exists() and candidate != candidate.parent:+ candidate = candidate.parent+ return candidate+++def public_file(item):+ return {+ "name": item["name"],+ "size": item["size"],+ "weighted_expert_bytes": item["weighted_expert_bytes"],+ "activated_experts": item["activated_experts"],+ }+++def create_plan(model, mirror, source_dirs, usage, budget, reserve):+ model = Path(model).expanduser().resolve()+ mirror = Path(mirror).expanduser().resolve()+ usage = Path(usage).expanduser().resolve()+ if model == mirror:+ raise MirrorError("Mirror directory must differ from the primary model directory")+ if budget <= 0 or reserve < 0:+ raise MirrorError("Mirror budget must be positive and reserve cannot be negative")+ directories, candidates = discover_shards(model, source_dirs)+ counts = usage_counts(usage)+ selected = select_shards(candidates, counts, budget)+ selected_bytes = sum(item["size"] for item in selected)+ existing_bytes = sum(item["size"] for item in selected+ if not (mirror / item["name"]).is_symlink()+ and (mirror / item["name"]).is_file()+ and (mirror / item["name"]).stat().st_size == item["size"])+ remaining = selected_bytes - existing_bytes+ free = shutil.disk_usage(existing_parent(mirror)).free+ warnings = []+ mirror_device = existing_parent(mirror).stat().st_dev+ if any(directory.stat().st_dev == mirror_device for directory in directories):+ warnings.append("mirror path shares a filesystem device with a source directory")+ if not counts:+ reason = "usage_history_missing"+ elif not selected:+ reason = "no_usage_ranked_shards_fit_the_budget"+ elif free - remaining < reserve:+ reason = "free_space_reserve"+ else:+ reason = None+ payload = {+ "schema": SCHEMA,+ "model_root": str(model),+ "source_dirs": [str(path) for path in directories],+ "mirror_root": str(mirror),+ "usage_file": str(usage),+ "usage_history_present": bool(counts),+ "usage_selections": sum(counts.values()),+ "budget_bytes": budget,+ "reserve_bytes": reserve,+ "free_bytes": free,+ "selected_bytes": selected_bytes,+ "remaining_copy_bytes": remaining,+ "projected_free_bytes": free - remaining,+ "admitted": reason is None,+ "reason": reason,+ "warnings": warnings,+ "files": [public_file(item) for item in selected],+ }+ return payload, selected+++def fsync_directory(path):+ if os.name == "nt":+ return+ descriptor = os.open(path, os.O_RDONLY)+ try:+ try:+ os.fsync(descriptor)+ except OSError as error:+ if error.errno not in (errno.EBADF, errno.EINVAL, errno.ENOTSUP):+ raise+ finally:+ os.close(descriptor)+++def atomic_json(path, payload):+ temporary = path.with_name(f".{path.name}.part")+ try:+ with temporary.open("w", encoding="utf-8", newline="\n") as stream:+ json.dump(payload, stream, indent=2)+ stream.write("\n")+ stream.flush()+ os.fsync(stream.fileno())+ os.replace(temporary, path)+ fsync_directory(path.parent)+ except Exception:+ temporary.unlink(missing_ok=True)+ raise+++def copy_verified(source, target, expected_digest):+ temporary = target.with_name(f".{target.name}.part")+ temporary.unlink(missing_ok=True)+ digest = hashlib.sha256()+ try:+ with source.open("rb") as input_stream, temporary.open("xb") as output_stream:+ for chunk in iter(lambda: input_stream.read(COPY_CHUNK), b""):+ output_stream.write(chunk)+ digest.update(chunk)+ output_stream.flush()+ os.fsync(output_stream.fileno())+ if digest.hexdigest() != expected_digest or sha256(temporary) != expected_digest:+ raise MirrorError(f"Copied shard failed SHA-256 verification: {source.name}")+ os.replace(temporary, target)+ fsync_directory(target.parent)+ except Exception:+ temporary.unlink(missing_ok=True)+ raise+++def stage_mirror(model, mirror, source_dirs, usage, budget, reserve):+ plan, selected = create_plan(model, mirror, source_dirs, usage, budget, reserve)+ if not selected or not plan["usage_history_present"]:+ return plan+ mirror = Path(mirror).expanduser().resolve()+ mirror.mkdir(parents=True, exist_ok=True)+ remaining = 0+ for item in selected:+ item["sha256"] = sha256(item["source"])+ target = mirror / item["name"]+ item["already_verified"] = (+ not target.is_symlink() and target.is_file()+ and target.stat().st_size == item["size"]+ and sha256(target) == item["sha256"]+ )+ if not item["already_verified"]:+ remaining += item["size"]+ free = shutil.disk_usage(mirror).free+ plan["free_bytes"] = free+ plan["remaining_copy_bytes"] = remaining+ plan["projected_free_bytes"] = free - remaining+ if free - remaining < reserve:+ plan["admitted"] = False+ plan["reason"] = "free_space_reserve"+ return plan+ plan["admitted"] = True+ plan["reason"] = None+ for item in selected:+ if not item["already_verified"]:+ copy_verified(item["source"], mirror / item["name"], item["sha256"])+ receipt = {+ "schema": SCHEMA,+ "created_at": datetime.now(timezone.utc).isoformat(),+ "model_root": plan["model_root"],+ "usage_file": plan["usage_file"],+ "budget_bytes": budget,+ "reserve_bytes": reserve,+ "files": [{**public_file(item), "sha256": item["sha256"]} for item in selected],+ }+ atomic_json(mirror / RECEIPT, receipt)+ result = verify_mirror(mirror)+ result["plan"] = plan+ return result+++def safe_receipt_name(value):+ return (isinstance(value, str) and value and "/" not in value and "\\" not in value+ and Path(value).name == value and value.endswith(".safetensors"))+++def verify_mirror(mirror):+ mirror = Path(mirror).expanduser().resolve()+ receipt_path = mirror / RECEIPT+ if not receipt_path.is_file():+ return {"schema": SCHEMA, "ready": False, "mirror_root": str(mirror),+ "reason": "receipt_missing", "failures": []}+ receipt = load_json(receipt_path)+ failures = []+ files = receipt.get("files")+ if receipt.get("schema") != SCHEMA or not isinstance(files, list) or not files:+ return {"schema": SCHEMA, "ready": False, "mirror_root": str(mirror),+ "reason": "invalid_receipt", "failures": []}+ mirrored_bytes = 0+ names = set()+ for item in files:+ if not isinstance(item, dict) or not safe_receipt_name(item.get("name")):+ failures.append("invalid_receipt_entry")+ continue+ name = item["name"]+ if name in names:+ failures.append(name + " (duplicate)")+ continue+ names.add(name)+ size = item.get("size")+ digest = item.get("sha256")+ target = mirror / name+ if (isinstance(size, bool) or not isinstance(size, int) or size <= 0 or+ not isinstance(digest, str) or SHA256_HEX.fullmatch(digest) is None):+ failures.append(name + " (metadata)")+ continue+ mirrored_bytes += size+ if target.is_symlink() or not target.is_file() or target.stat().st_size != size:+ failures.append(name + " (size)")+ elif sha256(target) != digest:+ failures.append(name + " (sha256)")+ return {+ "schema": SCHEMA,+ "ready": not failures,+ "mirror_root": str(mirror),+ "file_count": len(files),+ "mirrored_bytes": mirrored_bytes,+ "failures": failures,+ }+++def parse_args(argv=None):+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("action", choices=("plan", "stage", "verify"))+ parser.add_argument("--model", required=True, help="primary Colibri model directory")+ parser.add_argument("--mirror", required=True, help="partial mirror directory")+ parser.add_argument("--source-dir", action="append", default=[],+ help="additional split-model shard directory (repeatable)")+ parser.add_argument("--usage", help="usage history (default: MODEL/.coli_usage)")+ parser.add_argument("--budget-gib", type=float, default=0,+ help="maximum total mirror size; required for plan/stage")+ parser.add_argument("--reserve-gib", type=float, default=10,+ help="free space to preserve on the mirror filesystem")+ return parser.parse_args(argv)+++def main(argv=None):+ args = parse_args(argv)+ model = Path(args.model)+ mirror = Path(args.mirror)+ if args.action == "verify":+ payload = verify_mirror(mirror)+ print(json.dumps(payload, indent=2))+ return 0 if payload["ready"] else 4+ if (not math.isfinite(args.budget_gib) or args.budget_gib <= 0 or+ not math.isfinite(args.reserve_gib) or args.reserve_gib < 0):+ raise MirrorError("--budget-gib must be positive and --reserve-gib cannot be negative")+ usage = Path(args.usage) if args.usage else model / ".coli_usage"+ budget = int(args.budget_gib * GIB)+ reserve = int(args.reserve_gib * GIB)+ if args.action == "plan":+ payload, _selected = create_plan(model, mirror, args.source_dir, usage, budget, reserve)+ else:+ payload = stage_mirror(model, mirror, args.source_dir, usage, budget, reserve)+ print(json.dumps(payload, indent=2))+ ready = payload.get("ready", payload.get("admitted", False))+ return 0 if ready else 2+++if __name__ == "__main__":+ try:+ raise SystemExit(main())+ except (MirrorError, OSError) as error:+ print(json.dumps({"schema": SCHEMA, "ready": False, "error": str(error)}),+ file=sys.stderr)+ raise SystemExit(1)