Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat: add learned partial mirror planner

Bill Herald0324d1081fbc
diff --git a/README.md b/README.md
index 86aad79c..3b9bbc5e 100644
--- a/README.md
+++ b/README.md
@@ -135,6 +135,26 @@ the hottest ones automatically — colibrì literally gets faster the more you u
it. On multi-socket hosts, `COLI_NUMA=1` interleaves the resident weights across
memory controllers ([#82](https://github.com/JustVugg/colibri/issues/82)).
+For a second drive that cannot hold the whole model, Colibri can rank a partial
+mirror from the expert history it already learns. Run a few representative
+prompts first so `.coli_usage` reflects the workload, then plan, stage, and
+verify the mirror:
+
+```bash
+./c/coli mirror plan --model /fast/glm52_i4 --mirror /second/glm52_i4 \
+ --budget-gib 200 --reserve-gib 20
+./c/coli mirror stage --model /fast/glm52_i4 --mirror /second/glm52_i4 \
+ --budget-gib 200 --reserve-gib 20
+./c/coli mirror verify --model /fast/glm52_i4 --mirror /second/glm52_i4
+```
+
+The planner reads safetensors headers directly, follows split-model directories
+from `COLI_MODEL_DIRS`, and prioritizes shards that can serve the hottest routed
+experts. Staging never changes the primary model: it copies through temporary
+files, preserves the requested free-space reserve, verifies every shard with
+SHA-256, never deletes an existing mirror shard, and atomically publishes a
+receipt only after the selected mirror is ready.
+
### Never wait for the disk twice
Misses are expensive, so the engine spends most of its cleverness avoiding and
diff --git a/c/coli b/c/coli
index 5864e689..764dd1b0 100755
--- a/c/coli
+++ b/c/coli
@@ -8,6 +8,7 @@ Run GLM-5.2 (744B) locally on CPU with roughly 15-26 GB of RAM.
coli run "prompt" one-shot generation
coli info model, RAM, disk, and configuration status
coli plan Disk / RAM / VRAM resource plan
+ coli mirror Plan, stage, or verify a learned partial mirror
coli doctor installation and execution-plan diagnostics
coli bench [task...] quality benchmarks (MMLU/HellaSwag/...)
coli convert convert GLM-5.2-FP8 to int4, one shard at a time
@@ -949,6 +950,27 @@ def cmd_convert(a):
print(f" {C.dim}[2/2] int8 MTP head (speculative drafts){C.r}")
sys.exit(subprocess.call(mtp_cmd+["--mtp"]))
+
+def cmd_mirror(a):
+ if not a.mirror:
+ sys.exit("mirror path required: pass --mirror or set COLI_MODEL_MIRROR")
+ command = [sys.executable, os.path.join(TOOLS, "mirror_plan.py"), a.action,
+ "--model", a.model, "--mirror", a.mirror]
+ source_dirs = list(a.source_dir)
+ configured = os.environ.get("COLI_MODEL_DIRS", "")
+ if configured:
+ source_dirs.extend(part.strip() for part in re.split(r"[;,]", configured)
+ if part.strip())
+ for directory in source_dirs:
+ command += ["--source-dir", directory]
+ if a.usage:
+ command += ["--usage", a.usage]
+ if a.action != "verify":
+ command += ["--budget-gib", str(a.budget_gib),
+ "--reserve-gib", str(a.reserve_gib)]
+ return subprocess.call(command)
+
+
def main():
common=argparse.ArgumentParser(add_help=False)
common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile)
@@ -969,6 +991,14 @@ def main():
sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])
pp=sub.add_parser("plan",parents=[common])
pp.add_argument("--json",action="store_true")
+ pm=sub.add_parser("mirror", parents=[common],
+ help="plan, stage, or verify a usage-ranked partial model mirror")
+ pm.add_argument("action", choices=("plan", "stage", "verify"))
+ pm.add_argument("--mirror", default=os.environ.get("COLI_MODEL_MIRROR"))
+ pm.add_argument("--source-dir", action="append", default=[])
+ pm.add_argument("--usage")
+ pm.add_argument("--budget-gib", type=float, default=0)
+ pm.add_argument("--reserve-gib", type=float, default=10)
pd=sub.add_parser("doctor",parents=[common])
pd.add_argument("--json",action="store_true",help="emit a versioned JSON report")
pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")
@@ -1012,7 +1042,8 @@ def main():
pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0)
pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)")
a=ap.parse_args()
- handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,
+ handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"mirror":cmd_mirror,
+ "doctor":cmd_doctor,
"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench,
"convert":cmd_convert,"web":cmd_web}.get(a.cmd)
if handler: sys.exit(handler(a) or 0)
diff --git a/c/tests/test_mirror_plan.py b/c/tests/test_mirror_plan.py
new file mode 100644
index 00000000..4da76712
--- /dev/null
+++ b/c/tests/test_mirror_plan.py
@@ -0,0 +1,148 @@
+import json
+import struct
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from tools.mirror_plan import (RECEIPT, MirrorError, create_plan, discover_shards,
+ stage_mirror, usage_counts, verify_mirror)
+
+
+class MirrorPlannerTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.root = Path(self.temporary.name)
+ self.model = self.root / "model"
+ self.mirror = self.root / "mirror"
+ self.split = self.root / "split"
+ self.model.mkdir()
+ self.split.mkdir()
+ self.usage = self.model / ".coli_usage"
+
+ def tearDown(self):
+ self.temporary.cleanup()
+
+ @staticmethod
+ def write_shard(directory, name, tensors):
+ offset = 0
+ header = {}
+ payload = bytearray()
+ for tensor, size in tensors:
+ header[tensor] = {"dtype": "U8", "shape": [size],
+ "data_offsets": [offset, offset + size]}
+ payload.extend(bytes([len(header) % 251]) * size)
+ offset += size
+ encoded = json.dumps(header, separators=(",", ":")).encode()
+ path = directory / name
+ path.write_bytes(struct.pack("<Q", len(encoded)) + encoded + payload)
+ return path
+
+ def test_usage_parser_ignores_malformed_rows_and_accumulates(self):
+ self.usage.write_text("0 1 7\ninvalid\n0 1 5\n-1 2 9\n1 2 0\n", encoding="utf-8")
+ self.assertEqual(usage_counts(self.usage), {(0, 1): 12})
+
+ def test_plan_activates_hot_gate_shard_before_companion_shard(self):
+ gate = self.write_shard(self.model, "hot-gate.safetensors", [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 20),
+ ])
+ companion = self.write_shard(self.model, "hot-down.safetensors", [
+ ("model.layers.0.mlp.experts.0.down_proj.weight", 60),
+ ])
+ self.write_shard(self.model, "cold-gate.safetensors", [
+ ("model.layers.0.mlp.experts.1.gate_proj.weight", 20),
+ ])
+ self.usage.write_text("0 0 100\n0 1 1\n", encoding="utf-8")
+ budget = gate.stat().st_size + companion.stat().st_size
+
+ plan, selected = create_plan(self.model, self.mirror, [], self.usage, budget, 0)
+
+ self.assertTrue(plan["admitted"])
+ self.assertEqual([item["name"] for item in selected],
+ ["hot-gate.safetensors", "hot-down.safetensors"])
+ self.assertEqual(selected[0]["activated_experts"], 1)
+ self.assertEqual(selected[1]["activated_experts"], 0)
+
+ def test_plan_requires_learned_usage_instead_of_guessing(self):
+ shard = self.write_shard(self.model, "model.safetensors", [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 8),
+ ])
+ plan, selected = create_plan(
+ self.model, self.mirror, [], self.usage, shard.stat().st_size, 0)
+ self.assertFalse(plan["admitted"])
+ self.assertEqual(plan["reason"], "usage_history_missing")
+ self.assertEqual(selected, [])
+
+ def test_split_directories_are_searched_and_basenames_are_deduplicated(self):
+ primary = self.write_shard(self.model, "same.safetensors", [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 8),
+ ])
+ self.write_shard(self.split, "same.safetensors", [
+ ("model.layers.0.mlp.experts.1.gate_proj.weight", 16),
+ ])
+ extra = self.write_shard(self.split, "extra.safetensors", [
+ ("model.layers.0.mlp.experts.2.gate_proj.weight", 12),
+ ])
+ _directories, candidates = discover_shards(self.model, [self.split])
+ by_name = {item["name"]: item for item in candidates}
+ self.assertEqual(by_name["same.safetensors"]["source"], primary)
+ self.assertEqual(by_name["extra.safetensors"]["source"], extra)
+
+ def test_stage_is_atomic_resumable_and_sha256_verified(self):
+ source = self.write_shard(self.model, "hot.safetensors", [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 32),
+ ])
+ self.usage.write_text("0 0 25\n", encoding="utf-8")
+ budget = source.stat().st_size
+
+ result = stage_mirror(self.model, self.mirror, [], self.usage, budget, 0)
+
+ self.assertTrue(result["ready"])
+ target = self.mirror / source.name
+ self.assertEqual(target.read_bytes(), source.read_bytes())
+ self.assertTrue((self.mirror / RECEIPT).is_file())
+ first_mtime = target.stat().st_mtime_ns
+
+ repeated = stage_mirror(self.model, self.mirror, [], self.usage, budget, 0)
+ self.assertTrue(repeated["ready"])
+ self.assertEqual(repeated["plan"]["remaining_copy_bytes"], 0)
+ self.assertEqual(target.stat().st_mtime_ns, first_mtime)
+
+ content = target.read_bytes()
+ target.write_bytes(bytes([content[0] ^ 1]) + content[1:])
+ verification = verify_mirror(self.mirror)
+ self.assertFalse(verification["ready"])
+ self.assertEqual(verification["failures"], ["hot.safetensors (sha256)"])
+
+ def test_reserve_preflight_writes_no_shard_or_receipt(self):
+ source = self.write_shard(self.model, "hot.safetensors", [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", 32),
+ ])
+ self.usage.write_text("0 0 25\n", encoding="utf-8")
+ disk = mock.Mock(free=source.stat().st_size - 1)
+ with mock.patch("tools.mirror_plan.shutil.disk_usage", return_value=disk):
+ result = stage_mirror(
+ self.model, self.mirror, [], self.usage, source.stat().st_size, 0)
+ self.assertFalse(result["admitted"])
+ self.assertEqual(result["reason"], "free_space_reserve")
+ self.assertFalse((self.mirror / source.name).exists())
+ self.assertFalse((self.mirror / RECEIPT).exists())
+
+ def test_verify_rejects_receipt_path_traversal(self):
+ self.mirror.mkdir()
+ receipt = {"schema": "colibri.partial-mirror.v1", "files": [{
+ "name": "../outside.safetensors", "size": 1, "sha256": "0" * 64,
+ }]}
+ (self.mirror / RECEIPT).write_text(json.dumps(receipt), encoding="utf-8")
+ result = verify_mirror(self.mirror)
+ self.assertFalse(result["ready"])
+ self.assertEqual(result["failures"], ["invalid_receipt_entry"])
+
+ def test_invalid_safetensors_header_fails_closed(self):
+ (self.model / "bad.safetensors").write_bytes(struct.pack("<Q", 999) + b"{}")
+ with self.assertRaises(MirrorError):
+ discover_shards(self.model, [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/mirror_plan.py b/c/tools/mirror_plan.py
new file mode 100644
index 00000000..0e488979
--- /dev/null
+++ b/c/tools/mirror_plan.py
@@ -0,0 +1,436 @@
+#!/usr/bin/env python3
+"""Plan, stage, and verify a usage-ranked partial model mirror."""
+
+import argparse
+import errno
+import hashlib
+import json
+import math
+import os
+import re
+import shutil
+import struct
+import sys
+from datetime import datetime, timezone
+from fractions import Fraction
+from pathlib import Path
+
+
+SCHEMA = "colibri.partial-mirror.v1"
+RECEIPT = ".colibri-mirror.json"
+GIB = 1024 ** 3
+MAX_HEADER = 256 * 1024 * 1024
+COPY_CHUNK = 16 * 1024 * 1024
+EXPERT_KEY = re.compile(
+ r"(?:^|\.)layers\.(\d+)\.mlp\.experts\.(\d+)\."
+ r"(gate_proj|up_proj|down_proj)\.weight(?:\.qs)?$"
+)
+SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
+
+
+class MirrorError(RuntimeError):
+ pass
+
+
+def load_json(path):
+ try:
+ value = json.loads(path.read_text(encoding="utf-8-sig"))
+ except (OSError, UnicodeError, json.JSONDecodeError) as error:
+ raise MirrorError(f"Cannot read JSON from {path}: {error}") from error
+ if not isinstance(value, dict):
+ raise MirrorError(f"Expected a JSON object in {path}")
+ return value
+
+
+def sha256(path):
+ digest = hashlib.sha256()
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(COPY_CHUNK), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def usage_counts(path):
+ counts = {}
+ if not path.is_file():
+ return counts
+ try:
+ lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+ except OSError as error:
+ raise MirrorError(f"Cannot read usage history {path}: {error}") from error
+ for line in lines:
+ fields = line.split()
+ if len(fields) != 3:
+ continue
+ try:
+ layer, expert, count = map(int, fields)
+ except ValueError:
+ continue
+ if layer >= 0 and expert >= 0 and count > 0:
+ key = (layer, expert)
+ counts[key] = counts.get(key, 0) + count
+ return counts
+
+
+def shard_metadata(path):
+ size = path.stat().st_size
+ if size < 8:
+ raise MirrorError(f"Safetensors shard is truncated: {path}")
+ with path.open("rb") as stream:
+ prefix = stream.read(8)
+ header_size = struct.unpack("<Q", prefix)[0]
+ if not 0 < header_size <= MAX_HEADER or header_size > size - 8:
+ raise MirrorError(f"Invalid safetensors header length in {path}")
+ try:
+ header = json.loads(stream.read(header_size))
+ except (UnicodeError, json.JSONDecodeError) as error:
+ raise MirrorError(f"Invalid safetensors header in {path}: {error}") from error
+ if not isinstance(header, dict):
+ raise MirrorError(f"Safetensors header is not an object: {path}")
+
+ expert_bytes = {}
+ gate_experts = set()
+ for name, metadata in header.items():
+ if name == "__metadata__" or not isinstance(name, str) or not isinstance(metadata, dict):
+ continue
+ match = EXPERT_KEY.search(name)
+ offsets = metadata.get("data_offsets")
+ if match is None or not isinstance(offsets, list) or len(offsets) != 2:
+ continue
+ start, end = offsets
+ if (isinstance(start, bool) or isinstance(end, bool) or
+ not isinstance(start, int) or not isinstance(end, int) or
+ start < 0 or end < start or end > size - 8 - header_size):
+ raise MirrorError(f"Invalid tensor offsets for {name} in {path}")
+ expert = (int(match.group(1)), int(match.group(2)))
+ expert_bytes[expert] = expert_bytes.get(expert, 0) + end - start
+ if match.group(3) == "gate_proj" and name.endswith(".weight"):
+ gate_experts.add(expert)
+ return size, expert_bytes, gate_experts
+
+
+def discover_shards(model, source_dirs):
+ directories = []
+ for directory in (model, *source_dirs):
+ resolved = Path(directory).expanduser().resolve()
+ if resolved not in directories:
+ directories.append(resolved)
+ candidates = []
+ seen = set()
+ for directory in directories:
+ if not directory.is_dir():
+ raise MirrorError(f"Model shard directory does not exist: {directory}")
+ for path in sorted(directory.glob("*.safetensors")):
+ if path.name in seen or not path.is_file():
+ continue
+ seen.add(path.name)
+ size, expert_bytes, gate_experts = shard_metadata(path)
+ candidates.append({
+ "name": path.name,
+ "size": size,
+ "source": path,
+ "expert_bytes": expert_bytes,
+ "gate_experts": gate_experts,
+ })
+ if not candidates:
+ raise MirrorError("No safetensors shards were found in the model source directories")
+ return directories, candidates
+
+
+def select_shards(candidates, counts, budget):
+ # The engine decides an expert's replica from its gate_proj shard. A shard
+ # containing only up/down weights cannot serve that expert from the mirror
+ # until its gate shard is present, so score companions only after activation.
+ # The byte-weighted usage score estimates I/O moved off the primary drive.
+ selected = []
+ activated = set()
+ remaining = list(candidates)
+ used = 0
+ while True:
+ choices = []
+ for item in remaining:
+ if used + item["size"] > budget:
+ continue
+ eligible = activated | item["gate_experts"]
+ benefit = sum(counts.get(expert, 0) * byte_count
+ for expert, byte_count in item["expert_bytes"].items()
+ if expert in eligible)
+ if benefit:
+ choices.append((item, benefit))
+ if not choices:
+ break
+ item, benefit = min(
+ choices,
+ key=lambda choice: (
+ -Fraction(choice[1], choice[0]["size"]),
+ -choice[1],
+ choice[0]["size"],
+ choice[0]["name"],
+ ),
+ )
+ newly_activated = item["gate_experts"] - activated
+ selected.append({**item, "weighted_expert_bytes": benefit,
+ "activated_experts": len(newly_activated)})
+ activated.update(item["gate_experts"])
+ remaining.remove(item)
+ used += item["size"]
+ return selected
+
+
+def existing_parent(path):
+ candidate = path
+ while not candidate.exists() and candidate != candidate.parent:
+ candidate = candidate.parent
+ return candidate
+
+
+def public_file(item):
+ return {
+ "name": item["name"],
+ "size": item["size"],
+ "weighted_expert_bytes": item["weighted_expert_bytes"],
+ "activated_experts": item["activated_experts"],
+ }
+
+
+def create_plan(model, mirror, source_dirs, usage, budget, reserve):
+ model = Path(model).expanduser().resolve()
+ mirror = Path(mirror).expanduser().resolve()
+ usage = Path(usage).expanduser().resolve()
+ if model == mirror:
+ raise MirrorError("Mirror directory must differ from the primary model directory")
+ if budget <= 0 or reserve < 0:
+ raise MirrorError("Mirror budget must be positive and reserve cannot be negative")
+ directories, candidates = discover_shards(model, source_dirs)
+ counts = usage_counts(usage)
+ selected = select_shards(candidates, counts, budget)
+ selected_bytes = sum(item["size"] for item in selected)
+ existing_bytes = sum(item["size"] for item in selected
+ if not (mirror / item["name"]).is_symlink()
+ and (mirror / item["name"]).is_file()
+ and (mirror / item["name"]).stat().st_size == item["size"])
+ remaining = selected_bytes - existing_bytes
+ free = shutil.disk_usage(existing_parent(mirror)).free
+ warnings = []
+ mirror_device = existing_parent(mirror).stat().st_dev
+ if any(directory.stat().st_dev == mirror_device for directory in directories):
+ warnings.append("mirror path shares a filesystem device with a source directory")
+ if not counts:
+ reason = "usage_history_missing"
+ elif not selected:
+ reason = "no_usage_ranked_shards_fit_the_budget"
+ elif free - remaining < reserve:
+ reason = "free_space_reserve"
+ else:
+ reason = None
+ payload = {
+ "schema": SCHEMA,
+ "model_root": str(model),
+ "source_dirs": [str(path) for path in directories],
+ "mirror_root": str(mirror),
+ "usage_file": str(usage),
+ "usage_history_present": bool(counts),
+ "usage_selections": sum(counts.values()),
+ "budget_bytes": budget,
+ "reserve_bytes": reserve,
+ "free_bytes": free,
+ "selected_bytes": selected_bytes,
+ "remaining_copy_bytes": remaining,
+ "projected_free_bytes": free - remaining,
+ "admitted": reason is None,
+ "reason": reason,
+ "warnings": warnings,
+ "files": [public_file(item) for item in selected],
+ }
+ return payload, selected
+
+
+def fsync_directory(path):
+ if os.name == "nt":
+ return
+ descriptor = os.open(path, os.O_RDONLY)
+ try:
+ try:
+ os.fsync(descriptor)
+ except OSError as error:
+ if error.errno not in (errno.EBADF, errno.EINVAL, errno.ENOTSUP):
+ raise
+ finally:
+ os.close(descriptor)
+
+
+def atomic_json(path, payload):
+ temporary = path.with_name(f".{path.name}.part")
+ try:
+ with temporary.open("w", encoding="utf-8", newline="\n") as stream:
+ json.dump(payload, stream, indent=2)
+ stream.write("\n")
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(temporary, path)
+ fsync_directory(path.parent)
+ except Exception:
+ temporary.unlink(missing_ok=True)
+ raise
+
+
+def copy_verified(source, target, expected_digest):
+ temporary = target.with_name(f".{target.name}.part")
+ temporary.unlink(missing_ok=True)
+ digest = hashlib.sha256()
+ try:
+ with source.open("rb") as input_stream, temporary.open("xb") as output_stream:
+ for chunk in iter(lambda: input_stream.read(COPY_CHUNK), b""):
+ output_stream.write(chunk)
+ digest.update(chunk)
+ output_stream.flush()
+ os.fsync(output_stream.fileno())
+ if digest.hexdigest() != expected_digest or sha256(temporary) != expected_digest:
+ raise MirrorError(f"Copied shard failed SHA-256 verification: {source.name}")
+ os.replace(temporary, target)
+ fsync_directory(target.parent)
+ except Exception:
+ temporary.unlink(missing_ok=True)
+ raise
+
+
+def stage_mirror(model, mirror, source_dirs, usage, budget, reserve):
+ plan, selected = create_plan(model, mirror, source_dirs, usage, budget, reserve)
+ if not selected or not plan["usage_history_present"]:
+ return plan
+ mirror = Path(mirror).expanduser().resolve()
+ mirror.mkdir(parents=True, exist_ok=True)
+ remaining = 0
+ for item in selected:
+ item["sha256"] = sha256(item["source"])
+ target = mirror / item["name"]
+ item["already_verified"] = (
+ not target.is_symlink() and target.is_file()
+ and target.stat().st_size == item["size"]
+ and sha256(target) == item["sha256"]
+ )
+ if not item["already_verified"]:
+ remaining += item["size"]
+ free = shutil.disk_usage(mirror).free
+ plan["free_bytes"] = free
+ plan["remaining_copy_bytes"] = remaining
+ plan["projected_free_bytes"] = free - remaining
+ if free - remaining < reserve:
+ plan["admitted"] = False
+ plan["reason"] = "free_space_reserve"
+ return plan
+ plan["admitted"] = True
+ plan["reason"] = None
+ for item in selected:
+ if not item["already_verified"]:
+ copy_verified(item["source"], mirror / item["name"], item["sha256"])
+ receipt = {
+ "schema": SCHEMA,
+ "created_at": datetime.now(timezone.utc).isoformat(),
+ "model_root": plan["model_root"],
+ "usage_file": plan["usage_file"],
+ "budget_bytes": budget,
+ "reserve_bytes": reserve,
+ "files": [{**public_file(item), "sha256": item["sha256"]} for item in selected],
+ }
+ atomic_json(mirror / RECEIPT, receipt)
+ result = verify_mirror(mirror)
+ result["plan"] = plan
+ return result
+
+
+def safe_receipt_name(value):
+ return (isinstance(value, str) and value and "/" not in value and "\\" not in value
+ and Path(value).name == value and value.endswith(".safetensors"))
+
+
+def verify_mirror(mirror):
+ mirror = Path(mirror).expanduser().resolve()
+ receipt_path = mirror / RECEIPT
+ if not receipt_path.is_file():
+ return {"schema": SCHEMA, "ready": False, "mirror_root": str(mirror),
+ "reason": "receipt_missing", "failures": []}
+ receipt = load_json(receipt_path)
+ failures = []
+ files = receipt.get("files")
+ if receipt.get("schema") != SCHEMA or not isinstance(files, list) or not files:
+ return {"schema": SCHEMA, "ready": False, "mirror_root": str(mirror),
+ "reason": "invalid_receipt", "failures": []}
+ mirrored_bytes = 0
+ names = set()
+ for item in files:
+ if not isinstance(item, dict) or not safe_receipt_name(item.get("name")):
+ failures.append("invalid_receipt_entry")
+ continue
+ name = item["name"]
+ if name in names:
+ failures.append(name + " (duplicate)")
+ continue
+ names.add(name)
+ size = item.get("size")
+ digest = item.get("sha256")
+ target = mirror / name
+ if (isinstance(size, bool) or not isinstance(size, int) or size <= 0 or
+ not isinstance(digest, str) or SHA256_HEX.fullmatch(digest) is None):
+ failures.append(name + " (metadata)")
+ continue
+ mirrored_bytes += size
+ if target.is_symlink() or not target.is_file() or target.stat().st_size != size:
+ failures.append(name + " (size)")
+ elif sha256(target) != digest:
+ failures.append(name + " (sha256)")
+ return {
+ "schema": SCHEMA,
+ "ready": not failures,
+ "mirror_root": str(mirror),
+ "file_count": len(files),
+ "mirrored_bytes": mirrored_bytes,
+ "failures": failures,
+ }
+
+
+def parse_args(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("action", choices=("plan", "stage", "verify"))
+ parser.add_argument("--model", required=True, help="primary Colibri model directory")
+ parser.add_argument("--mirror", required=True, help="partial mirror directory")
+ parser.add_argument("--source-dir", action="append", default=[],
+ help="additional split-model shard directory (repeatable)")
+ parser.add_argument("--usage", help="usage history (default: MODEL/.coli_usage)")
+ parser.add_argument("--budget-gib", type=float, default=0,
+ help="maximum total mirror size; required for plan/stage")
+ parser.add_argument("--reserve-gib", type=float, default=10,
+ help="free space to preserve on the mirror filesystem")
+ return parser.parse_args(argv)
+
+
+def main(argv=None):
+ args = parse_args(argv)
+ model = Path(args.model)
+ mirror = Path(args.mirror)
+ if args.action == "verify":
+ payload = verify_mirror(mirror)
+ print(json.dumps(payload, indent=2))
+ return 0 if payload["ready"] else 4
+ if (not math.isfinite(args.budget_gib) or args.budget_gib <= 0 or
+ not math.isfinite(args.reserve_gib) or args.reserve_gib < 0):
+ raise MirrorError("--budget-gib must be positive and --reserve-gib cannot be negative")
+ usage = Path(args.usage) if args.usage else model / ".coli_usage"
+ budget = int(args.budget_gib * GIB)
+ reserve = int(args.reserve_gib * GIB)
+ if args.action == "plan":
+ payload, _selected = create_plan(model, mirror, args.source_dir, usage, budget, reserve)
+ else:
+ payload = stage_mirror(model, mirror, args.source_dir, usage, budget, reserve)
+ print(json.dumps(payload, indent=2))
+ ready = payload.get("ready", payload.get("admitted", False))
+ return 0 if ready else 2
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except (MirrorError, OSError) as error:
+ print(json.dumps({"schema": SCHEMA, "ready": False, "error": str(error)}),
+ file=sys.stderr)
+ raise SystemExit(1)