Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(tools): content-validate fp8 repack resume and refuse malformed inputs
monotophicae2f1dcc6722
diff --git a/c/tests/test_fp8_repack.py b/c/tests/test_fp8_repack.pyindex e6da135f..7b0facee 100644--- a/c/tests/test_fp8_repack.py+++ b/c/tests/test_fp8_repack.py@@ -41,7 +41,7 @@ stderr naming the condition), not exit 0 having emitted nothing -- an empty"container" nobody asked for; --dry-run's own "0 selected" report isunaffected (that IS the loud, honest answer dry-run exists to give)."""-import glob, json, os, re, struct, subprocess, sys, tempfile, unittest+import glob, hashlib, json, os, re, struct, subprocess, sys, tempfile, unittesttry:import torch@@ -838,5 +838,237 @@ class ZeroTargetRefusalTest(unittest.TestCase):self.assertIn("0 tensor(s) selected", proc.stdout)+def _sha(path):+ return hashlib.sha256(open(path, "rb").read()).hexdigest()+++class MalformedRepackTensorTest(unittest.TestCase):+ """U5 fix round, finding 2: a repack-KIND tensor with the wrong dtype or a+ missing `_scale_inv` sidecar used to fall through BOTH selection+ predicates (not repackable, kind not in PASSTHROUGH_KINDS) and was+ silently DROPPED from the container -- an exit-0 mint with a missing+ weight. It must instead refuse loudly, naming the tensor and the exact+ defect (MalformedRepackTensorError), with a nonzero exit."""++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.indir = os.path.join(self.tmp.name, "fp8src")+ os.makedirs(self.indir)+ self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")+ self.tool = os.path.join(os.path.dirname(__file__), "..", "tools",+ "repack_fp8_passthrough.py")++ def tearDown(self):+ self.tmp.cleanup()++ def _write_shard(self, sd):+ save_file(sd, self.shard)++ def _fp8_pair(self, name, O=256, I_=256):+ from glm_fp8_emit import fp8_block_quantize+ torch.manual_seed(1)+ w_fp8, scale = fp8_block_quantize(torch.randn(O, I_) * 0.02)+ return {name: w_fp8, name + "_scale_inv": scale}++ def test_missing_sidecar_refused_with_name_and_defect(self):+ name = "model.layers.0.self_attn.o_proj.weight"+ sd = self._fp8_pair(name)+ del sd[name + "_scale_inv"]+ # a well-formed tensor alongside, so the refusal is provably about the+ # malformed one and not about an otherwise-empty shard+ sd.update(self._fp8_pair("model.layers.0.self_attn.q_a_proj.weight"))+ self._write_shard(sd)+ for fn in (rp.repack_shard, rp.shard_inventory): # real mode AND --dry-run+ with self.assertRaises(rp.MalformedRepackTensorError) as cm:+ fn(self.shard, 5)+ self.assertIn(name, str(cm.exception))+ self.assertIn("_scale_inv", str(cm.exception))++ def test_wrong_dtype_refused_with_name_and_dtype(self):+ name = "model.layers.0.self_attn.kv_b_proj.weight"+ sd = self._fp8_pair("model.layers.0.self_attn.o_proj.weight")+ sd[name] = (torch.randn(256, 256) * 0.02).to(torch.bfloat16)+ self._write_shard(sd)+ with self.assertRaises(rp.MalformedRepackTensorError) as cm:+ rp.repack_shard(self.shard, 5)+ self.assertIn(name, str(cm.exception))+ self.assertIn("BF16", str(cm.exception))++ def test_cli_exits_nonzero_and_names_the_tensor(self):+ name = "model.layers.0.self_attn.o_proj.weight"+ sd = self._fp8_pair(name)+ del sd[name + "_scale_inv"]+ self._write_shard(sd)+ with open(os.path.join(self.indir, "config.json"), "w") as fh:+ json.dump({"model_type": "glm_moe_dsa_test_fixture"}, fh)+ with open(os.path.join(self.indir, "tokenizer.json"), "w") as fh:+ json.dump({"model": {"vocab": {"<|endoftext|>": 0}, "merges": []}}, fh)+ proc = subprocess.run(+ [sys.executable, self.tool, "--indir", self.indir,+ "--outdir", os.path.join(self.tmp.name, "out"), "--n-layers", "5"],+ capture_output=True, text=True)+ self.assertNotEqual(proc.returncode, 0,+ "a malformed repack-kind tensor must refuse, not drop silently")+ self.assertIn("MalformedRepackTensorError", proc.stderr)+ self.assertIn(name, proc.stderr)++ def test_control_wellformed_fixture_still_repacks(self):+ """Proof-of-bite control: the same shapes WITH their sidecars mint."""+ sd = self._fp8_pair("model.layers.0.self_attn.o_proj.weight")+ sd.update(self._fp8_pair("model.layers.0.self_attn.kv_b_proj.weight"))+ self._write_shard(sd)+ out, inv, fmt_map = rp.repack_shard(self.shard, 5)+ self.assertEqual(len(inv), 2)+ self.assertIn("model.layers.0.self_attn.o_proj.weight", out)+ self.assertIn("model.layers.0.self_attn.kv_b_proj.weight", out)+++class ResumeContentValidationTest(unittest.TestCase):+ """U5 fix round, finding 1: the resume manifest used to trust+ os.path.exists -- a truncated or mutated output shard resumed as "done",+ and an input mutated between runs went undetected. Resume must validate+ CONTENT: recorded size+sha256 per completed output (re-emit on mismatch),+ and a recorded input fingerprint whose mismatch aborts with+ InputChangedError."""++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.indir = os.path.join(self.tmp.name, "fp8src")+ os.makedirs(self.indir)+ self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")+ _make_fixture(self.shard)+ with open(os.path.join(self.indir, "config.json"), "w") as fh:+ json.dump({"model_type": "glm_moe_dsa_test_fixture"}, fh)+ with open(os.path.join(self.indir, "tokenizer.json"), "w") as fh:+ json.dump({"model": {"vocab": {"<|endoftext|>": 0}, "merges": []}}, fh)+ self.outdir = os.path.join(self.tmp.name, "out")+ self.tool = os.path.join(os.path.dirname(__file__), "..", "tools",+ "repack_fp8_passthrough.py")+ self.prog = os.path.join(self.outdir, ".fp8pass-progress.json")++ def tearDown(self):+ self.tmp.cleanup()++ def _run(self):+ return subprocess.run(+ [sys.executable, self.tool, "--indir", self.indir, "--outdir",+ self.outdir, "--n-layers", "5"], capture_output=True, text=True)++ def _out_shard(self):+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))+ self.assertEqual(len(outs), 1)+ return outs[0]++ def test_truncated_output_reemitted_byte_identical(self):+ self.assertEqual(self._run().returncode, 0)+ out = self._out_shard()+ good = _sha(out)+ stamped_before = json.loads(open(self.prog).read())["stamped"]+ with open(out, "r+b") as f:+ f.truncate(os.path.getsize(out) // 2)+ proc = self._run()+ self.assertEqual(proc.returncode, 0)+ self.assertIn("failed content validation", proc.stdout)+ self.assertIn("truncated", proc.stdout)+ self.assertEqual(_sha(out), good,+ "a truncated 'completed' shard must be re-emitted, byte-identical")+ # the re-emit hands its stamp share back before re-counting: no doubling+ self.assertEqual(json.loads(open(self.prog).read())["stamped"], stamped_before)++ def test_same_size_output_mutation_reemitted(self):+ """Size alone cannot catch a same-size byte flip -- sha256 must."""+ self.assertEqual(self._run().returncode, 0)+ out = self._out_shard()+ good = _sha(out)+ with open(out, "r+b") as f:+ f.seek(-1, 2)+ b = f.read(1)+ f.seek(-1, 2)+ f.write(bytes([b[0] ^ 0xFF]))+ proc = self._run()+ self.assertEqual(proc.returncode, 0)+ self.assertIn("sha256", proc.stdout)+ self.assertEqual(_sha(out), good)++ def test_mutated_input_refuses_with_named_error(self):+ self.assertEqual(self._run().returncode, 0)+ os.utime(self.shard) # content untouched; fingerprint (mtime_ns) moves+ proc = self._run()+ self.assertNotEqual(proc.returncode, 0,+ "an input that changed between runs must abort the resume")+ self.assertIn("InputChangedError", proc.stderr)+ self.assertIn("INPUT CHANGED", proc.stderr)+ self.assertIn("model-00001-of-00001.safetensors", proc.stderr)++ def test_legacy_string_manifest_refused(self):+ """A pre-fix manifest (string entries, no content records) has nothing+ to validate against; trusting it is the exact hole this fix closes."""+ self.assertEqual(self._run().returncode, 0)+ out_name = os.path.basename(self._out_shard())+ with open(self.prog, "w") as fh:+ json.dump({"shards": {"model-00001-of-00001.safetensors": out_name},+ "stamped": 5}, fh)+ proc = self._run()+ self.assertNotEqual(proc.returncode, 0)+ self.assertIn("ResumeManifestError", proc.stderr)+ self.assertIn("content-validated", proc.stderr)++ def test_untouched_resume_still_skips_without_rewrite(self):+ self.assertEqual(self._run().returncode, 0)+ out = self._out_shard()+ mtime = os.path.getmtime(out)+ proc = self._run()+ self.assertEqual(proc.returncode, 0)+ self.assertIn("[RESUME] 1 shard(s) already done", proc.stdout)+ self.assertEqual(os.path.getmtime(out), mtime,+ "content validation must not rewrite a valid shard")+++class CanonicalizerSlotFitTest(unittest.TestCase):+ """U5 fix round, finding 4: _canonicalize_metadata_order's slot-fit guard+ was a bare assert -- compiled out under python -O, where an oversized+ header rewrite (non-ASCII bytes json.dumps re-escapes longer) silently+ overwrote tensor payload bytes. It must be a real, always-on exception."""++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.path = os.path.join(self.tmp.name, "shard.safetensors")+ # non-ASCII metadata VALUE: the Rust writer stores it raw UTF-8; the+ # canonicalizer's json.dumps re-escapes it to \\uXXXX, inflating the+ # header past its slot.+ save_file({"w": torch.zeros(4, 4)}, self.path,+ metadata={"colibri.fmt": '{"pésééééééé": "x"}',+ "colibri.container.formats": '["fp8-e4m3-b128"]'})++ def tearDown(self):+ self.tmp.cleanup()++ def test_slot_fit_violation_raises_real_exception(self):+ before = _sha(self.path)+ with self.assertRaises(ValueError) as cm:+ rp._canonicalize_metadata_order(self.path)+ self.assertNotIsInstance(cm.exception, AssertionError)+ self.assertIn("HEADER SLOT-FIT", str(cm.exception))+ self.assertEqual(_sha(self.path), before, "the refusal must not touch the file")++ def test_slot_fit_guard_survives_python_dash_O(self):+ """The exact pre-fix failure: under -O the assert vanished and the file+ was corrupted silently. The guard must fire under -O too."""+ before = _sha(self.path)+ tools = os.path.join(os.path.dirname(__file__), "..", "tools")+ proc = subprocess.run(+ [sys.executable, "-O", "-c",+ "import sys; sys.path.insert(0, sys.argv[1]); "+ "import repack_fp8_passthrough as rp; "+ "rp._canonicalize_metadata_order(sys.argv[2])",+ tools, self.path],+ capture_output=True, text=True)+ self.assertNotEqual(proc.returncode, 0,+ "the slot-fit guard must survive python -O")+ self.assertIn("HEADER SLOT-FIT", proc.stderr)+ self.assertEqual(_sha(self.path), before,+ "under -O the file must stay untouched, not silently corrupt")++if __name__ == "__main__":unittest.main()diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.pyindex 1ebf92f9..97c06156 100644--- a/c/tools/repack_fp8_passthrough.py+++ b/c/tools/repack_fp8_passthrough.py@@ -212,7 +212,7 @@ Usage (synthetic fixtures / local testing only):python3 tools/repack_fp8_passthrough.py --indir <fp8_dir> --outdir <out> --n-layers 78python3 tools/repack_fp8_passthrough.py --indir <fp8_dir> --outdir <out> --n-layers 78 --mtp"""-import argparse, glob, json, os, shutil, struct, sys+import argparse, glob, hashlib, json, os, shutil, struct, syssys.path.insert(0, os.path.dirname(__file__))from convert_fp8_to_int4 import classify, check_or_record_params # reuse: same taxonomy,@@ -338,6 +338,62 @@ def is_passthrough_target(name, n_layers, keep_mtp=False):return kind in PASSTHROUGH_KINDS+class MalformedRepackTensorError(ValueError):+ """A repack-kind tensor (REPACK_KINDS) that CANNOT be repacked: wrong dtype+ for the fp8 path, or no `_scale_inv` sidecar. Named so the refusal is+ grep-able; raised instead of silently dropping the tensor (see+ _repack_defect)."""+++class InputChangedError(ValueError):+ """A source shard recorded done in the resume manifest no longer matches+ its recorded fingerprint (size + mtime_ns): the input changed between+ runs, and a resumed mint over it would emit a container silently mixing+ two different sources."""+++class ResumeManifestError(ValueError):+ """The resume manifest cannot be trusted as-is -- e.g. its entries predate+ content-validated resume and carry no size/sha256/fingerprint records to+ verify against."""+++def _repack_defect(name, dtype, keys, n_layers, keep_mtp=False):+ """The exact defect string if `name` is a repack-kind tensor (REPACK_KINDS)+ that CANNOT be repacked -- wrong dtype for the fp8 path, or a missing+ `_scale_inv` sidecar -- else None.++ This is the loud counterpart to is_repack_target's False: before this+ check existed, a repack-KIND tensor that failed the dtype or sidecar test+ fell through BOTH selection predicates (its kind is not in+ PASSTHROUGH_KINDS either) and was silently DROPPED from the container --+ an o_proj missing its scale sidecar produced an exit-0 mint with that+ weight absent, discovered only at engine load time (or worse, not at+ all on a checkpoint variant nobody loads immediately). Same "refuse+ rather than guess" discipline as _check_geometry, applied one step+ earlier: a malformed SOURCE tensor is caught at selection time with its+ name and the exact defect, instead of vanishing.++ `_scale_inv` sidecars themselves and non-repack kinds are None (not this+ check's concern -- sidecars are consumed with their weight, and+ passthrough/skip kinds have their own paths)."""+ if name.endswith("_scale_inv"):+ return None+ kind = classify(name, n_layers, keep_mtp=keep_mtp)+ if kind not in REPACK_KINDS:+ return None+ if dtype not in ("F8_E4M3", "float8_e4m3fn"):+ return (f"{name}: repack-kind ('{kind}') tensor has dtype {dtype}, not FP8 "+ f"e4m3 -- this tool byte-preserves FP8 weights only, and silently "+ f"dropping it would mint a container with a MISSING weight; refusing")+ if (name + "_scale_inv") not in keys:+ return (f"{name}: FP8 repack-kind ('{kind}') tensor has no "+ f"'{name}_scale_inv' block-scale sidecar in its shard -- an fmt=8 "+ f"tensor cannot be minted without its `.qs` scales, and silently "+ f"dropping it would mint a container with a MISSING weight; refusing")+ return None++def _stampable(kind):"""Will the ENGINE consult a `colibri.fmt` stamp for a tensor of this kind?@@ -565,10 +621,23 @@ def _canonicalize_metadata_order(path):returnhdr["__metadata__"] = dict(sorted(meta.items()))new = json.dumps(hdr, separators=(",", ":")).encode()- assert len(new) <= hlen, (- f"{path}: canonicalized header ({len(new)} bytes) no longer fits "- f"the original slot ({hlen} bytes) -- a same-keys/same-values "- f"reorder should never change length; something else changed")+ # SLOT-FIT guard. A real exception, NOT an assert: python -O compiles+ # asserts out, and with this check gone an oversized rewrite seeks to+ # offset 8 and overwrites the first tensor's payload bytes -- a silently+ # corrupted container from the one function whose whole job is byte+ # determinism. The known way to oversize: a non-ASCII name/metadata byte+ # the original writer stored as raw UTF-8 (2-4 bytes) that json.dumps+ # re-escapes as \\uXXXX (6 bytes), so "same keys, same values" is only+ # length-preserving for the ASCII headers this tool itself emits.+ if len(new) > hlen:+ raise ValueError(+ f"{path}: HEADER SLOT-FIT check failed -- canonicalized header "+ f"({len(new)} bytes) no longer fits the original {hlen}-byte slot. "+ f"A same-keys/same-values reorder is length-preserving only for "+ f"pure-ASCII headers; a non-ASCII tensor name or metadata byte "+ f"re-escapes longer under json.dumps. Writing it would overwrite "+ f"tensor payload bytes at offset 8+{hlen}; refusing to corrupt "+ f"{path}")new += b" " * (hlen - len(new))with open(path, "r+b") as f:f.seek(8)@@ -617,6 +686,79 @@ def _stamps_from_other_passes(outdir, prog_name):return total+def _sha256_file(path, bufsize=1 << 20):+ h = hashlib.sha256()+ with open(path, "rb") as f:+ for chunk in iter(lambda: f.read(bufsize), b""):+ h.update(chunk)+ return h.hexdigest()+++def _input_fingerprint(path):+ """Fingerprint of a SOURCE shard for the resume manifest: size + mtime_ns.++ Deliberately NOT a content hash: inputs are the part of a real mint that+ is hundreds of GB, and they are re-checked on EVERY resume for EVERY+ shard (including the completed ones), so an input hash would turn each+ resume into a full re-read of the source checkpoint. size+mtime_ns is the+ same class of evidence rsync/make trust for "unchanged", catches every+ normal mutation path (a re-download, an edit, a different checkpoint+ symlinked into place all change at least one of the two), and the failure+ mode it cannot catch -- a byte-level mutation that preserves size AND+ resets mtime_ns to the nanosecond -- requires deliberate forgery, against+ which the OUTPUT-side sha256 in the same manifest still bounds the blast+ radius to shards not yet minted. The output side hashes precisely because+ this reasoning does not apply there: outputs are validated one at a time+ and only when recorded done."""+ st = os.stat(path)+ return {"in_size": st.st_size, "in_mtime_ns": st.st_mtime_ns}+++def _require_content_records(done, prog_path):+ """CONTENT-VALIDATED RESUME: refuse a manifest whose entries predate the+ content records (plain string values, the pre-fix schema) -- there is+ nothing in such an entry to validate a "completed" output against, and+ resuming on trust is exactly the hole this fix closes. Refusing (rather+ than silently re-minting) keeps the operator in the loop: deleting the+ manifest is a one-line, safe reset (shards re-emit deterministically,+ byte-identical for byte-identical inputs)."""+ legacy = sorted(k for k, v in done.items() if not isinstance(v, dict))+ if legacy:+ raise ResumeManifestError(+ f"{prog_path}: {len(legacy)} manifest entr"+ f"{'y' if len(legacy) == 1 else 'ies'} (first: '{legacy[0]}') "+ f"predate content-validated resume -- no output size/sha256 and no "+ f"input fingerprint to verify against, so 'done' cannot be "+ f"trusted. Delete {prog_path} to re-mint; output shards re-emit "+ f"deterministically from unchanged inputs")+++def _resume_output_problem(entry, outdir):+ """Why a shard recorded done canNOT be trusted as done, or None if its+ recorded output validates. Content validation, not existence: the size+ check (a stat) catches truncation/partial writes first and cheaply, then+ sha256 catches same-size mutation. Whole-file sha256 is the same evidence+ class tools/repack_rans.py's manifest records per emitted shard."""+ out = entry.get("out")+ if out == "":+ return None # shard legitimately emitted nothing+ if not out or not isinstance(out, str):+ return "manifest entry has no usable output record"+ path = os.path.join(outdir, out)+ if not os.path.exists(path):+ return f"{out}: recorded output shard is missing"+ size = os.path.getsize(path)+ if size != entry.get("out_size"):+ return (f"{out}: size {size} != recorded {entry.get('out_size')} "+ f"(truncated or partially written)")+ digest = _sha256_file(path)+ if digest != entry.get("out_sha256"):+ return (f"{out}: sha256 {digest[:16]}... != recorded "+ f"{str(entry.get('out_sha256'))[:16]}... (content changed after "+ f"this shard was recorded done)")+ return None++# dtype -> bytes/element, for the dry-run byte-volume estimate on PASS-THROUGH# tensors only (shard_inventory reads shapes from get_slice, never the tensor# data itself, so it cannot ask torch for itemsize). Covers the dtypes a raw@@ -653,6 +795,9 @@ def shard_inventory(path, n_layers, keep_mtp=False):for name in f.keys():sl = f.get_slice(name)dt = sl.get_dtype()+ defect = _repack_defect(name, dt, keys, n_layers, keep_mtp)+ if defect is not None:+ raise MalformedRepackTensorError(defect)if is_repack_target(name, dt, keys, n_layers, keep_mtp):kind = classify(name, n_layers, keep_mtp=keep_mtp)O, I = sl.get_shape()@@ -712,6 +857,9 @@ def repack_shard(path, n_layers, keep_mtp=False):for name in f.keys():sl = f.get_slice(name)dt = sl.get_dtype()+ defect = _repack_defect(name, dt, keys, n_layers, keep_mtp)+ if defect is not None:+ raise MalformedRepackTensorError(defect)if is_repack_target(name, dt, keys, n_layers, keep_mtp):kind = classify(name, n_layers, keep_mtp=keep_mtp)stampable = _stampable(kind)@@ -904,12 +1052,19 @@ def main():if not check_or_record_params(a.outdir, guard_prefix, params):sys.exit(1)- # RESUME (same #383-class manifest idiom as convert_fp8_to_int4.py's --indir path):- # a sidecar records input-shard -> output-shard-name-or-"" (shards with no- # resident-FP8 tensors emit no file), written atomically so an interrupted run- # never leaves a half-written manifest for the next invocation to trust. The- # params guard above already owns "don't mix conversions" -- this manifest is- # purely which shards are done.+ # RESUME (same #383-class manifest idiom as convert_fp8_to_int4.py's --indir+ # path), CONTENT-VALIDATED: a sidecar records, per input shard, a dict --+ # {"out": output-shard-name-or-"" (shards with no target tensors emit no+ # file), "out_size"/"out_sha256": the emitted shard's content record,+ # "stamps": its share of the stamp budget, "in_size"/"in_mtime_ns": the+ # INPUT shard's fingerprint at completion} -- written atomically so an+ # interrupted run never leaves a half-written manifest for the next+ # invocation to trust. "Done" is re-verified on resume against the CONTENT+ # records, never against bare existence: a truncated/mutated output is+ # re-emitted (to its recorded name), and a mutated INPUT aborts the run+ # with InputChangedError (resuming over it would mix two sources into one+ # container). The params guard above already owns "don't mix conversions"+ # -- this manifest is purely which shards are done, and provably so.prog_path = os.path.join(a.outdir, prog_name)prog = {}if os.path.exists(prog_path):@@ -918,6 +1073,7 @@ def main():except (OSError, ValueError):prog = {}done = prog.setdefault("shards", {})+ _require_content_records(done, prog_path)# ST_FMT_STAMP_MAX is a CONTAINER-wide bound, and a resumed run only sees the# shards it processes itself -- so the running total lives in the manifest# next to the shard progress, not in a local counter that a resume resets.@@ -934,18 +1090,39 @@ def main():n = fresh = skipped = 0for sp in shards:key = os.path.basename(sp)+ fp = _input_fingerprint(sp)prev = done.get(key)- if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))):- if prev:- n += 1- skipped += 1- continue+ reuse_name = None+ if prev is not None:+ recorded = (prev.get("in_size"), prev.get("in_mtime_ns"))+ if recorded != (fp["in_size"], fp["in_mtime_ns"]):+ raise InputChangedError(+ f"{key}: INPUT CHANGED between runs -- fingerprint at "+ f"completion was size={recorded[0]} mtime_ns={recorded[1]}, "+ f"but {sp} is now size={fp['in_size']} "+ f"mtime_ns={fp['in_mtime_ns']}. Resuming would mix two "+ f"different sources into one container; delete "+ f"{prog_name} (and the emitted shards) in {a.outdir} to "+ f"re-mint from the current input")+ problem = _resume_output_problem(prev, a.outdir)+ if problem is None:+ if prev["out"]:+ n += 1+ skipped += 1+ continue+ # Recorded done, but the OUTPUT no longer validates: re-emit it,+ # to its recorded name (so the container's shard numbering is+ # stable), handing its stamp share back to the budget first.+ print(f"[RESUME] {key}: recorded output failed content validation "+ f"({problem}) -- re-emitting")+ reuse_name = prev.get("out") or None+ prog["stamped"] -= int(prev.get("stamps", 0))out, inv, fmt_map = repack_shard(sp, a.n_layers, a.mtp)all_inv.extend(inv)if not out:- done[key] = ""+ done[key] = {"out": "", "stamps": 0, **fp}else:- name = f"{out_prefix}{n:05d}.safetensors"+ name = reuse_name if reuse_name else f"{out_prefix}{n:05d}.safetensors"# I-4 STRUCTURAL GATE: last check before the bytes exist on disk,# derived from `out` itself rather than from the selection that built# it. See _emission_stamp_gate.@@ -970,7 +1147,13 @@ def main():# Fix up the two bytes that can vary, in place, right after the file# is written and before this shard is recorded done._canonicalize_metadata_order(out_path)- done[key] = name+ # Content record for resume: taken from the file just written+ # (post-canonicalization, so the hash matches what a resume will+ # re-read), plus the input fingerprint from before this shard was+ # processed.+ done[key] = {"out": name, "stamps": len(fmt_map),+ "out_size": os.path.getsize(out_path),+ "out_sha256": _sha256_file(out_path), **fp}n += 1fresh += 1tmp = prog_path + ".tmp"@@ -984,14 +1167,14 @@ def main():# FIX ROUND 2 (clean-room conformance trial, spec I6 -- loud failure, every# refusal names its condition): a run that selects ZERO repack-target tensors# across every shard under --indir (present run AND any prior resumed run --- # `done` has one entry per shard, "" meaning that shard has NEVER produced an- # output) used to exit 0 having emitted nothing -- a silent trap: an empty+ # `done` has one entry per shard, out=="" meaning that shard has NEVER+ # produced an output) used to exit 0 having emitted nothing -- a silent trap: an empty# "container" (just the resume/params sidecars, no actual .safetensors output)# that nobody asked for and no caller-side check would catch. Refuse loudly# instead. --dry-run is NOT covered here: reporting "0 tensors selected" IS# the loud, honest answer dry-run exists to give, not a silent no-op -- see# the early `return` above, before any of this resume/write bookkeeping.- if all(v == "" for v in done.values()):+ if all(v.get("out", "") == "" for v in done.values()):print(f"ERROR: no repack-target tensors found under {a.indir} (checked "f"{len(shards)} shard(s), 0 selected) -- nothing emitted; refusing "f"to exit 0 for an empty container nobody asked for", file=sys.stderr)