Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(tools): extend fp8 repack tool to full-family emission

monotophic886eacfea291
diff --git a/c/tests/test_fp8_e2e_repack_load.py b/c/tests/test_fp8_e2e_repack_load.py
index c38cd4d7..ee505676 100644
--- a/c/tests/test_fp8_e2e_repack_load.py
+++ b/c/tests/test_fp8_e2e_repack_load.py
@@ -18,9 +18,24 @@ test does, end to end:
Makefile's own CFLAGS, asserted to produce zero warnings -- into a
temp binary;
4. run that binary against the REAL repacked output directory, asserting
- every stamped tensor loads fmt=8 through the real qt_from_disk with
+ every selected tensor loads fmt=8 through the real qt_from_disk with
finite dequantized values.
+Three cases the round trip has to carry, all through the same real loader:
+
+ * a RESIDENT tensor at the COLLISION shape (nblkO*nblkI==O). Its stamp is the
+ only thing standing between fmt=8 and a silent int8-row misread, and this is
+ the first test in the tree that proves the stamp does that job on a
+ tool-produced container rather than a hand-authored one. Its proof-of-bite
+ is a second run over the SAME container with the `colibri.fmt` key deleted
+ from the header: the loader must then resolve the identical bytes to fmt=1.
+ * a ROUTED EXPERT, unstamped by design (colibri.c's routed-expert load sites
+ pass stamped_name=NULL), resolving to fmt=8 by byte arithmetic alone.
+ * the container-level `colibri.container.formats` declaration, which no engine
+ reads yet: an unknown `__metadata__` key must be IGNORED, not refused. That
+ is invariant I-1's half of this PR -- an engine that does not know the key
+ loads such a container exactly as it does today.
+
Hermetic: everything (checkpoint, repacked output, compiled harness binary)
lives under one TemporaryDirectory; nothing is left behind.
"""
@@ -70,8 +85,60 @@ def _cc_flags():
return cc, cflags, ldflags
+def _read_header(path):
+ with open(path, "rb") as f:
+ hlen = struct.unpack("<Q", f.read(8))[0]
+ return hlen, json.loads(f.read(hlen))
+
+
+def _rewrite_metadata(path, drop_key=None, set_key=None, value=None):
+ """Edit `__metadata__` IN PLACE, keeping every tensor's data offset valid by
+ re-padding the header JSON to its original byte length with spaces (the
+ safetensors header is length-prefixed and may carry trailing whitespace;
+ every JSON parser in the chain, including c/json.h's, skips it). Used to
+ build the proof-of-bite containers: identical weight bytes, identical
+ offsets, one metadata key changed."""
+ hlen, hdr = _read_header(path)
+ if drop_key is not None:
+ del hdr["__metadata__"][drop_key]
+ if set_key is not None:
+ hdr["__metadata__"][set_key] = value
+ new = json.dumps(hdr, separators=(",", ":")).encode()
+ assert len(new) <= hlen, "rewritten header must fit the original slot"
+ new += b" " * (hlen - len(new))
+ with open(path, "r+b") as f:
+ f.seek(8)
+ f.write(new)
+
+
class Fp8RepackLoadE2ETest(unittest.TestCase):
- """The real tools/repack_fp8_passthrough.py -> the real C loader, once."""
+ """The real tools/repack_fp8_passthrough.py -> the real C loader."""
+
+ # colibri.c is a big translation unit; build the harness once for the class
+ # rather than once per test.
+ _harness_dir = None
+ _harness_bin = None
+ _build_stderr = None
+
+ @classmethod
+ def setUpClass(cls):
+ cc, cflags, ldflags = _cc_flags()
+ if not cc:
+ raise unittest.SkipTest("no C compiler found on PATH")
+ cls._harness_dir = tempfile.TemporaryDirectory()
+ harness_src = os.path.join(HERE, "test_fp8_e2e_loader.c")
+ harness_bin = os.path.join(cls._harness_dir.name, "test_fp8_e2e_loader")
+ build = subprocess.run([cc] + cflags + [harness_src, "-o", harness_bin] + ldflags,
+ capture_output=True, text=True, cwd=C_DIR)
+ if build.returncode != 0:
+ raise AssertionError(f"e2e harness build failed:\n{build.stderr}")
+ cls._harness_bin = harness_bin
+ cls._build_stderr = build.stderr
+
+ @classmethod
+ def tearDownClass(cls):
+ if cls._harness_dir:
+ cls._harness_dir.cleanup()
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
@@ -79,23 +146,55 @@ class Fp8RepackLoadE2ETest(unittest.TestCase):
os.makedirs(self.indir)
self.outdir = os.path.join(self.tmp.name, "out")
self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")
+ # FULL-FAMILY EMISSION (M1 audit): config.json AND tokenizer.json are
+ # now MANDATORY in --indir -- the real repack tool refuses to run
+ # without either. This harness only exercises the C loader's
+ # tensor-decode path (qt_from_disk via test_fp8_e2e_loader.c), never
+ # load_cfg or tok_load, so a minimal placeholder is enough for both.
+ with open(os.path.join(self.indir, "config.json"), "w") as fh:
+ json.dump({"model_type": "glm_moe_dsa_test_fixture"}, fh)
+ with open(os.path.join(self.indir, "tokenizer.json"), "w") as fh:
+ json.dump({"model": {"vocab": {"<|endoftext|>": 0}, "merges": []}}, fh)
def tearDown(self):
self.tmp.cleanup()
+ def _repack(self):
+ tool = os.path.join(C_DIR, "tools", "repack_fp8_passthrough.py")
+ rc = subprocess.run([sys.executable, tool, "--indir", self.indir,
+ "--outdir", self.outdir, "--n-layers", "5"],
+ capture_output=True, text=True)
+ self.assertEqual(rc.returncode, 0,
+ f"real repack tool failed:\nSTDOUT:\n{rc.stdout}\nSTDERR:\n{rc.stderr}")
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ self.assertEqual(len(outs), 1, "expected exactly one repacked output shard")
+ return outs[0]
+
+ def _load(self, shapes):
+ args = [self._harness_bin, self.outdir]
+ for name, (O, I) in shapes.items():
+ args += [name, str(O), str(I)]
+ return subprocess.run(args, capture_output=True, text=True)
+
+ def test_harness_builds_without_warnings(self):
+ """Production flags require zero warnings (pr-cycle mechanical gate)."""
+ self.assertEqual(self._build_stderr.strip(), "",
+ f"e2e harness build produced warnings:\n{self._build_stderr}")
+
def test_real_repack_output_loads_through_real_c_loader(self):
D, I_ = 256, 384 # non-degenerate shapes: nblkO*nblkI never coincides with O (no
# THE DESIGN LANDMINE ambiguity here -- that boundary is
- # test_fp8_load.c's job, this test's job is the round trip)
+ # test_fp8_load.c's job and this file's collision test below)
torch.manual_seed(7)
sd = {
"model.layers.0.self_attn.o_proj.weight": torch.randn(D, D) * 0.02,
"model.layers.0.self_attn.q_a_proj.weight": torch.randn(D, D) * 0.02,
"model.layers.0.mlp.shared_experts.gate_proj.weight": torch.randn(D, I_) * 0.02,
"model.layers.0.mlp.gate_proj.weight": torch.randn(D, I_) * 0.02,
- # routed expert: must NOT be selected/stamped (stays int4-g64 path) -- included
- # as a negative control so this test also proves the real tool's selection
- # logic held on the real round trip, not just in test_fp8_repack.py's own suite.
+ # routed expert: selected and byte-preserved, but UNSTAMPED -- it must
+ # resolve to fmt=8 by byte arithmetic alone, because colibri.c's
+ # routed-expert load sites pass stamped_name=NULL and would never see a
+ # stamp even if one were written.
"model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(I_, D) * 0.02,
"model.layers.0.input_layernorm.weight": torch.randn(D), # f32: must NOT be selected
}
@@ -108,54 +207,119 @@ class Fp8RepackLoadE2ETest(unittest.TestCase):
"model.layers.0.self_attn.q_a_proj.weight": (D, D),
"model.layers.0.mlp.shared_experts.gate_proj.weight": (D, I_),
"model.layers.0.mlp.gate_proj.weight": (D, I_),
+ "model.layers.0.mlp.experts.0.gate_proj.weight": (I_, D),
}
save_fp8_safetensors(sd, self.shard)
-
- tool = os.path.join(C_DIR, "tools", "repack_fp8_passthrough.py")
- rc = subprocess.run([sys.executable, tool, "--indir", self.indir,
- "--outdir", self.outdir, "--n-layers", "5"],
- capture_output=True, text=True)
- self.assertEqual(rc.returncode, 0, f"real repack tool failed:\nSTDOUT:\n{rc.stdout}\nSTDERR:\n{rc.stderr}")
-
- outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
- self.assertEqual(len(outs), 1, "expected exactly one repacked output shard")
+ out = self._repack()
# Sanity-check the real tool's own selection/stamp BEFORE asking the C harness
# to load it: if this fails, the bug is in the tool, not the loader, and running
# the harness anyway would only muddy which side broke.
- with open(outs[0], "rb") as f:
- hlen = struct.unpack("<Q", f.read(8))[0]
- hdr = json.loads(f.read(hlen))
+ _, hdr = _read_header(out)
self.assertIn("__metadata__", hdr)
self.assertIn("colibri.fmt", hdr["__metadata__"])
stamp = json.loads(hdr["__metadata__"]["colibri.fmt"])
- self.assertEqual(set(stamp.keys()), set(shapes.keys()),
- "real tool's selection must match exactly the resident-kind tensors")
- self.assertNotIn("model.layers.0.mlp.experts.0.gate_proj.weight", hdr)
- self.assertNotIn("model.layers.0.input_layernorm.weight", hdr)
-
- cc, cflags, ldflags = _cc_flags()
- if not cc:
- raise unittest.SkipTest("no C compiler found on PATH")
-
- harness_src = os.path.join(HERE, "test_fp8_e2e_loader.c")
- harness_bin = os.path.join(self.tmp.name, "test_fp8_e2e_loader")
- build = subprocess.run([cc] + cflags + [harness_src, "-o", harness_bin] + ldflags,
- capture_output=True, text=True, cwd=C_DIR)
- self.assertEqual(build.returncode, 0, f"e2e harness build failed:\n{build.stderr}")
- self.assertEqual(build.stderr.strip(), "",
- f"e2e harness build produced warnings (production flags require zero):\n{build.stderr}")
+ self.assertEqual(set(stamp.keys()),
+ set(shapes) - {"model.layers.0.mlp.experts.0.gate_proj.weight"},
+ "stamp map must cover the residents and NOT the routed expert")
+ self.assertIn("model.layers.0.mlp.experts.0.gate_proj.weight", hdr,
+ "routed experts are repacked into the container")
+ # FULL-FAMILY EMISSION (M1 audit): input_layernorm is now emitted too
+ # -- as a raw F32 pass-through (is_passthrough_target), not through
+ # the fp8-repack path this harness's qt_from_disk-specific decode
+ # check below exercises (that's a different loader function, `ld`'s
+ # st_read_f32 fallback -- out of this C harness's scope, and out of
+ # this tool's scope per this task: no engine/loader changes here).
+ # What matters for THIS test is that it stays out of the fp8 stamp
+ # map and carries no `.qs` -- test_fp8_repack.py and
+ # test_fp8_repack_full_family.py cover the pass-through copy itself.
+ self.assertIn("model.layers.0.input_layernorm.weight", hdr)
+ self.assertEqual(hdr["model.layers.0.input_layernorm.weight"]["dtype"], "F32")
+ self.assertNotIn("model.layers.0.input_layernorm.weight.qs", hdr)
+ self.assertNotIn("model.layers.0.input_layernorm.weight", stamp)
+ # I-1: the container-level declaration is an UNKNOWN key to this engine.
+ self.assertIn("colibri.container.formats", hdr["__metadata__"])
- args = [harness_bin, self.outdir]
- for name, (O, I) in shapes.items():
- args += [name, str(O), str(I)]
- run = subprocess.run(args, capture_output=True, text=True)
+ run = self._load(shapes)
self.assertEqual(run.returncode, 0,
f"e2e harness (real loader) failed:\nSTDOUT:\n{run.stdout}\nSTDERR:\n{run.stderr}")
self.assertNotIn("FAIL", run.stdout)
for name in shapes:
self.assertIn(f"ok {name}:", run.stdout)
self.assertIn("fp8 e2e repack->load: ok", run.stdout)
+ # ...and the unknown container-level key produced no diagnostic at all.
+ self.assertEqual(run.stderr.strip(), "",
+ f"unknown __metadata__ key must be ignored silently:\n{run.stderr}")
+
+ def test_container_declaration_stays_inert_even_when_malformed(self):
+ """I-1's sharp edge. The well-formed declaration being ignored is checked
+ above; this checks the KEY is inert, not just this VALUE. An engine that
+ does not know `colibri.container.formats` must ignore whatever is under
+ it -- if it ever started validating the value, a container written by a
+ newer tool would stop loading on an older engine, which is exactly the
+ compatibility break the split between PR-A's writer half and the later
+ engine half is arranged to avoid. Contrast `colibri.fmt`, which IS
+ parsed and DOES refuse loudly on a malformed value (st.h's
+ st_fmt_stamp_ingest) -- that asymmetry is deliberate and is what this
+ test pins.
+
+ Non-square, non-degenerate shape on purpose: resolution here must come
+ from byte arithmetic, so nothing about this test depends on the stamp
+ path that the collision test exercises."""
+ torch.manual_seed(13)
+ name = "model.layers.0.self_attn.q_a_proj.weight"
+ save_fp8_safetensors({name: torch.randn(256, 384) * 0.02}, self.shard)
+ out = self._repack()
+ # Each replacement must fit the original value's serialized slot, so the
+ # tensor data offsets the header declares stay valid (see
+ # _rewrite_metadata): not-JSON, a JSON object where an array is
+ # expected, a JSON array of the wrong element type, and empty.
+ for bogus in ('not json', '{"a":1}', '[42]', ''):
+ _rewrite_metadata(out, set_key="colibri.container.formats", value=bogus)
+ run = self._load({name: (256, 384)})
+ self.assertEqual(run.returncode, 0,
+ f"unknown container key with value {bogus!r} must not "
+ f"break the load:\n{run.stdout}\n{run.stderr}")
+ self.assertIn(f"ok {name}:", run.stdout)
+ self.assertEqual(run.stderr.strip(), "",
+ f"value {bogus!r} produced a diagnostic:\n{run.stderr}")
+
+ def test_collision_shape_resident_loads_fmt8_and_the_stamp_is_what_does_it(self):
+ """I-4 end to end, both polarities, on a container the REAL tool produced.
+
+ [2,256] is the collision shape (nblkO=1, nblkI=2, product=2==O): the
+ block-scale array and a per-row int8 scale array are the same 8 bytes, so
+ byte arithmetic alone cannot tell fmt=8 from fmt=1 and qt_resolve_fmt's
+ INVERSION resolves an unstamped one to fmt=1. Stamped, the loader must
+ return fmt=8. PROOF-OF-BITE, on the SAME container with the SAME weight
+ bytes: delete `colibri.fmt` from the header and the loader must return
+ fmt=1 -- so this test would notice if the stamp stopped being consulted,
+ rather than passing on byte arithmetic that never needed it."""
+ torch.manual_seed(9)
+ name = "model.layers.0.self_attn.o_proj.weight"
+ # glm_fp8_emit's block-quantizer is the fixture generator everywhere else in
+ # this suite; [2,256] goes through it unchanged (one 128x128 block row, two
+ # block columns).
+ save_fp8_safetensors({name: torch.randn(2, 256) * 0.02}, self.shard)
+ out = self._repack()
+
+ _, hdr = _read_header(out)
+ self.assertEqual(json.loads(hdr["__metadata__"]["colibri.fmt"]),
+ {name: "fp8-e4m3-b128"})
+ self.assertEqual(hdr[name + ".qs"]["shape"], [2])
+ self.assertEqual(hdr[name]["shape"], [2, 256])
+
+ run = self._load({name: (2, 256)})
+ self.assertEqual(run.returncode, 0,
+ f"stamped collision-shape tensor must load fmt=8:\n{run.stdout}\n{run.stderr}")
+ self.assertIn(f"ok {name}:", run.stdout)
+
+ _rewrite_metadata(out, drop_key="colibri.fmt")
+ bite = self._load({name: (2, 256)})
+ self.assertNotEqual(bite.returncode, 0,
+ "unstamped, the identical bytes must NOT resolve to fmt=8")
+ self.assertIn("fmt=1, expected 8", bite.stdout,
+ f"expected the INVERSION's int8-row default:\n{bite.stdout}")
if __name__ == "__main__":
diff --git a/c/tests/test_fp8_repack.py b/c/tests/test_fp8_repack.py
index 3e69f061..e6da135f 100644
--- a/c/tests/test_fp8_repack.py
+++ b/c/tests/test_fp8_repack.py
@@ -10,23 +10,38 @@ to fmt=8).
Synthetic fixtures ONLY (tools/glm_fp8_emit.py, the exact real-checkpoint FP8
layout that convert_fp8_to_int4.py's dequant() reads) -- no real Z.ai shard is
read or written by this suite, per the build's hard constraint. Covers:
-selection (resident kinds byte-preserved, routed experts / io / f32 excluded),
-byte-for-byte preservation of the fp8 weight and the .qs scale rename, the
-scale-geometry refusal path in TWO forms (THE DESIGN LANDMINE's write-side
-twin) -- a malformed source shard (.qs shape doesn't match ceil(O/128)x
-ceil(I/128) for its weight) must be refused, AND a well-formed-but-AMBIGUOUS
-shape (nblkO*nblkI==O, where this tensor's block-scale byte count would
-coincide with a per-row int8 scale byte count) must ALSO be refused
-(maintainer review, #528: this is not hypothetical, GLM-5.2's own
-self_attn.o_proj.weight is a real instance) -- --dry-run writing nothing, and
-the #383-class resume/params-guard behavior. FIX ROUND 2 (clean-room
+selection (resident kinds AND routed experts byte-preserved; io / f32 /
+kv_b_proj excluded), byte-for-byte preservation of the fp8 weight and the .qs
+scale rename, the scale-geometry refusal path (a malformed source shard whose
+.qs shape doesn't match ceil(O/128)xceil(I/128) for its weight must be
+refused), --dry-run writing nothing, and the #383-class resume/params-guard
+behavior.
+
+THE COLLISION SHAPE, and what this suite pins. A well-formed but AMBIGUOUS
+shape (nblkO*nblkI==O, where a tensor's block-scale byte count coincides with a
+per-row int8 scale byte count -- GLM-5.2's own self_attn.o_proj.weight
+[6144,16384] is a real instance) used to be refused unconditionally. It is now
+refused only when nothing can resolve it on READ: qt_resolve_fmt's "THE STAMPED
+CASE IS UNCHANGED by this inversion" branch (colibri.c) resolves a STAMPED
+tensor at that shape through TRUST-VERIFY-REFUSE, which is the case the stamp
+mechanism was built for; an UNSTAMPED one still resolves to int8-row silently.
+Which tensors are stamp-resolvable is a reader-side fact, not a writer's
+choice: qt_from_disk consults the stamp map, while the three routed-expert load
+sites pass stamped_name=NULL. So this suite pins BOTH polarities --
+ * resident (stampable) at the collision shape: EMITTED, and stamped;
+ * routed expert (unstampable) at the collision shape: REFUSED;
+plus the emission-time structural gate, which re-derives the predicate from the
+OUTPUT tensors themselves so a stamp lost between selection and write is caught
+by something other than the selection logic that lost it.
+
+FIX ROUND 2 (clean-room
conformance trial, spec I6): a real (non-dry-run) run that selects ZERO
repack-target tensors across --indir must refuse loudly (nonzero exit,
stderr naming the condition), not exit 0 having emitted nothing -- an empty
"container" nobody asked for; --dry-run's own "0 selected" report is
unaffected (that IS the loud, honest answer dry-run exists to give).
"""
-import glob, json, os, struct, subprocess, sys, tempfile, unittest
+import glob, json, os, re, struct, subprocess, sys, tempfile, unittest
try:
import torch
@@ -37,6 +52,7 @@ except ImportError as e:
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
from glm_fp8_emit import save_fp8_safetensors
+from convert_fp8_to_int4 import classify
import repack_fp8_passthrough as rp
@@ -46,32 +62,65 @@ def _read_header(path):
return json.loads(f.read(hlen))
+SELECTED = {
+ "model.layers.0.self_attn.o_proj.weight",
+ "model.layers.0.self_attn.q_a_proj.weight",
+ "model.layers.0.self_attn.kv_b_proj.weight", # FULL-FAMILY EMISSION: now selected
+ "model.layers.0.mlp.shared_experts.gate_proj.weight",
+ "model.layers.0.mlp.gate_proj.weight",
+ "model.layers.0.mlp.experts.0.gate_proj.weight",
+ "model.layers.0.mlp.experts.1.up_proj.weight",
+}
+STAMPED = {n for n in SELECTED if ".mlp.experts." not in n}
+ROUTED = SELECTED - STAMPED
+# FULL-FAMILY EMISSION: the non-FP8 pass-through names (is_passthrough_target)
+# the fixture below also carries. Kept separate from SELECTED/STAMPED/ROUTED
+# (which describe the fp8-repack path only -- entries with a `.qs` sidecar)
+# since these never get one.
+PASSTHROUGH = {
+ "model.layers.0.input_layernorm.weight",
+ "model.layers.0.mlp.gate.weight",
+ "model.embed_tokens.weight",
+}
+
+
def _make_fixture(path, D=256, I_=384, E=4):
"""One synthetic shard covering every selection case: SELECTED resident
- kinds (attn/o/sh/dmlp), the EXCLUDED resident kind kvb (kv_b_proj -- valid
- per convert_fp8_to_int4.classify(), but the engine's CPU/CUDA MLA-absorb
- paths have no fmt=8 case yet, so this tool must not select it -- see the
- module docstring), routed experts (must be excluded), the router and a
- norm (f32-kept, never fp8), and embed_tokens (io kind -- excluded by kind
- even though glm_fp8_emit's simpler keep_f32() happens to FP8-quantize it,
- since it doesn't know the full classify() taxonomy; that mismatch is
- itself a useful case: is_repack_target must exclude by KIND, not by
- guessing at source dtype conventions)."""
+ kinds (attn/o/sh/dmlp/kvb) and SELECTED routed experts (kind "x" --
+ repacked but never stamped), plus PASSTHROUGH (router f32, a norm f32,
+ and embed_tokens io -- all copied byte-identically, no `.qs`, never
+ stamped; see is_passthrough_target).
+
+ Built by hand (not via glm_fp8_emit.save_fp8_safetensors' auto-detection)
+ so each family gets its REAL checkpoint dtype: fp8-repack tensors are
+ genuinely FP8 e4m3 with a block-scale sidecar; passthrough tensors are
+ genuinely raw (embed_tokens as BF16 -- glm_fp8_emit's simpler keep_f32()
+ doesn't know the "io" kind and would otherwise FP8-quantize it, which is
+ not what the real checkpoint does for embed_tokens -- see M1 audit)."""
torch.manual_seed(0)
- sd = {
+ from glm_fp8_emit import fp8_block_quantize
+ raw = {
+ "model.layers.0.input_layernorm.weight": torch.randn(D), # f32 norm: PASSTHROUGH
+ "model.layers.0.mlp.gate.weight": torch.randn(E, D), # router f32: PASSTHROUGH
+ "model.embed_tokens.weight": (torch.randn(64, D) * 0.02).to(torch.bfloat16), # io: PASSTHROUGH, real dtype
+ }
+ quant = {
"model.layers.0.self_attn.o_proj.weight": torch.randn(D, D) * 0.02,
"model.layers.0.self_attn.q_a_proj.weight": torch.randn(D, D) * 0.02,
"model.layers.0.self_attn.kv_b_proj.weight": torch.randn(D, D) * 0.02,
"model.layers.0.mlp.shared_experts.gate_proj.weight": torch.randn(D, I_) * 0.02,
"model.layers.0.mlp.gate_proj.weight": torch.randn(D, I_) * 0.02, # dmlp
- "model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(I_, D) * 0.02, # routed: EXCLUDE
- "model.layers.0.mlp.experts.1.up_proj.weight": torch.randn(I_, D) * 0.02, # routed: EXCLUDE
- "model.layers.0.input_layernorm.weight": torch.randn(D), # f32: EXCLUDE
- "model.layers.0.mlp.gate.weight": torch.randn(E, D), # router f32: EXCLUDE
- "model.embed_tokens.weight": torch.randn(64, D), # io: EXCLUDE
+ "model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(I_, D) * 0.02, # routed: SELECT, unstamped
+ "model.layers.0.mlp.experts.1.up_proj.weight": torch.randn(I_, D) * 0.02, # routed: SELECT, unstamped
}
- n_fp8, n_tot = save_fp8_safetensors(sd, path)
- return sd, n_fp8, n_tot
+ sd = dict(raw)
+ for name, t in quant.items():
+ w_fp8, scale = fp8_block_quantize(t)
+ sd[name] = w_fp8
+ sd[name + "_scale_inv"] = scale
+ save_file(sd, path)
+ n_fp8 = sum(1 for v in sd.values() if v.dtype == torch.float8_e4m3fn)
+ return sd, n_fp8, len(sd)
class SelectionTest(unittest.TestCase):
@@ -88,17 +137,22 @@ class SelectionTest(unittest.TestCase):
def test_dry_run_selection_and_no_writes(self):
inv = rp.shard_inventory(self.shard, n_layers=5)
names = {it["name"] for it in inv}
- self.assertEqual(names, {
- "model.layers.0.self_attn.o_proj.weight",
- "model.layers.0.self_attn.q_a_proj.weight",
- "model.layers.0.mlp.shared_experts.gate_proj.weight",
- "model.layers.0.mlp.gate_proj.weight",
- })
+ self.assertEqual(names, SELECTED | PASSTHROUGH)
kinds = {it["name"]: it["kind"] for it in inv}
self.assertEqual(kinds["model.layers.0.self_attn.o_proj.weight"], "o")
self.assertEqual(kinds["model.layers.0.self_attn.q_a_proj.weight"], "attn")
+ self.assertEqual(kinds["model.layers.0.self_attn.kv_b_proj.weight"], "kvb")
self.assertEqual(kinds["model.layers.0.mlp.shared_experts.gate_proj.weight"], "sh")
self.assertEqual(kinds["model.layers.0.mlp.gate_proj.weight"], "dmlp")
+ self.assertEqual(kinds["model.layers.0.mlp.experts.0.gate_proj.weight"], "x")
+ self.assertEqual(kinds["model.layers.0.input_layernorm.weight"], "f32")
+ self.assertEqual(kinds["model.layers.0.mlp.gate.weight"], "f32")
+ self.assertEqual(kinds["model.embed_tokens.weight"], "io")
+ passthrough = {it["name"]: it["passthrough"] for it in inv}
+ for name in PASSTHROUGH:
+ self.assertTrue(passthrough[name], name)
+ for name in SELECTED:
+ self.assertFalse(passthrough[name], name)
# --dry-run through the CLI must not create the outdir at all
outdir = os.path.join(self.tmp.name, "out_dry")
rc = os.system(
@@ -107,36 +161,92 @@ class SelectionTest(unittest.TestCase):
self.assertEqual(rc, 0)
self.assertFalse(os.path.exists(outdir), "--dry-run must not write anything")
- def test_routed_experts_excluded(self):
- inv = rp.shard_inventory(self.shard, n_layers=5)
+ def test_routed_experts_selected_but_never_stamped(self):
+ """Routed experts are repacked (they are the bulk of the checkpoint's fp8
+ bytes) but must NEVER appear in the stamp map: colibri.c's three
+ routed-expert load sites all call qt_resolve_fmt with stamped_name=NULL,
+ so a stamp there is written and never read -- and st.h's
+ ST_FMT_STAMP_MAX (4096) is sized on exactly that premise, so stamping
+ tens of thousands of them would make the engine refuse the container at
+ discovery time."""
+ out, inv, fmt_map = rp.repack_shard(self.shard, n_layers=5)
names = {it["name"] for it in inv}
- self.assertNotIn("model.layers.0.mlp.experts.0.gate_proj.weight", names)
- self.assertNotIn("model.layers.0.mlp.experts.1.up_proj.weight", names)
-
- def test_kv_b_proj_excluded(self):
- """Regression guard for a gap found in self-review: kv_b_proj is a valid
- resident kind per classify(), but colibri.c's CPU absorb path
- (qt_addrow/qt_matvec_rows) and the CUDA absorb kernels have no fmt=8
- case -- selecting it here would produce a container the engine
- silently misreads as int2. Must stay excluded until that path gains
- fmt=8 support (separate follow-up work)."""
+ self.assertEqual(names, SELECTED | PASSTHROUGH)
+ for name in ROUTED:
+ self.assertIn(name, out, f"{name}: routed expert must be repacked")
+ self.assertIn(name + ".qs", out)
+ self.assertNotIn(name, fmt_map, f"{name}: routed experts must not be stamped")
+ self.assertEqual(set(fmt_map), STAMPED)
+ stamped = {it["name"]: it["stamped"] for it in inv}
+ self.assertEqual({n for n, s in stamped.items() if s}, STAMPED)
+
+ def test_stamp_max_matches_the_c_constant(self):
+ """The writer-side stamp budget is a mirror of st.h's ST_FMT_STAMP_MAX.
+ A mirror that silently drifts is worse than no mirror: pin it to the C
+ header's own text so a change on either side fails here."""
+ st_h = os.path.join(os.path.dirname(__file__), "..", "st.h")
+ with open(st_h) as f:
+ src = f.read()
+ m = re.search(r"#define\s+ST_FMT_STAMP_MAX\s+(\d+)", src)
+ self.assertIsNotNone(m, "ST_FMT_STAMP_MAX not found in c/st.h")
+ self.assertEqual(int(m.group(1)), rp.ST_FMT_STAMP_MAX)
+
+ def test_kv_b_proj_included_and_stamped(self):
+ """FULL-FAMILY EMISSION (M1 audit): kv_b_proj is now repacked the same
+ way o_proj is -- byte-preserved fp8 weight + renamed `.qs`, stamped
+ (it clears the collision shape, same as o_proj -- M1 confirms no new
+ _check_geometry logic is needed). This tool change is independent of
+ the engine-side absorb work landing (separate branch, not this diff);
+ see the module docstring for the sequencing caveat."""
+ name = "model.layers.0.self_attn.kv_b_proj.weight"
+ self.assertIn("kvb", rp.RESIDENT_KINDS)
+ self.assertIn("kvb", rp.STAMPABLE_KINDS)
inv = rp.shard_inventory(self.shard, n_layers=5)
names = {it["name"] for it in inv}
- self.assertNotIn("model.layers.0.self_attn.kv_b_proj.weight", names)
- self.assertNotIn("kvb", rp.RESIDENT_KINDS)
-
- def test_io_and_f32_excluded(self):
+ self.assertIn(name, names)
+ kind = {it["name"]: it["kind"] for it in inv}[name]
+ self.assertEqual(kind, "kvb")
+ out, inv2, fmt_map = rp.repack_shard(self.shard, n_layers=5)
+ self.assertIn(name, out)
+ self.assertIn(name + ".qs", out)
+ self.assertEqual(out[name].dtype, torch.uint8)
+ self.assertEqual(fmt_map.get(name), rp.FORMAT_NAME)
+ with safe_open(self.shard, framework="pt") as f:
+ w_src = f.get_tensor(name).view(torch.uint8)
+ sc_src = f.get_tensor(name + "_scale_inv").reshape(-1)
+ self.assertTrue(torch.equal(out[name], w_src), "kv_b_proj weight bytes not preserved")
+ self.assertTrue(torch.equal(out[name + ".qs"], sc_src), "kv_b_proj scale not preserved")
+
+ def test_io_and_f32_pass_through(self):
+ """FULL-FAMILY EMISSION (M1 audit): embed_tokens/lm_head (io) and
+ norms/router (f32) are now copied byte-identically -- no `.qs`, never
+ stamped, dtype unchanged (BF16 for embed_tokens, F32 for the norm and
+ router in this fixture)."""
inv = rp.shard_inventory(self.shard, n_layers=5)
- names = {it["name"] for it in inv}
- self.assertNotIn("model.embed_tokens.weight", names) # io kind
- self.assertNotIn("model.layers.0.input_layernorm.weight", names) # f32 (also not FP8 dtype)
- self.assertNotIn("model.layers.0.mlp.gate.weight", names) # router, f32 kind
+ by_name = {it["name"]: it for it in inv}
+ for name in PASSTHROUGH:
+ self.assertIn(name, by_name, name)
+ self.assertTrue(by_name[name]["passthrough"], name)
+ self.assertFalse(by_name[name]["stamped"], name)
+ out, inv2, fmt_map = rp.repack_shard(self.shard, n_layers=5)
+ with safe_open(self.shard, framework="pt") as f:
+ for name in PASSTHROUGH:
+ self.assertIn(name, out, name)
+ self.assertNotIn(name + ".qs", out, name)
+ self.assertNotIn(name, fmt_map, name)
+ src = f.get_tensor(name)
+ self.assertEqual(out[name].dtype, src.dtype, name)
+ self.assertTrue(torch.equal(out[name], src), f"{name}: not byte-identical")
+ self.assertEqual(out["model.embed_tokens.weight"].dtype, torch.bfloat16)
+ self.assertEqual(out["model.layers.0.input_layernorm.weight"].dtype, torch.float32)
def test_byte_preservation_and_qs_rename(self):
out, inv, fmt_map = rp.repack_shard(self.shard, n_layers=5)
- self.assertEqual(len(inv), 4)
+ fp8_inv = [it for it in inv if not it["passthrough"]]
+ self.assertEqual(len(fp8_inv), len(SELECTED))
+ self.assertEqual(len(inv), len(SELECTED) + len(PASSTHROUGH))
with safe_open(self.shard, framework="pt") as f:
- for it in inv:
+ for it in fp8_inv:
name = it["name"]
w_src = f.get_tensor(name).view(torch.uint8)
sc_src = f.get_tensor(name + "_scale_inv").reshape(-1)
@@ -147,15 +257,14 @@ class SelectionTest(unittest.TestCase):
nblkO, nblkI = (O + 127) // 128, (I_ + 127) // 128
self.assertEqual(out[name + ".qs"].numel(), nblkO * nblkI)
- def test_fmt_map_covers_exactly_selected_weight_names(self):
+ def test_fmt_map_covers_exactly_the_stampable_weight_names(self):
"""fmt_map (the METADATA STAMP payload) must cover exactly the selected
- WEIGHT names -- never the ".qs" sidecars, never an excluded tensor -- and
- every value must be FORMAT_NAME (the format's public NAME, not the
- internal fmt=8 ordinal)."""
+ STAMPABLE weight names -- never the ".qs" sidecars, never an excluded
+ tensor, never a routed expert -- and every value must be FORMAT_NAME
+ (the format's public NAME, not the internal fmt=8 ordinal)."""
out, inv, fmt_map = rp.repack_shard(self.shard, n_layers=5)
- selected_names = {it["name"] for it in inv}
- self.assertEqual(set(fmt_map.keys()), selected_names)
- for name in selected_names:
+ self.assertEqual(set(fmt_map.keys()), STAMPED)
+ for name in STAMPED:
self.assertNotIn(name + ".qs", fmt_map)
self.assertEqual(fmt_map[name], rp.FORMAT_NAME)
self.assertEqual(rp.FORMAT_NAME, "fp8-e4m3-b128")
@@ -176,33 +285,261 @@ class SelectionTest(unittest.TestCase):
with self.assertRaises(ValueError):
rp.shard_inventory(bad_path, n_layers=5)
- def test_geometry_refusal_on_ambiguous_shape(self):
- """WRITER-SIDE REFUSAL (maintainer review, #528): a WELL-FORMED .qs sidecar
- (correct nblkOxnblkI shape for [O,I], unlike the malformed case above) must
- still be refused when nblkO*nblkI==O -- the exact shape where this tensor's
- block-scale byte count (nblkO*nblkI*4) would coincide with a per-row int8
- scale byte count (O*4). Not hypothetical: GLM-5.2's own
- self_attn.o_proj.weight ([6144,16384], nblkO=48 nblkI=128 product=6144==O)
- is a real instance of this exact family; O=2,I=256 here is the smallest
- realistic analog (nblkO=1, nblkI=2, product=2==O), the same degenerate
- shape family c/tests/test_fp8_load.c's test_disambiguation() sweeps. The
- engine's qt_resolve_fmt reader resolves an unstamped collision at this
- shape to fmt=1 (int8-row) rather than refusing (see colibri.c's
- INVERSION) -- so this tool refusing to ever EMIT an fmt=8 container here
- is what keeps that reader-side resolution correct: an fmt=8 tensor this
- tool produced at this shape would be silently read back as plain int8."""
- amb_path = os.path.join(self.tmp.name, "ambiguous.safetensors")
- O, I_ = 2, 256
+ def _ambiguous_shard(self, tensor_name, O=2, I_=256):
+ """A one-tensor shard at the collision shape with a WELL-FORMED .qs
+ sidecar (correct nblkOxnblkI for [O,I], so it passes the first
+ geometry check and reaches the collision predicate). O=2,I=256 is the
+ smallest realistic analog of GLM-5.2's own self_attn.o_proj.weight
+ ([6144,16384], nblkO=48 nblkI=128 product=6144==O): nblkO=1, nblkI=2,
+ product=2==O -- the same degenerate family c/tests/test_fp8_load.c's
+ test_disambiguation() sweeps."""
+ path = os.path.join(self.tmp.name, "ambiguous.safetensors")
w = torch.randn(O, I_).to(torch.float8_e4m3fn)
nblkO, nblkI = (O + 127) // 128, (I_ + 127) // 128
self.assertEqual(nblkO * nblkI, O, "fixture must actually hit the collision")
- scale = torch.ones(nblkO, nblkI, dtype=torch.float32) # WELL-FORMED for [O,I] -- passes the first check
- save_file({"model.layers.0.self_attn.o_proj.weight": w,
- "model.layers.0.self_attn.o_proj.weight_scale_inv": scale}, amb_path)
+ scale = torch.ones(nblkO, nblkI, dtype=torch.float32)
+ save_file({tensor_name: w, tensor_name + "_scale_inv": scale}, path)
+ return path
+
+ def test_ambiguous_shape_permitted_for_a_stamped_resident(self):
+ """#528's invariant, as the mechanism actually states it: a resident
+ tensor at the collision shape is EMITTED, and carries the stamp that
+ resolves it. qt_resolve_fmt's "THE STAMPED CASE IS UNCHANGED by this
+ inversion" branch (colibri.c) reads such a tensor back as fmt=8 through
+ TRUST-VERIFY-REFUSE. Nothing here relaxes the unstamped rule -- the
+ unstamped polarity is the next test."""
+ name = "model.layers.0.self_attn.o_proj.weight"
+ path = self._ambiguous_shard(name)
+ inv = rp.shard_inventory(path, n_layers=5)
+ self.assertEqual([it["name"] for it in inv], [name])
+ self.assertTrue(inv[0]["ambiguous"])
+ self.assertTrue(inv[0]["stamped"])
+ out, inv2, fmt_map = rp.repack_shard(path, n_layers=5)
+ self.assertEqual(fmt_map, {name: rp.FORMAT_NAME})
+ rp._emission_stamp_gate(out, fmt_map) # must not raise
+
+ def test_ambiguous_shape_still_refused_for_an_unstamped_routed_expert(self):
+ """I-4, the half that must NOT widen. A routed expert at the same shape
+ is refused, because nothing can resolve it on read: colibri.c's
+ routed-expert load sites pass stamped_name=NULL, so the reader's
+ INVERSION resolves the collision to fmt=1 (int8-row) and every weight is
+ silently misread. This is the case the old unconditional refusal was
+ protecting, and it is still refused."""
+ name = "model.layers.0.mlp.experts.3.gate_proj.weight"
+ path = self._ambiguous_shard(name)
+ with self.assertRaises(ValueError) as cm:
+ rp.repack_shard(path, n_layers=5)
+ self.assertIn("AMBIGUOUS", str(cm.exception))
+ self.assertIn("NOT stamp-resolvable", str(cm.exception))
with self.assertRaises(ValueError):
- rp.repack_shard(amb_path, n_layers=5)
+ rp.shard_inventory(path, n_layers=5)
+
+ def test_emission_gate_catches_a_lost_stamp(self):
+ """The structural half of I-4: even if selection produced a tensor at the
+ collision shape and then LOST its stamp, nothing reaches disk. The gate
+ re-derives the predicate from the output tensors' own [O,I], so it does
+ not depend on the selection logic that dropped the stamp. Proof-of-bite:
+ the same `out` passes with the real fmt_map (previous test) and fails
+ with an emptied one."""
+ name = "model.layers.0.self_attn.o_proj.weight"
+ path = self._ambiguous_shard(name)
+ out, inv, fmt_map = rp.repack_shard(path, n_layers=5)
+ with self.assertRaises(ValueError) as cm:
+ rp._emission_stamp_gate(out, {})
+ self.assertIn("no 'fp8-e4m3-b128' stamp", str(cm.exception))
+ self.assertIn("refusing", str(cm.exception))
+
+ def test_stamp_budget_refuses_over_the_c_cap(self):
+ with self.assertRaises(ValueError) as cm:
+ rp._check_stamp_budget(rp.ST_FMT_STAMP_MAX + 1)
+ self.assertIn("ST_FMT_STAMP_MAX", str(cm.exception))
+ rp._check_stamp_budget(rp.ST_FMT_STAMP_MAX) # exactly at the cap is fine
+
+ def test_second_landmine_shape_refused_for_an_unstamped_routed_expert(self):
+ """I=98, single block: the weight-byte count (O*98) and the 4-byte scale
+ array also match an E8/IQ3 (fmt=6) tensor's signature (qt_resolve_fmt's
+ SECOND DESIGN LANDMINE). A stamp naming fp8-e4m3-b128 resolves that
+ read-side; a routed expert has none, and the reader aborts the whole
+ model load. Refuse at write time instead. No Z.ai projection lands at
+ I=98 -- this guards the shape that only became reachable when this tool
+ started emitting unstampable tensors."""
+ path = os.path.join(self.tmp.name, "i98.safetensors")
+ name = "model.layers.0.mlp.experts.3.gate_proj.weight"
+ O, I_ = 64, 98
+ w = torch.randn(O, I_).to(torch.float8_e4m3fn)
+ save_file({name: w,
+ name + "_scale_inv": torch.ones(1, 1, dtype=torch.float32)}, path)
+ with self.assertRaises(ValueError) as cm:
+ rp.repack_shard(path, n_layers=5)
+ self.assertIn("SECOND DESIGN LANDMINE", str(cm.exception))
+
+
+class EmissionGateIndependenceTest(unittest.TestCase):
+ """FIX ROUND 3 (arbitration of the I-4 conflict between the blind validator
+ and the deep auditor, settled by execution).
+
+ Both reviewers exercised _emission_stamp_gate, but only ever with a RESIDENT
+ name; the validator's routed-expert attack went through repack_shard and was
+ refused one function earlier, by _check_geometry. The cell neither ran --
+ ROUTED name, stamp PRESENT -- was PERMITTED, because the gate was structural
+ on SHAPE but procedural on STAMPABILITY: it answered "will the reader consult
+ a stamp for this name?" by consulting the very map _stampable() had built, so
+ it agreed with itself. This class pins the whole 2x2 plus the cases the fix
+ could plausibly have broken, at BOTH collision shapes the reviewers used.
+
+ `out` is built here the way repack_shard builds it (uint8 [O,I] weight + flat
+ F32 .qs) deliberately WITHOUT going through repack_shard: the point of the
+ gate is to hold when the selection path is wrong."""
+
+ RESIDENT = "model.layers.0.self_attn.o_proj.weight"
+ ROUTED = "model.layers.0.mlp.experts.3.gate_proj.weight"
+ ODD = "model.layers.0.self_attn.some_new_proj.weight"
+ MTP = "model.layers.78.self_attn.o_proj.weight"
+
+ def _out(self, name, O, I_, nscale=None):
+ n = ((O + 127) // 128) * ((I_ + 127) // 128) if nscale is None else nscale
+ return {name: torch.zeros(O, I_, dtype=torch.uint8),
+ name + ".qs": torch.ones(n, dtype=torch.float32)}
+
+ def _permitted(self, out, fmt_map):
+ try:
+ rp._emission_stamp_gate(out, fmt_map)
+ return True
+ except ValueError:
+ return False
+
+ def test_the_2x2_over_name_kind_and_stamp_state(self):
+ """The decisive matrix. ROUTED/STAMPED is the cell that used to leak: a
+ stamp on a routed expert is written into __metadata__ and NEVER read
+ (colibri.c:2414/:2612/:2811 all pass stamped_name=NULL), so at the
+ collision shape qt_resolve_fmt's INVERSION resolves it to fmt=1 and every
+ weight in that expert is silently misread as int8. Shapes: [2,256] (the
+ auditor's) and [2,200] (the validator's) -- both genuine collisions
+ (nblkO*nblkI == O), the same family as GLM-5.2's o_proj [6144,16384]."""
+ want = {("RESIDENT", "UNSTAMPED"): False, ("RESIDENT", "STAMPED"): True,
+ ("ROUTED", "UNSTAMPED"): False, ("ROUTED", "STAMPED"): False}
+ for I_ in (256, 200):
+ for kind, name in (("RESIDENT", self.RESIDENT), ("ROUTED", self.ROUTED)):
+ for state in ("UNSTAMPED", "STAMPED"):
+ fmt_map = {name: rp.FORMAT_NAME} if state == "STAMPED" else {}
+ self.assertEqual(
+ self._permitted(self._out(name, 2, I_), fmt_map),
+ want[(kind, state)],
+ f"{kind}/{state} at [2,{I_}]")
+
+ def test_routed_refusal_names_the_unreadable_stamp_condition(self):
+ name = self.ROUTED
+ with self.assertRaises(ValueError) as cm:
+ rp._emission_stamp_gate(self._out(name, 2, 256), {name: rp.FORMAT_NAME})
+ self.assertIn("ROUTED EXPERT", str(cm.exception))
+ self.assertIn("stamped_name=NULL", str(cm.exception))
+
+ def test_the_three_legitimate_cells_are_unchanged(self):
+ """The regressions the routed test could have caused. A non-colliding
+ routed expert is the BULK of what this tool emits and must still be
+ permitted unstamped."""
+ self.assertTrue(self._permitted(
+ self._out(self.RESIDENT, 2, 256), {self.RESIDENT: rp.FORMAT_NAME}))
+ self.assertTrue(self._permitted(
+ self._out(self.RESIDENT, 2, 200), {self.RESIDENT: rp.FORMAT_NAME}))
+ self.assertTrue(self._permitted(self._out(self.ROUTED, 512, 1024), {}))
+
+ def test_mtp_layer78_resident_at_the_collision_shape_still_permitted(self):
+ """THE REGRESSION THE OTHER CANDIDATE FIX WOULD HAVE CAUSED, pinned.
+
+ Re-deriving stampability as _stampable(classify(name, n_layers)) needs
+ keep_mtp as well as n_layers: classify(l78 o_proj, 78, keep_mtp=False) is
+ "skip", so a version that plumbed n_layers alone falsely REFUSES the --mtp
+ pass's legitimate layer-78 residents -- and GLM-5.2's o_proj [6144,16384]
+ IS at the collision shape (48*128 == 6144), so that would stop the mint on
+ the real checkpoint. The name-derived test has no such dependency; this
+ pins that it does not acquire one."""
+ for O, I_ in ((2, 256), (2, 200), (6144, 16384)):
+ self.assertTrue(
+ self._permitted(self._out(self.MTP, O, I_), {self.MTP: rp.FORMAT_NAME}),
+ f"--mtp layer-78 o_proj at [{O},{I_}] must still be permitted")
+ # and the same tensor unstamped is still refused
+ self.assertFalse(self._permitted(self._out(self.MTP, 6144, 16384), {}))
+
+ def test_second_landmine_covered_by_the_emission_gate_too(self):
+ """I==98 single-block: _check_geometry covered this (SELECTION side) and
+ the emission gate did not -- it tested only nblkO*nblkI == O, and 1 != 64.
+ The reader ABORTS the whole model load on an unstamped one
+ (qt_resolve_fmt's fp8_blk_f32_also branch), so this is the same omission
+ as the routed cell, seen a second time."""
+ self.assertFalse(self._permitted(self._out(self.ROUTED, 64, 98, nscale=1), {}))
+ self.assertFalse(self._permitted(self._out(self.ROUTED, 64, 98, nscale=1),
+ {self.ROUTED: rp.FORMAT_NAME}))
+ with self.assertRaises(ValueError) as cm:
+ rp._emission_stamp_gate(self._out(self.RESIDENT, 64, 98, nscale=1), {})
+ self.assertIn("SECOND DESIGN LANDMINE", str(cm.exception))
+ # a stamped RESIDENT at that shape resolves on read, so it is permitted
+ self.assertTrue(self._permitted(self._out(self.RESIDENT, 64, 98, nscale=1),
+ {self.RESIDENT: rp.FORMAT_NAME}))
+
+ def test_gate_predicate_is_the_EMITTED_qs_length(self):
+ """The gate's collision predicate is len(.qs) == O -- what the reader
+ actually counts (ns == O*4 bytes, colibri.c) -- not _nblk(O)*_nblk(I) == O,
+ which coincides with it only because _check_geometry already pinned the
+ block shape. That pinning is the selection-side assumption this gate exists
+ NOT to depend on. [4,256] has _nblk product 2 != O=4, so the old predicate
+ would not fire; a .qs of length 4 makes the READER see a collision, and
+ this gate now refuses it."""
+ out = self._out(self.RESIDENT, 4, 256, nscale=4)
+ self.assertNotEqual(((4 + 127) // 128) * ((256 + 127) // 128), 4)
+ self.assertFalse(self._permitted(out, {}))
+ self.assertTrue(self._permitted(out, {self.RESIDENT: rp.FORMAT_NAME}))
+ # converse: _nblk product == O but the emitted .qs does NOT collide on
+ # read -- the reader sees ns=12 != O*4=8, so refusing would be wrong.
+ self.assertTrue(self._permitted(self._out(self.RESIDENT, 2, 256, nscale=3), {}))
+
+ def test_gate_skips_raw_passthrough_tensors_including_1d_norms(self):
+ """M1's fix-shape requirement, pinned directly: a 1-D raw pass-through
+ tensor (a norm) in `out` must not trip the 2-D collision-shape
+ assumption the gate applies to fp8 weights -- it must be recognized as
+ non-fp8 and skipped, whatever its shape. Regression target: before
+ this fix, `len(t.shape) != 2` fired unconditionally on every non-.qs
+ entry, so ANY 1-D norm added to `out` made a real mint run crash."""
+ out = {
+ "model.norm.weight": torch.randn(256), # 1-D F32 norm
+ "model.layers.0.self_attn.q_a_layernorm.weight": torch.randn(64), # 1-D F32 norm
+ "model.embed_tokens.weight": torch.randn(64, 256).to(torch.bfloat16), # 2-D BF16, no .qs
+ self.RESIDENT: torch.zeros(2, 256, dtype=torch.uint8),
+ self.RESIDENT + ".qs": torch.ones(2, dtype=torch.float32),
+ }
+ fmt_map = {self.RESIDENT: rp.FORMAT_NAME}
+ rp._emission_stamp_gate(out, fmt_map) # must not raise
+
+ def test_gate_refuses_a_weight_with_no_qs_sidecar(self):
+ """fmt=8 is resolved from the scale byte count; a weight emitted without
+ its scale array is unresolvable however it is stamped."""
+ with self.assertRaises(ValueError) as cm:
+ rp._emission_stamp_gate({self.RESIDENT: torch.zeros(2, 256, dtype=torch.uint8)},
+ {self.RESIDENT: rp.FORMAT_NAME})
+ self.assertIn(".qs", str(cm.exception))
+
+ def test_catch_all_kind_is_repacked_but_not_stampable(self):
+ """classify()'s "q" is a fallback for ANY unmatched name ending .weight,
+ so a stamp on such a tensor would rest on a fallback rather than on an
+ audited colibri.c load site. It stays a repack target (it is fp8 and
+ belongs in the container) but is not stampable, which makes both gates
+ refuse it at the collision shape instead of emitting it under an
+ unverified stamp. The kinds that DO have audited load sites are
+ unaffected -- that is the half that must not regress."""
+ self.assertEqual(classify(self.ODD, 78), "q")
+ self.assertIn("q", rp.REPACK_KINDS)
+ self.assertNotIn("q", rp.STAMPABLE_KINDS)
+ self.assertFalse(rp._stampable("q"))
+ self.assertEqual(rp.STAMPABLE_KINDS, frozenset({"sh", "o", "attn", "dmlp", "kvb"}))
+ for name in ("model.layers.0.self_attn.o_proj.weight",
+ "model.layers.0.self_attn.q_a_proj.weight",
+ "model.layers.0.mlp.shared_experts.gate_proj.weight",
+ "model.layers.0.mlp.gate_proj.weight"):
+ self.assertTrue(rp._stampable(classify(name, 78)), name)
with self.assertRaises(ValueError):
- rp.shard_inventory(amb_path, n_layers=5)
+ rp._check_geometry(self.ODD, 2, 256, 1, 2, rp._stampable(classify(self.ODD, 78)))
+ rp._check_geometry(self.RESIDENT, 2, 256, 1, 2,
+ rp._stampable(classify(self.RESIDENT, 78))) # must not raise
class ResumeAndParamsGuardTest(unittest.TestCase):
@@ -212,15 +549,27 @@ class ResumeAndParamsGuardTest(unittest.TestCase):
os.makedirs(self.indir)
self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")
_make_fixture(self.shard)
+ # M1 audit / FULL-FAMILY EMISSION: config.json AND tokenizer.json are
+ # now MANDATORY in --indir (main() refuses to run without either --
+ # see STANDALONE_LOAD_REQUIRED_FILES / _copy_standalone_load_files).
+ # Every subprocess CLI run in this class needs both present; content
+ # is a placeholder since this class never loads the mint output
+ # through the tokenizer, only inspects the emitted safetensors header.
+ with open(os.path.join(self.indir, "config.json"), "w") as fh:
+ json.dump({"model_type": "glm_moe_dsa_test_fixture"}, fh)
+ with open(os.path.join(self.indir, "tokenizer.json"), "w") as fh:
+ json.dump({"model": {"vocab": {"<|endoftext|>": 0}, "merges": []}}, fh)
self.outdir = os.path.join(self.tmp.name, "out")
self.tool = os.path.join(os.path.dirname(__file__), "..", "tools", "repack_fp8_passthrough.py")
def tearDown(self):
self.tmp.cleanup()
- def _run(self, n_layers):
- return os.system(f'{sys.executable} "{self.tool}" --indir "{self.indir}" '
- f'--outdir "{self.outdir}" --n-layers {n_layers} >/dev/null 2>&1')
+ def _run(self, n_layers, extra=()):
+ return subprocess.run(
+ [sys.executable, self.tool, "--indir", self.indir, "--outdir", self.outdir,
+ "--n-layers", str(n_layers), *extra],
+ capture_output=True, text=True).returncode
def test_output_container_geometry(self):
self.assertEqual(self._run(5), 0)
@@ -234,17 +583,33 @@ class ResumeAndParamsGuardTest(unittest.TestCase):
w = hdr["model.layers.0.self_attn.o_proj.weight"]
self.assertEqual(w["dtype"], "U8")
self.assertEqual(w["shape"], [256, 256])
- # experts / io / f32 / kv_b_proj must be absent from the output container entirely
- self.assertNotIn("model.layers.0.mlp.experts.0.gate_proj.weight", hdr)
- self.assertNotIn("model.embed_tokens.weight", hdr)
- self.assertNotIn("model.layers.0.input_layernorm.weight", hdr)
- self.assertNotIn("model.layers.0.self_attn.kv_b_proj.weight", hdr)
+ # routed experts are now IN the container (byte-preserved, with their .qs)
+ for name in ROUTED:
+ self.assertIn(name, hdr)
+ self.assertEqual(hdr[name]["dtype"], "U8")
+ self.assertIn(name + ".qs", hdr)
+ # kv_b_proj: FULL-FAMILY EMISSION -- now IN the container, fp8-repacked
+ # exactly like o_proj (byte-preserved, stamped, with its .qs).
+ kvb = hdr["model.layers.0.self_attn.kv_b_proj.weight"]
+ self.assertEqual(kvb["dtype"], "U8")
+ self.assertIn("model.layers.0.self_attn.kv_b_proj.weight.qs", hdr)
+ # io / f32: FULL-FAMILY EMISSION -- now IN the container, raw dtype
+ # unchanged, no .qs sidecar (pass-through, not fp8-repack).
+ embed = hdr["model.embed_tokens.weight"]
+ self.assertEqual(embed["dtype"], "BF16")
+ self.assertNotIn("model.embed_tokens.weight.qs", hdr)
+ norm = hdr["model.layers.0.input_layernorm.weight"]
+ self.assertEqual(norm["dtype"], "F32")
+ self.assertNotIn("model.layers.0.input_layernorm.weight.qs", hdr)
+ # config.json must have landed in --outdir (M1 standalone-load checklist)
+ self.assertTrue(os.path.exists(os.path.join(self.outdir, "config.json")))
def test_metadata_stamp_present_and_correct(self):
"""End-to-end through the real CLI: __metadata__["colibri.fmt"] must be
- present, parse as JSON, and name exactly the selected resident-kind
- tensors -- the real round trip a unit-level rp.repack_shard() call
- can't exercise (save_file's own metadata= handling)."""
+ present, parse as JSON, and name exactly the selected STAMPABLE tensors
+ -- routed experts are in the container but must not be in the stamp map
+ -- the real round trip a unit-level rp.repack_shard() call can't
+ exercise (save_file's own metadata= handling)."""
self.assertEqual(self._run(5), 0)
outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
self.assertEqual(len(outs), 1)
@@ -252,14 +617,114 @@ class ResumeAndParamsGuardTest(unittest.TestCase):
self.assertIn("__metadata__", hdr)
self.assertIn("colibri.fmt", hdr["__metadata__"])
stamp = json.loads(hdr["__metadata__"]["colibri.fmt"])
- self.assertEqual(set(stamp.keys()), {
- "model.layers.0.self_attn.o_proj.weight",
- "model.layers.0.self_attn.q_a_proj.weight",
- "model.layers.0.mlp.shared_experts.gate_proj.weight",
- "model.layers.0.mlp.gate_proj.weight",
- })
+ self.assertEqual(set(stamp.keys()), STAMPED)
for name, fmt_name in stamp.items():
self.assertEqual(fmt_name, rp.FORMAT_NAME, f"{name}: stamped format name mismatch")
+ for name in PASSTHROUGH:
+ self.assertNotIn(name, stamp, f"{name}: raw pass-through tensor must never be stamped")
+
+ def test_container_format_declaration_present(self):
+ """The container-level declaration (U-6a writer half): a JSON array of
+ format NAME strings, on every emitted shard. Inert today -- no engine
+ reads it -- so what this pins is the KEY and the VALUE SHAPE a later
+ engine-side check will consume, plus the rule that it names the format
+ by NAME and never by ordinal."""
+ self.assertEqual(self._run(5), 0)
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ self.assertTrue(outs)
+ for path in outs:
+ meta = _read_header(path)["__metadata__"]
+ self.assertIn(rp.CONTAINER_FORMATS_KEY, meta)
+ self.assertEqual(rp.CONTAINER_FORMATS_KEY, "colibri.container.formats")
+ declared = json.loads(meta[rp.CONTAINER_FORMATS_KEY])
+ self.assertEqual(declared, ["fp8-e4m3-b128"])
+ self.assertTrue(all(isinstance(v, str) for v in meta.values()),
+ "safetensors __metadata__ values must all be strings")
+
+ def test_repeated_mint_is_byte_identical(self):
+ """Regression test for D1's diagnosis
+ (colibri_lab/reports/D1_MINT_DIVERGENCE_2026-08-18.md): save_file's
+ Rust extension does not preserve `__metadata__`'s key order, so two
+ independent mints of the SAME checkpoint used to be able to disagree
+ byte-for-byte on header bytes alone -- identical tensor payloads,
+ different header bytes -- exactly the false-positive D1's cross-box
+ SHA-256 sweep hit. D1 read existing artifacts across two DIFFERENT
+ boxes and could not itself prove repeat-run stability on one box;
+ this closes that gap: two FRESH `python3 repack_fp8_passthrough.py`
+ subprocesses, same input, different --outdir, so no in-process state
+ (e.g. a warm hash seed) can launder the result -- their output
+ shard(s) must be byte-identical, headers included.
+
+ Also pins the MECHANISM directly rather than only the outcome: a
+ header whose `__metadata__` key order is already canonical (sorted)
+ is byte-identical to a re-mint DETERMINISTICALLY, not by the 50/50
+ luck a raw two-run comparison alone would rely on -- see
+ _canonicalize_metadata_order in repack_fp8_passthrough.py."""
+ outdir_a = os.path.join(self.tmp.name, "out_det_a")
+ outdir_b = os.path.join(self.tmp.name, "out_det_b")
+ for outdir in (outdir_a, outdir_b):
+ proc = subprocess.run(
+ [sys.executable, self.tool, "--indir", self.indir, "--outdir", outdir,
+ "--n-layers", "5"],
+ capture_output=True, text=True)
+ self.assertEqual(proc.returncode, 0, proc.stderr)
+ outs_a = sorted(glob.glob(os.path.join(outdir_a, "out-fp8pass-*.safetensors")))
+ outs_b = sorted(glob.glob(os.path.join(outdir_b, "out-fp8pass-*.safetensors")))
+ self.assertTrue(outs_a, "mint A produced no shard files")
+ self.assertEqual([os.path.basename(p) for p in outs_a],
+ [os.path.basename(p) for p in outs_b])
+ for pa, pb in zip(outs_a, outs_b):
+ with open(pa, "rb") as fa, open(pb, "rb") as fb:
+ ba, bb = fa.read(), fb.read()
+ self.assertEqual(
+ ba, bb,
+ f"{os.path.basename(pa)}: byte mismatch between two fresh mints of "
+ f"the identical input -- header-determinism regression")
+ hdr = _read_header(pa)
+ self.assertIn("__metadata__", hdr)
+ meta_keys = list(hdr["__metadata__"].keys())
+ self.assertEqual(
+ meta_keys, sorted(meta_keys),
+ f"{os.path.basename(pa)}: __metadata__ key order is not canonical "
+ f"-- _canonicalize_metadata_order did not run or did not take effect")
+
+ def test_mtp_pass_selects_only_the_mtp_layer(self):
+ """I-5: layer n_layers (the multi-token-prediction head) is reachable, via
+ a second pass into the SAME outdir with its own out-mtp-fp8pass- prefix,
+ its own resume manifest and its own #383-class params sidecar -- so the
+ two passes cannot be mistaken for resumes of each other. The engine
+ detects MTP by tensor NAME (colibri.c's has_mtp probe over
+ model.layers.<n_layers>.*), not by filename."""
+ mtp_shard = os.path.join(self.indir, "model-00002-of-00002.safetensors")
+ torch.manual_seed(11)
+ save_fp8_safetensors({
+ "model.layers.5.self_attn.o_proj.weight": torch.randn(256, 256) * 0.02,
+ "model.layers.5.mlp.experts.0.gate_proj.weight": torch.randn(384, 256) * 0.02,
+ "model.layers.5.eh_proj.weight": torch.randn(256, 512) * 0.02,
+ }, mtp_shard)
+ # the MAIN pass must skip layer 5 entirely (li >= n_layers -> "skip")
+ self.assertEqual(self._run(5), 0)
+ main_hdr = {}
+ for p in glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors")):
+ main_hdr.update(_read_header(p))
+ self.assertNotIn("model.layers.5.self_attn.o_proj.weight", main_hdr)
+ # the MTP pass selects layer 5 and ONLY layer 5
+ self.assertEqual(self._run(5, ["--mtp"]), 0)
+ outs = glob.glob(os.path.join(self.outdir, "out-mtp-fp8pass-*.safetensors"))
+ self.assertEqual(len(outs), 1)
+ hdr = _read_header(outs[0])
+ self.assertIn("model.layers.5.self_attn.o_proj.weight", hdr)
+ self.assertIn("model.layers.5.mlp.experts.0.gate_proj.weight", hdr)
+ self.assertIn("model.layers.5.eh_proj.weight", hdr)
+ self.assertNotIn("model.layers.0.self_attn.o_proj.weight", hdr)
+ stamp = json.loads(hdr["__metadata__"]["colibri.fmt"])
+ self.assertIn("model.layers.5.self_attn.o_proj.weight", stamp)
+ self.assertNotIn("model.layers.5.mlp.experts.0.gate_proj.weight", stamp)
+ # both passes coexist in one outdir: separate manifests, separate guards
+ self.assertTrue(os.path.exists(os.path.join(self.outdir, ".fp8pass-progress.json")))
+ self.assertTrue(os.path.exists(os.path.join(self.outdir, ".fp8pass-mtp-progress.json")))
+ self.assertTrue(os.path.exists(os.path.join(self.outdir, ".fp8pass-params.json")))
+ self.assertTrue(os.path.exists(os.path.join(self.outdir, ".fp8pass-mtp-params.json")))
def test_resume_skips_completed_shard(self):
self.assertEqual(self._run(5), 0)
@@ -269,6 +734,35 @@ class ResumeAndParamsGuardTest(unittest.TestCase):
self.assertEqual(len(outs), 1, "resume must not duplicate output shards")
self.assertEqual(os.path.getmtime(outs[0]), mtime1, "resume must not rewrite a completed shard")
+ def test_stamp_budget_is_container_wide_across_the_mtp_split(self):
+ """FIX ROUND 3: the main pass and the --mtp pass write into ONE --outdir
+ and therefore ONE container, but keep SEPARATE progress manifests. The
+ budget they enforce is st.h's ST_FMT_STAMP_MAX, which st_fmt_stamp_ingest
+ accumulates over EVERY shard in the directory -- so counted per-manifest,
+ two passes could each stay under the cap and still hand the engine a
+ container over it (the writer guarantee would be 2x the reader's bound).
+ The comment claimed container-wide; now the code is.
+
+ Simulated by planting an --mtp manifest already at the cap, then running
+ the main pass into the same outdir: it must refuse."""
+ self.assertEqual(rp.PROGRESS_MANIFESTS,
+ (".fp8pass-progress.json", ".fp8pass-mtp-progress.json"))
+ os.makedirs(self.outdir, exist_ok=True)
+ with open(os.path.join(self.outdir, ".fp8pass-mtp-progress.json"), "w") as fh:
+ json.dump({"shards": {}, "stamped": rp.ST_FMT_STAMP_MAX}, fh)
+ self.assertEqual(
+ rp._stamps_from_other_passes(self.outdir, ".fp8pass-progress.json"),
+ rp.ST_FMT_STAMP_MAX)
+ proc = subprocess.run(
+ [sys.executable, self.tool, "--indir", self.indir, "--outdir", self.outdir,
+ "--n-layers", "5"], capture_output=True, text=True)
+ self.assertNotEqual(proc.returncode, 0,
+ "stamps from the --mtp pass must count toward the cap")
+ self.assertIn("ST_FMT_STAMP_MAX", proc.stderr)
+ # and a pass counts only the OTHER manifests, never its own twice
+ self.assertEqual(
+ rp._stamps_from_other_passes(self.outdir, ".fp8pass-mtp-progress.json"), 0)
+
def test_params_mismatch_refused(self):
self.assertEqual(self._run(5), 0)
rc = self._run(78) # different n_layers, same outdir
@@ -279,16 +773,23 @@ class ResumeAndParamsGuardTest(unittest.TestCase):
def _make_zero_target_fixture(path, D=256, I_=384):
- """A shard with real FP8 tensors, but NONE matching a repack-target kind --
- mirrors the trial's own scenario (a non-resident-named FP8 tensor selects
- nothing): one routed expert (kind "x", explicitly excluded) and one f32
- norm (never a repack candidate at all)."""
+ """A shard with real tensor content, but NONE matching a repack-target OR
+ passthrough-target kind -- mirrors the trial's own zero-target scenario.
+ FULL-FAMILY EMISSION widened both is_repack_target (kv_b_proj now
+ included) and added is_passthrough_target (io/f32 now included), so
+ neither of those kinds is "zero targets" anymore -- a DSA indexer weight
+ is: classify() returns "skip" for any name containing "indexer" on the
+ main pass, regardless of dtype or n_layers, and indexer tensors are
+ explicitly out of this tool's scope (separate follow-on, D-FP8-3 -- M1
+ audit)."""
torch.manual_seed(2)
- sd = {
- "model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(I_, D) * 0.02, # routed: EXCLUDE
- "model.layers.0.input_layernorm.weight": torch.randn(D), # f32: EXCLUDE
- }
- save_fp8_safetensors(sd, path)
+ from glm_fp8_emit import fp8_block_quantize
+ w = torch.randn(I_, D) * 0.02
+ w_fp8, scale = fp8_block_quantize(w)
+ save_file({
+ "model.layers.0.self_attn.indexer.wq_b.weight": w_fp8,
+ "model.layers.0.self_attn.indexer.wq_b.weight_scale_inv": scale,
+ }, path)
class ZeroTargetRefusalTest(unittest.TestCase):
@@ -302,6 +803,12 @@ class ZeroTargetRefusalTest(unittest.TestCase):
os.makedirs(self.indir)
self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")
_make_zero_target_fixture(self.shard)
+ # See ResumeAndParamsGuardTest.setUp: config.json and tokenizer.json
+ # are both MANDATORY in --indir now.
+ with open(os.path.join(self.indir, "config.json"), "w") as fh:
+ json.dump({"model_type": "glm_moe_dsa_test_fixture"}, fh)
+ with open(os.path.join(self.indir, "tokenizer.json"), "w") as fh:
+ json.dump({"model": {"vocab": {"<|endoftext|>": 0}, "merges": []}}, fh)
self.outdir = os.path.join(self.tmp.name, "out")
self.tool = os.path.join(os.path.dirname(__file__), "..", "tools", "repack_fp8_passthrough.py")
diff --git a/c/tests/test_fp8_repack_full_family.py b/c/tests/test_fp8_repack_full_family.py
new file mode 100644
index 00000000..8db984d7
--- /dev/null
+++ b/c/tests/test_fp8_repack_full_family.py
@@ -0,0 +1,376 @@
+"""tools/repack_fp8_passthrough.py: FULL-FAMILY EMISSION coverage tests.
+
+Dispatch: builder B3, task M1_MINT_COVERAGE_AUDIT_2026-08-18 /
+fp8_integration_gate_v0_1.md item 6 ("the mint must emit ALL families ...
+so the output is standalone-loadable"). Synthetic fixtures ONLY -- no real
+Z.ai shard is read or written by this suite, per the tool's own hard
+constraint (see repack_fp8_passthrough.py's module docstring) and per this
+task's own scope limit: no read of the real 755 GB checkpoint, no model
+decode.
+
+Builds ONE small synthetic checkpoint directory covering every tensor family
+M1's own standalone-load checklist names (M1 audit section 3, quoting
+colibri.c's model_init req[] array, colibri.c:1843-1922):
+ - model-level: model.embed_tokens.weight, lm_head.weight, model.norm.weight
+ - a DENSE layer (layer 0): input_layernorm, post_attention_layernorm,
+ self_attn.{q_a_proj,q_a_layernorm,q_b_proj,kv_a_proj_with_mqa,
+ kv_a_layernorm,kv_b_proj,o_proj}.weight, mlp.{gate_proj,up_proj,
+ down_proj}.weight
+ - a SPARSE/MoE layer (layer 1): the same norm/self_attn set, plus
+ mlp.gate.weight (router), mlp.gate.e_score_correction_bias,
+ mlp.shared_experts.{gate_proj,up_proj,down_proj}.weight, and
+ mlp.experts.<e>.{gate_proj,up_proj,down_proj}.weight per expert
+ - a DSA indexer tensor and a beyond-n_layers ("MTP-shaped") tensor, both
+ of which must be EXCLUDED (M1: separately, safely gated -- out of this
+ audit's family list) -- included here as a negative control so the
+ "emitted set == M1's checklist" assertion is an EXACT match, not a
+ superset check that would miss over-selection.
+
+Also builds config.json, tokenizer.json (both MANDATORY -- F3) and
+generation_config.json (best-effort) in --indir, so the same fixture covers
+both the tensor-family checklist and the standalone-load aux-file checklist.
+
+Runs the real CLI end to end (subprocess, --n-layers 2, no --mtp) and checks,
+per the task's evidence bar:
+ 1. every family in M1's checklist is emitted, name-by-name (exact set
+ equality against the checklist, built independently of the tool's own
+ classify() so a shared bug in both wouldn't hide);
+ 2. byte-identity of BOTH pass-through payloads (fp8-repack AND raw
+ BF16/F32) via a SHA-256 hash of input bytes vs output bytes;
+ 3. a 1-D norm (several: model.norm.weight, per-layer input_layernorm,
+ self_attn.q_a_layernorm) passes _emission_stamp_gate without crashing
+ the run (the M1 fix-shape requirement);
+ 4. config.json and tokenizer.json (both MANDATORY) and generation_config.json
+ (best-effort) land in --outdir, byte-identical to --indir;
+ 5. kv_b_proj is stamped fp8; router/norms/embed/lm_head are present but
+ carry no stamp and no `.qs`.
+
+F3 (2026-08-18, colibri_lab/reports/F3_MINT_AUX_FILES_2026-08-18.md): V2's
+end-to-end smoke found the full mint emitted config.json but not
+tokenizer.json, so the loaded-standalone claim was false -- the engine's
+tok_load (c/tok.h:101, called from colibri.c:7294/7952/8134) hard-exits
+without it, exactly like load_cfg does without config.json. tokenizer.json
+joins config.json in STANDALONE_LOAD_REQUIRED_FILES; every other non-tensor
+file in a real GLM-5.2 checkpoint (tokenizer_config.json, chat_template.jinja,
+model.safetensors.index.json, README.md, LICENSE, .gitattributes) is verified
+NOT read anywhere in colibri.c / c/*.h / openai_server.py and stays excluded
+-- see the comment block above STANDALONE_LOAD_REQUIRED_FILES in
+repack_fp8_passthrough.py for the file:line evidence per file.
+"""
+import glob, hashlib, json, os, struct, subprocess, sys, tempfile, unittest
+
+try:
+ import torch
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+except ImportError as e:
+ raise unittest.SkipTest(f"torch/safetensors not installed: {e}")
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+from glm_fp8_emit import fp8_block_quantize
+import repack_fp8_passthrough as rp
+
+TOOL = os.path.join(os.path.dirname(__file__), "..", "tools", "repack_fp8_passthrough.py")
+
+# ---- fixture geometry (small, arbitrary, internally consistent) ----------
+D, DQA, DQB, DKVA, DKVB, I_, V, E = 32, 16, 24, 12, 20, 48, 50, 2
+N_LAYERS = 2 # layer 0 = dense, layer 1 = sparse/MoE; both < N_LAYERS ("main")
+
+
+def _sha256(t):
+ """Dtype-agnostic byte hash: view(uint8) is a bit-level reinterpret, the
+ same trick repack_shard itself uses for fp8 weights, so it works
+ identically for BF16/F32 raw tensors and FP8 weights alike."""
+ return hashlib.sha256(t.contiguous().view(torch.uint8).numpy().tobytes()).hexdigest()
+
+
+def _attn_block(prefix):
+ """self_attn.{q_a_proj,q_a_layernorm,q_b_proj,kv_a_proj_with_mqa,
+ kv_a_layernorm,kv_b_proj,o_proj} -- identical set for dense and sparse
+ layers per M1's checklist. Returns (raw_dict, quant_dict)."""
+ raw = {
+ f"{prefix}.self_attn.q_a_layernorm.weight": torch.randn(DQA).to(torch.bfloat16),
+ f"{prefix}.self_attn.kv_a_layernorm.weight": torch.randn(DKVA).to(torch.bfloat16),
+ }
+ quant = {
+ f"{prefix}.self_attn.q_a_proj.weight": torch.randn(DQA, D) * 0.02,
+ f"{prefix}.self_attn.q_b_proj.weight": torch.randn(DQB, DQA) * 0.02,
+ f"{prefix}.self_attn.kv_a_proj_with_mqa.weight": torch.randn(DKVA, D) * 0.02,
+ f"{prefix}.self_attn.kv_b_proj.weight": torch.randn(DKVB, DKVA) * 0.02,
+ f"{prefix}.self_attn.o_proj.weight": torch.randn(D, DQB) * 0.02,
+ }
+ return raw, quant
+
+
+def _make_checkpoint(indir):
+ """Writes one shard covering every family in M1's checklist (dense layer
+ 0, sparse layer 1, model-level tensors), PLUS two names that must be
+ EXCLUDED (indexer, beyond-n_layers) as a negative control, PLUS
+ config.json, tokenizer.json, generation_config.json (the external files
+ the mint must reproduce in --outdir for the container to be standalone-
+ loadable -- config.json and tokenizer.json mandatory, generation_config
+ .json best-effort)."""
+ torch.manual_seed(7)
+ raw = {
+ "model.embed_tokens.weight": torch.randn(V, D).to(torch.bfloat16),
+ "lm_head.weight": torch.randn(V, D).to(torch.bfloat16),
+ "model.norm.weight": torch.randn(D).to(torch.bfloat16), # 1-D
+ "model.layers.0.input_layernorm.weight": torch.randn(D).to(torch.bfloat16),
+ "model.layers.0.post_attention_layernorm.weight": torch.randn(D).to(torch.bfloat16),
+ "model.layers.1.input_layernorm.weight": torch.randn(D).to(torch.bfloat16),
+ "model.layers.1.post_attention_layernorm.weight": torch.randn(D).to(torch.bfloat16),
+ "model.layers.1.mlp.gate.weight": torch.randn(E, D).to(torch.bfloat16), # router
+ "model.layers.1.mlp.gate.e_score_correction_bias": torch.randn(E), # F32 bias
+ }
+ quant = {
+ # dense layer 0 MLP
+ "model.layers.0.mlp.gate_proj.weight": torch.randn(I_, D) * 0.02,
+ "model.layers.0.mlp.up_proj.weight": torch.randn(I_, D) * 0.02,
+ "model.layers.0.mlp.down_proj.weight": torch.randn(D, I_) * 0.02,
+ # sparse layer 1: shared expert + routed experts
+ "model.layers.1.mlp.shared_experts.gate_proj.weight": torch.randn(I_, D) * 0.02,
+ "model.layers.1.mlp.shared_experts.up_proj.weight": torch.randn(I_, D) * 0.02,
+ "model.layers.1.mlp.shared_experts.down_proj.weight": torch.randn(D, I_) * 0.02,
+ }
+ for e in range(E):
+ quant[f"model.layers.1.mlp.experts.{e}.gate_proj.weight"] = torch.randn(I_, D) * 0.02
+ quant[f"model.layers.1.mlp.experts.{e}.up_proj.weight"] = torch.randn(I_, D) * 0.02
+ quant[f"model.layers.1.mlp.experts.{e}.down_proj.weight"] = torch.randn(D, I_) * 0.02
+ for prefix in ("model.layers.0", "model.layers.1"):
+ r, q = _attn_block(prefix)
+ raw.update(r)
+ quant.update(q)
+ # NEGATIVE CONTROL: must be excluded from the emitted set.
+ quant["model.layers.0.self_attn.indexer.wq_b.weight"] = torch.randn(I_, D) * 0.02 # DSA indexer: "skip"
+ quant["model.layers.2.self_attn.o_proj.weight"] = torch.randn(D, DQB) * 0.02 # layer 2 >= N_LAYERS: "skip"
+
+ sd = dict(raw)
+ for name, t in quant.items():
+ w_fp8, scale = fp8_block_quantize(t)
+ sd[name] = w_fp8
+ sd[name + "_scale_inv"] = scale
+
+ os.makedirs(indir, exist_ok=True)
+ shard = os.path.join(indir, "model-00001-of-00001.safetensors")
+ save_file(sd, shard)
+ with open(os.path.join(indir, "config.json"), "w") as fh:
+ json.dump({"model_type": "glm_moe_dsa_test_fixture", "hidden_size": D}, fh)
+ with open(os.path.join(indir, "tokenizer.json"), "w") as fh:
+ json.dump({"model": {"vocab": {"<|endoftext|>": 0, "a": 1}, "merges": []},
+ "added_tokens": []}, fh)
+ with open(os.path.join(indir, "generation_config.json"), "w") as fh:
+ json.dump({"temperature": 0.0}, fh)
+ return shard, sd
+
+
+def _expected_checklist():
+ """M1's standalone-load checklist (audit section 3), built independently
+ of classify()/repack_fp8_passthrough.py -- a shared bug in both would not
+ be caught by an equality check derived from the same source."""
+ names = {"model.embed_tokens.weight", "lm_head.weight", "model.norm.weight"}
+ attn_suffixes = ("self_attn.q_a_proj.weight", "self_attn.q_a_layernorm.weight",
+ "self_attn.q_b_proj.weight", "self_attn.kv_a_proj_with_mqa.weight",
+ "self_attn.kv_a_layernorm.weight", "self_attn.kv_b_proj.weight",
+ "self_attn.o_proj.weight")
+ for i in (0, 1):
+ p = f"model.layers.{i}"
+ names.add(f"{p}.input_layernorm.weight")
+ names.add(f"{p}.post_attention_layernorm.weight")
+ for s in attn_suffixes:
+ names.add(f"{p}.{s}")
+ # layer 0: dense MLP
+ for s in ("gate_proj", "up_proj", "down_proj"):
+ names.add(f"model.layers.0.mlp.{s}.weight")
+ # layer 1: sparse MLP
+ names.add("model.layers.1.mlp.gate.weight")
+ names.add("model.layers.1.mlp.gate.e_score_correction_bias")
+ for s in ("gate_proj", "up_proj", "down_proj"):
+ names.add(f"model.layers.1.mlp.shared_experts.{s}.weight")
+ for e in range(E):
+ names.add(f"model.layers.1.mlp.experts.{e}.{s}.weight")
+ return names
+
+
+def _read_header(path):
+ with open(path, "rb") as f:
+ hlen = struct.unpack("<Q", f.read(8))[0]
+ return json.loads(f.read(hlen))
+
+
+class FullFamilyEmissionTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.indir = os.path.join(self.tmp.name, "fp8src")
+ self.outdir = os.path.join(self.tmp.name, "out")
+ self.shard, self.sd = _make_checkpoint(self.indir)
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def _run_cli(self, extra=()):
+ return subprocess.run(
+ [sys.executable, TOOL, "--indir", self.indir, "--outdir", self.outdir,
+ "--n-layers", str(N_LAYERS), *extra],
+ capture_output=True, text=True)
+
+ def test_dry_run_covers_the_full_checklist_without_writing(self):
+ """Cross-check via the tool's own shard_inventory (no subprocess) --
+ 1-D norms must not raise here either (_check_geometry is never
+ called on them; is_passthrough_target selects them directly)."""
+ inv = rp.shard_inventory(self.shard, n_layers=N_LAYERS)
+ names = {it["name"] for it in inv}
+ self.assertEqual(names, _expected_checklist())
+ self.assertFalse(os.path.exists(self.outdir), "--dry-run must not create --outdir")
+
+ def test_full_mint_emits_every_family_and_only_those(self):
+ proc = self._run_cli()
+ self.assertEqual(proc.returncode, 0, proc.stderr)
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ self.assertEqual(len(outs), 1)
+ hdr = _read_header(outs[0])
+ emitted = {k for k in hdr if k != "__metadata__" and not k.endswith(".qs")}
+ expected = _expected_checklist()
+ self.assertEqual(emitted, expected,
+ f"missing: {expected - emitted}; unexpected: {emitted - expected}")
+ # negative control: indexer and beyond-n_layers names must NOT appear anywhere
+ self.assertNotIn("model.layers.0.self_attn.indexer.wq_b.weight", hdr)
+ self.assertNotIn("model.layers.2.self_attn.o_proj.weight", hdr)
+
+ def test_byte_identity_fp8_and_raw_passthrough_hash_match(self):
+ """Byte-identity, hash in vs out, for BOTH categories: an fp8-repack
+ tensor (kv_b_proj, weight+qs) and raw pass-through tensors (a 1-D
+ BF16 norm, a 2-D BF16 embed table, a 1-D F32 bias)."""
+ self.assertEqual(self._run_cli().returncode, 0)
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ self.assertEqual(len(outs), 1)
+ with safe_open(self.shard, framework="pt") as f_in, \
+ safe_open(outs[0], framework="pt") as f_out:
+ # fp8-repack: kv_b_proj weight bytes + its renamed .qs sidecar
+ name = "model.layers.0.self_attn.kv_b_proj.weight"
+ self.assertEqual(_sha256(f_in.get_tensor(name).view(torch.uint8)),
+ _sha256(f_out.get_tensor(name)))
+ self.assertEqual(
+ _sha256(f_in.get_tensor(name + "_scale_inv").reshape(-1)),
+ _sha256(f_out.get_tensor(name + ".qs")))
+ # raw pass-through: 1-D norm, 2-D embed table, 1-D f32 bias
+ for name in ("model.norm.weight", "model.embed_tokens.weight",
+ "model.layers.1.mlp.gate.e_score_correction_bias",
+ "model.layers.0.self_attn.q_a_layernorm.weight"):
+ self.assertEqual(_sha256(f_in.get_tensor(name)),
+ _sha256(f_out.get_tensor(name)),
+ f"{name}: hash mismatch, not byte-identical")
+ self.assertNotIn(name + ".qs", f_out.keys(), f"{name}: must carry no .qs")
+
+ def test_1d_norms_pass_the_emission_stamp_gate_without_crashing(self):
+ """The M1 fix-shape requirement, end to end: several 1-D norms are in
+ this fixture (model.norm.weight, both layers' input_layernorm/
+ post_attention_layernorm, q_a_layernorm, kv_a_layernorm) and the run
+ must complete (rc==0), not crash on _emission_stamp_gate's old
+ unconditional `len(t.shape) != 2` check."""
+ proc = self._run_cli()
+ self.assertEqual(proc.returncode, 0, proc.stderr)
+ self.assertNotIn("Traceback", proc.stderr)
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ hdr = _read_header(outs[0])
+ expected_1d = {
+ "model.norm.weight": D,
+ "model.layers.0.input_layernorm.weight": D,
+ "model.layers.1.post_attention_layernorm.weight": D,
+ "model.layers.0.self_attn.q_a_layernorm.weight": DQA,
+ "model.layers.1.self_attn.kv_a_layernorm.weight": DKVA,
+ }
+ for name, size in expected_1d.items():
+ self.assertEqual(hdr[name]["shape"], [size], f"{name}: shape sanity")
+
+ def test_config_json_and_generation_config_land_in_outdir(self):
+ """M1 gap: main() previously never copied config.json into --outdir
+ -- a minted directory was not standalone-loadable for that reason
+ alone. config.json is mandatory; generation_config.json is
+ best-effort (both present in this fixture's --indir)."""
+ self.assertEqual(self._run_cli().returncode, 0)
+ out_cfg = os.path.join(self.outdir, "config.json")
+ self.assertTrue(os.path.exists(out_cfg))
+ with open(os.path.join(self.indir, "config.json")) as f_in, open(out_cfg) as f_out:
+ self.assertEqual(json.load(f_in), json.load(f_out))
+ out_gen = os.path.join(self.outdir, "generation_config.json")
+ self.assertTrue(os.path.exists(out_gen))
+ with open(os.path.join(self.indir, "generation_config.json")) as f_in, open(out_gen) as f_out:
+ self.assertEqual(json.load(f_in), json.load(f_out))
+
+ def test_config_json_missing_refuses_loudly(self):
+ """The mandatory half of the same fix: no config.json in --indir must
+ refuse the run, not silently mint an unloadable directory."""
+ os.remove(os.path.join(self.indir, "config.json"))
+ proc = self._run_cli()
+ self.assertNotEqual(proc.returncode, 0)
+ self.assertIn("config.json", proc.stderr)
+ self.assertFalse(os.path.exists(os.path.join(self.outdir, "config.json")))
+
+ def test_generation_config_missing_is_best_effort_not_fatal(self):
+ os.remove(os.path.join(self.indir, "generation_config.json"))
+ proc = self._run_cli()
+ self.assertEqual(proc.returncode, 0, proc.stderr)
+ self.assertFalse(os.path.exists(os.path.join(self.outdir, "generation_config.json")))
+
+ def test_tokenizer_json_lands_in_outdir_byte_identical(self):
+ """F3: V2's end-to-end smoke (2026-08-18) found the full mint emitted
+ config.json but not tokenizer.json, so a minted directory could not
+ actually be loaded standalone -- colibri.c's tok_load (c/tok.h:101)
+ hard-exits without it, exactly like load_cfg does without
+ config.json. Byte-identity, not just existence: this is a COPY, not a
+ regeneration, so the source and minted bytes must match exactly."""
+ self.assertEqual(self._run_cli().returncode, 0)
+ out_tok = os.path.join(self.outdir, "tokenizer.json")
+ self.assertTrue(os.path.exists(out_tok))
+ src = os.path.join(self.indir, "tokenizer.json")
+ with open(src, "rb") as f_in, open(out_tok, "rb") as f_out:
+ in_bytes, out_bytes = f_in.read(), f_out.read()
+ self.assertEqual(in_bytes, out_bytes,
+ "tokenizer.json must be a byte-identical copy, not a regeneration")
+
+ def test_tokenizer_json_missing_refuses_loudly(self):
+ """The mandatory half of the tokenizer.json fix: no tokenizer.json in
+ --indir must refuse the run, not silently mint a directory the
+ engine will fail to load (V2's exact finding, now a hard refusal
+ instead of a silent gap)."""
+ os.remove(os.path.join(self.indir, "tokenizer.json"))
+ proc = self._run_cli()
+ self.assertNotEqual(proc.returncode, 0)
+ self.assertIn("tokenizer.json", proc.stderr)
+ self.assertFalse(os.path.exists(os.path.join(self.outdir, "tokenizer.json")))
+
+ def test_index_json_and_docs_not_copied(self):
+ """Files confirmed NOT read by colibri.c/openai_server.py must stay
+ excluded, including model.safetensors.index.json -- copying it would
+ be actively wrong here since its weight_map keys the SOURCE shard
+ names (model-NNNNN-of-00141.safetensors), not the mint's
+ out-fp8pass-*.safetensors layout. tokenizer_config.json, chat_template
+ .jinja, README.md, LICENSE, and .gitattributes are likewise never
+ opened anywhere in the loader/server code path (see the file:line
+ citations above STANDALONE_LOAD_REQUIRED_FILES)."""
+ never_copy = ("model.safetensors.index.json", "tokenizer_config.json",
+ "chat_template.jinja", "README.md", "LICENSE", ".gitattributes")
+ for fname in never_copy:
+ with open(os.path.join(self.indir, fname), "w") as fh:
+ fh.write("placeholder -- must not be copied to --outdir\n")
+ self.assertEqual(self._run_cli().returncode, 0)
+ for fname in never_copy:
+ self.assertFalse(os.path.exists(os.path.join(self.outdir, fname)),
+ f"{fname} must NOT be copied into --outdir")
+
+ def test_kvb_stamped_router_norms_embed_unstamped(self):
+ self.assertEqual(self._run_cli().returncode, 0)
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ hdr = _read_header(outs[0])
+ stamp = json.loads(hdr["__metadata__"]["colibri.fmt"])
+ self.assertEqual(stamp.get("model.layers.0.self_attn.kv_b_proj.weight"), rp.FORMAT_NAME)
+ self.assertEqual(stamp.get("model.layers.1.self_attn.kv_b_proj.weight"), rp.FORMAT_NAME)
+ for name in ("model.embed_tokens.weight", "lm_head.weight", "model.norm.weight",
+ "model.layers.1.mlp.gate.weight",
+ "model.layers.1.mlp.gate.e_score_correction_bias"):
+ self.assertNotIn(name, stamp, f"{name}: raw pass-through must never be stamped")
+ self.assertNotIn(name + ".qs", hdr, f"{name}: raw pass-through must carry no .qs")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.py
index 93b6c8f2..1ebf92f9 100644
--- a/c/tools/repack_fp8_passthrough.py
+++ b/c/tools/repack_fp8_passthrough.py
@@ -20,24 +20,62 @@ THE DESIGN LANDMINE (see qt_resolve_fmt in colibri.c, c/quant.h's E4M3_LUT
comment): the engine tells fmt=8 (this tool's output) apart from fmt=1 (plain
int8) ONLY by scale-array geometry -- per-row for fmt=1, per-128x128-block for
fmt=8. This tool must therefore emit .qs sidecars whose byte count is EXACTLY
-ceil(O/128)*ceil(I/128)*4, never anything that could coincide with O*4 by
-accident for the shapes it targets -- ENFORCED, not just asserted, by
-_check_geometry below (maintainer review, #528): it refuses to repack any
-[O,I] where nblkO*nblkI==O, the exact shape where this tool's own block-scale
-byte count would coincide with a per-row int8 scale byte count. This is not
-hypothetical: GLM-5.2's own self_attn.o_proj.weight ([6144,16384]) is a real
-instance (nblkO=48, nblkI=128, product=6144==O). It matters more now than it
-used to, because the engine's UNSTAMPED read-side collision policy resolves an
-ambiguous shape to fmt=1 rather than refusing (see qt_resolve_fmt's
-INVERSION) -- so an fmt=8 container this tool emitted at such a shape would be
-silently read back as plain int8, not caught by a read-side refusal. Avoiding
-the collision at write time is therefore load-bearing, not a redundant
-belt-and-braces check. At I=98 specifically, a single-block fp8 tensor's raw
-weight-byte count ALSO coincides with a genuine fmt=6 (E8/IQ3) tensor's -- see
-qt_resolve_fmt's "SECOND DESIGN LANDMINE" comment; this tool does not target
-I=98 shapes in practice (Z.ai's real projections don't land there), and that
-narrower collision is unchanged by this PR (still an unconditional read-side
-refusal), so _check_geometry does not additionally guard against it here.
+ceil(O/128)*ceil(I/128)*4, and must never emit one that could coincide with
+O*4 for a tensor whose read-side resolution has no other evidence to go on --
+ENFORCED, not just asserted, by _check_geometry below (maintainer review,
+#528). This is not hypothetical: GLM-5.2's own self_attn.o_proj.weight
+([6144,16384]) is a real instance (nblkO=48, nblkI=128, product=6144==O), and
+the collision census over the whole checkpoint finds it is the ONLY ambiguous
+2-D fp8 family there.
+
+WHAT DECIDES THE COLLISION, and what changed: the engine's UNSTAMPED read-side
+policy resolves an ambiguous shape to fmt=1 rather than refusing (see
+qt_resolve_fmt's INVERSION) -- so an UNSTAMPED fmt=8 tensor at such a shape
+would be silently read back as plain int8, with no read-side refusal to catch
+it. A STAMPED one is different, and always has been: qt_resolve_fmt's
+"THE STAMPED CASE IS UNCHANGED by this inversion" branch resolves a tensor
+whose __metadata__ stamp names exactly one of the two colliding candidates
+through the existing TRUST-VERIFY-REFUSE path. So the write-side rule is not
+"never emit at the collision shape" but "never emit at the collision shape
+UNSTAMPED" -- the invariant #528 asked for, stated at the precision the
+mechanism actually has. _check_geometry therefore takes a `stampable` flag and
+refuses when it is false; _emission_stamp_gate below re-derives BOTH halves of
+that question from the OUTPUT TENSORS THEMSELVES immediately before the bytes
+hit disk -- the collision predicates from the emitted weight's [O,I] and its
+emitted .qs LENGTH, and stampability from the tensor NAME (not from the map the
+selection side built) -- so the guarantee is structural (a property of what is
+written) rather than procedural (a property of how the callers happen to be
+wired). The one axis the emitted bytes cannot answer, an unrecognized name that
+is neither routed nor an audited resident, is closed on the selection side by
+STAMPABLE_KINDS instead.
+
+WHICH TENSORS ARE STAMPABLE is a READER-side fact, not a writer's choice. Only
+qt_from_disk consults the stamp map; the three routed-expert load sites
+(expert_load_impl and friends, colibri.c) call qt_resolve_fmt with
+stamped_name=NULL, so a stamp on a routed expert would be written and never
+read. docs/FORMATS.md states the same thing from the registry's side ("the
+routed-expert load sites ... pass stamped_name=NULL, so the MANDATORY stamp is
+not consulted there today"), and st.h's ST_FMT_STAMP_MAX cap is sized on the
+same premise (stamps are "NEVER the tens of thousands of routed-expert tensors
+a large MoE model carries"). Routed experts are therefore emitted UNSTAMPED
+and stay subject to the unconditional collision refusal.
+
+The same reader-side standard applies in the other direction: a tensor is
+stampable only if a load site was actually audited for it. classify()'s "q" is a
+CATCH-ALL for any unmatched name ending ".weight", so no particular load site
+corresponds to it -- it is repacked but not stampable (see STAMPABLE_KINDS),
+which means an unexpected tensor from some checkpoint variant is refused loudly
+at the collision shape rather than emitted under a stamp resting on a fallback.
+
+At I=98 specifically, a single-block fp8 tensor's raw weight-byte count ALSO
+coincides with a genuine fmt=6 (E8/IQ3) tensor's -- see qt_resolve_fmt's
+"SECOND DESIGN LANDMINE" comment. That collision is resolved read-side by a
+stamp naming fp8-e4m3-b128 (qt_resolve_fmt's `sf==8 && fp8_blk_f32_also`
+branch) and refuses unconditionally without one. Z.ai's real projections do
+not land at I=98, but this tool now also emits routed experts, which are
+unstampable -- so _check_geometry guards that shape for unstampable tensors
+too, refusing at write time rather than emitting a container that would refuse
+at load time.
SCALE ENCODING: this tool always emits f32 block scales (4 bytes/block, the
`.to(torch.float32)` below) -- the ENCODING this build implements. It never
@@ -48,32 +86,54 @@ scales) is out of scope for this tool as written; the read side (qt_resolve_fmt)
recognizes and refuses that byte signature by name rather than misread it,
should a container carrying it ever reach the engine some other way.
-SELECTED kinds only: "resident" tensors (shared expert, o_proj, other
-attention projections, dense-MLP first layers, and the generic resident
-fallback) -- routed experts (kind "x" in convert_fp8_to_int4.classify) are
-EXCLUDED and stay on the existing int4-g64 streaming path; routers/norms/bias
-(kind "f32") and embed/lm_head (kind "io", BF16 in source, never FP8) are
-untouched by this tool -- carrying those into a mixed fmt=8+int4 loadout
-directory is separate, deferred "loadout index-rewrite blending" work.
-
-kv_b_proj (kind "kvb") is ALSO deliberately excluded, despite being a resident
-tensor classify() would otherwise select: colibri.c's MLA-absorption CPU path
-(qt_addrow/qt_matvec_rows, called only on l->kv_b -- the always-available
-fallback whenever the Metal fused decode kernel isn't used) has no fmt==8
-support. FIX ROUND 2 update: an fmt=8 (or fmt=6) tensor reaching either
-function now refuses loudly (exit(1), naming the function and the fmt) --
-before that fix it was worse than "misread as int2": qt_addrow SIGSEGV'd
-outright (t->q4 is NULL for fmt=8; the untouched int2 fall-through
-dereferenced it), and only fmt=6 (t->q4 non-NULL, wrong per-row scale
-geometry) actually produced silently-wrong values. Either way, this tool
-still does not select kv_b_proj: refusing loudly at the absorb call site is
-not the same as SUPPORTING fmt=8 there (no dequant path exists, whether it
-now trips a controlled refusal or, before the fix, undefined behavior). The
-CUDA absorb kernels (coli_cuda_attention_absorb/_kvdev in backend_cuda.cu)
-are similarly int4-specific and unaffected by that fix. Repacking kv_b_proj
-to fmt=8 needs real fmt=8 CPU+CUDA absorb-path support first -- out of scope
-for this build, so this tool refuses to produce a container the engine
-cannot safely read.
+SELECTED kinds (fp8-repack path): "resident" tensors (shared expert, o_proj,
+kv_b_proj, other attention projections, dense-MLP first layers, and the
+generic resident fallback) AND routed experts (kind "x" in
+convert_fp8_to_int4.classify). Routed experts were
+previously excluded and left on the int4-g64 streaming path; they are the bulk
+of a GLM-5.2 checkpoint's fp8 bytes, so byte-preserving them is the point of
+minting from the authoritative checkpoint at all. They differ from residents in
+exactly one respect this tool has to honour: they are never stamped (see WHICH
+TENSORS ARE STAMPABLE above), so their format is decided by byte arithmetic
+alone and _check_geometry keeps the unconditional collision refusal for them.
+The engine's CPU routed-expert path dispatches them correctly -- expert_ffn ->
+expert_gate_up/matmul_qt (colibri.c), whose fmt==8 branch calls matmul_fp8;
+the CUDA grouped-expert path refuses fmt>3 host-side and falls back to that CPU
+path rather than mis-decoding them.
+
+FULL-FAMILY EMISSION (M1 mint coverage audit, 2026-08-18): routers/norms/bias
+(kind "f32") and embed/lm_head (kind "io", BF16 or F32 in source, never FP8)
+are now copied byte-identically into the container -- see is_passthrough_target
+and PASSTHROUGH_KINDS below. This is a pure pass-through, not a repack: no
+`.qs` sidecar, no stamp, no `_check_geometry` involvement, dtype unchanged. It
+is exactly the code path the engine already runs today against a raw,
+unrepacked source directory (`ld`/`qt_from_disk`'s fallback to `st_read_f32`
+when no `.qs` sidecar exists, colibri.c) -- nothing new is exercised on the
+read side. `main()` also copies `config.json` (mandatory, colibri.c's
+load_cfg) and `generation_config.json` (best-effort) from --indir into
+--outdir. Together with kv_b_proj emission below, this makes the tool's
+output a standalone-loadable model directory on its own, with one caveat: the
+engine's fmt=8 absorb support (branch `f8/absorb-fmt8`) must land before a
+kv_b_proj-carrying container is safe to decode through the batched serving
+path -- see below.
+
+kv_b_proj (kind "kvb") is now emitted the same way as o_proj/attn: byte-
+preserved fp8 weight + renamed `.qs` scale sidecar, stamped (it clears the
+collision shape the same way o_proj does -- M1 audit, GLM-5.2's kv_b_proj is
+never at the ambiguous [O,I] this tool's `_check_geometry` guards). THIS TOOL
+CHANGE IS INDEPENDENT of, and does not wait on, the engine-side absorb work:
+colibri.c's MLA-absorption CPU path (qt_addrow/qt_matvec_rows, called only on
+l->kv_b) and the CUDA absorb kernels have no fmt==8 case as of this tool's own
+HEAD -- that support is being built in parallel on `f8/absorb-fmt8` and is
+NOT part of this diff. A container minted with this tool BEFORE that engine
+branch lands will load clean (qt_from_disk resolves the stamp exactly like
+o_proj's) but crash -- loudly, not silently -- on any decode that reaches the
+batched (`kvs`-nonNULL) serving path, because `ABSORB=0` cannot bypass absorb
+there. Sequencing that correctly (engine work before this container is used
+for decode) is the gate's job, not this tool's: repacking kv_b_proj here is
+the tool-side half of a two-sided integration, done now because engine and
+tool work can (and per the gate's assembly manifest, should) proceed in
+parallel.
METADATA STAMP (reference implementation of the FORMATS-registry FR -- see
docs/FORMATS.md): every output shard's safetensors `__metadata__` carries a
@@ -101,6 +161,46 @@ container this tool itself has ever produced before this feature existed)
are otherwise unaffected: no stamp means byte-arithmetic inference alone
decides, exactly as it always has.
+CONTAINER-LEVEL FORMAT DECLARATION: every output shard's `__metadata__` also
+carries a `colibri.container.formats` key whose VALUE is JSON text -- a sorted
+array of the format NAME strings this container declares it carries residents
+in (today: exactly ["fp8-e4m3-b128"]). It is a CONTAINER-wide statement, not a
+per-tensor one, and it is deliberately INERT in this revision: no engine reads
+it. Its purpose is the hole the per-tensor stamp cannot close by itself -- if
+any downstream tool rewrites shards without preserving `__metadata__`, a
+stamped fmt=8 tensor at the collision shape silently becomes an unstamped one
+and resolves to int8-row with no error. A container that DECLARES fp8-e4m3
+residents gives a reader the standing to treat an unstamped ambiguous tensor
+inside it as an ERROR instead of an int8 default, which converts a procedural
+guarantee back into a structural one. The engine-side check is separate work;
+containers minted now carry the declaration so that check has data to act on
+from the day it lands. Unknown `__metadata__` keys are ignored by the existing
+reader (st_fmt_stamp_ingest looks up "colibri.fmt" alone, and st_init_multi
+skips the whole "__metadata__" entry when indexing tensors), so an engine that
+does not know this key loads such a container exactly as it does today --
+verified by tests/test_fp8_e2e_repack_load.py.
+
+HEADER METADATA ORDER (fixed 2026-08-18, see
+colibri_lab/reports/D1_MINT_DIVERGENCE_2026-08-18.md and
+colibri_lab/reports/F2_MINT_DETERMINISM_FIX_2026-08-18.md): shards this tool
+emitted BEFORE this fix can differ from a re-mint of the same checkpoint in
+the byte order of the two keys inside `__metadata__` ("colibri.fmt" vs
+"colibri.container.formats") -- a property of the underlying safetensors
+`save_file` extension, which does not preserve Python-side dict order when
+serializing its `metadata=` argument. This affects ONLY those two header
+bytes' relative position; every tensor payload byte, every `data_offsets`
+value, and both metadata VALUES are unaffected and were always correct (D1's
+diagnosis closed the corruption question directly). A raw whole-file SHA-256
+comparison between two containers minted from the same input before and
+after this fix -- or between two pre-fix mints -- can therefore show a
+mismatch that is entirely this cosmetic reordering, not a faithfulness
+regression; compare tensor payload hashes (or `__metadata__`'s parsed JSON
+values) rather than raw file digests when checking such pairs.
+`_canonicalize_metadata_order` (this module) makes every container minted
+FROM THIS FIX ONWARD byte-identical for byte-identical inputs, including
+across repeated runs and across machines -- see
+tests/test_fp8_repack.py's determinism test.
+
HARD CONSTRAINT for this build: unit-tested on synthetic fixtures ONLY (see
tests/test_fp8_repack.py, built with tools/glm_fp8_emit.py's exact real-
checkpoint layout) plus --dry-run inventory mode against those same fixtures.
@@ -110,19 +210,60 @@ NO read of any real Z.ai shard, NO full or partial repack of a real checkpoint
Usage (synthetic fixtures / local testing only):
python3 tools/repack_fp8_passthrough.py --indir <fp8_dir> --outdir <out> --dry-run
python3 tools/repack_fp8_passthrough.py --indir <fp8_dir> --outdir <out> --n-layers 78
+ python3 tools/repack_fp8_passthrough.py --indir <fp8_dir> --outdir <out> --n-layers 78 --mtp
"""
-import argparse, glob, json, os, sys
+import argparse, glob, json, os, shutil, struct, sys
sys.path.insert(0, os.path.dirname(__file__))
from convert_fp8_to_int4 import classify, check_or_record_params # reuse: same taxonomy,
# same #383-class params guard
-# Kinds classify() can return that this tool targets: resident weights, NOT routed
-# experts ("x"), NOT the always-F32 set ("f32"), NOT embed/lm_head ("io" -- BF16 in
-# source, never FP8), NOT sidecars/skips ("consumed"/"skip"). "kvb" (kv_b_proj) is
-# ALSO excluded -- see the module docstring: the CPU absorb path (qt_addrow/
-# qt_matvec_rows) and the CUDA absorb kernels have no fmt=8 case yet.
-RESIDENT_KINDS = frozenset({"sh", "o", "attn", "dmlp", "q"})
+# Kinds classify() can return that this tool targets, split by whether the ENGINE
+# will consult a metadata stamp for them on read -- the split that decides which
+# tensors may sit at the collision shape (see the module docstring).
+#
+# RESIDENT_KINDS load through qt_from_disk, which looks the tensor's name up in the
+# stamp map (st_fmt_stamp) and passes the result into qt_resolve_fmt: STAMPABLE.
+# ROUTED_KINDS ("x") load through expert_load_impl and its two siblings, every one of
+# which passes stamped_name=NULL: NOT stampable, whatever this tool writes.
+#
+# "kvb" (kv_b_proj) is now included: FULL-FAMILY EMISSION (M1 audit) -- it is FP8
+# in the source and clears the collision shape the same way "o" does, so it needs
+# no new _check_geometry/stamp-gate logic. See the module docstring for the
+# engine-side absorb-support caveat (separate, parallel work, not part of this
+# tool's selection logic).
+#
+# STILL NOT targeted here (see is_passthrough_target/PASSTHROUGH_KINDS instead):
+# the always-BF16/F32 set ("f32", "io") -- those are copied byte-identically by a
+# separate, dtype-agnostic code path, not this fp8 one. Sidecars/skips
+# ("consumed"/"skip") are never targeted by either path.
+RESIDENT_KINDS = frozenset({"sh", "o", "attn", "dmlp", "q", "kvb"})
+ROUTED_KINDS = frozenset({"x"})
+REPACK_KINDS = RESIDENT_KINDS | ROUTED_KINDS
+
+# Non-FP8 resident kinds this tool copies byte-identically (dtype unchanged, no
+# `.qs` sidecar, no stamp): embed_tokens/lm_head ("io") and norms/router/bias
+# ("f32"). See is_passthrough_target below and the module docstring's
+# FULL-FAMILY EMISSION section.
+PASSTHROUGH_KINDS = frozenset({"io", "f32"})
+
+# STAMPABILITY is NARROWER than residency, and deliberately so. Being resident is
+# necessary for the engine to consult a stamp, but the claim a stamp underwrites --
+# "this exact name loads through qt_from_disk, which looks it up in st_fmt_stamp" --
+# was verified name by name against colibri.c's load sites ("sh" -> :1997-1999,
+# "o" -> :1979, "attn" -> :1975-1978, "dmlp" -> the same dense-MLP qt_load calls,
+# "kvb" -> :1707 (qt_load call for l->kv_b -- see M1 audit; the stamp is
+# consulted the same generic way for every qt_load call site, kv_b included),
+# plus the MTP twins at :2047ff). "q" is NOT one of those names: classify() returns
+# it as a CATCH-ALL for any unmatched tensor ending ".weight"
+# (convert_fp8_to_int4.py's last line before the f32 default), so a stamp on a "q"
+# tensor would rest on a fallback rather than on an audited reader-side fact. Such a
+# tensor is still REPACKED (it is fp8 in the source and belongs in the container);
+# it is simply not stampable, which means the collision-shape gates refuse it
+# LOUDLY instead of emitting it under a stamp nobody verified anyone reads. If a
+# future checkpoint variant brings a real new resident family, add its kind to
+# classify() and to this set together with the load site that justifies it.
+STAMPABLE_KINDS = frozenset({"sh", "o", "attn", "dmlp", "kvb"})
# The format's PUBLIC identity (what containers/FRs advertise) -- the internal
# fmt=8 ordinal is a colibri.c-only enum value, never itself persisted (see
@@ -136,6 +277,23 @@ FORMAT_NAME = "fp8-e4m3-b128"
# future tooling can cite the exact string instead of re-typing it.
METADATA_KEY = "colibri.fmt"
+# safetensors __metadata__ key carrying the CONTAINER-level format declaration:
+# JSON-encoded array of format NAME strings (never ordinals -- same rule as the
+# per-tensor stamp). See the module docstring's CONTAINER-LEVEL FORMAT
+# DECLARATION section. Inert in this revision: nothing reads it yet.
+CONTAINER_FORMATS_KEY = "colibri.container.formats"
+
+# Mirror of c/st.h's ST_FMT_STAMP_MAX. st_fmt_stamp_ingest refuses a container
+# whose combined colibri.fmt entries exceed this, so a writer that can exceed it
+# writes a container the engine will refuse at discovery time. GLM-5.2 stamps
+# ~7 residents x 79 layers, an order of magnitude under -- but this tool now also
+# selects routed experts (tens of thousands of tensors), and the ONLY thing
+# keeping them out of the stamp map is _stampable(). Checking the bound here
+# turns "we believe we do not stamp routed experts" into a run that stops if we
+# ever do. Kept as a literal (not imported) because the C constant is not
+# reachable from Python; tests/test_fp8_repack.py pins the two together.
+ST_FMT_STAMP_MAX = 4096
+
BLOCK = 128
@@ -143,59 +301,351 @@ def _nblk(n):
return (n + BLOCK - 1) // BLOCK
-def is_repack_target(name, dtype, keys, n_layers):
- """True if `name` is a resident-kind FP8 tensor this tool should byte-preserve
- into the fmt=8 container. `keys` is the full set of tensor names in the shard
- (needed to confirm the `_scale_inv` sidecar is actually present -- classify()
- alone can't tell FP8 tensors from any other dtype a resident-kind name might
- carry in a non-FP8 checkpoint variant)."""
+def is_repack_target(name, dtype, keys, n_layers, keep_mtp=False):
+ """True if `name` is an FP8 tensor this tool should byte-preserve into the
+ fmt=8 container (resident kinds and routed experts -- see REPACK_KINDS).
+ `keys` is the full set of tensor names in the shard (needed to confirm the
+ `_scale_inv` sidecar is actually present -- classify() alone can't tell FP8
+ tensors from any other dtype a target-kind name might carry in a non-FP8
+ checkpoint variant)."""
if name.endswith("_scale_inv"):
return False # sidecar, handled together with its weight
- kind = classify(name, n_layers)
- if kind not in RESIDENT_KINDS:
+ kind = classify(name, n_layers, keep_mtp=keep_mtp)
+ if kind not in REPACK_KINDS:
return False
if dtype not in ("F8_E4M3", "float8_e4m3fn"):
return False
return (name + "_scale_inv") in keys
-def _check_geometry(name, O, I, nblkO, nblkI):
+def is_passthrough_target(name, n_layers, keep_mtp=False):
+ """True if `name` is a non-FP8 resident tensor this tool should copy
+ byte-identically into the container: embed_tokens/lm_head (kind "io") and
+ norms/router/bias (kind "f32") -- see PASSTHROUGH_KINDS.
+
+ Unlike is_repack_target, this makes NO dtype restriction. M1's audit
+ (`knowledge/decisions/2026-08-16-fp8-faithful-container.md`) confirms these
+ families are BF16 or F32 in the real GLM-5.2-FP8 checkpoint, never FP8 --
+ but the copy itself is dtype-agnostic by design: the engine's raw-tensor
+ read path (`ld`'s `st_read_f32[_cap]`, colibri.c) already accepts
+ BF16/F16/F32 and converts to f32 output, the same fallback it takes today
+ against an unrepacked source directory, so there is no dtype this tool
+ needs to special-case here. `_scale_inv`-suffixed names are excluded the
+ same way is_repack_target excludes them: classify() itself returns
+ "consumed" for them (its very first check), which is not in
+ PASSTHROUGH_KINDS."""
+ kind = classify(name, n_layers, keep_mtp=keep_mtp)
+ return kind in PASSTHROUGH_KINDS
+
+
+def _stampable(kind):
+ """Will the ENGINE consult a `colibri.fmt` stamp for a tensor of this kind?
+
+ Resident kinds load through qt_from_disk, which looks the name up via
+ st_fmt_stamp and hands the result to qt_resolve_fmt -- so a stamp on them is
+ read and is load bearing. Routed experts ("x") load through
+ expert_load_impl/uring_load_finish/pin_load, every one of which calls
+ qt_resolve_fmt with stamped_name=NULL: a stamp on a routed expert would be
+ written and never consulted. This is a READER-side fact, mirrored here rather
+ than a policy this tool gets to choose -- see the module docstring, and
+ docs/FORMATS.md's own statement of it.
+
+ The allowlist is STAMPABLE_KINDS, not RESIDENT_KINDS: classify()'s "q" is a
+ catch-all for unmatched ".weight" names and no audited load site corresponds to
+ it, so it is repacked but never stamped (see STAMPABLE_KINDS' comment)."""
+ return kind in STAMPABLE_KINDS
+
+
+def _check_geometry(name, O, I, nblkO, nblkI, stampable):
"""Refuse (loud, ValueError) rather than silently emit a .qs whose geometry
doesn't match what qt_resolve_fmt expects for [O,I] -- the same "untrusted
container" discipline the C side applies on read, applied here on write so a
malformed source checkpoint is caught at repack time, not at load time three
steps later with a confusing engine-side refusal.
- WRITER-SIDE REFUSAL (maintainer review, #528): also refuses at the AMBIGUOUS
- shape THE DESIGN LANDMINE describes (qt_resolve_fmt in colibri.c) -- where
- O == ceil(O/128)*ceil(I/128), i.e. this tensor's block-scale byte count
- (nblkO*nblkI*4) exactly equals a per-row int8 scale byte count (O*4). GLM-5.2's
- own self_attn.o_proj.weight ([6144,16384]) is a REAL, non-hypothetical instance
- of this shape. The engine's reader now resolves an unstamped collision to fmt=1
- (the incumbent, decodable format) rather than refusing -- which makes it load
- bearing that THIS tool never emits an fmt=8 container at such a shape: this
- docstring's own opening promise ("never anything that could coincide with O*4
- by accident") is enforced here, not just asserted."""
+ WRITER-SIDE REFUSAL (maintainer review, #528), stated at the precision the
+ read side actually has: at the AMBIGUOUS shape THE DESIGN LANDMINE describes
+ (qt_resolve_fmt in colibri.c) -- where O == ceil(O/128)*ceil(I/128), i.e. this
+ tensor's block-scale byte count (nblkO*nblkI*4) exactly equals a per-row int8
+ scale byte count (O*4) -- an UNSTAMPED fmt=8 tensor is silently read back as
+ fmt=1, because the reader's INVERSION resolves an unstamped collision to the
+ incumbent decodable format rather than refusing. A STAMPED one is not: the
+ reader's "THE STAMPED CASE IS UNCHANGED by this inversion" branch resolves it
+ through TRUST-VERIFY-REFUSE, which is what the stamp mechanism exists for.
+ GLM-5.2's own self_attn.o_proj.weight ([6144,16384]) is a REAL, non-
+ hypothetical instance of this shape, and the census over the whole checkpoint
+ finds it is the only one -- so `stampable` is what separates "emit it, stamped"
+ from "refuse". Routed experts are never stampable (see _stampable), so for them
+ this stays the unconditional refusal it has always been.
+
+ The SECOND DESIGN LANDMINE (qt_resolve_fmt) is guarded on the same terms: a
+ single-block fp8 tensor at I=98 has nb==O*98 and ns==4, which is also fmt=6's
+ (E8/IQ3) byte signature. A stamp naming fp8-e4m3-b128 resolves that read-side;
+ without one the reader refuses outright. No Z.ai projection lands at I=98, but
+ this tool now emits unstampable routed experts, so refuse at write time rather
+ than emit a container that aborts the model load."""
exp_nblkO, exp_nblkI = _nblk(O), _nblk(I)
if (nblkO, nblkI) != (exp_nblkO, exp_nblkI):
raise ValueError(
f"{name}: scale_inv block shape ({nblkO},{nblkI}) != expected "
f"({exp_nblkO},{exp_nblkI}) for [{O},{I}] -- refusing to repack a shape "
f"the engine's qt_resolve_fmt would either refuse or (worse) misread")
- if nblkO * nblkI == O:
+ if nblkO * nblkI == O and not stampable:
raise ValueError(
- f"{name}: [{O},{I}] is an AMBIGUOUS fp8-e4m3-b128 shape -- its "
- f"block-scale byte count (nblkO*nblkI*4={nblkO*nblkI*4}) exactly "
- f"coincides with a per-row int8 scale byte count (O*4={O*4}) for the "
- f"same [O,I]; refusing to emit an fmt=8 container the engine's "
+ f"{name}: [{O},{I}] is an AMBIGUOUS fp8-e4m3-b128 shape and this tensor "
+ f"is NOT stamp-resolvable on read -- its block-scale byte count "
+ f"(nblkO*nblkI*4={nblkO * nblkI * 4}) exactly coincides with a per-row "
+ f"int8 scale byte count (O*4={O * 4}) for the same [O,I], and the "
+ f"engine's routed-expert load sites pass stamped_name=NULL, so nothing "
+ f"can resolve it; refusing to emit an fmt=8 container the engine's "
f"qt_resolve_fmt collision policy would resolve to fmt=1 (int8-row), "
- f"not fmt=8, on an unstamped read (THE DESIGN LANDMINE, colibri.c)")
+ f"not fmt=8 (THE DESIGN LANDMINE, colibri.c)")
+ if I == 98 and nblkO * nblkI == 1 and not stampable:
+ raise ValueError(
+ f"{name}: [{O},{I}] is a single-block fp8-e4m3-b128 shape whose weight "
+ f"bytes (O*I={O * I}) and 4-byte scale array also match an E8/IQ3 "
+ f"(fmt=6) tensor's byte signature, and this tensor is NOT "
+ f"stamp-resolvable on read -- refusing to emit a container the engine's "
+ f"qt_resolve_fmt would abort on (SECOND DESIGN LANDMINE, colibri.c)")
+
+
+def _reader_consults_a_stamp(name):
+ """Will the engine look this tensor's NAME up in the stamp map on read?
+
+ Derived from the NAME ALONE -- no `kind`, no classify(), no _stampable(), and
+ so no dependency on the selection-side predicate whose mistakes this gate
+ exists to catch. classify() returns "x" (routed expert) for exactly the names
+ carrying ".mlp.experts." and ending ".weight" (convert_fp8_to_int4.py), and
+ ROUTED_KINDS is exactly {"x"}; colibri.c's three routed-expert load sites
+ (:2414, :2612, :2811) each call qt_resolve_fmt with stamped_name=NULL. So a
+ stamp on such a name is written into __metadata__ and never consulted.
+
+ Deliberately NOT written as `_stampable(classify(name, n_layers, keep_mtp))`.
+ That form would (a) make the gate agree with the selection predicate by
+ construction -- the exact self-reference this gate must not have -- and
+ (b) require n_layers AND keep_mtp to be plumbed in; a version that plumbed
+ n_layers alone would classify the --mtp pass's legitimate layer-78 residents
+ as "skip" and falsely refuse GLM-5.2's own o_proj [6144,16384], which sits at
+ the collision shape. The substring test has neither problem."""
+ return ".mlp.experts." not in name
+
+
+def _emission_stamp_gate(out, fmt_map):
+ """I-4, STRUCTURALLY: no fmt=8 tensor leaves this tool at a shape the reader
+ cannot resolve without a stamp the reader will actually consult.
+
+ _check_geometry enforces the same rule, but from the SELECTION side -- it
+ trusts the caller to have classified the tensor correctly and to actually put
+ the promised stamp in fmt_map. This gate re-derives BOTH halves of the
+ question from the OUTPUT TENSORS THEMSELVES: the collision predicates from the
+ emitted weight's [O,I] and its emitted `.qs` LENGTH (which is what the reader
+ actually counts -- ns == O*4 bytes, i.e. len(.qs) == O), and stampABILITY from
+ the tensor NAME via _reader_consults_a_stamp, not from the map _stampable()
+ built. It runs immediately before save_file.
+
+ WHAT THIS GUARANTEES, stated at the precision it holds (an earlier revision
+ of this docstring claimed a third hazard it did not cover):
+ * a DROPPED STAMP is caught -- the stamp map is read here, three lines
+ before save_file, and an unresolvable shape without its stamp refuses;
+ * a SECOND WRITER PATH is caught -- the check is derived from `out`, and
+ `out` is the dict save_file serializes, so any path that builds one is
+ gated by construction;
+ * a RECLASSIFIED TENSOR is caught on the ROUTED axis -- a routed-expert
+ NAME at an unresolvable shape refuses whatever fmt_map says about it,
+ because the name test does not consult fmt_map.
+ What it does NOT and cannot catch from `out` alone: a name that is not a
+ routed expert and not a genuine resident either -- classify()'s "q"
+ catch-all. Nothing in the emitted bytes distinguishes such a tensor from an
+ audited resident. That axis is closed on the selection side instead, by
+ STAMPABLE_KINDS excluding "q", which makes _check_geometry refuse it at these
+ same shapes and keeps it out of fmt_map so this gate refuses it too.
+
+ NON-FP8 PASS-THROUGH TENSORS (M1 audit fix, FULL-FAMILY EMISSION): `out` now
+ also carries embed_tokens/lm_head/norms/router -- byte-identical BF16/F32
+ copies with no `.qs` sidecar and no stamp (see is_passthrough_target). Those
+ are not this gate's concern: the collision-shape checks below exist only to
+ protect fmt=8's scale-geometry inference on read, which never applies to a
+ raw copy. The gate tells the two apart the same way it derives everything
+ else here -- from the OUTPUT TENSOR ITSELF, not from a name or a kind the
+ selection side computed: every fmt=8 weight this tool emits is written as
+ `w.view(torch.uint8)` (see repack_shard), so `t.dtype == torch.uint8` is
+ exactly the set this gate must inspect. A raw pass-through tensor keeps its
+ original dtype (never uint8), so it is skipped -- structurally, including
+ when 1-D (norms), which the 2-D check below would otherwise reject.
+
+ The companion writer-side bound on the stamp map's SIZE is
+ _check_stamp_budget below (st.h's ST_FMT_STAMP_MAX is container-wide, so it is
+ counted across shards -- and across the main/--mtp passes -- by the caller,
+ not here)."""
+ import torch
+ for name, t in out.items():
+ if name.endswith(".qs"):
+ continue
+ if t.dtype != torch.uint8:
+ continue # raw BF16/F32 pass-through, not fp8
+ if len(t.shape) != 2:
+ raise ValueError(
+ f"{name}: emitted weight tensor has shape {tuple(t.shape)}, expected "
+ f"2-D [O,I] -- refusing to emit a container whose geometry the "
+ f"read-side collision check cannot be re-derived from")
+ O, I = int(t.shape[0]), int(t.shape[1])
+ qs = out.get(name + ".qs")
+ if qs is None:
+ raise ValueError(
+ f"{name}: about to emit an fmt=8 weight with no '{name}.qs' scale "
+ f"array -- the engine resolves fmt=8 from the scale byte count, so "
+ f"a weight without one is unresolvable; refusing")
+ nsc = int(qs.numel())
+ # Both predicates below are stated on the EMITTED scale count rather than
+ # on _nblk(O)*_nblk(I), so they do not inherit _check_geometry's pinning of
+ # (nblkO,nblkI) -- that pinning is a selection-side fact and this gate must
+ # not depend on it (it is the one this gate would have to catch if the
+ # scale-writing path ever changed). Read side: colibri.c's ns == O*4 is
+ # len(.qs) == O; the second landmine's ns == 4 is len(.qs) == 1.
+ unreadable = not _reader_consults_a_stamp(name)
+ unstamped = fmt_map.get(name) != FORMAT_NAME
+ if unreadable or unstamped:
+ why = ("its NAME marks it a ROUTED EXPERT, and every routed-expert load "
+ "site passes stamped_name=NULL, so a stamp on it is written and "
+ "never read" if unreadable else
+ f"it carries no '{FORMAT_NAME}' stamp")
+ if nsc == O:
+ raise ValueError(
+ f"{name}: about to emit an fmt=8 tensor at the AMBIGUOUS shape "
+ f"[{O},{I}] (len(.qs)={nsc}==O, so its scale byte count "
+ f"coincides with a per-row int8 scale array) that nothing can "
+ f"resolve on read -- {why}; the engine would resolve it to "
+ f"fmt=1 (int8-row) and silently misread every weight; refusing "
+ f"(THE DESIGN LANDMINE, colibri.c qt_resolve_fmt)")
+ if I == 98 and nsc == 1:
+ raise ValueError(
+ f"{name}: about to emit an fmt=8 tensor at the single-block "
+ f"shape [{O},{I}] (len(.qs)={nsc}, weight bytes O*I={O * I}) "
+ f"whose byte signature also matches an E8/IQ3 (fmt=6) tensor, "
+ f"and nothing can resolve it on read -- {why}; the engine "
+ f"aborts the whole model load on this; refusing "
+ f"(SECOND DESIGN LANDMINE, colibri.c qt_resolve_fmt)")
+
+
+def _canonicalize_metadata_order(path):
+ """Rewrite `path`'s safetensors header IN PLACE so `__metadata__`'s KEY
+ ORDER is deterministic, regardless of what save_file just wrote.
+
+ Diagnosis (colibri_lab/reports/D1_MINT_DIVERGENCE_2026-08-18.md,
+ reproduced locally on this Mac against the installed safetensors build):
+ `save_file`'s Rust extension serializes the `metadata=` argument through
+ an unordered map -- every VALUE and every tensor payload byte is always
+ correct, but the two-key `__metadata__` object's byte order in the
+ header is a coin flip, independently per save_file call (not just per
+ box: repeat calls in the SAME process differ too, confirmed by minting
+ the same fixture twice above this function's own test). CPython dict
+ order (the `meta` dict built at the call site) never reaches the
+ written bytes, so building `meta` more carefully cannot fix this --
+ only touching the bytes after the Rust side has already written them
+ can.
+
+ This function does exactly that: read the header back, re-serialize
+ `__metadata__` with a canonical (sorted) key order, and write it back
+ into the SAME length-prefixed slot. A permutation of the same keys and
+ values can only reorder bytes, never add or remove any -- so the
+ rewritten header is always <= the original slot (padded with trailing
+ spaces to match exactly, the same convention
+ tests/test_fp8_e2e_repack_load.py's `_rewrite_metadata` uses, and every
+ JSON parser in this tree, including c/json.h's, already skips trailing
+ whitespace). No tensor's data_offsets move, so no payload byte is ever
+ touched.
+
+ No-op if the header carries no `__metadata__` (should not happen at
+ this tool's own call site, but this function is generic)."""
+ with open(path, "rb") as f:
+ hlen = struct.unpack("<Q", f.read(8))[0]
+ hdr = json.loads(f.read(hlen))
+ meta = hdr.get("__metadata__")
+ if not meta:
+ return
+ hdr["__metadata__"] = dict(sorted(meta.items()))
+ new = json.dumps(hdr, separators=(",", ":")).encode()
+ assert len(new) <= hlen, (
+ f"{path}: canonicalized header ({len(new)} bytes) no longer fits "
+ f"the original slot ({hlen} bytes) -- a same-keys/same-values "
+ f"reorder should never change length; something else changed")
+ new += b" " * (hlen - len(new))
+ with open(path, "r+b") as f:
+ f.seek(8)
+ f.write(new)
+
+
+def _check_stamp_budget(total_stamped):
+ if total_stamped > ST_FMT_STAMP_MAX:
+ raise ValueError(
+ f"stamp map would carry {total_stamped} entries, over st.h's "
+ f"ST_FMT_STAMP_MAX ({ST_FMT_STAMP_MAX}) -- st_fmt_stamp_ingest refuses "
+ f"such a container at discovery time. Stamps are a resident-tensor "
+ f"convention; this many entries means routed experts are being stamped")
+
+
+# The main pass and the --mtp pass write into the SAME --outdir and therefore
+# produce ONE container, but each keeps its own progress manifest (a resume
+# manifest has to be per-selection -- see --mtp's comment in main()). The stamp
+# budget is not per-selection: st_fmt_stamp_ingest accumulates S->fmt_n over every
+# shard it discovers in the directory (c/st.h:372), so ST_FMT_STAMP_MAX is a
+# CONTAINER-wide bound. Counted per-manifest, two passes could each stay under the
+# cap and still hand the engine a container over it -- the writer guarantee would
+# be 2x the reader's bound. So the budget check sums this pass's running total with
+# what the other pass has already recorded into the same directory.
+PROGRESS_MANIFESTS = (".fp8pass-progress.json", ".fp8pass-mtp-progress.json")
+
+
+def _stamps_from_other_passes(outdir, prog_name):
+ """Stamp count already recorded by the OTHER pass(es) writing into `outdir`.
+
+ Read tolerantly (same tolerance the resume manifest itself is read with): an
+ unreadable sibling manifest means this run counts only its own stamps, which
+ is the behaviour before this became container-wide, not worse than it."""
+ total = 0
+ for other in PROGRESS_MANIFESTS:
+ if other == prog_name:
+ continue
+ path = os.path.join(outdir, other)
+ if not os.path.exists(path):
+ continue
+ try:
+ with open(path) as fh:
+ total += int(json.loads(fh.read()).get("stamped", 0))
+ except (OSError, ValueError, TypeError):
+ continue
+ return total
-def shard_inventory(path, n_layers):
+# dtype -> bytes/element, for the dry-run byte-volume estimate on PASS-THROUGH
+# tensors only (shard_inventory reads shapes from get_slice, never the tensor
+# data itself, so it cannot ask torch for itemsize). Covers the dtypes a raw
+# BF16/F32 resident tensor can plausibly carry; anything else reports 0 rather
+# than guessing -- the estimate is informational (_print_inventory_summary),
+# never load-bearing for a correctness check.
+_DTYPE_NBYTES = {"F64": 8, "I64": 8, "F32": 4, "I32": 4, "F16": 2, "BF16": 2,
+ "I16": 2, "U8": 1, "I8": 1, "BOOL": 1}
+
+
+def shard_inventory(path, n_layers, keep_mtp=False):
"""--dry-run: scan one shard's header only (get_slice, no tensor data pulled)
- and return the list of tensors that WOULD be repacked, without writing
- anything. Cheap: safe to run over an entire real checkpoint's headers."""
+ and return the list of tensors that WOULD be repacked or passed through,
+ without writing anything. Cheap: safe to run over an entire real
+ checkpoint's headers.
+
+ Applies the same _check_geometry gate a real run would, with the same
+ `stampable` verdict per tensor, so a dry run over a real checkpoint answers
+ the question a real run would answer -- including which tensors sit at the
+ collision shape and are being permitted only because they will be stamped
+ (reported per-tensor as "ambiguous", and summarized by
+ _print_inventory_summary).
+
+ FULL-FAMILY EMISSION: also reports the non-FP8 pass-through tensors
+ (embed_tokens/lm_head/norms/router -- is_passthrough_target) that
+ repack_shard will copy byte-identically. These carry no scale sidecar and
+ no collision-shape question, so _check_geometry does not apply to them;
+ each is marked `"passthrough": True` in its inventory record (fp8-repack
+ records get `"passthrough": False`) so callers can filter by category."""
from safetensors import safe_open
inv = []
with safe_open(path, framework="pt") as f:
@@ -203,19 +653,34 @@ def shard_inventory(path, n_layers):
for name in f.keys():
sl = f.get_slice(name)
dt = sl.get_dtype()
- if not is_repack_target(name, dt, keys, n_layers):
- continue
- O, I = sl.get_shape()
- sc_slice = f.get_slice(name + "_scale_inv")
- nblkO, nblkI = sc_slice.get_shape()
- _check_geometry(name, O, I, nblkO, nblkI)
- inv.append({"name": name, "kind": classify(name, n_layers),
- "O": O, "I": I, "nblkO": nblkO, "nblkI": nblkI,
- "weight_bytes": O * I, "scale_bytes": nblkO * nblkI * 4})
+ if is_repack_target(name, dt, keys, n_layers, keep_mtp):
+ kind = classify(name, n_layers, keep_mtp=keep_mtp)
+ O, I = sl.get_shape()
+ sc_slice = f.get_slice(name + "_scale_inv")
+ nblkO, nblkI = sc_slice.get_shape()
+ _check_geometry(name, O, I, nblkO, nblkI, _stampable(kind))
+ inv.append({"name": name, "kind": kind, "passthrough": False,
+ "O": O, "I": I, "nblkO": nblkO, "nblkI": nblkI,
+ "stamped": _stampable(kind),
+ "ambiguous": _nblk(O) * _nblk(I) == O,
+ "weight_bytes": O * I, "scale_bytes": nblkO * nblkI * 4})
+ elif is_passthrough_target(name, n_layers, keep_mtp):
+ kind = classify(name, n_layers, keep_mtp=keep_mtp)
+ shape = sl.get_shape()
+ nbytes = _DTYPE_NBYTES.get(dt, 0)
+ w = 1
+ for d in shape:
+ w *= d
+ inv.append({"name": name, "kind": kind, "passthrough": True,
+ "O": shape[0] if shape else 0,
+ "I": shape[1] if len(shape) > 1 else 0,
+ "nblkO": 0, "nblkI": 0, "stamped": False,
+ "ambiguous": False,
+ "weight_bytes": w * nbytes, "scale_bytes": 0})
return inv
-def repack_shard(path, n_layers):
+def repack_shard(path, n_layers, keep_mtp=False):
"""Real mode: byte-preserve every selected tensor's weight bytes (raw e4m3,
no dequant/requant -- `.view(torch.uint8)` is a pure bit-reinterpret, verified
byte-identical against torch.float8_e4m3fn's own storage) and rename/flatten
@@ -223,10 +688,20 @@ def repack_shard(path, n_layers):
nblkO*nblkI elements, row-major [blkO,blkI] -- matching
scale[(o/128)*nblkI+i/128] on the read side). Returns (out_dict, inventory,
fmt_map): fmt_map is {weight_tensor_name: FORMAT_NAME} for every selected
- tensor -- the caller JSON-encodes it into the output shard's __metadata__
- (see the module docstring's METADATA STAMP section). Keyed by the WEIGHT
- name only (not the ".qs" sidecar) -- that's the name qt_resolve_fmt/
- qt_verify_fmt_stamp look the stamp up by on the read side."""
+ tensor the ENGINE WILL CONSULT A STAMP FOR (residents; see _stampable) --
+ the caller JSON-encodes it into the output shard's __metadata__ (see the
+ module docstring's METADATA STAMP section). Keyed by the WEIGHT name only
+ (not the ".qs" sidecar) -- that's the name qt_resolve_fmt/qt_verify_fmt_stamp
+ look the stamp up by on the read side. Routed experts are repacked but NOT
+ stamped: the engine's routed-expert load sites pass stamped_name=NULL, and
+ st.h's ST_FMT_STAMP_MAX is sized on that premise.
+
+ FULL-FAMILY EMISSION: also copies every is_passthrough_target tensor
+ (embed_tokens/lm_head/norms/router/bias) into `out` UNMODIFIED -- original
+ dtype, original shape (including 1-D norms), no `.qs`, never added to
+ fmt_map. This is a plain `f.get_tensor(name).contiguous()`, not a
+ view/reinterpret: there is no dtype conversion to undo (unlike the fp8
+ weight's uint8 view), so a plain copy is already byte-identical."""
import torch
from safetensors import safe_open
out = {}
@@ -237,19 +712,35 @@ def repack_shard(path, n_layers):
for name in f.keys():
sl = f.get_slice(name)
dt = sl.get_dtype()
- if not is_repack_target(name, dt, keys, n_layers):
- continue
- w = f.get_tensor(name) # torch.float8_e4m3fn [O,I]
- sc = f.get_tensor(name + "_scale_inv") # torch.float32 [nblkO,nblkI]
- O, I = w.shape
- nblkO, nblkI = sc.shape
- _check_geometry(name, O, I, nblkO, nblkI)
- out[name] = w.view(torch.uint8).contiguous() # BYTE-PRESERVED
- out[name + ".qs"] = sc.to(torch.float32).reshape(-1).contiguous()
- fmt_map[name] = FORMAT_NAME
- inv.append({"name": name, "kind": classify(name, n_layers),
- "O": O, "I": I, "nblkO": nblkO, "nblkI": nblkI,
- "weight_bytes": O * I, "scale_bytes": nblkO * nblkI * 4})
+ if is_repack_target(name, dt, keys, n_layers, keep_mtp):
+ kind = classify(name, n_layers, keep_mtp=keep_mtp)
+ stampable = _stampable(kind)
+ w = f.get_tensor(name) # torch.float8_e4m3fn [O,I]
+ sc = f.get_tensor(name + "_scale_inv") # torch.float32 [nblkO,nblkI]
+ O, I = w.shape
+ nblkO, nblkI = sc.shape
+ _check_geometry(name, O, I, nblkO, nblkI, stampable)
+ out[name] = w.view(torch.uint8).contiguous() # BYTE-PRESERVED
+ out[name + ".qs"] = sc.to(torch.float32).reshape(-1).contiguous()
+ if stampable:
+ fmt_map[name] = FORMAT_NAME
+ inv.append({"name": name, "kind": kind, "passthrough": False,
+ "O": O, "I": I, "nblkO": nblkO, "nblkI": nblkI,
+ "stamped": stampable,
+ "ambiguous": _nblk(O) * _nblk(I) == O,
+ "weight_bytes": O * I, "scale_bytes": nblkO * nblkI * 4})
+ elif is_passthrough_target(name, n_layers, keep_mtp):
+ kind = classify(name, n_layers, keep_mtp=keep_mtp)
+ w = f.get_tensor(name) # original dtype, unchanged
+ out[name] = w.contiguous() # BYTE-PRESERVED (plain copy)
+ shape = tuple(w.shape)
+ nbytes = w.element_size() * w.numel()
+ inv.append({"name": name, "kind": kind, "passthrough": True,
+ "O": shape[0] if shape else 0,
+ "I": shape[1] if len(shape) > 1 else 0,
+ "nblkO": 0, "nblkI": 0, "stamped": False,
+ "ambiguous": False,
+ "weight_bytes": nbytes, "scale_bytes": 0})
return out, inv, fmt_map
@@ -265,7 +756,91 @@ def _print_inventory_summary(all_inv, dry_run):
for kind in sorted(by_kind):
items = by_kind[kind]
w = sum(it["weight_bytes"] for it in items)
- print(f"{tag} kind={kind:5s} {len(items):5d} tensor(s) {w/1e9:.3f} GB")
+ stamped = sum(1 for it in items if it["stamped"])
+ print(f"{tag} kind={kind:5s} {len(items):5d} tensor(s) {w/1e9:.3f} GB "
+ f"{stamped} stamped")
+ # Report the collision-shape population explicitly rather than leaving it
+ # implicit in "no refusal happened": these are the tensors whose correct
+ # read-side resolution depends entirely on the stamp surviving downstream,
+ # and a run that emits some deserves to say so out loud.
+ amb = [it for it in all_inv if it["ambiguous"]]
+ if amb:
+ fams = sorted({(it["O"], it["I"]) for it in amb})
+ print(f"{tag} {len(amb)} tensor(s) at the AMBIGUOUS shape "
+ f"(nblkO*nblkI==O), all stamped '{FORMAT_NAME}'; shape famil"
+ f"{'y' if len(fams) == 1 else 'ies'}: "
+ + ", ".join(f"[{o},{i}]" for o, i in fams))
+ print(f"{tag} {sum(1 for it in all_inv if it['stamped'])} stamp entr"
+ f"{'y' if sum(1 for it in all_inv if it['stamped']) == 1 else 'ies'} "
+ f"total (st.h ST_FMT_STAMP_MAX={ST_FMT_STAMP_MAX})")
+
+
+# M1 audit, section 3 (standalone-load checklist): colibri.c's st_init_multi
+# globs *.safetensors directly and builds its own tensor index -- no
+# HuggingFace-style model.safetensors.index.json is required or consulted
+# (and copying it byte-identical from --indir would be actively WRONG here:
+# its weight_map keys point at the SOURCE shard names, model-NNNNN-of-00141
+# .safetensors, which this tool does not reproduce -- the mint renames/repacks
+# into out-fp8pass-*.safetensors, so a copied index would describe a layout
+# the minted directory does not have).
+#
+# The EXTERNAL (non-tensor) files the loader/server actually open from a
+# model dir at runtime, verified against this worktree's HEAD:
+# - config.json cfg_root, colibri.c:1361 -- fopen(...); if(!f){
+# perror(p); exit(1); } -- MANDATORY, the run aborts without it. Also
+# read by openai_server.py's Engine.__init__ (:1773) for arch detection.
+# - generation_config.json colibri.c:1404-1405 -- fopen, comment "assente
+# = nessun problema: e' opzionale" -- best-effort; HF's authority for
+# generation defaults (extra EOS stop ids) when present.
+# - tokenizer.json c/tok.h:101 (tk_read_file, called from tok_load)
+# -- fopen(...); if(!f){ perror(path); exit(1); } -- MANDATORY. Called
+# from every serve/generate entry point that needs a tokenizer
+# (colibri.c:7294 run_text, :7952 run_serve_mux, :8134 main serve loop).
+# Before this fix main() never copied it into --outdir, so a minted
+# directory was not standalone-loadable for THIS reason alone (confirmed
+# by V2's end-to-end smoke test, 2026-08-18: load only succeeded via a
+# hand-symlinked tokenizer.json borrowed from the sibling source dir).
+#
+# Files confirmed NOT read anywhere in colibri.c / c/*.h / openai_server.py
+# (grepped this worktree; only docstrings/comments reference them, never an
+# fopen/Path().open against a model dir) and therefore excluded:
+# - tokenizer_config.json, chat_template.jinja -- the GLM-5.2 chat template
+# is reimplemented natively in code, not read from the .jinja file
+# (colibri.c:8211 comment: "template UFFICIALE GLM-5.2 (chat_template
+# .jinja): niente \n dopo i ruoli..."; openai_server.py:1016: "AUTHORITATIVE
+# GLM-5.2 tool-declaration block (byte-matches chat_template.jinja)" --
+# both hardcoded to match the file's behavior, not sourced from it).
+# - README.md, LICENSE, .gitattributes -- pure repo/documentation metadata,
+# zero references in any loader or server code path.
+# Before this fix, main() never copied config.json or generation_config.json
+# either, so a minted directory was not standalone-loadable for this reason
+# ALONE, independent of any tensor-family gap.
+STANDALONE_LOAD_REQUIRED_FILES = ("config.json", "tokenizer.json")
+STANDALONE_LOAD_OPTIONAL_FILES = ("generation_config.json",)
+
+
+def _copy_standalone_load_files(indir, outdir):
+ """Copy the external (non-tensor) files a minted --outdir needs to be
+ loadable on its own, per STANDALONE_LOAD_REQUIRED_FILES/_OPTIONAL_FILES.
+ Refuses loudly (like every other "about to emit an unloadable container"
+ check in this tool) if a REQUIRED file is missing from --indir; an
+ OPTIONAL file is copied if present and silently skipped otherwise (the
+ same best-effort posture load_cfg itself takes for
+ generation_config.json). Idempotent: safe to call once per pass (main
+ and --mtp each call it) since both write into the same --outdir."""
+ for fname in STANDALONE_LOAD_REQUIRED_FILES:
+ src = os.path.join(indir, fname)
+ if not os.path.exists(src):
+ print(f"ERROR: {src} not found -- the engine hard-exits without "
+ f"{fname} in the model directory (see the fopen site cited "
+ f"above STANDALONE_LOAD_REQUIRED_FILES); refusing to emit a "
+ f"container that cannot load", file=sys.stderr)
+ sys.exit(1)
+ shutil.copy2(src, os.path.join(outdir, fname))
+ for fname in STANDALONE_LOAD_OPTIONAL_FILES:
+ src = os.path.join(indir, fname)
+ if os.path.exists(src):
+ shutil.copy2(src, os.path.join(outdir, fname))
def main():
@@ -276,28 +851,57 @@ def main():
ap.add_argument("--n-layers", type=int, default=78)
ap.add_argument("--dry-run", action="store_true",
help="inventory only: scan headers, print selection counts, write nothing")
+ # Same shape as convert_fp8_to_int4.py's --mtp: a SEPARATE pass over the same
+ # --indir selecting ONLY layer n_layers (the multi-token-prediction head), into
+ # out-mtp-fp8pass-*.safetensors alongside the main pass's shards in the same
+ # --outdir. The engine detects MTP purely by tensor NAME presence
+ # (model.layers.<n_layers>.*, colibri.c's has_mtp probe) -- the "out-mtp-"
+ # prefix is a human/tooling convention (convert_fmt4_to_fmt2.py keys off it),
+ # not something the loader parses. Two passes rather than one flag on the main
+ # pass because the resume manifest and the #383-class params guard are both
+ # per-selection: mixing MTP into a resumed main run would make "shard done"
+ # mean two different things.
+ ap.add_argument("--mtp", action="store_true",
+ help="repack ONLY the MTP head (model.layers.<n_layers>.*) -> "
+ "out-mtp-fp8pass-*.safetensors")
a = ap.parse_args()
shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors")))
if not shards:
print(f"ERROR: no *.safetensors files in {a.indir}"); sys.exit(1)
+ out_prefix = "out-mtp-fp8pass-" if a.mtp else "out-fp8pass-"
+ guard_prefix = "fp8pass-mtp-" if a.mtp else "fp8pass-"
+ prog_name = PROGRESS_MANIFESTS[1] if a.mtp else PROGRESS_MANIFESTS[0]
+
if a.dry_run:
all_inv = []
for sp in shards:
- all_inv.extend(shard_inventory(sp, a.n_layers))
+ all_inv.extend(shard_inventory(sp, a.n_layers, a.mtp))
_print_inventory_summary(all_inv, dry_run=True)
+ # Container-wide, like the real run below: a dry run of one pass over an
+ # outdir the other pass has already written into must answer the question
+ # the ENGINE will ask (the whole directory), not just this pass's share.
+ _check_stamp_budget(sum(1 for it in all_inv if it["stamped"])
+ + _stamps_from_other_passes(a.outdir, prog_name))
return
os.makedirs(a.outdir, exist_ok=True)
+ # M1 audit / F3: config.json and tokenizer.json are mandatory for the
+ # container to be standalone-loadable; generation_config.json is
+ # best-effort. See STANDALONE_LOAD_REQUIRED_FILES/_OPTIONAL_FILES above
+ # for the per-file fopen citations. Called on every pass (main and --mtp
+ # both write into this --outdir) -- idempotent.
+ _copy_standalone_load_files(a.indir, a.outdir)
# #383-class guard, reused (not reimplemented): refuses a resumed run whose
# params differ from what's already partially in this outdir (the #355 failure
# mode -- a second pass with different flags silently mixing containers).
# Owns its own sidecar (.fp8pass-params.json), separate from the shard-progress
# manifest below, exactly like the --repo mtp/indexer loops in
# convert_fp8_to_int4.py use it.
- params = {"mode": "fp8-passthrough", "n_layers": a.n_layers}
- if not check_or_record_params(a.outdir, "fp8pass-", params):
+ params = {"mode": "fp8-passthrough-mtp" if a.mtp else "fp8-passthrough",
+ "n_layers": a.n_layers}
+ if not check_or_record_params(a.outdir, guard_prefix, params):
sys.exit(1)
# RESUME (same #383-class manifest idiom as convert_fp8_to_int4.py's --indir path):
@@ -306,7 +910,7 @@ def main():
# never leaves a half-written manifest for the next invocation to trust. The
# params guard above already owns "don't mix conversions" -- this manifest is
# purely which shards are done.
- prog_path = os.path.join(a.outdir, ".fp8pass-progress.json")
+ prog_path = os.path.join(a.outdir, prog_name)
prog = {}
if os.path.exists(prog_path):
try:
@@ -314,6 +918,16 @@ def main():
except (OSError, ValueError):
prog = {}
done = prog.setdefault("shards", {})
+ # ST_FMT_STAMP_MAX is a CONTAINER-wide bound, and a resumed run only sees the
+ # shards it processes itself -- so the running total lives in the manifest
+ # next to the shard progress, not in a local counter that a resume resets.
+ # "Container-wide" also spans the main/--mtp split, which share one --outdir and
+ # therefore one container while keeping separate manifests: this pass's total is
+ # only its own share, so the OTHER pass's recorded stamps are added at the check
+ # (see _stamps_from_other_passes). Read once here rather than per shard -- the
+ # two passes are sequential by construction (each is a separate invocation).
+ prog.setdefault("stamped", 0)
+ other_stamped = _stamps_from_other_passes(a.outdir, prog_name)
from safetensors.torch import save_file
all_inv = []
@@ -326,18 +940,36 @@ def main():
n += 1
skipped += 1
continue
- out, inv, fmt_map = repack_shard(sp, a.n_layers)
+ out, inv, fmt_map = repack_shard(sp, a.n_layers, a.mtp)
all_inv.extend(inv)
if not out:
done[key] = ""
else:
- name = f"out-fp8pass-{n:05d}.safetensors"
+ name = f"{out_prefix}{n:05d}.safetensors"
+ # I-4 STRUCTURAL GATE: last check before the bytes exist on disk,
+ # derived from `out` itself rather than from the selection that built
+ # it. See _emission_stamp_gate.
+ _emission_stamp_gate(out, fmt_map)
+ prog["stamped"] += len(fmt_map)
+ _check_stamp_budget(prog["stamped"] + other_stamped)
# METADATA STAMP: __metadata__ values must be strings (safetensors
# constraint) -- colibri.fmt's VALUE is itself JSON text (a map of
# tensor name -> format NAME), not a nested object, for exactly
# that reason. sort_keys for a deterministic, diffable stamp.
- meta = {METADATA_KEY: json.dumps(fmt_map, sort_keys=True)}
- save_file(out, os.path.join(a.outdir, name), metadata=meta)
+ # colibri.container.formats is the CONTAINER-level declaration (see
+ # the module docstring): written on every emitted shard, including
+ # shards that carry only unstamped routed experts -- it is a claim
+ # about the container, not about this shard's contents.
+ meta = {METADATA_KEY: json.dumps(fmt_map, sort_keys=True),
+ CONTAINER_FORMATS_KEY: json.dumps([FORMAT_NAME])}
+ out_path = os.path.join(a.outdir, name)
+ save_file(out, out_path, metadata=meta)
+ # DETERMINISM FIX (D1, colibri_lab/reports/D1_MINT_DIVERGENCE_2026-08-18.md):
+ # save_file's Rust extension does not preserve `meta`'s key order into
+ # the written header -- see _canonicalize_metadata_order's docstring.
+ # Fix up the two bytes that can vary, in place, right after the file
+ # is written and before this shard is recorded done.
+ _canonicalize_metadata_order(out_path)
done[key] = name
n += 1
fresh += 1