Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fp8: self-describing __metadata__ stamp, trust-verify-refuse reader

monotophic3a0f01fa3b6c
diff --git a/c/colibri.c b/c/colibri.c
index 7c8cca5a..6dc410bc 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1294,6 +1294,46 @@ static int detect_group_size(int O, int I, int64_t ns){
return 0;
}
+/* FORMAT NAME <-> internal fmt-int table. The NAME is the public identity a
+ * container or a Feature Request advertises; the int on the right is this
+ * build's internal enum value (colibri.c's QT.fmt / qt_resolve_fmt's return
+ * value), never itself persisted to a container -- a container's __metadata__
+ * stamp (below) carries the NAME, never the number, matching docs/FORMATS.md's
+ * own registry. Covers every format qt_resolve_fmt can return, not just the
+ * one this branch's tool stamps: a single-entry table could only ever
+ * exercise the "unrecognized name" refusal path, never a genuine
+ * recognized-but-different-format mismatch. "e8-iq3-lattice" is this build's
+ * own placeholder name for upstream fmt=6 (#465 never stamped containers --
+ * dev has no metadata-stamp feature of its own) -- listed so
+ * qt_fmt_by_name/qt_name_by_fmt are total over every value qt_resolve_fmt can
+ * return, matching this comment's own claim; a real name for fmt=6 belongs to
+ * whoever upstreams a stamp for it. "fp8-e4m3-b128" (fmt=8) names the WEIGHT
+ * geometry only, not a specific scale encoding -- a tensor stamped
+ * "fp8-e4m3-b128" whose scale sidecar carries UE8M0 bytes still refuses (see
+ * qt_resolve_fmt's "SCALE ENCODING IS A DECLARED PROPERTY" comment): the stamp
+ * confirms the WEIGHT format, it does not grant this build a decoder it
+ * doesn't have. */
+static const struct { const char *name; int fmt; } FMT_NAMES[] = {
+ { "f32", 0 },
+ { "int8-row", 1 },
+ { "int4-row", 2 },
+ { "int2-row", 3 },
+ { "int4-grouped", 4 },
+ { "int3-g64", 5 },
+ { "e8-iq3-lattice", 6 },
+ { "fp8-e4m3-b128", 8 },
+};
+#define N_FMT_NAMES (int)(sizeof(FMT_NAMES)/sizeof(FMT_NAMES[0]))
+
+static int qt_fmt_by_name(const char *name){
+ for(int i=0;i<N_FMT_NAMES;i++) if(!strcmp(FMT_NAMES[i].name,name)) return FMT_NAMES[i].fmt;
+ return -1; /* unrecognized name -- never a valid qt_resolve_fmt() return value */
+}
+static const char *qt_name_by_fmt(int fmt){
+ for(int i=0;i<N_FMT_NAMES;i++) if(FMT_NAMES[i].fmt==fmt) return FMT_NAMES[i].name;
+ return NULL; /* fmt has no registered public name yet (e.g. upstream 0-5) */
+}
+
/* SEC: risolve e VALIDA il formato quantizzato di un tensore [O,I] letto da un
* container non fidato (mirror). L'inferenza precedente (`?1:?2:3`) cadeva su
* int2 per QUALSIASI conteggio byte non riconosciuto: un peso troppo corto
@@ -1301,11 +1341,13 @@ static int detect_group_size(int O, int I, int64_t ns){
* 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte
* della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) — altrimenti
* si termina invece di sforare. Ritorna fmt (1/2/3/4/5/6/8) e scrive *gs.
- * No container-level format stamp is consulted here: a self-describing
- * __metadata__ stamp that could resolve a genuine byte-collision below
- * (instead of refusing it unconditionally) is a follow-up proposal, not
- * present in this build -- every ambiguous shape this function can see
- * refuses, full stop. */
+ * No container-level format stamp is consulted INSIDE this function yet in
+ * this commit -- qt_verify_fmt_stamp (below) checks one AFTER the fact, a
+ * pure post-hoc cross-check that cannot yet resolve a genuine byte-collision
+ * below (every ambiguous shape this function can see still refuses
+ * unconditionally here). A follow-up commit threads a `stamped_name`
+ * parameter into this function itself so a stamp can resolve those
+ * collisions instead of merely confirming an already-unambiguous result. */
static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){
int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4);
int64_t exp_i3=(int64_t)O*i3_rowbytes(I); /* int3-g64 (fmt=5): 24B per 64-input group */
@@ -1461,6 +1503,48 @@ static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns
name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); }
return fmt;
}
+
+/* TRUST-VERIFY-REFUSE: if `stamped` (the tensor's __metadata__ format-NAME
+ * stamp, or NULL if none -- see st_fmt_stamp/st_fmt_stamp_ingest in st.h)
+ * is present, verify it agrees with `fmt` (qt_resolve_fmt's byte-arithmetic
+ * result) and refuse loudly on disagreement -- same "untrusted container"
+ * discipline qt_resolve_fmt applies throughout. A stamp naming a format this
+ * build doesn't recognize is ALSO a refusal: silently accepting an
+ * unrecognized name would be indistinguishable from missing a real mismatch,
+ * and "refuse rather than guess" is the whole point of this function's
+ * design. No stamp at all is NOT an error -- the container simply predates
+ * this feature (or was never stamped by a stamping tool), and byte-arithmetic
+ * inference alone decides, exactly as before this function existed: zero
+ * behavior change for unstamped containers.
+ *
+ * Called from qt_from_disk right after qt_resolve_fmt -- the resident-tensor
+ * load path this branch's repack tool actually stamps. Deliberately NOT
+ * threaded into the routed-expert loader paths (expert_load_impl and
+ * friends, which call qt_resolve_fmt separately for g/u/d slab layout): this
+ * branch's tools/repack_fp8_passthrough.py never stamps routed experts (kind
+ * "x" is explicitly excluded, see that tool's module docstring), so there is
+ * no stamp for those paths to verify yet -- adding the plumbing there now
+ * would be framework-building ahead of any container that needs it, which
+ * this reference implementation deliberately avoids. */
+static void qt_verify_fmt_stamp(const char *name, const char *stamped, int fmt){
+ if(!stamped) return; /* unstamped: infer exactly as today */
+ int stamped_fmt = qt_fmt_by_name(stamped);
+ if(stamped_fmt == fmt) return; /* agree: silent pass-through, loads normally */
+ if(stamped_fmt < 0){
+ fprintf(stderr,
+ "%s: metadata stamp names format '%s', which this build does not recognize "
+ "(byte-arithmetic inference says fmt=%d) -- refusing (untrusted container, "
+ "unrecognized stamp name)\n", name, stamped, fmt);
+ exit(1);
+ }
+ const char *inferred_name = qt_name_by_fmt(fmt);
+ fprintf(stderr,
+ "%s: metadata stamp says format '%s' but byte-arithmetic inference says fmt=%d "
+ "(%s) -- refusing (untrusted container, stamp/inference mismatch)\n",
+ name, stamped, fmt, inferred_name ? inferred_name : "no registered name");
+ exit(1);
+}
+
/* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate).
* - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti
* - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni)
@@ -1475,6 +1559,8 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int
* non fidati (SEC). */
int gs=0;
int fmt = qt_resolve_fmt(name,O,I,nb,ns,&gs);
+ const char *stamped = st_fmt_stamp(&m->S,name); /* NULL if unstamped */
+ qt_verify_fmt_stamp(name,stamped,fmt); /* TRUST-VERIFY-REFUSE: no-op if unstamped */
if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); }
else if(fmt==4){ int ng=(I+gs-1)/gs;
/* METAL: t->s must be page-aligned + coli_metal_register'd like every other
diff --git a/c/st.h b/c/st.h
index 2b87f920..b01ffb03 100644
--- a/c/st.h
+++ b/c/st.h
@@ -49,6 +49,20 @@ typedef struct {
* (GLM: 256 expert x 78 layer x 3 x 2) la scansione lineare
* costava decine di secondi/token (misurato sul primo run reale) */
int hcap;
+ /* FORMAT METADATA STAMP (reference impl, see colibri.c's qt_verify_fmt_stamp):
+ * per-tensor {name -> format NAME string} pairs collected from every shard's
+ * __metadata__["colibri.fmt"] JSON blob (safetensors __metadata__ values are
+ * always strings, so colibri.fmt's value is itself JSON text, parsed a
+ * second time -- see st_init_multi below). Small in practice (only the
+ * tensors a stamping tool selected, a subset of S->t), so a flat array +
+ * linear st_fmt_stamp() lookup is fine; this is a reference implementation,
+ * not a framework -- no hash map for a handful-to-low-thousands of entries.
+ * Both arrays own strdup'd strings, intentionally leaked like the rest of
+ * st_init_multi's one-time startup parsing (see the json_parse callers
+ * below). */
+ char **fmt_name; /* stamped tensor name */
+ char **fmt_val; /* stamped format NAME string */
+ int fmt_n, fmt_cap;
} shards;
#define ST_MAX_SHARDS 512
@@ -225,6 +239,60 @@ static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char
}
}
+/* Parses one shard's __metadata__["colibri.fmt"] value (a safetensors metadata
+ * value is always a plain string, so colibri.fmt's VALUE is itself JSON text --
+ * a flat {tensor_name: format_name} object -- parsed a second time here) and
+ * appends every entry to S->fmt_name/fmt_val. Absent __metadata__, or a
+ * __metadata__ without a colibri.fmt key, is NOT an error: that's simply an
+ * unstamped container, and byte-arithmetic inference alone decides, exactly as
+ * before this feature existed (see qt_verify_fmt_stamp in colibri.c). A
+ * colibri.fmt key that IS present but doesn't parse into that shape is refused
+ * loudly -- same "untrusted container" discipline qt_resolve_fmt applies
+ * elsewhere: a stamp the engine cannot make sense of must not be silently
+ * ignored (that would be indistinguishable from a real mismatch going
+ * unnoticed). */
+static void st_fmt_stamp_ingest(shards *S, jval *root, const char *shard_path) {
+ jval *meta = json_get(root, "__metadata__");
+ if (!meta || meta->t != J_OBJ) return; /* no metadata object: unstamped, fine */
+ jval *stamp = json_get(meta, "colibri.fmt");
+ if (!stamp) return; /* no stamp key: unstamped, fine */
+ if (stamp->t != J_STR) {
+ fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] is not a JSON string -- malformed stamp, refusing (untrusted container)\n",
+ shard_path); exit(1); }
+ char *arena2 = NULL;
+ jval *inner = json_parse(stamp->str, &arena2);
+ if (!inner || inner->t != J_OBJ) {
+ fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] does not parse as a JSON object -- malformed stamp, refusing (untrusted container)\n",
+ shard_path); exit(1); }
+ for (int i = 0; i < inner->len; i++) {
+ jval *v = inner->kids[i];
+ if (v->t != J_STR) {
+ fprintf(stderr, "%s: colibri.fmt entry '%s' is not a string -- malformed stamp, refusing (untrusted container)\n",
+ shard_path, inner->keys[i]); exit(1); }
+ if (S->fmt_n == S->fmt_cap) {
+ S->fmt_cap = S->fmt_cap ? S->fmt_cap * 2 : 16;
+ S->fmt_name = realloc(S->fmt_name, S->fmt_cap * sizeof(char*));
+ S->fmt_val = realloc(S->fmt_val, S->fmt_cap * sizeof(char*));
+ }
+ S->fmt_name[S->fmt_n] = strdup(inner->keys[i]);
+ S->fmt_val[S->fmt_n] = strdup(v->str);
+ S->fmt_n++;
+ }
+ free(arena2); /* always NULL (json_parse never populates it -- see j_dup); the jval
+ * tree itself is intentionally leaked, same one-time-startup convention
+ * as st_init_multi's own root parse a few lines below. */
+}
+
+/* Stamped format NAME for `name`, or NULL if this tensor carries no stamp
+ * (either because no shard stamped it, or the container predates this
+ * feature). Linear scan: S->fmt_n is a subset of S->n (only stamped tensors),
+ * small in practice -- see the shards struct comment. */
+static const char *st_fmt_stamp(shards *S, const char *name) {
+ for (int i = 0; i < S->fmt_n; i++)
+ if (!strcmp(S->fmt_name[i], name)) return S->fmt_val[i];
+ return NULL;
+}
+
/* Scan one directory for *.safetensors shards, appending to files[] (dedup by
* basename, so a list of directories acts as a SEARCH PATH: the same shard
* present on two drives is taken from the first-listed one only). *added
@@ -311,6 +379,7 @@ static void st_init_multi(shards *S, const char *snap_dir, const char *extra_dir
jval *root = json_parse(hdr, &arena);
if (!root || root->t != J_OBJ) {
fprintf(stderr, "%s: safetensors header is not a JSON object\n", files[fi]); exit(1); }
+ st_fmt_stamp_ingest(S, root, files[fi]);
for (int i = 0; i < root->len; i++) {
const char *name = root->keys[i];
if (!strcmp(name, "__metadata__")) continue;
diff --git a/c/tests/test_fp8_e2e_loader.c b/c/tests/test_fp8_e2e_loader.c
index 0c847877..4cdebd2f 100644
--- a/c/tests/test_fp8_e2e_loader.c
+++ b/c/tests/test_fp8_e2e_loader.c
@@ -15,9 +15,9 @@
* Usage: test_fp8_e2e_loader <container_dir> [name O I]...
* For every (name,O,I) triple, calls the REAL qt_from_disk (the identical
* function every model load uses) and asserts:
- * (a) fmt==8 resolved (native fp8-e4m3-passthrough, via byte-arithmetic
- * inference alone -- this PR's repack tool writes no container
- * metadata stamp, see repack_fp8_passthrough.py's module docstring);
+ * (a) fmt==8 resolved (native fp8-e4m3-passthrough -- byte-arithmetic
+ * inference AND, since this tool stamps its output, metadata-stamp
+ * agreement, both exercised for real here, neither mocked);
* (b) the weight (q8) and scale (s) buffers are non-NULL;
* (c) every dequantized value (e4m3_decode(byte) * block scale) is finite
* -- catches a decode-table or block-index bug a pure byte-count
diff --git a/c/tests/test_fp8_e2e_repack_load.py b/c/tests/test_fp8_e2e_repack_load.py
index 024c13a0..844a8c82 100644
--- a/c/tests/test_fp8_e2e_repack_load.py
+++ b/c/tests/test_fp8_e2e_repack_load.py
@@ -18,9 +18,8 @@ test does, end to end:
Makefile's own CFLAGS, asserted to produce zero warnings -- into a
temp binary;
4. run that binary against the REAL repacked output directory, asserting
- every selected tensor loads fmt=8 through the real qt_from_disk with
- finite dequantized values (byte-arithmetic inference alone -- this PR's
- repack tool writes no container metadata stamp).
+ every stamped tensor loads fmt=8 through the real qt_from_disk with
+ finite dequantized values.
Hermetic: everything (checkpoint, repacked output, compiled harness binary)
lives under one TemporaryDirectory; nothing is left behind.
@@ -86,7 +85,7 @@ class Fp8RepackLoadE2ETest(unittest.TestCase):
"model.layers.0.self_attn.q_a_proj.weight": torch.randn(D, D) * 0.02,
"model.layers.0.mlp.shared_experts.gate_proj.weight": torch.randn(D, I_) * 0.02,
"model.layers.0.mlp.gate_proj.weight": torch.randn(D, I_) * 0.02,
- # routed expert: must NOT be selected (stays int4-g64 path) -- included
+ # routed expert: must NOT be selected/stamped (stays int4-g64 path) -- included
# as a negative control so this test also proves the real tool's selection
# logic held on the real round trip, not just in test_fp8_repack.py's own suite.
"model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(I_, D) * 0.02,
@@ -113,24 +112,19 @@ class Fp8RepackLoadE2ETest(unittest.TestCase):
outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
self.assertEqual(len(outs), 1, "expected exactly one repacked output shard")
- # Sanity-check the real tool's own selection BEFORE asking the C harness to load
- # it: if this fails, the bug is in the tool, not the loader, and running the
- # harness anyway would only muddy which side broke. This PR's repack tool writes
- # NO container metadata stamp (that's a separate, follow-up PR -- see the tool's
- # own module docstring), so selection is checked by tensor presence instead of a
- # stamp's tensor-name set, and the stamp's ABSENCE is itself asserted below: a
- # regression guard that this PR's tool really stays out of that scope.
+ # Sanity-check the real tool's own selection/stamp BEFORE asking the C harness
+ # to load it: if this fails, the bug is in the tool, not the loader, and running
+ # the harness anyway would only muddy which side broke.
with open(outs[0], "rb") as f:
hlen = struct.unpack("<Q", f.read(8))[0]
hdr = json.loads(f.read(hlen))
- for name in shapes:
- self.assertIn(name, hdr, f"{name}: expected tensor missing from repacked output")
- self.assertIn(name + ".qs", hdr, f"{name}.qs: expected scale sidecar missing")
+ self.assertIn("__metadata__", hdr)
+ self.assertIn("colibri.fmt", hdr["__metadata__"])
+ stamp = json.loads(hdr["__metadata__"]["colibri.fmt"])
+ self.assertEqual(set(stamp.keys()), set(shapes.keys()),
+ "real tool's selection must match exactly the resident-kind tensors")
self.assertNotIn("model.layers.0.mlp.experts.0.gate_proj.weight", hdr)
self.assertNotIn("model.layers.0.input_layernorm.weight", hdr)
- self.assertNotIn("__metadata__", hdr,
- "this PR's repack tool must not write a container metadata stamp "
- "(stamp writer is a separate, follow-up PR)")
cc, cflags, ldflags = _cc_flags()
if not cc:
diff --git a/c/tests/test_fp8_repack.py b/c/tests/test_fp8_repack.py
index f95366bc..3e69f061 100644
--- a/c/tests/test_fp8_repack.py
+++ b/c/tests/test_fp8_repack.py
@@ -133,7 +133,7 @@ class SelectionTest(unittest.TestCase):
self.assertNotIn("model.layers.0.mlp.gate.weight", names) # router, f32 kind
def test_byte_preservation_and_qs_rename(self):
- out, inv = rp.repack_shard(self.shard, n_layers=5)
+ out, inv, fmt_map = rp.repack_shard(self.shard, n_layers=5)
self.assertEqual(len(inv), 4)
with safe_open(self.shard, framework="pt") as f:
for it in inv:
@@ -147,6 +147,19 @@ class SelectionTest(unittest.TestCase):
nblkO, nblkI = (O + 127) // 128, (I_ + 127) // 128
self.assertEqual(out[name + ".qs"].numel(), nblkO * nblkI)
+ def test_fmt_map_covers_exactly_selected_weight_names(self):
+ """fmt_map (the METADATA STAMP payload) must cover exactly the selected
+ WEIGHT names -- never the ".qs" sidecars, never an excluded tensor -- and
+ every value must be FORMAT_NAME (the format's public NAME, not the
+ internal fmt=8 ordinal)."""
+ out, inv, fmt_map = rp.repack_shard(self.shard, n_layers=5)
+ selected_names = {it["name"] for it in inv}
+ self.assertEqual(set(fmt_map.keys()), selected_names)
+ for name in selected_names:
+ self.assertNotIn(name + ".qs", fmt_map)
+ self.assertEqual(fmt_map[name], rp.FORMAT_NAME)
+ self.assertEqual(rp.FORMAT_NAME, "fp8-e4m3-b128")
+
def test_geometry_refusal_on_malformed_scale(self):
"""A shard whose _scale_inv shape doesn't match ceil(O/128)xceil(I/128) for
its weight must be REFUSED (write-side twin of qt_resolve_fmt's read-side
@@ -227,6 +240,27 @@ class ResumeAndParamsGuardTest(unittest.TestCase):
self.assertNotIn("model.layers.0.input_layernorm.weight", hdr)
self.assertNotIn("model.layers.0.self_attn.kv_b_proj.weight", hdr)
+ def test_metadata_stamp_present_and_correct(self):
+ """End-to-end through the real CLI: __metadata__["colibri.fmt"] must be
+ present, parse as JSON, and name exactly the selected resident-kind
+ tensors -- the real round trip a unit-level rp.repack_shard() call
+ can't exercise (save_file's own metadata= handling)."""
+ self.assertEqual(self._run(5), 0)
+ outs = glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))
+ self.assertEqual(len(outs), 1)
+ hdr = _read_header(outs[0])
+ self.assertIn("__metadata__", hdr)
+ self.assertIn("colibri.fmt", hdr["__metadata__"])
+ stamp = json.loads(hdr["__metadata__"]["colibri.fmt"])
+ self.assertEqual(set(stamp.keys()), {
+ "model.layers.0.self_attn.o_proj.weight",
+ "model.layers.0.self_attn.q_a_proj.weight",
+ "model.layers.0.mlp.shared_experts.gate_proj.weight",
+ "model.layers.0.mlp.gate_proj.weight",
+ })
+ for name, fmt_name in stamp.items():
+ self.assertEqual(fmt_name, rp.FORMAT_NAME, f"{name}: stamped format name mismatch")
+
def test_resume_skips_completed_shard(self):
self.assertEqual(self._run(5), 0)
mtime1 = os.path.getmtime(glob.glob(os.path.join(self.outdir, "out-fp8pass-*.safetensors"))[0])
diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.py
index 15f27c34..354cd4ef 100644
--- a/c/tools/repack_fp8_passthrough.py
+++ b/c/tools/repack_fp8_passthrough.py
@@ -75,12 +75,23 @@ to fmt=8 needs real fmt=8 CPU+CUDA absorb-path support first -- out of scope
for this build, so this tool refuses to produce a container the engine
cannot safely read.
-This tool does NOT write a container metadata stamp: __metadata__-based
-self-description (writer + qt_verify_fmt_stamp reader + docs/FORMATS.md
-registry) is a separate, follow-up PR, per the maintainer's #524 scoping.
-This PR's collision/refusal logic (qt_resolve_fmt) therefore refuses
-UNCONDITIONALLY at every ambiguous shape this tool could produce -- there is
-no stamp to break a tie.
+METADATA STAMP (reference implementation of the FORMATS-registry FR -- see
+docs/FORMATS.md): every output shard's safetensors `__metadata__` carries a
+`colibri.fmt` key whose VALUE is itself a JSON-encoded object mapping each
+stamped tensor's exact name to its format NAME string (FORMAT_NAME below,
+"fp8-e4m3-b128" -- never the internal fmt=8 ordinal, which the container must
+never depend on, see colibri.c's QT struct comment). The reader
+(qt_from_disk's qt_verify_fmt_stamp, colibri.c) is TRUST-VERIFY-REFUSE: a
+stamp that agrees with the byte-arithmetic inference is a no-op (the tensor
+loads exactly as it would unstamped); a stamp that disagrees, or names a
+format this build doesn't recognize, is refused loudly (same "untrusted
+container" discipline as qt_resolve_fmt's own THE DESIGN LANDMINE refusal).
+As a bonus, a stamp can also RESOLVE a genuine byte-count collision (the
+fmt=1-vs-fmt=8 and fmt=6-vs-fmt=8 cases documented in qt_resolve_fmt) instead
+of the collision refusing unconditionally -- see qt_resolve_fmt's own
+documentation for the exact rule. Unstamped containers (any container from a
+tool that doesn't stamp) are unaffected: no stamp means inference alone
+decides, exactly as before this feature existed.
HARD CONSTRAINT for this build: unit-tested on synthetic fixtures ONLY (see
tests/test_fp8_repack.py, built with tools/glm_fp8_emit.py's exact real-
@@ -105,6 +116,18 @@ from convert_fp8_to_int4 import classify, check_or_record_params # reuse: same
# qt_matvec_rows) and the CUDA absorb kernels have no fmt=8 case yet.
RESIDENT_KINDS = frozenset({"sh", "o", "attn", "dmlp", "q"})
+# The format's PUBLIC identity (what containers/FRs advertise) -- the internal
+# fmt=8 ordinal is a colibri.c-only enum value, never itself persisted (see
+# that file's QT struct comment). Must match the reader's FMT_NAMES table
+# (colibri.c, near qt_resolve_fmt) or every stamp this tool writes will be
+# refused as "unrecognized format name" on load.
+FORMAT_NAME = "fp8-e4m3-b128"
+
+# safetensors __metadata__ key this tool stamps: JSON-encoded {tensor_name:
+# format_name}. Kept as a module constant so the reader-side doc and any
+# future tooling can cite the exact string instead of re-typing it.
+METADATA_KEY = "colibri.fmt"
+
BLOCK = 128
@@ -190,11 +213,17 @@ def repack_shard(path, n_layers):
byte-identical against torch.float8_e4m3fn's own storage) and rename/flatten
the `_scale_inv` sidecar to the engine's `name.qs` convention (flat F32,
nblkO*nblkI elements, row-major [blkO,blkI] -- matching
- scale[(o/128)*nblkI+i/128] on the read side). Returns (out_dict, inventory)."""
+ scale[(o/128)*nblkI+i/128] on the read side). Returns (out_dict, inventory,
+ fmt_map): fmt_map is {weight_tensor_name: FORMAT_NAME} for every selected
+ tensor -- the caller JSON-encodes it into the output shard's __metadata__
+ (see the module docstring's METADATA STAMP section). Keyed by the WEIGHT
+ name only (not the ".qs" sidecar) -- that's the name qt_resolve_fmt/
+ qt_verify_fmt_stamp look the stamp up by on the read side."""
import torch
from safetensors import safe_open
out = {}
inv = []
+ fmt_map = {}
with safe_open(path, framework="pt") as f:
keys = set(f.keys())
for name in f.keys():
@@ -209,10 +238,11 @@ def repack_shard(path, n_layers):
_check_geometry(name, O, I, nblkO, nblkI)
out[name] = w.view(torch.uint8).contiguous() # BYTE-PRESERVED
out[name + ".qs"] = sc.to(torch.float32).reshape(-1).contiguous()
+ fmt_map[name] = FORMAT_NAME
inv.append({"name": name, "kind": classify(name, n_layers),
"O": O, "I": I, "nblkO": nblkO, "nblkI": nblkI,
"weight_bytes": O * I, "scale_bytes": nblkO * nblkI * 4})
- return out, inv
+ return out, inv, fmt_map
def _print_inventory_summary(all_inv, dry_run):
@@ -288,13 +318,18 @@ def main():
n += 1
skipped += 1
continue
- out, inv = repack_shard(sp, a.n_layers)
+ out, inv, fmt_map = repack_shard(sp, a.n_layers)
all_inv.extend(inv)
if not out:
done[key] = ""
else:
name = f"out-fp8pass-{n:05d}.safetensors"
- save_file(out, os.path.join(a.outdir, name))
+ # METADATA STAMP: __metadata__ values must be strings (safetensors
+ # constraint) -- colibri.fmt's VALUE is itself JSON text (a map of
+ # tensor name -> format NAME), not a nested object, for exactly
+ # that reason. sort_keys for a deterministic, diffable stamp.
+ meta = {METADATA_KEY: json.dumps(fmt_map, sort_keys=True)}
+ save_file(out, os.path.join(a.outdir, name), metadata=meta)
done[key] = name
n += 1
fresh += 1