Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

test(fp8): e2e serve-batch pin, refusal canary, and mint-to-load regression

monotophicea8a7e4d55d9
diff --git a/c/Makefile b/c/Makefile
index 1ee880c1..eed82704 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -498,12 +498,17 @@ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(f
# "unsupported option '-msse4.1' for target arm64-...".
# Appended back below only on x86-64 hosts, alongside the other conditional
# platform tests already handled the same way.
+# test_e8x4g64_loader takes a minted container directory on argv and prints its
+# usage (exit 2) without one, so it is a harness rather than a gate: it has a
+# rule here so it compiles under the suite's own $(CFLAGS) and its warnings are
+# caught, and tests/test_e8x4g64_mint_load.py is what actually drives it.
# test_qwen38_tier_engine drives qwen38.c over the generated FP8 fixture
# (qwen38_tiny_fp8, gitignored); it runs from qwen38-tier-engine-check.
TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \
test_segment_adapters_registration test_segment_adapters_real \
test_edge_adapters_registration test_edge_adapters_real \
- test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine
+ test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine \
+ test_e8x4g64_loader
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
@@ -2096,6 +2101,14 @@ tests/test_olmoe_dot_i8_16_sse41$(EXE): tests/test_olmoe_dot_i8_16.c olmoe.c sse
tests/test_shard_kvb_refuse$(EXE): tests/test_shard_kvb_refuse.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ)
$(CC) $(CFLAGS) $< $(CUDA_OBJ) -o $@ $(LDFLAGS)
+# The e8x4g64 container loader harness. It takes a minted directory on argv and
+# is driven by tests/test_e8x4g64_mint_load.py, which builds it too -- this rule
+# exists so it also compiles under the suite's own $(CFLAGS) rather than only
+# under the driver's hand-copied flag list, and so a warning regression in it
+# fails the normal build.
+tests/test_e8x4g64_loader$(EXE): tests/test_e8x4g64_loader.c st.h quant.h fp8_format.h compat.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
test-c: $(TEST_BINS)
$(PYTHON) tools/run_tests.py $(TEST_BINS)
diff --git a/c/tests/test_e8x4g64_loader.c b/c/tests/test_e8x4g64_loader.c
new file mode 100644
index 00000000..32fb1707
--- /dev/null
+++ b/c/tests/test_e8x4g64_loader.c
@@ -0,0 +1,108 @@
+/* wq-v0-class (e8x4g64: int8 spine + grouped-int4 g64 experts) END-TO-END
+ * loader harness -- the C-side half of the mint->load regression case.
+ *
+ * tests/test_e8x4g64_mint_load.py drives it: builds a synthetic FP8 GLM-shaped
+ * checkpoint (tools/glm_fp8_emit.py, the same fixture helper the fp8
+ * passthrough e2e uses), mints it with the REAL tools/convert_fp8_to_int4.py
+ * CLI at the e8x4g64 recipe (--ebits 8 --xbits 4 --io-bits 8 --group-size 64),
+ * then invokes this binary against the real output directory. Sibling of
+ * tests/test_fp8_e2e_loader.c (fmt=8 passthrough), same division of labor:
+ * the Python side proves the TOOL ran for real; this side proves the REAL
+ * st_init/qt_from_disk (the identical functions every model load uses)
+ * resolve every minted tensor to the format the recipe promises.
+ *
+ * Usage: test_e8x4g64_loader <container_dir> [name O I wantfmt wantgs]...
+ * For every (name,O,I,wantfmt,wantgs) 5-tuple, calls the REAL qt_from_disk and
+ * asserts:
+ * (a) the resolved fmt equals wantfmt (1 = int8 per-row spine, 4 = grouped
+ * int4 experts) and, for fmt=4, the derived group size equals wantgs --
+ * both resolved by qt_resolve_fmt's byte arithmetic on the tool's actual
+ * output, neither mocked nor stamped;
+ * (b) the weight and scale buffers are non-NULL;
+ * (c) every dequantized value is finite -- catches a scale-layout or
+ * packing bug a pure byte-count check wouldn't.
+ *
+ * The D-2 duplicate-name refusal is exercised through this same binary: the
+ * Python driver runs it a second time against a copy of the container holding
+ * a duplicated shard, and st_init below then refuses (exit 1, naming both
+ * shards) before any tuple is checked -- the positive half of the D-2 pair,
+ * on a tool-produced container. The clean run doubles as the negative
+ * control: a legitimate mint loads with no refusal. */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv){
+ if(argc < 2){ fprintf(stderr,"usage: %s <dir> [name O I wantfmt wantgs]...\n", argv[0]); return 2; }
+ if((argc-2) % 5 != 0){
+ fprintf(stderr,"args after <dir> must come in (name,O,I,wantfmt,wantgs) 5-tuples (got %d)\n", argc-2);
+ return 2;
+ }
+ const char *dir = argv[1];
+ int ntensors = (argc-2)/5;
+ if(ntensors == 0){ fprintf(stderr,"no tuples given -- nothing to check\n"); return 2; }
+
+ static Model gm; memset(&gm,0,sizeof gm);
+ st_init(&gm.S, dir); /* D-2 duplicate-name detection lives in here (st.h) */
+
+ int fails = 0;
+ for(int i=0;i<ntensors;i++){
+ const char *name = argv[2+i*5];
+ int O = atoi(argv[3+i*5]);
+ int I = atoi(argv[4+i*5]);
+ int wantfmt = atoi(argv[5+i*5]);
+ int wantgs = atoi(argv[6+i*5]);
+
+ QT t; memset(&t,0,sizeof t);
+ qt_from_disk(&gm, name, O, I, 8, 0, &t); /* THE REAL LOADER PATH -- not mocked */
+
+ if(t.fmt != wantfmt){
+ printf("FAIL %s: fmt=%d, expected %d\n", name, t.fmt, wantfmt);
+ fails++; continue;
+ }
+ if(wantfmt == 4 && t.gs != wantgs){
+ printf("FAIL %s: fmt=4 resolved with gs=%d, expected g%d\n", name, t.gs, wantgs);
+ fails++; continue;
+ }
+ const void *w = t.fmt==1 ? (const void*)t.q8 : (const void*)t.q4;
+ if(!w || !t.s){
+ printf("FAIL %s: fmt=%d but weights=%p s=%p (expected both non-NULL)\n",
+ name, t.fmt, w, (void*)t.s);
+ fails++; continue;
+ }
+ /* (c) all-finite dequant, per format's own scale geometry */
+ int64_t bad = -1;
+ if(t.fmt == 1){
+ for(int64_t o=0; o<O && bad<0; o++){ float s=t.s[o];
+ for(int64_t ii=0; ii<I; ii++){
+ float v=(float)t.q8[o*(int64_t)I+ii]*s;
+ if(!isfinite(v)){ bad=o*(int64_t)I+ii; break; } } }
+ } else if(t.fmt == 4){
+ int ng=(I+t.gs-1)/t.gs;
+ for(int64_t o=0; o<O && bad<0; o++){ const float *scl=t.s+o*ng;
+ const uint8_t *row=t.q4+o*(int64_t)((I+1)/2);
+ for(int64_t ii=0; ii<I; ii++){
+ uint8_t b=row[ii>>1]; int nib=(ii&1)?(b>>4):(b&0xF);
+ float v=((int)nib-8)*scl[ii/t.gs];
+ if(!isfinite(v)){ bad=o*(int64_t)I+ii; break; } } }
+ } else {
+ printf("FAIL %s: this harness only knows the e8x4g64 formats (1, 4); "
+ "wantfmt=%d is a driver bug\n", name, wantfmt);
+ fails++; continue;
+ }
+ if(bad >= 0){
+ printf("FAIL %s: non-finite dequantized value at flat index %lld\n", name, (long long)bad);
+ fails++; continue;
+ }
+ printf("ok %s: fmt=%d%s O=%d I=%d, loaded through the real loader, all-finite\n",
+ name, t.fmt, t.fmt==4?" (g64)":"", O, I);
+ }
+ if(fails){ printf("e8x4g64 mint->load: %d/%d tensor(s) FAILED\n", fails, ntensors); return 1; }
+ printf("e8x4g64 mint->load: ok (%d tensor(s))\n", ntensors);
+ return 0;
+}
diff --git a/c/tests/test_e8x4g64_mint_load.py b/c/tests/test_e8x4g64_mint_load.py
new file mode 100644
index 00000000..7f506950
--- /dev/null
+++ b/c/tests/test_e8x4g64_mint_load.py
@@ -0,0 +1,297 @@
+"""wq-v0-class regression case: REAL tools/convert_fp8_to_int4.py output fed
+into the REAL C loader (st_init/qt_from_disk in colibri.c) at toy scale.
+
+The wq-v0 production container class is "e8x4g64": int8 per-row spine
+(attention / shared expert / dense MLP / embed / lm_head -> fmt=1) with
+grouped-int4 g64 routed experts (fmt=4, gs=64). This test mints that class
+each cycle at toy scale with GLM-shaped tensor names and the load-bearing
+real dimensions (kv_lora_rank-width kv_b contraction, group-64-divisible
+expert dims) through the real converter CLI -- not a reimplementation of its
+quantizers -- and then proves the REAL C loader resolves every tensor to the
+format the recipe promises. Regression coverage, not new-feature proof: the
+full-scale wq-v0 container is already banked and audited; what this pins is
+that the TOOLING and the LOADER still agree on the class after engine
+changes (the mint half of the mint->load->run regression bar; the run half
+executes at full scale against the banked container, outside this suite).
+
+Also carried here, because it belongs to exactly this container class: the
+D-2 duplicate-tensor-name pair on a TOOL-PRODUCED container.
+ * positive: a copy of the minted container with one shard duplicated must
+ refuse at st_init with the exact D-2 message (naming both shards), before
+ any tensor is read;
+ * negative control: the untouched mint loads clean through the same
+ binary -- the guard is duplicate-targeted, not a blanket gate.
+(tests/test_dup_name_refusal.c pins the same guard on hand-built fixtures;
+this is the tool-produced end of it.)
+
+Hermetic: checkpoint, minted output, duplicated copy, and the compiled
+harness all live under temporary directories; nothing is left behind.
+"""
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+try:
+ import torch
+except ImportError as e:
+ raise unittest.SkipTest(f"torch not installed: {e}")
+
+try:
+ import numpy as np
+except ImportError as e:
+ raise unittest.SkipTest(f"numpy not installed: {e}")
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+C_DIR = os.path.normpath(os.path.join(HERE, ".."))
+sys.path.insert(0, os.path.join(C_DIR, "tools"))
+# reuse: the real fp8 block-quantize fixture helpers, not reimplemented. The
+# quantize/dequantize pair anchors the numeric reference below to EXACTLY what
+# the converter ingests (its dequant() applies the same repeat_interleave
+# formula to the same saved fp8 bytes); the converter's own int8/int4
+# quantizers are NOT imported anywhere in this file -- their output is decoded
+# and bounded by independent code below.
+from glm_fp8_emit import save_fp8_safetensors, fp8_block_quantize, fp8_block_dequantize
+
+
+def _cc_flags():
+ """Mirror the Makefile's CFLAGS closely enough to compile colibri.c
+ cleanly (same arrangement as tests/test_fp8_e2e_repack_load.py)."""
+ cc = shutil.which("cc") or shutil.which("clang") or shutil.which("gcc")
+ if not cc:
+ return None, None, None
+ cflags = ["-O3", "-Wall", "-Wextra", "-Wno-unused-parameter",
+ "-Wno-misleading-indentation", "-Wno-unused-function"]
+ ldflags = ["-lm"]
+ if sys.platform not in ("darwin", "win32"):
+ cflags += ["-fopenmp"]
+ ldflags += ["-fopenmp"]
+ if sys.platform == "darwin":
+ # The Makefile builds this harness with OpenMP on macOS via Homebrew's
+ # libomp. Locating it must not fail quietly: a swallowed error here
+ # compiles a DIFFERENT binary from the one the suite builds, and the
+ # warning assertion below would then vouch for flags nobody ships.
+ # Report why it could not be found and let the caller decide.
+ try:
+ probe = subprocess.run(["brew", "--prefix", "libomp"], capture_output=True,
+ text=True, timeout=10)
+ prefix = probe.stdout.strip()
+ if probe.returncode != 0:
+ raise RuntimeError(
+ "`brew --prefix libomp` exited %d: %s"
+ % (probe.returncode, probe.stderr.strip() or "(no stderr)"))
+ except FileNotFoundError:
+ raise RuntimeError(
+ "libomp is required to build this harness on macOS the way the "
+ "Makefile builds it, and `brew` is not on PATH")
+ except (OSError, subprocess.TimeoutExpired) as exc:
+ raise RuntimeError("could not run `brew --prefix libomp`: %r" % (exc,))
+ inc, lib = os.path.join(prefix, "include"), os.path.join(prefix, "lib")
+ if not (prefix and os.path.exists(os.path.join(inc, "omp.h"))):
+ raise RuntimeError(
+ "`brew --prefix libomp` gave %r but %s/omp.h does not exist; "
+ "install libomp (`brew install libomp`) rather than compiling "
+ "this harness without OpenMP" % (prefix, inc))
+ cflags += ["-Xclang", "-fopenmp", "-I", inc]
+ ldflags += ["-L", lib, "-lomp"]
+ return cc, cflags, ldflags
+
+
+# Toy scale, real geometry where it is load-bearing:
+# * kv_b_proj keeps the REAL contraction width I=512 (GLM-5.2's
+# kv_lora_rank) with a toy head count -> O=160;
+# * routed-expert dims stay multiples of 64 (real containers are), so the
+# g64 grouping has no synthetic tail the production mint never sees --
+# while D_H (o_proj/q_a rows, expert contraction) is NOT a multiple of
+# 128, keeping the fp8 input's block scales on the partial-tail path;
+# * no tensor sits on the fmt=1-vs-fmt=8 collision boundary
+# (O == ceil(O/128)*ceil(I/128)), so byte-arithmetic resolution is
+# unambiguous, as it is at full scale.
+D_H = 320 # toy hidden size (multiple of 64, not of 128)
+KV_B_O, KV_B_I = 160, 512
+E_M = 192 # toy expert intermediate (multiple of 64)
+VOCAB = 512
+
+
+class E8x4g64MintLoadTest(unittest.TestCase):
+ """The real e8x4g64 mint recipe -> the real C loader."""
+
+ _harness_dir = None
+ _harness_bin = None
+ _build_stderr = None
+
+ @classmethod
+ def setUpClass(cls):
+ cc, cflags, ldflags = _cc_flags()
+ if not cc:
+ raise unittest.SkipTest("no C compiler found on PATH")
+ cls._harness_dir = tempfile.TemporaryDirectory()
+ harness_src = os.path.join(HERE, "test_e8x4g64_loader.c")
+ harness_bin = os.path.join(cls._harness_dir.name, "test_e8x4g64_loader")
+ build = subprocess.run([cc] + cflags + [harness_src, "-o", harness_bin] + ldflags,
+ capture_output=True, text=True, cwd=C_DIR)
+ if build.returncode != 0:
+ raise AssertionError(f"e8x4g64 harness build failed:\n{build.stderr}")
+ cls._harness_bin = harness_bin
+ cls._build_stderr = build.stderr
+
+ @classmethod
+ def tearDownClass(cls):
+ if cls._harness_dir:
+ cls._harness_dir.cleanup()
+
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.indir = os.path.join(self.tmp.name, "fp8src")
+ os.makedirs(self.indir)
+ self.outdir = os.path.join(self.tmp.name, "out")
+ self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def _emit_checkpoint(self):
+ """A GLM-shaped fp8 source checkpoint: every name below is one the real
+ converter's classify() routes exactly like the production checkpoint's
+ (kvb/attn/o -> ebits, sh/dmlp -> ebits, io -> io_bits, experts -> xbits,
+ norms -> f32 passthrough)."""
+ torch.manual_seed(11)
+ L = "model.layers.0"
+ sd = {
+ f"{L}.self_attn.kv_b_proj.weight": torch.randn(KV_B_O, KV_B_I) * 0.02,
+ f"{L}.self_attn.q_a_proj.weight": torch.randn(D_H, D_H) * 0.02,
+ f"{L}.self_attn.o_proj.weight": torch.randn(D_H, D_H) * 0.02,
+ f"{L}.mlp.shared_experts.gate_proj.weight": torch.randn(E_M, D_H) * 0.02,
+ f"{L}.mlp.experts.0.gate_proj.weight": torch.randn(E_M, D_H) * 0.02,
+ f"{L}.mlp.experts.0.up_proj.weight": torch.randn(E_M, D_H) * 0.02,
+ f"{L}.mlp.experts.0.down_proj.weight": torch.randn(D_H, E_M) * 0.02,
+ "model.embed_tokens.weight": torch.randn(VOCAB, D_H) * 0.02,
+ f"{L}.input_layernorm.weight": torch.randn(D_H), # f32 passthrough control
+ }
+ self.sd = sd # kept: the numeric round-trip test derives its reference from it
+ save_fp8_safetensors(sd, self.shard)
+
+ def _mint(self):
+ """The REAL converter CLI at the wq-v0 e8x4g64 recipe: int8 spine
+ (--ebits 8, --io-bits 8), grouped-int4 g64 routed experts (--xbits 4,
+ --group-size 64)."""
+ tool = os.path.join(C_DIR, "tools", "convert_fp8_to_int4.py")
+ rc = subprocess.run([sys.executable, tool, "--indir", self.indir,
+ "--outdir", self.outdir, "--n-layers", "1",
+ "--ebits", "8", "--xbits", "4", "--io-bits", "8",
+ "--group-size", "64"],
+ capture_output=True, text=True)
+ self.assertEqual(rc.returncode, 0,
+ f"real converter failed:\nSTDOUT:\n{rc.stdout}\nSTDERR:\n{rc.stderr}")
+ outs = glob.glob(os.path.join(self.outdir, "out-*.safetensors"))
+ self.assertEqual(len(outs), 1, f"expected exactly one minted shard, got {outs}")
+ return outs[0]
+
+ # 5-tuples for the C harness: (name, O, I, wantfmt, wantgs)
+ _EXPECT = [
+ ("model.layers.0.self_attn.kv_b_proj.weight", KV_B_O, KV_B_I, 1, 0),
+ ("model.layers.0.self_attn.q_a_proj.weight", D_H, D_H, 1, 0),
+ ("model.layers.0.self_attn.o_proj.weight", D_H, D_H, 1, 0),
+ ("model.layers.0.mlp.shared_experts.gate_proj.weight", E_M, D_H, 1, 0),
+ ("model.embed_tokens.weight", VOCAB, D_H, 1, 0),
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", E_M, D_H, 4, 64),
+ ("model.layers.0.mlp.experts.0.up_proj.weight", E_M, D_H, 4, 64),
+ ("model.layers.0.mlp.experts.0.down_proj.weight", D_H, E_M, 4, 64),
+ ]
+
+ def _load(self, container_dir):
+ args = [self._harness_bin, container_dir]
+ for name, n_out, n_in, fmt, gs in self._EXPECT:
+ args += [name, str(n_out), str(n_in), str(fmt), str(gs)]
+ return subprocess.run(args, capture_output=True, text=True)
+
+ def test_harness_builds_without_warnings(self):
+ """Production flags require zero warnings (pr-cycle mechanical gate)."""
+ self.assertEqual(self._build_stderr.strip(), "",
+ f"e8x4g64 harness build produced warnings:\n{self._build_stderr}")
+
+ def test_e8x4g64_mint_loads_through_real_c_loader(self):
+ self._emit_checkpoint()
+ self._mint()
+ rc = self._load(self.outdir)
+ self.assertEqual(rc.returncode, 0,
+ f"loader harness failed:\nSTDOUT:\n{rc.stdout}\nSTDERR:\n{rc.stderr}")
+ for name, _, _, fmt, _ in self._EXPECT:
+ self.assertIn(f"ok {name}: fmt={fmt}", rc.stdout)
+ # negative D-2 control rides along: the clean mint produced NO
+ # duplicate-name refusal anywhere in the load
+ self.assertNotIn("duplicate tensor name", rc.stderr)
+
+ def test_e8x4g64_numeric_round_trip(self):
+ """The minted VALUES, not just the format class (deep-audit MAJOR-1:
+ the fmt/finite checks alone passed a 37x scale corruption of
+ quant_int8 -- format class and finiteness both survive numeric
+ corruption). This closes DR-9(a)'s literal words, "mint ROUND-TRIP at
+ toy scale": dequantize the minted bytes with independent numpy code
+ (the converter's quantizers are never imported here) and require
+ agreement with the fp8-round-tripped source within the schemes' own
+ exact bound -- symmetric absmax rint quantization can never miss by
+ more than half a quantization step, so the tolerance is 0.5 steps
+ (+1e-3 for f32 arithmetic slop), derived from a scale RECOMPUTED from
+ the source, never from the minted .qs (a corrupted stored scale must
+ widen the error, not the bound)."""
+ self._emit_checkpoint()
+ minted = self._mint()
+ from safetensors.numpy import load_file
+ out = load_file(minted)
+ for name, n_out, n_in, fmt, gs in self._EXPECT:
+ # Reference: exactly what the converter ingested -- the fp8
+ # block-quantized source, dequantized (bit-identical formula to
+ # the converter's dequant(); see the import comment above).
+ w_fp8, scale = fp8_block_quantize(self.sd[name].float())
+ w_ref = fp8_block_dequantize(w_fp8, scale).numpy().astype(np.float32)
+ self.assertEqual(w_ref.shape, (n_out, n_in))
+ q, qs = out[name], out[name + ".qs"]
+ if fmt == 1:
+ # int8 per-row: one scale per output row, q in [-127,127]
+ # (amax maps to qmax=127, so -128 is unreachable).
+ deq = q.view(np.int8).reshape(n_out, n_in).astype(np.float32) * qs.reshape(n_out, 1)
+ s_true = np.maximum(np.abs(w_ref).max(axis=1, keepdims=True) / 127.0, 1e-8)
+ step = np.broadcast_to(s_true, (n_out, n_in))
+ else:
+ # int4 grouped g64: packed nibbles (low = even column), one
+ # scale per (row, 64-column group), q in [-7,7] likewise.
+ rb, ng = (n_in + 1) // 2, (n_in + gs - 1) // gs
+ b = q.reshape(n_out, rb)
+ deq = np.empty((n_out, n_in), np.float32)
+ deq[:, 0::2] = (b & 0xF).astype(np.float32)[:, : (n_in + 1) // 2] - 8.0
+ deq[:, 1::2] = (b >> 4).astype(np.float32)[:, : n_in // 2] - 8.0
+ deq *= np.repeat(qs.reshape(n_out, ng), gs, axis=1)[:, :n_in]
+ pad = np.zeros((n_out, ng * gs - n_in), np.float32)
+ grp = np.concatenate([np.abs(w_ref), pad], axis=1).reshape(n_out, ng, gs)
+ s_true = np.maximum(grp.max(axis=2) / 7.0, 1e-8) # [n_out, ng]
+ step = np.repeat(s_true, gs, axis=1)[:, :n_in]
+ worst = float((np.abs(deq - w_ref) / step).max())
+ self.assertLessEqual(
+ worst, 0.5 + 1e-3,
+ f"{name}: minted values diverge from the source by {worst:.4g} "
+ f"quantization steps (bound: half a step) -- the converter's "
+ f"numeric output regressed even though the format class may "
+ f"still be intact")
+
+ def test_d2_duplicate_shard_refuses_by_name(self):
+ """D-2 positive on the tool-produced container: duplicating the minted
+ shard under a second indexed name must refuse at st_init with the
+ exact duplicate-tensor-name message, before any tensor check runs."""
+ self._emit_checkpoint()
+ minted = self._mint()
+ dupdir = os.path.join(self.tmp.name, "dup")
+ shutil.copytree(self.outdir, dupdir)
+ shutil.copy(minted, os.path.join(dupdir, "out-99999.safetensors"))
+ rc = self._load(dupdir)
+ self.assertNotEqual(rc.returncode, 0, "duplicated container must refuse to load")
+ self.assertIn("duplicate tensor name across indexed shards, refusing", rc.stderr)
+ self.assertNotIn("ok model.", rc.stdout,
+ "no tensor may load from a container st_init refused")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_fp8_refusal_canary.py b/c/tests/test_fp8_refusal_canary.py
new file mode 100644
index 00000000..4d6dda30
--- /dev/null
+++ b/c/tests/test_fp8_refusal_canary.py
@@ -0,0 +1,120 @@
+"""Drift canary for the absorb-path fmt-refusal message shape.
+
+tests/test_fp8_serve_batch_e2e.py machine-matches the engine's death on the
+batched fmt=8 serve path with one regex (its REFUSAL constant, the family
+`(qt_addrow|qt_matvec_rows): unsupported fmt=`), in two load-bearing
+places: the pass lane's engine-death scan, and the expect-bite mode the
+fleet's old-binary half PASSES ONLY THROUGH. The product strings live in
+c/colibri.c (qt_addrow's refusal and qt_matvec_rows'); an innocent reword
+of either -- "unsupported" to "unhandled", dropping the function-name
+prefix -- would not break any build or unit suite, but it would silently
+degrade the bite proof's machine-match into a generic "engine died", and
+the e2e test only runs where a real fmt=8 container exists (never in CI).
+This canary closes that gap in CI: it fails, by name, the moment the
+matcher and the source strings drift apart, in either direction.
+
+What is load-bearing is exactly what this file asserts, no more:
+ * each function still has at least one refusal-class stderr message
+ (its runtime firing is pinned separately by tests/test_qt_addrow.c's
+ fork+waitpid refusal cases, which also require the "refus" word);
+ * at least ONE refusal-class message from each function matches the e2e
+ REFUSAL regex -- imported from the e2e module, never re-typed, so the
+ two files cannot agree by coincidence (existential, not universal: a
+ future second, differently-worded guard in the same function is a new
+ refusal, not drift, and must not fail this canary);
+ * the regex is still SELECTIVE -- it must not match an arbitrary death
+ line, or expect-bite would file any crash as the bite.
+Everything else about the messages (wording after the matched prefix,
+fmt lists, line breaks) is deliberately unpinned: message edits that keep
+the matchable shape must stay free.
+"""
+import re
+import sys
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE)) # sibling import under any invocation style
+from test_fp8_serve_batch_e2e import REFUSAL
+
+COLIBRI_C = HERE.parent / "colibri.c"
+FUNCTIONS = ("qt_addrow", "qt_matvec_rows")
+
+# One C string literal (escapes included), and an fprintf-to-stderr whose
+# message is one or more adjacent literals (the refusals wrap across lines).
+_STRING = r'"(?:[^"\\\n]|\\.)*"'
+_FPRINTF = re.compile(r'fprintf\s*\(\s*stderr\s*,\s*(' + _STRING + r'(?:\s*' + _STRING + r')*)')
+
+
+def stderr_messages(source):
+ """Every fprintf(stderr, ...) format string in `source`, with adjacent
+ literals concatenated (still escaped -- the matched prefix contains no
+ escapes, so matching against the raw literal is faithful to what the
+ runtime prints before the %d substitution)."""
+ messages = []
+ for match in _FPRINTF.finditer(source):
+ parts = re.findall(_STRING, match.group(1))
+ messages.append("".join(part[1:-1] for part in parts))
+ return messages
+
+
+class RefusalShapeCanaryTest(unittest.TestCase):
+ maxDiff = None
+
+ @classmethod
+ def setUpClass(cls):
+ cls.messages = stderr_messages(COLIBRI_C.read_text(encoding="utf-8"))
+
+ def messages_of(self, function):
+ return [m for m in self.messages if m.startswith(function + ":")]
+
+ def test_each_function_still_names_a_refusal_the_matcher_greps(self):
+ for function in FUNCTIONS:
+ named = self.messages_of(function)
+ self.assertTrue(
+ named,
+ f"colibri.c has no stderr message starting '{function}:' -- the "
+ f"refusal site was renamed or removed, and the D-I2 bite matcher "
+ f"(test_fp8_serve_batch_e2e.REFUSAL) can no longer identify this "
+ f"death; update the matcher and the fleet bite lane together")
+ refusals = [m for m in named if "refus" in m]
+ self.assertTrue(
+ refusals,
+ f"{function}: no refusal-class stderr message left (the 'refus' "
+ f"discipline tests/test_qt_addrow.c pins at runtime) -- if the "
+ f"refusal moved, this canary and the e2e matcher must follow it")
+ self.assertTrue(
+ any(REFUSAL.search(m) for m in refusals),
+ f"none of {function}'s refusal messages matches the e2e "
+ f"REFUSAL regex {REFUSAL.pattern!r} -- an innocent reword "
+ f"silently turns the fleet bite proof's machine-match into a "
+ f"generic 'engine died'; keep one matchable refusal per "
+ f"function or change test_fp8_serve_batch_e2e.REFUSAL in the "
+ f"same commit. (Existential on purpose: an ADDITIONAL, "
+ f"differently-worded guard in this function is new coverage, "
+ f"not drift.)\nmessages: {refusals}")
+
+ def test_matcher_stays_selective(self):
+ """The other drift direction: a loosened REFUSAL regex would make
+ expect-bite accept ANY death as the bite. Pin that it rejects a
+ representative non-refusal death line and the empty string, and
+ that it keys on the FUNCTION FAMILY, not the bare `unsupported
+ fmt=` words -- a matcher loosened to the words alone would accept
+ any future non-absorb `unsupported fmt=` message as the bite."""
+ self.assertIsNone(REFUSAL.search("malloc: out of memory allocating 42 GB"))
+ self.assertIsNone(REFUSAL.search("colibri engine exited unexpectedly"))
+ self.assertIsNone(REFUSAL.search(""))
+ self.assertIsNone(REFUSAL.search("qt_resolve_fmt: unsupported fmt=9 refusing"))
+ # The REAL in-tree near-miss (not synthetic): layer_cuda_shard_kvb's
+ # refusal shares the `unsupported ... fmt=` words but is a different
+ # guard family (shard-layout, pinned by tests/test_shard_kvb_refuse.c)
+ # -- a loosened matcher that swallowed it would misfile a shard
+ # refusal as the absorb bite in a fleet lane.
+ self.assertIsNone(REFUSAL.search(
+ "layer_cuda_shard_kvb: unsupported kv_b fmt=5 for the head-shard "
+ "upload (only fmt 1/2/3/4 match the per-row byte/scale strides "
+ "computed here) -- refusing the shard"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_fp8_serve_batch_e2e.py b/c/tests/test_fp8_serve_batch_e2e.py
new file mode 100644
index 00000000..52bb35b6
--- /dev/null
+++ b/c/tests/test_fp8_serve_batch_e2e.py
@@ -0,0 +1,668 @@
+"""Defect-closure pin for the batched-serve fmt=8 abort (D-I2): two-slot
+SERVE_BATCH mux against a REAL fp8 (fmt=8) `kv_b_proj` container.
+
+The defect: batched serve mode (openai_server's Engine always launches the
+engine with SERVE_BATCH=1; --kv-slots > 1 gives the mux real slots) decodes
+kv_b through the MLA absorb path -- qt_addrow / qt_matvec_rows in colibri.c.
+Before the fmt=8 absorb branches landed, those functions had no fp8 case:
+the first generation request against a container whose kv_b_proj resolves to
+fmt=8 (fp8-e4m3-b128 -- both production fp8 container classes, f8_full and
+f8x4g64, are in this class) killed the engine with the named refusal
+
+ qt_addrow: unsupported fmt=8 for the per-row-scale absorb path ...
+ -- refusing rather than misread t->s[row]/t->q4
+
+and the client saw a 500 `engine_error`. Single-stream chat never hit it,
+so nothing in the FakeEngine-backed server suite could: the abort lives
+below the engine wire protocol, on a code path only a real fmt=8 container
+reaches. This test pins the closure end to end: serve the real container
+batched, occupy BOTH KV slots concurrently, and require 200 + text on both
+requests with the engine still alive afterwards.
+
+A real fmt=8 container is hundreds of GB and exists only on the model
+hosts, never in CI, so discovery is by environment variable:
+
+ COLI_FP8_CONTAINER root of an engine-loadable container whose
+ kv_b_proj is fp8 (fmt=8). Unset => named SKIP.
+ COLI_ENGINE engine binary override (same convention as coli);
+ default: the built `colibri` next to the server.
+ SET but wrong => loud FAIL, never a skip.
+ COLI_FP8_EXPECT_BITE=1 proof-of-bite mode for the fleet's old-binary
+ half: the run PASSES only if the invocation FAILS
+ to serve AND the captured server stderr carries
+ the named `unsupported fmt=` refusal. A death
+ without the refusal, or a clean serve, FAILS.
+ COLI_FP8_READY_TIMEOUT seconds to wait for the server to come up
+ (default 1800 -- container loads are large).
+ COLI_FP8_GENERATE_TIMEOUT seconds per generation request (default
+ 1800; two concurrent cold prefills on a
+ storage-bound host can be slow).
+
+Set-but-wrong is a FAILURE, not a skip: if COLI_FP8_CONTAINER names a
+missing path, a container whose kv_b_proj is NOT fmt=8 (e.g. an int8
+spine), or a shard set with duplicate tensor names, this test would
+otherwise pass (or fail for the wrong reason) without ever entering the
+absorb branch it exists to pin -- the vacuous-gate hazard. The safetensors
+headers are checked first (pure header reads, no tensor data) and the
+mismatch is a loud failure naming what was found. The same doctrine covers
+COLI_ENGINE: a set-but-nonexistent engine path fails, it does not skip.
+(Known unreachable corner: a shape with O == ceil(O/128)*ceil(I/128) makes
+per-row and per-block scale counts collide and the unstamped loader would
+resolve fmt=1 where this check says fmt=8 -- real GLM kv_b O is in the
+thousands, so no planned container can reach it.)
+
+The engine/server child environment is constructed fresh: ambient COLI_*
+and engine-knob variables (KV8, KV_TQ, ...) are stripped except an
+explicit backend/location allowlist plus two value-restricted knobs --
+exactly ABSORB=0 (the ratified non-absorb fleet arm; any other ABSORB
+value is stripped), and exactly COLI_CUDA_ATTN=1 WHEN the lane opts in
+with COLI_FP8_E2E_CUDA_ABSORB=1 (default: stripped, so a leaked ambient
+COLI_CUDA_ATTN can never swap the code path under test) -- and a leaked
+COLI_API_KEY cannot 401 the ready poll; the kept/dropped set is printed
+and attached to every failure so the run artifact shows the env the
+invocation actually saw.
+
+Lane semantics, stated exactly: by DEFAULT every lane (CPU or CUDA host)
+exercises the CPU absorb arms -- the CUDA fmt=8 absorb decode is gated
+on COLI_CUDA_ATTN (path selection) AND CUDA_DENSE (kv_b's
+cuda_eligible), both of which this test strips unless the lane
+explicitly opts in. A CUDA lane that sets COLI_FP8_E2E_CUDA_ABSORB=1
+(plus its COLI_CUDA/COLI_GPU bindings and ambient COLI_CUDA_ATTN=1 and
+CUDA_DENSE=1) pins the CUDA absorb decode end-to-end, and the test then
+REQUIRES the engine's GPU-dense boot line as a witness -- an opt-in run
+whose engine reports resident-dense-on-CPU fails rather than banking a
+vacuous green. Without the opt-in, the CUDA absorb arm is pinned only
+by tests/test_backend_cuda.cu.
+"""
+import json
+import os
+import re
+import signal
+import socket
+import struct
+import subprocess
+import sys
+import threading
+import time
+import unittest
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+HERE = Path(__file__).resolve().parent
+C_DIR = HERE.parent
+CONTAINER = os.environ.get("COLI_FP8_CONTAINER")
+ENGINE_OVERRIDE = os.environ.get("COLI_ENGINE")
+EXPECT_BITE = os.environ.get("COLI_FP8_EXPECT_BITE") == "1"
+READY_TIMEOUT = float(os.environ.get("COLI_FP8_READY_TIMEOUT", "1800"))
+GENERATE_TIMEOUT = float(os.environ.get("COLI_FP8_GENERATE_TIMEOUT", "1800"))
+PROMPT = "The primary colors are" # the DR-10 fixed prompt: short, deterministic continuation
+MAX_TOKENS = 24
+# The refusal family this test exists to keep dead. Both absorb entry points
+# carry the same named message shape; match the family, not one function, so
+# a regression through either surfaces by name in the failure output.
+REFUSAL = re.compile(r"(qt_addrow|qt_matvec_rows): unsupported fmt=")
+ENGINE_DEATH = "colibri engine exited unexpectedly"
+
+# Child-environment policy (the invocation of record must not ride ambient
+# state): backend selection and read-only model LOCATION are the only knob
+# namespaces a lane may pass through -- CUDA lanes need COLI_CUDA/
+# COLI_GPU(S) per the DR-10 bindings, and a split/mirrored container needs
+# the engine to be TOLD where its shards live (location config says where
+# the weights ARE; a behavior knob changes what the engine DOES with them
+# -- only the former class passes):
+# COLI_MODEL_DIRS -- extra shard directories (st_init_multi's SPLIT
+# layout); stripping it hides the operator's shards
+# and fails as a bogus "not an fmt=8 container".
+# COLI_MODEL_MIRROR -- read-only replica dirs (multi-SSD read fan-out).
+# COLI_MMAP -- how weights are mapped from disk; load
+# placement/feasibility, not decode semantics.
+# COLI_DISK_WEIGHTS -- disk-resident weight policy; feasibility for
+# hundreds-of-GB loads, not decode semantics.
+# Every other COLI_* (COLI_API_KEY -> 401'd ready poll) and the bare
+# engine knobs below (KV8/KV_TQ change the KV format of record;
+# SNAP/SERVE/SERVE_BATCH/NGEN/KV_SLOTS belong to the server, which sets
+# its own) are stripped, and the strip is recorded. Documented legacy
+# aliases are stripped ALONGSIDE their primaries so a scrubbed primary
+# cannot resurface under its old name: SNAP_MIRROR (consulted only when
+# COLI_MODEL_MIRROR is unset/empty -- i.e. precisely after this scrub) is
+# name-stripped, and TEMP is stripped only when FULLY NUMERIC (matching
+# temp_from_env's strtod whole-string test: a numeric TEMP is the
+# deprecated sampling alias, a path TEMP is the Windows/ROCm temp
+# directory and must survive). Value-restricted exceptions: the ratified
+# non-absorb fleet arm selects itself with ABSORB=0, so exactly that
+# value passes (any other ambient ABSORB is stripped -- both decode paths
+# at ABSORB=0/default are ratified-equivalent, so a leaked "0" can shift
+# which path is pinned but never fake a pass).
+#
+# The CUDA-absorb opt-in (COLI_FP8_E2E_CUDA_ABSORB=1) is handled by
+# INJECTION, not passthrough -- see CUDA_ABSORB_INJECT below. A first
+# attempt value-allowed COLI_CUDA_ATTN=1 and CUDA_DENSE=1 through the scrub
+# and relied on the LANE to set both ambient; the fresh-env allowlist only
+# ever KEEPS a variable already in the environment, so when the lane set
+# COLI_CUDA_ATTN=1 but not the bare (non-COLI) CUDA_DENSE, the eligibility
+# knob never reached the child and the run booted resident-dense-on-CPU
+# (the boot-line witness caught it as a hard failure). The opt-in is the
+# single source of truth for the bundle the path needs, so the test now
+# SETS it directly and reports it, instead of hoping the lane assembled
+# the same set correctly.
+ENV_KNOBS_ALLOWED = ("COLI_CUDA", "COLI_GPU", "COLI_GPUS", "COLI_NO_OMP_TUNE",
+ "COLI_MODEL_DIRS", "COLI_MODEL_MIRROR", "COLI_MMAP",
+ "COLI_DISK_WEIGHTS")
+ENV_KNOBS_VALUE_ALLOWED = {"ABSORB": ("0",)}
+ENV_KNOBS_STRIPPED = ("KV8", "KV_TQ", "ABSORB", "CAP_RAISE", "CUDA_DENSE",
+ "COLI_CUDA_ATTN", "SNAP", "SNAP_MIRROR", "SERVE",
+ "SERVE_BATCH", "NGEN", "KV_SLOTS")
+CUDA_ABSORB_OPT_IN = "COLI_FP8_E2E_CUDA_ABSORB"
+# The exact engine knobs the CUDA fmt=8 absorb decode needs, INJECTED into
+# the child (with the lane's own COLI_CUDA=1 backend binding) when the lane
+# opts in. COLI_CUDA_ATTN=1 selects the CUDA absorb dispatch; CUDA_DENSE=1
+# makes kv_b/o cuda_eligible -- qt_load grants eligibility only under
+# g_cuda_dense (colibri.c:2123) and there is NO VRAM/budget gate on it
+# (colibri.c:10966), so CUDA_DENSE=1 reaching the child is sufficient on a
+# device that fits the dense tensors. CUDA_DENSE is a bare name (no COLI_
+# prefix), which is exactly why passthrough could not carry it and explicit
+# injection is required. Both are stripped from ambient (above) so an
+# unrequested value can never leak in; under the opt-in the test's own "1"
+# is what the child sees, recorded in the kept-knobs line as injected.
+CUDA_ABSORB_INJECT = {"COLI_CUDA_ATTN": "1", "CUDA_DENSE": "1"}
+
+# Boot-line witness for the opt-in arm: requesting a path is not proof it
+# ran. The engine announces its residency decision at boot
+# (colibri.c:11047-11049); under the opt-in this test asserts the GPU-dense
+# line and REFUSES the CPU-dense line, so a misconfigured lane (e.g. the
+# eligibility knob lost again) can never bank a vacuous green.
+CUDA_DENSE_BOOT = "[CUDA] mode: routed experts + resident dense tensors"
+CUDA_CPU_DENSE_BOOT = "[CUDA] mode: routed experts only (resident dense on CPU)"
+
+
+def observed_cuda_boot_mode(stderr_text):
+ """The engine's `[CUDA] mode: ...` boot line as it appears in stderr, or
+ None if it never printed. Echoed by the witness so a PASS artifact
+ SHOWS the observed residency, not just an assertion that succeeded."""
+ marker = "[CUDA] mode:"
+ start = stderr_text.find(marker)
+ if start == -1:
+ return None
+ end = stderr_text.find("\n", start)
+ return stderr_text[start:end if end != -1 else None]
+
+
+def cuda_absorb_witness_failure(stderr_text):
+ """None when the boot-line witness holds for the CUDA-absorb opt-in;
+ otherwise the failure message.
+
+ It checks two things against the engine's stderr: that the
+ resident-dense-on-CPU boot line is absent, and that the
+ routed+resident-dense line is present. Module-level so it can be called
+ without constructing the test case, and so it can be exercised directly
+ once there is a caller outside the container-gated test below -- there is
+ none today, so on a machine without a container this function has no
+ coverage at all."""
+ if CUDA_CPU_DENSE_BOOT in stderr_text:
+ return ("CUDA-absorb opt-in was requested but the engine booted with "
+ f"{CUDA_CPU_DENSE_BOOT!r} -- resident dense stayed on the CPU, "
+ "kv_b never became cuda_eligible, and the CUDA absorb path "
+ "cannot have run (vacuous green refused; check CUDA_DENSE=1 "
+ "reached the child, see the kept-knobs line)")
+ if CUDA_DENSE_BOOT not in stderr_text:
+ return ("CUDA-absorb opt-in was requested but the engine's boot "
+ f"witness {CUDA_DENSE_BOOT!r} never appeared on stderr -- "
+ "cannot certify the CUDA absorb path ran")
+ return None
+
+
+def _numeric_temp(value):
+ """temp_from_env's whole-string strtod test: TEMP acts as the deprecated
+ sampling alias only when fully numeric; a path value is the system temp
+ directory and is not a knob."""
+ if not value:
+ return False
+ try:
+ float(value)
+ return True
+ except ValueError:
+ return False
+
+
+def _child_env():
+ """Fresh server/engine environment per the policy above. Returns
+ (env, report): the report names kept knobs with values and dropped
+ knobs by NAME only (a dropped COLI_API_KEY must not leak its value
+ into a run artifact)."""
+ opt_in = os.environ.get(CUDA_ABSORB_OPT_IN) == "1"
+ env, kept, dropped = {}, [], []
+ for name, value in os.environ.items():
+ # The CUDA-absorb bundle is INJECTED below when opted in, never taken
+ # from ambient -- so the child sees the test's "1", not a leaked
+ # value, and an unrequested ambient copy is always stripped here.
+ if name in CUDA_ABSORB_INJECT:
+ dropped.append(name)
+ elif name in ENV_KNOBS_ALLOWED or value in ENV_KNOBS_VALUE_ALLOWED.get(name, ()):
+ env[name] = value
+ kept.append(f"{name}={value}")
+ elif name.startswith("COLI_") or name in ENV_KNOBS_STRIPPED:
+ dropped.append(name)
+ elif name == "TEMP" and _numeric_temp(value):
+ dropped.append(name) # numeric TEMP = deprecated COLI_TEMP alias
+ else:
+ env[name] = value # non-knob system env (PATH, HOME, LD_LIBRARY_PATH, ...)
+ if opt_in:
+ for name, value in CUDA_ABSORB_INJECT.items():
+ env[name] = value
+ kept.append(f"{name}={value} (injected)")
+ dropped = [d for d in dropped if d not in CUDA_ABSORB_INJECT]
+ return env, ("kept knobs: " + (", ".join(sorted(kept)) or "(none)")
+ + "; dropped knobs: " + (", ".join(sorted(dropped)) or "(none)"))
+
+
+def _default_engine():
+ """The built glm engine next to openai_server.py -- same candidates,
+ same order, as the server's own default_engine()."""
+ for name in ("colibri", "glm"):
+ for suffix in ("", ".exe"):
+ candidate = C_DIR / (name + suffix)
+ if candidate.exists():
+ return candidate
+ return C_DIR / "colibri"
+
+
+ENGINE = Path(ENGINE_OVERRIDE) if ENGINE_OVERRIDE else _default_engine()
+
+
+def _shard_headers(container):
+ """Merged {tensor name: (shape, nbytes)} across every *.safetensors shard
+ header in the container root, plus the duplicate-name collisions found on
+ the way. Header-only reads (u64 length + JSON): no tensor data is
+ touched, so scanning a 140-shard container is cheap. Duplicates are
+ returned rather than silently last-shard-wins: the engine refuses a
+ duplicate tensor name across shards (st_init's D-2 guard), so a merged
+ map that picked either copy could disagree with the load in either
+ direction."""
+ headers, owner, duplicates = {}, {}, []
+ shards = sorted(Path(container).glob("*.safetensors"))
+ for shard in shards:
+ with open(shard, "rb") as f:
+ (hlen,) = struct.unpack("<Q", f.read(8))
+ entries = json.loads(f.read(hlen))
+ for name, meta in entries.items():
+ if name == "__metadata__":
+ continue
+ if name in headers:
+ duplicates.append(f"{name} ({owner[name]} and {shard.name})")
+ continue
+ begin, end = meta["data_offsets"]
+ headers[name] = (meta["shape"], end - begin)
+ owner[name] = shard.name
+ return headers, len(shards), duplicates
+
+
+def _kv_b_fmt8_check(headers):
+ """Is the container's kv_b_proj in the fmt=8 class? Mirrors the loader's
+ own byte arithmetic (qt_resolve_fmt, colibri.c): fmt=8 carries O*I raw
+ e4m3 weight bytes plus ceil(O/128)*ceil(I/128) f32 block scales in the
+ .qs sidecar, where fmt=1 (int8 spine) carries O per-row f32 scales. Real
+ GLM kv_b shapes keep the two counts far apart, so the answer here is
+ unambiguous. Returns (verdict, detail) -- detail names what WAS found so
+ a wrong container fails readably."""
+ names = [n for n in headers
+ if n.endswith("self_attn.kv_b_proj.weight")]
+ if not names:
+ return False, "no self_attn.kv_b_proj.weight tensor in any shard header"
+ name = sorted(names)[0]
+ (shape, nb) = headers[name]
+ if len(shape) != 2:
+ return False, f"{name}: unexpected shape {shape}"
+ o, i = shape
+ qs = headers.get(name + ".qs")
+ if qs is None:
+ return False, f"{name}: no .qs scale sidecar (not a quantized-container tensor)"
+ ns = qs[1]
+ nblk = ((o + 127) // 128) * ((i + 127) // 128)
+ if nb != o * i:
+ return False, (f"{name}: [{o},{i}] weight is {nb} bytes, not the {o * i} "
+ f"of a one-byte-per-weight layout")
+ if ns == o * 4 and ns != nblk * 4:
+ return False, (f"{name}: [{o},{i}] carries per-ROW scales (ns={ns}) -- an "
+ f"int8-spine container, kv_b is fmt=1, the fmt=8 absorb "
+ f"path would never run")
+ if ns != nblk * 4:
+ return False, (f"{name}: [{o},{i}] scale sidecar is {ns} bytes; fmt=8 "
+ f"per-128x128-block f32 scales would be {nblk * 4}")
+ return True, f"{name}: [{o},{i}] weight {nb}B + {ns}B block scales (fmt=8)"
+
+
+class _Server:
+ """openai_server.py as a subprocess -- the production launch, not an
+ in-process shim: Engine's child environment (SERVE=1, SERVE_BATCH=1,
+ KV_SLOTS) and the real engine binary are exactly what a user gets.
+ Launched in its own session (POSIX) so close() can sweep the WHOLE
+ process group: during the container load the server has no SIGTERM
+ handler yet (serve() installs it only after Engine.__init__ returns),
+ so a plain terminate() on the ready-timeout path kills the server
+ without its engine-cleanup finally ever running, orphaning an engine
+ that streams hundreds of GB for the rest of its load. On Windows the
+ server's own KILL_ON_JOB_CLOSE job object ties the engine to the
+ server's lifetime instead."""
+
+ def __init__(self, port):
+ env, self.env_report = _child_env()
+ print(f"[fp8-e2e] child env of record -- {self.env_report}", flush=True)
+ self.proc = subprocess.Popen(
+ [sys.executable, str(C_DIR / "openai_server.py"),
+ "--model", CONTAINER, "--engine", str(ENGINE),
+ "--port", str(port), "--kv-slots", "2"],
+ cwd=C_DIR, env=env, stdin=subprocess.DEVNULL,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ start_new_session=(os.name == "posix"))
+ self.port = port
+ # Drain both pipes continuously: the interesting evidence (the named
+ # refusal, the engine-death notice) arrives on stderr AFTER the pipe
+ # would have filled and deadlocked a read-at-the-end design.
+ self._out, self._err = [], []
+ self._pumps = []
+ for pipe, sink in ((self.proc.stdout, self._out), (self.proc.stderr, self._err)):
+ t = threading.Thread(target=self._drain, args=(pipe, sink), daemon=True)
+ t.start()
+ self._pumps.append(t)
+
+ @staticmethod
+ def _drain(pipe, sink):
+ for line in iter(pipe.readline, b""):
+ sink.append(line.decode(errors="replace"))
+
+ def stderr(self):
+ return "".join(self._err)
+
+ def url(self, path):
+ return f"http://127.0.0.1:{self.port}{path}"
+
+ def _sweep_group(self):
+ """SIGKILL the server's whole session (server + engine + the OMP
+ re-exec). Harmless when everything already exited."""
+ if os.name != "posix":
+ return # Windows: job object covers the engine
+ try:
+ os.killpg(self.proc.pid, signal.SIGKILL)
+ except (ProcessLookupError, PermissionError, OSError):
+ pass
+
+ def close(self):
+ try:
+ if self.proc.poll() is None:
+ self.proc.terminate() # graceful first: serve()'s SIGTERM handler shuts the engine down
+ try:
+ self.proc.wait(timeout=60)
+ except subprocess.TimeoutExpired:
+ pass
+ finally:
+ # Unconditional group sweep: on the ready-timeout path the
+ # engine survives the server's death (no handler installed yet)
+ # and holds the write end of our stderr pipe -- without this
+ # kill the pumps never see EOF and the test process itself
+ # would hang for the orphan's remaining load time.
+ self._sweep_group()
+ if self.proc.poll() is None:
+ self.proc.kill()
+ try:
+ self.proc.wait(timeout=60)
+ except subprocess.TimeoutExpired:
+ pass
+ for t in self._pumps: # EOF is guaranteed once the group is swept
+ t.join(timeout=10)
+ if all(not t.is_alive() for t in self._pumps):
+ self.proc.stdout.close()
+ self.proc.stderr.close()
+ # else: leave the fds open -- a leaked descriptor in a dying test
+ # process beats deadlocking close() against a blocked reader.
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _get(url, timeout):
+ try:
+ with urlopen(url, timeout=timeout) as response:
+ return response.status, response.read()
+ except HTTPError as error:
+ return error.code, error.read()
+
+
+def _post(url, body, timeout):
+ """One generation request. Returns a tagged outcome so the caller can
+ tell an HTTP answer, a timed-out request, and a dropped connection
+ apart -- conflating them cost a whole diagnosis class (a timeout used
+ to kill the asking thread and read as 'request never completed')."""
+ request = Request(url, data=json.dumps(body).encode(),
+ headers={"Content-Type": "application/json"})
+ try:
+ with urlopen(request, timeout=timeout) as response:
+ return ("http", response.status, response.read(), response.headers)
+ except HTTPError as error:
+ return ("http", error.code, error.read(), error.headers)
+ except TimeoutError: # socket.timeout is this same type
+ return ("timeout", timeout, b"", None)
+ except URLError as error:
+ if isinstance(getattr(error, "reason", None), TimeoutError):
+ return ("timeout", timeout, b"", None)
+ return ("neterr", repr(error), b"", None)
+ except OSError as error: # e.g. connection reset by a dying engine
+ return ("neterr", repr(error), b"", None)
+
+
+ "COLI_FP8_CONTAINER is not set: no fmt=8 container on this host")
[email protected](ENGINE_OVERRIDE or ENGINE.exists(),
+ "engine binary is not built (make -C c colibri, or set COLI_ENGINE)")
+class ServeBatchFmt8E2ETest(unittest.TestCase):
+ maxDiff = None
+
+ def _fail_with_server_evidence(self, server, message):
+ """Every hard failure carries the server's stderr tail and the child
+ env of record: on a pre-fix engine that tail contains the named
+ qt_addrow refusal (see the expect-bite mode for the machine-checked
+ version of that claim)."""
+ tail = server.stderr()[-4000:]
+ self.fail(f"{message}\n--- child env of record ---\n{server.env_report}"
+ f"\n--- server stderr (tail) ---\n{tail}")
+
+ def _assert_bite(self, server, results):
+ """COLI_FP8_EXPECT_BITE=1: the fleet's old-binary half. PASS iff the
+ invocation failed to serve AND the named refusal is machine-matched
+ in the captured stderr. An engine that dies for ANY other reason
+ (OOM, protocol desync, bad path) FAILS -- a death without the named
+ refusal is not this defect and must never be filed as its bite."""
+ failed = [outcome for outcome in results
+ if outcome is None or outcome[0] != "http" or outcome[1] != 200]
+ stderr = server.stderr()
+ match = REFUSAL.search(stderr)
+ if not failed:
+ self._fail_with_server_evidence(
+ server, "expect-bite mode: both batched requests served 200 -- "
+ "no bite. Is COLI_ENGINE really the pre-fix binary?")
+ if match is None:
+ self._fail_with_server_evidence(
+ server, "expect-bite mode: the invocation failed WITHOUT the "
+ "named `unsupported fmt=` refusal -- a different death, "
+ "not this defect's bite")
+ # PASS: machine-matched refusal + failed serve. Record the matched
+ # line so the run artifact quotes the bite verbatim.
+ line_start = stderr.rfind("\n", 0, match.start()) + 1
+ line_end = stderr.find("\n", match.start())
+ print("[fp8-e2e] BITE CONFIRMED: "
+ + stderr[line_start:line_end if line_end != -1 else None], flush=True)
+
+ def test_batched_fmt8_serve_two_slots_200_and_engine_survives(self):
+ # Set-but-wrong engine is a FAILURE (same doctrine as the container):
+ # the decorator skips only when NO engine was named and none is
+ # built; a named engine that does not exist or cannot execute is a
+ # misconfigured lane, and skipping it would silently drop the one
+ # lane whose whole purpose is running that exact binary.
+ if ENGINE_OVERRIDE:
+ self.assertTrue(ENGINE.is_file(),
+ f"COLI_ENGINE={ENGINE_OVERRIDE} does not exist -- "
+ f"fix the path or unset it to use the built engine")
+ self.assertTrue(os.access(str(ENGINE), os.X_OK),
+ f"COLI_ENGINE={ENGINE_OVERRIDE} is not executable")
+ # Precondition, checked loudly (never skipped): the pointed-to
+ # container really is the fmt=8 kv_b class. A wrong container would
+ # pass every assertion below without the absorb path ever running.
+ self.assertTrue(os.path.isdir(CONTAINER),
+ f"COLI_FP8_CONTAINER={CONTAINER} is not a directory -- "
+ f"set it to a container root or unset it to skip")
+ headers, n_shards, duplicates = _shard_headers(CONTAINER)
+ self.assertTrue(n_shards > 0,
+ f"COLI_FP8_CONTAINER={CONTAINER} holds no *.safetensors shards")
+ self.assertFalse(
+ duplicates,
+ f"COLI_FP8_CONTAINER={CONTAINER} carries duplicate tensor names "
+ f"across shards (the engine's st_init refuses exactly this, and "
+ f"this check could otherwise judge a shard the engine never "
+ f"loads): {'; '.join(duplicates[:5])}")
+ is_fmt8, detail = _kv_b_fmt8_check(headers)
+ self.assertTrue(is_fmt8,
+ f"COLI_FP8_CONTAINER={CONTAINER} is not an fmt=8 kv_b "
+ f"container; this test would be vacuous against it: {detail}")
+
+ server = _Server(_free_port())
+ self.addCleanup(server.close)
+
+ # Startup: poll /v1/models (touches only the server, not the engine
+ # generate path) until it answers, and read the served model id from
+ # it rather than hardcoding one. Sleep EVERY iteration -- a fast
+ # non-200 answer must not turn the poll into a hot spin -- and fail
+ # fast on auth/host-guard rejections, which no amount of waiting
+ # will turn into a 200.
+ deadline = time.time() + READY_TIMEOUT
+ model_id = None
+ while time.time() < deadline:
+ if server.proc.poll() is not None:
+ self._fail_with_server_evidence(
+ server, f"server exited rc={server.proc.returncode} before READY")
+ status = None
+ try:
+ status, body = _get(server.url("/v1/models"), timeout=10)
+ except (URLError, OSError):
+ pass # not accepting yet -- keep waiting
+ if status == 200:
+ model_id = json.loads(body)["data"][0]["id"]
+ break
+ if status in (401, 403):
+ self._fail_with_server_evidence(
+ server, f"ready poll got HTTP {status} from /v1/models -- "
+ f"an auth/host-guard rejection, not a slow load; "
+ f"waiting longer cannot fix it")
+ time.sleep(2)
+ if model_id is None:
+ self._fail_with_server_evidence(
+ server, f"server not ready within {READY_TIMEOUT:.0f}s")
+
+ # The D-I2 invocation class: two generation requests IN FLIGHT
+ # TOGETHER, pinned to distinct KV slots so the 2-slot mux really
+ # multiplexes (conversation hashing would put one identical prompt in
+ # one slot). temperature 0 / fixed prompt per the invocation of
+ # record. On a pre-fix engine the FIRST decode through the absorb
+ # path kills the engine and one or both of these come back 500.
+ results = [None, None]
+
+ def ask(slot):
+ results[slot] = _post(server.url("/v1/completions"), {
+ "model": model_id, "prompt": PROMPT, "max_tokens": MAX_TOKENS,
+ "temperature": 0, "cache_slot": slot,
+ }, timeout=GENERATE_TIMEOUT)
+
+ threads = [threading.Thread(target=ask, args=(slot,)) for slot in (0, 1)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(GENERATE_TIMEOUT + 60)
+
+ if EXPECT_BITE:
+ self._assert_bite(server, results)
+ return
+
+ for slot, outcome in enumerate(results):
+ self.assertIsNotNone(outcome, f"slot {slot}: request never completed "
+ f"(asking thread still blocked)")
+ kind = outcome[0]
+ if kind == "timeout":
+ self._fail_with_server_evidence(
+ server, f"slot {slot}: generation request timed out after "
+ f"{outcome[1]:.0f}s (COLI_FP8_GENERATE_TIMEOUT to raise; "
+ f"a timeout is not a hang and not a protocol failure)")
+ if kind == "neterr":
+ self._fail_with_server_evidence(
+ server, f"slot {slot}: connection failed mid-request "
+ f"({outcome[1]}) -- engine/server dropped the stream")
+ _, status, body, resp_headers = outcome
+ if status != 200:
+ self._fail_with_server_evidence(
+ server,
+ f"slot {slot}: HTTP {status} (the D-I2 defect signature is a "
+ f"500 engine_error here)\nresponse body: "
+ f"{body.decode(errors='replace')[:2000]}")
+ payload = json.loads(body)
+ text = payload["choices"][0]["text"]
+ # "Coherent content", pinned mechanically: nonempty text with at
+ # least one real word, and the engine accounted for generated
+ # tokens. (Semantic quality belongs to the numeric battery, not
+ # this defect pin.)
+ self.assertTrue(text.strip(),
+ f"slot {slot}: 200 with empty text -- content-free "
+ f"success is not closure")
+ self.assertRegex(text, r"[A-Za-z]{2}",
+ f"slot {slot}: no word-like content in {text!r}")
+ self.assertGreater(payload["usage"]["completion_tokens"], 0,
+ f"slot {slot}: usage reports zero generated tokens")
+ # Concurrency witness: with two requests pinned to two distinct
+ # free slots, neither should queue. A silently serialized mux
+ # would park the second request for the first one's whole
+ # generation -- minutes, not milliseconds -- while every other
+ # assertion here stayed green.
+ queue_wait = resp_headers.get("x-colibri-queue-wait-ms")
+ self.assertIsNotNone(queue_wait,
+ f"slot {slot}: no x-colibri-queue-wait-ms response "
+ f"header -- the admission witness is gone")
+ self.assertLess(float(queue_wait), 1000.0,
+ f"slot {slot}: queued {queue_wait} ms behind the other "
+ f"slot -- the 2-slot mux is serializing, not batching")
+ # Echo the observed admission latency: the concurrency witness is
+ # a positive coverage claim, so its measured value belongs in the
+ # PASS artifact, not only in the failure path.
+ print(f"[fp8-e2e] witness: slot {slot} admission queue-wait = "
+ f"{queue_wait} ms", flush=True)
+
+ # No engine death, three ways: the named refusal family never fired,
+ # the server never recorded an engine exit, and the server still
+ # answers after both generations.
+ stderr = server.stderr()
+ refusal = REFUSAL.search(stderr)
+ self.assertIsNone(
+ refusal, "the engine printed the absorb-path refusal this test "
+ f"exists to keep dead:\n{stderr[stderr.rfind('qt_'):][:500]}")
+ self.assertNotIn(ENGINE_DEATH, stderr,
+ "the engine died during the batched run")
+ self.assertIsNone(server.proc.poll(), "server process exited mid-test")
+ status, _ = _get(server.url("/v1/models"), timeout=30)
+ self.assertEqual(status, 200, "server stopped answering after the batched pair")
+
+ # Boot-line witness (opt-in arm only): requesting the CUDA absorb
+ # path is not proof it ran -- certify it from the engine's own boot
+ # report, and refuse the CPU-dense line outright. The observed line
+ # is ECHOED on both pass and fail (positive coverage must be a
+ # captured artifact, not an inference from a green assertion), so a
+ # CI log carries the routed+dense observation verbatim.
+ if os.environ.get(CUDA_ABSORB_OPT_IN) == "1":
+ observed = observed_cuda_boot_mode(stderr)
+ print(f"[fp8-e2e] witness: observed boot mode = {observed!r}",
+ flush=True)
+ witness_failure = cuda_absorb_witness_failure(stderr)
+ if witness_failure:
+ self._fail_with_server_evidence(server, witness_failure)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.py
index 3e4d656f..c5452898 100644
--- a/c/tools/repack_fp8_passthrough.py
+++ b/c/tools/repack_fp8_passthrough.py
@@ -650,7 +650,7 @@ def _check_stamp_budget(total_stamped):
# produce ONE container, but each keeps its own progress manifest (a resume
# manifest has to be per-selection -- see --mtp's comment in main()). The stamp
# budget is not per-selection: st_fmt_stamp_ingest accumulates S->fmt_n over every
-# shard it discovers in the directory (c/st.h:372), so ST_FMT_STAMP_MAX is a
+# shard it discovers in the directory (c/st.h, the stamp scan), so ST_FMT_STAMP_MAX is a
# CONTAINER-wide bound. Counted per-manifest, two passes could each stay under the
# cap and still hand the engine a container over it -- the writer guarantee would
# be 2x the reader's bound. So the budget check sums this pass's running total with
@@ -926,17 +926,19 @@ def _print_inventory_summary(all_inv, dry_run):
# the minted directory does not have).
#
# The EXTERNAL (non-tensor) files the loader/server actually open from a
-# model dir at runtime, verified against this worktree's HEAD:
-# - config.json cfg_root, colibri.c:1361 -- fopen(...); if(!f){
+# model dir at runtime. Cited by SYMBOL, not line number: the previous table
+# carried line anchors that rotted twice, and a stale number is worse than no
+# number because it reads as a verification that was not performed.
+# - config.json colibri.c, cfg_root() -- fopen(...); if(!f){
# perror(p); exit(1); } -- MANDATORY, the run aborts without it. Also
-# read by openai_server.py's Engine.__init__ (:1773) for arch detection.
-# - generation_config.json colibri.c:1404-1405 -- fopen, comment "assente
+# read by openai_server.py's Engine.__init__ (:2635) for arch detection.
+# - generation_config.json colibri.c, cfg_root() -- fopen, comment "assente
# = nessun problema: e' opzionale" -- best-effort; HF's authority for
# generation defaults (extra EOS stop ids) when present.
-# - tokenizer.json c/tok.h:101 (tk_read_file, called from tok_load)
+# - tokenizer.json c/tok.h, tk_read_file() (called from tok_load)
# -- fopen(...); if(!f){ perror(path); exit(1); } -- MANDATORY. Called
# from every serve/generate entry point that needs a tokenizer
-# (colibri.c:7294 run_text, :7952 run_serve_mux, :8134 main serve loop).
+# (colibri.c: run_text, run_serve_mux, and the main serve loop).
# Before this fix main() never copied it into --outdir, so a minted
# directory was not standalone-loadable for THIS reason alone (confirmed
# by V2's end-to-end smoke test, 2026-08-18: load only succeeded via a
@@ -947,8 +949,8 @@ def _print_inventory_summary(all_inv, dry_run):
# fopen/Path().open against a model dir) and therefore excluded:
# - tokenizer_config.json, chat_template.jinja -- the GLM-5.2 chat template
# is reimplemented natively in code, not read from the .jinja file
-# (colibri.c:8211 comment: "template UFFICIALE GLM-5.2 (chat_template
-# .jinja): niente \n dopo i ruoli..."; openai_server.py:1016: "AUTHORITATIVE
+# (colibri.c, comment: "template UFFICIALE GLM-5.2 (chat_template
+# .jinja): niente \n dopo i ruoli..."; openai_server.py: "AUTHORITATIVE
# GLM-5.2 tool-declaration block (byte-matches chat_template.jinja)" --
# both hardcoded to match the file's behavior, not sourced from it).
# - README.md, LICENSE, .gitattributes -- pure repo/documentation metadata,
diff --git a/docs/FORMATS.md b/docs/FORMATS.md
index b52e90c8..16f38914 100644
--- a/docs/FORMATS.md
+++ b/docs/FORMATS.md
@@ -101,8 +101,9 @@ allowlist, :3379), consulted by both gate sites — `attention_rows` (:3448) and
`layer_forward_rows` (:5792) — and by the load-time notice
`metal_fmt_gate_notice` (:1866, called from `model_init`).
-Sources for all rows (`c/quant.h`/`c/colibri.c` line numbers at this PR
-pair's current restack, base dev `292ed4c`):
+Sources for all rows (`c/quant.h`/`c/colibri.c` line numbers verified at
+this branch's own head -- originally written against base dev `292ed4c`,
+re-anchored here because line numbers rot with the file, not the base):
- **fmt=0/1/2/3** — allocation policy: `qt_alloc`, `c/colibri.c`
(`bits>=16→fmt=0`, `bits>=5→fmt=1`, `bits>=4→fmt=2`, else `fmt=3`).