Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
test(fp8): e2e serve-batch pin, refusal canary, and mint-to-load regression
monotophicea8a7e4d55d9
diff --git a/c/Makefile b/c/Makefileindex 1ee880c1..eed82704 100644--- a/c/Makefile+++ b/c/Makefile@@ -498,12 +498,17 @@ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(f# "unsupported option '-msse4.1' for target arm64-...".# Appended back below only on x86-64 hosts, alongside the other conditional# platform tests already handled the same way.+# test_e8x4g64_loader takes a minted container directory on argv and prints its+# usage (exit 2) without one, so it is a harness rather than a gate: it has a+# rule here so it compiles under the suite's own $(CFLAGS) and its warnings are+# caught, and tests/test_e8x4g64_mint_load.py is what actually drives it.# test_qwen38_tier_engine drives qwen38.c over the generated FP8 fixture# (qwen38_tiny_fp8, gitignored); it runs from qwen38-tier-engine-check.TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \test_segment_adapters_registration test_segment_adapters_real \test_edge_adapters_registration test_edge_adapters_real \- test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine+ test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine \+ test_e8x4g64_loaderTEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)@@ -2096,6 +2101,14 @@ tests/test_olmoe_dot_i8_16_sse41$(EXE): tests/test_olmoe_dot_i8_16.c olmoe.c ssetests/test_shard_kvb_refuse$(EXE): tests/test_shard_kvb_refuse.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ)$(CC) $(CFLAGS) $< $(CUDA_OBJ) -o $@ $(LDFLAGS)+# The e8x4g64 container loader harness. It takes a minted directory on argv and+# is driven by tests/test_e8x4g64_mint_load.py, which builds it too -- this rule+# exists so it also compiles under the suite's own $(CFLAGS) rather than only+# under the driver's hand-copied flag list, and so a warning regression in it+# fails the normal build.+tests/test_e8x4g64_loader$(EXE): tests/test_e8x4g64_loader.c st.h quant.h fp8_format.h compat.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+test-c: $(TEST_BINS)$(PYTHON) tools/run_tests.py $(TEST_BINS)diff --git a/c/tests/test_e8x4g64_loader.c b/c/tests/test_e8x4g64_loader.cnew file mode 100644index 00000000..32fb1707--- /dev/null+++ b/c/tests/test_e8x4g64_loader.c@@ -0,0 +1,108 @@+/* wq-v0-class (e8x4g64: int8 spine + grouped-int4 g64 experts) END-TO-END+ * loader harness -- the C-side half of the mint->load regression case.+ *+ * tests/test_e8x4g64_mint_load.py drives it: builds a synthetic FP8 GLM-shaped+ * checkpoint (tools/glm_fp8_emit.py, the same fixture helper the fp8+ * passthrough e2e uses), mints it with the REAL tools/convert_fp8_to_int4.py+ * CLI at the e8x4g64 recipe (--ebits 8 --xbits 4 --io-bits 8 --group-size 64),+ * then invokes this binary against the real output directory. Sibling of+ * tests/test_fp8_e2e_loader.c (fmt=8 passthrough), same division of labor:+ * the Python side proves the TOOL ran for real; this side proves the REAL+ * st_init/qt_from_disk (the identical functions every model load uses)+ * resolve every minted tensor to the format the recipe promises.+ *+ * Usage: test_e8x4g64_loader <container_dir> [name O I wantfmt wantgs]...+ * For every (name,O,I,wantfmt,wantgs) 5-tuple, calls the REAL qt_from_disk and+ * asserts:+ * (a) the resolved fmt equals wantfmt (1 = int8 per-row spine, 4 = grouped+ * int4 experts) and, for fmt=4, the derived group size equals wantgs --+ * both resolved by qt_resolve_fmt's byte arithmetic on the tool's actual+ * output, neither mocked nor stamped;+ * (b) the weight and scale buffers are non-NULL;+ * (c) every dequantized value is finite -- catches a scale-layout or+ * packing bug a pure byte-count check wouldn't.+ *+ * The D-2 duplicate-name refusal is exercised through this same binary: the+ * Python driver runs it a second time against a copy of the container holding+ * a duplicated shard, and st_init below then refuses (exit 1, naming both+ * shards) before any tuple is checked -- the positive half of the D-2 pair,+ * on a tool-produced container. The clean run doubles as the negative+ * control: a legitimate mint loads with no refusal. */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++#include <stdio.h>+#include <string.h>+#include <math.h>+#include <stdlib.h>++int main(int argc, char **argv){+ if(argc < 2){ fprintf(stderr,"usage: %s <dir> [name O I wantfmt wantgs]...\n", argv[0]); return 2; }+ if((argc-2) % 5 != 0){+ fprintf(stderr,"args after <dir> must come in (name,O,I,wantfmt,wantgs) 5-tuples (got %d)\n", argc-2);+ return 2;+ }+ const char *dir = argv[1];+ int ntensors = (argc-2)/5;+ if(ntensors == 0){ fprintf(stderr,"no tuples given -- nothing to check\n"); return 2; }++ static Model gm; memset(&gm,0,sizeof gm);+ st_init(&gm.S, dir); /* D-2 duplicate-name detection lives in here (st.h) */++ int fails = 0;+ for(int i=0;i<ntensors;i++){+ const char *name = argv[2+i*5];+ int O = atoi(argv[3+i*5]);+ int I = atoi(argv[4+i*5]);+ int wantfmt = atoi(argv[5+i*5]);+ int wantgs = atoi(argv[6+i*5]);++ QT t; memset(&t,0,sizeof t);+ qt_from_disk(&gm, name, O, I, 8, 0, &t); /* THE REAL LOADER PATH -- not mocked */++ if(t.fmt != wantfmt){+ printf("FAIL %s: fmt=%d, expected %d\n", name, t.fmt, wantfmt);+ fails++; continue;+ }+ if(wantfmt == 4 && t.gs != wantgs){+ printf("FAIL %s: fmt=4 resolved with gs=%d, expected g%d\n", name, t.gs, wantgs);+ fails++; continue;+ }+ const void *w = t.fmt==1 ? (const void*)t.q8 : (const void*)t.q4;+ if(!w || !t.s){+ printf("FAIL %s: fmt=%d but weights=%p s=%p (expected both non-NULL)\n",+ name, t.fmt, w, (void*)t.s);+ fails++; continue;+ }+ /* (c) all-finite dequant, per format's own scale geometry */+ int64_t bad = -1;+ if(t.fmt == 1){+ for(int64_t o=0; o<O && bad<0; o++){ float s=t.s[o];+ for(int64_t ii=0; ii<I; ii++){+ float v=(float)t.q8[o*(int64_t)I+ii]*s;+ if(!isfinite(v)){ bad=o*(int64_t)I+ii; break; } } }+ } else if(t.fmt == 4){+ int ng=(I+t.gs-1)/t.gs;+ for(int64_t o=0; o<O && bad<0; o++){ const float *scl=t.s+o*ng;+ const uint8_t *row=t.q4+o*(int64_t)((I+1)/2);+ for(int64_t ii=0; ii<I; ii++){+ uint8_t b=row[ii>>1]; int nib=(ii&1)?(b>>4):(b&0xF);+ float v=((int)nib-8)*scl[ii/t.gs];+ if(!isfinite(v)){ bad=o*(int64_t)I+ii; break; } } }+ } else {+ printf("FAIL %s: this harness only knows the e8x4g64 formats (1, 4); "+ "wantfmt=%d is a driver bug\n", name, wantfmt);+ fails++; continue;+ }+ if(bad >= 0){+ printf("FAIL %s: non-finite dequantized value at flat index %lld\n", name, (long long)bad);+ fails++; continue;+ }+ printf("ok %s: fmt=%d%s O=%d I=%d, loaded through the real loader, all-finite\n",+ name, t.fmt, t.fmt==4?" (g64)":"", O, I);+ }+ if(fails){ printf("e8x4g64 mint->load: %d/%d tensor(s) FAILED\n", fails, ntensors); return 1; }+ printf("e8x4g64 mint->load: ok (%d tensor(s))\n", ntensors);+ return 0;+}diff --git a/c/tests/test_e8x4g64_mint_load.py b/c/tests/test_e8x4g64_mint_load.pynew file mode 100644index 00000000..7f506950--- /dev/null+++ b/c/tests/test_e8x4g64_mint_load.py@@ -0,0 +1,297 @@+"""wq-v0-class regression case: REAL tools/convert_fp8_to_int4.py output fed+into the REAL C loader (st_init/qt_from_disk in colibri.c) at toy scale.++The wq-v0 production container class is "e8x4g64": int8 per-row spine+(attention / shared expert / dense MLP / embed / lm_head -> fmt=1) with+grouped-int4 g64 routed experts (fmt=4, gs=64). This test mints that class+each cycle at toy scale with GLM-shaped tensor names and the load-bearing+real dimensions (kv_lora_rank-width kv_b contraction, group-64-divisible+expert dims) through the real converter CLI -- not a reimplementation of its+quantizers -- and then proves the REAL C loader resolves every tensor to the+format the recipe promises. Regression coverage, not new-feature proof: the+full-scale wq-v0 container is already banked and audited; what this pins is+that the TOOLING and the LOADER still agree on the class after engine+changes (the mint half of the mint->load->run regression bar; the run half+executes at full scale against the banked container, outside this suite).++Also carried here, because it belongs to exactly this container class: the+D-2 duplicate-tensor-name pair on a TOOL-PRODUCED container.+ * positive: a copy of the minted container with one shard duplicated must+ refuse at st_init with the exact D-2 message (naming both shards), before+ any tensor is read;+ * negative control: the untouched mint loads clean through the same+ binary -- the guard is duplicate-targeted, not a blanket gate.+(tests/test_dup_name_refusal.c pins the same guard on hand-built fixtures;+this is the tool-produced end of it.)++Hermetic: checkpoint, minted output, duplicated copy, and the compiled+harness all live under temporary directories; nothing is left behind.+"""+import glob+import os+import shutil+import subprocess+import sys+import tempfile+import unittest++try:+ import torch+except ImportError as e:+ raise unittest.SkipTest(f"torch not installed: {e}")++try:+ import numpy as np+except ImportError as e:+ raise unittest.SkipTest(f"numpy not installed: {e}")++HERE = os.path.dirname(os.path.abspath(__file__))+C_DIR = os.path.normpath(os.path.join(HERE, ".."))+sys.path.insert(0, os.path.join(C_DIR, "tools"))+# reuse: the real fp8 block-quantize fixture helpers, not reimplemented. The+# quantize/dequantize pair anchors the numeric reference below to EXACTLY what+# the converter ingests (its dequant() applies the same repeat_interleave+# formula to the same saved fp8 bytes); the converter's own int8/int4+# quantizers are NOT imported anywhere in this file -- their output is decoded+# and bounded by independent code below.+from glm_fp8_emit import save_fp8_safetensors, fp8_block_quantize, fp8_block_dequantize+++def _cc_flags():+ """Mirror the Makefile's CFLAGS closely enough to compile colibri.c+ cleanly (same arrangement as tests/test_fp8_e2e_repack_load.py)."""+ cc = shutil.which("cc") or shutil.which("clang") or shutil.which("gcc")+ if not cc:+ return None, None, None+ cflags = ["-O3", "-Wall", "-Wextra", "-Wno-unused-parameter",+ "-Wno-misleading-indentation", "-Wno-unused-function"]+ ldflags = ["-lm"]+ if sys.platform not in ("darwin", "win32"):+ cflags += ["-fopenmp"]+ ldflags += ["-fopenmp"]+ if sys.platform == "darwin":+ # The Makefile builds this harness with OpenMP on macOS via Homebrew's+ # libomp. Locating it must not fail quietly: a swallowed error here+ # compiles a DIFFERENT binary from the one the suite builds, and the+ # warning assertion below would then vouch for flags nobody ships.+ # Report why it could not be found and let the caller decide.+ try:+ probe = subprocess.run(["brew", "--prefix", "libomp"], capture_output=True,+ text=True, timeout=10)+ prefix = probe.stdout.strip()+ if probe.returncode != 0:+ raise RuntimeError(+ "`brew --prefix libomp` exited %d: %s"+ % (probe.returncode, probe.stderr.strip() or "(no stderr)"))+ except FileNotFoundError:+ raise RuntimeError(+ "libomp is required to build this harness on macOS the way the "+ "Makefile builds it, and `brew` is not on PATH")+ except (OSError, subprocess.TimeoutExpired) as exc:+ raise RuntimeError("could not run `brew --prefix libomp`: %r" % (exc,))+ inc, lib = os.path.join(prefix, "include"), os.path.join(prefix, "lib")+ if not (prefix and os.path.exists(os.path.join(inc, "omp.h"))):+ raise RuntimeError(+ "`brew --prefix libomp` gave %r but %s/omp.h does not exist; "+ "install libomp (`brew install libomp`) rather than compiling "+ "this harness without OpenMP" % (prefix, inc))+ cflags += ["-Xclang", "-fopenmp", "-I", inc]+ ldflags += ["-L", lib, "-lomp"]+ return cc, cflags, ldflags+++# Toy scale, real geometry where it is load-bearing:+# * kv_b_proj keeps the REAL contraction width I=512 (GLM-5.2's+# kv_lora_rank) with a toy head count -> O=160;+# * routed-expert dims stay multiples of 64 (real containers are), so the+# g64 grouping has no synthetic tail the production mint never sees --+# while D_H (o_proj/q_a rows, expert contraction) is NOT a multiple of+# 128, keeping the fp8 input's block scales on the partial-tail path;+# * no tensor sits on the fmt=1-vs-fmt=8 collision boundary+# (O == ceil(O/128)*ceil(I/128)), so byte-arithmetic resolution is+# unambiguous, as it is at full scale.+D_H = 320 # toy hidden size (multiple of 64, not of 128)+KV_B_O, KV_B_I = 160, 512+E_M = 192 # toy expert intermediate (multiple of 64)+VOCAB = 512+++class E8x4g64MintLoadTest(unittest.TestCase):+ """The real e8x4g64 mint recipe -> the real C loader."""++ _harness_dir = None+ _harness_bin = None+ _build_stderr = None++ @classmethod+ def setUpClass(cls):+ cc, cflags, ldflags = _cc_flags()+ if not cc:+ raise unittest.SkipTest("no C compiler found on PATH")+ cls._harness_dir = tempfile.TemporaryDirectory()+ harness_src = os.path.join(HERE, "test_e8x4g64_loader.c")+ harness_bin = os.path.join(cls._harness_dir.name, "test_e8x4g64_loader")+ build = subprocess.run([cc] + cflags + [harness_src, "-o", harness_bin] + ldflags,+ capture_output=True, text=True, cwd=C_DIR)+ if build.returncode != 0:+ raise AssertionError(f"e8x4g64 harness build failed:\n{build.stderr}")+ cls._harness_bin = harness_bin+ cls._build_stderr = build.stderr++ @classmethod+ def tearDownClass(cls):+ if cls._harness_dir:+ cls._harness_dir.cleanup()++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.indir = os.path.join(self.tmp.name, "fp8src")+ os.makedirs(self.indir)+ self.outdir = os.path.join(self.tmp.name, "out")+ self.shard = os.path.join(self.indir, "model-00001-of-00001.safetensors")++ def tearDown(self):+ self.tmp.cleanup()++ def _emit_checkpoint(self):+ """A GLM-shaped fp8 source checkpoint: every name below is one the real+ converter's classify() routes exactly like the production checkpoint's+ (kvb/attn/o -> ebits, sh/dmlp -> ebits, io -> io_bits, experts -> xbits,+ norms -> f32 passthrough)."""+ torch.manual_seed(11)+ L = "model.layers.0"+ sd = {+ f"{L}.self_attn.kv_b_proj.weight": torch.randn(KV_B_O, KV_B_I) * 0.02,+ f"{L}.self_attn.q_a_proj.weight": torch.randn(D_H, D_H) * 0.02,+ f"{L}.self_attn.o_proj.weight": torch.randn(D_H, D_H) * 0.02,+ f"{L}.mlp.shared_experts.gate_proj.weight": torch.randn(E_M, D_H) * 0.02,+ f"{L}.mlp.experts.0.gate_proj.weight": torch.randn(E_M, D_H) * 0.02,+ f"{L}.mlp.experts.0.up_proj.weight": torch.randn(E_M, D_H) * 0.02,+ f"{L}.mlp.experts.0.down_proj.weight": torch.randn(D_H, E_M) * 0.02,+ "model.embed_tokens.weight": torch.randn(VOCAB, D_H) * 0.02,+ f"{L}.input_layernorm.weight": torch.randn(D_H), # f32 passthrough control+ }+ self.sd = sd # kept: the numeric round-trip test derives its reference from it+ save_fp8_safetensors(sd, self.shard)++ def _mint(self):+ """The REAL converter CLI at the wq-v0 e8x4g64 recipe: int8 spine+ (--ebits 8, --io-bits 8), grouped-int4 g64 routed experts (--xbits 4,+ --group-size 64)."""+ tool = os.path.join(C_DIR, "tools", "convert_fp8_to_int4.py")+ rc = subprocess.run([sys.executable, tool, "--indir", self.indir,+ "--outdir", self.outdir, "--n-layers", "1",+ "--ebits", "8", "--xbits", "4", "--io-bits", "8",+ "--group-size", "64"],+ capture_output=True, text=True)+ self.assertEqual(rc.returncode, 0,+ f"real converter failed:\nSTDOUT:\n{rc.stdout}\nSTDERR:\n{rc.stderr}")+ outs = glob.glob(os.path.join(self.outdir, "out-*.safetensors"))+ self.assertEqual(len(outs), 1, f"expected exactly one minted shard, got {outs}")+ return outs[0]++ # 5-tuples for the C harness: (name, O, I, wantfmt, wantgs)+ _EXPECT = [+ ("model.layers.0.self_attn.kv_b_proj.weight", KV_B_O, KV_B_I, 1, 0),+ ("model.layers.0.self_attn.q_a_proj.weight", D_H, D_H, 1, 0),+ ("model.layers.0.self_attn.o_proj.weight", D_H, D_H, 1, 0),+ ("model.layers.0.mlp.shared_experts.gate_proj.weight", E_M, D_H, 1, 0),+ ("model.embed_tokens.weight", VOCAB, D_H, 1, 0),+ ("model.layers.0.mlp.experts.0.gate_proj.weight", E_M, D_H, 4, 64),+ ("model.layers.0.mlp.experts.0.up_proj.weight", E_M, D_H, 4, 64),+ ("model.layers.0.mlp.experts.0.down_proj.weight", D_H, E_M, 4, 64),+ ]++ def _load(self, container_dir):+ args = [self._harness_bin, container_dir]+ for name, n_out, n_in, fmt, gs in self._EXPECT:+ args += [name, str(n_out), str(n_in), str(fmt), str(gs)]+ return subprocess.run(args, capture_output=True, text=True)++ def test_harness_builds_without_warnings(self):+ """Production flags require zero warnings (pr-cycle mechanical gate)."""+ self.assertEqual(self._build_stderr.strip(), "",+ f"e8x4g64 harness build produced warnings:\n{self._build_stderr}")++ def test_e8x4g64_mint_loads_through_real_c_loader(self):+ self._emit_checkpoint()+ self._mint()+ rc = self._load(self.outdir)+ self.assertEqual(rc.returncode, 0,+ f"loader harness failed:\nSTDOUT:\n{rc.stdout}\nSTDERR:\n{rc.stderr}")+ for name, _, _, fmt, _ in self._EXPECT:+ self.assertIn(f"ok {name}: fmt={fmt}", rc.stdout)+ # negative D-2 control rides along: the clean mint produced NO+ # duplicate-name refusal anywhere in the load+ self.assertNotIn("duplicate tensor name", rc.stderr)++ def test_e8x4g64_numeric_round_trip(self):+ """The minted VALUES, not just the format class (deep-audit MAJOR-1:+ the fmt/finite checks alone passed a 37x scale corruption of+ quant_int8 -- format class and finiteness both survive numeric+ corruption). This closes DR-9(a)'s literal words, "mint ROUND-TRIP at+ toy scale": dequantize the minted bytes with independent numpy code+ (the converter's quantizers are never imported here) and require+ agreement with the fp8-round-tripped source within the schemes' own+ exact bound -- symmetric absmax rint quantization can never miss by+ more than half a quantization step, so the tolerance is 0.5 steps+ (+1e-3 for f32 arithmetic slop), derived from a scale RECOMPUTED from+ the source, never from the minted .qs (a corrupted stored scale must+ widen the error, not the bound)."""+ self._emit_checkpoint()+ minted = self._mint()+ from safetensors.numpy import load_file+ out = load_file(minted)+ for name, n_out, n_in, fmt, gs in self._EXPECT:+ # Reference: exactly what the converter ingested -- the fp8+ # block-quantized source, dequantized (bit-identical formula to+ # the converter's dequant(); see the import comment above).+ w_fp8, scale = fp8_block_quantize(self.sd[name].float())+ w_ref = fp8_block_dequantize(w_fp8, scale).numpy().astype(np.float32)+ self.assertEqual(w_ref.shape, (n_out, n_in))+ q, qs = out[name], out[name + ".qs"]+ if fmt == 1:+ # int8 per-row: one scale per output row, q in [-127,127]+ # (amax maps to qmax=127, so -128 is unreachable).+ deq = q.view(np.int8).reshape(n_out, n_in).astype(np.float32) * qs.reshape(n_out, 1)+ s_true = np.maximum(np.abs(w_ref).max(axis=1, keepdims=True) / 127.0, 1e-8)+ step = np.broadcast_to(s_true, (n_out, n_in))+ else:+ # int4 grouped g64: packed nibbles (low = even column), one+ # scale per (row, 64-column group), q in [-7,7] likewise.+ rb, ng = (n_in + 1) // 2, (n_in + gs - 1) // gs+ b = q.reshape(n_out, rb)+ deq = np.empty((n_out, n_in), np.float32)+ deq[:, 0::2] = (b & 0xF).astype(np.float32)[:, : (n_in + 1) // 2] - 8.0+ deq[:, 1::2] = (b >> 4).astype(np.float32)[:, : n_in // 2] - 8.0+ deq *= np.repeat(qs.reshape(n_out, ng), gs, axis=1)[:, :n_in]+ pad = np.zeros((n_out, ng * gs - n_in), np.float32)+ grp = np.concatenate([np.abs(w_ref), pad], axis=1).reshape(n_out, ng, gs)+ s_true = np.maximum(grp.max(axis=2) / 7.0, 1e-8) # [n_out, ng]+ step = np.repeat(s_true, gs, axis=1)[:, :n_in]+ worst = float((np.abs(deq - w_ref) / step).max())+ self.assertLessEqual(+ worst, 0.5 + 1e-3,+ f"{name}: minted values diverge from the source by {worst:.4g} "+ f"quantization steps (bound: half a step) -- the converter's "+ f"numeric output regressed even though the format class may "+ f"still be intact")++ def test_d2_duplicate_shard_refuses_by_name(self):+ """D-2 positive on the tool-produced container: duplicating the minted+ shard under a second indexed name must refuse at st_init with the+ exact duplicate-tensor-name message, before any tensor check runs."""+ self._emit_checkpoint()+ minted = self._mint()+ dupdir = os.path.join(self.tmp.name, "dup")+ shutil.copytree(self.outdir, dupdir)+ shutil.copy(minted, os.path.join(dupdir, "out-99999.safetensors"))+ rc = self._load(dupdir)+ self.assertNotEqual(rc.returncode, 0, "duplicated container must refuse to load")+ self.assertIn("duplicate tensor name across indexed shards, refusing", rc.stderr)+ self.assertNotIn("ok model.", rc.stdout,+ "no tensor may load from a container st_init refused")+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_fp8_refusal_canary.py b/c/tests/test_fp8_refusal_canary.pynew file mode 100644index 00000000..4d6dda30--- /dev/null+++ b/c/tests/test_fp8_refusal_canary.py@@ -0,0 +1,120 @@+"""Drift canary for the absorb-path fmt-refusal message shape.++tests/test_fp8_serve_batch_e2e.py machine-matches the engine's death on the+batched fmt=8 serve path with one regex (its REFUSAL constant, the family+`(qt_addrow|qt_matvec_rows): unsupported fmt=`), in two load-bearing+places: the pass lane's engine-death scan, and the expect-bite mode the+fleet's old-binary half PASSES ONLY THROUGH. The product strings live in+c/colibri.c (qt_addrow's refusal and qt_matvec_rows'); an innocent reword+of either -- "unsupported" to "unhandled", dropping the function-name+prefix -- would not break any build or unit suite, but it would silently+degrade the bite proof's machine-match into a generic "engine died", and+the e2e test only runs where a real fmt=8 container exists (never in CI).+This canary closes that gap in CI: it fails, by name, the moment the+matcher and the source strings drift apart, in either direction.++What is load-bearing is exactly what this file asserts, no more:+ * each function still has at least one refusal-class stderr message+ (its runtime firing is pinned separately by tests/test_qt_addrow.c's+ fork+waitpid refusal cases, which also require the "refus" word);+ * at least ONE refusal-class message from each function matches the e2e+ REFUSAL regex -- imported from the e2e module, never re-typed, so the+ two files cannot agree by coincidence (existential, not universal: a+ future second, differently-worded guard in the same function is a new+ refusal, not drift, and must not fail this canary);+ * the regex is still SELECTIVE -- it must not match an arbitrary death+ line, or expect-bite would file any crash as the bite.+Everything else about the messages (wording after the matched prefix,+fmt lists, line breaks) is deliberately unpinned: message edits that keep+the matchable shape must stay free.+"""+import re+import sys+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE)) # sibling import under any invocation style+from test_fp8_serve_batch_e2e import REFUSAL++COLIBRI_C = HERE.parent / "colibri.c"+FUNCTIONS = ("qt_addrow", "qt_matvec_rows")++# One C string literal (escapes included), and an fprintf-to-stderr whose+# message is one or more adjacent literals (the refusals wrap across lines).+_STRING = r'"(?:[^"\\\n]|\\.)*"'+_FPRINTF = re.compile(r'fprintf\s*\(\s*stderr\s*,\s*(' + _STRING + r'(?:\s*' + _STRING + r')*)')+++def stderr_messages(source):+ """Every fprintf(stderr, ...) format string in `source`, with adjacent+ literals concatenated (still escaped -- the matched prefix contains no+ escapes, so matching against the raw literal is faithful to what the+ runtime prints before the %d substitution)."""+ messages = []+ for match in _FPRINTF.finditer(source):+ parts = re.findall(_STRING, match.group(1))+ messages.append("".join(part[1:-1] for part in parts))+ return messages+++class RefusalShapeCanaryTest(unittest.TestCase):+ maxDiff = None++ @classmethod+ def setUpClass(cls):+ cls.messages = stderr_messages(COLIBRI_C.read_text(encoding="utf-8"))++ def messages_of(self, function):+ return [m for m in self.messages if m.startswith(function + ":")]++ def test_each_function_still_names_a_refusal_the_matcher_greps(self):+ for function in FUNCTIONS:+ named = self.messages_of(function)+ self.assertTrue(+ named,+ f"colibri.c has no stderr message starting '{function}:' -- the "+ f"refusal site was renamed or removed, and the D-I2 bite matcher "+ f"(test_fp8_serve_batch_e2e.REFUSAL) can no longer identify this "+ f"death; update the matcher and the fleet bite lane together")+ refusals = [m for m in named if "refus" in m]+ self.assertTrue(+ refusals,+ f"{function}: no refusal-class stderr message left (the 'refus' "+ f"discipline tests/test_qt_addrow.c pins at runtime) -- if the "+ f"refusal moved, this canary and the e2e matcher must follow it")+ self.assertTrue(+ any(REFUSAL.search(m) for m in refusals),+ f"none of {function}'s refusal messages matches the e2e "+ f"REFUSAL regex {REFUSAL.pattern!r} -- an innocent reword "+ f"silently turns the fleet bite proof's machine-match into a "+ f"generic 'engine died'; keep one matchable refusal per "+ f"function or change test_fp8_serve_batch_e2e.REFUSAL in the "+ f"same commit. (Existential on purpose: an ADDITIONAL, "+ f"differently-worded guard in this function is new coverage, "+ f"not drift.)\nmessages: {refusals}")++ def test_matcher_stays_selective(self):+ """The other drift direction: a loosened REFUSAL regex would make+ expect-bite accept ANY death as the bite. Pin that it rejects a+ representative non-refusal death line and the empty string, and+ that it keys on the FUNCTION FAMILY, not the bare `unsupported+ fmt=` words -- a matcher loosened to the words alone would accept+ any future non-absorb `unsupported fmt=` message as the bite."""+ self.assertIsNone(REFUSAL.search("malloc: out of memory allocating 42 GB"))+ self.assertIsNone(REFUSAL.search("colibri engine exited unexpectedly"))+ self.assertIsNone(REFUSAL.search(""))+ self.assertIsNone(REFUSAL.search("qt_resolve_fmt: unsupported fmt=9 refusing"))+ # The REAL in-tree near-miss (not synthetic): layer_cuda_shard_kvb's+ # refusal shares the `unsupported ... fmt=` words but is a different+ # guard family (shard-layout, pinned by tests/test_shard_kvb_refuse.c)+ # -- a loosened matcher that swallowed it would misfile a shard+ # refusal as the absorb bite in a fleet lane.+ self.assertIsNone(REFUSAL.search(+ "layer_cuda_shard_kvb: unsupported kv_b fmt=5 for the head-shard "+ "upload (only fmt 1/2/3/4 match the per-row byte/scale strides "+ "computed here) -- refusing the shard"))+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_fp8_serve_batch_e2e.py b/c/tests/test_fp8_serve_batch_e2e.pynew file mode 100644index 00000000..52bb35b6--- /dev/null+++ b/c/tests/test_fp8_serve_batch_e2e.py@@ -0,0 +1,668 @@+"""Defect-closure pin for the batched-serve fmt=8 abort (D-I2): two-slot+SERVE_BATCH mux against a REAL fp8 (fmt=8) `kv_b_proj` container.++The defect: batched serve mode (openai_server's Engine always launches the+engine with SERVE_BATCH=1; --kv-slots > 1 gives the mux real slots) decodes+kv_b through the MLA absorb path -- qt_addrow / qt_matvec_rows in colibri.c.+Before the fmt=8 absorb branches landed, those functions had no fp8 case:+the first generation request against a container whose kv_b_proj resolves to+fmt=8 (fp8-e4m3-b128 -- both production fp8 container classes, f8_full and+f8x4g64, are in this class) killed the engine with the named refusal++ qt_addrow: unsupported fmt=8 for the per-row-scale absorb path ...+ -- refusing rather than misread t->s[row]/t->q4++and the client saw a 500 `engine_error`. Single-stream chat never hit it,+so nothing in the FakeEngine-backed server suite could: the abort lives+below the engine wire protocol, on a code path only a real fmt=8 container+reaches. This test pins the closure end to end: serve the real container+batched, occupy BOTH KV slots concurrently, and require 200 + text on both+requests with the engine still alive afterwards.++A real fmt=8 container is hundreds of GB and exists only on the model+hosts, never in CI, so discovery is by environment variable:++ COLI_FP8_CONTAINER root of an engine-loadable container whose+ kv_b_proj is fp8 (fmt=8). Unset => named SKIP.+ COLI_ENGINE engine binary override (same convention as coli);+ default: the built `colibri` next to the server.+ SET but wrong => loud FAIL, never a skip.+ COLI_FP8_EXPECT_BITE=1 proof-of-bite mode for the fleet's old-binary+ half: the run PASSES only if the invocation FAILS+ to serve AND the captured server stderr carries+ the named `unsupported fmt=` refusal. A death+ without the refusal, or a clean serve, FAILS.+ COLI_FP8_READY_TIMEOUT seconds to wait for the server to come up+ (default 1800 -- container loads are large).+ COLI_FP8_GENERATE_TIMEOUT seconds per generation request (default+ 1800; two concurrent cold prefills on a+ storage-bound host can be slow).++Set-but-wrong is a FAILURE, not a skip: if COLI_FP8_CONTAINER names a+missing path, a container whose kv_b_proj is NOT fmt=8 (e.g. an int8+spine), or a shard set with duplicate tensor names, this test would+otherwise pass (or fail for the wrong reason) without ever entering the+absorb branch it exists to pin -- the vacuous-gate hazard. The safetensors+headers are checked first (pure header reads, no tensor data) and the+mismatch is a loud failure naming what was found. The same doctrine covers+COLI_ENGINE: a set-but-nonexistent engine path fails, it does not skip.+(Known unreachable corner: a shape with O == ceil(O/128)*ceil(I/128) makes+per-row and per-block scale counts collide and the unstamped loader would+resolve fmt=1 where this check says fmt=8 -- real GLM kv_b O is in the+thousands, so no planned container can reach it.)++The engine/server child environment is constructed fresh: ambient COLI_*+and engine-knob variables (KV8, KV_TQ, ...) are stripped except an+explicit backend/location allowlist plus two value-restricted knobs --+exactly ABSORB=0 (the ratified non-absorb fleet arm; any other ABSORB+value is stripped), and exactly COLI_CUDA_ATTN=1 WHEN the lane opts in+with COLI_FP8_E2E_CUDA_ABSORB=1 (default: stripped, so a leaked ambient+COLI_CUDA_ATTN can never swap the code path under test) -- and a leaked+COLI_API_KEY cannot 401 the ready poll; the kept/dropped set is printed+and attached to every failure so the run artifact shows the env the+invocation actually saw.++Lane semantics, stated exactly: by DEFAULT every lane (CPU or CUDA host)+exercises the CPU absorb arms -- the CUDA fmt=8 absorb decode is gated+on COLI_CUDA_ATTN (path selection) AND CUDA_DENSE (kv_b's+cuda_eligible), both of which this test strips unless the lane+explicitly opts in. A CUDA lane that sets COLI_FP8_E2E_CUDA_ABSORB=1+(plus its COLI_CUDA/COLI_GPU bindings and ambient COLI_CUDA_ATTN=1 and+CUDA_DENSE=1) pins the CUDA absorb decode end-to-end, and the test then+REQUIRES the engine's GPU-dense boot line as a witness -- an opt-in run+whose engine reports resident-dense-on-CPU fails rather than banking a+vacuous green. Without the opt-in, the CUDA absorb arm is pinned only+by tests/test_backend_cuda.cu.+"""+import json+import os+import re+import signal+import socket+import struct+import subprocess+import sys+import threading+import time+import unittest+from pathlib import Path+from urllib.error import HTTPError, URLError+from urllib.request import Request, urlopen++HERE = Path(__file__).resolve().parent+C_DIR = HERE.parent+CONTAINER = os.environ.get("COLI_FP8_CONTAINER")+ENGINE_OVERRIDE = os.environ.get("COLI_ENGINE")+EXPECT_BITE = os.environ.get("COLI_FP8_EXPECT_BITE") == "1"+READY_TIMEOUT = float(os.environ.get("COLI_FP8_READY_TIMEOUT", "1800"))+GENERATE_TIMEOUT = float(os.environ.get("COLI_FP8_GENERATE_TIMEOUT", "1800"))+PROMPT = "The primary colors are" # the DR-10 fixed prompt: short, deterministic continuation+MAX_TOKENS = 24+# The refusal family this test exists to keep dead. Both absorb entry points+# carry the same named message shape; match the family, not one function, so+# a regression through either surfaces by name in the failure output.+REFUSAL = re.compile(r"(qt_addrow|qt_matvec_rows): unsupported fmt=")+ENGINE_DEATH = "colibri engine exited unexpectedly"++# Child-environment policy (the invocation of record must not ride ambient+# state): backend selection and read-only model LOCATION are the only knob+# namespaces a lane may pass through -- CUDA lanes need COLI_CUDA/+# COLI_GPU(S) per the DR-10 bindings, and a split/mirrored container needs+# the engine to be TOLD where its shards live (location config says where+# the weights ARE; a behavior knob changes what the engine DOES with them+# -- only the former class passes):+# COLI_MODEL_DIRS -- extra shard directories (st_init_multi's SPLIT+# layout); stripping it hides the operator's shards+# and fails as a bogus "not an fmt=8 container".+# COLI_MODEL_MIRROR -- read-only replica dirs (multi-SSD read fan-out).+# COLI_MMAP -- how weights are mapped from disk; load+# placement/feasibility, not decode semantics.+# COLI_DISK_WEIGHTS -- disk-resident weight policy; feasibility for+# hundreds-of-GB loads, not decode semantics.+# Every other COLI_* (COLI_API_KEY -> 401'd ready poll) and the bare+# engine knobs below (KV8/KV_TQ change the KV format of record;+# SNAP/SERVE/SERVE_BATCH/NGEN/KV_SLOTS belong to the server, which sets+# its own) are stripped, and the strip is recorded. Documented legacy+# aliases are stripped ALONGSIDE their primaries so a scrubbed primary+# cannot resurface under its old name: SNAP_MIRROR (consulted only when+# COLI_MODEL_MIRROR is unset/empty -- i.e. precisely after this scrub) is+# name-stripped, and TEMP is stripped only when FULLY NUMERIC (matching+# temp_from_env's strtod whole-string test: a numeric TEMP is the+# deprecated sampling alias, a path TEMP is the Windows/ROCm temp+# directory and must survive). Value-restricted exceptions: the ratified+# non-absorb fleet arm selects itself with ABSORB=0, so exactly that+# value passes (any other ambient ABSORB is stripped -- both decode paths+# at ABSORB=0/default are ratified-equivalent, so a leaked "0" can shift+# which path is pinned but never fake a pass).+#+# The CUDA-absorb opt-in (COLI_FP8_E2E_CUDA_ABSORB=1) is handled by+# INJECTION, not passthrough -- see CUDA_ABSORB_INJECT below. A first+# attempt value-allowed COLI_CUDA_ATTN=1 and CUDA_DENSE=1 through the scrub+# and relied on the LANE to set both ambient; the fresh-env allowlist only+# ever KEEPS a variable already in the environment, so when the lane set+# COLI_CUDA_ATTN=1 but not the bare (non-COLI) CUDA_DENSE, the eligibility+# knob never reached the child and the run booted resident-dense-on-CPU+# (the boot-line witness caught it as a hard failure). The opt-in is the+# single source of truth for the bundle the path needs, so the test now+# SETS it directly and reports it, instead of hoping the lane assembled+# the same set correctly.+ENV_KNOBS_ALLOWED = ("COLI_CUDA", "COLI_GPU", "COLI_GPUS", "COLI_NO_OMP_TUNE",+ "COLI_MODEL_DIRS", "COLI_MODEL_MIRROR", "COLI_MMAP",+ "COLI_DISK_WEIGHTS")+ENV_KNOBS_VALUE_ALLOWED = {"ABSORB": ("0",)}+ENV_KNOBS_STRIPPED = ("KV8", "KV_TQ", "ABSORB", "CAP_RAISE", "CUDA_DENSE",+ "COLI_CUDA_ATTN", "SNAP", "SNAP_MIRROR", "SERVE",+ "SERVE_BATCH", "NGEN", "KV_SLOTS")+CUDA_ABSORB_OPT_IN = "COLI_FP8_E2E_CUDA_ABSORB"+# The exact engine knobs the CUDA fmt=8 absorb decode needs, INJECTED into+# the child (with the lane's own COLI_CUDA=1 backend binding) when the lane+# opts in. COLI_CUDA_ATTN=1 selects the CUDA absorb dispatch; CUDA_DENSE=1+# makes kv_b/o cuda_eligible -- qt_load grants eligibility only under+# g_cuda_dense (colibri.c:2123) and there is NO VRAM/budget gate on it+# (colibri.c:10966), so CUDA_DENSE=1 reaching the child is sufficient on a+# device that fits the dense tensors. CUDA_DENSE is a bare name (no COLI_+# prefix), which is exactly why passthrough could not carry it and explicit+# injection is required. Both are stripped from ambient (above) so an+# unrequested value can never leak in; under the opt-in the test's own "1"+# is what the child sees, recorded in the kept-knobs line as injected.+CUDA_ABSORB_INJECT = {"COLI_CUDA_ATTN": "1", "CUDA_DENSE": "1"}++# Boot-line witness for the opt-in arm: requesting a path is not proof it+# ran. The engine announces its residency decision at boot+# (colibri.c:11047-11049); under the opt-in this test asserts the GPU-dense+# line and REFUSES the CPU-dense line, so a misconfigured lane (e.g. the+# eligibility knob lost again) can never bank a vacuous green.+CUDA_DENSE_BOOT = "[CUDA] mode: routed experts + resident dense tensors"+CUDA_CPU_DENSE_BOOT = "[CUDA] mode: routed experts only (resident dense on CPU)"+++def observed_cuda_boot_mode(stderr_text):+ """The engine's `[CUDA] mode: ...` boot line as it appears in stderr, or+ None if it never printed. Echoed by the witness so a PASS artifact+ SHOWS the observed residency, not just an assertion that succeeded."""+ marker = "[CUDA] mode:"+ start = stderr_text.find(marker)+ if start == -1:+ return None+ end = stderr_text.find("\n", start)+ return stderr_text[start:end if end != -1 else None]+++def cuda_absorb_witness_failure(stderr_text):+ """None when the boot-line witness holds for the CUDA-absorb opt-in;+ otherwise the failure message.++ It checks two things against the engine's stderr: that the+ resident-dense-on-CPU boot line is absent, and that the+ routed+resident-dense line is present. Module-level so it can be called+ without constructing the test case, and so it can be exercised directly+ once there is a caller outside the container-gated test below -- there is+ none today, so on a machine without a container this function has no+ coverage at all."""+ if CUDA_CPU_DENSE_BOOT in stderr_text:+ return ("CUDA-absorb opt-in was requested but the engine booted with "+ f"{CUDA_CPU_DENSE_BOOT!r} -- resident dense stayed on the CPU, "+ "kv_b never became cuda_eligible, and the CUDA absorb path "+ "cannot have run (vacuous green refused; check CUDA_DENSE=1 "+ "reached the child, see the kept-knobs line)")+ if CUDA_DENSE_BOOT not in stderr_text:+ return ("CUDA-absorb opt-in was requested but the engine's boot "+ f"witness {CUDA_DENSE_BOOT!r} never appeared on stderr -- "+ "cannot certify the CUDA absorb path ran")+ return None+++def _numeric_temp(value):+ """temp_from_env's whole-string strtod test: TEMP acts as the deprecated+ sampling alias only when fully numeric; a path value is the system temp+ directory and is not a knob."""+ if not value:+ return False+ try:+ float(value)+ return True+ except ValueError:+ return False+++def _child_env():+ """Fresh server/engine environment per the policy above. Returns+ (env, report): the report names kept knobs with values and dropped+ knobs by NAME only (a dropped COLI_API_KEY must not leak its value+ into a run artifact)."""+ opt_in = os.environ.get(CUDA_ABSORB_OPT_IN) == "1"+ env, kept, dropped = {}, [], []+ for name, value in os.environ.items():+ # The CUDA-absorb bundle is INJECTED below when opted in, never taken+ # from ambient -- so the child sees the test's "1", not a leaked+ # value, and an unrequested ambient copy is always stripped here.+ if name in CUDA_ABSORB_INJECT:+ dropped.append(name)+ elif name in ENV_KNOBS_ALLOWED or value in ENV_KNOBS_VALUE_ALLOWED.get(name, ()):+ env[name] = value+ kept.append(f"{name}={value}")+ elif name.startswith("COLI_") or name in ENV_KNOBS_STRIPPED:+ dropped.append(name)+ elif name == "TEMP" and _numeric_temp(value):+ dropped.append(name) # numeric TEMP = deprecated COLI_TEMP alias+ else:+ env[name] = value # non-knob system env (PATH, HOME, LD_LIBRARY_PATH, ...)+ if opt_in:+ for name, value in CUDA_ABSORB_INJECT.items():+ env[name] = value+ kept.append(f"{name}={value} (injected)")+ dropped = [d for d in dropped if d not in CUDA_ABSORB_INJECT]+ return env, ("kept knobs: " + (", ".join(sorted(kept)) or "(none)")+ + "; dropped knobs: " + (", ".join(sorted(dropped)) or "(none)"))+++def _default_engine():+ """The built glm engine next to openai_server.py -- same candidates,+ same order, as the server's own default_engine()."""+ for name in ("colibri", "glm"):+ for suffix in ("", ".exe"):+ candidate = C_DIR / (name + suffix)+ if candidate.exists():+ return candidate+ return C_DIR / "colibri"+++ENGINE = Path(ENGINE_OVERRIDE) if ENGINE_OVERRIDE else _default_engine()+++def _shard_headers(container):+ """Merged {tensor name: (shape, nbytes)} across every *.safetensors shard+ header in the container root, plus the duplicate-name collisions found on+ the way. Header-only reads (u64 length + JSON): no tensor data is+ touched, so scanning a 140-shard container is cheap. Duplicates are+ returned rather than silently last-shard-wins: the engine refuses a+ duplicate tensor name across shards (st_init's D-2 guard), so a merged+ map that picked either copy could disagree with the load in either+ direction."""+ headers, owner, duplicates = {}, {}, []+ shards = sorted(Path(container).glob("*.safetensors"))+ for shard in shards:+ with open(shard, "rb") as f:+ (hlen,) = struct.unpack("<Q", f.read(8))+ entries = json.loads(f.read(hlen))+ for name, meta in entries.items():+ if name == "__metadata__":+ continue+ if name in headers:+ duplicates.append(f"{name} ({owner[name]} and {shard.name})")+ continue+ begin, end = meta["data_offsets"]+ headers[name] = (meta["shape"], end - begin)+ owner[name] = shard.name+ return headers, len(shards), duplicates+++def _kv_b_fmt8_check(headers):+ """Is the container's kv_b_proj in the fmt=8 class? Mirrors the loader's+ own byte arithmetic (qt_resolve_fmt, colibri.c): fmt=8 carries O*I raw+ e4m3 weight bytes plus ceil(O/128)*ceil(I/128) f32 block scales in the+ .qs sidecar, where fmt=1 (int8 spine) carries O per-row f32 scales. Real+ GLM kv_b shapes keep the two counts far apart, so the answer here is+ unambiguous. Returns (verdict, detail) -- detail names what WAS found so+ a wrong container fails readably."""+ names = [n for n in headers+ if n.endswith("self_attn.kv_b_proj.weight")]+ if not names:+ return False, "no self_attn.kv_b_proj.weight tensor in any shard header"+ name = sorted(names)[0]+ (shape, nb) = headers[name]+ if len(shape) != 2:+ return False, f"{name}: unexpected shape {shape}"+ o, i = shape+ qs = headers.get(name + ".qs")+ if qs is None:+ return False, f"{name}: no .qs scale sidecar (not a quantized-container tensor)"+ ns = qs[1]+ nblk = ((o + 127) // 128) * ((i + 127) // 128)+ if nb != o * i:+ return False, (f"{name}: [{o},{i}] weight is {nb} bytes, not the {o * i} "+ f"of a one-byte-per-weight layout")+ if ns == o * 4 and ns != nblk * 4:+ return False, (f"{name}: [{o},{i}] carries per-ROW scales (ns={ns}) -- an "+ f"int8-spine container, kv_b is fmt=1, the fmt=8 absorb "+ f"path would never run")+ if ns != nblk * 4:+ return False, (f"{name}: [{o},{i}] scale sidecar is {ns} bytes; fmt=8 "+ f"per-128x128-block f32 scales would be {nblk * 4}")+ return True, f"{name}: [{o},{i}] weight {nb}B + {ns}B block scales (fmt=8)"+++class _Server:+ """openai_server.py as a subprocess -- the production launch, not an+ in-process shim: Engine's child environment (SERVE=1, SERVE_BATCH=1,+ KV_SLOTS) and the real engine binary are exactly what a user gets.+ Launched in its own session (POSIX) so close() can sweep the WHOLE+ process group: during the container load the server has no SIGTERM+ handler yet (serve() installs it only after Engine.__init__ returns),+ so a plain terminate() on the ready-timeout path kills the server+ without its engine-cleanup finally ever running, orphaning an engine+ that streams hundreds of GB for the rest of its load. On Windows the+ server's own KILL_ON_JOB_CLOSE job object ties the engine to the+ server's lifetime instead."""++ def __init__(self, port):+ env, self.env_report = _child_env()+ print(f"[fp8-e2e] child env of record -- {self.env_report}", flush=True)+ self.proc = subprocess.Popen(+ [sys.executable, str(C_DIR / "openai_server.py"),+ "--model", CONTAINER, "--engine", str(ENGINE),+ "--port", str(port), "--kv-slots", "2"],+ cwd=C_DIR, env=env, stdin=subprocess.DEVNULL,+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ start_new_session=(os.name == "posix"))+ self.port = port+ # Drain both pipes continuously: the interesting evidence (the named+ # refusal, the engine-death notice) arrives on stderr AFTER the pipe+ # would have filled and deadlocked a read-at-the-end design.+ self._out, self._err = [], []+ self._pumps = []+ for pipe, sink in ((self.proc.stdout, self._out), (self.proc.stderr, self._err)):+ t = threading.Thread(target=self._drain, args=(pipe, sink), daemon=True)+ t.start()+ self._pumps.append(t)++ @staticmethod+ def _drain(pipe, sink):+ for line in iter(pipe.readline, b""):+ sink.append(line.decode(errors="replace"))++ def stderr(self):+ return "".join(self._err)++ def url(self, path):+ return f"http://127.0.0.1:{self.port}{path}"++ def _sweep_group(self):+ """SIGKILL the server's whole session (server + engine + the OMP+ re-exec). Harmless when everything already exited."""+ if os.name != "posix":+ return # Windows: job object covers the engine+ try:+ os.killpg(self.proc.pid, signal.SIGKILL)+ except (ProcessLookupError, PermissionError, OSError):+ pass++ def close(self):+ try:+ if self.proc.poll() is None:+ self.proc.terminate() # graceful first: serve()'s SIGTERM handler shuts the engine down+ try:+ self.proc.wait(timeout=60)+ except subprocess.TimeoutExpired:+ pass+ finally:+ # Unconditional group sweep: on the ready-timeout path the+ # engine survives the server's death (no handler installed yet)+ # and holds the write end of our stderr pipe -- without this+ # kill the pumps never see EOF and the test process itself+ # would hang for the orphan's remaining load time.+ self._sweep_group()+ if self.proc.poll() is None:+ self.proc.kill()+ try:+ self.proc.wait(timeout=60)+ except subprocess.TimeoutExpired:+ pass+ for t in self._pumps: # EOF is guaranteed once the group is swept+ t.join(timeout=10)+ if all(not t.is_alive() for t in self._pumps):+ self.proc.stdout.close()+ self.proc.stderr.close()+ # else: leave the fds open -- a leaked descriptor in a dying test+ # process beats deadlocking close() against a blocked reader.+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _get(url, timeout):+ try:+ with urlopen(url, timeout=timeout) as response:+ return response.status, response.read()+ except HTTPError as error:+ return error.code, error.read()+++def _post(url, body, timeout):+ """One generation request. Returns a tagged outcome so the caller can+ tell an HTTP answer, a timed-out request, and a dropped connection+ apart -- conflating them cost a whole diagnosis class (a timeout used+ to kill the asking thread and read as 'request never completed')."""+ request = Request(url, data=json.dumps(body).encode(),+ headers={"Content-Type": "application/json"})+ try:+ with urlopen(request, timeout=timeout) as response:+ return ("http", response.status, response.read(), response.headers)+ except HTTPError as error:+ return ("http", error.code, error.read(), error.headers)+ except TimeoutError: # socket.timeout is this same type+ return ("timeout", timeout, b"", None)+ except URLError as error:+ if isinstance(getattr(error, "reason", None), TimeoutError):+ return ("timeout", timeout, b"", None)+ return ("neterr", repr(error), b"", None)+ except OSError as error: # e.g. connection reset by a dying engine+ return ("neterr", repr(error), b"", None)++[email protected](CONTAINER,+ "COLI_FP8_CONTAINER is not set: no fmt=8 container on this host")[email protected](ENGINE_OVERRIDE or ENGINE.exists(),+ "engine binary is not built (make -C c colibri, or set COLI_ENGINE)")+class ServeBatchFmt8E2ETest(unittest.TestCase):+ maxDiff = None++ def _fail_with_server_evidence(self, server, message):+ """Every hard failure carries the server's stderr tail and the child+ env of record: on a pre-fix engine that tail contains the named+ qt_addrow refusal (see the expect-bite mode for the machine-checked+ version of that claim)."""+ tail = server.stderr()[-4000:]+ self.fail(f"{message}\n--- child env of record ---\n{server.env_report}"+ f"\n--- server stderr (tail) ---\n{tail}")++ def _assert_bite(self, server, results):+ """COLI_FP8_EXPECT_BITE=1: the fleet's old-binary half. PASS iff the+ invocation failed to serve AND the named refusal is machine-matched+ in the captured stderr. An engine that dies for ANY other reason+ (OOM, protocol desync, bad path) FAILS -- a death without the named+ refusal is not this defect and must never be filed as its bite."""+ failed = [outcome for outcome in results+ if outcome is None or outcome[0] != "http" or outcome[1] != 200]+ stderr = server.stderr()+ match = REFUSAL.search(stderr)+ if not failed:+ self._fail_with_server_evidence(+ server, "expect-bite mode: both batched requests served 200 -- "+ "no bite. Is COLI_ENGINE really the pre-fix binary?")+ if match is None:+ self._fail_with_server_evidence(+ server, "expect-bite mode: the invocation failed WITHOUT the "+ "named `unsupported fmt=` refusal -- a different death, "+ "not this defect's bite")+ # PASS: machine-matched refusal + failed serve. Record the matched+ # line so the run artifact quotes the bite verbatim.+ line_start = stderr.rfind("\n", 0, match.start()) + 1+ line_end = stderr.find("\n", match.start())+ print("[fp8-e2e] BITE CONFIRMED: "+ + stderr[line_start:line_end if line_end != -1 else None], flush=True)++ def test_batched_fmt8_serve_two_slots_200_and_engine_survives(self):+ # Set-but-wrong engine is a FAILURE (same doctrine as the container):+ # the decorator skips only when NO engine was named and none is+ # built; a named engine that does not exist or cannot execute is a+ # misconfigured lane, and skipping it would silently drop the one+ # lane whose whole purpose is running that exact binary.+ if ENGINE_OVERRIDE:+ self.assertTrue(ENGINE.is_file(),+ f"COLI_ENGINE={ENGINE_OVERRIDE} does not exist -- "+ f"fix the path or unset it to use the built engine")+ self.assertTrue(os.access(str(ENGINE), os.X_OK),+ f"COLI_ENGINE={ENGINE_OVERRIDE} is not executable")+ # Precondition, checked loudly (never skipped): the pointed-to+ # container really is the fmt=8 kv_b class. A wrong container would+ # pass every assertion below without the absorb path ever running.+ self.assertTrue(os.path.isdir(CONTAINER),+ f"COLI_FP8_CONTAINER={CONTAINER} is not a directory -- "+ f"set it to a container root or unset it to skip")+ headers, n_shards, duplicates = _shard_headers(CONTAINER)+ self.assertTrue(n_shards > 0,+ f"COLI_FP8_CONTAINER={CONTAINER} holds no *.safetensors shards")+ self.assertFalse(+ duplicates,+ f"COLI_FP8_CONTAINER={CONTAINER} carries duplicate tensor names "+ f"across shards (the engine's st_init refuses exactly this, and "+ f"this check could otherwise judge a shard the engine never "+ f"loads): {'; '.join(duplicates[:5])}")+ is_fmt8, detail = _kv_b_fmt8_check(headers)+ self.assertTrue(is_fmt8,+ f"COLI_FP8_CONTAINER={CONTAINER} is not an fmt=8 kv_b "+ f"container; this test would be vacuous against it: {detail}")++ server = _Server(_free_port())+ self.addCleanup(server.close)++ # Startup: poll /v1/models (touches only the server, not the engine+ # generate path) until it answers, and read the served model id from+ # it rather than hardcoding one. Sleep EVERY iteration -- a fast+ # non-200 answer must not turn the poll into a hot spin -- and fail+ # fast on auth/host-guard rejections, which no amount of waiting+ # will turn into a 200.+ deadline = time.time() + READY_TIMEOUT+ model_id = None+ while time.time() < deadline:+ if server.proc.poll() is not None:+ self._fail_with_server_evidence(+ server, f"server exited rc={server.proc.returncode} before READY")+ status = None+ try:+ status, body = _get(server.url("/v1/models"), timeout=10)+ except (URLError, OSError):+ pass # not accepting yet -- keep waiting+ if status == 200:+ model_id = json.loads(body)["data"][0]["id"]+ break+ if status in (401, 403):+ self._fail_with_server_evidence(+ server, f"ready poll got HTTP {status} from /v1/models -- "+ f"an auth/host-guard rejection, not a slow load; "+ f"waiting longer cannot fix it")+ time.sleep(2)+ if model_id is None:+ self._fail_with_server_evidence(+ server, f"server not ready within {READY_TIMEOUT:.0f}s")++ # The D-I2 invocation class: two generation requests IN FLIGHT+ # TOGETHER, pinned to distinct KV slots so the 2-slot mux really+ # multiplexes (conversation hashing would put one identical prompt in+ # one slot). temperature 0 / fixed prompt per the invocation of+ # record. On a pre-fix engine the FIRST decode through the absorb+ # path kills the engine and one or both of these come back 500.+ results = [None, None]++ def ask(slot):+ results[slot] = _post(server.url("/v1/completions"), {+ "model": model_id, "prompt": PROMPT, "max_tokens": MAX_TOKENS,+ "temperature": 0, "cache_slot": slot,+ }, timeout=GENERATE_TIMEOUT)++ threads = [threading.Thread(target=ask, args=(slot,)) for slot in (0, 1)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(GENERATE_TIMEOUT + 60)++ if EXPECT_BITE:+ self._assert_bite(server, results)+ return++ for slot, outcome in enumerate(results):+ self.assertIsNotNone(outcome, f"slot {slot}: request never completed "+ f"(asking thread still blocked)")+ kind = outcome[0]+ if kind == "timeout":+ self._fail_with_server_evidence(+ server, f"slot {slot}: generation request timed out after "+ f"{outcome[1]:.0f}s (COLI_FP8_GENERATE_TIMEOUT to raise; "+ f"a timeout is not a hang and not a protocol failure)")+ if kind == "neterr":+ self._fail_with_server_evidence(+ server, f"slot {slot}: connection failed mid-request "+ f"({outcome[1]}) -- engine/server dropped the stream")+ _, status, body, resp_headers = outcome+ if status != 200:+ self._fail_with_server_evidence(+ server,+ f"slot {slot}: HTTP {status} (the D-I2 defect signature is a "+ f"500 engine_error here)\nresponse body: "+ f"{body.decode(errors='replace')[:2000]}")+ payload = json.loads(body)+ text = payload["choices"][0]["text"]+ # "Coherent content", pinned mechanically: nonempty text with at+ # least one real word, and the engine accounted for generated+ # tokens. (Semantic quality belongs to the numeric battery, not+ # this defect pin.)+ self.assertTrue(text.strip(),+ f"slot {slot}: 200 with empty text -- content-free "+ f"success is not closure")+ self.assertRegex(text, r"[A-Za-z]{2}",+ f"slot {slot}: no word-like content in {text!r}")+ self.assertGreater(payload["usage"]["completion_tokens"], 0,+ f"slot {slot}: usage reports zero generated tokens")+ # Concurrency witness: with two requests pinned to two distinct+ # free slots, neither should queue. A silently serialized mux+ # would park the second request for the first one's whole+ # generation -- minutes, not milliseconds -- while every other+ # assertion here stayed green.+ queue_wait = resp_headers.get("x-colibri-queue-wait-ms")+ self.assertIsNotNone(queue_wait,+ f"slot {slot}: no x-colibri-queue-wait-ms response "+ f"header -- the admission witness is gone")+ self.assertLess(float(queue_wait), 1000.0,+ f"slot {slot}: queued {queue_wait} ms behind the other "+ f"slot -- the 2-slot mux is serializing, not batching")+ # Echo the observed admission latency: the concurrency witness is+ # a positive coverage claim, so its measured value belongs in the+ # PASS artifact, not only in the failure path.+ print(f"[fp8-e2e] witness: slot {slot} admission queue-wait = "+ f"{queue_wait} ms", flush=True)++ # No engine death, three ways: the named refusal family never fired,+ # the server never recorded an engine exit, and the server still+ # answers after both generations.+ stderr = server.stderr()+ refusal = REFUSAL.search(stderr)+ self.assertIsNone(+ refusal, "the engine printed the absorb-path refusal this test "+ f"exists to keep dead:\n{stderr[stderr.rfind('qt_'):][:500]}")+ self.assertNotIn(ENGINE_DEATH, stderr,+ "the engine died during the batched run")+ self.assertIsNone(server.proc.poll(), "server process exited mid-test")+ status, _ = _get(server.url("/v1/models"), timeout=30)+ self.assertEqual(status, 200, "server stopped answering after the batched pair")++ # Boot-line witness (opt-in arm only): requesting the CUDA absorb+ # path is not proof it ran -- certify it from the engine's own boot+ # report, and refuse the CPU-dense line outright. The observed line+ # is ECHOED on both pass and fail (positive coverage must be a+ # captured artifact, not an inference from a green assertion), so a+ # CI log carries the routed+dense observation verbatim.+ if os.environ.get(CUDA_ABSORB_OPT_IN) == "1":+ observed = observed_cuda_boot_mode(stderr)+ print(f"[fp8-e2e] witness: observed boot mode = {observed!r}",+ flush=True)+ witness_failure = cuda_absorb_witness_failure(stderr)+ if witness_failure:+ self._fail_with_server_evidence(server, witness_failure)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/repack_fp8_passthrough.py b/c/tools/repack_fp8_passthrough.pyindex 3e4d656f..c5452898 100644--- a/c/tools/repack_fp8_passthrough.py+++ b/c/tools/repack_fp8_passthrough.py@@ -650,7 +650,7 @@ def _check_stamp_budget(total_stamped):# produce ONE container, but each keeps its own progress manifest (a resume# manifest has to be per-selection -- see --mtp's comment in main()). The stamp# budget is not per-selection: st_fmt_stamp_ingest accumulates S->fmt_n over every-# shard it discovers in the directory (c/st.h:372), so ST_FMT_STAMP_MAX is a+# shard it discovers in the directory (c/st.h, the stamp scan), so ST_FMT_STAMP_MAX is a# CONTAINER-wide bound. Counted per-manifest, two passes could each stay under the# cap and still hand the engine a container over it -- the writer guarantee would# be 2x the reader's bound. So the budget check sums this pass's running total with@@ -926,17 +926,19 @@ def _print_inventory_summary(all_inv, dry_run):# the minted directory does not have).## The EXTERNAL (non-tensor) files the loader/server actually open from a-# model dir at runtime, verified against this worktree's HEAD:-# - config.json cfg_root, colibri.c:1361 -- fopen(...); if(!f){+# model dir at runtime. Cited by SYMBOL, not line number: the previous table+# carried line anchors that rotted twice, and a stale number is worse than no+# number because it reads as a verification that was not performed.+# - config.json colibri.c, cfg_root() -- fopen(...); if(!f){# perror(p); exit(1); } -- MANDATORY, the run aborts without it. Also-# read by openai_server.py's Engine.__init__ (:1773) for arch detection.-# - generation_config.json colibri.c:1404-1405 -- fopen, comment "assente+# read by openai_server.py's Engine.__init__ (:2635) for arch detection.+# - generation_config.json colibri.c, cfg_root() -- fopen, comment "assente# = nessun problema: e' opzionale" -- best-effort; HF's authority for# generation defaults (extra EOS stop ids) when present.-# - tokenizer.json c/tok.h:101 (tk_read_file, called from tok_load)+# - tokenizer.json c/tok.h, tk_read_file() (called from tok_load)# -- fopen(...); if(!f){ perror(path); exit(1); } -- MANDATORY. Called# from every serve/generate entry point that needs a tokenizer-# (colibri.c:7294 run_text, :7952 run_serve_mux, :8134 main serve loop).+# (colibri.c: run_text, run_serve_mux, and the main serve loop).# Before this fix main() never copied it into --outdir, so a minted# directory was not standalone-loadable for THIS reason alone (confirmed# by V2's end-to-end smoke test, 2026-08-18: load only succeeded via a@@ -947,8 +949,8 @@ def _print_inventory_summary(all_inv, dry_run):# fopen/Path().open against a model dir) and therefore excluded:# - tokenizer_config.json, chat_template.jinja -- the GLM-5.2 chat template# is reimplemented natively in code, not read from the .jinja file-# (colibri.c:8211 comment: "template UFFICIALE GLM-5.2 (chat_template-# .jinja): niente \n dopo i ruoli..."; openai_server.py:1016: "AUTHORITATIVE+# (colibri.c, comment: "template UFFICIALE GLM-5.2 (chat_template+# .jinja): niente \n dopo i ruoli..."; openai_server.py: "AUTHORITATIVE# GLM-5.2 tool-declaration block (byte-matches chat_template.jinja)" --# both hardcoded to match the file's behavior, not sourced from it).# - README.md, LICENSE, .gitattributes -- pure repo/documentation metadata,diff --git a/docs/FORMATS.md b/docs/FORMATS.mdindex b52e90c8..16f38914 100644--- a/docs/FORMATS.md+++ b/docs/FORMATS.md@@ -101,8 +101,9 @@ allowlist, :3379), consulted by both gate sites — `attention_rows` (:3448) and`layer_forward_rows` (:5792) — and by the load-time notice`metal_fmt_gate_notice` (:1866, called from `model_init`).-Sources for all rows (`c/quant.h`/`c/colibri.c` line numbers at this PR-pair's current restack, base dev `292ed4c`):+Sources for all rows (`c/quant.h`/`c/colibri.c` line numbers verified at+this branch's own head -- originally written against base dev `292ed4c`,+re-anchored here because line numbers rot with the file, not the base):- **fmt=0/1/2/3** — allocation policy: `qt_alloc`, `c/colibri.c`(`bits>=16→fmt=0`, `bits>=5→fmt=1`, `bits>=4→fmt=2`, else `fmt=3`).