Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
security(glm53): refuse f32 tensors shorter than the config reads
Marcello Costagliolaf65edabfdc13
diff --git a/c/glm53.c b/c/glm53.cindex a5cbecf1..42ce4a11 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -724,11 +724,20 @@ typedef struct {ColiVisionBlock *vblocks;} GModel;-static const float *load_f32(GModel *m, const char *fmt, ...) {+/* `want` e' quanti valori legge chi usa il tensore, contati dalla config. Il+ * buffer e' grande quanto dice l'header del file: se dice meno, la prima norma+ * o il router leggerebbero oltre la fine. Di piu' si accetta, come fa+ * deepseek_v4 (GHSA-9gjf): un checkpoint col padding resta buono. */+static const float *load_f32(GModel *m, int64_t want, const char *fmt, ...) {char name[512];va_list args; va_start(args, fmt); vsnprintf(name, sizeof(name), fmt, args); va_end(args);st_tensor *t = st_find(&m->S, name);if (!t) { fprintf(stderr, "missing tensor %s\n", name); exit(1); }+ if (t->numel < want) {+ fprintf(stderr, "%s: %lld values, the config needs %lld\n", name,+ (long long)t->numel, (long long)want);+ exit(1);+ }float *buffer = malloc((size_t)t->numel * sizeof(float));if (!buffer) { fprintf(stderr, "OOM on %s\n", name); exit(1); }st_read_f32_cap(&m->S, name, buffer, t->numel, 0);@@ -2008,6 +2017,10 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,snprintf(probe, sizeof(probe), "%sembed_tokens.weight", m->prefix);if (!st_find(&m->S, probe)) snprintf(m->prefix, sizeof(m->prefix), "model.");const char *P = m->prefix;+ /* Le misure con cui il forward legge i vettori f32 (vedi load_f32). Le+ * matrici mHC sono [(2+hc)*hc, hc*hidden], come in hyper_connections.h. */+ const Cfg *c = &m->c;+ const int64_t D = c->hidden, hc = c->hc_mult, hc_mix = (2 + hc) * hc;if (layer_end < 0 || layer_end > m->c.n_layers) layer_end = m->c.n_layers;if (layer_begin < 0) layer_begin = 0;@@ -2017,8 +2030,8 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,m->has_io = load_io;if (load_io) {- m->embed = load_f32(m, "%sembed_tokens.weight", P);- m->final_norm = load_f32(m, "%snorm.weight", P);+ m->embed = load_f32(m, c->vocab * D, "%sembed_tokens.weight", P);+ m->final_norm = load_f32(m, D, "%snorm.weight", P);}/* La testa e' l'unica matrice grande fuori dagli esperti: a vocab 154880* per hidden 4096 sono 2,5 GB in f32, quindi passa dallo stesso@@ -2055,20 +2068,20 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,for (int i = layer_begin; i < layer_end; i++) {GLayer *l = &m->layer[i];- l->in_ln = load_f32(m, "%slayers.%d.input_layernorm.weight", P, i);- l->post_ln = load_f32(m, "%slayers.%d.post_attention_layernorm.weight", P, i);- l->hc_attn_fn = load_f32(m, "%slayers.%d.hc_attn_fn", P, i);- l->hc_attn_base = load_f32(m, "%slayers.%d.hc_attn_base", P, i);- l->hc_attn_scale = load_f32(m, "%slayers.%d.hc_attn_scale", P, i);- l->hc_ffn_fn = load_f32(m, "%slayers.%d.hc_ffn_fn", P, i);- l->hc_ffn_base = load_f32(m, "%slayers.%d.hc_ffn_base", P, i);- l->hc_ffn_scale = load_f32(m, "%slayers.%d.hc_ffn_scale", P, i);+ l->in_ln = load_f32(m, D, "%slayers.%d.input_layernorm.weight", P, i);+ l->post_ln = load_f32(m, D, "%slayers.%d.post_attention_layernorm.weight", P, i);+ l->hc_attn_fn = load_f32(m, hc_mix * hc * D, "%slayers.%d.hc_attn_fn", P, i);+ l->hc_attn_base = load_f32(m, hc_mix, "%slayers.%d.hc_attn_base", P, i);+ l->hc_attn_scale = load_f32(m, 3, "%slayers.%d.hc_attn_scale", P, i);+ l->hc_ffn_fn = load_f32(m, hc_mix * hc * D, "%slayers.%d.hc_ffn_fn", P, i);+ l->hc_ffn_base = load_f32(m, hc_mix, "%slayers.%d.hc_ffn_base", P, i);+ l->hc_ffn_scale = load_f32(m, 3, "%slayers.%d.hc_ffn_scale", P, i);if (m->c.is_full[i]) {l->qa = load_mat(m, "%slayers.%d.self_attn.q_a_proj.weight", P, i);- l->qa_ln = load_f32(m, "%slayers.%d.self_attn.q_a_layernorm.weight", P, i);+ l->qa_ln = load_f32(m, c->q_lora, "%slayers.%d.self_attn.q_a_layernorm.weight", P, i);l->qb = load_mat(m, "%slayers.%d.self_attn.q_b_proj.weight", P, i);l->kva = load_mat(m, "%slayers.%d.self_attn.kv_a_proj_with_mqa.weight", P, i);- l->kva_ln = load_f32(m, "%slayers.%d.self_attn.kv_a_layernorm.weight", P, i);+ l->kva_ln = load_f32(m, c->kv_lora, "%slayers.%d.self_attn.kv_a_layernorm.weight", P, i);{char kvb_name[512];snprintf(kvb_name, sizeof(kvb_name),@@ -2079,10 +2092,11 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,l->iwq = load_mat(m, "%slayers.%d.self_attn.indexer.wq_b.weight", P, i);l->iwk = load_mat(m, "%slayers.%d.self_attn.indexer.wk.weight", P, i);l->iwp = load_mat(m, "%slayers.%d.self_attn.indexer.weights_proj.weight", P, i);- l->ik_nw = load_f32(m, "%slayers.%d.self_attn.indexer.k_norm.weight", P, i);- l->ik_nb = load_f32(m, "%slayers.%d.self_attn.indexer.k_norm.bias", P, i);+ l->ik_nw = load_f32(m, c->index_hd, "%slayers.%d.self_attn.indexer.k_norm.weight", P, i);+ l->ik_nb = load_f32(m, c->index_hd, "%slayers.%d.self_attn.indexer.k_norm.bias", P, i);if (m->c.index_kpool > 1) {- l->ikpa = load_f32(m, "%slayers.%d.self_attn.indexer.index_kpool_compress_ape", P, i);+ l->ikpa = load_f32(m, (int64_t)c->index_kpool * c->index_hd,+ "%slayers.%d.self_attn.indexer.index_kpool_compress_ape", P, i);l->ikpg = load_mat(m, "%slayers.%d.self_attn.indexer.index_kpool_compress_gate", P, i);}} else {@@ -2095,9 +2109,9 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,l->kfa = load_mat(m, "%slayers.%d.self_attn.f_a_proj.weight", P, i);l->kfb = load_mat(m, "%slayers.%d.self_attn.f_b_proj.weight", P, i);l->kb = load_mat(m, "%slayers.%d.self_attn.b_proj.weight", P, i);- l->dt = load_f32(m, "%slayers.%d.self_attn.dt_bias", P, i);- l->alog = load_f32(m, "%slayers.%d.self_attn.A_log", P, i);- l->onorm = load_f32(m, "%slayers.%d.self_attn.o_norm.weight", P, i);+ l->dt = load_f32(m, c->kda_proj, "%slayers.%d.self_attn.dt_bias", P, i);+ l->alog = load_f32(m, c->kda_heads, "%slayers.%d.self_attn.A_log", P, i);+ l->onorm = load_f32(m, c->kda_hd, "%slayers.%d.self_attn.o_norm.weight", P, i);/* Il checkpoint tiene q/k/v conv separate; la ricorrenza le vuole* concatenate nello stesso ordine di qkv. */{@@ -2105,7 +2119,7 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,float *conv = malloc((size_t)3 * width * sizeof(float));const char *parts[3] = { "q_conv1d", "k_conv1d", "v_conv1d" };for (int p = 0; p < 3; p++) {- const float *piece = load_f32(m, "%slayers.%d.self_attn.%s.weight",+ const float *piece = load_f32(m, width, "%slayers.%d.self_attn.%s.weight",P, i, parts[p]);memcpy(conv + (size_t)p * width, piece, (size_t)width * sizeof(float));free((void *)piece);@@ -2118,10 +2132,10 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,l->du = load_mat(m, "%slayers.%d.mlp.up_proj.weight", P, i);l->dd = load_mat(m, "%slayers.%d.mlp.down_proj.weight", P, i);} else {- l->router = load_f32(m, "%slayers.%d.mlp.gate.weight", P, i);+ l->router = load_f32(m, c->n_experts * D, "%slayers.%d.mlp.gate.weight", P, i);l->rbias = st_find(&m->S, (snprintf(probe, sizeof(probe),"%slayers.%d.mlp.gate.e_score_correction_bias", P, i), probe))- ? load_f32(m, "%s", probe) : NULL;+ ? load_f32(m, c->n_experts, "%s", probe) : NULL;l->rg = load_mat(m, "%slayers.%d.mlp.shared_experts.gate_proj.weight", P, i);l->ru = load_mat(m, "%slayers.%d.mlp.shared_experts.up_proj.weight", P, i);l->rd = load_mat(m, "%slayers.%d.mlp.shared_experts.down_proj.weight", P, i);@@ -2197,36 +2211,41 @@ static void vision_load(GModel *m) {.eps = c->vis_eps, .swiglu_limit = c->vis_swiglu_limit,.rope_theta = 10000.0f,};- m->vision.patch_w = load_f32(m, "%spatch_embed.proj.weight", V);- m->vision.patch_b = load_f32(m, "%spatch_embed.proj.bias", V);- m->vision.post_norm = load_f32(m, "%spost_layernorm.weight", V);- m->vision.down_w = load_f32(m, "%sdownsample.weight", V);- m->vision.down_b = load_f32(m, "%sdownsample.bias", V);- m->vision.merger_proj = load_f32(m, "%smerger.proj.weight", V);- m->vision.merger_norm_w = load_f32(m, "%smerger.post_projection_norm.weight", V);- m->vision.merger_norm_b = load_f32(m, "%smerger.post_projection_norm.bias", V);- m->vision.merger_gate = load_f32(m, "%smerger.gate_proj.weight", V);- m->vision.merger_up = load_f32(m, "%smerger.up_proj.weight", V);- m->vision.merger_down = load_f32(m, "%smerger.down_proj.weight", V);+ /* Le forme sono quelle scritte accanto ai campi in vision_tower.h. */+ const ColiVisionConfig *vc = &m->vision.config;+ const int64_t hidden = vc->hidden, inter = vc->intermediate, out = vc->out_hidden;+ const int64_t proj = vc->proj_intermediate, merge = vc->merge;+ const int64_t patch = (int64_t)vc->in_channels * vc->temporal * vc->patch * vc->patch;+ m->vision.patch_w = load_f32(m, hidden * patch, "%spatch_embed.proj.weight", V);+ m->vision.patch_b = load_f32(m, hidden, "%spatch_embed.proj.bias", V);+ m->vision.post_norm = load_f32(m, hidden, "%spost_layernorm.weight", V);+ m->vision.down_w = load_f32(m, out * hidden * merge * merge, "%sdownsample.weight", V);+ m->vision.down_b = load_f32(m, out, "%sdownsample.bias", V);+ m->vision.merger_proj = load_f32(m, out * out, "%smerger.proj.weight", V);+ m->vision.merger_norm_w = load_f32(m, out, "%smerger.post_projection_norm.weight", V);+ m->vision.merger_norm_b = load_f32(m, out, "%smerger.post_projection_norm.bias", V);+ m->vision.merger_gate = load_f32(m, proj * out, "%smerger.gate_proj.weight", V);+ m->vision.merger_up = load_f32(m, proj * out, "%smerger.up_proj.weight", V);+ m->vision.merger_down = load_f32(m, out * proj, "%smerger.down_proj.weight", V);m->vblocks = calloc((size_t)c->vis_layers, sizeof(*m->vblocks));if (!m->vblocks) { fprintf(stderr, "OOM allocating vision blocks\n"); exit(1); }for (int b = 0; b < c->vis_layers; b++) {ColiVisionBlock *vb = &m->vblocks[b];- vb->norm1 = load_f32(m, "%sblocks.%d.norm1.weight", V, b);- vb->norm2 = load_f32(m, "%sblocks.%d.norm2.weight", V, b);- vb->qkv_w = load_f32(m, "%sblocks.%d.attn.qkv.weight", V, b);- vb->qkv_b = load_f32(m, "%sblocks.%d.attn.qkv.bias", V, b);- vb->q_norm = load_f32(m, "%sblocks.%d.attn.q_norm.weight", V, b);- vb->k_norm = load_f32(m, "%sblocks.%d.attn.k_norm.weight", V, b);- vb->proj_w = load_f32(m, "%sblocks.%d.attn.proj.weight", V, b);- vb->proj_b = load_f32(m, "%sblocks.%d.attn.proj.bias", V, b);- vb->gate_w = load_f32(m, "%sblocks.%d.mlp.gate_proj.weight", V, b);- vb->gate_b = load_f32(m, "%sblocks.%d.mlp.gate_proj.bias", V, b);- vb->up_w = load_f32(m, "%sblocks.%d.mlp.up_proj.weight", V, b);- vb->up_b = load_f32(m, "%sblocks.%d.mlp.up_proj.bias", V, b);- vb->down_w = load_f32(m, "%sblocks.%d.mlp.down_proj.weight", V, b);- vb->down_b = load_f32(m, "%sblocks.%d.mlp.down_proj.bias", V, b);+ vb->norm1 = load_f32(m, hidden, "%sblocks.%d.norm1.weight", V, b);+ vb->norm2 = load_f32(m, hidden, "%sblocks.%d.norm2.weight", V, b);+ vb->qkv_w = load_f32(m, 3 * hidden * hidden, "%sblocks.%d.attn.qkv.weight", V, b);+ vb->qkv_b = load_f32(m, 3 * hidden, "%sblocks.%d.attn.qkv.bias", V, b);+ vb->q_norm = load_f32(m, vc->head_dim, "%sblocks.%d.attn.q_norm.weight", V, b);+ vb->k_norm = load_f32(m, vc->head_dim, "%sblocks.%d.attn.k_norm.weight", V, b);+ vb->proj_w = load_f32(m, hidden * hidden, "%sblocks.%d.attn.proj.weight", V, b);+ vb->proj_b = load_f32(m, hidden, "%sblocks.%d.attn.proj.bias", V, b);+ vb->gate_w = load_f32(m, inter * hidden, "%sblocks.%d.mlp.gate_proj.weight", V, b);+ vb->gate_b = load_f32(m, inter, "%sblocks.%d.mlp.gate_proj.bias", V, b);+ vb->up_w = load_f32(m, inter * hidden, "%sblocks.%d.mlp.up_proj.weight", V, b);+ vb->up_b = load_f32(m, inter, "%sblocks.%d.mlp.up_proj.bias", V, b);+ vb->down_w = load_f32(m, hidden * inter, "%sblocks.%d.mlp.down_proj.weight", V, b);+ vb->down_b = load_f32(m, hidden, "%sblocks.%d.mlp.down_proj.bias", V, b);}m->vision.blocks = m->vblocks;m->has_vision = 1;diff --git a/c/tests/safetensors_edit.py b/c/tests/safetensors_edit.pyindex 67ccc3d5..40e6e39e 100644--- a/c/tests/safetensors_edit.py+++ b/c/tests/safetensors_edit.py@@ -28,6 +28,10 @@ def _shard_of(directory, name):raise KeyError(f"{name} is in no shard of {directory}")+def shape_of(directory, name):+ return list(_shard_of(directory, name)[1][name]["shape"])++def copy_fixture(source, destination):shutil.copytree(source, destination)return Path(destination)@@ -57,3 +61,29 @@ def add_f32(directory, beside, name, count, value=0x41414141):encoded += b" " * (-len(encoded) % 8)shard.write_bytes(struct.pack("<Q", len(encoded)) + encoded + data+ struct.pack("<I", value) * count)+++def shrink(directory, name, count):+ """Keep the first `count` elements of tensor `name`, declared as a flat vector.++ The header stays honest: shape and offsets agree, and the tensors stored after+ it move up so the data section has no hole. Only the length is hostile.+ """+ shard, header, start = _shard_of(directory, name)+ data = shard.read_bytes()[start:]+ width = len(_NAN[header[name]["dtype"]])+ tensors = sorted((entry["data_offsets"][0], key) for key, entry in header.items()+ if key != "__metadata__")+ body, at = [], 0+ for _, key in tensors:+ entry = header[key]+ first, last = entry["data_offsets"]+ if key == name:+ last = first + width * count+ entry["shape"] = [count]+ body.append(data[first:last])+ entry["data_offsets"] = [at, at + last - first]+ at += last - first+ encoded = json.dumps(header, separators=(",", ":")).encode()+ encoded += b" " * (-len(encoded) % 8)+ shard.write_bytes(struct.pack("<Q", len(encoded)) + encoded + b"".join(body))diff --git a/c/tests/test_glm53_hostile_weights.py b/c/tests/test_glm53_hostile_weights.pyindex 8d854377..2befe84e 100644--- a/c/tests/test_glm53_hostile_weights.py+++ b/c/tests/test_glm53_hostile_weights.py@@ -1,15 +1,21 @@-"""glm53 on a container whose router weights were chosen by someone else.+"""glm53 on a container whose weights were chosen by someone else.An all-NaN router row: NaN > best is false for every expert, so the top-k pickstayed at -1, and the gate weight was read as score[-1] before the id reached theexpert cache. GHSA-5xpg-vw35-2687 fixed this shape in inkling, kimi_k3 and olmoethrough rt_router_pick() in route_trace.h, which glm53 already includes.+A short f32 tensor: load_f32() sized the buffer by the numel the header declares,+while the forward pass reads norms, router, mHC and vision weights by the sizes in+config.json. One element short was a heap read past the buffer.+Runs in the GLM-5.3 oracle job of check.yml, on the streaming fixture it builds(the same one test_glm53_context_exceeded uses):COLI_GLM53_FIXTURE=/tmp/glm53_stream-i4 python -m unittest -v tests.test_glm53_hostile_weights"""+import json+import mathimport osimport subprocessimport sys@@ -18,7 +24,7 @@ import unittestfrom pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parent))-from safetensors_edit import copy_fixture, fill_nan # noqa: E402+from safetensors_edit import copy_fixture, fill_nan, shape_of, shrink # noqa: E402HERE = Path(__file__).resolve().parent.parentBINARY = next((path for path in (HERE / "glm53.exe", HERE / "glm53") if path.exists()), None)@@ -26,6 +32,47 @@ FIXTURE = Path(os.environ.get("COLI_GLM53_FIXTURE", ""))ROUTER = "model.language_model.layers.3.mlp.gate.weight" # the fixture's one MoE layerSANITIZER = ("ERROR: AddressSanitizer", "runtime error:")+# Every tensor glm53 loads through load_f32(), once per sizing rule: layer 0 is a+# KDA layer, layer 3 the MLA + indexer + MoE one, block 0 stands for every vision+# block. The q/k/v conv pieces are copied out by a memcpy of the configured width.+F32_TENSORS = ["model.language_model." + name for name in (+ "embed_tokens.weight", "norm.weight",+ "layers.0.input_layernorm.weight", "layers.0.post_attention_layernorm.weight",+ "layers.0.hc_attn_fn", "layers.0.hc_attn_base", "layers.0.hc_attn_scale",+ "layers.0.hc_ffn_fn", "layers.0.hc_ffn_base", "layers.0.hc_ffn_scale",+ "layers.0.self_attn.dt_bias", "layers.0.self_attn.A_log",+ "layers.0.self_attn.o_norm.weight", "layers.0.self_attn.q_conv1d.weight",+ "layers.0.self_attn.k_conv1d.weight", "layers.0.self_attn.v_conv1d.weight",+ "layers.3.self_attn.q_a_layernorm.weight", "layers.3.self_attn.kv_a_layernorm.weight",+ "layers.3.self_attn.indexer.k_norm.weight", "layers.3.self_attn.indexer.k_norm.bias",+ "layers.3.self_attn.indexer.index_kpool_compress_ape",+ "layers.3.mlp.gate.weight", "layers.3.mlp.gate.e_score_correction_bias",+)] + ["model.visual." + name for name in (+ "patch_embed.proj.weight", "patch_embed.proj.bias", "post_layernorm.weight",+ "downsample.weight", "downsample.bias", "merger.proj.weight",+ "merger.post_projection_norm.weight", "merger.post_projection_norm.bias",+ "merger.gate_proj.weight", "merger.up_proj.weight", "merger.down_proj.weight",+ "blocks.0.norm1.weight", "blocks.0.norm2.weight",+ "blocks.0.attn.qkv.weight", "blocks.0.attn.qkv.bias",+ "blocks.0.attn.q_norm.weight", "blocks.0.attn.k_norm.weight",+ "blocks.0.attn.proj.weight", "blocks.0.attn.proj.bias",+ "blocks.0.mlp.gate_proj.weight", "blocks.0.mlp.gate_proj.bias",+ "blocks.0.mlp.up_proj.weight", "blocks.0.mlp.up_proj.bias",+ "blocks.0.mlp.down_proj.weight", "blocks.0.mlp.down_proj.bias",+)]+++def image_run(snapshot):+ """The fixture's own prompt with its image: every text and vision weight is read."""+ reference = json.loads((FIXTURE / "ref.json").read_text())+ grid_h, grid_w = reference["grid"]+ command = [str(BINARY), "--model", str(snapshot),+ "--ids", ",".join(str(token) for token in reference["prompt"]),+ "--patches", str(snapshot / "patches.f32"), "--grid", f"{grid_h}x{grid_w}",+ "--greedy", "2"]+ return subprocess.run(command, capture_output=True, text=True, errors="replace",+ timeout=300, env={**os.environ, "GLM53_BITS": "32"})+def serve_turn(snapshot, prompt=b"abc"):"""One SERVE turn; return (the line that ends it, the engine's stderr)."""@@ -81,5 +128,30 @@ class Glm53HostileWeightsTest(unittest.TestCase):self.assertIn("[router] non-finite logits", stderr)[email protected](BINARY, "glm53 engine is not built")[email protected](FIXTURE.name and (FIXTURE / "patches.f32").is_file(),+ "COLI_GLM53_FIXTURE not set to a GLM-5.3-Flash container with an image")+class Glm53ShortTensorTest(unittest.TestCase):+ def test_the_intact_fixture_passes_the_same_checks(self):+ with tempfile.TemporaryDirectory() as scratch:+ result = image_run(copy_fixture(FIXTURE, Path(scratch) / "model"))+ for marker in SANITIZER:+ self.assertNotIn(marker, result.stderr, result.stderr[-4000:])+ self.assertEqual(result.returncode, 0, result.stderr[-4000:])+ self.assertIn("greedy", result.stdout)++ def test_an_f32_tensor_one_element_short_is_refused_at_load(self):+ for name in F32_TENSORS:+ with self.subTest(name), tempfile.TemporaryDirectory() as scratch:+ snapshot = copy_fixture(FIXTURE, Path(scratch) / "model")+ shrink(snapshot, name, math.prod(shape_of(snapshot, name)) - 1)+ result = image_run(snapshot)+ for marker in SANITIZER:+ self.assertNotIn(marker, result.stderr, result.stderr[-4000:])+ self.assertNotEqual(result.returncode, 0, result.stdout)+ self.assertIn(f"{name}: ", result.stderr)+ self.assertIn("the config needs", result.stderr)++if __name__ == "__main__":unittest.main()