Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
security(olmoe): hold every dense tensor to the shape config.json gives
Marcello Costagliola507ef188f0aa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 40f4c0ec..751529e3 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -883,6 +883,11 @@ jobs:SNAP=olmoe_tiny_c ./olmoe 8 8 olmoe_tiny/ref_olmoe.json | tee oracle.log# the engine prints the score and exits 0 either way: the gate is heregrep -qE 'Matching tokens: ([0-9]+)/\1$' oracle.log+ - name: "Dense tensors: the shape config.json gives, or the load refuses"+ run: |+ cd c+ COLI_OLMOE_FIXTURE=olmoe_tiny_c COLI_OLMOE_REFERENCE=olmoe_tiny/ref_olmoe.json \+ python3 -m unittest -v tests.test_olmoe_dense_shape- name: "Cache cap: the sentinel takes the budget, an explicit cap is left alone"run: |cd cdiff --git a/c/olmoe.c b/c/olmoe.cindex ebd5a6f8..1d8f2b0d 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -535,10 +535,29 @@ static void load_cfg(Cfg *c, const char *snap) {free(buf); free(arena);}-static float *load_t(Model *m, const char *name) {- int64_t n = st_numel(&m->S, name);- if (n < 0) { fprintf(stderr, "missing %s\n", name); exit(1); }- float *p = falloc(n);+/* rows x columns is the shape the forward uses the tensor with, counted from+ * the config: q/k/v/o are [hidden, hidden], the router [num_experts, hidden],+ * embed_tokens and lm_head [vocab_size, hidden], a norm [hidden] (columns 0).+ * Allocated from the header's element count instead, a tensor shorter than+ * that was read past its end, and one of another shape was used as if it had+ * this one. Exact, like the experts in load_expert_merged(). */+static float *load_t(Model *m, const char *name, int64_t rows, int64_t columns) {+ st_tensor *t = st_find(&m->S, name);+ if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }+ if (t->rank != (columns ? 2 : 1) || t->shape[0] != rows || (columns && t->shape[1] != columns)) {+ char got[192] = "[";+ for (int k = 0; k < t->rank; k++)+ snprintf(got + strlen(got), sizeof(got) - strlen(got), "%s%lld", k ? ", " : "",+ (long long)t->shape[k]);+ strncat(got, "]", sizeof(got) - strlen(got) - 1);+ char want[64];+ if (columns) snprintf(want, sizeof(want), "[%lld, %lld]", (long long)rows, (long long)columns);+ else snprintf(want, sizeof(want), "[%lld]", (long long)rows);+ fprintf(stderr, "%s: shape %s, expected %s from config.json, refusing (untrusted container)\n",+ name, got, want);+ exit(1);+ }+ float *p = falloc(t->numel);st_read_f32(&m->S, name, p, 0); /* densa: niente DONTNEED, resta residente */return p;}@@ -560,21 +579,23 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,}double t0 = now_s();if (load_boundaries) {- m->embed = load_t(m, "model.embed_tokens.weight");- m->lm_head = load_t(m, "lm_head.weight");- m->final_norm = load_t(m, "model.norm.weight");+ m->embed = load_t(m, "model.embed_tokens.weight", c->vocab, c->hidden);+ m->lm_head = load_t(m, "lm_head.weight", c->vocab, c->hidden);+ m->final_norm = load_t(m, "model.norm.weight", c->hidden, 0);}m->L = calloc(c->n_layers, sizeof(Layer));char nm[256];+ const int64_t D = c->hidden;for (int i = layer_begin; i < layer_end; i++) {Layer *l = &m->L[i];- #define LD(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm)- LD(in_ln, "input_layernorm.weight");- LD(post_ln,"post_attention_layernorm.weight");- LD(q, "self_attn.q_proj.weight"); LD(k, "self_attn.k_proj.weight");- LD(v, "self_attn.v_proj.weight"); LD(o, "self_attn.o_proj.weight");- LD(qn,"self_attn.q_norm.weight"); LD(kn,"self_attn.k_norm.weight");- LD(gate, "mlp.gate.weight");+ #define LD(field, suffix, rows, columns) \+ snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm,rows,columns)+ LD(in_ln, "input_layernorm.weight", D, 0);+ LD(post_ln,"post_attention_layernorm.weight", D, 0);+ LD(q, "self_attn.q_proj.weight", D, D); LD(k, "self_attn.k_proj.weight", D, D);+ LD(v, "self_attn.v_proj.weight", D, D); LD(o, "self_attn.o_proj.weight", D, D);+ LD(qn,"self_attn.q_norm.weight", D, 0); LD(kn,"self_attn.k_norm.weight", D, 0);+ LD(gate, "mlp.gate.weight", c->n_experts, D);#undef LD}/* cap <= 0 is "you decide", the sentinel the launcher sends when nobody@@ -2696,9 +2717,10 @@ static int olmoe_edge_engine_open("out of memory opening OLMoE Edge");load_cfg(&engine->model.c, options->model_dir);st_init(&engine->model.S, options->model_dir);- engine->model.embed = load_t(&engine->model, "model.embed_tokens.weight");- engine->model.lm_head = load_t(&engine->model, "lm_head.weight");- engine->model.final_norm = load_t(&engine->model, "model.norm.weight");+ const Cfg *ec = &engine->model.c;+ engine->model.embed = load_t(&engine->model, "model.embed_tokens.weight", ec->vocab, ec->hidden);+ engine->model.lm_head = load_t(&engine->model, "lm_head.weight", ec->vocab, ec->hidden);+ engine->model.final_norm = load_t(&engine->model, "model.norm.weight", ec->hidden, 0);char tokenizer_path[4096];snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",options->model_dir);diff --git a/c/tests/test_olmoe_dense_shape.py b/c/tests/test_olmoe_dense_shape.pynew file mode 100644index 00000000..22e69b2c--- /dev/null+++ b/c/tests/test_olmoe_dense_shape.py@@ -0,0 +1,164 @@+"""OLMoE checks every dense tensor's shape against config.json at load.++load_t() allocated each dense tensor from the element count in the+safetensors header, and the forward then used it with the config's+dimensions: q/k/v/o as [hidden, hidden], the router as [num_experts, hidden],+embed_tokens and lm_head as [vocab_size, hidden], every norm as [hidden].+Nothing compared the two. A tensor shorter than its config shape was read+past its end (a key projection written for grouped-query attention, which+this engine does not run, is one), and one of another shape with enough+elements (more rows, a transposed router) was used as if it had the right+one. The routed experts are already held to an exact size in+load_expert_merged(); this holds the dense tensors to the same rule.++Each test copies the tiny OLMoE container, rewrites ONE dense tensor with+another shape and runs the real engine on the container's own reference.+Needs a converted tiny OLMoE container and a built engine, the ones the OLMoE+tiny oracle job builds.+"""+import json+import os+import shutil+import struct+import subprocess+import tempfile+import unittest+from math import prod+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("olmoe.exe" if os.name == "nt" else "olmoe")+FIXTURE = Path(os.environ.get("COLI_OLMOE_FIXTURE", ""))+REFERENCE = Path(os.environ.get("COLI_OLMOE_REFERENCE", ""))+ELEMENT = {"F32": 4, "F16": 2, "BF16": 2}+++def reshape(container, name, shape):+ """Rewrite `name` in its shard with `shape`, its bytes cut or repeated to fit."""+ for shard in sorted(container.glob("*.safetensors")):+ raw = shard.read_bytes()+ size = struct.unpack("<Q", raw[:8])[0]+ header = json.loads(raw[8:8 + size])+ if name not in header:+ continue+ data = raw[8 + size:]+ rebuilt, blob = {}, bytearray()+ for key, entry in sorted(((k, v) for k, v in header.items() if k != "__metadata__"),+ key=lambda item: item[1]["data_offsets"][0]):+ begin, end = entry["data_offsets"]+ payload = data[begin:end]+ if key == name:+ want = prod(shape) * ELEMENT[entry["dtype"]]+ payload = (payload * (want // len(payload) + 1))[:want]+ entry = {**entry, "shape": list(shape)}+ rebuilt[key] = {**entry, "data_offsets": [len(blob), len(blob) + len(payload)]}+ blob += payload+ if "__metadata__" in header:+ rebuilt["__metadata__"] = header["__metadata__"]+ text = json.dumps(rebuilt).encode()+ text += b" " * (-len(text) % 8)+ shard.write_bytes(struct.pack("<Q", len(text)) + text + bytes(blob))+ return+ raise AssertionError(f"{name} is not in {container}")+++def run(changes):+ with tempfile.TemporaryDirectory() as tmp:+ container = Path(tmp) / "olmoe"+ shutil.copytree(FIXTURE, container)+ for name, shape in changes:+ reshape(container, name, shape)+ environment = {**os.environ, "SNAP": str(container)}+ return subprocess.run([str(ENGINE), "8", "8", str(REFERENCE)], env=environment,+ capture_output=True, text=True, errors="replace",+ timeout=300)++[email protected](ENGINE.exists(), "olmoe engine is not built")[email protected](FIXTURE.name and (FIXTURE / "config.json").is_file()+ and REFERENCE.name and REFERENCE.is_file(),+ "COLI_OLMOE_FIXTURE / COLI_OLMOE_REFERENCE not set to a converted "+ "OLMoE container and its reference")+class OlmoeDenseShapeTest(unittest.TestCase):++ @classmethod+ def setUpClass(cls):+ config = json.loads((FIXTURE / "config.json").read_text())+ cls.hidden, cls.vocab = config["hidden_size"], config["vocab_size"]+ cls.experts = config["num_experts"]++ def assertRefused(self, result, name, shape, expected):+ message = f"{name}: shape {list(shape)}, expected {list(expected)} from config.json"+ self.assertEqual(result.returncode, 1, result.stderr[-2000:])+ self.assertIn(message, result.stderr, result.stderr[-2000:])+ self.assertNotIn("Matching tokens", result.stdout)++ def refused(self, name, shape, expected):+ self.assertRefused(run([(name, shape)]), name, shape, expected)++ def test_the_container_as_converted_runs_token_exact(self):+ result = run([])+ self.assertEqual(result.returncode, 0, result.stderr[-2000:])+ self.assertRegex(result.stdout, r"Matching tokens: (\d+)/\1\n")++ def test_a_short_query_projection_is_refused(self):+ D = self.hidden+ self.refused("model.layers.1.self_attn.q_proj.weight", [D // 2, D], [D, D])++ def test_a_grouped_query_key_projection_is_refused(self):+ D = self.hidden+ self.refused("model.layers.0.self_attn.k_proj.weight", [D // 4, D], [D, D])++ def test_a_value_projection_with_extra_rows_is_refused(self):+ D = self.hidden+ self.refused("model.layers.2.self_attn.v_proj.weight", [2 * D, D], [D, D])++ def test_an_output_projection_with_short_columns_is_refused(self):+ D = self.hidden+ self.refused("model.layers.3.self_attn.o_proj.weight", [D, D // 2], [D, D])++ def test_a_query_projection_with_extra_columns_is_refused(self):+ D = self.hidden+ self.refused("model.layers.0.self_attn.q_proj.weight", [D, 2 * D], [D, D])++ def test_a_short_router_is_refused(self):+ D, E = self.hidden, self.experts+ self.refused("model.layers.0.mlp.gate.weight", [E // 2, D], [E, D])++ def test_a_transposed_router_is_refused(self):+ D, E = self.hidden, self.experts+ self.refused("model.layers.1.mlp.gate.weight", [D, E], [E, D])++ def test_a_short_lm_head_is_refused(self):+ D, V = self.hidden, self.vocab+ self.refused("lm_head.weight", [V - 8, D], [V, D])++ def test_a_short_embedding_is_refused(self):+ D, V = self.hidden, self.vocab+ self.refused("model.embed_tokens.weight", [V // 2, D], [V, D])++ def test_short_norms_are_refused(self):+ D = self.hidden+ for name in ("model.norm.weight",+ "model.layers.0.input_layernorm.weight",+ "model.layers.0.post_attention_layernorm.weight",+ "model.layers.0.self_attn.q_norm.weight",+ "model.layers.0.self_attn.k_norm.weight"):+ with self.subTest(name=name):+ self.refused(name, [D // 2], [D])++ def test_a_norm_with_an_extra_axis_is_refused(self):+ D = self.hidden+ self.refused("model.layers.2.input_layernorm.weight", [D, 1], [D])++ def test_a_projection_with_an_extra_axis_is_refused(self):+ D = self.hidden+ self.refused("model.layers.3.self_attn.k_proj.weight", [D, D, 1], [D, D])++ def test_a_flat_projection_is_refused(self):+ D = self.hidden+ self.refused("model.layers.2.self_attn.q_proj.weight", [D * D], [D, D])+++if __name__ == "__main__":+ unittest.main()