Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

security(olmoe): hold every dense tensor to the shape config.json gives

Marcello Costagliola507ef188f0aa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 40f4c0ec..751529e3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -883,6 +883,11 @@ jobs:
SNAP=olmoe_tiny_c ./olmoe 8 8 olmoe_tiny/ref_olmoe.json | tee oracle.log
# the engine prints the score and exits 0 either way: the gate is here
grep -qE 'Matching tokens: ([0-9]+)/\1$' oracle.log
+ - name: "Dense tensors: the shape config.json gives, or the load refuses"
+ run: |
+ cd c
+ COLI_OLMOE_FIXTURE=olmoe_tiny_c COLI_OLMOE_REFERENCE=olmoe_tiny/ref_olmoe.json \
+ python3 -m unittest -v tests.test_olmoe_dense_shape
- name: "Cache cap: the sentinel takes the budget, an explicit cap is left alone"
run: |
cd c
diff --git a/c/olmoe.c b/c/olmoe.c
index ebd5a6f8..1d8f2b0d 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -535,10 +535,29 @@ static void load_cfg(Cfg *c, const char *snap) {
free(buf); free(arena);
}
-static float *load_t(Model *m, const char *name) {
- int64_t n = st_numel(&m->S, name);
- if (n < 0) { fprintf(stderr, "missing %s\n", name); exit(1); }
- float *p = falloc(n);
+/* rows x columns is the shape the forward uses the tensor with, counted from
+ * the config: q/k/v/o are [hidden, hidden], the router [num_experts, hidden],
+ * embed_tokens and lm_head [vocab_size, hidden], a norm [hidden] (columns 0).
+ * Allocated from the header's element count instead, a tensor shorter than
+ * that was read past its end, and one of another shape was used as if it had
+ * this one. Exact, like the experts in load_expert_merged(). */
+static float *load_t(Model *m, const char *name, int64_t rows, int64_t columns) {
+ st_tensor *t = st_find(&m->S, name);
+ if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }
+ if (t->rank != (columns ? 2 : 1) || t->shape[0] != rows || (columns && t->shape[1] != columns)) {
+ char got[192] = "[";
+ for (int k = 0; k < t->rank; k++)
+ snprintf(got + strlen(got), sizeof(got) - strlen(got), "%s%lld", k ? ", " : "",
+ (long long)t->shape[k]);
+ strncat(got, "]", sizeof(got) - strlen(got) - 1);
+ char want[64];
+ if (columns) snprintf(want, sizeof(want), "[%lld, %lld]", (long long)rows, (long long)columns);
+ else snprintf(want, sizeof(want), "[%lld]", (long long)rows);
+ fprintf(stderr, "%s: shape %s, expected %s from config.json, refusing (untrusted container)\n",
+ name, got, want);
+ exit(1);
+ }
+ float *p = falloc(t->numel);
st_read_f32(&m->S, name, p, 0); /* densa: niente DONTNEED, resta residente */
return p;
}
@@ -560,21 +579,23 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
}
double t0 = now_s();
if (load_boundaries) {
- m->embed = load_t(m, "model.embed_tokens.weight");
- m->lm_head = load_t(m, "lm_head.weight");
- m->final_norm = load_t(m, "model.norm.weight");
+ m->embed = load_t(m, "model.embed_tokens.weight", c->vocab, c->hidden);
+ m->lm_head = load_t(m, "lm_head.weight", c->vocab, c->hidden);
+ m->final_norm = load_t(m, "model.norm.weight", c->hidden, 0);
}
m->L = calloc(c->n_layers, sizeof(Layer));
char nm[256];
+ const int64_t D = c->hidden;
for (int i = layer_begin; i < layer_end; i++) {
Layer *l = &m->L[i];
- #define LD(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm)
- LD(in_ln, "input_layernorm.weight");
- LD(post_ln,"post_attention_layernorm.weight");
- LD(q, "self_attn.q_proj.weight"); LD(k, "self_attn.k_proj.weight");
- LD(v, "self_attn.v_proj.weight"); LD(o, "self_attn.o_proj.weight");
- LD(qn,"self_attn.q_norm.weight"); LD(kn,"self_attn.k_norm.weight");
- LD(gate, "mlp.gate.weight");
+ #define LD(field, suffix, rows, columns) \
+ snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm,rows,columns)
+ LD(in_ln, "input_layernorm.weight", D, 0);
+ LD(post_ln,"post_attention_layernorm.weight", D, 0);
+ LD(q, "self_attn.q_proj.weight", D, D); LD(k, "self_attn.k_proj.weight", D, D);
+ LD(v, "self_attn.v_proj.weight", D, D); LD(o, "self_attn.o_proj.weight", D, D);
+ LD(qn,"self_attn.q_norm.weight", D, 0); LD(kn,"self_attn.k_norm.weight", D, 0);
+ LD(gate, "mlp.gate.weight", c->n_experts, D);
#undef LD
}
/* cap <= 0 is "you decide", the sentinel the launcher sends when nobody
@@ -2696,9 +2717,10 @@ static int olmoe_edge_engine_open(
"out of memory opening OLMoE Edge");
load_cfg(&engine->model.c, options->model_dir);
st_init(&engine->model.S, options->model_dir);
- engine->model.embed = load_t(&engine->model, "model.embed_tokens.weight");
- engine->model.lm_head = load_t(&engine->model, "lm_head.weight");
- engine->model.final_norm = load_t(&engine->model, "model.norm.weight");
+ const Cfg *ec = &engine->model.c;
+ engine->model.embed = load_t(&engine->model, "model.embed_tokens.weight", ec->vocab, ec->hidden);
+ engine->model.lm_head = load_t(&engine->model, "lm_head.weight", ec->vocab, ec->hidden);
+ engine->model.final_norm = load_t(&engine->model, "model.norm.weight", ec->hidden, 0);
char tokenizer_path[4096];
snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",
options->model_dir);
diff --git a/c/tests/test_olmoe_dense_shape.py b/c/tests/test_olmoe_dense_shape.py
new file mode 100644
index 00000000..22e69b2c
--- /dev/null
+++ b/c/tests/test_olmoe_dense_shape.py
@@ -0,0 +1,164 @@
+"""OLMoE checks every dense tensor's shape against config.json at load.
+
+load_t() allocated each dense tensor from the element count in the
+safetensors header, and the forward then used it with the config's
+dimensions: q/k/v/o as [hidden, hidden], the router as [num_experts, hidden],
+embed_tokens and lm_head as [vocab_size, hidden], every norm as [hidden].
+Nothing compared the two. A tensor shorter than its config shape was read
+past its end (a key projection written for grouped-query attention, which
+this engine does not run, is one), and one of another shape with enough
+elements (more rows, a transposed router) was used as if it had the right
+one. The routed experts are already held to an exact size in
+load_expert_merged(); this holds the dense tensors to the same rule.
+
+Each test copies the tiny OLMoE container, rewrites ONE dense tensor with
+another shape and runs the real engine on the container's own reference.
+Needs a converted tiny OLMoE container and a built engine, the ones the OLMoE
+tiny oracle job builds.
+"""
+import json
+import os
+import shutil
+import struct
+import subprocess
+import tempfile
+import unittest
+from math import prod
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+ENGINE = HERE / ("olmoe.exe" if os.name == "nt" else "olmoe")
+FIXTURE = Path(os.environ.get("COLI_OLMOE_FIXTURE", ""))
+REFERENCE = Path(os.environ.get("COLI_OLMOE_REFERENCE", ""))
+ELEMENT = {"F32": 4, "F16": 2, "BF16": 2}
+
+
+def reshape(container, name, shape):
+ """Rewrite `name` in its shard with `shape`, its bytes cut or repeated to fit."""
+ for shard in sorted(container.glob("*.safetensors")):
+ raw = shard.read_bytes()
+ size = struct.unpack("<Q", raw[:8])[0]
+ header = json.loads(raw[8:8 + size])
+ if name not in header:
+ continue
+ data = raw[8 + size:]
+ rebuilt, blob = {}, bytearray()
+ for key, entry in sorted(((k, v) for k, v in header.items() if k != "__metadata__"),
+ key=lambda item: item[1]["data_offsets"][0]):
+ begin, end = entry["data_offsets"]
+ payload = data[begin:end]
+ if key == name:
+ want = prod(shape) * ELEMENT[entry["dtype"]]
+ payload = (payload * (want // len(payload) + 1))[:want]
+ entry = {**entry, "shape": list(shape)}
+ rebuilt[key] = {**entry, "data_offsets": [len(blob), len(blob) + len(payload)]}
+ blob += payload
+ if "__metadata__" in header:
+ rebuilt["__metadata__"] = header["__metadata__"]
+ text = json.dumps(rebuilt).encode()
+ text += b" " * (-len(text) % 8)
+ shard.write_bytes(struct.pack("<Q", len(text)) + text + bytes(blob))
+ return
+ raise AssertionError(f"{name} is not in {container}")
+
+
+def run(changes):
+ with tempfile.TemporaryDirectory() as tmp:
+ container = Path(tmp) / "olmoe"
+ shutil.copytree(FIXTURE, container)
+ for name, shape in changes:
+ reshape(container, name, shape)
+ environment = {**os.environ, "SNAP": str(container)}
+ return subprocess.run([str(ENGINE), "8", "8", str(REFERENCE)], env=environment,
+ capture_output=True, text=True, errors="replace",
+ timeout=300)
+
+
[email protected](ENGINE.exists(), "olmoe engine is not built")
[email protected](FIXTURE.name and (FIXTURE / "config.json").is_file()
+ and REFERENCE.name and REFERENCE.is_file(),
+ "COLI_OLMOE_FIXTURE / COLI_OLMOE_REFERENCE not set to a converted "
+ "OLMoE container and its reference")
+class OlmoeDenseShapeTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ config = json.loads((FIXTURE / "config.json").read_text())
+ cls.hidden, cls.vocab = config["hidden_size"], config["vocab_size"]
+ cls.experts = config["num_experts"]
+
+ def assertRefused(self, result, name, shape, expected):
+ message = f"{name}: shape {list(shape)}, expected {list(expected)} from config.json"
+ self.assertEqual(result.returncode, 1, result.stderr[-2000:])
+ self.assertIn(message, result.stderr, result.stderr[-2000:])
+ self.assertNotIn("Matching tokens", result.stdout)
+
+ def refused(self, name, shape, expected):
+ self.assertRefused(run([(name, shape)]), name, shape, expected)
+
+ def test_the_container_as_converted_runs_token_exact(self):
+ result = run([])
+ self.assertEqual(result.returncode, 0, result.stderr[-2000:])
+ self.assertRegex(result.stdout, r"Matching tokens: (\d+)/\1\n")
+
+ def test_a_short_query_projection_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.1.self_attn.q_proj.weight", [D // 2, D], [D, D])
+
+ def test_a_grouped_query_key_projection_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.0.self_attn.k_proj.weight", [D // 4, D], [D, D])
+
+ def test_a_value_projection_with_extra_rows_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.2.self_attn.v_proj.weight", [2 * D, D], [D, D])
+
+ def test_an_output_projection_with_short_columns_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.3.self_attn.o_proj.weight", [D, D // 2], [D, D])
+
+ def test_a_query_projection_with_extra_columns_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.0.self_attn.q_proj.weight", [D, 2 * D], [D, D])
+
+ def test_a_short_router_is_refused(self):
+ D, E = self.hidden, self.experts
+ self.refused("model.layers.0.mlp.gate.weight", [E // 2, D], [E, D])
+
+ def test_a_transposed_router_is_refused(self):
+ D, E = self.hidden, self.experts
+ self.refused("model.layers.1.mlp.gate.weight", [D, E], [E, D])
+
+ def test_a_short_lm_head_is_refused(self):
+ D, V = self.hidden, self.vocab
+ self.refused("lm_head.weight", [V - 8, D], [V, D])
+
+ def test_a_short_embedding_is_refused(self):
+ D, V = self.hidden, self.vocab
+ self.refused("model.embed_tokens.weight", [V // 2, D], [V, D])
+
+ def test_short_norms_are_refused(self):
+ D = self.hidden
+ for name in ("model.norm.weight",
+ "model.layers.0.input_layernorm.weight",
+ "model.layers.0.post_attention_layernorm.weight",
+ "model.layers.0.self_attn.q_norm.weight",
+ "model.layers.0.self_attn.k_norm.weight"):
+ with self.subTest(name=name):
+ self.refused(name, [D // 2], [D])
+
+ def test_a_norm_with_an_extra_axis_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.2.input_layernorm.weight", [D, 1], [D])
+
+ def test_a_projection_with_an_extra_axis_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.3.self_attn.k_proj.weight", [D, D, 1], [D, D])
+
+ def test_a_flat_projection_is_refused(self):
+ D = self.hidden
+ self.refused("model.layers.2.self_attn.q_proj.weight", [D * D], [D, D])
+
+
+if __name__ == "__main__":
+ unittest.main()