Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix RoPE docs, preflight expert check, run numerical tests in CI with a new job

matteobee8af0ee825
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0d3f41e6..6a054e2a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -784,6 +784,30 @@ jobs:
- name: Python test suite
run: cd c && python3 -m unittest discover -s tests -p 'test_*.py'
+ gguf:
+ name: GGUF reader + converter tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ cache-dependency-path: c/tools/requirements-gguf.txt
+ # This job is what makes the numerical GGUF evidence real: without these
+ # deps those tests skip at import, so the generic `python` job alone only
+ # exercises test_gguf_reader.py (pure stdlib).
+ - name: Install GGUF test dependencies
+ run: pip install -r c/tools/requirements-gguf.txt
+ - name: GGUF reader, dequant, profile and converter tests
+ run: |
+ cd c
+ python3 -m unittest -v \
+ tests.test_gguf_reader \
+ tests.test_gguf_dequant \
+ tests.test_gguf_olmoe_profile \
+ tests.test_convert_gguf_to_olmoe
+
windows-python-focused:
name: Python tests (Windows focused)
runs-on: windows-latest
diff --git a/c/tests/fixtures/olmoe_quantize_row_golden.npz b/c/tests/fixtures/olmoe_quantize_row_golden.npz
new file mode 100644
index 00000000..5059eb19
Binary files /dev/null and b/c/tests/fixtures/olmoe_quantize_row_golden.npz differ
diff --git a/c/tests/test_gguf_olmoe_profile.py b/c/tests/test_gguf_olmoe_profile.py
index d679bb2d..42dd6404 100644
--- a/c/tests/test_gguf_olmoe_profile.py
+++ b/c/tests/test_gguf_olmoe_profile.py
@@ -1,11 +1,14 @@
#!/usr/bin/env python3
"""tools/gguf_olmoe_profile.py: OLMoE name mapping, layout and int8 quantization.
-The strongest check here is test_matches_reference_quantizer: our numpy
-quantize_row is compared against the torch implementation in
-tools/convert_olmoe_merged.py, which is the code the existing OLMoE container
-was built with. If the two disagree, the new GGUF converter would produce a
-container the engine reads differently from the known-good one.
+The strongest check here is the quantizer comparison against
+tools/convert_olmoe_merged.py, the torch code the existing OLMoE container was
+built with. It runs in two forms: test_matches_reference_quantizer_golden uses
+the committed fixture (tests/fixtures/olmoe_quantize_row_golden.npz), generated
+once with that torch reference, so CI needs only numpy; the live variant
+test_matches_reference_quantizer_live re-runs the torch reference when it is
+installed. If the two disagree, the new GGUF converter would produce a container
+the engine reads differently from the known-good one.
"""
import sys
@@ -21,6 +24,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
import gguf_olmoe_profile as profile
+GOLDEN_QUANT = (Path(__file__).resolve().parent / "fixtures"
+ / "olmoe_quantize_row_golden.npz")
+
class MappingTest(unittest.TestCase):
def test_parse_expert(self):
@@ -106,18 +112,41 @@ class QuantizationTest(unittest.TestCase):
with self.assertRaises(ValueError):
profile.quantize_row(np.zeros(8, dtype=np.float32))
- def test_matches_reference_quantizer(self):
+ def test_matches_reference_quantizer_golden(self):
+ """quantize_row pinned to the torch reference via a committed fixture.
+
+ The .npz holds the deterministic inputs plus convert_olmoe_merged.py's
+ int8 weights and scales (seed 20260825, 20 variable-shape cases), so CI
+ runs this check with only numpy installed.
+ """
+ if not GOLDEN_QUANT.is_file():
+ self.skipTest("golden fixture missing: %s" % GOLDEN_QUANT)
+ golden = np.load(str(GOLDEN_QUANT))
+ for i in range(int(golden["count"])):
+ w = golden["case_%02d_w" % i]
+ q_got, s_got = profile.quantize_row(w)
+ np.testing.assert_array_equal(
+ q_got, golden["case_%02d_q" % i], err_msg="case %d int8" % i)
+ np.testing.assert_allclose(
+ s_got, golden["case_%02d_s" % i], rtol=0, atol=0,
+ err_msg="case %d scales" % i)
+
+ def test_matches_reference_quantizer_live(self):
try:
import torch
import convert_olmoe_merged as reference
except ImportError as exc:
self.skipTest("torch/convert_olmoe_merged unavailable: %s" % exc)
+ golden = np.load(str(GOLDEN_QUANT)) if GOLDEN_QUANT.is_file() else None
rng = np.random.default_rng(20260825)
- for _ in range(20):
+ for case in range(20):
rows = int(rng.integers(1, 8))
cols = int(rng.integers(1, 40))
w = rng.normal(0.0, 1.0, size=(rows, cols)).astype(np.float32)
+ if golden is not None:
+ # the committed fixture and the live reference must agree
+ np.testing.assert_array_equal(w, golden["case_%02d_w" % case])
q_ref, s_ref = reference.quantize_row(torch.from_numpy(w))
q_got, s_got = profile.quantize_row(w)
np.testing.assert_array_equal(q_got, q_ref.cpu().numpy())
diff --git a/c/tools/convert_gguf_to_olmoe.py b/c/tools/convert_gguf_to_olmoe.py
index 89cb5782..aec91239 100644
--- a/c/tools/convert_gguf_to_olmoe.py
+++ b/c/tools/convert_gguf_to_olmoe.py
@@ -12,9 +12,9 @@ Output, per tensor, matches what c/olmoe.c loads:
model.layers.N.mlp.experts.E.merged_weight (I8) + .qs (F32);
* config.json reconstructed from the GGUF olmoe.* metadata. GGUF does not
carry norm_topk_prob; the default is false (matching the official OLMoE
- checkpoints) and --norm-topk-prob overrides it. q/k are taken as stored
- (HuggingFace layout); --qk-permuted undoes llama.cpp's RoPE permutation for
- GGUFs that carry it;
+ checkpoints) and --norm-topk-prob overrides it. q/k are taken as stored:
+ the official OLMoE GGUF is already in the HuggingFace layout. --qk-permuted
+ undoes llama.cpp's RoPE permutation for a GGUF that carries it;
* tokenizer.json rebuilt from tokenizer.ggml.* (byte-level GPT-2), validated
token-for-token against the official OLMoE tokenizer, so the container is
usable for chat/serve, not only loadable for validation.
@@ -170,8 +170,9 @@ def convert_dense(reader, dense, writer, config, qk_permuted):
flat = dequantize(raw, tensor["ggml_type"], tensor["numel"])
logical = profile.to_logical(flat, tensor["dims"])
if qk_permuted:
- # Some GGUF producers (recent llama.cpp olmo.py) RoPE-permute q/k;
- # c/olmoe.c uses the plain HuggingFace layout, so undo it.
+ # llama.cpp's dense OLMo converter (OlmoModel, not OlmoeModel)
+ # RoPE-permutes q/k; c/olmoe.c uses the plain HuggingFace layout, so
+ # undo it. The official OLMoE GGUF is not permuted.
name = tensor["name"]
if name.endswith("attn_q.weight"):
logical = profile.restore_rope_layout(logical, config["num_attention_heads"],
@@ -193,10 +194,7 @@ def convert_experts(reader, experts, writer, config):
done_experts = 0
for layer in range(layer_count):
- kinds = experts.get(layer)
- if not kinds or set(kinds) != {"gate", "up", "down"}:
- raise ConversionError("layer %d: incomplete expert trio: %s"
- % (layer, sorted(kinds) if kinds else None))
+ kinds = experts[layer]
raw_bytes, expert_layout = {}, {}
for kind in ("gate", "up", "down"):
tensor = kinds[kind]
@@ -270,6 +268,11 @@ def run(args):
missing = sorted(required_names(config) - present)
if missing:
raise ConversionError("missing required tensors: %s" % ", ".join(missing[:8]))
+ for layer in range(config["num_hidden_layers"]):
+ kinds = experts.get(layer)
+ if not kinds or set(kinds) != {"gate", "up", "down"}:
+ raise ConversionError("layer %d: incomplete expert trio: %s"
+ % (layer, sorted(kinds) if kinds else None))
experts_total = config["num_hidden_layers"] * config["num_experts"]
if args.dry_run:
@@ -371,9 +374,9 @@ def main():
"weights). GGUF does not carry this field; the default is false, "
"which matches the official OLMoE checkpoints")
parser.add_argument("--qk-permuted", action="store_true",
- help="undo llama.cpp's RoPE permutation of q_proj/k_proj. Off by "
- "default because the official OLMoE GGUF is already in the "
- "HuggingFace layout; use it only for GGUFs that were permuted")
+ help="undo llama.cpp's dense-OLMo RoPE permutation of q_proj/k_proj. "
+ "Off by default because the official OLMoE GGUF is already in "
+ "the HuggingFace layout; use it only for GGUFs that were permuted")
parser.add_argument("--flush-every", type=int, default=512, metavar="N",
help="flush an output safetensors shard every N tensors "
"(default: 512; smaller = more, smaller shards)")
diff --git a/c/tools/gguf_olmoe_profile.py b/c/tools/gguf_olmoe_profile.py
index f0a755ba..f9faa960 100644
--- a/c/tools/gguf_olmoe_profile.py
+++ b/c/tools/gguf_olmoe_profile.py
@@ -90,14 +90,21 @@ def to_logical(flat, gguf_dims):
def restore_rope_layout(weights, n_head, n_kv_head=None):
"""Undo llama.cpp's RoPE permutation of q_proj/k_proj weights.
- conversion/olmo.py permutes q_proj and k_proj exactly like Llama:
+ Only llama.cpp's *dense* OLMo converter does this: conversion/olmo.py's
+ OlmoModel (OlmoForCausalLM) permutes q_proj and k_proj exactly like Llama:
forward: w.reshape(n, 2, out // n // 2, *rest).swapaxes(1, 2).reshape(w.shape)
with n = n_kv_head when it differs from n_head (k_proj), else n_head. That
is NOT an involution unless out // n // 2 == 2, so the inverse is the axis
transpose of the (m, 2) decomposition:
inverse: w.reshape(n, m, 2, *rest).swapaxes(1, 2).reshape(w.shape)
- with m = out // n // 2. c/olmoe.c uses HF-style RoPE (adjacent pairs per
- head), so the GGUF's permuted q/k must be restored to the HF layout.
+ with m = out // n // 2.
+
+ The OLMoE converter is a different class, OlmoeModel (OlmoeForCausalLM), and
+ does NOT permute: llama.cpp runs LLM_ARCH_OLMOE with NEOX RoPE -- pairs of
+ head values offset by head_dim/2 -- the same layout as HuggingFace's
+ rotate_half and c/olmoe.c's rope_head. The official OLMoE GGUF is therefore
+ already in the HF layout; this function is only for a GGUF that a permuting
+ converter produced.
"""
n = n_kv_head if (n_kv_head and n_head != n_kv_head) else n_head
w = np.asarray(weights)
diff --git a/c/tools/requirements-gguf.txt b/c/tools/requirements-gguf.txt
new file mode 100644
index 00000000..16d5e9e9
--- /dev/null
+++ b/c/tools/requirements-gguf.txt
@@ -0,0 +1,14 @@
+# Dependencies for the GGUF reader / dequant / OLMoE-profile / converter tests
+# (c/tests/test_gguf_reader.py, test_gguf_dequant.py,
+# test_gguf_olmoe_profile.py, test_convert_gguf_to_olmoe.py). Installed only by
+# the ci.yml `gguf` job; the engine-build and generic Python jobs stay lean.
+#
+# gguf is llama.cpp's gguf-py reference: test_gguf_dequant.py cross-checks our
+# dequantization against gguf.quants.dequantize. The committed golden fixture is
+# the frozen pin, so this is a minimum version that lets upstream drift surface.
+# torch is deliberately absent: the quantizer is pinned by the committed golden
+# fixture (tests/fixtures/olmoe_quantize_row_golden.npz) and the torch-based
+# cross-checks stay optional for local runs.
+numpy>=1.26
+safetensors>=0.4
+gguf>=0.19