Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

added general tool wrapper, gguf dequant and special olmoe conversion file, added tests

matteoc054523eae2c
diff --git a/c/tests/fixtures/gguf_dequant_golden.npz b/c/tests/fixtures/gguf_dequant_golden.npz
new file mode 100644
index 00000000..d45387bc
Binary files /dev/null and b/c/tests/fixtures/gguf_dequant_golden.npz differ
diff --git a/c/tests/test_convert_gguf_to_olmoe.py b/c/tests/test_convert_gguf_to_olmoe.py
new file mode 100644
index 00000000..e434ab69
--- /dev/null
+++ b/c/tests/test_convert_gguf_to_olmoe.py
@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+"""tools/convert_gguf_to_olmoe.py: end-to-end conversion of a tiny synthetic GGUF.
+
+Builds a minimal OLMoE-shaped GGUF (all F16/F32, no K-quants) with the smallest
+dims the engine accepts, runs the real CLI, and asserts the container c/olmoe.c
+expects: config.json keys, tokenizer.json reconstruction, the dense name
+mapping, and per-expert merged_weight (I8) + .qs (F32) at the exact sizes the
+engine requires. Also covers --dry-run (writes nothing, creates no dir) and the
+refusals for a non-OLMoE GGUF / unknown tensor; --remove-source-file removes
+the source only after full success.
+"""
+
+import json
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+TESTDIR = Path(__file__).resolve().parent
+TOOLS = TESTDIR.parent / "tools"
+sys.path.insert(0, str(TOOLS))
+sys.path.insert(0, str(TESTDIR))
+
+from gguf_fixture import ( # noqa: E402 (encoder helpers reused verbatim)
+ METADATA_ARRAY,
+ METADATA_BOOL,
+ METADATA_FLOAT32,
+ METADATA_STRING,
+ METADATA_UINT32,
+ _encode_string,
+ _encode_value,
+)
+
+import convert_gguf_to_olmoe # noqa: E402
+
+# tiny but engine-valid dims
+HIDDEN, INTER, N_EXPERTS, LAYERS, VOCAB, HEADS = 8, 4, 2, 1, 16, 2
+
+
+def _align(value, alignment):
+ return (value + alignment - 1) // alignment * alignment
+
+
+def _f16_bytes(values):
+ return np.asarray(values, dtype=np.float16).tobytes()
+
+
+def _f32_bytes(values):
+ return np.asarray(values, dtype=np.float32).tobytes()
+
+
+def build_tiny_olmoe_gguf(path, arch="olmoe", extra_tensor=None, override=None):
+ rng = np.random.default_rng(20260825)
+ override = override or {}
+
+ def m2d(hf_shape):
+ return list(reversed(hf_shape))
+
+ tensors = []
+ # HF shape -> GGUF dims (reversed). Dense matrices F16, 1-D norms F32.
+ def add_gguf(name, gguf_dims, ggml_type, payload):
+ tensors.append({"name": name, "dims": gguf_dims,
+ "ggml_type": ggml_type, "payload": override.get(name, payload)})
+
+ add_gguf("token_embd.weight", m2d([VOCAB, HIDDEN]), 1,
+ _f16_bytes(rng.normal(size=VOCAB * HIDDEN)))
+ add_gguf("output.weight", m2d([VOCAB, HIDDEN]), 1,
+ _f16_bytes(rng.normal(size=VOCAB * HIDDEN)))
+ add_gguf("output_norm.weight", [HIDDEN], 0, _f32_bytes(rng.normal(size=HIDDEN)))
+ for layer in range(LAYERS):
+ add_gguf("blk.%d.attn_norm.weight" % layer, [HIDDEN], 0,
+ _f32_bytes(rng.normal(size=HIDDEN)))
+ add_gguf("blk.%d.ffn_norm.weight" % layer, [HIDDEN], 0,
+ _f32_bytes(rng.normal(size=HIDDEN)))
+ for w in ("q", "k", "v", "output"):
+ add_gguf("blk.%d.attn_%s.weight" % (layer, w), m2d([HIDDEN, HIDDEN]), 1,
+ _f16_bytes(rng.normal(size=HIDDEN * HIDDEN)))
+ add_gguf("blk.%d.attn_q_norm.weight" % layer, [HIDDEN], 0,
+ _f32_bytes(rng.normal(size=HIDDEN)))
+ add_gguf("blk.%d.attn_k_norm.weight" % layer, [HIDDEN], 0,
+ _f32_bytes(rng.normal(size=HIDDEN)))
+ add_gguf("blk.%d.ffn_gate_inp.weight" % layer, m2d([N_EXPERTS, HIDDEN]), 1,
+ _f16_bytes(rng.normal(size=N_EXPERTS * HIDDEN)))
+ add_gguf("blk.%d.ffn_gate_exps.weight" % layer, m2d([N_EXPERTS, INTER, HIDDEN]), 1,
+ _f16_bytes(rng.normal(size=N_EXPERTS * INTER * HIDDEN)))
+ add_gguf("blk.%d.ffn_up_exps.weight" % layer, m2d([N_EXPERTS, INTER, HIDDEN]), 1,
+ _f16_bytes(rng.normal(size=N_EXPERTS * INTER * HIDDEN)))
+ add_gguf("blk.%d.ffn_down_exps.weight" % layer, m2d([N_EXPERTS, HIDDEN, INTER]), 1,
+ _f16_bytes(rng.normal(size=N_EXPERTS * INTER * HIDDEN)))
+ if extra_tensor:
+ tensors.append(extra_tensor)
+
+ metadata = [
+ (METADATA_STRING, "general.architecture", arch),
+ (METADATA_UINT32, "general.alignment", 32),
+ (METADATA_UINT32, "olmoe.embedding_length", HIDDEN),
+ (METADATA_UINT32, "olmoe.block_count", LAYERS),
+ (METADATA_UINT32, "olmoe.attention.head_count", HEADS),
+ (METADATA_UINT32, "olmoe.attention.head_count_kv", HEADS),
+ (METADATA_UINT32, "olmoe.expert_count", N_EXPERTS),
+ (METADATA_UINT32, "olmoe.expert_used_count", 2),
+ (METADATA_UINT32, "olmoe.feed_forward_length", INTER),
+ (METADATA_FLOAT32, "olmoe.rope.freq_base", 10000.0),
+ (METADATA_FLOAT32, "olmoe.attention.layer_norm_rms_epsilon", 1e-5),
+ (METADATA_ARRAY, "tokenizer.ggml.tokens",
+ (METADATA_STRING, ["aa", "bb", "<unk>", "Ġt"])),
+ (METADATA_ARRAY, "tokenizer.ggml.token_type",
+ (METADATA_UINT32, [1, 1, 3, 4])),
+ (METADATA_ARRAY, "tokenizer.ggml.merges",
+ (METADATA_STRING, ["Ġ t"])),
+ (METADATA_UINT32, "tokenizer.ggml.bos_token_id", 2),
+ (METADATA_UINT32, "tokenizer.ggml.eos_token_id", 2),
+ (METADATA_BOOL, "tokenizer.ggml.add_bos_token", False),
+ (METADATA_BOOL, "tokenizer.ggml.add_eos_token", False),
+ ]
+ return write_gguf(path, metadata, tensors)
+
+
+def write_gguf(path, metadata, tensors):
+ alignment = 32
+ out = b"GGUF"
+ out += struct.pack("<I", 3)
+ out += struct.pack("<Q", len(tensors))
+ out += struct.pack("<Q", len(metadata))
+ for value_type, key, value in metadata:
+ out += _encode_string(key)
+ out += struct.pack("<I", value_type)
+ out += _encode_value(value_type, value)
+
+ data_offset = 0
+ data_base = _align(len(out) + sum(
+ 8 + len(t["name"].encode("utf-8")) + 4 + 8 * len(t["dims"]) + 4 + 8
+ for t in tensors), alignment)
+ for tensor in tensors:
+ out += _encode_string(tensor["name"])
+ out += struct.pack("<I", len(tensor["dims"]))
+ for dim in tensor["dims"]:
+ out += struct.pack("<Q", dim)
+ out += struct.pack("<I", tensor["ggml_type"])
+ out += struct.pack("<Q", data_offset)
+ data_offset += len(tensor["payload"])
+ data_offset = _align(data_offset, alignment)
+
+ out += b"\x00" * (data_base - len(out))
+ for tensor in tensors:
+ out += tensor["payload"]
+ out += b"\x00" * (_align(len(tensor["payload"]), alignment) - len(tensor["payload"]))
+ Path(path).write_bytes(out)
+
+
+def run_converter(input_path, output_path, *extra):
+ return subprocess.run(
+ [sys.executable, str(TOOLS / "convert_gguf_to_olmoe.py"),
+ "--input", str(input_path), "--output", str(output_path), *extra],
+ cwd=str(TOOLS.parent), capture_output=True, text=True,
+ timeout=120)
+
+
+def _rope_permute(w, n_head, n_kv_head=None):
+ """llama.cpp's q/k RoPE permutation (conversion/llama.py), for the fixture."""
+ n = n_kv_head if (n_kv_head and n_head != n_kv_head) else n_head
+ return np.ascontiguousarray(
+ w.reshape((n, 2, w.shape[0] // n // 2) + w.shape[1:]).swapaxes(1, 2).reshape(w.shape))
+
+
+def expected_names():
+ names = {"model.embed_tokens.weight", "lm_head.weight", "model.norm.weight"}
+ for layer in range(LAYERS):
+ base = "model.layers.%d" % layer
+ names.update({
+ base + ".input_layernorm.weight",
+ base + ".post_attention_layernorm.weight",
+ base + ".self_attn.q_proj.weight",
+ base + ".self_attn.k_proj.weight",
+ base + ".self_attn.v_proj.weight",
+ base + ".self_attn.o_proj.weight",
+ base + ".self_attn.q_norm.weight",
+ base + ".self_attn.k_norm.weight",
+ base + ".mlp.gate.weight",
+ })
+ for layer in range(LAYERS):
+ for expert in range(N_EXPERTS):
+ names.add(base_merge_name(layer, expert))
+ names.add("model.layers.%d.mlp.experts.%d.qs" % (layer, expert))
+ return names
+
+
+def base_merge_name(layer, expert):
+ return "model.layers.%d.mlp.experts.%d.merged_weight" % (layer, expert)
+
+
+class ConverterEndToEndTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.dir = Path(self.tmp.name)
+ self.gguf = self.dir / "model.gguf"
+ self.out = self.dir / "out"
+ build_tiny_olmoe_gguf(str(self.gguf))
+
+ def test_converts_full_container(self):
+ result = run_converter(self.gguf, self.out)
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ config = json.loads((self.out / "config.json").read_text())
+ self.assertEqual(config["hidden_size"], HIDDEN)
+ self.assertEqual(config["num_experts"], N_EXPERTS)
+ self.assertEqual(config["vocab_size"], 4)
+ self.assertFalse(config["norm_topk_prob"])
+
+ tokenizer = json.loads((self.out / "tokenizer.json").read_text())
+ self.assertEqual(tokenizer["model"]["vocab"], {"aa": 0, "bb": 1, "<unk>": 2, "Ġt": 3})
+
+ present = set()
+ from safetensors import safe_open
+ for shard in sorted(self.out.glob("model-*.safetensors")):
+ with safe_open(str(shard), framework="np") as handle:
+ present.update(handle.keys())
+
+ self.assertEqual(present, expected_names())
+
+ with safe_open(str(next(self.out.glob("model-*.safetensors"))), framework="np") as handle:
+ merged = handle.get_tensor(base_merge_name(0, 0))
+ qs = handle.get_tensor("model.layers.0.mlp.experts.0.qs")
+ self.assertEqual(merged.dtype, np.int8)
+ self.assertEqual(qs.dtype, np.float32)
+ self.assertEqual(merged.size, INTER * HIDDEN * 2 + HIDDEN * INTER)
+ self.assertEqual(qs.size, INTER + INTER + HIDDEN)
+
+ def test_dry_run_creates_nothing(self):
+ result = run_converter(self.gguf, self.out, "--dry-run")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertFalse(self.out.exists())
+
+ def test_refuses_wrong_architecture(self):
+ wrong = self.dir / "wrong.gguf"
+ build_tiny_olmoe_gguf(str(wrong), arch="llama")
+ result = run_converter(wrong, self.out)
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("general.architecture", result.stderr)
+ self.assertFalse(self.out.exists())
+
+ def test_refuses_unknown_tensor(self):
+ weird = self.dir / "weird.gguf"
+ build_tiny_olmoe_gguf(str(weird), extra_tensor={
+ "name": "blk.0.not_a_real_tensor",
+ "dims": [HIDDEN],
+ "ggml_type": 0,
+ "payload": _f32_bytes(np.zeros(HIDDEN)),
+ })
+ result = run_converter(weird, self.out)
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("unrecognized tensors", result.stderr)
+
+ def test_remove_source_only_on_success(self):
+ result = run_converter(self.gguf, self.out, "--remove-source-file")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertFalse(self.gguf.exists())
+ self.assertTrue((self.out / "config.json").is_file())
+
+ def test_qk_left_as_stored_by_default(self):
+ rng = np.random.default_rng(99)
+ hf_q_proj = rng.normal(size=(HIDDEN, HIDDEN)).astype(np.float16)
+ permuted_q_proj = _rope_permute(hf_q_proj, HEADS)
+ self.assertFalse(np.array_equal(permuted_q_proj, hf_q_proj))
+ build_tiny_olmoe_gguf(str(self.gguf), override={
+ "blk.0.attn_q.weight": permuted_q_proj.tobytes(),
+ })
+
+ result = run_converter(self.gguf, self.out)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ got = self._read("model.layers.0.self_attn.q_proj.weight")
+ # no flag: the stored (permuted) layout is passed through unchanged
+ np.testing.assert_array_equal(got, permuted_q_proj)
+
+ def test_qk_permuted_flag_undoes_rope_permutation(self):
+ rng = np.random.default_rng(99)
+ hf_q_proj = rng.normal(size=(HIDDEN, HIDDEN)).astype(np.float16)
+ hf_k_proj = rng.normal(size=(HIDDEN, HIDDEN)).astype(np.float16)
+ build_tiny_olmoe_gguf(str(self.gguf), override={
+ "blk.0.attn_q.weight": _rope_permute(hf_q_proj, HEADS).tobytes(),
+ "blk.0.attn_k.weight": _rope_permute(hf_k_proj, HEADS).tobytes(),
+ })
+
+ result = run_converter(self.gguf, self.out, "--qk-permuted")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ np.testing.assert_array_equal(
+ self._read("model.layers.0.self_attn.q_proj.weight"), hf_q_proj)
+ np.testing.assert_array_equal(
+ self._read("model.layers.0.self_attn.k_proj.weight"), hf_k_proj)
+
+ def _read(self, name):
+ from safetensors import safe_open
+ for shard in sorted(self.out.glob("model-*.safetensors")):
+ with safe_open(str(shard), framework="np") as handle:
+ if name in handle.keys():
+ return handle.get_tensor(name)
+ raise KeyError(name)
+
+ def test_overwrite_rewrites_existing_container(self):
+ self.assertEqual(run_converter(self.gguf, self.out).returncode, 0)
+ (self.out / "model-99999.safetensors").write_bytes(b"stale")
+ result = run_converter(self.gguf, self.out, "--overwrite")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertFalse((self.out / "model-99999.safetensors").exists())
+
+ def test_refuses_existing_container_without_flag(self):
+ self.assertEqual(run_converter(self.gguf, self.out).returncode, 0)
+ before = {p.name: p.read_bytes() for p in self.out.glob("model-*.safetensors")}
+ result = run_converter(self.gguf, self.out)
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("already contains a converted container", result.stderr)
+ self.assertIn("--overwrite", result.stderr)
+ after = {p.name: p.read_bytes() for p in self.out.glob("model-*.safetensors")}
+ self.assertEqual(before, after)
+
+ def test_dense_storage_dtypes(self):
+ self.assertEqual(
+ convert_gguf_to_olmoe._dense_to_storage(np.zeros(HIDDEN, np.float32)).dtype,
+ np.float32)
+ self.assertEqual(
+ convert_gguf_to_olmoe._dense_to_storage(np.zeros((HIDDEN, HIDDEN), np.float32)).dtype,
+ np.float16)
+
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/c/tests/test_gguf_dequant.py b/c/tests/test_gguf_dequant.py
new file mode 100644
index 00000000..2e3df9dc
--- /dev/null
+++ b/c/tests/test_gguf_dequant.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""tools/gguf_dequant.py: ggml block dequantization, pinned to ggml's math.
+
+Two independent checks:
+ * committed golden vectors (tests/fixtures/gguf_dequant_golden.npz), generated
+ with llama.cpp's gguf-py reference on deterministic random blocks -- run in
+ CI with only numpy;
+ * if the gguf package is importable, a live comparison against gguf.quants.
+ dequantize on fresh random blocks, so drift from the reference is caught.
+
+Also covers the scalar types (F32/F16/BF16) and the refusal paths (unsupported
+type, numel/byte-count mismatch).
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import gguf_reader
+from gguf_reader import GGUFError
+from gguf_dequant import (
+ dequantize,
+ dequantize_q2_K,
+ dequantize_q3_K,
+ dequantize_q6_K,
+)
+
+GOLDEN = Path(__file__).resolve().parent / "fixtures" / "gguf_dequant_golden.npz"
+
+BLOCK_CASES = [
+ ("Q2_K", gguf_reader.GGML_TYPE_Q2_K, 84, dequantize_q2_K),
+ ("Q3_K", gguf_reader.GGML_TYPE_Q3_K, 110, dequantize_q3_K),
+ ("Q6_K", gguf_reader.GGML_TYPE_Q6_K, 210, dequantize_q6_K),
+]
+
+
+class GoldenDequantTest(unittest.TestCase):
+ def setUp(self):
+ if not GOLDEN.is_file():
+ self.skipTest("golden fixture missing: %s" % GOLDEN)
+ self.golden = np.load(str(GOLDEN))
+
+ def test_block_types_match_golden(self):
+ for name, ggml_type, block_bytes, _func in BLOCK_CASES:
+ raw = self.golden["%s_raw" % name]
+ expected = self.golden["%s_expected" % name].reshape(-1)
+ got = dequantize(raw.tobytes(), ggml_type, expected.size)
+ self.assertEqual(got.shape, expected.shape)
+ np.testing.assert_allclose(got, expected, rtol=0, atol=0, equal_nan=True,
+ err_msg="%s dequant mismatch" % name)
+
+ def test_dispatcher_uses_same_functions(self):
+ for name, ggml_type, _bb, func in BLOCK_CASES:
+ raw = self.golden["%s_raw" % name]
+ direct = func(raw).reshape(-1)
+ dispatched = dequantize(raw.tobytes(), ggml_type, direct.size)
+ np.testing.assert_array_equal(dispatched, direct)
+
+
+class ScalarDequantTest(unittest.TestCase):
+ def test_f32(self):
+ values = np.array([0.0, 1.5, -2.25, 3.5e10], dtype=np.float32)
+ got = dequantize(values.tobytes(), gguf_reader.GGML_TYPE_F32, values.size)
+ np.testing.assert_array_equal(got, values)
+
+ def test_f16(self):
+ values = np.array([0.0, 1.5, -2.25, 0.125], dtype=np.float16)
+ got = dequantize(values.tobytes(), gguf_reader.GGML_TYPE_F16, values.size)
+ np.testing.assert_array_equal(got, values.astype(np.float32))
+
+ def test_bf16(self):
+ u16 = np.array([0x0000, 0x3FC0, 0xC040, 0x3E00], dtype=np.uint16)
+ got = dequantize(u16.tobytes(), gguf_reader.GGML_TYPE_BF16, u16.size)
+ expected = (u16.astype(np.uint32) << 16).view(np.float32)
+ np.testing.assert_array_equal(got, expected)
+
+
+class DequantRefusalTest(unittest.TestCase):
+ def test_unsupported_type(self):
+ with self.assertRaisesRegex(GGUFError, "unsupported ggml type"):
+ dequantize(b"\x00" * 16, 100, 16)
+
+ def test_numel_not_multiple_of_block(self):
+ with self.assertRaisesRegex(GGUFError, "not a multiple of block"):
+ dequantize(b"\x00" * 84, gguf_reader.GGML_TYPE_Q2_K, 255)
+
+ def test_byte_count_mismatch(self):
+ with self.assertRaisesRegex(GGUFError, "expected"):
+ dequantize(b"\x00" * 84, gguf_reader.GGML_TYPE_Q2_K, 512)
+
+
+class LiveReferenceTest(unittest.TestCase):
+ def test_matches_gguf_py_on_random_blocks(self):
+ try:
+ import gguf
+ from gguf.quants import dequantize as reference
+ except ImportError as exc:
+ self.skipTest("gguf package not installed: %s" % exc)
+
+ rng = np.random.default_rng(20260825)
+ mapping = {
+ "Q2_K": (gguf.GGMLQuantizationType.Q2_K, gguf_reader.GGML_TYPE_Q2_K, 84),
+ "Q3_K": (gguf.GGMLQuantizationType.Q3_K, gguf_reader.GGML_TYPE_Q3_K, 110),
+ "Q6_K": (gguf.GGMLQuantizationType.Q6_K, gguf_reader.GGML_TYPE_Q6_K, 210),
+ }
+ for name, (ref_type, our_type, block_bytes) in mapping.items():
+ for _ in range(25):
+ nb = int(rng.integers(1, 5))
+ raw = rng.integers(0, 256, size=nb * block_bytes, dtype=np.uint8)
+ expected = reference(raw.reshape(nb, block_bytes).copy(), ref_type).reshape(-1)
+ got = dequantize(raw.tobytes(), our_type, nb * 256)
+ np.testing.assert_allclose(got, expected, rtol=0, atol=0, equal_nan=True,
+ err_msg="%s live mismatch" % name)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_gguf_olmoe_profile.py b/c/tests/test_gguf_olmoe_profile.py
new file mode 100644
index 00000000..82ae7508
--- /dev/null
+++ b/c/tests/test_gguf_olmoe_profile.py
@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""tools/gguf_olmoe_profile.py: OLMoE name mapping, layout and int8 quantization.
+
+The strongest check here is test_matches_reference_quantizer: our numpy
+quantize_row is compared against the torch implementation in
+tools/convert_olmoe_merged.py, which is the code the existing OLMoE container
+was built with. If the two disagree, the new GGUF converter would produce a
+container the engine reads differently from the known-good one.
+"""
+
+import sys
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import gguf_olmoe_profile as profile
+
+
+class MappingTest(unittest.TestCase):
+ def test_parse_expert(self):
+ self.assertEqual(profile.parse_expert("blk.3.ffn_gate_exps.weight"), (3, "gate"))
+ self.assertEqual(profile.parse_expert("blk.0.ffn_down_exps.weight"), (0, "down"))
+ self.assertIsNone(profile.parse_expert("blk.0.attn_q.weight"))
+ self.assertIsNone(profile.parse_expert("token_embd.weight"))
+
+ def test_dense_target(self):
+ self.assertEqual(profile.dense_target("token_embd.weight"),
+ "model.embed_tokens.weight")
+ self.assertEqual(profile.dense_target("output.weight"), "lm_head.weight")
+ self.assertEqual(profile.dense_target("output_norm.weight"), "model.norm.weight")
+ self.assertEqual(profile.dense_target("blk.5.attn_q.weight"),
+ "model.layers.5.self_attn.q_proj.weight")
+ self.assertEqual(profile.dense_target("blk.11.ffn_gate_inp.weight"),
+ "model.layers.11.mlp.gate.weight")
+ self.assertEqual(profile.dense_target("blk.0.ffn_norm.weight"),
+ "model.layers.0.post_attention_layernorm.weight")
+ self.assertIsNone(profile.dense_target("blk.0.some_unknown.weight"))
+
+
+class LayoutTest(unittest.TestCase):
+ def test_to_logical_reverses_dims(self):
+ flat = np.arange(6, dtype=np.float32)
+ logical = profile.to_logical(flat, [3, 2])
+ self.assertEqual(logical.shape, (2, 3))
+ np.testing.assert_array_equal(logical, flat.reshape(2, 3))
+
+ def test_to_logical_1d_is_identity(self):
+ flat = np.arange(5, dtype=np.float32)
+ np.testing.assert_array_equal(profile.to_logical(flat, [5]), flat)
+
+ def test_expert_matrix_shape(self):
+ self.assertEqual(profile.expert_matrix_shape([2048, 1024, 64]), (1024, 2048))
+ self.assertEqual(profile.expert_matrix_shape([1024, 2048, 64]), (2048, 1024))
+
+ def test_restore_rope_layout_inverts_llama_permutation(self):
+ def forward(w, n_head, n_kv=None):
+ n = n_kv if (n_kv and n_head != n_kv) else n_head
+ return np.ascontiguousarray(
+ w.reshape((n, 2, w.shape[0] // n // 2) + w.shape[1:]).swapaxes(1, 2).reshape(w.shape))
+
+ rng = np.random.default_rng(11)
+ for (out, n_head, n_kv) in ((16, 4, 4), (24, 4, 2), (12, 2, None), (64, 4, None)):
+ hf = rng.normal(size=(out, 3)).astype(np.float32)
+ permuted = forward(hf, n_head, n_kv)
+ if out // (n_kv if (n_kv and n_head != n_kv) else n_head) // 2 != 2:
+ self.assertFalse(np.array_equal(permuted, hf))
+ restored = profile.restore_rope_layout(permuted, n_head, n_kv)
+ np.testing.assert_array_equal(restored, hf)
+
+ def test_restore_rope_layout_matches_torch_reference(self):
+ try:
+ import torch
+ except ImportError as exc:
+ self.skipTest("torch unavailable: %s" % exc)
+ for (out, n_head, n_kv) in ((16, 4, 4), (24, 4, 2), (12, 2, None)):
+ rng = np.random.default_rng(out)
+ hf = torch.from_numpy(rng.normal(size=(out, 7)).astype(np.float32))
+ n = n_kv if (n_kv and n_head != n_kv) else n_head
+ permuted = (hf.reshape(n, 2, out // n // 2, *hf.shape[1:])
+ .swapaxes(1, 2).reshape(hf.shape))
+ restored = profile.restore_rope_layout(permuted.numpy(), n_head, n_kv)
+ np.testing.assert_array_equal(restored, hf.numpy())
+
+
+class QuantizationTest(unittest.TestCase):
+ def test_quantize_row_math(self):
+ w = np.array([[1.0, -2.0, 0.5], [0.0, 0.0, 0.0]], dtype=np.float32)
+ q, s = profile.quantize_row(w)
+ self.assertEqual(q.dtype, np.int8)
+ self.assertEqual(q.shape, (2, 3))
+ self.assertEqual(s.shape, (2,))
+ # row 0: absmax 2 -> scale 2/127, q = round(w/scale)
+ self.assertAlmostEqual(float(s[0]), 2.0 / 127.0, places=7)
+ expected = np.clip(np.rint(w[0] / s[0]), -128, 127).astype(np.int8)
+ np.testing.assert_array_equal(q[0], expected)
+ # reconstruction W ~= q*scale
+ np.testing.assert_allclose(q.astype(np.float32) * s[:, None], w, atol=2.0 / 127.0)
+
+ def test_quantize_row_rejects_non_2d(self):
+ with self.assertRaises(ValueError):
+ profile.quantize_row(np.zeros(8, dtype=np.float32))
+
+ def test_matches_reference_quantizer(self):
+ try:
+ import torch
+ import convert_olmoe_merged as reference
+ except ImportError as exc:
+ self.skipTest("torch/convert_olmoe_merged unavailable: %s" % exc)
+
+ rng = np.random.default_rng(20260825)
+ for _ in range(20):
+ rows = int(rng.integers(1, 8))
+ cols = int(rng.integers(1, 40))
+ w = rng.normal(0.0, 1.0, size=(rows, cols)).astype(np.float32)
+ q_ref, s_ref = reference.quantize_row(torch.from_numpy(w))
+ q_got, s_got = profile.quantize_row(w)
+ np.testing.assert_array_equal(q_got, q_ref.cpu().numpy())
+ np.testing.assert_allclose(s_got, s_ref.cpu().numpy(), rtol=0, atol=0)
+
+ def test_merge_expert_layout(self):
+ rng = np.random.default_rng(7)
+ hidden, inter = 4, 6
+ gate = rng.normal(0, 1, (inter, hidden)).astype(np.float32)
+ up = rng.normal(0, 1, (inter, hidden)).astype(np.float32)
+ down = rng.normal(0, 1, (hidden, inter)).astype(np.float32)
+ merged, scales = profile.merge_expert(gate, up, down)
+
+ qg, sg = profile.quantize_row(gate)
+ qu, su = profile.quantize_row(up)
+ qd, sd = profile.quantize_row(down)
+ np.testing.assert_array_equal(
+ merged, np.concatenate((qg.reshape(-1), qu.reshape(-1), qd.reshape(-1))))
+ np.testing.assert_array_equal(
+ scales, np.concatenate((sg, su, sd)))
+ self.assertEqual(merged.dtype, np.int8)
+ self.assertEqual(scales.dtype, np.float32)
+ self.assertEqual(merged.size, inter * hidden * 2 + hidden * inter)
+ self.assertEqual(scales.size, inter + inter + hidden)
+
+
+class ConfigTest(unittest.TestCase):
+ def _meta(self):
+ return {
+ "olmoe.embedding_length": 2048,
+ "olmoe.block_count": 16,
+ "olmoe.attention.head_count": 16,
+ "olmoe.attention.head_count_kv": 16,
+ "olmoe.expert_count": 64,
+ "olmoe.expert_used_count": 8,
+ "olmoe.feed_forward_length": 1024,
+ "olmoe.rope.freq_base": 10000.0,
+ "olmoe.attention.layer_norm_rms_epsilon": 1e-5,
+ "tokenizer.ggml.tokens": ["a"] * 50304,
+ }
+
+ def test_build_config(self):
+ config = profile.build_config(self._meta())
+ self.assertEqual(config["hidden_size"], 2048)
+ self.assertEqual(config["num_hidden_layers"], 16)
+ self.assertEqual(config["num_attention_heads"], 16)
+ self.assertEqual(config["num_key_value_heads"], 16)
+ self.assertEqual(config["num_experts"], 64)
+ self.assertEqual(config["num_experts_per_tok"], 8)
+ self.assertEqual(config["intermediate_size"], 1024)
+ self.assertEqual(config["vocab_size"], 50304)
+ self.assertFalse(config["norm_topk_prob"])
+ self.assertEqual(config["model_type"], "olmoe")
+ self.assertEqual(config["architectures"], ["OlmoeForCausalLM"])
+
+ def test_build_config_norm_topk_prob_override(self):
+ config = profile.build_config(self._meta(), norm_topk_prob=True)
+ self.assertTrue(config["norm_topk_prob"])
+
+ def test_build_config_missing_key(self):
+ meta = self._meta()
+ del meta["olmoe.expert_count"]
+ with self.assertRaises(ValueError):
+ profile.build_config(meta)
+
+ def test_build_config_missing_tokens(self):
+ meta = self._meta()
+ del meta["tokenizer.ggml.tokens"]
+ with self.assertRaises(ValueError):
+ profile.build_config(meta)
+
+
+class TokenizerTest(unittest.TestCase):
+ def _meta(self):
+ return {
+ "tokenizer.ggml.tokens": ["hello", " world", "<unk>", "Ġt", "a"],
+ "tokenizer.ggml.merges": ["Ġ t", "Ġ Ġ"],
+ "tokenizer.ggml.token_type": [1, 1, 3, 4, 6],
+ "tokenizer.ggml.bos_token_id": 2,
+ "tokenizer.ggml.eos_token_id": 2,
+ "tokenizer.ggml.padding_token_id": 0,
+ "tokenizer.ggml.add_bos_token": False,
+ "tokenizer.ggml.add_eos_token": False,
+ "tokenizer.chat_template": "{{ bos_token }}...",
+ }
+
+ def test_vocab_and_merges(self):
+ tokenizer = profile.build_tokenizer(self._meta())
+ self.assertEqual(tokenizer["model"]["type"], "BPE")
+ self.assertEqual(tokenizer["model"]["vocab"], {
+ "hello": 0, " world": 1, "<unk>": 2, "Ġt": 3, "a": 4})
+ self.assertEqual(tokenizer["model"]["merges"], ["Ġ t", "Ġ Ġ"])
+
+ def test_added_tokens_only_special_and_user_defined(self):
+ tokenizer = profile.build_tokenizer(self._meta())
+ self.assertEqual(tokenizer["added_tokens"], [
+ {"id": 2, "content": "<unk>", "special": True},
+ {"id": 3, "content": "Ġt", "special": False},
+ ])
+
+ def test_special_ids(self):
+ tokenizer = profile.build_tokenizer(self._meta())
+ self.assertEqual(tokenizer["bos_token"], "<unk>")
+ self.assertEqual(tokenizer["eos_token"], "<unk>")
+ self.assertEqual(tokenizer["pad_token"], "hello")
+ self.assertFalse(tokenizer["add_bos_token"])
+ self.assertFalse(tokenizer["add_eos_token"])
+ self.assertEqual(tokenizer["chat_template"], "{{ bos_token }}...")
+
+ def test_missing_tokens_raises(self):
+ meta = self._meta()
+ del meta["tokenizer.ggml.tokens"]
+ with self.assertRaises(ValueError):
+ profile.build_tokenizer(meta)
+
+ def test_byte_token_not_added(self):
+ meta = self._meta()
+ meta["tokenizer.ggml.token_type"] = [6, 6, 6, 6, 6]
+ tokenizer = profile.build_tokenizer(meta)
+ self.assertEqual(tokenizer["added_tokens"], [])
+
+ def test_missing_token_type_defaults_to_normal(self):
+ meta = self._meta()
+ del meta["tokenizer.ggml.token_type"]
+ tokenizer = profile.build_tokenizer(meta)
+ self.assertEqual(tokenizer["added_tokens"], [])
+
+ def test_partial_token_type_defaults_to_normal(self):
+ meta = self._meta()
+ meta["tokenizer.ggml.token_type"] = [3] # only the first token is typed
+ tokenizer = profile.build_tokenizer(meta)
+ self.assertEqual(tokenizer["added_tokens"], [
+ {"id": 0, "content": "hello", "special": True},
+ ])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/convert_gguf_to_olmoe.py b/c/tools/convert_gguf_to_olmoe.py
new file mode 100644
index 00000000..89cb5782
--- /dev/null
+++ b/c/tools/convert_gguf_to_olmoe.py
@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""Convert a GGUF OLMoE checkpoint into a colibri container for c/olmoe.c.
+
+Flow (one user-facing entry point):
+ wrapper (this file) -> gguf_reader (structure) -> gguf_dequant (numbers)
+ -> gguf_olmoe_profile (OLMoE mapping/layout/quant)
+ -> safetensors shards + config.json + tokenizer.json
+
+Output, per tensor, matches what c/olmoe.c loads:
+ * dense tensors dequantized to F16 (1-D norms kept F32), names remapped;
+ * routed experts dequantized and re-quantized to int8 row-wise, merged into
+ model.layers.N.mlp.experts.E.merged_weight (I8) + .qs (F32);
+ * config.json reconstructed from the GGUF olmoe.* metadata. GGUF does not
+ carry norm_topk_prob; the default is false (matching the official OLMoE
+ checkpoints) and --norm-topk-prob overrides it. q/k are taken as stored
+ (HuggingFace layout); --qk-permuted undoes llama.cpp's RoPE permutation for
+ GGUFs that carry it;
+ * tokenizer.json rebuilt from tokenizer.ggml.* (byte-level GPT-2), validated
+ token-for-token against the official OLMoE tokenizer, so the container is
+ usable for chat/serve, not only loadable for validation.
+
+Safety:
+ * refuses a non-OLMoE GGUF, unknown tensor names, and missing required tensors;
+ * refuses to touch an --output that already contains a container unless the user
+ passes --overwrite (rebuild from scratch); it never silently skips work;
+ * writes shards atomically (tmp + rename), so a crash leaves no half shard;
+ * --remove-source-file deletes the source GGUF ONLY after a fully successful
+ conversion (never on a partial run);
+ * --dry-run prints the plan without writing anything or creating the output
+ directory.
+
+Usage:
+ python tools/convert_gguf_to_olmoe.py \\
+ --input /path/to/model.gguf # or a dir holding exactly one .gguf
+ --output /path/to/olmoe-colibri # created if missing
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+
+from gguf_reader import GGUFReader
+from gguf_dequant import dequantize
+import gguf_olmoe_profile as profile
+
+try:
+ from safetensors.numpy import save_file
+except ImportError as exc:
+ sys.exit("Missing dependency: %s. Install: pip install numpy safetensors" % exc)
+
+
+class ConversionError(Exception):
+ pass
+
+
+def resolve_sources(model):
+ path = Path(model).expanduser()
+ if path.is_dir():
+ files = sorted(p for p in path.iterdir() if p.suffix == ".gguf")
+ if not files:
+ raise ConversionError("no .gguf files in %s" % path)
+ if len(files) > 1:
+ raise ConversionError(
+ "%s holds %d .gguf files; OLMoE is converted from a single file "
+ "for now, pass one explicitly" % (path, len(files)))
+ return [files[0]]
+ if not path.is_file():
+ raise ConversionError("model not found: %s" % path)
+ return [path]
+
+
+def required_names(config):
+ names = {
+ "model.embed_tokens.weight",
+ "lm_head.weight",
+ "model.norm.weight",
+ }
+ for layer in range(config["num_hidden_layers"]):
+ base = "model.layers.%d" % layer
+ names.update({
+ base + ".input_layernorm.weight",
+ base + ".post_attention_layernorm.weight",
+ base + ".self_attn.q_proj.weight",
+ base + ".self_attn.k_proj.weight",
+ base + ".self_attn.v_proj.weight",
+ base + ".self_attn.o_proj.weight",
+ base + ".self_attn.q_norm.weight",
+ base + ".self_attn.k_norm.weight",
+ base + ".mlp.gate.weight",
+ })
+ return names
+
+
+def plan_tensors(reader):
+ """Return (dense, experts, unmapped) planned from the GGUF tensor list."""
+ dense = []
+ experts = {}
+ unmapped = []
+ for tensor in reader.tensors:
+ name = tensor["name"]
+ parsed = profile.parse_expert(name)
+ if parsed is not None:
+ layer, kind = parsed
+ experts.setdefault(layer, {})[kind] = tensor
+ continue
+ target = profile.dense_target(name)
+ if target is None:
+ unmapped.append(name)
+ continue
+ dense.append((target, tensor))
+ return dense, experts, unmapped
+
+
+class OutputWriter:
+ def __init__(self, out_dir, flush_every):
+ self.out_dir = out_dir
+ self.flush_every = flush_every
+ self.buf = {}
+ self.shard_idx = 0
+
+ def add_many(self, items):
+ """Buffer related tensors and flush once the buffer reaches flush_every.
+
+ Contract: every tensor of one logical unit -- e.g. an expert's int8
+ merged_weight AND its f32 .qs -- is passed in a single call, so a flush
+ never splits the pair across shards during a normal run.
+ """
+ for name, array in items:
+ self.buf[name] = np.ascontiguousarray(array)
+ if len(self.buf) >= self.flush_every:
+ self.flush()
+
+ def flush(self):
+ if not self.buf:
+ return
+ destination = self.out_dir / ("model-%05d.safetensors" % self.shard_idx)
+ temporary = Path(str(destination) + ".tmp")
+ temporary.unlink(missing_ok=True)
+ try:
+ save_file(self.buf, str(temporary))
+ os.replace(temporary, destination)
+ finally:
+ temporary.unlink(missing_ok=True)
+ print(" wrote %s (%d tensors)" % (destination.name, len(self.buf)), flush=True)
+ self.buf = {}
+ self.shard_idx += 1
+
+
+def _write_json_atomic(path, payload):
+ temporary = Path(str(path) + ".tmp")
+ temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n",
+ encoding="utf-8")
+ os.replace(temporary, path)
+
+
+def _dense_to_storage(logical):
+ if logical.ndim == 1:
+ return logical.astype(np.float32)
+ return logical.astype(np.float16)
+
+
+def convert_dense(reader, dense, writer, config, qk_permuted):
+ for target, tensor in dense:
+ raw = reader.read_tensor(tensor)
+ flat = dequantize(raw, tensor["ggml_type"], tensor["numel"])
+ logical = profile.to_logical(flat, tensor["dims"])
+ if qk_permuted:
+ # Some GGUF producers (recent llama.cpp olmo.py) RoPE-permute q/k;
+ # c/olmoe.c uses the plain HuggingFace layout, so undo it.
+ name = tensor["name"]
+ if name.endswith("attn_q.weight"):
+ logical = profile.restore_rope_layout(logical, config["num_attention_heads"],
+ config["num_attention_heads"])
+ elif name.endswith("attn_k.weight"):
+ logical = profile.restore_rope_layout(logical, config["num_attention_heads"],
+ config["num_key_value_heads"])
+ writer.add_many(((target, _dense_to_storage(logical)),))
+
+
+def convert_experts(reader, experts, writer, config):
+ layer_count = config["num_hidden_layers"]
+ expert_count = config["num_experts"]
+ hidden = config["hidden_size"]
+ inter = config["intermediate_size"]
+ want_w = inter * hidden + inter * hidden + hidden * inter
+ want_s = inter + inter + hidden
+ total = layer_count * expert_count
+ done_experts = 0
+
+ for layer in range(layer_count):
+ kinds = experts.get(layer)
+ if not kinds or set(kinds) != {"gate", "up", "down"}:
+ raise ConversionError("layer %d: incomplete expert trio: %s"
+ % (layer, sorted(kinds) if kinds else None))
+ raw_bytes, expert_layout = {}, {}
+ for kind in ("gate", "up", "down"):
+ tensor = kinds[kind]
+ raw_bytes[kind] = reader.read_tensor(tensor)
+ expert_layout[kind] = (profile.expert_matrix_shape(tensor["dims"]),
+ tensor["ggml_type"],
+ profile.expert_bytes_per_slice(tensor["dims"], tensor["ggml_type"]))
+ for expert in range(expert_count):
+ merged_name = "model.layers.%d.mlp.experts.%d.merged_weight" % (layer, expert)
+ qs_name = "model.layers.%d.mlp.experts.%d.qs" % (layer, expert)
+ per_kind = {}
+ for kind in ("gate", "up", "down"):
+ expert_shape, ggml_type, chunk = expert_layout[kind]
+ start = expert * chunk
+ flat = dequantize(raw_bytes[kind][start:start + chunk], ggml_type,
+ expert_shape[0] * expert_shape[1])
+ per_kind[kind] = flat.reshape(expert_shape)
+ merged, scales = profile.merge_expert(per_kind["gate"], per_kind["up"], per_kind["down"])
+ if merged.size != want_w or scales.size != want_s:
+ raise ConversionError(
+ "expert [%d,%d] produced %d bytes / %d scales, expected %d / %d"
+ % (layer, expert, merged.size, scales.size, want_w, want_s))
+ # The int8 weights and their f32 scales are ONE logical unit: one
+ # add_many call keeps them in the same shard (see OutputWriter).
+ writer.add_many(((merged_name, merged), (qs_name, scales)))
+ done_experts += 1
+ print(" layer %d/%d experts done (%d/%d)"
+ % (layer + 1, layer_count, done_experts, total), flush=True)
+
+
+def _format_duration(seconds):
+ seconds = int(round(seconds))
+ if seconds < 60:
+ return "%ds" % seconds
+ minutes, secs = divmod(seconds, 60)
+ if minutes < 60:
+ return "%dm %02ds" % (minutes, secs)
+ hours, minutes = divmod(minutes, 60)
+ return "%dh %02dm %02ds" % (hours, minutes, secs)
+
+
+def run(args):
+ started = time.perf_counter()
+ sources = resolve_sources(args.input)
+ source = sources[0]
+ out_dir = Path(args.output).expanduser()
+
+ print("colibri: converting a GGUF OLMoE checkpoint into a colibri container", flush=True)
+ print(" flow: %s" % source.name, flush=True)
+ print(" -> gguf_reader (header/metadata/tensor index)", flush=True)
+ print(" -> gguf_dequant + gguf_olmoe_profile (dequant, layout, int8 experts)", flush=True)
+ print(" -> safetensors shards + config.json + tokenizer.json", flush=True)
+ print(" input: %s" % source, flush=True)
+ print(" output: %s" % out_dir, flush=True)
+
+ reader = GGUFReader(str(source))
+ try:
+ arch = reader.metadata.get("general.architecture")
+ if arch != "olmoe":
+ raise ConversionError("general.architecture is %r, expected 'olmoe'" % arch)
+ config = profile.build_config(reader.metadata,
+ norm_topk_prob=args.norm_topk_prob)
+ dense, experts, unmapped = plan_tensors(reader)
+ if unmapped:
+ raise ConversionError("unrecognized tensors: %s" % ", ".join(sorted(unmapped)[:8]))
+
+ present = {target for target, _ in dense}
+ for layer, kinds in experts.items():
+ for expert in range(config["num_experts"]):
+ present.add("model.layers.%d.mlp.experts.%d.merged_weight" % (layer, expert))
+ missing = sorted(required_names(config) - present)
+ if missing:
+ raise ConversionError("missing required tensors: %s" % ", ".join(missing[:8]))
+
+ experts_total = config["num_hidden_layers"] * config["num_experts"]
+ if args.dry_run:
+ print("dry-run: %s" % source)
+ print(" architecture: olmoe, %d layers, %d experts (top-%d), hidden %d, inter %d, vocab %d"
+ % (config["num_hidden_layers"], config["num_experts"],
+ config["num_experts_per_tok"], config["hidden_size"],
+ config["intermediate_size"], config["vocab_size"]))
+ print(" dense tensors: %d" % len(dense))
+ print(" expert tensors: %d (%d merged_weight + %d qs)"
+ % (experts_total * 2, experts_total, experts_total))
+ if out_dir.exists() and any(out_dir.glob("model-*.safetensors")):
+ print(" note: output already contains a container; --overwrite rebuilds it")
+ return 0
+
+ existing_shards = sorted(out_dir.glob("model-*.safetensors"))
+ if existing_shards and not args.overwrite:
+ raise ConversionError(
+ "output already contains a converted container (%d safetensors shard(s)):\n"
+ " %s\n"
+ "refusing to overwrite it. Pass --overwrite to rebuild it from scratch."
+ % (len(existing_shards), out_dir))
+
+ out_dir.mkdir(parents=True, exist_ok=True)
+ if args.overwrite:
+ for path in list(out_dir.glob("model-*.safetensors")):
+ path.unlink()
+ for name in ("config.json", "tokenizer.json"):
+ sidecar = out_dir / name
+ if sidecar.is_file():
+ sidecar.unlink()
+ print(" --overwrite: cleared previous container outputs", flush=True)
+ writer = OutputWriter(out_dir, args.flush_every)
+ _write_json_atomic(out_dir / "config.json", config)
+ print(" wrote config.json", flush=True)
+ tokenizer = profile.build_tokenizer(reader.metadata)
+ _write_json_atomic(out_dir / "tokenizer.json", tokenizer)
+ print(" wrote tokenizer.json (%d vocab, %d merges, %d added)"
+ % (len(tokenizer["model"]["vocab"]),
+ len(tokenizer["model"].get("merges", [])),
+ len(tokenizer["added_tokens"])), flush=True)
+ convert_dense(reader, dense, writer, config, args.qk_permuted)
+ convert_experts(reader, experts, writer, config)
+ writer.flush()
+ finally:
+ reader.close()
+
+ if args.remove_source_file:
+ os.remove(source)
+ print(" removed source %s" % source, flush=True)
+ print("Done:")
+ print("- path: %s" % out_dir)
+ print("- time: %s" % _format_duration(time.perf_counter() - started))
+ return 0
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ prog="convert_gguf_to_olmoe.py",
+ description="Convert an OLMoE GGUF checkpoint into a colibri container "
+ "readable by the c/olmoe.c engine (safetensors shards + "
+ "config.json + tokenizer.json).",
+ epilog=(
+ "examples:\n"
+ " convert_gguf_to_olmoe.py --input model.gguf --output ./olmoe-colibri\n"
+ " convert_gguf_to_olmoe.py --input ./gguf-dir --output ~/llm/olmoe --dry-run\n"
+ " convert_gguf_to_olmoe.py --input model.gguf --output ./olmoe --remove-source-file\n"
+ "\n"
+ "notes:\n"
+ " --input may be a single .gguf file or a directory holding exactly one\n"
+ " .gguf (more than one is refused: OLMoE is converted from one file).\n"
+ " --output is created (with any missing parent) if it does not exist.\n"
+ " If --output already holds a container, the run REFUSES with a message;\n"
+ " pass --overwrite to rebuild it from scratch.\n"
+ " --remove-source-file deletes the input .gguf only after the whole\n"
+ " conversion finished successfully; it never removes on a partial run.\n"
+ " --dry-run prints the plan, writes nothing and creates no directory.\n"
+ " The engine can load the result for validation; chat/serve work because\n"
+ " tokenizer.json is reconstructed from the GGUF tokenizer metadata.\n"),
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument("--input", required=True, metavar="PATH",
+ help="source GGUF file, or a directory holding exactly one .gguf")
+ parser.add_argument("--output", required=True, metavar="DIR",
+ help="destination directory for the colibri container "
+ "(created, including missing parents, if absent)")
+ parser.add_argument("--dry-run", action="store_true",
+ help="print the conversion plan (layers, experts, output dir) "
+ "and exit without writing anything or creating --output")
+ parser.add_argument("--remove-source-file", action="store_true",
+ help="after a fully successful conversion, delete the source "
+ ".gguf (never on a partial or failed run)")
+ parser.add_argument("--overwrite", action="store_true",
+ help="rebuild an existing container in --output from scratch. "
+ "Without it, the run refuses when --output already holds "
+ "model-*.safetensors")
+ parser.add_argument("--norm-topk-prob", action="store_true",
+ help="set config.json norm_topk_prob=true (renormalize top-k router "
+ "weights). GGUF does not carry this field; the default is false, "
+ "which matches the official OLMoE checkpoints")
+ parser.add_argument("--qk-permuted", action="store_true",
+ help="undo llama.cpp's RoPE permutation of q_proj/k_proj. Off by "
+ "default because the official OLMoE GGUF is already in the "
+ "HuggingFace layout; use it only for GGUFs that were permuted")
+ parser.add_argument("--flush-every", type=int, default=512, metavar="N",
+ help="flush an output safetensors shard every N tensors "
+ "(default: 512; smaller = more, smaller shards)")
+ args = parser.parse_args()
+ try:
+ return run(args)
+ except ConversionError as exc:
+ sys.stderr.write("ERROR: %s\n" % exc)
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/c/tools/gguf_dequant.py b/c/tools/gguf_dequant.py
new file mode 100644
index 00000000..24bd56b8
--- /dev/null
+++ b/c/tools/gguf_dequant.py
@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Dequantization of GGUF/ggml tensor blocks to float32, pure numpy.
+
+Ported block-for-block from ggml-org/ggml src/ggml-quants.c (dequantize_row_*),
+using the block layouts in src/ggml-common.h. This module owns the numerical
+decode used by the converter profiles; gguf_reader.py owns structure only.
+
+Only the types listed in _DEQUANTIZERS are implemented. dequantize() raises
+GGUFError for any other type, so a profile never silently mis-decodes bytes.
+Correctness is pinned by tests/test_gguf_dequant.py, which compares against
+llama.cpp's gguf-py reference on random blocks and against committed vectors.
+"""
+
+import numpy as np
+
+from gguf_reader import (
+ GGML_TYPE_BF16,
+ GGML_TYPE_F16,
+ GGML_TYPE_F32,
+ GGML_TYPE_Q2_K,
+ GGML_TYPE_Q3_K,
+ GGML_TYPE_Q6_K,
+ GGUFError,
+ ggml_block_spec,
+ ggml_type_name,
+)
+
+QK_K = 256
+_MASK_KMASK1 = 0x03030303
+_MASK_KMASK2 = 0x0F0F0F0F
+
+
+def _f16_to_f32(pair_bytes):
+ """(n, 2) uint8 or (n,) uint16 -> float32."""
+ u16 = pair_bytes.view("<u2").reshape(-1)
+ return u16.view(np.float16).astype(np.float32)
+
+
+def _bf16_to_f32(word_bytes):
+ u16 = word_bytes.view("<u2").reshape(-1).astype(np.uint32)
+ return (u16 << 16).view(np.float32)
+
+
+def dequantize_q2_K(blocks):
+ # Q2_K (ggml-common.h block_q2_K, 84 bytes/256 elems):
+ # scales[16] (per-16 low nibble=scale, high nibble=min, 4-bit),
+ # qs[64] (2-bit packed quants), then fp16 block scale (d) and min (dmin).
+ num_blocks = blocks.shape[0]
+ scales = blocks[:, 0:16].astype(np.int32)
+ packed_q = blocks[:, 16:80]
+ block_scale = _f16_to_f32(blocks[:, 80:82])
+ block_min = _f16_to_f32(blocks[:, 82:84])
+ out = np.empty((num_blocks, QK_K), dtype=np.float32)
+ is_ = 0
+ for half in range(2):
+ base = half * 32
+ for j in range(4):
+ shift = j * 2
+ for sub in range(2):
+ sc = scales[:, is_]
+ is_ += 1
+ group_scale = block_scale * (sc & 0xF)
+ group_min = block_min * (sc >> 4)
+ cols = base + sub * 16 + np.arange(16)
+ qv = ((packed_q[:, cols] >> shift) & 3).astype(np.float32)
+ lo = half * 128 + j * 32 + sub * 16
+ out[:, lo:lo + 16] = group_scale[:, None] * qv - group_min[:, None]
+ return out
+
+
+def dequantize_q3_K(blocks):
+ # Q3_K (ggml-common.h block_q3_K, 110 bytes/256 elems):
+ # hmask[32] high bits, qs[64] low 2 bits, scales[12] (sixteen 6-bit scales
+ # packed into 12 bytes, reassembled by the kmask shuffle below), fp16 scale.
+ num_blocks = blocks.shape[0]
+ hmask = blocks[:, 0:32]
+ packed_q = blocks[:, 32:96]
+ scale_bytes = blocks[:, 96:108]
+ block_scale = _f16_to_f32(blocks[:, 108:110])
+
+ # Sixteen 6-bit scales live packed in 12 bytes: three 32-bit words, then a
+ # bit-shuffle reassembles them (kmask1/kmask2 from ggml-quants.c).
+ a0 = (scale_bytes[:, 0].astype(np.uint32) | (scale_bytes[:, 1].astype(np.uint32) << 8)
+ | (scale_bytes[:, 2].astype(np.uint32) << 16) | (scale_bytes[:, 3].astype(np.uint32) << 24))
+ a1 = (scale_bytes[:, 4].astype(np.uint32) | (scale_bytes[:, 5].astype(np.uint32) << 8)
+ | (scale_bytes[:, 6].astype(np.uint32) << 16) | (scale_bytes[:, 7].astype(np.uint32) << 24))
+ a2 = (scale_bytes[:, 8].astype(np.uint32) | (scale_bytes[:, 9].astype(np.uint32) << 8)
+ | (scale_bytes[:, 10].astype(np.uint32) << 16) | (scale_bytes[:, 11].astype(np.uint32) << 24))
+ tmp = a2
+ a2n = ((a0 >> 4) & _MASK_KMASK2) | (((tmp >> 4) & _MASK_KMASK1) << 4)
+ a3n = ((a1 >> 4) & _MASK_KMASK2) | (((tmp >> 6) & _MASK_KMASK1) << 4)
+ a0n = (a0 & _MASK_KMASK2) | (((tmp >> 0) & _MASK_KMASK1) << 4)
+ a1n = (a1 & _MASK_KMASK2) | (((tmp >> 2) & _MASK_KMASK1) << 4)
+
+ row_scales = np.empty((num_blocks, 16), dtype=np.int8)
+ for i, word in enumerate((a0n, a1n, a2n, a3n)):
+ row_scales[:, i * 4:(i + 1) * 4] = word.astype("<u4").view(np.uint8).reshape(num_blocks, 4)
+
+ out = np.empty((num_blocks, QK_K), dtype=np.float32)
+ is_ = 0
+ for half in range(2):
+ base = half * 32
+ for j in range(4):
+ shift = j * 2
+ m = 1 << (half * 4 + j)
+ for sub in range(2):
+ group_scale = block_scale * (row_scales[:, is_] - 32).astype(np.float32)
+ is_ += 1
+ qcols = base + sub * 16 + np.arange(16)
+ hcols = sub * 16 + np.arange(16)
+ qv = ((packed_q[:, qcols] >> shift) & 3).astype(np.float32)
+ high_bit = ((hmask[:, hcols] & m) != 0).astype(np.float32)
+ # low 2-bit code, minus 4 when the high bit of the mask is clear
+ centered_q = qv - 4.0 * (1.0 - high_bit)
+ lo = half * 128 + j * 32 + sub * 16
+ out[:, lo:lo + 16] = group_scale[:, None] * centered_q
+ return out
+
+
+def dequantize_q6_K(blocks):
+ # Q6_K (ggml-common.h block_q6_K, 210 bytes/256 elems):
+ # ql[128] low 4 bits, qh[64] upper 2 bits, scales[16] int8 per-16 scales,
+ # fp16 block scale. Two 128-elem halves; codes are (low | high<<4) - 32.
+ num_blocks = blocks.shape[0]
+ packed_lo = blocks[:, 0:128]
+ packed_hi = blocks[:, 128:192]
+ row_scales = blocks[:, 192:208].view(np.int8)
+ block_scale = _f16_to_f32(blocks[:, 208:210])
+ out = np.empty((num_blocks, QK_K), dtype=np.float32)
+ for half in range(2):
+ lo_base = half * 64
+ hi_base = half * 32
+ scale_base = half * 8
+ l = np.arange(32)
+ group = l // 16
+ code0 = ((packed_lo[:, lo_base + l] & 0xF).astype(np.int32)
+ | (((packed_hi[:, hi_base + l] >> 0) & 3).astype(np.int32) << 4)) - 32
+ code1 = ((packed_lo[:, lo_base + l + 32] & 0xF).astype(np.int32)
+ | (((packed_hi[:, hi_base + l] >> 2) & 3).astype(np.int32) << 4)) - 32
+ code2 = ((packed_lo[:, lo_base + l] >> 4).astype(np.int32)
+ | (((packed_hi[:, hi_base + l] >> 4) & 3).astype(np.int32) << 4)) - 32
+ code3 = ((packed_lo[:, lo_base + l + 32] >> 4).astype(np.int32)
+ | (((packed_hi[:, hi_base + l] >> 6) & 3).astype(np.int32) << 4)) - 32
+ scale0 = row_scales[:, scale_base + group + 0].astype(np.float32)
+ scale1 = row_scales[:, scale_base + group + 2].astype(np.float32)
+ scale2 = row_scales[:, scale_base + group + 4].astype(np.float32)
+ scale3 = row_scales[:, scale_base + group + 6].astype(np.float32)
+ base = half * 128
+ out[:, base + l] = block_scale[:, None] * scale0 * code0.astype(np.float32)
+ out[:, base + l + 32] = block_scale[:, None] * scale1 * code1.astype(np.float32)
+ out[:, base + l + 64] = block_scale[:, None] * scale2 * code2.astype(np.float32)
+ out[:, base + l + 96] = block_scale[:, None] * scale3 * code3.astype(np.float32)
+ return out
+
+
+_DEQUANTIZERS = {
+ GGML_TYPE_Q2_K: dequantize_q2_K,
+ GGML_TYPE_Q3_K: dequantize_q3_K,
+ GGML_TYPE_Q6_K: dequantize_q6_K,
+}
+
+
+def dequantize(raw, ggml_type, numel):
+ """Decode a tensor's stored bytes to a flat float32 array (GGUF element order)."""
+ if ggml_type in _DEQUANTIZERS:
+ block_elems, block_bytes = ggml_block_spec(ggml_type)
+ if numel % block_elems != 0:
+ raise GGUFError(
+ "dequant %s: numel %d is not a multiple of block %d"
+ % (ggml_type_name(ggml_type), numel, block_elems))
+ nblocks = numel // block_elems
+ data = np.frombuffer(raw, dtype=np.uint8)
+ if data.size != nblocks * block_bytes:
+ raise GGUFError(
+ "dequant %s: %d bytes for %d blocks of %d, expected %d"
+ % (ggml_type_name(ggml_type), data.size, nblocks, block_bytes,
+ nblocks * block_bytes))
+ blocks = data.reshape(nblocks, block_bytes)
+ return _DEQUANTIZERS[ggml_type](blocks).reshape(-1)
+
+ element_size = {GGML_TYPE_F32: 4, GGML_TYPE_F16: 2, GGML_TYPE_BF16: 2}.get(ggml_type)
+ if element_size is not None:
+ expected = numel * element_size
+ if len(raw) != expected:
+ raise GGUFError(
+ "dequant %s: %d bytes for %d elements, expected %d"
+ % (ggml_type_name(ggml_type), len(raw), numel, expected))
+ if ggml_type == GGML_TYPE_F32:
+ return np.frombuffer(raw, dtype="<f4").astype(np.float32)
+ if ggml_type == GGML_TYPE_F16:
+ return np.frombuffer(raw, dtype="<f2").astype(np.float32)
+ return _bf16_to_f32(np.frombuffer(raw, dtype=np.uint8))
+
+ raise GGUFError("dequant: unsupported ggml type %d (%s)"
+ % (ggml_type, ggml_type_name(ggml_type)))
diff --git a/c/tools/gguf_olmoe_profile.py b/c/tools/gguf_olmoe_profile.py
new file mode 100644
index 00000000..f0a755ba
--- /dev/null
+++ b/c/tools/gguf_olmoe_profile.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""OLMoE profile: GGUF tensor layout -> colibri container layout.
+
+This module owns everything OLMoE-specific about the conversion and nothing
+about file I/O or CLI:
+
+ * name mapping from GGUF (llama.cpp) tensor names to the names c/olmoe.c
+ loads: model.embed_tokens.weight, model.layers.N.self_attn.q_proj.weight,
+ and per-expert model.layers.N.mlp.experts.E.{merged_weight,qs};
+ * dequantized GGUF order -> logical HuggingFace order (GGUF stores dims
+ reversed);
+ * row-wise int8 quantization identical to olmoe.c's decoder contract
+ W[o,i] ~= q[o,i]*scale[o] (same math as tools/convert_olmoe_merged.py);
+ * per-expert gate|up|down merge into one merged_weight + one .qs;
+ * config.json reconstruction from GGUF olmoe.* metadata.
+
+It is imported by convert_gguf_to_olmoe.py (CLI) and unit-tested directly.
+"""
+
+import re
+
+import numpy as np
+
+EXPERT_RE = re.compile(r"^blk\.(\d+)\.ffn_(gate|up|down)_exps\.weight$")
+
+DENSE_SIMPLE = {
+ "token_embd.weight": "model.embed_tokens.weight",
+ "output.weight": "lm_head.weight",
+ "output_norm.weight": "model.norm.weight",
+}
+
+DENSE_LAYERED_RE = re.compile(
+ r"^blk\.(\d+)\.(attn_norm|ffn_norm|attn_q|attn_k|attn_v|attn_output|"
+ r"attn_q_norm|attn_k_norm|ffn_gate_inp)\.weight$")
+
+LAYER_SUFFIX = {
+ "attn_norm": "input_layernorm.weight",
+ "ffn_norm": "post_attention_layernorm.weight",
+ "attn_q": "self_attn.q_proj.weight",
+ "attn_k": "self_attn.k_proj.weight",
+ "attn_v": "self_attn.v_proj.weight",
+ "attn_output": "self_attn.o_proj.weight",
+ "attn_q_norm": "self_attn.q_norm.weight",
+ "attn_k_norm": "self_attn.k_norm.weight",
+ "ffn_gate_inp": "mlp.gate.weight",
+}
+
+CONFIG_KEYS = {
+ "olmoe.embedding_length": "hidden_size",
+ "olmoe.block_count": "num_hidden_layers",
+ "olmoe.attention.head_count": "num_attention_heads",
+ "olmoe.attention.head_count_kv": "num_key_value_heads",
+ "olmoe.expert_count": "num_experts",
+ "olmoe.expert_used_count": "num_experts_per_tok",
+ "olmoe.feed_forward_length": "intermediate_size",
+ "olmoe.rope.freq_base": "rope_theta",
+ "olmoe.attention.layer_norm_rms_epsilon": "rms_norm_eps",
+}
+
+
+def parse_expert(name):
+ """(layer, kind) for an expert tensor name, or None."""
+ match = EXPERT_RE.match(name)
+ if not match:
+ return None
+ return int(match.group(1)), match.group(2)
+
+
+def dense_target(name):
+ """colibri name for a GGUF dense tensor, or None if not a known dense tensor."""
+ if name in DENSE_SIMPLE:
+ return DENSE_SIMPLE[name]
+ match = DENSE_LAYERED_RE.match(name)
+ if match:
+ layer, kind = int(match.group(1)), match.group(2)
+ return "model.layers.%d.%s" % (layer, LAYER_SUFFIX[kind])
+ return None
+
+
+def to_logical(flat, gguf_dims):
+ """Reshape GGUF-order flat values into the logical (HuggingFace) shape.
+
+ GGUF stores the source tensor's bytes unchanged and only reverses the dims
+ labels, so the flat data is already C-order of the original shape: reshape
+ to reversed(gguf_dims) reproduces it. No transpose is involved.
+ """
+ return np.ascontiguousarray(flat.reshape(tuple(reversed(gguf_dims))))
+
+
+def restore_rope_layout(weights, n_head, n_kv_head=None):
+ """Undo llama.cpp's RoPE permutation of q_proj/k_proj weights.
+
+ conversion/olmo.py permutes q_proj and k_proj exactly like Llama:
+ forward: w.reshape(n, 2, out // n // 2, *rest).swapaxes(1, 2).reshape(w.shape)
+ with n = n_kv_head when it differs from n_head (k_proj), else n_head. That
+ is NOT an involution unless out // n // 2 == 2, so the inverse is the axis
+ transpose of the (m, 2) decomposition:
+ inverse: w.reshape(n, m, 2, *rest).swapaxes(1, 2).reshape(w.shape)
+ with m = out // n // 2. c/olmoe.c uses HF-style RoPE (adjacent pairs per
+ head), so the GGUF's permuted q/k must be restored to the HF layout.
+ """
+ n = n_kv_head if (n_kv_head and n_head != n_kv_head) else n_head
+ w = np.asarray(weights)
+ if w.ndim < 1 or n < 1 or w.shape[0] % n != 0 or (w.shape[0] // n) % 2 != 0:
+ raise ValueError("cannot restore RoPE layout for shape %r with %d heads"
+ % (w.shape, n))
+ m = w.shape[0] // n // 2
+ rest = w.shape[1:]
+ return np.ascontiguousarray(
+ w.reshape((n, m, 2) + rest).swapaxes(1, 2).reshape(w.shape))
+
+
+def quantize_row(weights):
+ """Row-wise int8 quantization, identical math to c/olmoe.c's decoder.
+
+ olmoe.c computes y[o] = scale[o] * sum_i x[i]*q[o,i], so the stored bytes
+ must satisfy W[o,i] ~= q[o,i]*scale[o] with scale[o] = row_absmax/127.
+ Rounding is numpy's rint (round-half-to-even), matching convert_olmoe_merged.py.
+ """
+ w = np.asarray(weights, dtype=np.float32)
+ if w.ndim != 2:
+ raise ValueError("quantize_row expects a 2-D weight, got shape %r" % (w.shape,))
+ row_max = np.maximum(np.max(np.abs(w), axis=1, keepdims=True), np.float32(1e-12))
+ scales = (row_max / np.float32(127.0)).astype(np.float32)
+ quantized = np.clip(np.rint(w / scales), -128.0, 127.0).astype(np.int8)
+ return quantized, scales.reshape(-1)
+
+
+def merge_expert(gate, up, down):
+ """Quantize and merge one expert's gate/up/down into colibri's layout.
+
+ merged_weight = int8(gate | up | down) flattened;
+ qs = f32(gate rows | up rows | down rows).
+ gate/up: [inter, hidden]; down: [hidden, inter].
+ """
+ q_gate, s_gate = quantize_row(gate)
+ q_up, s_up = quantize_row(up)
+ q_down, s_down = quantize_row(down)
+ merged = np.concatenate((q_gate.reshape(-1), q_up.reshape(-1), q_down.reshape(-1)))
+ scales = np.concatenate((s_gate, s_up, s_down)).astype(np.float32)
+ return merged, scales
+
+
+def expert_matrix_shape(gguf_dims):
+ """HF shape of one expert's matrix inside a fused [d0, d1, E] expert tensor."""
+ return (gguf_dims[1], gguf_dims[0])
+
+
+def expert_bytes_per_slice(gguf_dims, ggml_type):
+ """Bytes of one expert inside a fused expert tensor (contiguous slice)."""
+ from gguf_reader import ggml_block_spec
+ block_elems, block_bytes = ggml_block_spec(ggml_type)
+ if gguf_dims[0] % block_elems != 0:
+ raise ValueError(
+ "expert tensor dims[0]=%d is not a multiple of the block %d"
+ % (gguf_dims[0], block_elems))
+ row_blocks = gguf_dims[0] // block_elems
+ return row_blocks * block_bytes * gguf_dims[1]
+
+
+def _token_type_of(meta, index):
+ entries = meta.get("tokenizer.ggml.token_type")
+ if not entries or index >= len(entries):
+ return 1
+ return int(entries[index])
+
+
+def build_tokenizer(meta):
+ """Reconstruct an HF-style tokenizer.json from GGUF tokenizer metadata.
+
+ Feed shape matches what c/tok.h loads for GPT-2/byte-level BPE:
+ model.vocab (token string -> id), model.merges (space-separated "left
+ right" entries), added_tokens (id/content/special), pre_tokenizer that
+ does not select the o200k/Kimi maybe-flags, plus bos/eos/pad and the chat
+ template. token_type 3=CONTROL (special) and 4=USER_DEFINED become added
+ tokens; 5=UNUSED and 6=BYTE stay plain vocab entries.
+ """
+ tokens = meta.get("tokenizer.ggml.tokens")
+ if not tokens:
+ raise ValueError("GGUF metadata missing tokenizer.ggml.tokens")
+ merges = meta.get("tokenizer.ggml.merges")
+
+ vocab = {}
+ added = []
+ # ggml token_type: 1=NORMAL stays plain BPE vocab; 3=CONTROL and
+ # 4=USER_DEFINED become added tokens (atomic, literal); 5=UNUSED and 6=BYTE
+ # stay plain vocab entries (not atomic specials).
+ for index, token in enumerate(tokens):
+ vocab[token] = index
+ kind = _token_type_of(meta, index)
+ if kind in (3, 4):
+ added.append({"id": index, "content": token, "special": kind == 3})
+
+ model = {"type": "BPE", "vocab": vocab}
+ if merges:
+ model["merges"] = list(merges)
+
+ tokenizer = {
+ "add_bos_token": bool(meta.get("tokenizer.ggml.add_bos_token")),
+ "add_eos_token": bool(meta.get("tokenizer.ggml.add_eos_token")),
+ "model": model,
+ "pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": False},
+ "added_tokens": added,
+ }
+ for gguf_key, hf_key in (
+ ("tokenizer.ggml.bos_token_id", "bos_token"),
+ ("tokenizer.ggml.eos_token_id", "eos_token"),
+ ("tokenizer.ggml.padding_token_id", "pad_token")):
+ token_id = meta.get(gguf_key)
+ if token_id is not None and int(token_id) < len(tokens):
+ tokenizer[hf_key] = tokens[int(token_id)]
+ if "tokenizer.chat_template" in meta:
+ tokenizer["chat_template"] = meta["tokenizer.chat_template"]
+ return tokenizer
+
+
+def build_config(meta, norm_topk_prob=False, extra=None):
+ """Reconstruct the config.json c/olmoe.c loads from GGUF metadata.
+
+ norm_topk_prob is NOT in GGUF metadata; the official OLMoE checkpoints use
+ false (transformers' OlmoeConfig default), so that is the default here. Use
+ the CLI flag to override for a checkpoint that trained with it on.
+ """
+ config = {}
+ for gguf_key, cfg_key in CONFIG_KEYS.items():
+ if gguf_key not in meta:
+ raise ValueError("GGUF metadata missing %s" % gguf_key)
+ config[cfg_key] = meta[gguf_key]
+ tokens = meta.get("tokenizer.ggml.tokens")
+ if not tokens:
+ raise ValueError("GGUF metadata missing tokenizer.ggml.tokens (needed for vocab_size)")
+ config["vocab_size"] = len(tokens)
+ config["norm_topk_prob"] = bool(norm_topk_prob)
+ config["model_type"] = "olmoe"
+ config["architectures"] = ["OlmoeForCausalLM"]
+ if extra:
+ config.update(extra)
+ return config