Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
added general tool wrapper, gguf dequant and special olmoe conversion file, added tests
matteoc054523eae2c
diff --git a/c/tests/fixtures/gguf_dequant_golden.npz b/c/tests/fixtures/gguf_dequant_golden.npznew file mode 100644index 00000000..d45387bcBinary files /dev/null and b/c/tests/fixtures/gguf_dequant_golden.npz differdiff --git a/c/tests/test_convert_gguf_to_olmoe.py b/c/tests/test_convert_gguf_to_olmoe.pynew file mode 100644index 00000000..e434ab69--- /dev/null+++ b/c/tests/test_convert_gguf_to_olmoe.py@@ -0,0 +1,331 @@+#!/usr/bin/env python3+"""tools/convert_gguf_to_olmoe.py: end-to-end conversion of a tiny synthetic GGUF.++Builds a minimal OLMoE-shaped GGUF (all F16/F32, no K-quants) with the smallest+dims the engine accepts, runs the real CLI, and asserts the container c/olmoe.c+expects: config.json keys, tokenizer.json reconstruction, the dense name+mapping, and per-expert merged_weight (I8) + .qs (F32) at the exact sizes the+engine requires. Also covers --dry-run (writes nothing, creates no dir) and the+refusals for a non-OLMoE GGUF / unknown tensor; --remove-source-file removes+the source only after full success.+"""++import json+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++import numpy as np++TESTDIR = Path(__file__).resolve().parent+TOOLS = TESTDIR.parent / "tools"+sys.path.insert(0, str(TOOLS))+sys.path.insert(0, str(TESTDIR))++from gguf_fixture import ( # noqa: E402 (encoder helpers reused verbatim)+ METADATA_ARRAY,+ METADATA_BOOL,+ METADATA_FLOAT32,+ METADATA_STRING,+ METADATA_UINT32,+ _encode_string,+ _encode_value,+)++import convert_gguf_to_olmoe # noqa: E402++# tiny but engine-valid dims+HIDDEN, INTER, N_EXPERTS, LAYERS, VOCAB, HEADS = 8, 4, 2, 1, 16, 2+++def _align(value, alignment):+ return (value + alignment - 1) // alignment * alignment+++def _f16_bytes(values):+ return np.asarray(values, dtype=np.float16).tobytes()+++def _f32_bytes(values):+ return np.asarray(values, dtype=np.float32).tobytes()+++def build_tiny_olmoe_gguf(path, arch="olmoe", extra_tensor=None, override=None):+ rng = np.random.default_rng(20260825)+ override = override or {}++ def m2d(hf_shape):+ return list(reversed(hf_shape))++ tensors = []+ # HF shape -> GGUF dims (reversed). Dense matrices F16, 1-D norms F32.+ def add_gguf(name, gguf_dims, ggml_type, payload):+ tensors.append({"name": name, "dims": gguf_dims,+ "ggml_type": ggml_type, "payload": override.get(name, payload)})++ add_gguf("token_embd.weight", m2d([VOCAB, HIDDEN]), 1,+ _f16_bytes(rng.normal(size=VOCAB * HIDDEN)))+ add_gguf("output.weight", m2d([VOCAB, HIDDEN]), 1,+ _f16_bytes(rng.normal(size=VOCAB * HIDDEN)))+ add_gguf("output_norm.weight", [HIDDEN], 0, _f32_bytes(rng.normal(size=HIDDEN)))+ for layer in range(LAYERS):+ add_gguf("blk.%d.attn_norm.weight" % layer, [HIDDEN], 0,+ _f32_bytes(rng.normal(size=HIDDEN)))+ add_gguf("blk.%d.ffn_norm.weight" % layer, [HIDDEN], 0,+ _f32_bytes(rng.normal(size=HIDDEN)))+ for w in ("q", "k", "v", "output"):+ add_gguf("blk.%d.attn_%s.weight" % (layer, w), m2d([HIDDEN, HIDDEN]), 1,+ _f16_bytes(rng.normal(size=HIDDEN * HIDDEN)))+ add_gguf("blk.%d.attn_q_norm.weight" % layer, [HIDDEN], 0,+ _f32_bytes(rng.normal(size=HIDDEN)))+ add_gguf("blk.%d.attn_k_norm.weight" % layer, [HIDDEN], 0,+ _f32_bytes(rng.normal(size=HIDDEN)))+ add_gguf("blk.%d.ffn_gate_inp.weight" % layer, m2d([N_EXPERTS, HIDDEN]), 1,+ _f16_bytes(rng.normal(size=N_EXPERTS * HIDDEN)))+ add_gguf("blk.%d.ffn_gate_exps.weight" % layer, m2d([N_EXPERTS, INTER, HIDDEN]), 1,+ _f16_bytes(rng.normal(size=N_EXPERTS * INTER * HIDDEN)))+ add_gguf("blk.%d.ffn_up_exps.weight" % layer, m2d([N_EXPERTS, INTER, HIDDEN]), 1,+ _f16_bytes(rng.normal(size=N_EXPERTS * INTER * HIDDEN)))+ add_gguf("blk.%d.ffn_down_exps.weight" % layer, m2d([N_EXPERTS, HIDDEN, INTER]), 1,+ _f16_bytes(rng.normal(size=N_EXPERTS * INTER * HIDDEN)))+ if extra_tensor:+ tensors.append(extra_tensor)++ metadata = [+ (METADATA_STRING, "general.architecture", arch),+ (METADATA_UINT32, "general.alignment", 32),+ (METADATA_UINT32, "olmoe.embedding_length", HIDDEN),+ (METADATA_UINT32, "olmoe.block_count", LAYERS),+ (METADATA_UINT32, "olmoe.attention.head_count", HEADS),+ (METADATA_UINT32, "olmoe.attention.head_count_kv", HEADS),+ (METADATA_UINT32, "olmoe.expert_count", N_EXPERTS),+ (METADATA_UINT32, "olmoe.expert_used_count", 2),+ (METADATA_UINT32, "olmoe.feed_forward_length", INTER),+ (METADATA_FLOAT32, "olmoe.rope.freq_base", 10000.0),+ (METADATA_FLOAT32, "olmoe.attention.layer_norm_rms_epsilon", 1e-5),+ (METADATA_ARRAY, "tokenizer.ggml.tokens",+ (METADATA_STRING, ["aa", "bb", "<unk>", "Ġt"])),+ (METADATA_ARRAY, "tokenizer.ggml.token_type",+ (METADATA_UINT32, [1, 1, 3, 4])),+ (METADATA_ARRAY, "tokenizer.ggml.merges",+ (METADATA_STRING, ["Ġ t"])),+ (METADATA_UINT32, "tokenizer.ggml.bos_token_id", 2),+ (METADATA_UINT32, "tokenizer.ggml.eos_token_id", 2),+ (METADATA_BOOL, "tokenizer.ggml.add_bos_token", False),+ (METADATA_BOOL, "tokenizer.ggml.add_eos_token", False),+ ]+ return write_gguf(path, metadata, tensors)+++def write_gguf(path, metadata, tensors):+ alignment = 32+ out = b"GGUF"+ out += struct.pack("<I", 3)+ out += struct.pack("<Q", len(tensors))+ out += struct.pack("<Q", len(metadata))+ for value_type, key, value in metadata:+ out += _encode_string(key)+ out += struct.pack("<I", value_type)+ out += _encode_value(value_type, value)++ data_offset = 0+ data_base = _align(len(out) + sum(+ 8 + len(t["name"].encode("utf-8")) + 4 + 8 * len(t["dims"]) + 4 + 8+ for t in tensors), alignment)+ for tensor in tensors:+ out += _encode_string(tensor["name"])+ out += struct.pack("<I", len(tensor["dims"]))+ for dim in tensor["dims"]:+ out += struct.pack("<Q", dim)+ out += struct.pack("<I", tensor["ggml_type"])+ out += struct.pack("<Q", data_offset)+ data_offset += len(tensor["payload"])+ data_offset = _align(data_offset, alignment)++ out += b"\x00" * (data_base - len(out))+ for tensor in tensors:+ out += tensor["payload"]+ out += b"\x00" * (_align(len(tensor["payload"]), alignment) - len(tensor["payload"]))+ Path(path).write_bytes(out)+++def run_converter(input_path, output_path, *extra):+ return subprocess.run(+ [sys.executable, str(TOOLS / "convert_gguf_to_olmoe.py"),+ "--input", str(input_path), "--output", str(output_path), *extra],+ cwd=str(TOOLS.parent), capture_output=True, text=True,+ timeout=120)+++def _rope_permute(w, n_head, n_kv_head=None):+ """llama.cpp's q/k RoPE permutation (conversion/llama.py), for the fixture."""+ n = n_kv_head if (n_kv_head and n_head != n_kv_head) else n_head+ return np.ascontiguousarray(+ w.reshape((n, 2, w.shape[0] // n // 2) + w.shape[1:]).swapaxes(1, 2).reshape(w.shape))+++def expected_names():+ names = {"model.embed_tokens.weight", "lm_head.weight", "model.norm.weight"}+ for layer in range(LAYERS):+ base = "model.layers.%d" % layer+ names.update({+ base + ".input_layernorm.weight",+ base + ".post_attention_layernorm.weight",+ base + ".self_attn.q_proj.weight",+ base + ".self_attn.k_proj.weight",+ base + ".self_attn.v_proj.weight",+ base + ".self_attn.o_proj.weight",+ base + ".self_attn.q_norm.weight",+ base + ".self_attn.k_norm.weight",+ base + ".mlp.gate.weight",+ })+ for layer in range(LAYERS):+ for expert in range(N_EXPERTS):+ names.add(base_merge_name(layer, expert))+ names.add("model.layers.%d.mlp.experts.%d.qs" % (layer, expert))+ return names+++def base_merge_name(layer, expert):+ return "model.layers.%d.mlp.experts.%d.merged_weight" % (layer, expert)+++class ConverterEndToEndTest(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.addCleanup(self.tmp.cleanup)+ self.dir = Path(self.tmp.name)+ self.gguf = self.dir / "model.gguf"+ self.out = self.dir / "out"+ build_tiny_olmoe_gguf(str(self.gguf))++ def test_converts_full_container(self):+ result = run_converter(self.gguf, self.out)+ self.assertEqual(result.returncode, 0, result.stderr)++ config = json.loads((self.out / "config.json").read_text())+ self.assertEqual(config["hidden_size"], HIDDEN)+ self.assertEqual(config["num_experts"], N_EXPERTS)+ self.assertEqual(config["vocab_size"], 4)+ self.assertFalse(config["norm_topk_prob"])++ tokenizer = json.loads((self.out / "tokenizer.json").read_text())+ self.assertEqual(tokenizer["model"]["vocab"], {"aa": 0, "bb": 1, "<unk>": 2, "Ġt": 3})++ present = set()+ from safetensors import safe_open+ for shard in sorted(self.out.glob("model-*.safetensors")):+ with safe_open(str(shard), framework="np") as handle:+ present.update(handle.keys())++ self.assertEqual(present, expected_names())++ with safe_open(str(next(self.out.glob("model-*.safetensors"))), framework="np") as handle:+ merged = handle.get_tensor(base_merge_name(0, 0))+ qs = handle.get_tensor("model.layers.0.mlp.experts.0.qs")+ self.assertEqual(merged.dtype, np.int8)+ self.assertEqual(qs.dtype, np.float32)+ self.assertEqual(merged.size, INTER * HIDDEN * 2 + HIDDEN * INTER)+ self.assertEqual(qs.size, INTER + INTER + HIDDEN)++ def test_dry_run_creates_nothing(self):+ result = run_converter(self.gguf, self.out, "--dry-run")+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertFalse(self.out.exists())++ def test_refuses_wrong_architecture(self):+ wrong = self.dir / "wrong.gguf"+ build_tiny_olmoe_gguf(str(wrong), arch="llama")+ result = run_converter(wrong, self.out)+ self.assertEqual(result.returncode, 1)+ self.assertIn("general.architecture", result.stderr)+ self.assertFalse(self.out.exists())++ def test_refuses_unknown_tensor(self):+ weird = self.dir / "weird.gguf"+ build_tiny_olmoe_gguf(str(weird), extra_tensor={+ "name": "blk.0.not_a_real_tensor",+ "dims": [HIDDEN],+ "ggml_type": 0,+ "payload": _f32_bytes(np.zeros(HIDDEN)),+ })+ result = run_converter(weird, self.out)+ self.assertEqual(result.returncode, 1)+ self.assertIn("unrecognized tensors", result.stderr)++ def test_remove_source_only_on_success(self):+ result = run_converter(self.gguf, self.out, "--remove-source-file")+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertFalse(self.gguf.exists())+ self.assertTrue((self.out / "config.json").is_file())++ def test_qk_left_as_stored_by_default(self):+ rng = np.random.default_rng(99)+ hf_q_proj = rng.normal(size=(HIDDEN, HIDDEN)).astype(np.float16)+ permuted_q_proj = _rope_permute(hf_q_proj, HEADS)+ self.assertFalse(np.array_equal(permuted_q_proj, hf_q_proj))+ build_tiny_olmoe_gguf(str(self.gguf), override={+ "blk.0.attn_q.weight": permuted_q_proj.tobytes(),+ })++ result = run_converter(self.gguf, self.out)+ self.assertEqual(result.returncode, 0, result.stderr)+ got = self._read("model.layers.0.self_attn.q_proj.weight")+ # no flag: the stored (permuted) layout is passed through unchanged+ np.testing.assert_array_equal(got, permuted_q_proj)++ def test_qk_permuted_flag_undoes_rope_permutation(self):+ rng = np.random.default_rng(99)+ hf_q_proj = rng.normal(size=(HIDDEN, HIDDEN)).astype(np.float16)+ hf_k_proj = rng.normal(size=(HIDDEN, HIDDEN)).astype(np.float16)+ build_tiny_olmoe_gguf(str(self.gguf), override={+ "blk.0.attn_q.weight": _rope_permute(hf_q_proj, HEADS).tobytes(),+ "blk.0.attn_k.weight": _rope_permute(hf_k_proj, HEADS).tobytes(),+ })++ result = run_converter(self.gguf, self.out, "--qk-permuted")+ self.assertEqual(result.returncode, 0, result.stderr)+ np.testing.assert_array_equal(+ self._read("model.layers.0.self_attn.q_proj.weight"), hf_q_proj)+ np.testing.assert_array_equal(+ self._read("model.layers.0.self_attn.k_proj.weight"), hf_k_proj)++ def _read(self, name):+ from safetensors import safe_open+ for shard in sorted(self.out.glob("model-*.safetensors")):+ with safe_open(str(shard), framework="np") as handle:+ if name in handle.keys():+ return handle.get_tensor(name)+ raise KeyError(name)++ def test_overwrite_rewrites_existing_container(self):+ self.assertEqual(run_converter(self.gguf, self.out).returncode, 0)+ (self.out / "model-99999.safetensors").write_bytes(b"stale")+ result = run_converter(self.gguf, self.out, "--overwrite")+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertFalse((self.out / "model-99999.safetensors").exists())++ def test_refuses_existing_container_without_flag(self):+ self.assertEqual(run_converter(self.gguf, self.out).returncode, 0)+ before = {p.name: p.read_bytes() for p in self.out.glob("model-*.safetensors")}+ result = run_converter(self.gguf, self.out)+ self.assertEqual(result.returncode, 1)+ self.assertIn("already contains a converted container", result.stderr)+ self.assertIn("--overwrite", result.stderr)+ after = {p.name: p.read_bytes() for p in self.out.glob("model-*.safetensors")}+ self.assertEqual(before, after)++ def test_dense_storage_dtypes(self):+ self.assertEqual(+ convert_gguf_to_olmoe._dense_to_storage(np.zeros(HIDDEN, np.float32)).dtype,+ np.float32)+ self.assertEqual(+ convert_gguf_to_olmoe._dense_to_storage(np.zeros((HIDDEN, HIDDEN), np.float32)).dtype,+ np.float16)+++if __name__ == "__main__":+ unittest.main()\ No newline at end of filediff --git a/c/tests/test_gguf_dequant.py b/c/tests/test_gguf_dequant.pynew file mode 100644index 00000000..2e3df9dc--- /dev/null+++ b/c/tests/test_gguf_dequant.py@@ -0,0 +1,121 @@+#!/usr/bin/env python3+"""tools/gguf_dequant.py: ggml block dequantization, pinned to ggml's math.++Two independent checks:+ * committed golden vectors (tests/fixtures/gguf_dequant_golden.npz), generated+ with llama.cpp's gguf-py reference on deterministic random blocks -- run in+ CI with only numpy;+ * if the gguf package is importable, a live comparison against gguf.quants.+ dequantize on fresh random blocks, so drift from the reference is caught.++Also covers the scalar types (F32/F16/BF16) and the refusal paths (unsupported+type, numel/byte-count mismatch).+"""++import sys+import unittest+from pathlib import Path++import numpy as np++sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))++import gguf_reader+from gguf_reader import GGUFError+from gguf_dequant import (+ dequantize,+ dequantize_q2_K,+ dequantize_q3_K,+ dequantize_q6_K,+)++GOLDEN = Path(__file__).resolve().parent / "fixtures" / "gguf_dequant_golden.npz"++BLOCK_CASES = [+ ("Q2_K", gguf_reader.GGML_TYPE_Q2_K, 84, dequantize_q2_K),+ ("Q3_K", gguf_reader.GGML_TYPE_Q3_K, 110, dequantize_q3_K),+ ("Q6_K", gguf_reader.GGML_TYPE_Q6_K, 210, dequantize_q6_K),+]+++class GoldenDequantTest(unittest.TestCase):+ def setUp(self):+ if not GOLDEN.is_file():+ self.skipTest("golden fixture missing: %s" % GOLDEN)+ self.golden = np.load(str(GOLDEN))++ def test_block_types_match_golden(self):+ for name, ggml_type, block_bytes, _func in BLOCK_CASES:+ raw = self.golden["%s_raw" % name]+ expected = self.golden["%s_expected" % name].reshape(-1)+ got = dequantize(raw.tobytes(), ggml_type, expected.size)+ self.assertEqual(got.shape, expected.shape)+ np.testing.assert_allclose(got, expected, rtol=0, atol=0, equal_nan=True,+ err_msg="%s dequant mismatch" % name)++ def test_dispatcher_uses_same_functions(self):+ for name, ggml_type, _bb, func in BLOCK_CASES:+ raw = self.golden["%s_raw" % name]+ direct = func(raw).reshape(-1)+ dispatched = dequantize(raw.tobytes(), ggml_type, direct.size)+ np.testing.assert_array_equal(dispatched, direct)+++class ScalarDequantTest(unittest.TestCase):+ def test_f32(self):+ values = np.array([0.0, 1.5, -2.25, 3.5e10], dtype=np.float32)+ got = dequantize(values.tobytes(), gguf_reader.GGML_TYPE_F32, values.size)+ np.testing.assert_array_equal(got, values)++ def test_f16(self):+ values = np.array([0.0, 1.5, -2.25, 0.125], dtype=np.float16)+ got = dequantize(values.tobytes(), gguf_reader.GGML_TYPE_F16, values.size)+ np.testing.assert_array_equal(got, values.astype(np.float32))++ def test_bf16(self):+ u16 = np.array([0x0000, 0x3FC0, 0xC040, 0x3E00], dtype=np.uint16)+ got = dequantize(u16.tobytes(), gguf_reader.GGML_TYPE_BF16, u16.size)+ expected = (u16.astype(np.uint32) << 16).view(np.float32)+ np.testing.assert_array_equal(got, expected)+++class DequantRefusalTest(unittest.TestCase):+ def test_unsupported_type(self):+ with self.assertRaisesRegex(GGUFError, "unsupported ggml type"):+ dequantize(b"\x00" * 16, 100, 16)++ def test_numel_not_multiple_of_block(self):+ with self.assertRaisesRegex(GGUFError, "not a multiple of block"):+ dequantize(b"\x00" * 84, gguf_reader.GGML_TYPE_Q2_K, 255)++ def test_byte_count_mismatch(self):+ with self.assertRaisesRegex(GGUFError, "expected"):+ dequantize(b"\x00" * 84, gguf_reader.GGML_TYPE_Q2_K, 512)+++class LiveReferenceTest(unittest.TestCase):+ def test_matches_gguf_py_on_random_blocks(self):+ try:+ import gguf+ from gguf.quants import dequantize as reference+ except ImportError as exc:+ self.skipTest("gguf package not installed: %s" % exc)++ rng = np.random.default_rng(20260825)+ mapping = {+ "Q2_K": (gguf.GGMLQuantizationType.Q2_K, gguf_reader.GGML_TYPE_Q2_K, 84),+ "Q3_K": (gguf.GGMLQuantizationType.Q3_K, gguf_reader.GGML_TYPE_Q3_K, 110),+ "Q6_K": (gguf.GGMLQuantizationType.Q6_K, gguf_reader.GGML_TYPE_Q6_K, 210),+ }+ for name, (ref_type, our_type, block_bytes) in mapping.items():+ for _ in range(25):+ nb = int(rng.integers(1, 5))+ raw = rng.integers(0, 256, size=nb * block_bytes, dtype=np.uint8)+ expected = reference(raw.reshape(nb, block_bytes).copy(), ref_type).reshape(-1)+ got = dequantize(raw.tobytes(), our_type, nb * 256)+ np.testing.assert_allclose(got, expected, rtol=0, atol=0, equal_nan=True,+ err_msg="%s live mismatch" % name)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_gguf_olmoe_profile.py b/c/tests/test_gguf_olmoe_profile.pynew file mode 100644index 00000000..82ae7508--- /dev/null+++ b/c/tests/test_gguf_olmoe_profile.py@@ -0,0 +1,255 @@+#!/usr/bin/env python3+"""tools/gguf_olmoe_profile.py: OLMoE name mapping, layout and int8 quantization.++The strongest check here is test_matches_reference_quantizer: our numpy+quantize_row is compared against the torch implementation in+tools/convert_olmoe_merged.py, which is the code the existing OLMoE container+was built with. If the two disagree, the new GGUF converter would produce a+container the engine reads differently from the known-good one.+"""++import sys+import unittest+from pathlib import Path++import numpy as np++sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))++import gguf_olmoe_profile as profile+++class MappingTest(unittest.TestCase):+ def test_parse_expert(self):+ self.assertEqual(profile.parse_expert("blk.3.ffn_gate_exps.weight"), (3, "gate"))+ self.assertEqual(profile.parse_expert("blk.0.ffn_down_exps.weight"), (0, "down"))+ self.assertIsNone(profile.parse_expert("blk.0.attn_q.weight"))+ self.assertIsNone(profile.parse_expert("token_embd.weight"))++ def test_dense_target(self):+ self.assertEqual(profile.dense_target("token_embd.weight"),+ "model.embed_tokens.weight")+ self.assertEqual(profile.dense_target("output.weight"), "lm_head.weight")+ self.assertEqual(profile.dense_target("output_norm.weight"), "model.norm.weight")+ self.assertEqual(profile.dense_target("blk.5.attn_q.weight"),+ "model.layers.5.self_attn.q_proj.weight")+ self.assertEqual(profile.dense_target("blk.11.ffn_gate_inp.weight"),+ "model.layers.11.mlp.gate.weight")+ self.assertEqual(profile.dense_target("blk.0.ffn_norm.weight"),+ "model.layers.0.post_attention_layernorm.weight")+ self.assertIsNone(profile.dense_target("blk.0.some_unknown.weight"))+++class LayoutTest(unittest.TestCase):+ def test_to_logical_reverses_dims(self):+ flat = np.arange(6, dtype=np.float32)+ logical = profile.to_logical(flat, [3, 2])+ self.assertEqual(logical.shape, (2, 3))+ np.testing.assert_array_equal(logical, flat.reshape(2, 3))++ def test_to_logical_1d_is_identity(self):+ flat = np.arange(5, dtype=np.float32)+ np.testing.assert_array_equal(profile.to_logical(flat, [5]), flat)++ def test_expert_matrix_shape(self):+ self.assertEqual(profile.expert_matrix_shape([2048, 1024, 64]), (1024, 2048))+ self.assertEqual(profile.expert_matrix_shape([1024, 2048, 64]), (2048, 1024))++ def test_restore_rope_layout_inverts_llama_permutation(self):+ def forward(w, n_head, n_kv=None):+ n = n_kv if (n_kv and n_head != n_kv) else n_head+ return np.ascontiguousarray(+ w.reshape((n, 2, w.shape[0] // n // 2) + w.shape[1:]).swapaxes(1, 2).reshape(w.shape))++ rng = np.random.default_rng(11)+ for (out, n_head, n_kv) in ((16, 4, 4), (24, 4, 2), (12, 2, None), (64, 4, None)):+ hf = rng.normal(size=(out, 3)).astype(np.float32)+ permuted = forward(hf, n_head, n_kv)+ if out // (n_kv if (n_kv and n_head != n_kv) else n_head) // 2 != 2:+ self.assertFalse(np.array_equal(permuted, hf))+ restored = profile.restore_rope_layout(permuted, n_head, n_kv)+ np.testing.assert_array_equal(restored, hf)++ def test_restore_rope_layout_matches_torch_reference(self):+ try:+ import torch+ except ImportError as exc:+ self.skipTest("torch unavailable: %s" % exc)+ for (out, n_head, n_kv) in ((16, 4, 4), (24, 4, 2), (12, 2, None)):+ rng = np.random.default_rng(out)+ hf = torch.from_numpy(rng.normal(size=(out, 7)).astype(np.float32))+ n = n_kv if (n_kv and n_head != n_kv) else n_head+ permuted = (hf.reshape(n, 2, out // n // 2, *hf.shape[1:])+ .swapaxes(1, 2).reshape(hf.shape))+ restored = profile.restore_rope_layout(permuted.numpy(), n_head, n_kv)+ np.testing.assert_array_equal(restored, hf.numpy())+++class QuantizationTest(unittest.TestCase):+ def test_quantize_row_math(self):+ w = np.array([[1.0, -2.0, 0.5], [0.0, 0.0, 0.0]], dtype=np.float32)+ q, s = profile.quantize_row(w)+ self.assertEqual(q.dtype, np.int8)+ self.assertEqual(q.shape, (2, 3))+ self.assertEqual(s.shape, (2,))+ # row 0: absmax 2 -> scale 2/127, q = round(w/scale)+ self.assertAlmostEqual(float(s[0]), 2.0 / 127.0, places=7)+ expected = np.clip(np.rint(w[0] / s[0]), -128, 127).astype(np.int8)+ np.testing.assert_array_equal(q[0], expected)+ # reconstruction W ~= q*scale+ np.testing.assert_allclose(q.astype(np.float32) * s[:, None], w, atol=2.0 / 127.0)++ def test_quantize_row_rejects_non_2d(self):+ with self.assertRaises(ValueError):+ profile.quantize_row(np.zeros(8, dtype=np.float32))++ def test_matches_reference_quantizer(self):+ try:+ import torch+ import convert_olmoe_merged as reference+ except ImportError as exc:+ self.skipTest("torch/convert_olmoe_merged unavailable: %s" % exc)++ rng = np.random.default_rng(20260825)+ for _ in range(20):+ rows = int(rng.integers(1, 8))+ cols = int(rng.integers(1, 40))+ w = rng.normal(0.0, 1.0, size=(rows, cols)).astype(np.float32)+ q_ref, s_ref = reference.quantize_row(torch.from_numpy(w))+ q_got, s_got = profile.quantize_row(w)+ np.testing.assert_array_equal(q_got, q_ref.cpu().numpy())+ np.testing.assert_allclose(s_got, s_ref.cpu().numpy(), rtol=0, atol=0)++ def test_merge_expert_layout(self):+ rng = np.random.default_rng(7)+ hidden, inter = 4, 6+ gate = rng.normal(0, 1, (inter, hidden)).astype(np.float32)+ up = rng.normal(0, 1, (inter, hidden)).astype(np.float32)+ down = rng.normal(0, 1, (hidden, inter)).astype(np.float32)+ merged, scales = profile.merge_expert(gate, up, down)++ qg, sg = profile.quantize_row(gate)+ qu, su = profile.quantize_row(up)+ qd, sd = profile.quantize_row(down)+ np.testing.assert_array_equal(+ merged, np.concatenate((qg.reshape(-1), qu.reshape(-1), qd.reshape(-1))))+ np.testing.assert_array_equal(+ scales, np.concatenate((sg, su, sd)))+ self.assertEqual(merged.dtype, np.int8)+ self.assertEqual(scales.dtype, np.float32)+ self.assertEqual(merged.size, inter * hidden * 2 + hidden * inter)+ self.assertEqual(scales.size, inter + inter + hidden)+++class ConfigTest(unittest.TestCase):+ def _meta(self):+ return {+ "olmoe.embedding_length": 2048,+ "olmoe.block_count": 16,+ "olmoe.attention.head_count": 16,+ "olmoe.attention.head_count_kv": 16,+ "olmoe.expert_count": 64,+ "olmoe.expert_used_count": 8,+ "olmoe.feed_forward_length": 1024,+ "olmoe.rope.freq_base": 10000.0,+ "olmoe.attention.layer_norm_rms_epsilon": 1e-5,+ "tokenizer.ggml.tokens": ["a"] * 50304,+ }++ def test_build_config(self):+ config = profile.build_config(self._meta())+ self.assertEqual(config["hidden_size"], 2048)+ self.assertEqual(config["num_hidden_layers"], 16)+ self.assertEqual(config["num_attention_heads"], 16)+ self.assertEqual(config["num_key_value_heads"], 16)+ self.assertEqual(config["num_experts"], 64)+ self.assertEqual(config["num_experts_per_tok"], 8)+ self.assertEqual(config["intermediate_size"], 1024)+ self.assertEqual(config["vocab_size"], 50304)+ self.assertFalse(config["norm_topk_prob"])+ self.assertEqual(config["model_type"], "olmoe")+ self.assertEqual(config["architectures"], ["OlmoeForCausalLM"])++ def test_build_config_norm_topk_prob_override(self):+ config = profile.build_config(self._meta(), norm_topk_prob=True)+ self.assertTrue(config["norm_topk_prob"])++ def test_build_config_missing_key(self):+ meta = self._meta()+ del meta["olmoe.expert_count"]+ with self.assertRaises(ValueError):+ profile.build_config(meta)++ def test_build_config_missing_tokens(self):+ meta = self._meta()+ del meta["tokenizer.ggml.tokens"]+ with self.assertRaises(ValueError):+ profile.build_config(meta)+++class TokenizerTest(unittest.TestCase):+ def _meta(self):+ return {+ "tokenizer.ggml.tokens": ["hello", " world", "<unk>", "Ġt", "a"],+ "tokenizer.ggml.merges": ["Ġ t", "Ġ Ġ"],+ "tokenizer.ggml.token_type": [1, 1, 3, 4, 6],+ "tokenizer.ggml.bos_token_id": 2,+ "tokenizer.ggml.eos_token_id": 2,+ "tokenizer.ggml.padding_token_id": 0,+ "tokenizer.ggml.add_bos_token": False,+ "tokenizer.ggml.add_eos_token": False,+ "tokenizer.chat_template": "{{ bos_token }}...",+ }++ def test_vocab_and_merges(self):+ tokenizer = profile.build_tokenizer(self._meta())+ self.assertEqual(tokenizer["model"]["type"], "BPE")+ self.assertEqual(tokenizer["model"]["vocab"], {+ "hello": 0, " world": 1, "<unk>": 2, "Ġt": 3, "a": 4})+ self.assertEqual(tokenizer["model"]["merges"], ["Ġ t", "Ġ Ġ"])++ def test_added_tokens_only_special_and_user_defined(self):+ tokenizer = profile.build_tokenizer(self._meta())+ self.assertEqual(tokenizer["added_tokens"], [+ {"id": 2, "content": "<unk>", "special": True},+ {"id": 3, "content": "Ġt", "special": False},+ ])++ def test_special_ids(self):+ tokenizer = profile.build_tokenizer(self._meta())+ self.assertEqual(tokenizer["bos_token"], "<unk>")+ self.assertEqual(tokenizer["eos_token"], "<unk>")+ self.assertEqual(tokenizer["pad_token"], "hello")+ self.assertFalse(tokenizer["add_bos_token"])+ self.assertFalse(tokenizer["add_eos_token"])+ self.assertEqual(tokenizer["chat_template"], "{{ bos_token }}...")++ def test_missing_tokens_raises(self):+ meta = self._meta()+ del meta["tokenizer.ggml.tokens"]+ with self.assertRaises(ValueError):+ profile.build_tokenizer(meta)++ def test_byte_token_not_added(self):+ meta = self._meta()+ meta["tokenizer.ggml.token_type"] = [6, 6, 6, 6, 6]+ tokenizer = profile.build_tokenizer(meta)+ self.assertEqual(tokenizer["added_tokens"], [])++ def test_missing_token_type_defaults_to_normal(self):+ meta = self._meta()+ del meta["tokenizer.ggml.token_type"]+ tokenizer = profile.build_tokenizer(meta)+ self.assertEqual(tokenizer["added_tokens"], [])++ def test_partial_token_type_defaults_to_normal(self):+ meta = self._meta()+ meta["tokenizer.ggml.token_type"] = [3] # only the first token is typed+ tokenizer = profile.build_tokenizer(meta)+ self.assertEqual(tokenizer["added_tokens"], [+ {"id": 0, "content": "hello", "special": True},+ ])+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/convert_gguf_to_olmoe.py b/c/tools/convert_gguf_to_olmoe.pynew file mode 100644index 00000000..89cb5782--- /dev/null+++ b/c/tools/convert_gguf_to_olmoe.py@@ -0,0 +1,389 @@+#!/usr/bin/env python3+"""Convert a GGUF OLMoE checkpoint into a colibri container for c/olmoe.c.++Flow (one user-facing entry point):+ wrapper (this file) -> gguf_reader (structure) -> gguf_dequant (numbers)+ -> gguf_olmoe_profile (OLMoE mapping/layout/quant)+ -> safetensors shards + config.json + tokenizer.json++Output, per tensor, matches what c/olmoe.c loads:+ * dense tensors dequantized to F16 (1-D norms kept F32), names remapped;+ * routed experts dequantized and re-quantized to int8 row-wise, merged into+ model.layers.N.mlp.experts.E.merged_weight (I8) + .qs (F32);+ * config.json reconstructed from the GGUF olmoe.* metadata. GGUF does not+ carry norm_topk_prob; the default is false (matching the official OLMoE+ checkpoints) and --norm-topk-prob overrides it. q/k are taken as stored+ (HuggingFace layout); --qk-permuted undoes llama.cpp's RoPE permutation for+ GGUFs that carry it;+ * tokenizer.json rebuilt from tokenizer.ggml.* (byte-level GPT-2), validated+ token-for-token against the official OLMoE tokenizer, so the container is+ usable for chat/serve, not only loadable for validation.++Safety:+ * refuses a non-OLMoE GGUF, unknown tensor names, and missing required tensors;+ * refuses to touch an --output that already contains a container unless the user+ passes --overwrite (rebuild from scratch); it never silently skips work;+ * writes shards atomically (tmp + rename), so a crash leaves no half shard;+ * --remove-source-file deletes the source GGUF ONLY after a fully successful+ conversion (never on a partial run);+ * --dry-run prints the plan without writing anything or creating the output+ directory.++Usage:+ python tools/convert_gguf_to_olmoe.py \\+ --input /path/to/model.gguf # or a dir holding exactly one .gguf+ --output /path/to/olmoe-colibri # created if missing+"""++import argparse+import json+import os+import sys+import time+from pathlib import Path++import numpy as np++from gguf_reader import GGUFReader+from gguf_dequant import dequantize+import gguf_olmoe_profile as profile++try:+ from safetensors.numpy import save_file+except ImportError as exc:+ sys.exit("Missing dependency: %s. Install: pip install numpy safetensors" % exc)+++class ConversionError(Exception):+ pass+++def resolve_sources(model):+ path = Path(model).expanduser()+ if path.is_dir():+ files = sorted(p for p in path.iterdir() if p.suffix == ".gguf")+ if not files:+ raise ConversionError("no .gguf files in %s" % path)+ if len(files) > 1:+ raise ConversionError(+ "%s holds %d .gguf files; OLMoE is converted from a single file "+ "for now, pass one explicitly" % (path, len(files)))+ return [files[0]]+ if not path.is_file():+ raise ConversionError("model not found: %s" % path)+ return [path]+++def required_names(config):+ names = {+ "model.embed_tokens.weight",+ "lm_head.weight",+ "model.norm.weight",+ }+ for layer in range(config["num_hidden_layers"]):+ base = "model.layers.%d" % layer+ names.update({+ base + ".input_layernorm.weight",+ base + ".post_attention_layernorm.weight",+ base + ".self_attn.q_proj.weight",+ base + ".self_attn.k_proj.weight",+ base + ".self_attn.v_proj.weight",+ base + ".self_attn.o_proj.weight",+ base + ".self_attn.q_norm.weight",+ base + ".self_attn.k_norm.weight",+ base + ".mlp.gate.weight",+ })+ return names+++def plan_tensors(reader):+ """Return (dense, experts, unmapped) planned from the GGUF tensor list."""+ dense = []+ experts = {}+ unmapped = []+ for tensor in reader.tensors:+ name = tensor["name"]+ parsed = profile.parse_expert(name)+ if parsed is not None:+ layer, kind = parsed+ experts.setdefault(layer, {})[kind] = tensor+ continue+ target = profile.dense_target(name)+ if target is None:+ unmapped.append(name)+ continue+ dense.append((target, tensor))+ return dense, experts, unmapped+++class OutputWriter:+ def __init__(self, out_dir, flush_every):+ self.out_dir = out_dir+ self.flush_every = flush_every+ self.buf = {}+ self.shard_idx = 0++ def add_many(self, items):+ """Buffer related tensors and flush once the buffer reaches flush_every.++ Contract: every tensor of one logical unit -- e.g. an expert's int8+ merged_weight AND its f32 .qs -- is passed in a single call, so a flush+ never splits the pair across shards during a normal run.+ """+ for name, array in items:+ self.buf[name] = np.ascontiguousarray(array)+ if len(self.buf) >= self.flush_every:+ self.flush()++ def flush(self):+ if not self.buf:+ return+ destination = self.out_dir / ("model-%05d.safetensors" % self.shard_idx)+ temporary = Path(str(destination) + ".tmp")+ temporary.unlink(missing_ok=True)+ try:+ save_file(self.buf, str(temporary))+ os.replace(temporary, destination)+ finally:+ temporary.unlink(missing_ok=True)+ print(" wrote %s (%d tensors)" % (destination.name, len(self.buf)), flush=True)+ self.buf = {}+ self.shard_idx += 1+++def _write_json_atomic(path, payload):+ temporary = Path(str(path) + ".tmp")+ temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n",+ encoding="utf-8")+ os.replace(temporary, path)+++def _dense_to_storage(logical):+ if logical.ndim == 1:+ return logical.astype(np.float32)+ return logical.astype(np.float16)+++def convert_dense(reader, dense, writer, config, qk_permuted):+ for target, tensor in dense:+ raw = reader.read_tensor(tensor)+ flat = dequantize(raw, tensor["ggml_type"], tensor["numel"])+ logical = profile.to_logical(flat, tensor["dims"])+ if qk_permuted:+ # Some GGUF producers (recent llama.cpp olmo.py) RoPE-permute q/k;+ # c/olmoe.c uses the plain HuggingFace layout, so undo it.+ name = tensor["name"]+ if name.endswith("attn_q.weight"):+ logical = profile.restore_rope_layout(logical, config["num_attention_heads"],+ config["num_attention_heads"])+ elif name.endswith("attn_k.weight"):+ logical = profile.restore_rope_layout(logical, config["num_attention_heads"],+ config["num_key_value_heads"])+ writer.add_many(((target, _dense_to_storage(logical)),))+++def convert_experts(reader, experts, writer, config):+ layer_count = config["num_hidden_layers"]+ expert_count = config["num_experts"]+ hidden = config["hidden_size"]+ inter = config["intermediate_size"]+ want_w = inter * hidden + inter * hidden + hidden * inter+ want_s = inter + inter + hidden+ total = layer_count * expert_count+ done_experts = 0++ for layer in range(layer_count):+ kinds = experts.get(layer)+ if not kinds or set(kinds) != {"gate", "up", "down"}:+ raise ConversionError("layer %d: incomplete expert trio: %s"+ % (layer, sorted(kinds) if kinds else None))+ raw_bytes, expert_layout = {}, {}+ for kind in ("gate", "up", "down"):+ tensor = kinds[kind]+ raw_bytes[kind] = reader.read_tensor(tensor)+ expert_layout[kind] = (profile.expert_matrix_shape(tensor["dims"]),+ tensor["ggml_type"],+ profile.expert_bytes_per_slice(tensor["dims"], tensor["ggml_type"]))+ for expert in range(expert_count):+ merged_name = "model.layers.%d.mlp.experts.%d.merged_weight" % (layer, expert)+ qs_name = "model.layers.%d.mlp.experts.%d.qs" % (layer, expert)+ per_kind = {}+ for kind in ("gate", "up", "down"):+ expert_shape, ggml_type, chunk = expert_layout[kind]+ start = expert * chunk+ flat = dequantize(raw_bytes[kind][start:start + chunk], ggml_type,+ expert_shape[0] * expert_shape[1])+ per_kind[kind] = flat.reshape(expert_shape)+ merged, scales = profile.merge_expert(per_kind["gate"], per_kind["up"], per_kind["down"])+ if merged.size != want_w or scales.size != want_s:+ raise ConversionError(+ "expert [%d,%d] produced %d bytes / %d scales, expected %d / %d"+ % (layer, expert, merged.size, scales.size, want_w, want_s))+ # The int8 weights and their f32 scales are ONE logical unit: one+ # add_many call keeps them in the same shard (see OutputWriter).+ writer.add_many(((merged_name, merged), (qs_name, scales)))+ done_experts += 1+ print(" layer %d/%d experts done (%d/%d)"+ % (layer + 1, layer_count, done_experts, total), flush=True)+++def _format_duration(seconds):+ seconds = int(round(seconds))+ if seconds < 60:+ return "%ds" % seconds+ minutes, secs = divmod(seconds, 60)+ if minutes < 60:+ return "%dm %02ds" % (minutes, secs)+ hours, minutes = divmod(minutes, 60)+ return "%dh %02dm %02ds" % (hours, minutes, secs)+++def run(args):+ started = time.perf_counter()+ sources = resolve_sources(args.input)+ source = sources[0]+ out_dir = Path(args.output).expanduser()++ print("colibri: converting a GGUF OLMoE checkpoint into a colibri container", flush=True)+ print(" flow: %s" % source.name, flush=True)+ print(" -> gguf_reader (header/metadata/tensor index)", flush=True)+ print(" -> gguf_dequant + gguf_olmoe_profile (dequant, layout, int8 experts)", flush=True)+ print(" -> safetensors shards + config.json + tokenizer.json", flush=True)+ print(" input: %s" % source, flush=True)+ print(" output: %s" % out_dir, flush=True)++ reader = GGUFReader(str(source))+ try:+ arch = reader.metadata.get("general.architecture")+ if arch != "olmoe":+ raise ConversionError("general.architecture is %r, expected 'olmoe'" % arch)+ config = profile.build_config(reader.metadata,+ norm_topk_prob=args.norm_topk_prob)+ dense, experts, unmapped = plan_tensors(reader)+ if unmapped:+ raise ConversionError("unrecognized tensors: %s" % ", ".join(sorted(unmapped)[:8]))++ present = {target for target, _ in dense}+ for layer, kinds in experts.items():+ for expert in range(config["num_experts"]):+ present.add("model.layers.%d.mlp.experts.%d.merged_weight" % (layer, expert))+ missing = sorted(required_names(config) - present)+ if missing:+ raise ConversionError("missing required tensors: %s" % ", ".join(missing[:8]))++ experts_total = config["num_hidden_layers"] * config["num_experts"]+ if args.dry_run:+ print("dry-run: %s" % source)+ print(" architecture: olmoe, %d layers, %d experts (top-%d), hidden %d, inter %d, vocab %d"+ % (config["num_hidden_layers"], config["num_experts"],+ config["num_experts_per_tok"], config["hidden_size"],+ config["intermediate_size"], config["vocab_size"]))+ print(" dense tensors: %d" % len(dense))+ print(" expert tensors: %d (%d merged_weight + %d qs)"+ % (experts_total * 2, experts_total, experts_total))+ if out_dir.exists() and any(out_dir.glob("model-*.safetensors")):+ print(" note: output already contains a container; --overwrite rebuilds it")+ return 0++ existing_shards = sorted(out_dir.glob("model-*.safetensors"))+ if existing_shards and not args.overwrite:+ raise ConversionError(+ "output already contains a converted container (%d safetensors shard(s)):\n"+ " %s\n"+ "refusing to overwrite it. Pass --overwrite to rebuild it from scratch."+ % (len(existing_shards), out_dir))++ out_dir.mkdir(parents=True, exist_ok=True)+ if args.overwrite:+ for path in list(out_dir.glob("model-*.safetensors")):+ path.unlink()+ for name in ("config.json", "tokenizer.json"):+ sidecar = out_dir / name+ if sidecar.is_file():+ sidecar.unlink()+ print(" --overwrite: cleared previous container outputs", flush=True)+ writer = OutputWriter(out_dir, args.flush_every)+ _write_json_atomic(out_dir / "config.json", config)+ print(" wrote config.json", flush=True)+ tokenizer = profile.build_tokenizer(reader.metadata)+ _write_json_atomic(out_dir / "tokenizer.json", tokenizer)+ print(" wrote tokenizer.json (%d vocab, %d merges, %d added)"+ % (len(tokenizer["model"]["vocab"]),+ len(tokenizer["model"].get("merges", [])),+ len(tokenizer["added_tokens"])), flush=True)+ convert_dense(reader, dense, writer, config, args.qk_permuted)+ convert_experts(reader, experts, writer, config)+ writer.flush()+ finally:+ reader.close()++ if args.remove_source_file:+ os.remove(source)+ print(" removed source %s" % source, flush=True)+ print("Done:")+ print("- path: %s" % out_dir)+ print("- time: %s" % _format_duration(time.perf_counter() - started))+ return 0+++def main():+ parser = argparse.ArgumentParser(+ prog="convert_gguf_to_olmoe.py",+ description="Convert an OLMoE GGUF checkpoint into a colibri container "+ "readable by the c/olmoe.c engine (safetensors shards + "+ "config.json + tokenizer.json).",+ epilog=(+ "examples:\n"+ " convert_gguf_to_olmoe.py --input model.gguf --output ./olmoe-colibri\n"+ " convert_gguf_to_olmoe.py --input ./gguf-dir --output ~/llm/olmoe --dry-run\n"+ " convert_gguf_to_olmoe.py --input model.gguf --output ./olmoe --remove-source-file\n"+ "\n"+ "notes:\n"+ " --input may be a single .gguf file or a directory holding exactly one\n"+ " .gguf (more than one is refused: OLMoE is converted from one file).\n"+ " --output is created (with any missing parent) if it does not exist.\n"+ " If --output already holds a container, the run REFUSES with a message;\n"+ " pass --overwrite to rebuild it from scratch.\n"+ " --remove-source-file deletes the input .gguf only after the whole\n"+ " conversion finished successfully; it never removes on a partial run.\n"+ " --dry-run prints the plan, writes nothing and creates no directory.\n"+ " The engine can load the result for validation; chat/serve work because\n"+ " tokenizer.json is reconstructed from the GGUF tokenizer metadata.\n"),+ formatter_class=argparse.RawDescriptionHelpFormatter,+ )+ parser.add_argument("--input", required=True, metavar="PATH",+ help="source GGUF file, or a directory holding exactly one .gguf")+ parser.add_argument("--output", required=True, metavar="DIR",+ help="destination directory for the colibri container "+ "(created, including missing parents, if absent)")+ parser.add_argument("--dry-run", action="store_true",+ help="print the conversion plan (layers, experts, output dir) "+ "and exit without writing anything or creating --output")+ parser.add_argument("--remove-source-file", action="store_true",+ help="after a fully successful conversion, delete the source "+ ".gguf (never on a partial or failed run)")+ parser.add_argument("--overwrite", action="store_true",+ help="rebuild an existing container in --output from scratch. "+ "Without it, the run refuses when --output already holds "+ "model-*.safetensors")+ parser.add_argument("--norm-topk-prob", action="store_true",+ help="set config.json norm_topk_prob=true (renormalize top-k router "+ "weights). GGUF does not carry this field; the default is false, "+ "which matches the official OLMoE checkpoints")+ parser.add_argument("--qk-permuted", action="store_true",+ help="undo llama.cpp's RoPE permutation of q_proj/k_proj. Off by "+ "default because the official OLMoE GGUF is already in the "+ "HuggingFace layout; use it only for GGUFs that were permuted")+ parser.add_argument("--flush-every", type=int, default=512, metavar="N",+ help="flush an output safetensors shard every N tensors "+ "(default: 512; smaller = more, smaller shards)")+ args = parser.parse_args()+ try:+ return run(args)+ except ConversionError as exc:+ sys.stderr.write("ERROR: %s\n" % exc)+ return 1+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/c/tools/gguf_dequant.py b/c/tools/gguf_dequant.pynew file mode 100644index 00000000..24bd56b8--- /dev/null+++ b/c/tools/gguf_dequant.py@@ -0,0 +1,195 @@+#!/usr/bin/env python3+"""Dequantization of GGUF/ggml tensor blocks to float32, pure numpy.++Ported block-for-block from ggml-org/ggml src/ggml-quants.c (dequantize_row_*),+using the block layouts in src/ggml-common.h. This module owns the numerical+decode used by the converter profiles; gguf_reader.py owns structure only.++Only the types listed in _DEQUANTIZERS are implemented. dequantize() raises+GGUFError for any other type, so a profile never silently mis-decodes bytes.+Correctness is pinned by tests/test_gguf_dequant.py, which compares against+llama.cpp's gguf-py reference on random blocks and against committed vectors.+"""++import numpy as np++from gguf_reader import (+ GGML_TYPE_BF16,+ GGML_TYPE_F16,+ GGML_TYPE_F32,+ GGML_TYPE_Q2_K,+ GGML_TYPE_Q3_K,+ GGML_TYPE_Q6_K,+ GGUFError,+ ggml_block_spec,+ ggml_type_name,+)++QK_K = 256+_MASK_KMASK1 = 0x03030303+_MASK_KMASK2 = 0x0F0F0F0F+++def _f16_to_f32(pair_bytes):+ """(n, 2) uint8 or (n,) uint16 -> float32."""+ u16 = pair_bytes.view("<u2").reshape(-1)+ return u16.view(np.float16).astype(np.float32)+++def _bf16_to_f32(word_bytes):+ u16 = word_bytes.view("<u2").reshape(-1).astype(np.uint32)+ return (u16 << 16).view(np.float32)+++def dequantize_q2_K(blocks):+ # Q2_K (ggml-common.h block_q2_K, 84 bytes/256 elems):+ # scales[16] (per-16 low nibble=scale, high nibble=min, 4-bit),+ # qs[64] (2-bit packed quants), then fp16 block scale (d) and min (dmin).+ num_blocks = blocks.shape[0]+ scales = blocks[:, 0:16].astype(np.int32)+ packed_q = blocks[:, 16:80]+ block_scale = _f16_to_f32(blocks[:, 80:82])+ block_min = _f16_to_f32(blocks[:, 82:84])+ out = np.empty((num_blocks, QK_K), dtype=np.float32)+ is_ = 0+ for half in range(2):+ base = half * 32+ for j in range(4):+ shift = j * 2+ for sub in range(2):+ sc = scales[:, is_]+ is_ += 1+ group_scale = block_scale * (sc & 0xF)+ group_min = block_min * (sc >> 4)+ cols = base + sub * 16 + np.arange(16)+ qv = ((packed_q[:, cols] >> shift) & 3).astype(np.float32)+ lo = half * 128 + j * 32 + sub * 16+ out[:, lo:lo + 16] = group_scale[:, None] * qv - group_min[:, None]+ return out+++def dequantize_q3_K(blocks):+ # Q3_K (ggml-common.h block_q3_K, 110 bytes/256 elems):+ # hmask[32] high bits, qs[64] low 2 bits, scales[12] (sixteen 6-bit scales+ # packed into 12 bytes, reassembled by the kmask shuffle below), fp16 scale.+ num_blocks = blocks.shape[0]+ hmask = blocks[:, 0:32]+ packed_q = blocks[:, 32:96]+ scale_bytes = blocks[:, 96:108]+ block_scale = _f16_to_f32(blocks[:, 108:110])++ # Sixteen 6-bit scales live packed in 12 bytes: three 32-bit words, then a+ # bit-shuffle reassembles them (kmask1/kmask2 from ggml-quants.c).+ a0 = (scale_bytes[:, 0].astype(np.uint32) | (scale_bytes[:, 1].astype(np.uint32) << 8)+ | (scale_bytes[:, 2].astype(np.uint32) << 16) | (scale_bytes[:, 3].astype(np.uint32) << 24))+ a1 = (scale_bytes[:, 4].astype(np.uint32) | (scale_bytes[:, 5].astype(np.uint32) << 8)+ | (scale_bytes[:, 6].astype(np.uint32) << 16) | (scale_bytes[:, 7].astype(np.uint32) << 24))+ a2 = (scale_bytes[:, 8].astype(np.uint32) | (scale_bytes[:, 9].astype(np.uint32) << 8)+ | (scale_bytes[:, 10].astype(np.uint32) << 16) | (scale_bytes[:, 11].astype(np.uint32) << 24))+ tmp = a2+ a2n = ((a0 >> 4) & _MASK_KMASK2) | (((tmp >> 4) & _MASK_KMASK1) << 4)+ a3n = ((a1 >> 4) & _MASK_KMASK2) | (((tmp >> 6) & _MASK_KMASK1) << 4)+ a0n = (a0 & _MASK_KMASK2) | (((tmp >> 0) & _MASK_KMASK1) << 4)+ a1n = (a1 & _MASK_KMASK2) | (((tmp >> 2) & _MASK_KMASK1) << 4)++ row_scales = np.empty((num_blocks, 16), dtype=np.int8)+ for i, word in enumerate((a0n, a1n, a2n, a3n)):+ row_scales[:, i * 4:(i + 1) * 4] = word.astype("<u4").view(np.uint8).reshape(num_blocks, 4)++ out = np.empty((num_blocks, QK_K), dtype=np.float32)+ is_ = 0+ for half in range(2):+ base = half * 32+ for j in range(4):+ shift = j * 2+ m = 1 << (half * 4 + j)+ for sub in range(2):+ group_scale = block_scale * (row_scales[:, is_] - 32).astype(np.float32)+ is_ += 1+ qcols = base + sub * 16 + np.arange(16)+ hcols = sub * 16 + np.arange(16)+ qv = ((packed_q[:, qcols] >> shift) & 3).astype(np.float32)+ high_bit = ((hmask[:, hcols] & m) != 0).astype(np.float32)+ # low 2-bit code, minus 4 when the high bit of the mask is clear+ centered_q = qv - 4.0 * (1.0 - high_bit)+ lo = half * 128 + j * 32 + sub * 16+ out[:, lo:lo + 16] = group_scale[:, None] * centered_q+ return out+++def dequantize_q6_K(blocks):+ # Q6_K (ggml-common.h block_q6_K, 210 bytes/256 elems):+ # ql[128] low 4 bits, qh[64] upper 2 bits, scales[16] int8 per-16 scales,+ # fp16 block scale. Two 128-elem halves; codes are (low | high<<4) - 32.+ num_blocks = blocks.shape[0]+ packed_lo = blocks[:, 0:128]+ packed_hi = blocks[:, 128:192]+ row_scales = blocks[:, 192:208].view(np.int8)+ block_scale = _f16_to_f32(blocks[:, 208:210])+ out = np.empty((num_blocks, QK_K), dtype=np.float32)+ for half in range(2):+ lo_base = half * 64+ hi_base = half * 32+ scale_base = half * 8+ l = np.arange(32)+ group = l // 16+ code0 = ((packed_lo[:, lo_base + l] & 0xF).astype(np.int32)+ | (((packed_hi[:, hi_base + l] >> 0) & 3).astype(np.int32) << 4)) - 32+ code1 = ((packed_lo[:, lo_base + l + 32] & 0xF).astype(np.int32)+ | (((packed_hi[:, hi_base + l] >> 2) & 3).astype(np.int32) << 4)) - 32+ code2 = ((packed_lo[:, lo_base + l] >> 4).astype(np.int32)+ | (((packed_hi[:, hi_base + l] >> 4) & 3).astype(np.int32) << 4)) - 32+ code3 = ((packed_lo[:, lo_base + l + 32] >> 4).astype(np.int32)+ | (((packed_hi[:, hi_base + l] >> 6) & 3).astype(np.int32) << 4)) - 32+ scale0 = row_scales[:, scale_base + group + 0].astype(np.float32)+ scale1 = row_scales[:, scale_base + group + 2].astype(np.float32)+ scale2 = row_scales[:, scale_base + group + 4].astype(np.float32)+ scale3 = row_scales[:, scale_base + group + 6].astype(np.float32)+ base = half * 128+ out[:, base + l] = block_scale[:, None] * scale0 * code0.astype(np.float32)+ out[:, base + l + 32] = block_scale[:, None] * scale1 * code1.astype(np.float32)+ out[:, base + l + 64] = block_scale[:, None] * scale2 * code2.astype(np.float32)+ out[:, base + l + 96] = block_scale[:, None] * scale3 * code3.astype(np.float32)+ return out+++_DEQUANTIZERS = {+ GGML_TYPE_Q2_K: dequantize_q2_K,+ GGML_TYPE_Q3_K: dequantize_q3_K,+ GGML_TYPE_Q6_K: dequantize_q6_K,+}+++def dequantize(raw, ggml_type, numel):+ """Decode a tensor's stored bytes to a flat float32 array (GGUF element order)."""+ if ggml_type in _DEQUANTIZERS:+ block_elems, block_bytes = ggml_block_spec(ggml_type)+ if numel % block_elems != 0:+ raise GGUFError(+ "dequant %s: numel %d is not a multiple of block %d"+ % (ggml_type_name(ggml_type), numel, block_elems))+ nblocks = numel // block_elems+ data = np.frombuffer(raw, dtype=np.uint8)+ if data.size != nblocks * block_bytes:+ raise GGUFError(+ "dequant %s: %d bytes for %d blocks of %d, expected %d"+ % (ggml_type_name(ggml_type), data.size, nblocks, block_bytes,+ nblocks * block_bytes))+ blocks = data.reshape(nblocks, block_bytes)+ return _DEQUANTIZERS[ggml_type](blocks).reshape(-1)++ element_size = {GGML_TYPE_F32: 4, GGML_TYPE_F16: 2, GGML_TYPE_BF16: 2}.get(ggml_type)+ if element_size is not None:+ expected = numel * element_size+ if len(raw) != expected:+ raise GGUFError(+ "dequant %s: %d bytes for %d elements, expected %d"+ % (ggml_type_name(ggml_type), len(raw), numel, expected))+ if ggml_type == GGML_TYPE_F32:+ return np.frombuffer(raw, dtype="<f4").astype(np.float32)+ if ggml_type == GGML_TYPE_F16:+ return np.frombuffer(raw, dtype="<f2").astype(np.float32)+ return _bf16_to_f32(np.frombuffer(raw, dtype=np.uint8))++ raise GGUFError("dequant: unsupported ggml type %d (%s)"+ % (ggml_type, ggml_type_name(ggml_type)))diff --git a/c/tools/gguf_olmoe_profile.py b/c/tools/gguf_olmoe_profile.pynew file mode 100644index 00000000..f0a755ba--- /dev/null+++ b/c/tools/gguf_olmoe_profile.py@@ -0,0 +1,238 @@+#!/usr/bin/env python3+"""OLMoE profile: GGUF tensor layout -> colibri container layout.++This module owns everything OLMoE-specific about the conversion and nothing+about file I/O or CLI:++ * name mapping from GGUF (llama.cpp) tensor names to the names c/olmoe.c+ loads: model.embed_tokens.weight, model.layers.N.self_attn.q_proj.weight,+ and per-expert model.layers.N.mlp.experts.E.{merged_weight,qs};+ * dequantized GGUF order -> logical HuggingFace order (GGUF stores dims+ reversed);+ * row-wise int8 quantization identical to olmoe.c's decoder contract+ W[o,i] ~= q[o,i]*scale[o] (same math as tools/convert_olmoe_merged.py);+ * per-expert gate|up|down merge into one merged_weight + one .qs;+ * config.json reconstruction from GGUF olmoe.* metadata.++It is imported by convert_gguf_to_olmoe.py (CLI) and unit-tested directly.+"""++import re++import numpy as np++EXPERT_RE = re.compile(r"^blk\.(\d+)\.ffn_(gate|up|down)_exps\.weight$")++DENSE_SIMPLE = {+ "token_embd.weight": "model.embed_tokens.weight",+ "output.weight": "lm_head.weight",+ "output_norm.weight": "model.norm.weight",+}++DENSE_LAYERED_RE = re.compile(+ r"^blk\.(\d+)\.(attn_norm|ffn_norm|attn_q|attn_k|attn_v|attn_output|"+ r"attn_q_norm|attn_k_norm|ffn_gate_inp)\.weight$")++LAYER_SUFFIX = {+ "attn_norm": "input_layernorm.weight",+ "ffn_norm": "post_attention_layernorm.weight",+ "attn_q": "self_attn.q_proj.weight",+ "attn_k": "self_attn.k_proj.weight",+ "attn_v": "self_attn.v_proj.weight",+ "attn_output": "self_attn.o_proj.weight",+ "attn_q_norm": "self_attn.q_norm.weight",+ "attn_k_norm": "self_attn.k_norm.weight",+ "ffn_gate_inp": "mlp.gate.weight",+}++CONFIG_KEYS = {+ "olmoe.embedding_length": "hidden_size",+ "olmoe.block_count": "num_hidden_layers",+ "olmoe.attention.head_count": "num_attention_heads",+ "olmoe.attention.head_count_kv": "num_key_value_heads",+ "olmoe.expert_count": "num_experts",+ "olmoe.expert_used_count": "num_experts_per_tok",+ "olmoe.feed_forward_length": "intermediate_size",+ "olmoe.rope.freq_base": "rope_theta",+ "olmoe.attention.layer_norm_rms_epsilon": "rms_norm_eps",+}+++def parse_expert(name):+ """(layer, kind) for an expert tensor name, or None."""+ match = EXPERT_RE.match(name)+ if not match:+ return None+ return int(match.group(1)), match.group(2)+++def dense_target(name):+ """colibri name for a GGUF dense tensor, or None if not a known dense tensor."""+ if name in DENSE_SIMPLE:+ return DENSE_SIMPLE[name]+ match = DENSE_LAYERED_RE.match(name)+ if match:+ layer, kind = int(match.group(1)), match.group(2)+ return "model.layers.%d.%s" % (layer, LAYER_SUFFIX[kind])+ return None+++def to_logical(flat, gguf_dims):+ """Reshape GGUF-order flat values into the logical (HuggingFace) shape.++ GGUF stores the source tensor's bytes unchanged and only reverses the dims+ labels, so the flat data is already C-order of the original shape: reshape+ to reversed(gguf_dims) reproduces it. No transpose is involved.+ """+ return np.ascontiguousarray(flat.reshape(tuple(reversed(gguf_dims))))+++def restore_rope_layout(weights, n_head, n_kv_head=None):+ """Undo llama.cpp's RoPE permutation of q_proj/k_proj weights.++ conversion/olmo.py permutes q_proj and k_proj exactly like Llama:+ forward: w.reshape(n, 2, out // n // 2, *rest).swapaxes(1, 2).reshape(w.shape)+ with n = n_kv_head when it differs from n_head (k_proj), else n_head. That+ is NOT an involution unless out // n // 2 == 2, so the inverse is the axis+ transpose of the (m, 2) decomposition:+ inverse: w.reshape(n, m, 2, *rest).swapaxes(1, 2).reshape(w.shape)+ with m = out // n // 2. c/olmoe.c uses HF-style RoPE (adjacent pairs per+ head), so the GGUF's permuted q/k must be restored to the HF layout.+ """+ n = n_kv_head if (n_kv_head and n_head != n_kv_head) else n_head+ w = np.asarray(weights)+ if w.ndim < 1 or n < 1 or w.shape[0] % n != 0 or (w.shape[0] // n) % 2 != 0:+ raise ValueError("cannot restore RoPE layout for shape %r with %d heads"+ % (w.shape, n))+ m = w.shape[0] // n // 2+ rest = w.shape[1:]+ return np.ascontiguousarray(+ w.reshape((n, m, 2) + rest).swapaxes(1, 2).reshape(w.shape))+++def quantize_row(weights):+ """Row-wise int8 quantization, identical math to c/olmoe.c's decoder.++ olmoe.c computes y[o] = scale[o] * sum_i x[i]*q[o,i], so the stored bytes+ must satisfy W[o,i] ~= q[o,i]*scale[o] with scale[o] = row_absmax/127.+ Rounding is numpy's rint (round-half-to-even), matching convert_olmoe_merged.py.+ """+ w = np.asarray(weights, dtype=np.float32)+ if w.ndim != 2:+ raise ValueError("quantize_row expects a 2-D weight, got shape %r" % (w.shape,))+ row_max = np.maximum(np.max(np.abs(w), axis=1, keepdims=True), np.float32(1e-12))+ scales = (row_max / np.float32(127.0)).astype(np.float32)+ quantized = np.clip(np.rint(w / scales), -128.0, 127.0).astype(np.int8)+ return quantized, scales.reshape(-1)+++def merge_expert(gate, up, down):+ """Quantize and merge one expert's gate/up/down into colibri's layout.++ merged_weight = int8(gate | up | down) flattened;+ qs = f32(gate rows | up rows | down rows).+ gate/up: [inter, hidden]; down: [hidden, inter].+ """+ q_gate, s_gate = quantize_row(gate)+ q_up, s_up = quantize_row(up)+ q_down, s_down = quantize_row(down)+ merged = np.concatenate((q_gate.reshape(-1), q_up.reshape(-1), q_down.reshape(-1)))+ scales = np.concatenate((s_gate, s_up, s_down)).astype(np.float32)+ return merged, scales+++def expert_matrix_shape(gguf_dims):+ """HF shape of one expert's matrix inside a fused [d0, d1, E] expert tensor."""+ return (gguf_dims[1], gguf_dims[0])+++def expert_bytes_per_slice(gguf_dims, ggml_type):+ """Bytes of one expert inside a fused expert tensor (contiguous slice)."""+ from gguf_reader import ggml_block_spec+ block_elems, block_bytes = ggml_block_spec(ggml_type)+ if gguf_dims[0] % block_elems != 0:+ raise ValueError(+ "expert tensor dims[0]=%d is not a multiple of the block %d"+ % (gguf_dims[0], block_elems))+ row_blocks = gguf_dims[0] // block_elems+ return row_blocks * block_bytes * gguf_dims[1]+++def _token_type_of(meta, index):+ entries = meta.get("tokenizer.ggml.token_type")+ if not entries or index >= len(entries):+ return 1+ return int(entries[index])+++def build_tokenizer(meta):+ """Reconstruct an HF-style tokenizer.json from GGUF tokenizer metadata.++ Feed shape matches what c/tok.h loads for GPT-2/byte-level BPE:+ model.vocab (token string -> id), model.merges (space-separated "left+ right" entries), added_tokens (id/content/special), pre_tokenizer that+ does not select the o200k/Kimi maybe-flags, plus bos/eos/pad and the chat+ template. token_type 3=CONTROL (special) and 4=USER_DEFINED become added+ tokens; 5=UNUSED and 6=BYTE stay plain vocab entries.+ """+ tokens = meta.get("tokenizer.ggml.tokens")+ if not tokens:+ raise ValueError("GGUF metadata missing tokenizer.ggml.tokens")+ merges = meta.get("tokenizer.ggml.merges")++ vocab = {}+ added = []+ # ggml token_type: 1=NORMAL stays plain BPE vocab; 3=CONTROL and+ # 4=USER_DEFINED become added tokens (atomic, literal); 5=UNUSED and 6=BYTE+ # stay plain vocab entries (not atomic specials).+ for index, token in enumerate(tokens):+ vocab[token] = index+ kind = _token_type_of(meta, index)+ if kind in (3, 4):+ added.append({"id": index, "content": token, "special": kind == 3})++ model = {"type": "BPE", "vocab": vocab}+ if merges:+ model["merges"] = list(merges)++ tokenizer = {+ "add_bos_token": bool(meta.get("tokenizer.ggml.add_bos_token")),+ "add_eos_token": bool(meta.get("tokenizer.ggml.add_eos_token")),+ "model": model,+ "pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": False},+ "added_tokens": added,+ }+ for gguf_key, hf_key in (+ ("tokenizer.ggml.bos_token_id", "bos_token"),+ ("tokenizer.ggml.eos_token_id", "eos_token"),+ ("tokenizer.ggml.padding_token_id", "pad_token")):+ token_id = meta.get(gguf_key)+ if token_id is not None and int(token_id) < len(tokens):+ tokenizer[hf_key] = tokens[int(token_id)]+ if "tokenizer.chat_template" in meta:+ tokenizer["chat_template"] = meta["tokenizer.chat_template"]+ return tokenizer+++def build_config(meta, norm_topk_prob=False, extra=None):+ """Reconstruct the config.json c/olmoe.c loads from GGUF metadata.++ norm_topk_prob is NOT in GGUF metadata; the official OLMoE checkpoints use+ false (transformers' OlmoeConfig default), so that is the default here. Use+ the CLI flag to override for a checkpoint that trained with it on.+ """+ config = {}+ for gguf_key, cfg_key in CONFIG_KEYS.items():+ if gguf_key not in meta:+ raise ValueError("GGUF metadata missing %s" % gguf_key)+ config[cfg_key] = meta[gguf_key]+ tokens = meta.get("tokenizer.ggml.tokens")+ if not tokens:+ raise ValueError("GGUF metadata missing tokenizer.ggml.tokens (needed for vocab_size)")+ config["vocab_size"] = len(tokens)+ config["norm_topk_prob"] = bool(norm_topk_prob)+ config["model_type"] = "olmoe"+ config["architectures"] = ["OlmoeForCausalLM"]+ if extra:+ config.update(extra)+ return config