Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tools(qwen36): snap-view generator to run qwen36 over a qpack container
Léon Avic Simmons307eb993ba77
diff --git a/c/tests/test_make_qwen36_qpack_snap.py b/c/tests/test_make_qwen36_qpack_snap.pynew file mode 100644index 00000000..b49bef42--- /dev/null+++ b/c/tests/test_make_qwen36_qpack_snap.py@@ -0,0 +1,274 @@+"""make_qwen36_qpack_snap: snap view over a qpack container, plus the C round-trip.++Builds a synthetic MLX-affine container (nested multimodal config.json with+per-module quantization overrides, a hand-written model.safetensors with+packed U32 triples) and checks:++ * the view's config.json is the FLATTENED text config (load_cfg reads+ hidden_size at the top level);+ * qwen36_meta.json derives head dims from the LOGICAL weight shapes --+ o_proj's input axis is stored packed (uint32 words), so a derivation+ that forgets to unpack reports o_in 8x too small and fails here;+ * layer_types / n_active / DeltaNet dims / rope_theta (nested under+ rope_parameters) survive the trip;+ * model.safetensors and tokenizer.json are links into the container, and+ the container itself is never written;+ * refusal when --out points inside the container.++Round-trip (the independent cross-implementation evidence): the fixture+carries a seeded-random Q4 tensor whose dequantized rows are computed HERE in+pure Python (bf16 scalars decoded exactly, scale*q + bias per group) and+handed to tests/test_qwen36_dense_affine's compare mode, which loads the SAME+bytes through the engine's own st_init + load_t_n path -- through the snap+view's symlink, exactly as a real run reads them. Skips loudly if the C gate+binary is not built (`make test-c` builds it first in a green `make test`).+"""+import json+import os+import random+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent+C_DIR = HERE.parent+TOOL = C_DIR / "tools" / "make_qwen36_qpack_snap.py"+GATE = C_DIR / "tests" / ("test_qwen36_dense_affine.exe" if os.name == "nt"+ else "test_qwen36_dense_affine")+++def bf16_bits(value: float) -> int:+ return struct.unpack("<I", struct.pack("<f", value))[0] >> 16+++def bf16_value(bits: int) -> float:+ return struct.unpack("<f", struct.pack("<I", bits << 16))[0]+++def write_safetensors(path: Path, tensors: list[tuple[str, str, list[int], bytes]]):+ header, offset = {}, 0+ for name, dtype, shape, blob in tensors:+ header[name] = {"dtype": dtype, "shape": shape,+ "data_offsets": [offset, offset + len(blob)]}+ offset += len(blob)+ encoded = json.dumps(header).encode("utf-8")+ with open(path, "wb") as f:+ f.write(struct.pack("<Q", len(encoded)))+ f.write(encoded)+ for _, _, _, blob in tensors:+ f.write(blob)+++class MakeQwen36QpackSnapTest(unittest.TestCase):+ HIDDEN, N_LAYERS = 32, 4+ N_Q, N_KV, HEAD_DIM = 2, 1, 8+ Q_HEAD_DIM = 16 # attn_output_gate: q_proj carries value+gate+ GS = 16 # container-wide affine group size++ @classmethod+ def setUpClass(cls):+ cls.tmp = tempfile.TemporaryDirectory()+ root = Path(cls.tmp.name)+ cls.container = root / "tiny.qpack"+ cls.view = root / "snap-view"+ cls.container.mkdir()+ cls.rng = random.Random(42)+ cls._write_config()+ cls._write_model()+ (cls.container / "tokenizer.json").write_text(+ json.dumps({"model": {"vocab": {"a": 0}}}), encoding="utf-8")+ (cls.container / "manifest.json").write_text(+ json.dumps({"magic": "QPACK", "version": 1,+ "modelName": "tiny", "sourceCheckpoint": "tiny"}),+ encoding="utf-8")+ cls.before = sorted(p.name for p in cls.container.iterdir())+ result = subprocess.run(+ [sys.executable, str(TOOL), "--container", str(cls.container),+ "--out", str(cls.view)],+ capture_output=True, text=True, cwd=C_DIR)+ assert result.returncode == 0, result.stderr + result.stdout+ cls.tool_output = result.stdout++ @classmethod+ def tearDownClass(cls):+ cls.tmp.cleanup()++ # ---- fixture ---------------------------------------------------------++ @classmethod+ def _write_config(cls):+ text_config = {+ "model_type": "qwen3_5_moe_text",+ "hidden_size": cls.HIDDEN,+ "num_hidden_layers": cls.N_LAYERS,+ "vocab_size": 64,+ "rms_norm_eps": 1e-6,+ "layer_types": ["linear_attention"] * 3 + ["full_attention"],+ "num_attention_heads": cls.N_Q,+ "num_key_value_heads": cls.N_KV,+ "head_dim": cls.HEAD_DIM,+ "attn_output_gate": True,+ "partial_rotary_factor": 0.25,+ "num_experts": 8,+ "num_experts_per_tok": 2,+ "moe_intermediate_size": 16,+ "shared_expert_intermediate_size": 16,+ "linear_num_value_heads": 2,+ "linear_num_key_heads": 1,+ "linear_key_head_dim": 8,+ "linear_value_head_dim": 8,+ "linear_conv_kernel_dim": 4,+ "rope_parameters": {"rope_theta": 12345678.0,+ "mrope_section": [2, 1, 1]},+ }+ config = {+ "model_type": "qwen3_5_moe",+ "architectures": ["Qwen3_5MoeForConditionalGeneration"],+ "text_config": text_config,+ "quantization": {+ "bits": 4, "group_size": cls.GS, "mode": "affine",+ "language_model.model.layers.3.mlp.gate":+ {"bits": 8, "group_size": cls.GS},+ },+ }+ (cls.container / "config.json").write_text(json.dumps(config),+ encoding="utf-8")++ @classmethod+ def _quantized(cls, name: str, rows: int, cols: int, bits: int):+ """Author a random affine triple; returns (tensors, logical rows)."""+ per_word, mask = 32 // bits, (1 << bits) - 1+ words = [cls.rng.getrandbits(32) for _ in range(rows * cols // per_word)]+ groups = cols // cls.GS+ scales = [bf16_bits(cls.rng.uniform(0.01, 0.2))+ for _ in range(rows * groups)]+ biases = [bf16_bits(cls.rng.uniform(-1.0, 1.0))+ for _ in range(rows * groups)]+ logical = []+ for r in range(rows):+ row = []+ for c in range(cols):+ word = words[r * (cols // per_word) + c // per_word]+ q = (word >> (bits * (c % per_word))) & mask+ g = r * groups + c // cls.GS+ row.append(bf16_value(scales[g]) * q + bf16_value(biases[g]))+ logical.append(row)+ tensors = [+ (name + ".weight", "U32", [rows, cols // per_word],+ struct.pack(f"<{len(words)}I", *words)),+ (name + ".scales", "BF16", [rows, groups],+ struct.pack(f"<{len(scales)}H", *scales)),+ (name + ".biases", "BF16", [rows, groups],+ struct.pack(f"<{len(biases)}H", *biases)),+ ]+ return tensors, logical++ @classmethod+ def _write_model(cls):+ p = "language_model.model.layers.3.self_attn."+ tensors = []+ for proj, rows in (("q_proj", cls.N_Q * cls.Q_HEAD_DIM),+ ("k_proj", cls.N_KV * cls.HEAD_DIM),+ ("v_proj", cls.N_KV * cls.HEAD_DIM)):+ t, _ = cls._quantized(p + proj, rows, cls.HIDDEN, bits=4)+ tensors += t+ # o_proj input axis = n_q*head_dim = 16 LOGICAL columns -> 2 packed+ # words; the o_in derivation must report 16, not 2.+ t, _ = cls._quantized(p + "o_proj", cls.HIDDEN,+ cls.N_Q * cls.HEAD_DIM, bits=4)+ tensors += t+ tensors.append((p + "q_norm.weight", "BF16", [cls.HEAD_DIM],+ struct.pack(f"<{cls.HEAD_DIM}H",+ *[bf16_bits(1.0)] * cls.HEAD_DIM)))+ # Q8 override module: 8 experts x hidden router.+ t, _ = cls._quantized("language_model.model.layers.3.mlp.gate",+ 8, cls.HIDDEN, bits=8)+ tensors += t+ # Round-trip tensor for the C compare-mode gate.+ t, cls.roundtrip = cls._quantized("language_model.model.roundtrip",+ 4, cls.HIDDEN, bits=4)+ tensors += t+ write_safetensors(cls.container / "model.safetensors", tensors)++ # ---- gates -----------------------------------------------------------++ def test_flat_config(self):+ cfg = json.loads((self.view / "config.json").read_text())+ self.assertEqual(cfg["hidden_size"], self.HIDDEN)+ self.assertEqual(cfg["num_hidden_layers"], self.N_LAYERS)+ self.assertEqual(cfg["vocab_size"], 64)+ self.assertNotIn("text_config", cfg)+ hf = json.loads((self.view / "config.hf.json").read_text())+ self.assertIn("text_config", hf)++ def test_meta_head_dims_unpacked(self):+ meta = json.loads((self.view / "qwen36_meta.json").read_text())+ self.assertEqual(meta["q_heads"], self.N_Q)+ self.assertEqual(meta["kv_heads"], self.N_KV)+ self.assertEqual(meta["q_head_dim"], self.Q_HEAD_DIM)+ self.assertEqual(meta["k_head_dim"], self.HEAD_DIM)+ self.assertEqual(meta["v_head_dim"], self.HEAD_DIM)+ self.assertEqual(meta["o_in"], self.N_Q * self.HEAD_DIM) # unpacked!+ self.assertEqual(meta["head_dim"], self.HEAD_DIM)+ self.assertEqual(meta["qk_rope_head_dim"], self.HEAD_DIM)++ def test_meta_layers_dn_rope(self):+ meta = json.loads((self.view / "qwen36_meta.json").read_text())+ self.assertEqual(meta["n_layers"], self.N_LAYERS)+ self.assertEqual(meta["n_active"], 1)+ self.assertEqual(meta["layer_types"],+ ["linear_attention"] * 3 + ["full_attention"])+ self.assertEqual(meta["num_experts"], 8)+ self.assertEqual(meta["topk"], 2)+ self.assertEqual(meta["moe_inter"], 16)+ self.assertEqual(meta["dn_vheads"], 2)+ self.assertEqual(meta["dn_kheads"], 1)+ self.assertEqual(meta["dn_conv_dim"], 2 * 1 * 8 + 2 * 8)+ self.assertEqual(meta["rope_theta"], 12345678.0)+ self.assertTrue(meta["attn_output_gate"])+ self.assertEqual(meta["expert_gs"], 0)+ # MLX-derived containers carry full-gamma norm weights; the engine+ # must be told to undo the shift (load_norm_n) or the model is noise.+ self.assertIs(meta["zero_centered_norms"], False)++ def test_links_and_readonly_container(self):+ for name in ("model.safetensors", "tokenizer.json"):+ link = self.view / name+ self.assertTrue(link.exists(), name)+ self.assertEqual(os.stat(link).st_ino,+ os.stat(self.container / name).st_ino,+ f"{name} is a copy, not a link")+ after = sorted(p.name for p in self.container.iterdir())+ self.assertEqual(self.before, after, "container was written to")++ def test_refuses_out_inside_container(self):+ result = subprocess.run(+ [sys.executable, str(TOOL), "--container", str(self.container),+ "--out", str(self.container / "view")],+ capture_output=True, text=True, cwd=C_DIR)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("OUTSIDE", result.stderr + result.stdout)++ def test_roundtrip_against_engine_loader(self):+ if not GATE.exists():+ self.skipTest(f"{GATE} not built (run `make test-c` first): the "+ "C-vs-Python round-trip was NOT checked")+ expected = struct.pack(+ f"<{4 * self.HIDDEN}f",+ *[v for row in self.roundtrip for v in row])+ expected_path = Path(self.tmp.name) / "roundtrip.f32"+ expected_path.write_bytes(expected)+ result = subprocess.run(+ [str(GATE), str(self.view), "model.roundtrip.weight",+ str(4 * self.HIDDEN), str(expected_path)],+ capture_output=True, text=True, cwd=C_DIR)+ self.assertEqual(result.returncode, 0,+ result.stderr + result.stdout)+ self.assertIn("round-trip", result.stdout)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/make_qwen36_qpack_snap.py b/c/tools/make_qwen36_qpack_snap.pynew file mode 100644index 00000000..4d4237a0--- /dev/null+++ b/c/tools/make_qwen36_qpack_snap.py@@ -0,0 +1,248 @@+#!/usr/bin/env python3+"""Build a qwen36 snap view over a Swiftlet qpack container (READ-ONLY source).++The engine's contract is the one convert_qwen36.py established: a snapshot+directory with a FLAT config.json, a qwen36_meta.json whose head dims were+derived from the actual weight shapes, the *.safetensors holding the dense+weights, and tokenizer.json. A Swiftlet qpack container already carries all+of the bytes -- model.safetensors (dense half, MLX affine triples that+c/qwen36.c expands at load), tokenizer.json, and a nested multimodal+config.json -- but not the two engine-side metadata files, and the container+itself must never be written to (it is a production artifact shared with+Swiftlet, hash-pinned by hashes.json).++So the bridge is a VIEW: a fresh directory holding the generated flat+config.json + qwen36_meta.json next to symlinks into the container. No+tensor bytes are copied or converted on disk; st_init follows the symlink and+the engine's affine dense loader does the expansion in memory.++Usage:+ python tools/make_qwen36_qpack_snap.py \+ --container ~/models/qwen3.6-35b.qpack --out ~/build/qwen36-35b-snap++Run the engine against it:+ SNAP=<out> QWEN36_QPACK=<container> ./qwen36 16 4 prompt.txt++Head-dim derivation mirrors convert_qwen36.py ("derived from the actual+weight shapes, authoritative"), with one extra step: quantized tensors+declare PACKED shapes (uint32 words along the input axis), so logical dims+are unpacked through the config's MLX quantization spec (default bits/group+plus per-module overrides, the same matching Swiftlet's Checkpoint.quantSpec+uses). Stdlib only -- no torch, no numpy, no safetensors dependency; the+safetensors header is 8 bytes of length plus JSON.+"""+import argparse+import json+import os+import struct+import sys+from pathlib import Path+++def read_safetensors_header(path: Path) -> dict:+ with open(path, "rb") as f:+ (hlen,) = struct.unpack("<Q", f.read(8))+ if hlen > (512 << 20):+ sys.exit(f"{path}: implausible safetensors header length {hlen}")+ header = json.loads(f.read(hlen))+ header.pop("__metadata__", None)+ return header+++class QuantSpec:+ """Config `quantization` block: default bits/group + per-module overrides."""++ def __init__(self, cfg: dict):+ block = cfg.get("quantization") or cfg.get("quantization_config") or {}+ self.default = (int(block.get("bits", 4)), int(block.get("group_size", 64)))+ self.overrides = {+ key: (int(spec["bits"]), int(spec["group_size"]))+ for key, spec in block.items()+ if isinstance(spec, dict) and "bits" in spec+ }++ def for_module(self, module: str):+ for key, spec in self.overrides.items():+ if module.endswith(key) or key.endswith(module):+ return spec+ return self.default+++class DenseShapes:+ """Logical (unpacked) shapes for the dense tensors, prefix-resolved."""++ def __init__(self, header: dict, quant: QuantSpec):+ self.header = header+ self.quant = quant++ def resolve(self, name: str):+ if name in self.header:+ return name+ prefixed = "language_model." + name+ return prefixed if prefixed in self.header else None++ def logical_shape(self, name: str):+ resolved = self.resolve(name)+ if resolved is None:+ return None+ info = self.header[resolved]+ shape = list(info["shape"])+ if info["dtype"] in ("U32", "I32") and self.resolve(+ name.removesuffix(".weight") + ".scales") is not None:+ bits, _ = self.quant.for_module(+ resolved.removesuffix(".weight"))+ shape[-1] *= 32 // bits+ return shape+++def derive_meta(cfg_full: dict, mcfg: dict, shapes: DenseShapes) -> dict:+ n_layers = int(mcfg["num_hidden_layers"])+ layer_types = mcfg.get("layer_types")+ if not layer_types:+ interval = int(mcfg.get("full_attention_interval", 4))+ layer_types = ["full_attention" if (i + 1) % interval == 0+ else "linear_attention" for i in range(n_layers)]+ full_idx = [i for i, t in enumerate(layer_types) if t == "full_attention"]+ rope = mcfg.get("rope_parameters") or {}+ meta = {+ "model_type": cfg_full.get("model_type"),+ "hidden": int(mcfg["hidden_size"]),+ "n_layers": n_layers,+ "n_active": len(full_idx),+ "layer_types": layer_types,+ "num_experts": int(mcfg["num_experts"]),+ "topk": int(mcfg["num_experts_per_tok"]),+ "moe_inter": int(mcfg["moe_intermediate_size"]),+ "shared_inter": int(mcfg.get("shared_expert_intermediate_size",+ mcfg.get("moe_intermediate_size", 0))),+ "rms_eps": float(mcfg.get("rms_norm_eps", 1e-6)),+ "scoring_func": mcfg.get("scoring_func", "softmax"),+ "n_group": int(mcfg.get("n_group", 1)),+ "topk_group": int(mcfg.get("topk_group", 1)),+ "norm_topk_prob": bool(mcfg.get("norm_topk_prob", False)),+ "attn_output_gate": bool(mcfg.get("attn_output_gate", False)),+ "rope_theta": float(rope.get("rope_theta",+ mcfg.get("rope_theta", 10000000.0))),+ "mrope_section": rope.get("mrope_section", [16, 16, 16]),+ "partial_rotary_factor": float(mcfg.get("partial_rotary_factor", 0.25)),+ "expert_gs": 0, # snapshot-expert group scaling; unused with a qpack store+ # HF Qwen3.6 stores the rmsnorm_row weights zero-centered and the+ # engine applies (1 + w); mlx-lm materialises the +1 at conversion, so+ # an MLX-derived container carries FULL gamma. This flag makes the+ # engine undo the shift at load (c/qwen36.c load_norm_n) -- without it+ # every normalised activation is ~doubled and the model is noise.+ "zero_centered_norms": False,+ }+ n_q = int(mcfg["num_attention_heads"])+ n_kv = int(mcfg["num_key_value_heads"])+ meta["q_heads"] = n_q+ meta["kv_heads"] = n_kv+ if not full_idx:+ sys.exit("no full_attention layer to derive head dims from")+ fi = full_idx[0]++ def logical(proj):+ return shapes.logical_shape(f"model.layers.{fi}.self_attn.{proj}.weight")++ qp, kp, vp, op = (logical(p) for p in ("q_proj", "k_proj", "v_proj", "o_proj"))+ qn = shapes.logical_shape(f"model.layers.{fi}.self_attn.q_norm.weight")+ if qp is None or kp is None or vp is None or op is None:+ sys.exit(f"layer {fi}: attention projections missing from model.safetensors")+ meta["q_head_dim"] = qp[0] // n_q+ meta["k_head_dim"] = kp[0] // n_kv+ meta["v_head_dim"] = vp[0] // n_kv+ meta["o_in"] = op[1]+ if qn is not None:+ meta["qk_rope_head_dim"] = qn[0]+ meta["head_dim"] = meta["k_head_dim"]+ meta["rope_dim"] = meta.get("qk_rope_head_dim", meta["head_dim"] // 4)+ # DeltaNet dims are explicit in config (same source convert_qwen36.py uses).+ meta["dn_vheads"] = int(mcfg.get("linear_num_value_heads",+ mcfg.get("num_value_heads", 0)))+ meta["dn_kheads"] = int(mcfg.get("linear_num_key_heads",+ mcfg.get("num_key_heads", 0)))+ meta["dn_kdim"] = int(mcfg.get("linear_key_head_dim",+ mcfg.get("key_head_dim", 0)))+ meta["dn_vdim"] = int(mcfg.get("linear_value_head_dim",+ mcfg.get("value_head_dim", 0)))+ meta["dn_convk"] = int(mcfg.get("linear_conv_kernel_dim",+ mcfg.get("conv_kernel_size", 0)))+ meta["dn_conv_dim"] = (meta["dn_kheads"] * meta["dn_kdim"] * 2 ++ meta["dn_vheads"] * meta["dn_vdim"])+ return meta+++def place_link(target: Path, link: Path):+ """Symlink preferred (zero copies, container stays the single source of+ truth); hard link as the fallback for hosts without symlink privileges."""+ if link.is_symlink() or link.exists():+ link.unlink()+ try:+ link.symlink_to(target)+ return "symlink"+ except OSError:+ os.link(target, link) # let a cross-device failure surface loudly+ return "hardlink"+++def main():+ # __doc__ is None under python -OO (docstrings stripped): fall back to a+ # literal rather than crashing before argparse can even print usage.+ ap = argparse.ArgumentParser(+ description=__doc__.splitlines()[0] if __doc__ else+ "Build a qwen36 snap view over a Swiftlet qpack container")+ ap.add_argument("--container", required=True,+ help="qpack container directory (never written to)")+ ap.add_argument("--out", required=True,+ help="snap view directory to create")+ args = ap.parse_args()+ container = Path(args.container).expanduser().resolve()+ out = Path(args.out).expanduser().resolve()+ if container == out or container in out.parents:+ sys.exit("refusing: --out must live OUTSIDE the read-only container")+ st_path = container / "model.safetensors"+ cfg_path = container / "config.json"+ for required in (st_path, cfg_path):+ if not required.is_file():+ sys.exit(f"{required}: missing -- not a qpack container?")+ manifest = container / "manifest.json"+ if manifest.is_file():+ magic = json.loads(manifest.read_text(encoding="utf-8")).get("magic")+ if magic != "QPACK":+ sys.exit(f"{manifest}: magic {magic!r} is not QPACK")+ else:+ print(f"[warn] {manifest} missing; proceeding on model.safetensors alone")++ cfg_full = json.loads(cfg_path.read_text(encoding="utf-8"))+ mcfg = cfg_full.get("text_config", cfg_full)+ quant = QuantSpec(cfg_full)+ shapes = DenseShapes(read_safetensors_header(st_path), quant)+ meta = derive_meta(cfg_full, mcfg, shapes)++ out.mkdir(parents=True, exist_ok=True)+ # The engine reads FLAT keys from config.json (load_cfg), but multimodal+ # checkpoints nest them under text_config -- same flattening+ # convert_qwen36.py performs, original kept as config.hf.json.+ (out / "config.json").write_text(json.dumps(mcfg, indent=2), encoding="utf-8")+ (out / "config.hf.json").write_text(json.dumps(cfg_full, indent=2),+ encoding="utf-8")+ (out / "qwen36_meta.json").write_text(json.dumps(meta, indent=2),+ encoding="utf-8")+ links = {"model.safetensors": st_path}+ tok = container / "tokenizer.json"+ if tok.is_file():+ links["tokenizer.json"] = tok+ else:+ print(f"[warn] {tok} missing; set TOK=<path> when running the engine")+ for name, target in links.items():+ kind = place_link(target, out / name)+ print(f"[link] {name} -> {target} ({kind})")+ print(f"[meta] {out / 'qwen36_meta.json'} "+ f"(q_head_dim={meta['q_head_dim']} k/v={meta['k_head_dim']}/"+ f"{meta['v_head_dim']} o_in={meta['o_in']} "+ f"n_active={meta['n_active']}/{meta['n_layers']})")+ print(f"\nRun: SNAP={out} QWEN36_QPACK={container} ./qwen36 16 4 prompt.txt")+++if __name__ == "__main__":+ main()