Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen38 MTP: tiny fixtures with a head, a torch reference, the lossless gate
JustVugg439c838d5963
diff --git a/.gitignore b/.gitignoreindex 4fbe610a..03d9c9be 100644--- a/.gitignore+++ b/.gitignore@@ -159,6 +159,9 @@ c/qwen38_vision_tiny/c/qwen38_mm_tiny/c/qwen38_tiny_fp8/c/qwen38_tiny_int4/+c/qwen38_tiny_mtp/+c/qwen38_tiny_fp8_mtp/+c/qwen38_tiny_int4_mtp/# MiMo tiny fixture: config.json and ref.json are committed, the rest is regeneratedc/mimo_tiny/*.safetensorsc/mimo_tiny/model.safetensors.index.jsondiff --git a/c/Makefile b/c/Makefileindex d2f677d2..667d5098 100644--- a/c/Makefile+++ b/c/Makefile@@ -1390,6 +1390,34 @@ qwen38-tiny-int4-check: qwen38-tiny-int4-generate qwen38-tiny-fp8-generate qwen3echo "qwen38 int4-g64 experts: ref_int4.json reproduced in $$n configurations, last logits identical across paths; FP8 control ok"./tests/test_qwen38_tier_engine$(EXE) ./qwen38_tiny_int4+# Speculative decoding with the checkpoint's MTP head (Q38_MTP=1; qwen38_core.h,+# "the MTP head"). The fixtures are the three above plus an MTP head (--mtp:+# random head weights from their own seed under the release's mtp.* names; the+# model and its references do not change) and a byte tokenizer for the text+# and serve gates. tests/test_qwen38_mtp.py checks:+# (a) the head's draft logits equal tools/qwen38_mtp_ref.py's float32+# reference for every Q38_MTP_WIRING, through the one-row and the+# two-row (after an accepted draft) paths, on BF16 and FP8 experts;+# (b) decoding with drafts is byte-identical to Q38_MTP=0 (stdout and last+# logits) at caps 1/2/8, prefill batching on and off, BF16 and int8+# trunk, on BF16, FP8 and int4-g64 experts, in the normal mode and with+# Q38_MTP_FORCE=reject (a rollback at every token), accept and mixed,+# whose acceptance and tokens/forward must be the loop's own count;+# (c) a 40-token text decode and a five-turn serve session (prompt cache,+# pin, logprobs, a sampled turn) are byte-identical with drafts, and the+# head's drafts after a restored prefix or pin equal a fresh session's.+.PHONY: qwen38-tiny-mtp-generate qwen38-tiny-mtp-check+qwen38-tiny-mtp-generate:+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_mtp --mtp+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_fp8_mtp --fp8-experts --mtp+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp+ @for d in qwen38_tiny_mtp qwen38_tiny_fp8_mtp qwen38_tiny_int4_mtp; do \+ $(PYTHON) tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./$$d || exit $$?; done++qwen38-tiny-mtp-check: qwen38-tiny-mtp-generate qwen38$(EXE)+ $(PYTHON) tests/test_qwen38_mtp.py --engine ./qwen38$(EXE) --bf16 ./qwen38_tiny_mtp \+ --fp8 ./qwen38_tiny_fp8_mtp --int4 ./qwen38_tiny_int4_mtp+# Context-size gates: KV layout, growth across requests, and the attention# capacity. Includes qwen36.c directly, so no model file is needed.# Same tier objects as the engine: the test includes qwen36.c, so with CUDA=1diff --git a/c/tests/test_qwen38_mtp.py b/c/tests/test_qwen38_mtp.pynew file mode 100644index 00000000..b78f855d--- /dev/null+++ b/c/tests/test_qwen38_mtp.py@@ -0,0 +1,358 @@+#!/usr/bin/env python3+"""Qwen3.8 MTP head and speculative decoding, on the tiny fixtures with a head.++make qwen38-tiny-mtp-check runs it on the BF16, FP8 and int4-g64 fixtures that+tools/make_qwen38_tiny.py --mtp writes (with a byte tokenizer for text and+serve). Gates:++ head the engine's draft logits (Q38_MTP_DUMP, one record per draft)+ equal tools/qwen38_mtp_ref.py's float32 reference of the head to+ float tolerance, for each wiring of Q38_MTP_WIRING, through the+ one-row and the two-row (after an accepted draft) paths; and the+ wirings really differ.+ lossless with the head drafting, stdout and the last logits (DUMP) are+ byte-identical to Q38_MTP=0 at caps 1/2/8, prefill batching on+ and off, BF16 and int8 trunk, in the normal mode and in the three+ Q38_MTP_FORCE modes: reject (every draft rolled back), accept+ (every draft right) and mixed (alternating); the forced modes'+ acceptance and tokens/forward match the count the loop must give.+ text a 40-token text-prompt decode, MTP on and off, byte-identical.+ serve a multi-turn serve session (prefix reuse, a pinned prefix,+ logprobs, a sampled turn) gives the same frames MTP on and off.++ python tests/test_qwen38_mtp.py --engine ./qwen38 --bf16 ./qwen38_tiny_mtp \\+ --fp8 ./qwen38_tiny_fp8_mtp --int4 ./qwen38_tiny_int4_mtp+"""+import argparse+import json+import os+import re+import subprocess+import sys+import tempfile+from pathlib import Path++import numpy as np++HERE = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(HERE / "tools"))++FAILS = []+BASE_ENV = {"OMP_NUM_THREADS": "2", "COLI_NO_OMP_TUNE": "1", "COLI_CUDA": "0",+ "Q38_TRUNK_GPU": "0", "NOSTREAM": "1", "USAGE_SAVE": "0"}+MTP_KEYS = ("Q38_MTP", "Q38_MTP_WIRING", "Q38_MTP_FORCE", "Q38_MTP_DUMP", "DUMP",+ "Q38_PREFILL_BATCH", "Q38_TRUNK_MIN_KB", "Q38_EXPERT_INT4")+++def check(ok, what):+ print((" ok " if ok else " FAIL ") + what, flush=True)+ if not ok:+ FAILS.append(what)+++def run(engine, model, args, extra, stdin=None):+ env = {k: v for k, v in os.environ.items() if k not in MTP_KEYS}+ env.update(BASE_ENV)+ env["SNAP"] = str(model)+ env.update(extra)+ return subprocess.run([str(Path(engine).resolve()), *map(str, args)], env=env, input=stdin,+ capture_output=True, timeout=600)+++SPEC_LINE = re.compile(r"\[qwen38 MTP\] (\S+): ([0-9.]+) tokens/forward \((\d+) forwards per (\d+) tokens\) "+ r"\| acceptance [0-9.]+% \((\d+)/(\d+) drafts\)")+++def spec_counts(stderr):+ """(forwards, tokens, accepted, drafts) of the run's speculation line."""+ found = SPEC_LINE.search(stderr.decode(errors="replace"))+ return tuple(int(found.group(i)) for i in (3, 4, 5, 6)) if found else None+++def expected_counts(n_new, mode):+ """What q38_spec_step must count for a CLI run of n_new tokens in a forced+ mode: the prompt's forward gives token 1, then one call per token fed; a+ call drafts when two more tokens may follow, and the next call takes the+ draft's logits with no forward when the draft stands."""+ drafts = accepted = forwards = tokens = 0+ ahead = None+ for s in range(n_new - 1):+ tokens += 1+ if ahead is not None:+ standing, ahead = ahead, None+ if standing:+ accepted += 1+ continue+ if n_new - 1 - s >= 2:+ standing = mode == "accept" or (mode == "mixed" and drafts % 2 == 0)+ drafts += 1+ ahead = standing+ forwards += 1+ return forwards, tokens, accepted, drafts+++# ---------- head: draft logits against the torch reference ----------++def read_dump(path, vocab):+ raw = Path(path).read_bytes()+ record = 8 + 4 * vocab+ if len(raw) % record:+ raise ValueError(f"{path}: {len(raw)} bytes is not a whole number of {record}-byte records")+ out = []+ for at in range(0, len(raw), record):+ row, token = np.frombuffer(raw[at:at + 8], np.int32)+ out.append((int(row), int(token), np.frombuffer(raw[at + 8:at + record], np.float32)))+ return out+++def check_head(engine, model):+ """The head's numbers. The fixture's four streams end with nearly equal+ norms, so wirings a and b part by only ~1e-5 on its logits; the engine+ meets its own wiring's float32 reference ~100x closer than that, which is+ what tells the wirings apart here."""+ import qwen38_mtp_ref as ref+ data = json.loads((model / "ref.json").read_text(encoding="utf-8"))+ ids, vocab = data["full_ids"], len(data["final_logits"])+ print(f"head: {model}")+ references = {w: ref.mtp_logits(model, ids, w).numpy() for w in "abc"}+ for a, b in (("a", "b"), ("a", "c"), ("b", "c")):+ gap = float(np.abs(references[a] - references[b]).max())+ check(gap > 2e-6, f"wirings {a} and {b} give different logits (max |diff| {gap:.3g})")+ with tempfile.TemporaryDirectory() as tmp:+ for wiring in "abc":+ rows_seen = set()+ for mode in ("reject", "mixed"):+ dump = Path(tmp) / f"{wiring}-{mode}.f32"+ # reject drafts at every token (rows past the prompt, one pending+ # row each); mixed reaches the head with two pending rows after+ # each accepted draft. The BF16 trunk: every fixture matrix is+ # under the 1 MiB int8 threshold.+ r = run(engine, model, [2, 8, model / "ref.json"],+ {"Q38_MTP": "1", "Q38_MTP_WIRING": wiring, "Q38_MTP_FORCE": mode,+ "Q38_MTP_DUMP": str(dump), "Q38_TRUNK_MIN_KB": "1024"})+ if r.returncode:+ check(False, f"wiring {wiring} {mode}: engine exit {r.returncode}\n"+ + r.stderr.decode(errors="replace")[-2000:])+ continue+ worst, nearest_other, argmax_ok, rows = 0.0, float("inf"), True, 0+ for row, token, logits in read_dump(dump, vocab):+ if token != ids[row + 1]:+ check(False, f"wiring {wiring} {mode}: draft row {row} paired token {token}, "+ f"the sequence has {ids[row + 1]}")+ continue+ errors = {w: float(np.abs(logits - references[w][row]).max()) for w in "abc"}+ worst = max(worst, errors[wiring])+ nearest_other = min(nearest_other, *(e for w, e in errors.items() if w != wiring))+ argmax_ok &= int(logits.argmax()) == int(references[wiring][row].argmax())+ rows += 1+ rows_seen.add(row)+ check(rows > 0 and worst <= 1e-6 and 20 * worst <= nearest_other and argmax_ok,+ f"wiring {wiring} {mode}: {rows} draft rows within {worst:.2g} of the reference "+ f"(the other wirings' are {nearest_other:.2g} away), argmax {'equal' if argmax_ok else 'DIFFERS'}")+ check(rows_seen >= set(range(len(data["prompt_ids"]) - 1, len(ids) - 3)),+ f"wiring {wiring}: rows {sorted(rows_seen)} compared")+++# ---------- lossless: MTP on == MTP off ----------++def check_lossless(engine, model, ref_name, extra=None):+ extra = dict(extra or {})+ data = json.loads((model / ref_name).read_text(encoding="utf-8"))+ n_new = len(data["full_ids"]) - len(data["prompt_ids"])+ print(f"lossless: {model} ({ref_name}{', ' + ' '.join(f'{k}={v}' for k, v in extra.items()) if extra else ''})")+ configs, failed_before = 0, len(FAILS)+ with tempfile.TemporaryDirectory() as tmp:+ for trunk in ("1024", "0"):+ for batch in ("0", "1"):+ for cap in (1, 2, 8):+ tag = f"trunk={trunk} batch={batch} cap={cap}"+ env = dict(extra, Q38_TRUNK_MIN_KB=trunk, Q38_PREFILL_BATCH=batch)+ base_dump = Path(tmp) / "off.f32"+ off = run(engine, model, [cap, 8, model / ref_name], dict(env, DUMP=str(base_dump)))+ if off.returncode:+ check(False, f"{tag}: MTP off fails the oracle\n" + off.stdout.decode(errors="replace"))+ continue+ base_logits = base_dump.read_bytes()+ for mode in ("", "reject", "accept", "mixed"):+ dump = Path(tmp) / "on.f32"+ on = run(engine, model, [cap, 8, model / ref_name],+ dict(env, DUMP=str(dump), Q38_MTP="1", Q38_MTP_FORCE=mode))+ name = f"{tag} {mode or 'normal'}"+ same = on.returncode == 0 and on.stdout == off.stdout and dump.read_bytes() == base_logits+ counts = spec_counts(on.stderr)+ if mode:+ want = expected_counts(n_new, mode)+ same &= counts == want+ else:+ same &= counts is not None and counts[2] <= counts[3] and counts[0] <= counts[1]+ if not same:+ check(False, f"{name}: output differs from MTP off or wrong accounting "+ f"(counts {counts}, want {expected_counts(n_new, mode) if mode else 'consistent'})\n"+ + on.stdout.decode(errors="replace") + on.stderr.decode(errors="replace")[-1500:])+ configs += 1+ off = run(engine, model, [8, 8, model / ref_name], dict(extra))+ for knob in ({"Q38_MTP_WIRING": "b"}, {"Q38_MTP_WIRING": "c"}, {"Q38_MTP_CAP": "1"}):+ on = run(engine, model, [8, 8, model / ref_name], dict(extra, Q38_MTP="1", **knob))+ name = " ".join(f"{k}={v}" for k, v in knob.items())+ check(on.returncode == 0 and on.stdout == off.stdout, f"{name}: output equals MTP off")+ want = {m: expected_counts(n_new, m) for m in ("reject", "accept", "mixed")}+ check(len(FAILS) == failed_before,+ f"{configs} runs byte-identical to MTP off (stdout and last logits); forced counts "+ + ", ".join(f"{m} {w[0]} forwards/{w[1]} tokens {w[2]}/{w[3]} accepted" for m, w in want.items()))+++# ---------- text: a longer greedy decode ----------++def check_text(engine, model, extra=None):+ print(f"text: {model}")+ with tempfile.TemporaryDirectory() as tmp:+ prompt = Path(tmp) / "prompt.txt"+ prompt.write_bytes(b"123 456: 7")+ outs = {}+ for mode in ("off", "", "reject"):+ dump = Path(tmp) / f"{mode}.f32"+ env = dict(extra or {}, N_NEW="40", DUMP=str(dump), TOK=str(model / "tokenizer.json"))+ if mode != "off":+ env.update(Q38_MTP="1", Q38_MTP_FORCE=mode)+ r = run(engine, model, [2, 8, prompt], env)+ outs[mode] = (r.returncode, r.stdout, re.findall(rb"Text +:.*", r.stderr), dump.read_bytes() if dump.exists() else b"")+ if mode == "":+ counts = spec_counts(r.stderr)+ print(f" normal mode on 40 tokens: forwards/tokens/accepted/drafts = {counts}")+ check(outs["off"][0] == 0 and outs["off"][2], "40-token text decode runs with MTP off")+ for mode in ("", "reject"):+ check(outs[mode] == outs["off"], f"40-token text decode, MTP {mode or 'normal'}: same text and last logits")+++# ---------- serve: several turns through the wire protocol ----------++SERVE_TURNS = [+ (1, b"123 456:", 24, "0", "1", ""),+ (2, b"123 456: 0", 24, "0", "1", " pin=1"),+ (3, b"123 456: 01", 20, "0", "1", " logprobs=2"),+ (4, b"789 012:", 16, "0.9", "0.95", ""),+ (5, b"123 456: 0123", 20, "0", "1", ""),+]+++def serve_session(engine, model, extra, turns=SERVE_TURNS):+ """Turns sent one at a time, each when the previous one is DONE (a SUBMIT+ during a turn is refused as busy). Returns the exit code, the frames that+ must not depend on drafting, and stderr."""+ env = {k: v for k, v in os.environ.items() if k not in MTP_KEYS}+ env.update(BASE_ENV)+ env.update(extra, SNAP=str(model), SERVE="1", TOK=str(model / "tokenizer.json"), Q38_MAXT="128")+ with tempfile.TemporaryFile() as err:+ p = subprocess.Popen([str(Path(engine).resolve()), "1", "8"], env=env, stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, stderr=err)+ frames = []++ def frame():+ line = p.stdout.readline()+ if not line:+ raise EOFError("engine closed")+ fields = line.replace(b"\x01", b"").split()+ if fields and fields[0] in (b"DATA", b"ECHO"):+ n = int(fields[2])+ line = line.rstrip(b"\n") + b"|" + p.stdout.read(n)+ p.stdout.read(1)+ return fields, line++ try:+ while frame()[0][:1] != [b"READY"]:+ pass+ for rid, text, count, temp, top_p, ext in turns:+ p.stdin.write(f"SUBMIT {rid} 0 {len(text)} {count} {temp} {top_p}{ext}\n".encode() + text + b"\n")+ p.stdin.flush()+ while True:+ fields, line = frame()+ if not fields or fields[0] in (b"STAT", b"PROF", b"EMAP", b"HITS", b"CAPS"):+ continue # timings, and the expert residency the draft rows change+ if fields[0] == b"DONE": # tokens/s and RSS vary: keep the count and the reason+ line = b" ".join(fields[:4] + fields[-2:])+ frames.append(line)+ if fields[0] in (b"DONE", b"ERROR"):+ break+ p.stdin.close()+ rc = p.wait(timeout=60)+ except Exception as exc: # noqa: BLE001 -- reported as a failed gate+ p.kill()+ p.wait()+ frames.append(f"harness: {exc!r}".encode())+ rc = -1+ err.seek(0)+ return rc, frames, err.read()+++def check_serve(engine, model, extra=None):+ extra = dict(extra or {})+ print(f"serve: {model}")+ rc, base, _ = serve_session(engine, model, extra)+ done = [f for f in base if f.startswith(b"DONE")]+ check(rc == 0 and len(done) == 5, f"MTP off: five turns served ({len(done)} DONE)")+ for mode in ("", "reject"):+ rc, frames, err = serve_session(engine, model, dict(extra, Q38_MTP="1", Q38_MTP_FORCE=mode))+ turns = len(re.findall(rb"\[qwen38 MTP\] turn \d+:", err))+ check(rc == 0 and frames == base and turns == 5,+ f"MTP {mode or 'normal'}: every frame equals MTP off (prefix reuse, pin, logprobs, "+ f"sampled turn), {turns} per-turn MTP lines")+ if frames != base:+ for a, b in zip(base, frames):+ if a != b:+ print(f" first difference:\n off {a[:160]!r}\n on {b[:160]!r}")+ break+ # The head's place survives a restored prefix: after the prompt cache+ # (turn 2 extends turn 1) and after a pin (turn 8 extends turn 6's pin;+ # turn 7 moved the prompt cache to a longer prompt that turn 8 leaves+ # at position 9, and kept the pinned rows), its drafts are bit for bit+ # those of a session that sees the turn first.+ config = json.loads((model / "config.json").read_text(encoding="utf-8"))+ vocab = config.get("text_config", config)["vocab_size"]+ turn = {t[0]: t for t in SERVE_TURNS + [(6, b"123 456:", 4, "0", "1", " pin=1"),+ (7, b"123 456: 01", 4, "0", "1", ""),+ (8, b"123 456: 9", 20, "0", "1", "")]}+ with tempfile.TemporaryDirectory() as tmp:+ for history, last, how in (((1,), 2, "the prompt cache"), ((6, 7), 8, "a pin")):+ dumps = []+ for turns in ([turn[i] for i in history] + [turn[last]], [turn[last]]):+ dump = Path(tmp) / f"{len(dumps)}.f32"+ rc, _, _ = serve_session(engine, model, dict(extra, Q38_MTP="1", Q38_MTP_DUMP=str(dump)), turns)+ dumps.append(read_dump(dump, vocab) if rc == 0 else [])+ warm, fresh = dumps+ same = bool(fresh) and len(warm) > len(fresh) and all(+ a[0] == b[0] and a[1] == b[1] and a[2].tobytes() == b[2].tobytes()+ for a, b in zip(warm[-len(fresh):], fresh))+ check(same, f"turn {last} after {how}: its {len(fresh)} drafts' logits equal a fresh session's")+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--engine", required=True, type=Path)+ ap.add_argument("--bf16", type=Path, help="tools/make_qwen38_tiny.py --mtp")+ ap.add_argument("--fp8", type=Path, help="... --fp8-experts --mtp")+ ap.add_argument("--int4", type=Path, help="... --fp8-experts --int4-experts --expert-gain 3 --mtp")+ ap.add_argument("--skip-head", action="store_true", help="no torch reference (lossless gates only)")+ a = ap.parse_args()+ for model in (a.bf16, a.fp8):+ if model and not a.skip_head:+ check_head(a.engine, model)+ for model in (a.bf16, a.fp8):+ if model:+ check_lossless(a.engine, model, "ref.json")+ if a.int4:+ check_lossless(a.engine, a.int4, "ref_int4.json", {"Q38_EXPERT_INT4": "1"})+ for model in (a.bf16, a.fp8, a.int4):+ if model and (model / "tokenizer.json").exists():+ extra = {"Q38_EXPERT_INT4": "1"} if model == a.int4 else {}+ check_text(a.engine, model, extra)+ check_serve(a.engine, model, extra)+ if FAILS:+ print(f"qwen38 MTP: {len(FAILS)} check(s) failed")+ return 1+ print("qwen38 MTP: head matches the reference for wirings a, b, c; decoding with drafts is "+ "byte-identical to decoding without them")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/make_qwen38_tiny.py b/c/tools/make_qwen38_tiny.pyindex 7e53aeac..287e4cdf 100644--- a/c/tools/make_qwen38_tiny.py+++ b/c/tools/make_qwen38_tiny.py@@ -7,7 +7,8 @@ wrapper. Consequently its state-dict names start at ``model.embed_tokens``;the released multimodal checkpoint adds the extra ``model.language_model``prefix. This keeps vision and MTP weights out while retaining the exact textmodule names (``layers.*.linear_attn``, ``layers.*.self_attn``, ``layers.*.ple``-and ``layers.*.mlp``) an engine needs to support.+and ``layers.*.mlp``) an engine needs to support. ``--mtp`` adds an MTP head+under the release's ``mtp.*`` names (transformers has none; see _add_mtp).Usage::@@ -221,6 +222,67 @@ def _rewrite_shard_fp8(out: Path, packed):print(f"fp8 experts: {len(packed)} matrices rewritten as F8_E4M3 + BF16 weight_scale_inv")+MTP_SEED_OFFSET = 0x4D5450 # "MTP": the head's weights come from their own stream+MTP_ROPE_THETA = 50000.0 # not the model's 10000, so a head on the wrong base shows+++def _add_mtp(out: Path, config, seed, fp8_experts=False, expert_gain=1.0):+ """Add an MTP head with the release's tensor names to the saved fixture.++ Transformers has no MTP module (it ignores ``mtp.*`` on load), so the head+ is written here: random weights, drawn from a generator of their own after+ the model is saved, so the model, its tensors and ref.json are exactly+ those of the same fixture without ``--mtp``. The head's decoder layer has+ the tensors of the model's last attention layer (``mtp.layers.0.*``, same+ names after the layer prefix and same shapes); around it sit+ ``pre_fc_norm_embedding`` [H], ``pre_fc_norm_hidden`` [hc_count*H],+ ``fc_embedding`` and ``fc_hidden`` [H, H] and a ``hyper_connection_mixer``+ shaped like the model's. Norm weights get small random values (the+ model's are zero, Qwen4-Exp norms scale by 1+w) so a norm reading the+ wrong slice of its weight shows. With ``fp8_experts`` the head's routed+ experts are e4m3 with 128x128 block scales like the model's."""+ from safetensors.torch import load_file, save_file+ path = out / "model.safetensors"+ tensors = load_file(str(path))+ gen = torch.Generator().manual_seed(seed + MTP_SEED_OFFSET)+ attn = [i for i, kind in enumerate(config.layer_types) if kind != "linear_attention"+ and (i + 1) not in (config.ple_layer_ids or [])]+ source = f"model.layers.{attn[-1]}."+ std = config.initializer_range++ def draw(shape, scale):+ return (torch.randn(shape, generator=gen) * scale).to(torch.bfloat16)++ head = {}+ for name in sorted(k for k in tensors if k.startswith(source)):+ shape = tuple(tensors[name].shape)+ suffix = name[len(source):]+ scale = 0.1 if len(shape) == 1 else std * (expert_gain if ".mlp.experts." in suffix else 1.0)+ head["mtp.layers.0." + suffix] = draw(shape, scale)+ H, W = config.hidden_size, config.hidden_size * config.hc_count+ head["mtp.pre_fc_norm_embedding.weight"] = draw((H,), 0.1)+ head["mtp.pre_fc_norm_hidden.weight"] = draw((W,), 0.1)+ head["mtp.fc_embedding.weight"] = draw((H, H), 0.1)+ head["mtp.fc_hidden.weight"] = draw((H, H), 0.1)+ for name in sorted(k for k in tensors if k.startswith("model.hyper_connection_mixer.")):+ shape = tuple(tensors[name].shape)+ head["mtp." + name[len("model."):]] = draw(shape, 0.1 if len(shape) == 1 else std)+ if fp8_experts:+ for name in sorted(k for k in head if ".mlp.experts." in k):+ q, scale_inv, _ = _fp8_block_quant(head[name])+ head[name] = q.contiguous()+ head[name + "_scale_inv"] = scale_inv.to(torch.bfloat16).contiguous()+ tensors.update(head)+ save_file(tensors, str(path), metadata={"format": "pt"})+ cfg_path = out / "config.json"+ cfg = json.loads(cfg_path.read_text(encoding="utf-8"))+ cfg["mtp_num_hidden_layers"] = 1+ cfg["mtp"] = {"layer_types": ["full_attention"], "rope_theta": MTP_ROPE_THETA, "hybrid": True}+ cfg["mtp_use_dedicated_embeddings"] = False+ cfg_path.write_text(json.dumps(cfg, indent=2) + "\n", encoding="utf-8")+ print(f"mtp head: {len(head)} tensors (decoder layer shaped like {source}*)")++def _int4_experts_in_model(model, out: Path):"""Write the int4-g64 sidecar of the fixture just saved, with the converterthe real model goes through (tools/convert_qwen38_experts_int4.py), then read its bytes@@ -259,7 +321,7 @@ def _int4_experts_in_model(model, out: Path):def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_experts=False,- int4_experts=False, expert_gain=1.0):+ int4_experts=False, expert_gain=1.0, mtp=False):if max_new < 1:raise ValueError("max_new must be at least 1")random.seed(seed)@@ -351,6 +413,8 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e# from, and qwen38 picks it up by itself: regenerating the fixture drops it.shutil.rmtree(out / "experts-int4g64", ignore_errors=True)model.save_pretrained(str(out), safe_serialization=True)+ if mtp:+ _add_mtp(out, config, seed, fp8_experts=fp8_experts, expert_gain=expert_gain)if packed:_rewrite_shard_fp8(out, packed)@@ -414,11 +478,14 @@ def main():parser.add_argument("--int4-experts", action="store_true",help="also convert the routed experts to the experts-int4g64/ sidecar ""and write ref_int4.json, the reference of the dequantized sidecar")+ parser.add_argument("--mtp", action="store_true",+ help="add an MTP head (mtp.*, random weights from their own seed) with the "+ "release's tensor names; the model and ref.json do not change")args = parser.parse_args()prompt = [int(x) for x in args.prompt_ids.split(",") if x.strip()] if args.prompt_ids else Nonebuild(args.out, prompt_ids=prompt, max_new=args.max_new, seed=args.seed, emit_ref=not args.no_ref,fp8_experts=args.fp8_experts, int4_experts=args.int4_experts,- expert_gain=args.expert_gain)+ expert_gain=args.expert_gain, mtp=args.mtp)if __name__ == "__main__":diff --git a/c/tools/qwen38_mtp_ref.py b/c/tools/qwen38_mtp_ref.pynew file mode 100644index 00000000..414207fe--- /dev/null+++ b/c/tools/qwen38_mtp_ref.py@@ -0,0 +1,225 @@+#!/usr/bin/env python3+"""Reference logits of the Qwen3.8 MTP head on a tiny fixture, in torch.++The head is built from our own reading of its tensors, the same reading+qwen38_core.h documents next to q38_mtp_input (the MTP head):++* the model's four hyper-connection streams at position p -- the+ ``hidden_size * hc_count`` vector the final mixer reads, captured here as the+ input of ``model.hyper_connection_mixer`` -- and the embedding of the token+ at p+1 become four streams through ``pre_fc_norm_*`` and ``fc_*``, in one of+ the wirings a, b or c (``fuse`` below);+* those streams run through one decoder layer with the tensors under+ ``mtp.layers.0`` -- the transformers ``Qwen4ExpTextDecoderLayer``, the oracle+ the engine's own layers are checked against, built as an attention layer+ without PLE -- with RoPE on ``mtp.rope_theta`` and causal attention over the+ pairs;+* ``mtp.hyper_connection_mixer`` (a ``Qwen4ExpTextGatedResidual`` without+ inject) and the shared ``lm_head`` give the logits.++Row i of the result pairs the streams at position i with ``ids[i + 1]``: the+logits that predict ``ids[i + 2]``. Everything runs in float32 on the bytes on+disk: BF16 widens exactly, and the FP8 routed experts (the model's and the+head's) are dequantized the way qwen38's expanded path does it, e4m3 value+times the block's ``weight_scale_inv`` in f32, so the reference sees the+engine's weights and not the BF16 rounding the fixture generator kept in+memory.++ python tools/qwen38_mtp_ref.py --model ./qwen38_tiny_mtp --wiring a+"""++import argparse+import copy+import json+import math+from pathlib import Path++import numpy as np+import torch++FP8_BLOCK = 128+++def _e4m3_table():+ """quant.h's E4M3 table (float8_e4m3fn: bias 7, subnormals, 0x7F/0xFF NaN)."""+ table = np.empty(256, np.float32)+ for byte in range(256):+ exponent, mantissa = (byte >> 3) & 0xF, byte & 7+ if exponent == 0xF and mantissa == 7:+ value = math.nan+ elif exponent == 0:+ value = mantissa / 8.0 * 2.0 ** -6+ else:+ value = (1.0 + mantissa / 8.0) * 2.0 ** (exponent - 7)+ table[byte] = -value if byte & 0x80 else value+ return torch.from_numpy(table)+++E4M3 = _e4m3_table()+++def load_tensors(model_dir: Path):+ """Every tensor of the snapshot as float32; FP8 matrices dequantized with+ their block scales, the scale tensors themselves dropped."""+ from safetensors.torch import load_file+ raw = {}+ for path in sorted(model_dir.glob("*.safetensors")):+ raw.update(load_file(str(path)))+ out = {}+ for name, tensor in raw.items():+ if name.endswith("_scale_inv"):+ continue+ if tensor.dtype == torch.float8_e4m3fn:+ scale = raw[name + "_scale_inv"].float()+ rows, cols = tensor.shape+ expand = scale.repeat_interleave(FP8_BLOCK, 0)[:rows].repeat_interleave(FP8_BLOCK, 1)[:, :cols]+ out[name] = E4M3[tensor.view(torch.uint8).long()] * expand+ elif tensor.is_floating_point():+ out[name] = tensor.float()+ else:+ out[name] = tensor # the PLE hash tables are int64+ return out+++def _fuse_experts(weights, prefix, experts):+ """Per-expert gate/up/down -> transformers' fused gate_up_proj / down_proj."""+ gate_up = torch.stack([torch.cat([weights[f"{prefix}{e}.gate_proj.weight"],+ weights[f"{prefix}{e}.up_proj.weight"]], 0) for e in range(experts)])+ down = torch.stack([weights[f"{prefix}{e}.down_proj.weight"] for e in range(experts)])+ return gate_up, down+++def load_model(model_dir: Path):+ """The fixture's model in float32 with the exact weights on disk, its+ config, and the f32 tensors."""+ from transformers import Qwen4ExpForCausalLM, Qwen4ExpTextConfig+ cfg = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))+ config = Qwen4ExpTextConfig.from_pretrained(str(model_dir))+ if hasattr(config, "quantization_config"):+ del config.quantization_config # the experts are dequantized here, not by transformers+ config._attn_implementation = "eager"+ model = Qwen4ExpForCausalLM(config).float().eval()+ weights = load_tensors(model_dir)+ state = model.state_dict()+ with torch.no_grad():+ for name, param in state.items():+ if name.endswith(".mlp.experts.gate_up_proj") or name.endswith(".mlp.experts.down_proj"):+ base = name[: name.rindex(".") + 1]+ gate_up, down = _fuse_experts(weights, base, config.num_experts)+ param.copy_(gate_up if name.endswith("gate_up_proj") else down)+ elif name in weights:+ param.copy_(weights[name])+ elif name.endswith(("layer_multipliers", "ngram_heads_vocab_sizes", "ngram_heads_offsets")):+ continue # buffers the model computes itself+ else:+ shards = sorted(k for k in weights if k.startswith(name.replace(".weight", ".shard_")))+ if not shards:+ raise KeyError(f"{name}: not in the snapshot")+ param.copy_(torch.cat([weights[k] for k in shards], 0))+ return model, config, cfg, weights+++def model_streams(model, ids):+ """The four streams the final mixer reads, [len(ids), hc_count * hidden]."""+ captured = {}++ def hook(module, args):+ captured["streams"] = args[0].detach()++ handle = model.model.hyper_connection_mixer.register_forward_pre_hook(hook)+ try:+ with torch.no_grad():+ model(input_ids=torch.tensor([ids]), use_cache=False)+ finally:+ handle.remove()+ return captured["streams"][0]+++def rms0(x, weight, eps):+ """Qwen4-Exp's zero-centered RMSNorm over the last dimension: x / rms * (1 + w)."""+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + weight)+++def fuse(streams, embedded, weights, config, wiring):+ """The head's input streams [S, hc_count*H] from the model's streams at p+ [S, hc_count*H] and the embeddings of the tokens at p+1 [S, H]."""+ C, H, eps = config.hc_count, config.hidden_size, config.rms_norm_eps+ w_hid = weights["mtp.pre_fc_norm_hidden.weight"]+ fe = rms0(embedded, weights["mtp.pre_fc_norm_embedding.weight"], eps) @ weights["mtp.fc_embedding.weight"].T+ fc_hidden = weights["mtp.fc_hidden.weight"]+ S = streams.shape[0]+ if wiring == "a": # one norm over the whole vector, fc_hidden per stream, embedding into all four+ normed = rms0(streams, w_hid, eps).view(S, C, H)+ return (normed @ fc_hidden.T + fe[:, None, :]).reshape(S, C * H)+ per_stream = rms0(streams.view(S, C, H), w_hid.view(C, H), eps)+ if wiring == "b": # a norm per stream, fc_hidden per stream, embedding into all four+ return (per_stream @ fc_hidden.T + fe[:, None, :]).reshape(S, C * H)+ if wiring == "c": # a norm per stream, the streams' mean, one fc_hidden, copied to all four+ one = per_stream.mean(1) @ fc_hidden.T + fe+ return one.repeat(1, C)+ raise ValueError(f"wiring {wiring!r}: a, b or c")+++def mtp_layer(config, cfg, weights):+ """The head's decoder layer: transformers' layer, as one of the model's+ attention layers without PLE, holding the mtp.layers.0 tensors."""+ from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextDecoderLayer+ attn = [i for i, kind in enumerate(config.layer_types) if kind != "linear_attention"+ and (i + 1) not in (config.ple_layer_ids or [])]+ layer = Qwen4ExpTextDecoderLayer(config, attn[-1]).float().eval()+ state = layer.state_dict()+ with torch.no_grad():+ for name, param in state.items():+ if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.down_proj"):+ gate_up, down = _fuse_experts(weights, "mtp.layers.0.mlp.experts.", config.num_experts)+ param.copy_(gate_up if name.endswith("gate_up_proj") else down)+ else:+ param.copy_(weights["mtp.layers.0." + name])+ return layer+++def mtp_logits(model_dir, ids, wiring):+ """[len(ids) - 1, vocab]: row i is the head's logits for the pair+ (streams at i, ids[i + 1])."""+ from transformers.models.qwen4_exp.modeling_qwen4_exp import (+ Qwen4ExpTextGatedResidual, Qwen4ExpTextRotaryEmbedding)+ model_dir = Path(model_dir)+ model, config, cfg, weights = load_model(model_dir)+ streams = model_streams(model, ids)[:-1]+ nxt = torch.tensor(ids[1:])+ embedded = weights["model.embed_tokens.weight"][nxt]+ hyper = fuse(streams, embedded, weights, config, wiring)+ S = hyper.shape[0]+ head_config = copy.deepcopy(config)+ theta = (cfg.get("mtp") or {}).get("rope_theta", config.rope_parameters["rope_theta"])+ head_config.rope_parameters = dict(config.rope_parameters, rope_theta=theta)+ rotary = Qwen4ExpTextRotaryEmbedding(config=head_config)+ positions = torch.arange(S).view(1, 1, -1).expand(3, 1, -1)+ position_embeddings = rotary(hyper[None], positions)+ mask = torch.full((1, 1, S, S), torch.finfo(torch.float32).min).triu(1)+ layer = mtp_layer(config, cfg, weights)+ mixer = Qwen4ExpTextGatedResidual(config, use_combine=False).float().eval()+ with torch.no_grad():+ for name, param in mixer.state_dict().items():+ param.copy_(weights["mtp.hyper_connection_mixer." + name])+ out = layer(hyper[None], position_embeddings=position_embeddings, attention_mask=mask,+ conv_mask=None, past_key_values=None, ple_input_ids=None)+ logits = mixer(out)[0] @ weights["lm_head.weight"].T+ return logits+++def main():+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--model", required=True, type=Path)+ parser.add_argument("--wiring", default="a", choices="abc")+ parser.add_argument("--ids", help="comma-separated ids (default: ref.json full_ids)")+ args = parser.parse_args()+ ids = ([int(x) for x in args.ids.split(",")] if args.ids else+ json.loads((args.model / "ref.json").read_text(encoding="utf-8"))["full_ids"])+ logits = mtp_logits(args.model, ids, args.wiring)+ for i, row in enumerate(logits):+ print(f"row {i}: token {ids[i + 1]} -> draft {int(row.argmax())} (max {float(row.max()):.6f})")+++if __name__ == "__main__":+ main()