Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38 MTP: tiny fixtures with a head, a torch reference, the lossless gate

JustVugg439c838d5963
diff --git a/.gitignore b/.gitignore
index 4fbe610a..03d9c9be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -159,6 +159,9 @@ c/qwen38_vision_tiny/
c/qwen38_mm_tiny/
c/qwen38_tiny_fp8/
c/qwen38_tiny_int4/
+c/qwen38_tiny_mtp/
+c/qwen38_tiny_fp8_mtp/
+c/qwen38_tiny_int4_mtp/
# MiMo tiny fixture: config.json and ref.json are committed, the rest is regenerated
c/mimo_tiny/*.safetensors
c/mimo_tiny/model.safetensors.index.json
diff --git a/c/Makefile b/c/Makefile
index d2f677d2..667d5098 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1390,6 +1390,34 @@ qwen38-tiny-int4-check: qwen38-tiny-int4-generate qwen38-tiny-fp8-generate qwen3
echo "qwen38 int4-g64 experts: ref_int4.json reproduced in $$n configurations, last logits identical across paths; FP8 control ok"
./tests/test_qwen38_tier_engine$(EXE) ./qwen38_tiny_int4
+# Speculative decoding with the checkpoint's MTP head (Q38_MTP=1; qwen38_core.h,
+# "the MTP head"). The fixtures are the three above plus an MTP head (--mtp:
+# random head weights from their own seed under the release's mtp.* names; the
+# model and its references do not change) and a byte tokenizer for the text
+# and serve gates. tests/test_qwen38_mtp.py checks:
+# (a) the head's draft logits equal tools/qwen38_mtp_ref.py's float32
+# reference for every Q38_MTP_WIRING, through the one-row and the
+# two-row (after an accepted draft) paths, on BF16 and FP8 experts;
+# (b) decoding with drafts is byte-identical to Q38_MTP=0 (stdout and last
+# logits) at caps 1/2/8, prefill batching on and off, BF16 and int8
+# trunk, on BF16, FP8 and int4-g64 experts, in the normal mode and with
+# Q38_MTP_FORCE=reject (a rollback at every token), accept and mixed,
+# whose acceptance and tokens/forward must be the loop's own count;
+# (c) a 40-token text decode and a five-turn serve session (prompt cache,
+# pin, logprobs, a sampled turn) are byte-identical with drafts, and the
+# head's drafts after a restored prefix or pin equal a fresh session's.
+.PHONY: qwen38-tiny-mtp-generate qwen38-tiny-mtp-check
+qwen38-tiny-mtp-generate:
+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_mtp --mtp
+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_fp8_mtp --fp8-experts --mtp
+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp
+ @for d in qwen38_tiny_mtp qwen38_tiny_fp8_mtp qwen38_tiny_int4_mtp; do \
+ $(PYTHON) tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./$$d || exit $$?; done
+
+qwen38-tiny-mtp-check: qwen38-tiny-mtp-generate qwen38$(EXE)
+ $(PYTHON) tests/test_qwen38_mtp.py --engine ./qwen38$(EXE) --bf16 ./qwen38_tiny_mtp \
+ --fp8 ./qwen38_tiny_fp8_mtp --int4 ./qwen38_tiny_int4_mtp
+
# Context-size gates: KV layout, growth across requests, and the attention
# capacity. Includes qwen36.c directly, so no model file is needed.
# Same tier objects as the engine: the test includes qwen36.c, so with CUDA=1
diff --git a/c/tests/test_qwen38_mtp.py b/c/tests/test_qwen38_mtp.py
new file mode 100644
index 00000000..b78f855d
--- /dev/null
+++ b/c/tests/test_qwen38_mtp.py
@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+"""Qwen3.8 MTP head and speculative decoding, on the tiny fixtures with a head.
+
+make qwen38-tiny-mtp-check runs it on the BF16, FP8 and int4-g64 fixtures that
+tools/make_qwen38_tiny.py --mtp writes (with a byte tokenizer for text and
+serve). Gates:
+
+ head the engine's draft logits (Q38_MTP_DUMP, one record per draft)
+ equal tools/qwen38_mtp_ref.py's float32 reference of the head to
+ float tolerance, for each wiring of Q38_MTP_WIRING, through the
+ one-row and the two-row (after an accepted draft) paths; and the
+ wirings really differ.
+ lossless with the head drafting, stdout and the last logits (DUMP) are
+ byte-identical to Q38_MTP=0 at caps 1/2/8, prefill batching on
+ and off, BF16 and int8 trunk, in the normal mode and in the three
+ Q38_MTP_FORCE modes: reject (every draft rolled back), accept
+ (every draft right) and mixed (alternating); the forced modes'
+ acceptance and tokens/forward match the count the loop must give.
+ text a 40-token text-prompt decode, MTP on and off, byte-identical.
+ serve a multi-turn serve session (prefix reuse, a pinned prefix,
+ logprobs, a sampled turn) gives the same frames MTP on and off.
+
+ python tests/test_qwen38_mtp.py --engine ./qwen38 --bf16 ./qwen38_tiny_mtp \\
+ --fp8 ./qwen38_tiny_fp8_mtp --int4 ./qwen38_tiny_int4_mtp
+"""
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import numpy as np
+
+HERE = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(HERE / "tools"))
+
+FAILS = []
+BASE_ENV = {"OMP_NUM_THREADS": "2", "COLI_NO_OMP_TUNE": "1", "COLI_CUDA": "0",
+ "Q38_TRUNK_GPU": "0", "NOSTREAM": "1", "USAGE_SAVE": "0"}
+MTP_KEYS = ("Q38_MTP", "Q38_MTP_WIRING", "Q38_MTP_FORCE", "Q38_MTP_DUMP", "DUMP",
+ "Q38_PREFILL_BATCH", "Q38_TRUNK_MIN_KB", "Q38_EXPERT_INT4")
+
+
+def check(ok, what):
+ print((" ok " if ok else " FAIL ") + what, flush=True)
+ if not ok:
+ FAILS.append(what)
+
+
+def run(engine, model, args, extra, stdin=None):
+ env = {k: v for k, v in os.environ.items() if k not in MTP_KEYS}
+ env.update(BASE_ENV)
+ env["SNAP"] = str(model)
+ env.update(extra)
+ return subprocess.run([str(Path(engine).resolve()), *map(str, args)], env=env, input=stdin,
+ capture_output=True, timeout=600)
+
+
+SPEC_LINE = re.compile(r"\[qwen38 MTP\] (\S+): ([0-9.]+) tokens/forward \((\d+) forwards per (\d+) tokens\) "
+ r"\| acceptance [0-9.]+% \((\d+)/(\d+) drafts\)")
+
+
+def spec_counts(stderr):
+ """(forwards, tokens, accepted, drafts) of the run's speculation line."""
+ found = SPEC_LINE.search(stderr.decode(errors="replace"))
+ return tuple(int(found.group(i)) for i in (3, 4, 5, 6)) if found else None
+
+
+def expected_counts(n_new, mode):
+ """What q38_spec_step must count for a CLI run of n_new tokens in a forced
+ mode: the prompt's forward gives token 1, then one call per token fed; a
+ call drafts when two more tokens may follow, and the next call takes the
+ draft's logits with no forward when the draft stands."""
+ drafts = accepted = forwards = tokens = 0
+ ahead = None
+ for s in range(n_new - 1):
+ tokens += 1
+ if ahead is not None:
+ standing, ahead = ahead, None
+ if standing:
+ accepted += 1
+ continue
+ if n_new - 1 - s >= 2:
+ standing = mode == "accept" or (mode == "mixed" and drafts % 2 == 0)
+ drafts += 1
+ ahead = standing
+ forwards += 1
+ return forwards, tokens, accepted, drafts
+
+
+# ---------- head: draft logits against the torch reference ----------
+
+def read_dump(path, vocab):
+ raw = Path(path).read_bytes()
+ record = 8 + 4 * vocab
+ if len(raw) % record:
+ raise ValueError(f"{path}: {len(raw)} bytes is not a whole number of {record}-byte records")
+ out = []
+ for at in range(0, len(raw), record):
+ row, token = np.frombuffer(raw[at:at + 8], np.int32)
+ out.append((int(row), int(token), np.frombuffer(raw[at + 8:at + record], np.float32)))
+ return out
+
+
+def check_head(engine, model):
+ """The head's numbers. The fixture's four streams end with nearly equal
+ norms, so wirings a and b part by only ~1e-5 on its logits; the engine
+ meets its own wiring's float32 reference ~100x closer than that, which is
+ what tells the wirings apart here."""
+ import qwen38_mtp_ref as ref
+ data = json.loads((model / "ref.json").read_text(encoding="utf-8"))
+ ids, vocab = data["full_ids"], len(data["final_logits"])
+ print(f"head: {model}")
+ references = {w: ref.mtp_logits(model, ids, w).numpy() for w in "abc"}
+ for a, b in (("a", "b"), ("a", "c"), ("b", "c")):
+ gap = float(np.abs(references[a] - references[b]).max())
+ check(gap > 2e-6, f"wirings {a} and {b} give different logits (max |diff| {gap:.3g})")
+ with tempfile.TemporaryDirectory() as tmp:
+ for wiring in "abc":
+ rows_seen = set()
+ for mode in ("reject", "mixed"):
+ dump = Path(tmp) / f"{wiring}-{mode}.f32"
+ # reject drafts at every token (rows past the prompt, one pending
+ # row each); mixed reaches the head with two pending rows after
+ # each accepted draft. The BF16 trunk: every fixture matrix is
+ # under the 1 MiB int8 threshold.
+ r = run(engine, model, [2, 8, model / "ref.json"],
+ {"Q38_MTP": "1", "Q38_MTP_WIRING": wiring, "Q38_MTP_FORCE": mode,
+ "Q38_MTP_DUMP": str(dump), "Q38_TRUNK_MIN_KB": "1024"})
+ if r.returncode:
+ check(False, f"wiring {wiring} {mode}: engine exit {r.returncode}\n"
+ + r.stderr.decode(errors="replace")[-2000:])
+ continue
+ worst, nearest_other, argmax_ok, rows = 0.0, float("inf"), True, 0
+ for row, token, logits in read_dump(dump, vocab):
+ if token != ids[row + 1]:
+ check(False, f"wiring {wiring} {mode}: draft row {row} paired token {token}, "
+ f"the sequence has {ids[row + 1]}")
+ continue
+ errors = {w: float(np.abs(logits - references[w][row]).max()) for w in "abc"}
+ worst = max(worst, errors[wiring])
+ nearest_other = min(nearest_other, *(e for w, e in errors.items() if w != wiring))
+ argmax_ok &= int(logits.argmax()) == int(references[wiring][row].argmax())
+ rows += 1
+ rows_seen.add(row)
+ check(rows > 0 and worst <= 1e-6 and 20 * worst <= nearest_other and argmax_ok,
+ f"wiring {wiring} {mode}: {rows} draft rows within {worst:.2g} of the reference "
+ f"(the other wirings' are {nearest_other:.2g} away), argmax {'equal' if argmax_ok else 'DIFFERS'}")
+ check(rows_seen >= set(range(len(data["prompt_ids"]) - 1, len(ids) - 3)),
+ f"wiring {wiring}: rows {sorted(rows_seen)} compared")
+
+
+# ---------- lossless: MTP on == MTP off ----------
+
+def check_lossless(engine, model, ref_name, extra=None):
+ extra = dict(extra or {})
+ data = json.loads((model / ref_name).read_text(encoding="utf-8"))
+ n_new = len(data["full_ids"]) - len(data["prompt_ids"])
+ print(f"lossless: {model} ({ref_name}{', ' + ' '.join(f'{k}={v}' for k, v in extra.items()) if extra else ''})")
+ configs, failed_before = 0, len(FAILS)
+ with tempfile.TemporaryDirectory() as tmp:
+ for trunk in ("1024", "0"):
+ for batch in ("0", "1"):
+ for cap in (1, 2, 8):
+ tag = f"trunk={trunk} batch={batch} cap={cap}"
+ env = dict(extra, Q38_TRUNK_MIN_KB=trunk, Q38_PREFILL_BATCH=batch)
+ base_dump = Path(tmp) / "off.f32"
+ off = run(engine, model, [cap, 8, model / ref_name], dict(env, DUMP=str(base_dump)))
+ if off.returncode:
+ check(False, f"{tag}: MTP off fails the oracle\n" + off.stdout.decode(errors="replace"))
+ continue
+ base_logits = base_dump.read_bytes()
+ for mode in ("", "reject", "accept", "mixed"):
+ dump = Path(tmp) / "on.f32"
+ on = run(engine, model, [cap, 8, model / ref_name],
+ dict(env, DUMP=str(dump), Q38_MTP="1", Q38_MTP_FORCE=mode))
+ name = f"{tag} {mode or 'normal'}"
+ same = on.returncode == 0 and on.stdout == off.stdout and dump.read_bytes() == base_logits
+ counts = spec_counts(on.stderr)
+ if mode:
+ want = expected_counts(n_new, mode)
+ same &= counts == want
+ else:
+ same &= counts is not None and counts[2] <= counts[3] and counts[0] <= counts[1]
+ if not same:
+ check(False, f"{name}: output differs from MTP off or wrong accounting "
+ f"(counts {counts}, want {expected_counts(n_new, mode) if mode else 'consistent'})\n"
+ + on.stdout.decode(errors="replace") + on.stderr.decode(errors="replace")[-1500:])
+ configs += 1
+ off = run(engine, model, [8, 8, model / ref_name], dict(extra))
+ for knob in ({"Q38_MTP_WIRING": "b"}, {"Q38_MTP_WIRING": "c"}, {"Q38_MTP_CAP": "1"}):
+ on = run(engine, model, [8, 8, model / ref_name], dict(extra, Q38_MTP="1", **knob))
+ name = " ".join(f"{k}={v}" for k, v in knob.items())
+ check(on.returncode == 0 and on.stdout == off.stdout, f"{name}: output equals MTP off")
+ want = {m: expected_counts(n_new, m) for m in ("reject", "accept", "mixed")}
+ check(len(FAILS) == failed_before,
+ f"{configs} runs byte-identical to MTP off (stdout and last logits); forced counts "
+ + ", ".join(f"{m} {w[0]} forwards/{w[1]} tokens {w[2]}/{w[3]} accepted" for m, w in want.items()))
+
+
+# ---------- text: a longer greedy decode ----------
+
+def check_text(engine, model, extra=None):
+ print(f"text: {model}")
+ with tempfile.TemporaryDirectory() as tmp:
+ prompt = Path(tmp) / "prompt.txt"
+ prompt.write_bytes(b"123 456: 7")
+ outs = {}
+ for mode in ("off", "", "reject"):
+ dump = Path(tmp) / f"{mode}.f32"
+ env = dict(extra or {}, N_NEW="40", DUMP=str(dump), TOK=str(model / "tokenizer.json"))
+ if mode != "off":
+ env.update(Q38_MTP="1", Q38_MTP_FORCE=mode)
+ r = run(engine, model, [2, 8, prompt], env)
+ outs[mode] = (r.returncode, r.stdout, re.findall(rb"Text +:.*", r.stderr), dump.read_bytes() if dump.exists() else b"")
+ if mode == "":
+ counts = spec_counts(r.stderr)
+ print(f" normal mode on 40 tokens: forwards/tokens/accepted/drafts = {counts}")
+ check(outs["off"][0] == 0 and outs["off"][2], "40-token text decode runs with MTP off")
+ for mode in ("", "reject"):
+ check(outs[mode] == outs["off"], f"40-token text decode, MTP {mode or 'normal'}: same text and last logits")
+
+
+# ---------- serve: several turns through the wire protocol ----------
+
+SERVE_TURNS = [
+ (1, b"123 456:", 24, "0", "1", ""),
+ (2, b"123 456: 0", 24, "0", "1", " pin=1"),
+ (3, b"123 456: 01", 20, "0", "1", " logprobs=2"),
+ (4, b"789 012:", 16, "0.9", "0.95", ""),
+ (5, b"123 456: 0123", 20, "0", "1", ""),
+]
+
+
+def serve_session(engine, model, extra, turns=SERVE_TURNS):
+ """Turns sent one at a time, each when the previous one is DONE (a SUBMIT
+ during a turn is refused as busy). Returns the exit code, the frames that
+ must not depend on drafting, and stderr."""
+ env = {k: v for k, v in os.environ.items() if k not in MTP_KEYS}
+ env.update(BASE_ENV)
+ env.update(extra, SNAP=str(model), SERVE="1", TOK=str(model / "tokenizer.json"), Q38_MAXT="128")
+ with tempfile.TemporaryFile() as err:
+ p = subprocess.Popen([str(Path(engine).resolve()), "1", "8"], env=env, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=err)
+ frames = []
+
+ def frame():
+ line = p.stdout.readline()
+ if not line:
+ raise EOFError("engine closed")
+ fields = line.replace(b"\x01", b"").split()
+ if fields and fields[0] in (b"DATA", b"ECHO"):
+ n = int(fields[2])
+ line = line.rstrip(b"\n") + b"|" + p.stdout.read(n)
+ p.stdout.read(1)
+ return fields, line
+
+ try:
+ while frame()[0][:1] != [b"READY"]:
+ pass
+ for rid, text, count, temp, top_p, ext in turns:
+ p.stdin.write(f"SUBMIT {rid} 0 {len(text)} {count} {temp} {top_p}{ext}\n".encode() + text + b"\n")
+ p.stdin.flush()
+ while True:
+ fields, line = frame()
+ if not fields or fields[0] in (b"STAT", b"PROF", b"EMAP", b"HITS", b"CAPS"):
+ continue # timings, and the expert residency the draft rows change
+ if fields[0] == b"DONE": # tokens/s and RSS vary: keep the count and the reason
+ line = b" ".join(fields[:4] + fields[-2:])
+ frames.append(line)
+ if fields[0] in (b"DONE", b"ERROR"):
+ break
+ p.stdin.close()
+ rc = p.wait(timeout=60)
+ except Exception as exc: # noqa: BLE001 -- reported as a failed gate
+ p.kill()
+ p.wait()
+ frames.append(f"harness: {exc!r}".encode())
+ rc = -1
+ err.seek(0)
+ return rc, frames, err.read()
+
+
+def check_serve(engine, model, extra=None):
+ extra = dict(extra or {})
+ print(f"serve: {model}")
+ rc, base, _ = serve_session(engine, model, extra)
+ done = [f for f in base if f.startswith(b"DONE")]
+ check(rc == 0 and len(done) == 5, f"MTP off: five turns served ({len(done)} DONE)")
+ for mode in ("", "reject"):
+ rc, frames, err = serve_session(engine, model, dict(extra, Q38_MTP="1", Q38_MTP_FORCE=mode))
+ turns = len(re.findall(rb"\[qwen38 MTP\] turn \d+:", err))
+ check(rc == 0 and frames == base and turns == 5,
+ f"MTP {mode or 'normal'}: every frame equals MTP off (prefix reuse, pin, logprobs, "
+ f"sampled turn), {turns} per-turn MTP lines")
+ if frames != base:
+ for a, b in zip(base, frames):
+ if a != b:
+ print(f" first difference:\n off {a[:160]!r}\n on {b[:160]!r}")
+ break
+ # The head's place survives a restored prefix: after the prompt cache
+ # (turn 2 extends turn 1) and after a pin (turn 8 extends turn 6's pin;
+ # turn 7 moved the prompt cache to a longer prompt that turn 8 leaves
+ # at position 9, and kept the pinned rows), its drafts are bit for bit
+ # those of a session that sees the turn first.
+ config = json.loads((model / "config.json").read_text(encoding="utf-8"))
+ vocab = config.get("text_config", config)["vocab_size"]
+ turn = {t[0]: t for t in SERVE_TURNS + [(6, b"123 456:", 4, "0", "1", " pin=1"),
+ (7, b"123 456: 01", 4, "0", "1", ""),
+ (8, b"123 456: 9", 20, "0", "1", "")]}
+ with tempfile.TemporaryDirectory() as tmp:
+ for history, last, how in (((1,), 2, "the prompt cache"), ((6, 7), 8, "a pin")):
+ dumps = []
+ for turns in ([turn[i] for i in history] + [turn[last]], [turn[last]]):
+ dump = Path(tmp) / f"{len(dumps)}.f32"
+ rc, _, _ = serve_session(engine, model, dict(extra, Q38_MTP="1", Q38_MTP_DUMP=str(dump)), turns)
+ dumps.append(read_dump(dump, vocab) if rc == 0 else [])
+ warm, fresh = dumps
+ same = bool(fresh) and len(warm) > len(fresh) and all(
+ a[0] == b[0] and a[1] == b[1] and a[2].tobytes() == b[2].tobytes()
+ for a, b in zip(warm[-len(fresh):], fresh))
+ check(same, f"turn {last} after {how}: its {len(fresh)} drafts' logits equal a fresh session's")
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--engine", required=True, type=Path)
+ ap.add_argument("--bf16", type=Path, help="tools/make_qwen38_tiny.py --mtp")
+ ap.add_argument("--fp8", type=Path, help="... --fp8-experts --mtp")
+ ap.add_argument("--int4", type=Path, help="... --fp8-experts --int4-experts --expert-gain 3 --mtp")
+ ap.add_argument("--skip-head", action="store_true", help="no torch reference (lossless gates only)")
+ a = ap.parse_args()
+ for model in (a.bf16, a.fp8):
+ if model and not a.skip_head:
+ check_head(a.engine, model)
+ for model in (a.bf16, a.fp8):
+ if model:
+ check_lossless(a.engine, model, "ref.json")
+ if a.int4:
+ check_lossless(a.engine, a.int4, "ref_int4.json", {"Q38_EXPERT_INT4": "1"})
+ for model in (a.bf16, a.fp8, a.int4):
+ if model and (model / "tokenizer.json").exists():
+ extra = {"Q38_EXPERT_INT4": "1"} if model == a.int4 else {}
+ check_text(a.engine, model, extra)
+ check_serve(a.engine, model, extra)
+ if FAILS:
+ print(f"qwen38 MTP: {len(FAILS)} check(s) failed")
+ return 1
+ print("qwen38 MTP: head matches the reference for wirings a, b, c; decoding with drafts is "
+ "byte-identical to decoding without them")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/make_qwen38_tiny.py b/c/tools/make_qwen38_tiny.py
index 7e53aeac..287e4cdf 100644
--- a/c/tools/make_qwen38_tiny.py
+++ b/c/tools/make_qwen38_tiny.py
@@ -7,7 +7,8 @@ wrapper. Consequently its state-dict names start at ``model.embed_tokens``;
the released multimodal checkpoint adds the extra ``model.language_model``
prefix. This keeps vision and MTP weights out while retaining the exact text
module names (``layers.*.linear_attn``, ``layers.*.self_attn``, ``layers.*.ple``
-and ``layers.*.mlp``) an engine needs to support.
+and ``layers.*.mlp``) an engine needs to support. ``--mtp`` adds an MTP head
+under the release's ``mtp.*`` names (transformers has none; see _add_mtp).
Usage::
@@ -221,6 +222,67 @@ def _rewrite_shard_fp8(out: Path, packed):
print(f"fp8 experts: {len(packed)} matrices rewritten as F8_E4M3 + BF16 weight_scale_inv")
+MTP_SEED_OFFSET = 0x4D5450 # "MTP": the head's weights come from their own stream
+MTP_ROPE_THETA = 50000.0 # not the model's 10000, so a head on the wrong base shows
+
+
+def _add_mtp(out: Path, config, seed, fp8_experts=False, expert_gain=1.0):
+ """Add an MTP head with the release's tensor names to the saved fixture.
+
+ Transformers has no MTP module (it ignores ``mtp.*`` on load), so the head
+ is written here: random weights, drawn from a generator of their own after
+ the model is saved, so the model, its tensors and ref.json are exactly
+ those of the same fixture without ``--mtp``. The head's decoder layer has
+ the tensors of the model's last attention layer (``mtp.layers.0.*``, same
+ names after the layer prefix and same shapes); around it sit
+ ``pre_fc_norm_embedding`` [H], ``pre_fc_norm_hidden`` [hc_count*H],
+ ``fc_embedding`` and ``fc_hidden`` [H, H] and a ``hyper_connection_mixer``
+ shaped like the model's. Norm weights get small random values (the
+ model's are zero, Qwen4-Exp norms scale by 1+w) so a norm reading the
+ wrong slice of its weight shows. With ``fp8_experts`` the head's routed
+ experts are e4m3 with 128x128 block scales like the model's."""
+ from safetensors.torch import load_file, save_file
+ path = out / "model.safetensors"
+ tensors = load_file(str(path))
+ gen = torch.Generator().manual_seed(seed + MTP_SEED_OFFSET)
+ attn = [i for i, kind in enumerate(config.layer_types) if kind != "linear_attention"
+ and (i + 1) not in (config.ple_layer_ids or [])]
+ source = f"model.layers.{attn[-1]}."
+ std = config.initializer_range
+
+ def draw(shape, scale):
+ return (torch.randn(shape, generator=gen) * scale).to(torch.bfloat16)
+
+ head = {}
+ for name in sorted(k for k in tensors if k.startswith(source)):
+ shape = tuple(tensors[name].shape)
+ suffix = name[len(source):]
+ scale = 0.1 if len(shape) == 1 else std * (expert_gain if ".mlp.experts." in suffix else 1.0)
+ head["mtp.layers.0." + suffix] = draw(shape, scale)
+ H, W = config.hidden_size, config.hidden_size * config.hc_count
+ head["mtp.pre_fc_norm_embedding.weight"] = draw((H,), 0.1)
+ head["mtp.pre_fc_norm_hidden.weight"] = draw((W,), 0.1)
+ head["mtp.fc_embedding.weight"] = draw((H, H), 0.1)
+ head["mtp.fc_hidden.weight"] = draw((H, H), 0.1)
+ for name in sorted(k for k in tensors if k.startswith("model.hyper_connection_mixer.")):
+ shape = tuple(tensors[name].shape)
+ head["mtp." + name[len("model."):]] = draw(shape, 0.1 if len(shape) == 1 else std)
+ if fp8_experts:
+ for name in sorted(k for k in head if ".mlp.experts." in k):
+ q, scale_inv, _ = _fp8_block_quant(head[name])
+ head[name] = q.contiguous()
+ head[name + "_scale_inv"] = scale_inv.to(torch.bfloat16).contiguous()
+ tensors.update(head)
+ save_file(tensors, str(path), metadata={"format": "pt"})
+ cfg_path = out / "config.json"
+ cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
+ cfg["mtp_num_hidden_layers"] = 1
+ cfg["mtp"] = {"layer_types": ["full_attention"], "rope_theta": MTP_ROPE_THETA, "hybrid": True}
+ cfg["mtp_use_dedicated_embeddings"] = False
+ cfg_path.write_text(json.dumps(cfg, indent=2) + "\n", encoding="utf-8")
+ print(f"mtp head: {len(head)} tensors (decoder layer shaped like {source}*)")
+
+
def _int4_experts_in_model(model, out: Path):
"""Write the int4-g64 sidecar of the fixture just saved, with the converter
the real model goes through (tools/convert_qwen38_experts_int4.py), then read its bytes
@@ -259,7 +321,7 @@ def _int4_experts_in_model(model, out: Path):
def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_experts=False,
- int4_experts=False, expert_gain=1.0):
+ int4_experts=False, expert_gain=1.0, mtp=False):
if max_new < 1:
raise ValueError("max_new must be at least 1")
random.seed(seed)
@@ -351,6 +413,8 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e
# from, and qwen38 picks it up by itself: regenerating the fixture drops it.
shutil.rmtree(out / "experts-int4g64", ignore_errors=True)
model.save_pretrained(str(out), safe_serialization=True)
+ if mtp:
+ _add_mtp(out, config, seed, fp8_experts=fp8_experts, expert_gain=expert_gain)
if packed:
_rewrite_shard_fp8(out, packed)
@@ -414,11 +478,14 @@ def main():
parser.add_argument("--int4-experts", action="store_true",
help="also convert the routed experts to the experts-int4g64/ sidecar "
"and write ref_int4.json, the reference of the dequantized sidecar")
+ parser.add_argument("--mtp", action="store_true",
+ help="add an MTP head (mtp.*, random weights from their own seed) with the "
+ "release's tensor names; the model and ref.json do not change")
args = parser.parse_args()
prompt = [int(x) for x in args.prompt_ids.split(",") if x.strip()] if args.prompt_ids else None
build(args.out, prompt_ids=prompt, max_new=args.max_new, seed=args.seed, emit_ref=not args.no_ref,
fp8_experts=args.fp8_experts, int4_experts=args.int4_experts,
- expert_gain=args.expert_gain)
+ expert_gain=args.expert_gain, mtp=args.mtp)
if __name__ == "__main__":
diff --git a/c/tools/qwen38_mtp_ref.py b/c/tools/qwen38_mtp_ref.py
new file mode 100644
index 00000000..414207fe
--- /dev/null
+++ b/c/tools/qwen38_mtp_ref.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Reference logits of the Qwen3.8 MTP head on a tiny fixture, in torch.
+
+The head is built from our own reading of its tensors, the same reading
+qwen38_core.h documents next to q38_mtp_input (the MTP head):
+
+* the model's four hyper-connection streams at position p -- the
+ ``hidden_size * hc_count`` vector the final mixer reads, captured here as the
+ input of ``model.hyper_connection_mixer`` -- and the embedding of the token
+ at p+1 become four streams through ``pre_fc_norm_*`` and ``fc_*``, in one of
+ the wirings a, b or c (``fuse`` below);
+* those streams run through one decoder layer with the tensors under
+ ``mtp.layers.0`` -- the transformers ``Qwen4ExpTextDecoderLayer``, the oracle
+ the engine's own layers are checked against, built as an attention layer
+ without PLE -- with RoPE on ``mtp.rope_theta`` and causal attention over the
+ pairs;
+* ``mtp.hyper_connection_mixer`` (a ``Qwen4ExpTextGatedResidual`` without
+ inject) and the shared ``lm_head`` give the logits.
+
+Row i of the result pairs the streams at position i with ``ids[i + 1]``: the
+logits that predict ``ids[i + 2]``. Everything runs in float32 on the bytes on
+disk: BF16 widens exactly, and the FP8 routed experts (the model's and the
+head's) are dequantized the way qwen38's expanded path does it, e4m3 value
+times the block's ``weight_scale_inv`` in f32, so the reference sees the
+engine's weights and not the BF16 rounding the fixture generator kept in
+memory.
+
+ python tools/qwen38_mtp_ref.py --model ./qwen38_tiny_mtp --wiring a
+"""
+
+import argparse
+import copy
+import json
+import math
+from pathlib import Path
+
+import numpy as np
+import torch
+
+FP8_BLOCK = 128
+
+
+def _e4m3_table():
+ """quant.h's E4M3 table (float8_e4m3fn: bias 7, subnormals, 0x7F/0xFF NaN)."""
+ table = np.empty(256, np.float32)
+ for byte in range(256):
+ exponent, mantissa = (byte >> 3) & 0xF, byte & 7
+ if exponent == 0xF and mantissa == 7:
+ value = math.nan
+ elif exponent == 0:
+ value = mantissa / 8.0 * 2.0 ** -6
+ else:
+ value = (1.0 + mantissa / 8.0) * 2.0 ** (exponent - 7)
+ table[byte] = -value if byte & 0x80 else value
+ return torch.from_numpy(table)
+
+
+E4M3 = _e4m3_table()
+
+
+def load_tensors(model_dir: Path):
+ """Every tensor of the snapshot as float32; FP8 matrices dequantized with
+ their block scales, the scale tensors themselves dropped."""
+ from safetensors.torch import load_file
+ raw = {}
+ for path in sorted(model_dir.glob("*.safetensors")):
+ raw.update(load_file(str(path)))
+ out = {}
+ for name, tensor in raw.items():
+ if name.endswith("_scale_inv"):
+ continue
+ if tensor.dtype == torch.float8_e4m3fn:
+ scale = raw[name + "_scale_inv"].float()
+ rows, cols = tensor.shape
+ expand = scale.repeat_interleave(FP8_BLOCK, 0)[:rows].repeat_interleave(FP8_BLOCK, 1)[:, :cols]
+ out[name] = E4M3[tensor.view(torch.uint8).long()] * expand
+ elif tensor.is_floating_point():
+ out[name] = tensor.float()
+ else:
+ out[name] = tensor # the PLE hash tables are int64
+ return out
+
+
+def _fuse_experts(weights, prefix, experts):
+ """Per-expert gate/up/down -> transformers' fused gate_up_proj / down_proj."""
+ gate_up = torch.stack([torch.cat([weights[f"{prefix}{e}.gate_proj.weight"],
+ weights[f"{prefix}{e}.up_proj.weight"]], 0) for e in range(experts)])
+ down = torch.stack([weights[f"{prefix}{e}.down_proj.weight"] for e in range(experts)])
+ return gate_up, down
+
+
+def load_model(model_dir: Path):
+ """The fixture's model in float32 with the exact weights on disk, its
+ config, and the f32 tensors."""
+ from transformers import Qwen4ExpForCausalLM, Qwen4ExpTextConfig
+ cfg = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
+ config = Qwen4ExpTextConfig.from_pretrained(str(model_dir))
+ if hasattr(config, "quantization_config"):
+ del config.quantization_config # the experts are dequantized here, not by transformers
+ config._attn_implementation = "eager"
+ model = Qwen4ExpForCausalLM(config).float().eval()
+ weights = load_tensors(model_dir)
+ state = model.state_dict()
+ with torch.no_grad():
+ for name, param in state.items():
+ if name.endswith(".mlp.experts.gate_up_proj") or name.endswith(".mlp.experts.down_proj"):
+ base = name[: name.rindex(".") + 1]
+ gate_up, down = _fuse_experts(weights, base, config.num_experts)
+ param.copy_(gate_up if name.endswith("gate_up_proj") else down)
+ elif name in weights:
+ param.copy_(weights[name])
+ elif name.endswith(("layer_multipliers", "ngram_heads_vocab_sizes", "ngram_heads_offsets")):
+ continue # buffers the model computes itself
+ else:
+ shards = sorted(k for k in weights if k.startswith(name.replace(".weight", ".shard_")))
+ if not shards:
+ raise KeyError(f"{name}: not in the snapshot")
+ param.copy_(torch.cat([weights[k] for k in shards], 0))
+ return model, config, cfg, weights
+
+
+def model_streams(model, ids):
+ """The four streams the final mixer reads, [len(ids), hc_count * hidden]."""
+ captured = {}
+
+ def hook(module, args):
+ captured["streams"] = args[0].detach()
+
+ handle = model.model.hyper_connection_mixer.register_forward_pre_hook(hook)
+ try:
+ with torch.no_grad():
+ model(input_ids=torch.tensor([ids]), use_cache=False)
+ finally:
+ handle.remove()
+ return captured["streams"][0]
+
+
+def rms0(x, weight, eps):
+ """Qwen4-Exp's zero-centered RMSNorm over the last dimension: x / rms * (1 + w)."""
+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + weight)
+
+
+def fuse(streams, embedded, weights, config, wiring):
+ """The head's input streams [S, hc_count*H] from the model's streams at p
+ [S, hc_count*H] and the embeddings of the tokens at p+1 [S, H]."""
+ C, H, eps = config.hc_count, config.hidden_size, config.rms_norm_eps
+ w_hid = weights["mtp.pre_fc_norm_hidden.weight"]
+ fe = rms0(embedded, weights["mtp.pre_fc_norm_embedding.weight"], eps) @ weights["mtp.fc_embedding.weight"].T
+ fc_hidden = weights["mtp.fc_hidden.weight"]
+ S = streams.shape[0]
+ if wiring == "a": # one norm over the whole vector, fc_hidden per stream, embedding into all four
+ normed = rms0(streams, w_hid, eps).view(S, C, H)
+ return (normed @ fc_hidden.T + fe[:, None, :]).reshape(S, C * H)
+ per_stream = rms0(streams.view(S, C, H), w_hid.view(C, H), eps)
+ if wiring == "b": # a norm per stream, fc_hidden per stream, embedding into all four
+ return (per_stream @ fc_hidden.T + fe[:, None, :]).reshape(S, C * H)
+ if wiring == "c": # a norm per stream, the streams' mean, one fc_hidden, copied to all four
+ one = per_stream.mean(1) @ fc_hidden.T + fe
+ return one.repeat(1, C)
+ raise ValueError(f"wiring {wiring!r}: a, b or c")
+
+
+def mtp_layer(config, cfg, weights):
+ """The head's decoder layer: transformers' layer, as one of the model's
+ attention layers without PLE, holding the mtp.layers.0 tensors."""
+ from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextDecoderLayer
+ attn = [i for i, kind in enumerate(config.layer_types) if kind != "linear_attention"
+ and (i + 1) not in (config.ple_layer_ids or [])]
+ layer = Qwen4ExpTextDecoderLayer(config, attn[-1]).float().eval()
+ state = layer.state_dict()
+ with torch.no_grad():
+ for name, param in state.items():
+ if name.endswith("mlp.experts.gate_up_proj") or name.endswith("mlp.experts.down_proj"):
+ gate_up, down = _fuse_experts(weights, "mtp.layers.0.mlp.experts.", config.num_experts)
+ param.copy_(gate_up if name.endswith("gate_up_proj") else down)
+ else:
+ param.copy_(weights["mtp.layers.0." + name])
+ return layer
+
+
+def mtp_logits(model_dir, ids, wiring):
+ """[len(ids) - 1, vocab]: row i is the head's logits for the pair
+ (streams at i, ids[i + 1])."""
+ from transformers.models.qwen4_exp.modeling_qwen4_exp import (
+ Qwen4ExpTextGatedResidual, Qwen4ExpTextRotaryEmbedding)
+ model_dir = Path(model_dir)
+ model, config, cfg, weights = load_model(model_dir)
+ streams = model_streams(model, ids)[:-1]
+ nxt = torch.tensor(ids[1:])
+ embedded = weights["model.embed_tokens.weight"][nxt]
+ hyper = fuse(streams, embedded, weights, config, wiring)
+ S = hyper.shape[0]
+ head_config = copy.deepcopy(config)
+ theta = (cfg.get("mtp") or {}).get("rope_theta", config.rope_parameters["rope_theta"])
+ head_config.rope_parameters = dict(config.rope_parameters, rope_theta=theta)
+ rotary = Qwen4ExpTextRotaryEmbedding(config=head_config)
+ positions = torch.arange(S).view(1, 1, -1).expand(3, 1, -1)
+ position_embeddings = rotary(hyper[None], positions)
+ mask = torch.full((1, 1, S, S), torch.finfo(torch.float32).min).triu(1)
+ layer = mtp_layer(config, cfg, weights)
+ mixer = Qwen4ExpTextGatedResidual(config, use_combine=False).float().eval()
+ with torch.no_grad():
+ for name, param in mixer.state_dict().items():
+ param.copy_(weights["mtp.hyper_connection_mixer." + name])
+ out = layer(hyper[None], position_embeddings=position_embeddings, attention_mask=mask,
+ conv_mask=None, past_key_values=None, ple_input_ids=None)
+ logits = mixer(out)[0] @ weights["lm_head.weight"].T
+ return logits
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model", required=True, type=Path)
+ parser.add_argument("--wiring", default="a", choices="abc")
+ parser.add_argument("--ids", help="comma-separated ids (default: ref.json full_ids)")
+ args = parser.parse_args()
+ ids = ([int(x) for x in args.ids.split(",")] if args.ids else
+ json.loads((args.model / "ref.json").read_text(encoding="utf-8"))["full_ids"])
+ logits = mtp_logits(args.model, ids, args.wiring)
+ for i, row in enumerate(logits):
+ print(f"row {i}: token {ids[i + 1]} -> draft {int(row.argmax())} (max {float(row.max()):.6f})")
+
+
+if __name__ == "__main__":
+ main()