Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: python wrapper, Hadamard helpers, .tqx container, Gyro re-encode and original-weight comparison scripts, pilot driver
Niko12217e9531b3daac
diff --git a/tools/tqenc.py b/tools/tqenc.pynew file mode 100644index 000000000..334063e2f--- /dev/null+++ b/tools/tqenc.py@@ -0,0 +1,188 @@+"""tqenc.py - Python side of the trellis encoder (tools/trellis/tqenc.cpp, built to libtqenc.so by+`sh tools/trellis/build.sh lib`): encode / decode blocks of TQ2_T, TQK6, TQK7 and the experimental TQK5X / TQK4X, the+input-only block Hadamard of the DeepSeek plan, and error metrics.++Rows of a tensor are blocks of 128 consecutive weights along the INPUT dimension (the GGUF fastest axis), exactly as the+decoders read them. A tensor of shape [rows, n] (n a multiple of 128) is encoded row by row; the output has+rows * n/128 * block_bytes bytes.+"""+from __future__ import annotations++import ctypes+import os+import pathlib++import numpy as np++TQ2_T, TQK6, TQK7, TQK5X, TQK4X = 144, 145, 146, 224, 225+TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X"}+TYPE_BY_NAME = {v: k for k, v in TYPE_NAMES.items()}+KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4}+BLOCK = 128+++def block_bytes(t: int) -> int:+ return 2 + 4 * KBITS[t]+++def bpw(t: int) -> float:+ return block_bytes(t) * 8 / BLOCK+++_lib = None+++def lib() -> ctypes.CDLL:+ global _lib+ if _lib is None:+ cands = [os.environ.get("TQENC_LIB", ""),+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc.so"),+ "libtqenc.so"]+ last = None+ for c in cands:+ if not c:+ continue+ try:+ _lib = ctypes.CDLL(c)+ break+ except OSError as e:+ last = e+ if _lib is None:+ raise RuntimeError("libtqenc.so not found (sh tools/trellis/build.sh lib): %s" % last)+ _lib.tqe_encode.restype = ctypes.c_double+ _lib.tqe_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,+ ctypes.c_void_p, ctypes.c_void_p]+ _lib.tqe_decode.restype = None+ _lib.tqe_decode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p]+ return _lib+++def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, ncand: int = 1, exact: int = 0,+ scale_tries: int = 1, threads: int = 0):+ """w: float32 array whose last axis is a multiple of 128. Returns (packed uint8 [nblocks, block_bytes],+ per-block error float64 [nblocks], total error, stats dict). Error = sum h (w - decoded)^2 (h = 1 if None)."""+ w = np.ascontiguousarray(w, dtype=np.float32)+ if w.shape[-1] % BLOCK:+ raise ValueError("last axis %d is not a multiple of 128" % w.shape[-1])+ nb = w.size // BLOCK+ out = np.zeros((nb, block_bytes(t)), dtype=np.uint8)+ err = np.zeros(nb, dtype=np.float64)+ stats = np.zeros(2, dtype=np.int64)+ hp = None+ if h is not None:+ h = np.ascontiguousarray(h, dtype=np.float32)+ if h.shape != w.shape:+ raise ValueError("importance weights must have the shape of w")+ hp = h.ctypes.data+ tot = lib().tqe_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, exact, scale_tries, threads,+ err.ctypes.data, stats.ctypes.data)+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}+++def decode(packed: np.ndarray, t: int) -> np.ndarray:+ """packed: uint8 array of whole blocks. Returns float32 [nblocks * 128] (d * codebook)."""+ packed = np.ascontiguousarray(packed, dtype=np.uint8)+ bb = block_bytes(t)+ if packed.size % bb:+ raise ValueError("%d bytes is not a whole number of %s blocks" % (packed.size, TYPE_NAMES[t]))+ nb = packed.size // bb+ out = np.zeros(nb * BLOCK, dtype=np.float32)+ lib().tqe_decode(t, packed.ctypes.data, nb, out.ctypes.data)+ return out+++# ---------------------------------------------------------------------------------------------------- Hadamard++def hadamard_matrix(n: int = BLOCK) -> np.ndarray:+ """The normalized Sylvester-Walsh-Hadamard matrix (entry (-1)^popcount(r & c) / sqrt(n)); symmetric, orthonormal."""+ h = np.array([[1.0]])+ while h.shape[0] < n:+ h = np.block([[h, h], [h, -h]])+ return (h / np.sqrt(n)).astype(np.float32)+++def make_signs(width: int, seed: int) -> np.ndarray:+ """A reproducible +1/-1 vector (the plan's D). The vector itself, not the seed, is what a file must store."""+ r = np.random.default_rng(seed)+ return (r.integers(0, 2, size=width) * 2 - 1).astype(np.float32)+++def pack_signs(signs: np.ndarray) -> bytes:+ return np.packbits((signs > 0).astype(np.uint8), bitorder="little").tobytes()+++def unpack_signs(b: bytes, width: int) -> np.ndarray:+ bits = np.unpackbits(np.frombuffer(b, dtype=np.uint8), bitorder="little")[:width]+ return bits.astype(np.float32) * 2 - 1+++def rotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:+ """The input transform U = H D on the last axis, per 128-block: Wrot = W U^T, i.e. each row w -> H (signs * w).+ The same function applied to an activation x gives U x."""+ w = np.asarray(w, dtype=np.float32)+ if signs is not None:+ w = w * signs+ shp = w.shape+ return (w.reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp) # H is symmetric+++def unrotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:+ """Inverse of rotate (H is its own inverse: orthonormal and symmetric; then the signs)."""+ shp = w.shape+ x = (np.asarray(w, dtype=np.float32).reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp)+ return x * signs if signs is not None else x+++# ---------------------------------------------------------------------------------------------------- metrics++def sqnr_db(ref: np.ndarray, approx: np.ndarray) -> float:+ ref = np.asarray(ref, dtype=np.float64).ravel()+ e = np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2)+ return float("inf") if e == 0 else float(10 * np.log10(np.sum(ref ** 2) / e))+++def rel_mse(ref: np.ndarray, approx: np.ndarray) -> float:+ ref = np.asarray(ref, dtype=np.float64).ravel()+ return float(np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2) / max(np.sum(ref ** 2), 1e-300))+++# ---------------------------------------------------------------------------------------------------- .tqx container+# A small versioned file for ONE encoded matrix (research artifacts; not a GGUF). Little endian:+# "TQXP" | u32 version (1) | u32 type id | u32 rows | u32 cols | u32 flags (bit 0: input Hadamard, 128-blocks)+# | u32 n_sign_bytes | sign bits (1 = +1, LSB first, one per input column) | u32 hadamard block (128) | packed blocks+# An unknown version or type is refused before any data is read.+TQX_MAGIC = b"TQXP"+TQX_VERSION = 1+++def write_tqx(path, t: int, rows: int, cols: int, packed: np.ndarray, signs: np.ndarray | None) -> None:+ import struct+ sb = pack_signs(signs) if signs is not None else b""+ with open(path, "wb") as f:+ f.write(TQX_MAGIC + struct.pack("<IIIIII", TQX_VERSION, t, rows, cols, 1 if signs is not None else 0, len(sb)))+ f.write(sb)+ f.write(struct.pack("<I", BLOCK))+ f.write(np.ascontiguousarray(packed, dtype=np.uint8).tobytes())+++def read_tqx(path):+ import struct+ data = open(path, "rb").read()+ if data[:4] != TQX_MAGIC:+ raise ValueError("not a TQXP file")+ ver, t, rows, cols, flags, nsb = struct.unpack_from("<IIIIII", data, 4)+ if ver != TQX_VERSION:+ raise ValueError("unsupported TQXP version %d (this reader handles %d)" % (ver, TQX_VERSION))+ if t not in KBITS:+ raise ValueError("unknown trellis type %d" % t)+ at = 4 + 24+ signs = unpack_signs(data[at:at + nsb], cols) if flags & 1 else None+ at += nsb+ (blk,) = struct.unpack_from("<I", data, at)+ at += 4+ if blk != BLOCK:+ raise ValueError("unsupported Hadamard block %d" % blk)+ nb = rows * (cols // BLOCK)+ packed = np.frombuffer(data, dtype=np.uint8, count=nb * block_bytes(t), offset=at).reshape(nb, block_bytes(t))+ return t, rows, cols, packed, signsdiff --git a/tools/trellis/gyro_reencode.py b/tools/trellis/gyro_reencode.pynew file mode 100644index 000000000..e0a66d95b--- /dev/null+++ b/tools/trellis/gyro_reencode.py@@ -0,0 +1,107 @@+"""gyro_reencode.py - gate 2/3 quality check of the trellis encoder against agentionai's Gyro files.++Decodes expert tensors of a Gyro GGUF (TQK6 gate/up, TQK7 down in Gyro-S; TQ2_T in Gyro-M) with the Strata decoder+(libtqenc's decode, bit-identical to tq_states.hpp, see tqenc_test), re-encodes the decoded weights at the same type+with tools/tqenc.py and reports the reconstruction error. The decoded weights are a feasible trellis point, so an+ideal encoder recovers them exactly (error 0): the error measures the search, not the quantization.++ python3 tools/trellis/gyro_reencode.py --gguf /data/models/gyro/Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf \+ --layers 0,20,39 --experts 0,7 --rows 256 [--cfg default,rot3,scale3]+"""+from __future__ import annotations++import argparse+import mmap+import pathlib+import sys+import time++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+import gguf_reader # noqa: E402+import tqenc # noqa: E402++CFGS = {+ "fast": dict(iters=0, ncand=1, scale_tries=1),+ "default": dict(iters=1, ncand=1, scale_tries=1),+ "iters3": dict(iters=3, ncand=1, scale_tries=1),+ "rot3": dict(iters=3, ncand=3, scale_tries=1),+ "scale3": dict(iters=3, ncand=3, scale_tries=3),+ "scale5": dict(iters=3, ncand=3, scale_tries=5),+}+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--gguf", required=True)+ ap.add_argument("--layers", default="0,20,39")+ ap.add_argument("--experts", default="0")+ ap.add_argument("--rows", type=int, default=256, help="rows (output channels) of each expert slice to test")+ ap.add_argument("--cfg", default="fast,default,rot3")+ ap.add_argument("--roles", default="gate,up,down")+ ap.add_argument("--threads", type=int, default=0)+ a = ap.parse_args()++ g = gguf_reader.GGUFFile(a.gguf)+ fh = open(a.gguf, "rb")+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)+ by_name = {t.name: t for t in g.tensors}+ print("file", pathlib.Path(a.gguf).name, "tensors", len(g.tensors), "types", g.by_type())+ hk = {k: v for k, v in g.metadata.items() if k.startswith("prism.") and not isinstance(v, list)}+ print("prism metadata:", hk)+ cfgs = a.cfg.split(",")+ rows_total = {c: [0.0, 0.0, 0, 0, 0.0] for c in cfgs} # err, energy, blocks, exact blocks, seconds+ for layer in [int(x) for x in a.layers.split(",")]:+ for role in a.roles.split(","):+ name = "blk.%d.ffn_%s_exps.weight" % (layer, role)+ ti = by_name.get(name)+ if ti is None:+ print("no tensor", name)+ continue+ t = tqenc.TYPE_BY_NAME.get(ti.type_name)+ if t is None:+ print(name, "type", ti.type_name, "is not a trellis type: skipped")+ continue+ n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]+ bb = tqenc.block_bytes(t)+ row_bytes = n_in // 128 * bb+ for e in [int(x) for x in a.experts.split(",")]:+ nrows = min(a.rows, n_out)+ off = g.data_start + ti.offset + (e * n_out) * row_bytes+ raw = np.frombuffer(mm, dtype=np.uint8, count=nrows * row_bytes, offset=off)+ wd = tqenc.decode(raw, t).reshape(nrows, n_in)+ en = float(np.sum(wd.astype(np.float64) ** 2))+ scales = raw.reshape(-1, bb)[:, :2].copy().view(np.float16).astype(np.float32).ravel()+ print("%s expert %d: %d rows x %d, %s %.3f bpw, |w| rms %.4g, agentionai block scales: median %.4g, "+ "p1 %.4g, p99 %.4g, zero-scale blocks %d" % (+ name, e, nrows, n_in, ti.type_name, tqenc.bpw(t), np.sqrt(en / wd.size),+ np.median(scales), np.percentile(scales, 1), np.percentile(scales, 99), int((scales == 0).sum())))+ for c in cfgs:+ t0 = time.time()+ packed, err, tot, st = tqenc.encode(wd, t, threads=a.threads, **CFGS[c])+ sec = time.time() - t0+ dec = tqenc.decode(packed, t)+ chk = float(np.sum((wd.ravel().astype(np.float64) - dec) ** 2))+ nexact = int((err == 0).sum())+ same_bytes = int((packed.reshape(-1) == raw).sum()) / raw.size+ print(" %-8s rel MSE %.3e SQNR %7.2f dB exact blocks %5d/%d (%.1f%%) scale==theirs %.1f%% "+ "%.0f weights/s (recheck %.3e)" % (+ c, tot / en, tqenc.sqnr_db(wd, dec), nexact, err.size, 100.0 * nexact / err.size,+ 100.0 * float(np.mean(packed[:, :2].reshape(-1, 2).view(np.uint16).ravel()+ == raw.reshape(-1, bb)[:, :2].copy().view(np.uint16).ravel())),+ wd.size / sec, chk / en))+ r = rows_total[c]+ r[0] += tot; r[1] += en; r[2] += err.size; r[3] += nexact; r[4] += sec+ print("\nTOTAL over all tested slices")+ for c in cfgs:+ r = rows_total[c]+ if r[1] > 0:+ print(" %-8s rel MSE %.3e SQNR %.2f dB exact blocks %.1f%% %.0f weights/s" % (+ c, r[0] / r[1], 10 * np.log10(r[1] / max(r[0], 1e-300)), 100.0 * r[3] / r[2], r[2] * 128 / max(r[4], 1e-9)))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/gyro_vs_original.py b/tools/trellis/gyro_vs_original.pynew file mode 100644index 000000000..d27629449--- /dev/null+++ b/tools/trellis/gyro_vs_original.py@@ -0,0 +1,107 @@+"""gyro_vs_original.py - the fair comparison: agentionai's Gyro quantization of Qwen3.8-Flash-Next against OUR encoder, both+measured against the original BF16 expert weights (fetched tensor by tensor with HTTP range requests, ~10 MB per expert).++For a layer and expert: W = original (gate, up from the fused gate_up_proj, down), rotated into the file's basis with the+file's own sign vectors (prism.hadamard.sign_values, widths 2560 / 640): W' = H (s * w) per row and 128-block.+ agentionai: Wd = decode of the file's blocks+ ours: Wo = decode(encode(W')) at the same type (TQK6 gate/up, TQK7 down), several search settings+Reported: relative MSE and SQNR of each against W'.++ python3 tools/trellis/gyro_vs_original.py --gguf <Gyro-S> --layers 0,20,39 --experts 0,5 --cfg fast,default,rot3+"""+from __future__ import annotations++import argparse+import mmap+import pathlib+import sys+import time++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import gguf_reader # noqa: E402+import hf_slice # noqa: E402+import tqenc # noqa: E402++CFGS = {+ "fast": dict(iters=0, ncand=1, scale_tries=1),+ "default": dict(iters=1, ncand=1, scale_tries=1),+ "iters3": dict(iters=3, ncand=1, scale_tries=1),+ "rot3": dict(iters=3, ncand=3, scale_tries=1),+ "scale3": dict(iters=3, ncand=3, scale_tries=3),+}+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--gguf", required=True)+ ap.add_argument("--repo", default="Qwen/Qwen3.8-Flash-Next")+ ap.add_argument("--layers", default="0,20,39")+ ap.add_argument("--experts", default="0")+ ap.add_argument("--cfg", default="fast,default,rot3")+ ap.add_argument("--threads", type=int, default=0)+ a = ap.parse_args()++ g = gguf_reader.GGUFFile(a.gguf)+ fh = open(a.gguf, "rb")+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)+ by_name = {t.name: t for t in g.tensors}+ sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]+ sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)+ signs, at = {}, 0+ for w in sw:+ signs[w] = sv[at:at + w]+ at += w+ print("sign widths", sw)+ repo = hf_slice.Repo(a.repo)+ cfgs = a.cfg.split(",")+ tot = {k: [0.0, 0.0] for k in ["agentionai"] + cfgs}+ tot_role = {}+ for layer in [int(x) for x in a.layers.split(",")]:+ pre = "model.language_model.layers.%d.mlp.experts." % layer+ for e in [int(x) for x in a.experts.split(",")]:+ t0 = time.time()+ gu = repo.tensor(pre + "gate_up_proj", e)[0] # [1280, 2560]+ dn = repo.tensor(pre + "down_proj", e)[0] # [2560, 640]+ print("layer %d expert %d fetched in %.1f s: gate_up %s down %s" % (layer, e, time.time() - t0, gu.shape, dn.shape))+ mats = {"gate": gu[:640], "up": gu[640:], "down": dn}+ for role, W in mats.items():+ ti = by_name["blk.%d.ffn_%s_exps.weight" % (layer, role)]+ t = tqenc.TYPE_BY_NAME[ti.type_name]+ n_in, n_out = ti.shape[0], ti.shape[1]+ assert W.shape == (n_out, n_in), (W.shape, ti.shape)+ bb = tqenc.block_bytes(t)+ row_bytes = n_in // 128 * bb+ off = g.data_start + ti.offset + e * n_out * row_bytes+ raw = np.frombuffer(mm, dtype=np.uint8, count=n_out * row_bytes, offset=off)+ wd = tqenc.decode(raw, t).reshape(n_out, n_in)+ Wr = tqenc.rotate(W, signs[n_in])+ en = float(np.sum(Wr.astype(np.float64) ** 2))+ ea = float(np.sum((Wr.astype(np.float64) - wd) ** 2))+ # sanity: the fused gate_up order (gate first) - if swapped the SQNR collapses+ line = ["agentionai %s %.2f dB" % (ti.type_name, 10 * np.log10(en / ea))]+ key = "%s" % role+ tot["agentionai"][0] += ea; tot["agentionai"][1] += en+ tr = tot_role.setdefault(key, {k: [0.0, 0.0] for k in ["agentionai"] + cfgs})+ tr["agentionai"][0] += ea; tr["agentionai"][1] += en+ for c in cfgs:+ t1 = time.time()+ packed, err, totr, st = tqenc.encode(Wr, t, threads=a.threads, **CFGS[c])+ sec = time.time() - t1+ line.append("%s %.2f dB (%.0fk w/s)" % (c, 10 * np.log10(en / totr), Wr.size / sec / 1e3))+ tot[c][0] += totr; tot[c][1] += en+ tr[c][0] += totr; tr[c][1] += en+ print(" L%d e%d %-4s " % (layer, e, role) + " | ".join(line))+ print("\nTOTAL SQNR (dB) against the rotated original, all tested matrices")+ for k, (er, en) in tot.items():+ print(" %-10s %.3f dB (rel MSE %.4e)" % (k, 10 * np.log10(en / er), er / en))+ for role, d in tot_role.items():+ print(" %-5s " % role + " ".join("%s %.3f" % (k, 10 * np.log10(v[1] / v[0])) for k, v in d.items()))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/hf_slice.py b/tools/trellis/hf_slice.pynew file mode 100644index 000000000..b93d55914--- /dev/null+++ b/tools/trellis/hf_slice.py@@ -0,0 +1,74 @@+"""hf_slice.py - read single tensors (or slices of them) out of a remote safetensors repo with HTTP range requests, so a+quality test needs megabytes of a model, not the model. No token is read or printed (public repos only; set HF_TOKEN in+the environment for gated ones - it is sent as a header and never printed).++ from hf_slice import Repo+ r = Repo("Qwen/Qwen3.8-Flash-Next")+ w = r.tensor("model.language_model.layers.0.mlp.experts.gate_up_proj", rows=slice(0, 1)) # expert 0 only+"""+from __future__ import annotations++import json+import os+import struct+import urllib.request++import numpy as np++_DT = {"BF16": (2, None), "F16": (2, np.float16), "F32": (4, np.float32), "F8_E4M3": (1, None), "I8": (1, np.int8),+ "U8": (1, np.uint8), "I32": (4, np.int32), "I64": (8, np.int64)}+++def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:+ req = urllib.request.Request(url)+ tok = os.environ.get("HF_TOKEN")+ if tok:+ req.add_header("Authorization", "Bearer " + tok)+ if start is not None:+ req.add_header("Range", "bytes=%d-%d" % (start, end))+ with urllib.request.urlopen(req, timeout=120) as r:+ return r.read()+++def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:+ u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16+ return u.view(np.float32)+++class Repo:+ def __init__(self, repo: str, rev: str = "main"):+ self.repo, self.rev = repo, rev+ self.base = "https://huggingface.co/%s/resolve/%s/" % (repo, rev)+ self.map = json.loads(_get(self.base + "model.safetensors.index.json"))["weight_map"]+ self._hdr: dict[str, tuple[dict, int]] = {}++ def header(self, shard: str):+ if shard not in self._hdr:+ n = struct.unpack("<Q", _get(self.base + shard, 0, 7))[0]+ h = json.loads(_get(self.base + shard, 8, 8 + n - 1))+ self._hdr[shard] = (h, 8 + n)+ return self._hdr[shard]++ def info(self, name: str):+ shard = self.map[name]+ h, base = self.header(shard)+ return shard, h[name], base++ def tensor(self, name: str, lead: slice | int | None = None) -> np.ndarray:+ """The tensor `name` as float32. lead: an index or slice on the first axis (e.g. an expert), fetched alone."""+ shard, t, base = self.info(name)+ shape = t["shape"]+ item = _DT[t["dtype"]][0]+ s, e = t["data_offsets"]+ inner = int(np.prod(shape[1:])) * item if len(shape) > 1 else item+ if lead is None:+ i0, i1 = 0, shape[0]+ elif isinstance(lead, int):+ i0, i1 = lead, lead + 1+ else:+ i0, i1 = lead.start or 0, lead.stop+ raw = _get(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1)+ out_shape = [i1 - i0] + shape[1:]+ if t["dtype"] == "BF16":+ return bf16_to_f32(raw).reshape(out_shape)+ return np.frombuffer(raw, dtype=_DT[t["dtype"]][1]).astype(np.float32).reshape(out_shape)diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.pynew file mode 100644index 000000000..cd21dfb86--- /dev/null+++ b/tools/trellis/pilot.py@@ -0,0 +1,184 @@+"""pilot.py - gate-3 block-quality pilot: a few experts from several layers of a source checkpoint, encoded at TQK4X / TQK5X /+TQK6 / TQK7 with and without the input-only block-128 Hadamard, and measured.++Sources: --gguf <shard1 or shard dir files...> (UD-Q8_K_XL: MXFP4 / Q8_0 / BF16 / F16 tensors, decoded exactly: the decoded+ source is the teacher, as the plan says) or --hf Qwen/... (range requests; test of the pipeline)++Metrics per (layer, role, expert, rate, transform), over a random row subset of the projection (--rows):+ mse relative weight MSE ||W - W^||^2 / ||W||^2 in the original basis (the Hadamard is orthonormal: the same in either basis)+ act_iso ||(W - W^) X||^2 / ||W X||^2 with X i.i.d. N(0,1) columns - PLACEHOLDER; in expectation equal to mse+ act_syn ||(W - W^) X||^2 / ||W X||^2 with X whose input channels have heavy-tailed scales (a few outlier channels,+ log-normal rest) - SYNTHETIC PLACEHOLDER for real calibration activations, labelled as such in every output+Real activation-weighted numbers need calibration activations from the reference runtime (not done here).+"""+from __future__ import annotations++import argparse+import json+import mmap+import pathlib+import sys+import time++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import gguf_reader # noqa: E402+import tqenc # noqa: E402++MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)+RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7}+CFGS = {+ "fast": dict(iters=0, ncand=1, scale_tries=1),+ "default": dict(iters=1, ncand=1, scale_tries=1),+ "rot3": dict(iters=3, ncand=3, scale_tries=1),+}+++def decode_mxfp4(raw: np.ndarray, n_in: int) -> np.ndarray:+ """raw: [rows * n_in/32 * 17] bytes -> float32 [rows, n_in]. ggml block_mxfp4: e8m0 scale, 16 bytes, low nibble =+ element j, high nibble = element j + 16, value = kvalues[nibble] * 2^(e - 128) (the e8m0 half scale)."""+ b = raw.reshape(-1, 17)+ e = b[:, 0].astype(np.int32)+ d = np.where(e == 0, np.float32(2.0 ** -127), np.ldexp(np.float32(1.0), e - 128)).astype(np.float32)+ q = b[:, 1:]+ vals = np.concatenate([MXFP4_LUT[q & 15], MXFP4_LUT[q >> 4]], axis=1) * d[:, None]+ return vals.reshape(-1, n_in)+++def decode_q8_0(raw: np.ndarray, n_in: int) -> np.ndarray:+ b = raw.reshape(-1, 34)+ d = b[:, :2].copy().view(np.float16).astype(np.float32)+ q = b[:, 2:].copy().view(np.int8).astype(np.float32)+ return (q * d).reshape(-1, n_in)+++class Source:+ """Yields float32 [rows, n_in] row subsets of one expert's projection."""++ def __init__(self, gguf_paths):+ self.files = []+ self.tensors = {}+ for p in gguf_paths:+ g = gguf_reader.GGUFFile(p)+ fh = open(p, "rb")+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)+ self.files.append((g, fh, mm))+ for t in g.tensors:+ self.tensors[t.name] = (g, mm, t)++ def names(self, needle):+ return sorted(n for n in self.tensors if needle in n)++ def expert_rows(self, name: str, expert: int, rows: np.ndarray) -> np.ndarray:+ g, mm, ti = self.tensors[name]+ n_in, n_out = ti.shape[0], ti.shape[1]+ if ti.type_name == "MXFP4":+ rb = n_in // 32 * 17+ elif ti.type_name == "Q8_0":+ rb = n_in // 32 * 34+ elif ti.type_name in ("BF16", "F16"):+ rb = n_in * 2+ elif ti.type_name == "F32":+ rb = n_in * 4+ else:+ raise ValueError("%s: tensor type %s is not readable by this pilot" % (name, ti.type_name))+ base = g.data_start + ti.offset + expert * n_out * rb+ out = np.empty((len(rows), n_in), dtype=np.float32)+ for k, r in enumerate(rows):+ raw = np.frombuffer(mm, dtype=np.uint8, count=rb, offset=base + int(r) * rb)+ if ti.type_name == "MXFP4":+ out[k] = decode_mxfp4(raw, n_in)[0]+ elif ti.type_name == "Q8_0":+ out[k] = decode_q8_0(raw, n_in)[0]+ elif ti.type_name == "BF16":+ out[k] = (raw.view(np.uint16).astype(np.uint32) << 16).view(np.float32)+ elif ti.type_name == "F16":+ out[k] = raw.view(np.float16).astype(np.float32)+ else:+ out[k] = raw.view(np.float32)+ return out+++def synth_x(n_in: int, T: int, seed: int) -> np.ndarray:+ """SYNTHETIC activations: channel scale exp(N(0, 0.5)), 1% of the channels x (10..40)."""+ r = np.random.default_rng(seed)+ s = np.exp(0.5 * r.standard_normal(n_in)).astype(np.float32)+ out = r.choice(n_in, size=max(1, n_in // 100), replace=False)+ s[out] *= r.uniform(10, 40, size=out.size).astype(np.float32)+ return (r.standard_normal((n_in, T)).astype(np.float32) * s[:, None])+++def act_err(W, What, X):+ D = (W - What).astype(np.float32) @ X+ Y = W.astype(np.float32) @ X+ return float(np.sum(D.astype(np.float64) ** 2) / max(np.sum(Y.astype(np.float64) ** 2), 1e-300))+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--gguf", nargs="+", required=True)+ ap.add_argument("--layers", default="0,7,15,23,31,39")+ ap.add_argument("--experts", default="0,1,128,255")+ ap.add_argument("--roles", default="gate,up,down")+ ap.add_argument("--rates", default="K4,K5,K6,K7")+ ap.add_argument("--rows", type=int, default=256)+ ap.add_argument("--tokens", type=int, default=256, help="columns of the placeholder X")+ ap.add_argument("--cfg", default="default")+ ap.add_argument("--threads", type=int, default=0)+ ap.add_argument("--out", required=True, help="jsonl results")+ ap.add_argument("--seed", type=int, default=1)+ ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")+ a = ap.parse_args()++ src = Source(a.gguf)+ rng = np.random.default_rng(a.seed)+ cfg = CFGS[a.cfg]+ sign_cache, x_cache = {}, {}+ out = open(a.out, "a")+ for layer in [int(x) for x in a.layers.split(",")]:+ for role in a.roles.split(","):+ name = a.pattern.format(L=layer, R=role)+ if name not in src.tensors:+ print("no tensor", name, "(have e.g. %s)" % src.names("exps")[:3])+ continue+ g, mm, ti = src.tensors[name]+ n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]+ for e in [int(x) for x in a.experts.split(",")]:+ if e >= n_exp:+ continue+ rows = np.sort(rng.choice(n_out, size=min(a.rows, n_out), replace=False))+ W = src.expert_rows(name, e, rows)+ en = float(np.sum(W.astype(np.float64) ** 2))+ if n_in not in sign_cache:+ sign_cache[n_in] = tqenc.make_signs(n_in, 1000 + n_in)+ x_cache[n_in] = (np.random.default_rng(7).standard_normal((n_in, a.tokens)).astype(np.float32),+ synth_x(n_in, a.tokens, 9))+ Xi, Xs = x_cache[n_in]+ for had in (False, True):+ sg = sign_cache[n_in] if had else None+ Wr = tqenc.rotate(W, sg) if had else W+ for rname in a.rates.split(","):+ t = RATES[rname]+ t0 = time.time()+ packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, **cfg)+ sec = time.time() - t0+ dec = tqenc.decode(packed, t).reshape(Wr.shape)+ What = tqenc.unrotate(dec, sg) if had else dec+ rec = dict(layer=layer, role=role, expert=e, rate=rname, had=had, n_in=n_in, rows=len(rows),+ mse=float(np.sum((W.astype(np.float64) - What) ** 2) / en),+ act_iso=act_err(W, What, Xi), act_syn=act_err(W, What, Xs),+ bpw=tqenc.bpw(t), seconds=sec, zero_blocks=st["zero_blocks"], cfg=a.cfg,+ x_kind="placeholder: iso gaussian / synthetic outlier channels")+ out.write(json.dumps(rec) + "\n")+ out.flush()+ print("L%-3d %-4s e%-3d %s %-5s mse %.4f (%.2f dB) act_iso %.4f act_syn %.4f %.1fs" % (+ layer, role, e, rname, "had" if had else "plain", rec["mse"], -10 * np.log10(rec["mse"]),+ rec["act_iso"], rec["act_syn"], sec), flush=True)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/test_tqenc_py.py b/tools/trellis/test_tqenc_py.pynew file mode 100644index 000000000..28f8bfb43--- /dev/null+++ b/tools/trellis/test_tqenc_py.py@@ -0,0 +1,52 @@+"""Python-level checks of tools/tqenc.py (run on the box with libtqenc.so built): Hadamard orthonormality and the+Q(Ux) == Q_orig x identity through the encoder, signs and .tqx round trips, K5/K4 sizes."""+import os+import sys+import tempfile+import pathlib++import numpy as np++sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))+import tqenc # noqa: E402++fails = 0+++def check(c, msg):+ global fails+ if not c:+ fails += 1+ print("FAIL", msg)+++H = tqenc.hadamard_matrix()+check(np.abs(H @ H.T - np.eye(128)).max() < 1e-6 and np.abs(H - H.T).max() == 0, "H128 orthonormal and symmetric")+rng = np.random.default_rng(3)+for n in (2304, 5120):+ s = tqenc.make_signs(n, 5)+ check(set(np.unique(s)) <= {-1.0, 1.0}, "signs +-1")+ check(np.array_equal(tqenc.unpack_signs(tqenc.pack_signs(s), n), s), "sign pack round trip")+ W = rng.standard_normal((8, n)).astype(np.float32)+ x = rng.standard_normal(n).astype(np.float32)+ Wr = tqenc.rotate(W, s)+ check(np.abs(tqenc.unrotate(Wr, s) - W).max() < 1e-5, "unrotate(rotate(W)) == W")+ Ux = tqenc.rotate(x[None, :], s)[0]+ check(np.abs(Wr @ Ux - W @ x).max() < 1e-3, "Wrot (U x) == W x before quantization")+ for t in (tqenc.TQK4X, tqenc.TQK5X, tqenc.TQK6, tqenc.TQK7, tqenc.TQ2_T):+ packed, err, tot, st = tqenc.encode(Wr, t, iters=0)+ check(packed.shape == (8 * n // 128, tqenc.block_bytes(t)), "packed shape")+ Q = tqenc.decode(packed, t).reshape(8, n)+ y1 = Q @ Ux+ y2 = tqenc.unrotate(Q, s) @ x+ check(np.abs(y1 - y2).max() < 1e-3 * max(1.0, np.abs(y1).max()), "Q(Ux) == unrotate(Q) x type %d" % t)+ check(abs(tot - np.sum((Wr.astype(np.float64) - Q) ** 2)) < 1e-4 * tot, "encoder error == decoded error")+ with tempfile.TemporaryDirectory() as d:+ p = os.path.join(d, "m.tqx")+ tqenc.write_tqx(p, t, 8, n, packed, s)+ t2, r2, c2, pk2, s2 = tqenc.read_tqx(p)+ check(t2 == t and r2 == 8 and c2 == n and np.array_equal(pk2, packed) and np.array_equal(s2, s), "tqx round trip")+check(tqenc.block_bytes(tqenc.TQK5X) == 22 and tqenc.block_bytes(tqenc.TQK4X) == 18, "K5 22 bytes, K4 18 bytes")+check(abs(tqenc.bpw(tqenc.TQK5X) - 1.375) < 1e-9 and abs(tqenc.bpw(tqenc.TQK4X) - 1.125) < 1e-9, "bpw")+print("python checks:", "OK" if not fails else "%d FAILED" % fails)+sys.exit(1 if fails else 0)