Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis: python wrapper, Hadamard helpers, .tqx container, Gyro re-encode and original-weight comparison scripts, pilot driver

Niko12217e9531b3daac
diff --git a/tools/tqenc.py b/tools/tqenc.py
new file mode 100644
index 000000000..334063e2f
--- /dev/null
+++ b/tools/tqenc.py
@@ -0,0 +1,188 @@
+"""tqenc.py - Python side of the trellis encoder (tools/trellis/tqenc.cpp, built to libtqenc.so by
+`sh tools/trellis/build.sh lib`): encode / decode blocks of TQ2_T, TQK6, TQK7 and the experimental TQK5X / TQK4X, the
+input-only block Hadamard of the DeepSeek plan, and error metrics.
+
+Rows of a tensor are blocks of 128 consecutive weights along the INPUT dimension (the GGUF fastest axis), exactly as the
+decoders read them. A tensor of shape [rows, n] (n a multiple of 128) is encoded row by row; the output has
+rows * n/128 * block_bytes bytes.
+"""
+from __future__ import annotations
+
+import ctypes
+import os
+import pathlib
+
+import numpy as np
+
+TQ2_T, TQK6, TQK7, TQK5X, TQK4X = 144, 145, 146, 224, 225
+TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X"}
+TYPE_BY_NAME = {v: k for k, v in TYPE_NAMES.items()}
+KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4}
+BLOCK = 128
+
+
+def block_bytes(t: int) -> int:
+ return 2 + 4 * KBITS[t]
+
+
+def bpw(t: int) -> float:
+ return block_bytes(t) * 8 / BLOCK
+
+
+_lib = None
+
+
+def lib() -> ctypes.CDLL:
+ global _lib
+ if _lib is None:
+ cands = [os.environ.get("TQENC_LIB", ""),
+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc.so"),
+ "libtqenc.so"]
+ last = None
+ for c in cands:
+ if not c:
+ continue
+ try:
+ _lib = ctypes.CDLL(c)
+ break
+ except OSError as e:
+ last = e
+ if _lib is None:
+ raise RuntimeError("libtqenc.so not found (sh tools/trellis/build.sh lib): %s" % last)
+ _lib.tqe_encode.restype = ctypes.c_double
+ _lib.tqe_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,
+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
+ ctypes.c_void_p, ctypes.c_void_p]
+ _lib.tqe_decode.restype = None
+ _lib.tqe_decode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p]
+ return _lib
+
+
+def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, ncand: int = 1, exact: int = 0,
+ scale_tries: int = 1, threads: int = 0):
+ """w: float32 array whose last axis is a multiple of 128. Returns (packed uint8 [nblocks, block_bytes],
+ per-block error float64 [nblocks], total error, stats dict). Error = sum h (w - decoded)^2 (h = 1 if None)."""
+ w = np.ascontiguousarray(w, dtype=np.float32)
+ if w.shape[-1] % BLOCK:
+ raise ValueError("last axis %d is not a multiple of 128" % w.shape[-1])
+ nb = w.size // BLOCK
+ out = np.zeros((nb, block_bytes(t)), dtype=np.uint8)
+ err = np.zeros(nb, dtype=np.float64)
+ stats = np.zeros(2, dtype=np.int64)
+ hp = None
+ if h is not None:
+ h = np.ascontiguousarray(h, dtype=np.float32)
+ if h.shape != w.shape:
+ raise ValueError("importance weights must have the shape of w")
+ hp = h.ctypes.data
+ tot = lib().tqe_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, exact, scale_tries, threads,
+ err.ctypes.data, stats.ctypes.data)
+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}
+
+
+def decode(packed: np.ndarray, t: int) -> np.ndarray:
+ """packed: uint8 array of whole blocks. Returns float32 [nblocks * 128] (d * codebook)."""
+ packed = np.ascontiguousarray(packed, dtype=np.uint8)
+ bb = block_bytes(t)
+ if packed.size % bb:
+ raise ValueError("%d bytes is not a whole number of %s blocks" % (packed.size, TYPE_NAMES[t]))
+ nb = packed.size // bb
+ out = np.zeros(nb * BLOCK, dtype=np.float32)
+ lib().tqe_decode(t, packed.ctypes.data, nb, out.ctypes.data)
+ return out
+
+
+# ---------------------------------------------------------------------------------------------------- Hadamard
+
+def hadamard_matrix(n: int = BLOCK) -> np.ndarray:
+ """The normalized Sylvester-Walsh-Hadamard matrix (entry (-1)^popcount(r & c) / sqrt(n)); symmetric, orthonormal."""
+ h = np.array([[1.0]])
+ while h.shape[0] < n:
+ h = np.block([[h, h], [h, -h]])
+ return (h / np.sqrt(n)).astype(np.float32)
+
+
+def make_signs(width: int, seed: int) -> np.ndarray:
+ """A reproducible +1/-1 vector (the plan's D). The vector itself, not the seed, is what a file must store."""
+ r = np.random.default_rng(seed)
+ return (r.integers(0, 2, size=width) * 2 - 1).astype(np.float32)
+
+
+def pack_signs(signs: np.ndarray) -> bytes:
+ return np.packbits((signs > 0).astype(np.uint8), bitorder="little").tobytes()
+
+
+def unpack_signs(b: bytes, width: int) -> np.ndarray:
+ bits = np.unpackbits(np.frombuffer(b, dtype=np.uint8), bitorder="little")[:width]
+ return bits.astype(np.float32) * 2 - 1
+
+
+def rotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:
+ """The input transform U = H D on the last axis, per 128-block: Wrot = W U^T, i.e. each row w -> H (signs * w).
+ The same function applied to an activation x gives U x."""
+ w = np.asarray(w, dtype=np.float32)
+ if signs is not None:
+ w = w * signs
+ shp = w.shape
+ return (w.reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp) # H is symmetric
+
+
+def unrotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:
+ """Inverse of rotate (H is its own inverse: orthonormal and symmetric; then the signs)."""
+ shp = w.shape
+ x = (np.asarray(w, dtype=np.float32).reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp)
+ return x * signs if signs is not None else x
+
+
+# ---------------------------------------------------------------------------------------------------- metrics
+
+def sqnr_db(ref: np.ndarray, approx: np.ndarray) -> float:
+ ref = np.asarray(ref, dtype=np.float64).ravel()
+ e = np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2)
+ return float("inf") if e == 0 else float(10 * np.log10(np.sum(ref ** 2) / e))
+
+
+def rel_mse(ref: np.ndarray, approx: np.ndarray) -> float:
+ ref = np.asarray(ref, dtype=np.float64).ravel()
+ return float(np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2) / max(np.sum(ref ** 2), 1e-300))
+
+
+# ---------------------------------------------------------------------------------------------------- .tqx container
+# A small versioned file for ONE encoded matrix (research artifacts; not a GGUF). Little endian:
+# "TQXP" | u32 version (1) | u32 type id | u32 rows | u32 cols | u32 flags (bit 0: input Hadamard, 128-blocks)
+# | u32 n_sign_bytes | sign bits (1 = +1, LSB first, one per input column) | u32 hadamard block (128) | packed blocks
+# An unknown version or type is refused before any data is read.
+TQX_MAGIC = b"TQXP"
+TQX_VERSION = 1
+
+
+def write_tqx(path, t: int, rows: int, cols: int, packed: np.ndarray, signs: np.ndarray | None) -> None:
+ import struct
+ sb = pack_signs(signs) if signs is not None else b""
+ with open(path, "wb") as f:
+ f.write(TQX_MAGIC + struct.pack("<IIIIII", TQX_VERSION, t, rows, cols, 1 if signs is not None else 0, len(sb)))
+ f.write(sb)
+ f.write(struct.pack("<I", BLOCK))
+ f.write(np.ascontiguousarray(packed, dtype=np.uint8).tobytes())
+
+
+def read_tqx(path):
+ import struct
+ data = open(path, "rb").read()
+ if data[:4] != TQX_MAGIC:
+ raise ValueError("not a TQXP file")
+ ver, t, rows, cols, flags, nsb = struct.unpack_from("<IIIIII", data, 4)
+ if ver != TQX_VERSION:
+ raise ValueError("unsupported TQXP version %d (this reader handles %d)" % (ver, TQX_VERSION))
+ if t not in KBITS:
+ raise ValueError("unknown trellis type %d" % t)
+ at = 4 + 24
+ signs = unpack_signs(data[at:at + nsb], cols) if flags & 1 else None
+ at += nsb
+ (blk,) = struct.unpack_from("<I", data, at)
+ at += 4
+ if blk != BLOCK:
+ raise ValueError("unsupported Hadamard block %d" % blk)
+ nb = rows * (cols // BLOCK)
+ packed = np.frombuffer(data, dtype=np.uint8, count=nb * block_bytes(t), offset=at).reshape(nb, block_bytes(t))
+ return t, rows, cols, packed, signs
diff --git a/tools/trellis/gyro_reencode.py b/tools/trellis/gyro_reencode.py
new file mode 100644
index 000000000..e0a66d95b
--- /dev/null
+++ b/tools/trellis/gyro_reencode.py
@@ -0,0 +1,107 @@
+"""gyro_reencode.py - gate 2/3 quality check of the trellis encoder against agentionai's Gyro files.
+
+Decodes expert tensors of a Gyro GGUF (TQK6 gate/up, TQK7 down in Gyro-S; TQ2_T in Gyro-M) with the Strata decoder
+(libtqenc's decode, bit-identical to tq_states.hpp, see tqenc_test), re-encodes the decoded weights at the same type
+with tools/tqenc.py and reports the reconstruction error. The decoded weights are a feasible trellis point, so an
+ideal encoder recovers them exactly (error 0): the error measures the search, not the quantization.
+
+ python3 tools/trellis/gyro_reencode.py --gguf /data/models/gyro/Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf \
+ --layers 0,20,39 --experts 0,7 --rows 256 [--cfg default,rot3,scale3]
+"""
+from __future__ import annotations
+
+import argparse
+import mmap
+import pathlib
+import sys
+import time
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+import gguf_reader # noqa: E402
+import tqenc # noqa: E402
+
+CFGS = {
+ "fast": dict(iters=0, ncand=1, scale_tries=1),
+ "default": dict(iters=1, ncand=1, scale_tries=1),
+ "iters3": dict(iters=3, ncand=1, scale_tries=1),
+ "rot3": dict(iters=3, ncand=3, scale_tries=1),
+ "scale3": dict(iters=3, ncand=3, scale_tries=3),
+ "scale5": dict(iters=3, ncand=3, scale_tries=5),
+}
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--gguf", required=True)
+ ap.add_argument("--layers", default="0,20,39")
+ ap.add_argument("--experts", default="0")
+ ap.add_argument("--rows", type=int, default=256, help="rows (output channels) of each expert slice to test")
+ ap.add_argument("--cfg", default="fast,default,rot3")
+ ap.add_argument("--roles", default="gate,up,down")
+ ap.add_argument("--threads", type=int, default=0)
+ a = ap.parse_args()
+
+ g = gguf_reader.GGUFFile(a.gguf)
+ fh = open(a.gguf, "rb")
+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
+ by_name = {t.name: t for t in g.tensors}
+ print("file", pathlib.Path(a.gguf).name, "tensors", len(g.tensors), "types", g.by_type())
+ hk = {k: v for k, v in g.metadata.items() if k.startswith("prism.") and not isinstance(v, list)}
+ print("prism metadata:", hk)
+ cfgs = a.cfg.split(",")
+ rows_total = {c: [0.0, 0.0, 0, 0, 0.0] for c in cfgs} # err, energy, blocks, exact blocks, seconds
+ for layer in [int(x) for x in a.layers.split(",")]:
+ for role in a.roles.split(","):
+ name = "blk.%d.ffn_%s_exps.weight" % (layer, role)
+ ti = by_name.get(name)
+ if ti is None:
+ print("no tensor", name)
+ continue
+ t = tqenc.TYPE_BY_NAME.get(ti.type_name)
+ if t is None:
+ print(name, "type", ti.type_name, "is not a trellis type: skipped")
+ continue
+ n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]
+ bb = tqenc.block_bytes(t)
+ row_bytes = n_in // 128 * bb
+ for e in [int(x) for x in a.experts.split(",")]:
+ nrows = min(a.rows, n_out)
+ off = g.data_start + ti.offset + (e * n_out) * row_bytes
+ raw = np.frombuffer(mm, dtype=np.uint8, count=nrows * row_bytes, offset=off)
+ wd = tqenc.decode(raw, t).reshape(nrows, n_in)
+ en = float(np.sum(wd.astype(np.float64) ** 2))
+ scales = raw.reshape(-1, bb)[:, :2].copy().view(np.float16).astype(np.float32).ravel()
+ print("%s expert %d: %d rows x %d, %s %.3f bpw, |w| rms %.4g, agentionai block scales: median %.4g, "
+ "p1 %.4g, p99 %.4g, zero-scale blocks %d" % (
+ name, e, nrows, n_in, ti.type_name, tqenc.bpw(t), np.sqrt(en / wd.size),
+ np.median(scales), np.percentile(scales, 1), np.percentile(scales, 99), int((scales == 0).sum())))
+ for c in cfgs:
+ t0 = time.time()
+ packed, err, tot, st = tqenc.encode(wd, t, threads=a.threads, **CFGS[c])
+ sec = time.time() - t0
+ dec = tqenc.decode(packed, t)
+ chk = float(np.sum((wd.ravel().astype(np.float64) - dec) ** 2))
+ nexact = int((err == 0).sum())
+ same_bytes = int((packed.reshape(-1) == raw).sum()) / raw.size
+ print(" %-8s rel MSE %.3e SQNR %7.2f dB exact blocks %5d/%d (%.1f%%) scale==theirs %.1f%% "
+ "%.0f weights/s (recheck %.3e)" % (
+ c, tot / en, tqenc.sqnr_db(wd, dec), nexact, err.size, 100.0 * nexact / err.size,
+ 100.0 * float(np.mean(packed[:, :2].reshape(-1, 2).view(np.uint16).ravel()
+ == raw.reshape(-1, bb)[:, :2].copy().view(np.uint16).ravel())),
+ wd.size / sec, chk / en))
+ r = rows_total[c]
+ r[0] += tot; r[1] += en; r[2] += err.size; r[3] += nexact; r[4] += sec
+ print("\nTOTAL over all tested slices")
+ for c in cfgs:
+ r = rows_total[c]
+ if r[1] > 0:
+ print(" %-8s rel MSE %.3e SQNR %.2f dB exact blocks %.1f%% %.0f weights/s" % (
+ c, r[0] / r[1], 10 * np.log10(r[1] / max(r[0], 1e-300)), 100.0 * r[3] / r[2], r[2] * 128 / max(r[4], 1e-9)))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/gyro_vs_original.py b/tools/trellis/gyro_vs_original.py
new file mode 100644
index 000000000..d27629449
--- /dev/null
+++ b/tools/trellis/gyro_vs_original.py
@@ -0,0 +1,107 @@
+"""gyro_vs_original.py - the fair comparison: agentionai's Gyro quantization of Qwen3.8-Flash-Next against OUR encoder, both
+measured against the original BF16 expert weights (fetched tensor by tensor with HTTP range requests, ~10 MB per expert).
+
+For a layer and expert: W = original (gate, up from the fused gate_up_proj, down), rotated into the file's basis with the
+file's own sign vectors (prism.hadamard.sign_values, widths 2560 / 640): W' = H (s * w) per row and 128-block.
+ agentionai: Wd = decode of the file's blocks
+ ours: Wo = decode(encode(W')) at the same type (TQK6 gate/up, TQK7 down), several search settings
+Reported: relative MSE and SQNR of each against W'.
+
+ python3 tools/trellis/gyro_vs_original.py --gguf <Gyro-S> --layers 0,20,39 --experts 0,5 --cfg fast,default,rot3
+"""
+from __future__ import annotations
+
+import argparse
+import mmap
+import pathlib
+import sys
+import time
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import gguf_reader # noqa: E402
+import hf_slice # noqa: E402
+import tqenc # noqa: E402
+
+CFGS = {
+ "fast": dict(iters=0, ncand=1, scale_tries=1),
+ "default": dict(iters=1, ncand=1, scale_tries=1),
+ "iters3": dict(iters=3, ncand=1, scale_tries=1),
+ "rot3": dict(iters=3, ncand=3, scale_tries=1),
+ "scale3": dict(iters=3, ncand=3, scale_tries=3),
+}
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--gguf", required=True)
+ ap.add_argument("--repo", default="Qwen/Qwen3.8-Flash-Next")
+ ap.add_argument("--layers", default="0,20,39")
+ ap.add_argument("--experts", default="0")
+ ap.add_argument("--cfg", default="fast,default,rot3")
+ ap.add_argument("--threads", type=int, default=0)
+ a = ap.parse_args()
+
+ g = gguf_reader.GGUFFile(a.gguf)
+ fh = open(a.gguf, "rb")
+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
+ by_name = {t.name: t for t in g.tensors}
+ sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]
+ sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)
+ signs, at = {}, 0
+ for w in sw:
+ signs[w] = sv[at:at + w]
+ at += w
+ print("sign widths", sw)
+ repo = hf_slice.Repo(a.repo)
+ cfgs = a.cfg.split(",")
+ tot = {k: [0.0, 0.0] for k in ["agentionai"] + cfgs}
+ tot_role = {}
+ for layer in [int(x) for x in a.layers.split(",")]:
+ pre = "model.language_model.layers.%d.mlp.experts." % layer
+ for e in [int(x) for x in a.experts.split(",")]:
+ t0 = time.time()
+ gu = repo.tensor(pre + "gate_up_proj", e)[0] # [1280, 2560]
+ dn = repo.tensor(pre + "down_proj", e)[0] # [2560, 640]
+ print("layer %d expert %d fetched in %.1f s: gate_up %s down %s" % (layer, e, time.time() - t0, gu.shape, dn.shape))
+ mats = {"gate": gu[:640], "up": gu[640:], "down": dn}
+ for role, W in mats.items():
+ ti = by_name["blk.%d.ffn_%s_exps.weight" % (layer, role)]
+ t = tqenc.TYPE_BY_NAME[ti.type_name]
+ n_in, n_out = ti.shape[0], ti.shape[1]
+ assert W.shape == (n_out, n_in), (W.shape, ti.shape)
+ bb = tqenc.block_bytes(t)
+ row_bytes = n_in // 128 * bb
+ off = g.data_start + ti.offset + e * n_out * row_bytes
+ raw = np.frombuffer(mm, dtype=np.uint8, count=n_out * row_bytes, offset=off)
+ wd = tqenc.decode(raw, t).reshape(n_out, n_in)
+ Wr = tqenc.rotate(W, signs[n_in])
+ en = float(np.sum(Wr.astype(np.float64) ** 2))
+ ea = float(np.sum((Wr.astype(np.float64) - wd) ** 2))
+ # sanity: the fused gate_up order (gate first) - if swapped the SQNR collapses
+ line = ["agentionai %s %.2f dB" % (ti.type_name, 10 * np.log10(en / ea))]
+ key = "%s" % role
+ tot["agentionai"][0] += ea; tot["agentionai"][1] += en
+ tr = tot_role.setdefault(key, {k: [0.0, 0.0] for k in ["agentionai"] + cfgs})
+ tr["agentionai"][0] += ea; tr["agentionai"][1] += en
+ for c in cfgs:
+ t1 = time.time()
+ packed, err, totr, st = tqenc.encode(Wr, t, threads=a.threads, **CFGS[c])
+ sec = time.time() - t1
+ line.append("%s %.2f dB (%.0fk w/s)" % (c, 10 * np.log10(en / totr), Wr.size / sec / 1e3))
+ tot[c][0] += totr; tot[c][1] += en
+ tr[c][0] += totr; tr[c][1] += en
+ print(" L%d e%d %-4s " % (layer, e, role) + " | ".join(line))
+ print("\nTOTAL SQNR (dB) against the rotated original, all tested matrices")
+ for k, (er, en) in tot.items():
+ print(" %-10s %.3f dB (rel MSE %.4e)" % (k, 10 * np.log10(en / er), er / en))
+ for role, d in tot_role.items():
+ print(" %-5s " % role + " ".join("%s %.3f" % (k, 10 * np.log10(v[1] / v[0])) for k, v in d.items()))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/hf_slice.py b/tools/trellis/hf_slice.py
new file mode 100644
index 000000000..b93d55914
--- /dev/null
+++ b/tools/trellis/hf_slice.py
@@ -0,0 +1,74 @@
+"""hf_slice.py - read single tensors (or slices of them) out of a remote safetensors repo with HTTP range requests, so a
+quality test needs megabytes of a model, not the model. No token is read or printed (public repos only; set HF_TOKEN in
+the environment for gated ones - it is sent as a header and never printed).
+
+ from hf_slice import Repo
+ r = Repo("Qwen/Qwen3.8-Flash-Next")
+ w = r.tensor("model.language_model.layers.0.mlp.experts.gate_up_proj", rows=slice(0, 1)) # expert 0 only
+"""
+from __future__ import annotations
+
+import json
+import os
+import struct
+import urllib.request
+
+import numpy as np
+
+_DT = {"BF16": (2, None), "F16": (2, np.float16), "F32": (4, np.float32), "F8_E4M3": (1, None), "I8": (1, np.int8),
+ "U8": (1, np.uint8), "I32": (4, np.int32), "I64": (8, np.int64)}
+
+
+def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:
+ req = urllib.request.Request(url)
+ tok = os.environ.get("HF_TOKEN")
+ if tok:
+ req.add_header("Authorization", "Bearer " + tok)
+ if start is not None:
+ req.add_header("Range", "bytes=%d-%d" % (start, end))
+ with urllib.request.urlopen(req, timeout=120) as r:
+ return r.read()
+
+
+def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:
+ u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
+ return u.view(np.float32)
+
+
+class Repo:
+ def __init__(self, repo: str, rev: str = "main"):
+ self.repo, self.rev = repo, rev
+ self.base = "https://huggingface.co/%s/resolve/%s/" % (repo, rev)
+ self.map = json.loads(_get(self.base + "model.safetensors.index.json"))["weight_map"]
+ self._hdr: dict[str, tuple[dict, int]] = {}
+
+ def header(self, shard: str):
+ if shard not in self._hdr:
+ n = struct.unpack("<Q", _get(self.base + shard, 0, 7))[0]
+ h = json.loads(_get(self.base + shard, 8, 8 + n - 1))
+ self._hdr[shard] = (h, 8 + n)
+ return self._hdr[shard]
+
+ def info(self, name: str):
+ shard = self.map[name]
+ h, base = self.header(shard)
+ return shard, h[name], base
+
+ def tensor(self, name: str, lead: slice | int | None = None) -> np.ndarray:
+ """The tensor `name` as float32. lead: an index or slice on the first axis (e.g. an expert), fetched alone."""
+ shard, t, base = self.info(name)
+ shape = t["shape"]
+ item = _DT[t["dtype"]][0]
+ s, e = t["data_offsets"]
+ inner = int(np.prod(shape[1:])) * item if len(shape) > 1 else item
+ if lead is None:
+ i0, i1 = 0, shape[0]
+ elif isinstance(lead, int):
+ i0, i1 = lead, lead + 1
+ else:
+ i0, i1 = lead.start or 0, lead.stop
+ raw = _get(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1)
+ out_shape = [i1 - i0] + shape[1:]
+ if t["dtype"] == "BF16":
+ return bf16_to_f32(raw).reshape(out_shape)
+ return np.frombuffer(raw, dtype=_DT[t["dtype"]][1]).astype(np.float32).reshape(out_shape)
diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.py
new file mode 100644
index 000000000..cd21dfb86
--- /dev/null
+++ b/tools/trellis/pilot.py
@@ -0,0 +1,184 @@
+"""pilot.py - gate-3 block-quality pilot: a few experts from several layers of a source checkpoint, encoded at TQK4X / TQK5X /
+TQK6 / TQK7 with and without the input-only block-128 Hadamard, and measured.
+
+Sources: --gguf <shard1 or shard dir files...> (UD-Q8_K_XL: MXFP4 / Q8_0 / BF16 / F16 tensors, decoded exactly: the decoded
+ source is the teacher, as the plan says) or --hf Qwen/... (range requests; test of the pipeline)
+
+Metrics per (layer, role, expert, rate, transform), over a random row subset of the projection (--rows):
+ mse relative weight MSE ||W - W^||^2 / ||W||^2 in the original basis (the Hadamard is orthonormal: the same in either basis)
+ act_iso ||(W - W^) X||^2 / ||W X||^2 with X i.i.d. N(0,1) columns - PLACEHOLDER; in expectation equal to mse
+ act_syn ||(W - W^) X||^2 / ||W X||^2 with X whose input channels have heavy-tailed scales (a few outlier channels,
+ log-normal rest) - SYNTHETIC PLACEHOLDER for real calibration activations, labelled as such in every output
+Real activation-weighted numbers need calibration activations from the reference runtime (not done here).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import mmap
+import pathlib
+import sys
+import time
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import gguf_reader # noqa: E402
+import tqenc # noqa: E402
+
+MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)
+RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7}
+CFGS = {
+ "fast": dict(iters=0, ncand=1, scale_tries=1),
+ "default": dict(iters=1, ncand=1, scale_tries=1),
+ "rot3": dict(iters=3, ncand=3, scale_tries=1),
+}
+
+
+def decode_mxfp4(raw: np.ndarray, n_in: int) -> np.ndarray:
+ """raw: [rows * n_in/32 * 17] bytes -> float32 [rows, n_in]. ggml block_mxfp4: e8m0 scale, 16 bytes, low nibble =
+ element j, high nibble = element j + 16, value = kvalues[nibble] * 2^(e - 128) (the e8m0 half scale)."""
+ b = raw.reshape(-1, 17)
+ e = b[:, 0].astype(np.int32)
+ d = np.where(e == 0, np.float32(2.0 ** -127), np.ldexp(np.float32(1.0), e - 128)).astype(np.float32)
+ q = b[:, 1:]
+ vals = np.concatenate([MXFP4_LUT[q & 15], MXFP4_LUT[q >> 4]], axis=1) * d[:, None]
+ return vals.reshape(-1, n_in)
+
+
+def decode_q8_0(raw: np.ndarray, n_in: int) -> np.ndarray:
+ b = raw.reshape(-1, 34)
+ d = b[:, :2].copy().view(np.float16).astype(np.float32)
+ q = b[:, 2:].copy().view(np.int8).astype(np.float32)
+ return (q * d).reshape(-1, n_in)
+
+
+class Source:
+ """Yields float32 [rows, n_in] row subsets of one expert's projection."""
+
+ def __init__(self, gguf_paths):
+ self.files = []
+ self.tensors = {}
+ for p in gguf_paths:
+ g = gguf_reader.GGUFFile(p)
+ fh = open(p, "rb")
+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
+ self.files.append((g, fh, mm))
+ for t in g.tensors:
+ self.tensors[t.name] = (g, mm, t)
+
+ def names(self, needle):
+ return sorted(n for n in self.tensors if needle in n)
+
+ def expert_rows(self, name: str, expert: int, rows: np.ndarray) -> np.ndarray:
+ g, mm, ti = self.tensors[name]
+ n_in, n_out = ti.shape[0], ti.shape[1]
+ if ti.type_name == "MXFP4":
+ rb = n_in // 32 * 17
+ elif ti.type_name == "Q8_0":
+ rb = n_in // 32 * 34
+ elif ti.type_name in ("BF16", "F16"):
+ rb = n_in * 2
+ elif ti.type_name == "F32":
+ rb = n_in * 4
+ else:
+ raise ValueError("%s: tensor type %s is not readable by this pilot" % (name, ti.type_name))
+ base = g.data_start + ti.offset + expert * n_out * rb
+ out = np.empty((len(rows), n_in), dtype=np.float32)
+ for k, r in enumerate(rows):
+ raw = np.frombuffer(mm, dtype=np.uint8, count=rb, offset=base + int(r) * rb)
+ if ti.type_name == "MXFP4":
+ out[k] = decode_mxfp4(raw, n_in)[0]
+ elif ti.type_name == "Q8_0":
+ out[k] = decode_q8_0(raw, n_in)[0]
+ elif ti.type_name == "BF16":
+ out[k] = (raw.view(np.uint16).astype(np.uint32) << 16).view(np.float32)
+ elif ti.type_name == "F16":
+ out[k] = raw.view(np.float16).astype(np.float32)
+ else:
+ out[k] = raw.view(np.float32)
+ return out
+
+
+def synth_x(n_in: int, T: int, seed: int) -> np.ndarray:
+ """SYNTHETIC activations: channel scale exp(N(0, 0.5)), 1% of the channels x (10..40)."""
+ r = np.random.default_rng(seed)
+ s = np.exp(0.5 * r.standard_normal(n_in)).astype(np.float32)
+ out = r.choice(n_in, size=max(1, n_in // 100), replace=False)
+ s[out] *= r.uniform(10, 40, size=out.size).astype(np.float32)
+ return (r.standard_normal((n_in, T)).astype(np.float32) * s[:, None])
+
+
+def act_err(W, What, X):
+ D = (W - What).astype(np.float32) @ X
+ Y = W.astype(np.float32) @ X
+ return float(np.sum(D.astype(np.float64) ** 2) / max(np.sum(Y.astype(np.float64) ** 2), 1e-300))
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--gguf", nargs="+", required=True)
+ ap.add_argument("--layers", default="0,7,15,23,31,39")
+ ap.add_argument("--experts", default="0,1,128,255")
+ ap.add_argument("--roles", default="gate,up,down")
+ ap.add_argument("--rates", default="K4,K5,K6,K7")
+ ap.add_argument("--rows", type=int, default=256)
+ ap.add_argument("--tokens", type=int, default=256, help="columns of the placeholder X")
+ ap.add_argument("--cfg", default="default")
+ ap.add_argument("--threads", type=int, default=0)
+ ap.add_argument("--out", required=True, help="jsonl results")
+ ap.add_argument("--seed", type=int, default=1)
+ ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")
+ a = ap.parse_args()
+
+ src = Source(a.gguf)
+ rng = np.random.default_rng(a.seed)
+ cfg = CFGS[a.cfg]
+ sign_cache, x_cache = {}, {}
+ out = open(a.out, "a")
+ for layer in [int(x) for x in a.layers.split(",")]:
+ for role in a.roles.split(","):
+ name = a.pattern.format(L=layer, R=role)
+ if name not in src.tensors:
+ print("no tensor", name, "(have e.g. %s)" % src.names("exps")[:3])
+ continue
+ g, mm, ti = src.tensors[name]
+ n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]
+ for e in [int(x) for x in a.experts.split(",")]:
+ if e >= n_exp:
+ continue
+ rows = np.sort(rng.choice(n_out, size=min(a.rows, n_out), replace=False))
+ W = src.expert_rows(name, e, rows)
+ en = float(np.sum(W.astype(np.float64) ** 2))
+ if n_in not in sign_cache:
+ sign_cache[n_in] = tqenc.make_signs(n_in, 1000 + n_in)
+ x_cache[n_in] = (np.random.default_rng(7).standard_normal((n_in, a.tokens)).astype(np.float32),
+ synth_x(n_in, a.tokens, 9))
+ Xi, Xs = x_cache[n_in]
+ for had in (False, True):
+ sg = sign_cache[n_in] if had else None
+ Wr = tqenc.rotate(W, sg) if had else W
+ for rname in a.rates.split(","):
+ t = RATES[rname]
+ t0 = time.time()
+ packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, **cfg)
+ sec = time.time() - t0
+ dec = tqenc.decode(packed, t).reshape(Wr.shape)
+ What = tqenc.unrotate(dec, sg) if had else dec
+ rec = dict(layer=layer, role=role, expert=e, rate=rname, had=had, n_in=n_in, rows=len(rows),
+ mse=float(np.sum((W.astype(np.float64) - What) ** 2) / en),
+ act_iso=act_err(W, What, Xi), act_syn=act_err(W, What, Xs),
+ bpw=tqenc.bpw(t), seconds=sec, zero_blocks=st["zero_blocks"], cfg=a.cfg,
+ x_kind="placeholder: iso gaussian / synthetic outlier channels")
+ out.write(json.dumps(rec) + "\n")
+ out.flush()
+ print("L%-3d %-4s e%-3d %s %-5s mse %.4f (%.2f dB) act_iso %.4f act_syn %.4f %.1fs" % (
+ layer, role, e, rname, "had" if had else "plain", rec["mse"], -10 * np.log10(rec["mse"]),
+ rec["act_iso"], rec["act_syn"], sec), flush=True)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/test_tqenc_py.py b/tools/trellis/test_tqenc_py.py
new file mode 100644
index 000000000..28f8bfb43
--- /dev/null
+++ b/tools/trellis/test_tqenc_py.py
@@ -0,0 +1,52 @@
+"""Python-level checks of tools/tqenc.py (run on the box with libtqenc.so built): Hadamard orthonormality and the
+Q(Ux) == Q_orig x identity through the encoder, signs and .tqx round trips, K5/K4 sizes."""
+import os
+import sys
+import tempfile
+import pathlib
+
+import numpy as np
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+import tqenc # noqa: E402
+
+fails = 0
+
+
+def check(c, msg):
+ global fails
+ if not c:
+ fails += 1
+ print("FAIL", msg)
+
+
+H = tqenc.hadamard_matrix()
+check(np.abs(H @ H.T - np.eye(128)).max() < 1e-6 and np.abs(H - H.T).max() == 0, "H128 orthonormal and symmetric")
+rng = np.random.default_rng(3)
+for n in (2304, 5120):
+ s = tqenc.make_signs(n, 5)
+ check(set(np.unique(s)) <= {-1.0, 1.0}, "signs +-1")
+ check(np.array_equal(tqenc.unpack_signs(tqenc.pack_signs(s), n), s), "sign pack round trip")
+ W = rng.standard_normal((8, n)).astype(np.float32)
+ x = rng.standard_normal(n).astype(np.float32)
+ Wr = tqenc.rotate(W, s)
+ check(np.abs(tqenc.unrotate(Wr, s) - W).max() < 1e-5, "unrotate(rotate(W)) == W")
+ Ux = tqenc.rotate(x[None, :], s)[0]
+ check(np.abs(Wr @ Ux - W @ x).max() < 1e-3, "Wrot (U x) == W x before quantization")
+ for t in (tqenc.TQK4X, tqenc.TQK5X, tqenc.TQK6, tqenc.TQK7, tqenc.TQ2_T):
+ packed, err, tot, st = tqenc.encode(Wr, t, iters=0)
+ check(packed.shape == (8 * n // 128, tqenc.block_bytes(t)), "packed shape")
+ Q = tqenc.decode(packed, t).reshape(8, n)
+ y1 = Q @ Ux
+ y2 = tqenc.unrotate(Q, s) @ x
+ check(np.abs(y1 - y2).max() < 1e-3 * max(1.0, np.abs(y1).max()), "Q(Ux) == unrotate(Q) x type %d" % t)
+ check(abs(tot - np.sum((Wr.astype(np.float64) - Q) ** 2)) < 1e-4 * tot, "encoder error == decoded error")
+ with tempfile.TemporaryDirectory() as d:
+ p = os.path.join(d, "m.tqx")
+ tqenc.write_tqx(p, t, 8, n, packed, s)
+ t2, r2, c2, pk2, s2 = tqenc.read_tqx(p)
+ check(t2 == t and r2 == 8 and c2 == n and np.array_equal(pk2, packed) and np.array_equal(s2, s), "tqx round trip")
+check(tqenc.block_bytes(tqenc.TQK5X) == 22 and tqenc.block_bytes(tqenc.TQK4X) == 18, "K5 22 bytes, K4 18 bytes")
+check(abs(tqenc.bpw(tqenc.TQK5X) - 1.375) < 1e-9 and abs(tqenc.bpw(tqenc.TQK4X) - 1.125) < 1e-9, "bpw")
+print("python checks:", "OK" if not fails else "%d FAILED" % fails)
+sys.exit(1 if fails else 0)