Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: tq_convert.py (re-encode a pack's MXFP4 experts with the trellis encoder into the engine's file layout) and pack_gguf --tq-experts

Niko12217df82763e386
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.py
index d1fd7a5e4..b56d57825 100644
--- a/tools/dsv41/pack_gguf.py
+++ b/tools/dsv41/pack_gguf.py
@@ -40,7 +40,10 @@ TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1
# the low-bit expert types (the Maya quants' experts; the engine's ds4_iq kernels)
10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),
14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),
- 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136)}
+ 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),
+ # the trellis types (tools/trellis/tqenc.hpp; 145 / 146 are the agentionai fork's ggml ids, 224-227 Strata's own)
+ 145: ("TQK6", 128, 26), 146: ("TQK7", 128, 30), 224: ("TQK5X", 128, 22), 225: ("TQK4X", 128, 18),
+ 226: ("TQK5T", 128, 22), 227: ("TQK4T", 128, 18)}
# the Engram set a quant without the tables takes them from by default, beside it (or inside its folder): unsloth's
# MXFP8 shard of them, where setups before v1.1.17 put it for Maya-S (--engram: any other)
ENGRAM_SETS = ("Engram",)
@@ -328,6 +331,8 @@ def main() -> int:
"in <model>/.. when there is one)")
ap.add_argument("--mmproj", help="unsloth's vision tower GGUF (mmproj-*.gguf, clip / deepseek41v): its tensors become the "
"checkpoint's vision.* / aligner.* (default: the first mmproj*.gguf in <model>/..)")
+ ap.add_argument("--tq-experts", help="a folder tq_convert.py wrote (tq.json + tq-L<layer>.bin): the routed experts of the layers "
+ "listed there are read from it as trellis types (TQK5X ..), the others keep their MXFP4")
ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")
ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out "
"(default: <model>/../Engram, when it is whole)")
@@ -544,6 +549,47 @@ def main() -> int:
"vision.rope_theta": 10000, "vision.max_image_tokens": 1024, "vision.min_pixels": k("image_min_pixels"),
"image_token_id": img_id}
print("vision tower from %s: %d blocks, image token %d" % (mm_path.name, mm_geo["vision.num_hidden_layers"], img_id))
+ # trellis experts from tq_convert.py: per layer the three roles stacked [n_exp][rows][row bytes] in one file
+ tq_layers = []
+ if a.tq_experts:
+ tqd = pathlib.Path(a.tq_experts).expanduser().resolve()
+ tq_idx = json.loads((tqd / "tq.json").read_text())
+ by_layer = {}
+ for key, ent in tq_idx.items():
+ m = re.match(r"layers\.(\d+)\.ffn\.experts\.(w[123])$", key)
+ if not m:
+ raise SystemExit("tq.json: %s is not a layer's expert role" % key)
+ by_layer.setdefault(int(m.group(1)), {})[m.group(2)] = ent
+ TQTYPES = {v[0]: k for k, v in TYPES.items()}
+ for il, roles in sorted(by_layer.items()):
+ if sorted(roles) != ["w1", "w2", "w3"]:
+ raise SystemExit("tq.json: layer %d lacks a role (has %s)" % (il, sorted(roles)))
+ if len({r[0] for r in roles.values()}) != 1 or roles["w1"][0] != roles["w3"][0]:
+ raise SystemExit("tq.json: layer %d: gate and up must be one type" % il)
+ drop = re.compile(r"layers\.%d\.ffn\.experts\.\d+\.w[123]\.weight$" % il)
+ kept = [t for t in tensors if not drop.match(t[0])]
+ if len(kept) == len(tensors):
+ raise SystemExit("layer %d has no routed experts in the GGUF to replace" % il)
+ tensors[:] = kept
+ for role, (tname, fname, off, total, shape) in roles.items():
+ if tname not in TQTYPES:
+ raise SystemExit("tq.json: unknown type %s" % tname)
+ n_exp, rows, cols = shape
+ per = total // n_exp
+ f = tqd / fname
+ if not f.is_file() or f.stat().st_size < off + total:
+ raise SystemExit("%s: missing or short" % f)
+ try:
+ relf = os.path.relpath(f, model).replace(os.sep, "/")
+ except ValueError:
+ raise SystemExit("%s must be on the model's drive" % f)
+ if relf not in files:
+ files.append(relf)
+ fi = files.index(relf)
+ for e in range(n_exp):
+ tensors.append(("layers.%d.ffn.experts.%d.%s.weight" % (il, e, role), tname, fi, off + e * per, per, [rows, cols]))
+ tq_layers.append(il)
+ print("trellis experts from %s: layers %s" % (tqd, tq_layers))
drafter = pathlib.Path(a.drafter).expanduser().resolve() if a.drafter else model.parent / "dspark" / "dspark.json"
if a.drafter and not drafter.is_file():
raise SystemExit("%s: no drafter there" % drafter)
@@ -607,7 +653,7 @@ def main() -> int:
by = {}
for t in tensors:
by[t[1]] = by.get(t[1], 0) + t[4]
- extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision))) if there]
+ extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision)), ("trellis experts in layers %s" % tq_layers, bool(tq_layers))) if there]
print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %
(model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),
nl, cols, "".join(", " + w for w in extras), out))
diff --git a/tools/dsv41/tq_convert.py b/tools/dsv41/tq_convert.py
new file mode 100644
index 000000000..298dca622
--- /dev/null
+++ b/tools/dsv41/tq_convert.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""tq_convert.py - re-encode the routed experts of some layers of a DeepSeek-V4.1 pack with the trellis encoder
+(tools/tqenc.py, HIP) and write them in the format the engine reads (strata-dsv41: ds4_tq.cu / ds4_tq_cpu.cpp):
+
+ HIP_VISIBLE_DEVICES=2 TQENC_GPU_LOCK=2 python tools/dsv41/tq_convert.py --pack PACK --out DIR --layers 1,7 --type K5
+
+PACK is the pack pack_gguf.py wrote (its tensors.txt names the source MXFP4 experts in the GGUF shards). Per layer one file
+DIR/tq-L<layer>.bin holds the three roles one after another (gate = w1, up = w3, down = w2), each stacked [n_exp][rows][row
+bytes] in the trellis types' file layout (a row = k / 128 blocks of {fp16 d; 4 K stream bytes}, tools/trellis/tqenc.hpp), 4096-aligned;
+DIR/tq.json lists them as {"layers.N.ffn.experts.w1": [type name, file, offset, bytes, [n_exp, rows, cols]], ...}. Then
+
+ python tools/dsv41/pack_gguf.py --model <gguf dir> --out PACK2 --tq-experts DIR
+
+writes a pack whose layers listed there read these experts (the other layers keep their MXFP4). The weights are encoded as they
+are (plain error, no rotation, no activation weights): what the dense override experiments of gate 3 measure. Types:
+K4 = TQK4X, K5 = TQK5X (computed codebook, version 2), K4T / K5T (the table, version 1), K6 = TQK6, K7 = TQK7.
+"""
+import argparse
+import concurrent.futures as cf
+import json
+import os
+import pathlib
+import sys
+import threading
+import time
+
+import numpy as np
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+import tqenc # noqa: E402
+
+TYPES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K4T": tqenc.TQK4T, "K5T": tqenc.TQK5T, "K6": tqenc.TQK6, "K7": tqenc.TQK7}
+CFGS = {"fast": dict(iters=0, ncand=1, scale_tries=1), "default": dict(iters=1, ncand=1, scale_tries=1)}
+MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)
+ROLES = (("w1", "gate"), ("w3", "up"), ("w2", "down"))
+
+
+def decode_mxfp4(raw, n_in):
+ """ggml MXFP4 blocks (32 values in 17 bytes: E8M0 scale, 16 code bytes) -> float32 [rows, n_in]"""
+ b = raw.reshape(-1, 17)
+ e = b[:, 0].astype(np.int32)
+ d = np.where(e == 0, np.float32(2.0 ** -127), np.ldexp(np.float32(1.0), e - 128)).astype(np.float32)
+ q = b[:, 1:]
+ return (np.concatenate([MXFP4_LUT[q & 15], MXFP4_LUT[q >> 4]], axis=1) * d[:, None]).reshape(-1, n_in)
+
+
+def read_pack(path):
+ files, tensors, cfg = {}, {}, {}
+ for line in open(os.path.join(path, "tensors.txt"), encoding="utf-8"):
+ f = line.split()
+ if f and f[0] == "file":
+ files[int(f[1])] = line.split(None, 2)[2].strip()
+ elif f and f[0] == "t":
+ tensors[f[1]] = (f[2], int(f[3]), int(f[4]), int(f[5]), [int(x) for x in f[6:]])
+ for line in open(os.path.join(path, "ds4.txt"), encoding="utf-8"):
+ k, _, v = line.strip().partition(" ")
+ cfg[k] = v
+ return files, tensors, cfg
+
+
+def align(n, a=4096):
+ return (n + a - 1) // a * a
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--pack", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--layers", required=True, help="comma list")
+ ap.add_argument("--type", required=True, choices=sorted(TYPES))
+ ap.add_argument("--cfg", default="fast", choices=sorted(CFGS))
+ ap.add_argument("--experts", type=int, default=0, help="only the first N experts (tests; the others stay zero streams)")
+ ap.add_argument("--threads", type=int, default=4)
+ ap.add_argument("--cpu", action="store_true", help="the CPU encoder instead of the card")
+ a = ap.parse_args()
+ t = TYPES[a.type]
+ cfg = CFGS[a.cfg]
+ files, tensors, pcfg = read_pack(a.pack)
+ model_dir = pcfg["model_dir"]
+ n_exp = int(pcfg["n_routed_experts"])
+ ff, dim = int(pcfg["moe_intermediate_size"]), int(pcfg["hidden_size"])
+ n_do = a.experts or n_exp
+ os.makedirs(a.out, exist_ok=True)
+ idx_path = os.path.join(a.out, "tq.json")
+ idx = json.load(open(idx_path)) if os.path.exists(idx_path) else {}
+ bb = tqenc.block_bytes(t)
+ mm = {}
+ gpu_lock = threading.Lock()
+ for L in [int(x) for x in a.layers.split(",")]:
+ path = os.path.join(a.out, "tq-L%d.bin" % L)
+ sizes = {}
+ offs = {}
+ off = 0
+ for role, _ in ROLES:
+ rows, cols = (ff, dim) if role != "w2" else (dim, ff)
+ sizes[role] = (rows, cols, n_exp * rows * (cols // 128) * bb)
+ offs[role] = off
+ off += align(sizes[role][2])
+ with open(path, "wb") as f:
+ f.truncate(off)
+ fd = os.open(path, os.O_RDWR)
+ for role, rname in ROLES:
+ rows, cols, total = sizes[role]
+ t0 = time.time()
+ per_out = rows * (cols // 128) * bb
+ names = ["layers.%d.ffn.experts.%d.%s.weight" % (L, e, role) for e in range(n_do)]
+ err2 = np.zeros(n_do)
+ ref2 = np.zeros(n_do)
+
+ def one(e):
+ dt, fi, off_, nb, shape = tensors[names[e]]
+ if dt != "MXFP4":
+ raise SystemExit("%s is %s, not MXFP4: only MXFP4 sources are re-encoded" % (names[e], dt))
+ key = files[fi]
+ if key not in mm:
+ mm[key] = np.memmap(os.path.join(model_dir, key), dtype=np.uint8, mode="r")
+ w = decode_mxfp4(np.asarray(mm[key][off_:off_ + nb]), cols)
+ with gpu_lock:
+ packed, err, tot, st = tqenc.encode(w, t, gpu=not a.cpu, **cfg)
+ dec = tqenc.decode(packed, t).reshape(rows, cols)
+ err2[e] = ((w.astype(np.float64) - dec) ** 2).sum()
+ ref2[e] = (w.astype(np.float64) ** 2).sum()
+ os.pwrite(fd, packed.tobytes(), offs[role] + e * per_out)
+ return st["nonfinite_blocks"]
+
+ with cf.ThreadPoolExecutor(a.threads) as ex:
+ nonfin = sum(ex.map(one, range(n_do)))
+ print("layer %d %s: %d experts %s %.0f s, rel mse %.4f (sqnr %.2f dB), nonfinite blocks %d" %
+ (L, rname, n_do, a.type, time.time() - t0, err2.sum() / ref2.sum(), 10 * np.log10(ref2.sum() / err2.sum()), nonfin), flush=True)
+ idx["layers.%d.ffn.experts.%s" % (L, role)] = [tqenc.TYPE_NAMES[t], os.path.basename(path), offs[role], total, [n_exp, rows, cols]]
+ os.fsync(fd)
+ os.close(fd)
+ json.dump(idx, open(idx_path, "w"), indent=1)
+
+
+if __name__ == "__main__":
+ main()