Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: tq_convert.py (re-encode a pack's MXFP4 experts with the trellis encoder into the engine's file layout) and pack_gguf --tq-experts
Niko12217df82763e386
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex d1fd7a5e4..b56d57825 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -40,7 +40,10 @@ TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1# the low-bit expert types (the Maya quants' experts; the engine's ds4_iq kernels)10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),- 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136)}+ 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),+ # the trellis types (tools/trellis/tqenc.hpp; 145 / 146 are the agentionai fork's ggml ids, 224-227 Strata's own)+ 145: ("TQK6", 128, 26), 146: ("TQK7", 128, 30), 224: ("TQK5X", 128, 22), 225: ("TQK4X", 128, 18),+ 226: ("TQK5T", 128, 22), 227: ("TQK4T", 128, 18)}# the Engram set a quant without the tables takes them from by default, beside it (or inside its folder): unsloth's# MXFP8 shard of them, where setups before v1.1.17 put it for Maya-S (--engram: any other)ENGRAM_SETS = ("Engram",)@@ -328,6 +331,8 @@ def main() -> int:"in <model>/.. when there is one)")ap.add_argument("--mmproj", help="unsloth's vision tower GGUF (mmproj-*.gguf, clip / deepseek41v): its tensors become the ""checkpoint's vision.* / aligner.* (default: the first mmproj*.gguf in <model>/..)")+ ap.add_argument("--tq-experts", help="a folder tq_convert.py wrote (tq.json + tq-L<layer>.bin): the routed experts of the layers "+ "listed there are read from it as trellis types (TQK5X ..), the others keep their MXFP4")ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out ""(default: <model>/../Engram, when it is whole)")@@ -544,6 +549,47 @@ def main() -> int:"vision.rope_theta": 10000, "vision.max_image_tokens": 1024, "vision.min_pixels": k("image_min_pixels"),"image_token_id": img_id}print("vision tower from %s: %d blocks, image token %d" % (mm_path.name, mm_geo["vision.num_hidden_layers"], img_id))+ # trellis experts from tq_convert.py: per layer the three roles stacked [n_exp][rows][row bytes] in one file+ tq_layers = []+ if a.tq_experts:+ tqd = pathlib.Path(a.tq_experts).expanduser().resolve()+ tq_idx = json.loads((tqd / "tq.json").read_text())+ by_layer = {}+ for key, ent in tq_idx.items():+ m = re.match(r"layers\.(\d+)\.ffn\.experts\.(w[123])$", key)+ if not m:+ raise SystemExit("tq.json: %s is not a layer's expert role" % key)+ by_layer.setdefault(int(m.group(1)), {})[m.group(2)] = ent+ TQTYPES = {v[0]: k for k, v in TYPES.items()}+ for il, roles in sorted(by_layer.items()):+ if sorted(roles) != ["w1", "w2", "w3"]:+ raise SystemExit("tq.json: layer %d lacks a role (has %s)" % (il, sorted(roles)))+ if len({r[0] for r in roles.values()}) != 1 or roles["w1"][0] != roles["w3"][0]:+ raise SystemExit("tq.json: layer %d: gate and up must be one type" % il)+ drop = re.compile(r"layers\.%d\.ffn\.experts\.\d+\.w[123]\.weight$" % il)+ kept = [t for t in tensors if not drop.match(t[0])]+ if len(kept) == len(tensors):+ raise SystemExit("layer %d has no routed experts in the GGUF to replace" % il)+ tensors[:] = kept+ for role, (tname, fname, off, total, shape) in roles.items():+ if tname not in TQTYPES:+ raise SystemExit("tq.json: unknown type %s" % tname)+ n_exp, rows, cols = shape+ per = total // n_exp+ f = tqd / fname+ if not f.is_file() or f.stat().st_size < off + total:+ raise SystemExit("%s: missing or short" % f)+ try:+ relf = os.path.relpath(f, model).replace(os.sep, "/")+ except ValueError:+ raise SystemExit("%s must be on the model's drive" % f)+ if relf not in files:+ files.append(relf)+ fi = files.index(relf)+ for e in range(n_exp):+ tensors.append(("layers.%d.ffn.experts.%d.%s.weight" % (il, e, role), tname, fi, off + e * per, per, [rows, cols]))+ tq_layers.append(il)+ print("trellis experts from %s: layers %s" % (tqd, tq_layers))drafter = pathlib.Path(a.drafter).expanduser().resolve() if a.drafter else model.parent / "dspark" / "dspark.json"if a.drafter and not drafter.is_file():raise SystemExit("%s: no drafter there" % drafter)@@ -607,7 +653,7 @@ def main() -> int:by = {}for t in tensors:by[t[1]] = by.get(t[1], 0) + t[4]- extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision))) if there]+ extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision)), ("trellis experts in layers %s" % tq_layers, bool(tq_layers))) if there]print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %(model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),nl, cols, "".join(", " + w for w in extras), out))diff --git a/tools/dsv41/tq_convert.py b/tools/dsv41/tq_convert.pynew file mode 100644index 000000000..298dca622--- /dev/null+++ b/tools/dsv41/tq_convert.py@@ -0,0 +1,137 @@+#!/usr/bin/env python3+"""tq_convert.py - re-encode the routed experts of some layers of a DeepSeek-V4.1 pack with the trellis encoder+(tools/tqenc.py, HIP) and write them in the format the engine reads (strata-dsv41: ds4_tq.cu / ds4_tq_cpu.cpp):++ HIP_VISIBLE_DEVICES=2 TQENC_GPU_LOCK=2 python tools/dsv41/tq_convert.py --pack PACK --out DIR --layers 1,7 --type K5++PACK is the pack pack_gguf.py wrote (its tensors.txt names the source MXFP4 experts in the GGUF shards). Per layer one file+DIR/tq-L<layer>.bin holds the three roles one after another (gate = w1, up = w3, down = w2), each stacked [n_exp][rows][row+bytes] in the trellis types' file layout (a row = k / 128 blocks of {fp16 d; 4 K stream bytes}, tools/trellis/tqenc.hpp), 4096-aligned;+DIR/tq.json lists them as {"layers.N.ffn.experts.w1": [type name, file, offset, bytes, [n_exp, rows, cols]], ...}. Then++ python tools/dsv41/pack_gguf.py --model <gguf dir> --out PACK2 --tq-experts DIR++writes a pack whose layers listed there read these experts (the other layers keep their MXFP4). The weights are encoded as they+are (plain error, no rotation, no activation weights): what the dense override experiments of gate 3 measure. Types:+K4 = TQK4X, K5 = TQK5X (computed codebook, version 2), K4T / K5T (the table, version 1), K6 = TQK6, K7 = TQK7.+"""+import argparse+import concurrent.futures as cf+import json+import os+import pathlib+import sys+import threading+import time++import numpy as np++sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))+import tqenc # noqa: E402++TYPES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K4T": tqenc.TQK4T, "K5T": tqenc.TQK5T, "K6": tqenc.TQK6, "K7": tqenc.TQK7}+CFGS = {"fast": dict(iters=0, ncand=1, scale_tries=1), "default": dict(iters=1, ncand=1, scale_tries=1)}+MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)+ROLES = (("w1", "gate"), ("w3", "up"), ("w2", "down"))+++def decode_mxfp4(raw, n_in):+ """ggml MXFP4 blocks (32 values in 17 bytes: E8M0 scale, 16 code bytes) -> float32 [rows, n_in]"""+ b = raw.reshape(-1, 17)+ e = b[:, 0].astype(np.int32)+ d = np.where(e == 0, np.float32(2.0 ** -127), np.ldexp(np.float32(1.0), e - 128)).astype(np.float32)+ q = b[:, 1:]+ return (np.concatenate([MXFP4_LUT[q & 15], MXFP4_LUT[q >> 4]], axis=1) * d[:, None]).reshape(-1, n_in)+++def read_pack(path):+ files, tensors, cfg = {}, {}, {}+ for line in open(os.path.join(path, "tensors.txt"), encoding="utf-8"):+ f = line.split()+ if f and f[0] == "file":+ files[int(f[1])] = line.split(None, 2)[2].strip()+ elif f and f[0] == "t":+ tensors[f[1]] = (f[2], int(f[3]), int(f[4]), int(f[5]), [int(x) for x in f[6:]])+ for line in open(os.path.join(path, "ds4.txt"), encoding="utf-8"):+ k, _, v = line.strip().partition(" ")+ cfg[k] = v+ return files, tensors, cfg+++def align(n, a=4096):+ return (n + a - 1) // a * a+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--pack", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--layers", required=True, help="comma list")+ ap.add_argument("--type", required=True, choices=sorted(TYPES))+ ap.add_argument("--cfg", default="fast", choices=sorted(CFGS))+ ap.add_argument("--experts", type=int, default=0, help="only the first N experts (tests; the others stay zero streams)")+ ap.add_argument("--threads", type=int, default=4)+ ap.add_argument("--cpu", action="store_true", help="the CPU encoder instead of the card")+ a = ap.parse_args()+ t = TYPES[a.type]+ cfg = CFGS[a.cfg]+ files, tensors, pcfg = read_pack(a.pack)+ model_dir = pcfg["model_dir"]+ n_exp = int(pcfg["n_routed_experts"])+ ff, dim = int(pcfg["moe_intermediate_size"]), int(pcfg["hidden_size"])+ n_do = a.experts or n_exp+ os.makedirs(a.out, exist_ok=True)+ idx_path = os.path.join(a.out, "tq.json")+ idx = json.load(open(idx_path)) if os.path.exists(idx_path) else {}+ bb = tqenc.block_bytes(t)+ mm = {}+ gpu_lock = threading.Lock()+ for L in [int(x) for x in a.layers.split(",")]:+ path = os.path.join(a.out, "tq-L%d.bin" % L)+ sizes = {}+ offs = {}+ off = 0+ for role, _ in ROLES:+ rows, cols = (ff, dim) if role != "w2" else (dim, ff)+ sizes[role] = (rows, cols, n_exp * rows * (cols // 128) * bb)+ offs[role] = off+ off += align(sizes[role][2])+ with open(path, "wb") as f:+ f.truncate(off)+ fd = os.open(path, os.O_RDWR)+ for role, rname in ROLES:+ rows, cols, total = sizes[role]+ t0 = time.time()+ per_out = rows * (cols // 128) * bb+ names = ["layers.%d.ffn.experts.%d.%s.weight" % (L, e, role) for e in range(n_do)]+ err2 = np.zeros(n_do)+ ref2 = np.zeros(n_do)++ def one(e):+ dt, fi, off_, nb, shape = tensors[names[e]]+ if dt != "MXFP4":+ raise SystemExit("%s is %s, not MXFP4: only MXFP4 sources are re-encoded" % (names[e], dt))+ key = files[fi]+ if key not in mm:+ mm[key] = np.memmap(os.path.join(model_dir, key), dtype=np.uint8, mode="r")+ w = decode_mxfp4(np.asarray(mm[key][off_:off_ + nb]), cols)+ with gpu_lock:+ packed, err, tot, st = tqenc.encode(w, t, gpu=not a.cpu, **cfg)+ dec = tqenc.decode(packed, t).reshape(rows, cols)+ err2[e] = ((w.astype(np.float64) - dec) ** 2).sum()+ ref2[e] = (w.astype(np.float64) ** 2).sum()+ os.pwrite(fd, packed.tobytes(), offs[role] + e * per_out)+ return st["nonfinite_blocks"]++ with cf.ThreadPoolExecutor(a.threads) as ex:+ nonfin = sum(ex.map(one, range(n_do)))+ print("layer %d %s: %d experts %s %.0f s, rel mse %.4f (sqnr %.2f dB), nonfinite blocks %d" %+ (L, rname, n_do, a.type, time.time() - t0, err2.sum() / ref2.sum(), 10 * np.log10(ref2.sum() / err2.sum()), nonfin), flush=True)+ idx["layers.%d.ffn.experts.%s" % (L, role)] = [tqenc.TYPE_NAMES[t], os.path.basename(path), offs[role], total, [n_exp, rows, cols]]+ os.fsync(fd)+ os.close(fd)+ json.dump(idx, open(idx_path, "w"), indent=1)+++if __name__ == "__main__":+ main()