Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: GPU path in tqenc.py and the pilot, diagonal activation second moments (importance-weighted encode), error profile script
Niko12214292678b8f5b
diff --git a/tools/tqenc.py b/tools/tqenc.pyindex 334063e2f..a92feb7a6 100644--- a/tools/tqenc.py+++ b/tools/tqenc.py@@ -58,8 +58,35 @@ def lib() -> ctypes.CDLL:return _lib+_hip = None+++def hip_lib() -> ctypes.CDLL:+ """libtqenc_hip.so (sh tools/trellis/build.sh hip; run under the box GPU locks, HIP_VISIBLE_DEVICES picks the card)."""+ global _hip+ if _hip is None:+ cands = [os.environ.get("TQENC_HIP_LIB", ""),+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc_hip.so")]+ last = None+ for c in cands:+ if not c:+ continue+ try:+ _hip = ctypes.CDLL(c)+ break+ except OSError as e:+ last = e+ if _hip is None:+ raise RuntimeError("libtqenc_hip.so not found: %s" % last)+ _hip.tqh_encode.restype = ctypes.c_double+ _hip.tqh_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]+ return _hip++def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, ncand: int = 1, exact: int = 0,- scale_tries: int = 1, threads: int = 0):+ scale_tries: int = 1, threads: int = 0, gpu: bool = False):"""w: float32 array whose last axis is a multiple of 128. Returns (packed uint8 [nblocks, block_bytes],per-block error float64 [nblocks], total error, stats dict). Error = sum h (w - decoded)^2 (h = 1 if None)."""w = np.ascontiguousarray(w, dtype=np.float32)@@ -75,6 +102,12 @@ def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, nif h.shape != w.shape:raise ValueError("importance weights must have the shape of w")hp = h.ctypes.data+ if gpu:+ if exact or t == TQ2_T:+ raise ValueError("the GPU encoder has no exact search and no TQ2_T")+ tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,+ err.ctypes.data, stats.ctypes.data, None, 0)+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}tot = lib().tqe_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, exact, scale_tries, threads,err.ctypes.data, stats.ctypes.data)return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}diff --git a/tools/trellis/gyro_error_profile.py b/tools/trellis/gyro_error_profile.pynew file mode 100644index 000000000..3ead8b027--- /dev/null+++ b/tools/trellis/gyro_error_profile.py@@ -0,0 +1,91 @@+"""gyro_error_profile.py - how is agentionai's quantization error distributed, compared with a plain-MSE encode?++For a few expert matrices of a Gyro file: W' = rotated original (hf_slice), Wd = the file's decoded weights, Wo = our plain+encode. Reports, per column of the (rotated) input axis and per row, how uneven the squared error is (a plain-MSE encoder+spreads it evenly; an activation-weighted one concentrates it where the weights matter less), the scale ratio of their+blocks to the least-squares scale of their own path, and the correlation of their error with ours.+"""+from __future__ import annotations++import argparse+import mmap+import pathlib+import sys++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import gguf_reader # noqa: E402+import hf_slice # noqa: E402+import tqenc # noqa: E402+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--gguf", required=True)+ ap.add_argument("--repo", default="Qwen/Qwen3.8-Flash-Next")+ ap.add_argument("--layers", default="10")+ ap.add_argument("--experts", default="0")+ ap.add_argument("--roles", default="gate,down")+ a = ap.parse_args()+ g = gguf_reader.GGUFFile(a.gguf)+ mm = mmap.mmap(open(a.gguf, "rb").fileno(), 0, access=mmap.ACCESS_READ)+ by = {t.name: t for t in g.tensors}+ sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]+ sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)+ signs, at = {}, 0+ for w in sw:+ signs[w] = sv[at:at + w]+ at += w+ repo = hf_slice.Repo(a.repo)+ for layer in [int(x) for x in a.layers.split(",")]:+ pre = "model.language_model.layers.%d.mlp.experts." % layer+ for e in [int(x) for x in a.experts.split(",")]:+ gu = repo.tensor(pre + "gate_up_proj", e)[0]+ dn = repo.tensor(pre + "down_proj", e)[0]+ mats = {"gate": gu[:640], "up": gu[640:], "down": dn}+ for role in a.roles.split(","):+ W = mats[role]+ ti = by["blk.%d.ffn_%s_exps.weight" % (layer, role)]+ t = tqenc.TYPE_BY_NAME[ti.type_name]+ n_in, n_out = ti.shape[0], ti.shape[1]+ bb = tqenc.block_bytes(t)+ rb = n_in // 128 * bb+ raw = np.frombuffer(mm, dtype=np.uint8, count=n_out * rb, offset=g.data_start + ti.offset + e * n_out * rb)+ Wd = tqenc.decode(raw, t).reshape(n_out, n_in)+ Wr = tqenc.rotate(W, signs[n_in])+ packed, _, _, _ = tqenc.encode(Wr, t, iters=3, ncand=3)+ Wo = tqenc.decode(packed, t).reshape(n_out, n_in)+ Ed, Eo = (Wr - Wd).astype(np.float64), (Wr - Wo).astype(np.float64)+ print("L%d e%d %s %s" % (layer, e, role, ti.type_name))+ for name, E in (("agentionai", Ed), ("ours (plain MSE)", Eo)):+ col = (E ** 2).mean(axis=0)+ row = (E ** 2).mean(axis=1)+ blk = (E ** 2).reshape(n_out, n_in // 128, 128).mean(axis=2)+ pos = (E ** 2).reshape(-1, 128).mean(axis=0)+ print(" %-18s rel MSE %.4f | per-column error: cv %.3f, max/mean %.2f | per-row: cv %.3f | per-128-block: cv %.3f | "+ "position-in-block cv %.3f | corr(error, |w|) %.3f" % (+ name, (E ** 2).sum() / (Wr.astype(np.float64) ** 2).sum(), col.std() / col.mean(), col.max() / col.mean(),+ row.std() / row.mean(), blk.std() / blk.mean(), pos.std() / pos.mean(),+ np.corrcoef(np.abs(Wr).ravel()[::7], np.abs(E).ravel()[::7])[0, 1]))+ # their scale vs the least-squares scale for their own path+ dv = raw.reshape(-1, bb)[:, :2].copy().view(np.float16).astype(np.float64).ravel()+ cw = tqenc.decode(np.concatenate([np.zeros((raw.size // bb, 2), np.uint8), raw.reshape(-1, bb)[:, 2:]], axis=1), t) # d = 0 -> zeros+ # unit-scale path values: decode with d = 1.0 (fp16 0x3c00)+ one = raw.reshape(-1, bb).copy()+ one[:, 0] = 0x00+ one[:, 1] = 0x3c+ cu = tqenc.decode(one, t).reshape(-1, 128).astype(np.float64)+ wb = Wr.reshape(-1, 128).astype(np.float64)+ ls = (wb * cu).sum(axis=1) / (cu * cu).sum(axis=1)+ print(" their scale / LS-optimal scale for their own path: median %.3f p10 %.3f p90 %.3f" % (+ np.median(dv / ls), np.percentile(dv / ls, 10), np.percentile(dv / ls, 90)))+ print(" error correlation (theirs, ours): %.3f energy of their error that is orthogonal to ours: %.3f" % (+ np.corrcoef(Ed.ravel()[::5], Eo.ravel()[::5])[0, 1], 1 - (Ed * Eo).sum() ** 2 / ((Ed ** 2).sum() * (Eo ** 2).sum())))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.pyindex 7d722f07c..9781251a8 100644--- a/tools/trellis/pilot.py+++ b/tools/trellis/pilot.py@@ -150,6 +150,11 @@ def main() -> int:ap.add_argument("--out", required=True, help="jsonl results")ap.add_argument("--seed", type=int, default=1)ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")+ ap.add_argument("--gpu", action="store_true", help="encode on the GPU (run under the box locks, HIP_VISIBLE_DEVICES)")+ ap.add_argument("--act-diag-dir", default=None,+ help="directory with L<layer>_<role>.npy = E[x^2] per input channel of that projection (calibration "+ "second moments, diagonal). Adds act_diag (the exact diagonal-Hessian error) and an importance-weighted "+ "encode (h = E[x^2]; in the Hadamard basis h = the block average of it)")a = ap.parse_args()if a.remote:@@ -180,13 +185,19 @@ def main() -> int:x_cache[n_in] = (np.random.default_rng(7).standard_normal((n_in, a.tokens)).astype(np.float32),synth_x(n_in, a.tokens, 9))Xi, Xs = x_cache[n_in]+ diag = None+ if a.act_diag_dir:+ dp = pathlib.Path(a.act_diag_dir) / ("L%d_%s.npy" % (layer, role))+ if dp.exists():+ diag = np.load(dp).astype(np.float64)+ assert diag.shape == (n_in,), (dp, diag.shape)for had in (False, True):sg = sign_cache[n_in] if had else NoneWr = tqenc.rotate(W, sg) if had else Wfor rname in a.rates.split(","):t = RATES[rname]t0 = time.time()- packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, **cfg)+ packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, gpu=a.gpu, **cfg)sec = time.time() - t0dec = tqenc.decode(packed, t).reshape(Wr.shape)What = tqenc.unrotate(dec, sg) if had else dec@@ -195,6 +206,16 @@ def main() -> int:act_iso=act_err(W, What, Xi), act_syn=act_err(W, What, Xs),bpw=tqenc.bpw(t), seconds=sec, zero_blocks=st["zero_blocks"], cfg=a.cfg,x_kind="placeholder: iso gaussian / synthetic outlier channels")+ if diag is not None:+ hv = np.maximum(diag, 1e-30).astype(np.float32)+ rec["act_diag"] = float(np.sum(((W - What).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))+ hb = hv if not had else np.repeat(hv.reshape(-1, 128).mean(axis=1), 128)+ hw = np.broadcast_to(hb, Wr.shape).astype(np.float32)+ packed2, _, _, _ = tqenc.encode(Wr, t, h=hw, threads=a.threads, gpu=a.gpu, **cfg)+ dec2 = tqenc.decode(packed2, t).reshape(Wr.shape)+ What2 = tqenc.unrotate(dec2, sg) if had else dec2+ rec["act_diag_weighted_encode"] = float(np.sum(((W - What2).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))+ rec["mse_weighted_encode"] = float(np.sum((W.astype(np.float64) - What2) ** 2) / en)out.write(json.dumps(rec) + "\n")out.flush()print("L%-3d %-4s e%-3d %s %-5s mse %.4f (%.2f dB) act_iso %.4f act_syn %.4f %.1fs" % (diff --git a/tools/trellis/pilot_summary.py b/tools/trellis/pilot_summary.pynew file mode 100644index 000000000..76c3c80c9--- /dev/null+++ b/tools/trellis/pilot_summary.py@@ -0,0 +1,42 @@+"""pilot_summary.py - tables from the jsonl of pilot.py.+ python3 tools/trellis/pilot_summary.py pilot1.jsonl+"""+import collections+import json+import math+import sys++rows = [json.loads(l) for l in open(sys.argv[1])]+print("records:", len(rows), "layers:", sorted({r["layer"] for r in rows}), "experts:", sorted({r["expert"] for r in rows}),+ "rows per matrix:", sorted({r["rows"] for r in rows}))+print("activation columns are PLACEHOLDERS (iso gaussian / synthetic outlier channels), not calibration data\n")+++def db(x):+ return -10 * math.log10(x)+++def table(keyf, title, cols=("mse", "act_iso", "act_syn")):+ g = collections.defaultdict(list)+ for r in rows:+ g[keyf(r)].append(r)+ print(title)+ for k in sorted(g):+ v = g[k]+ print(" %-40s n=%-3d " % (k, len(v)) + " ".join("%s %.4f (%.2f dB)" % (c, sum(x[c] for x in v) / len(v), db(sum(x[c] for x in v) / len(v))) for c in cols))+ print()+++rate_bpw = {"K4": 1.125, "K5": 1.375, "K6": 1.625, "K7": 1.875}+table(lambda r: "%s %.3f bpw %s" % (r["rate"], rate_bpw[r["rate"]], "hadamard" if r["had"] else "plain "), "mean over everything")+table(lambda r: "%s %s %s" % (r["role"], r["rate"], "had" if r["had"] else "plain"), "by projection")+table(lambda r: "L%02d %s %s" % (r["layer"], r["rate"], "had" if r["had"] else "plain"), "by layer")+# spread of the weight mse across experts for K5 plain+v = [r["mse"] for r in rows if r["rate"] == "K5" and not r["had"]]+print("K5 plain weight mse over all %d matrices: min %.4f max %.4f" % (len(v), min(v), max(v)))+# hadamard gain on the synthetic outlier activations+print("\nHadamard effect on act_syn (relative error, plain / hadamard), per rate")+for rate in ("K4", "K5", "K6", "K7"):+ a = [r["act_syn"] for r in rows if r["rate"] == rate and not r["had"]]+ b = [r["act_syn"] for r in rows if r["rate"] == rate and r["had"]]+ print(" %s plain %.4f had %.4f ratio %.3f" % (rate, sum(a) / len(a), sum(b) / len(b), (sum(a) / len(a)) / (sum(b) / len(b))))