Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis: GPU path in tqenc.py and the pilot, diagonal activation second moments (importance-weighted encode), error profile script

Niko12214292678b8f5b
diff --git a/tools/tqenc.py b/tools/tqenc.py
index 334063e2f..a92feb7a6 100644
--- a/tools/tqenc.py
+++ b/tools/tqenc.py
@@ -58,8 +58,35 @@ def lib() -> ctypes.CDLL:
return _lib
+_hip = None
+
+
+def hip_lib() -> ctypes.CDLL:
+ """libtqenc_hip.so (sh tools/trellis/build.sh hip; run under the box GPU locks, HIP_VISIBLE_DEVICES picks the card)."""
+ global _hip
+ if _hip is None:
+ cands = [os.environ.get("TQENC_HIP_LIB", ""),
+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc_hip.so")]
+ last = None
+ for c in cands:
+ if not c:
+ continue
+ try:
+ _hip = ctypes.CDLL(c)
+ break
+ except OSError as e:
+ last = e
+ if _hip is None:
+ raise RuntimeError("libtqenc_hip.so not found: %s" % last)
+ _hip.tqh_encode.restype = ctypes.c_double
+ _hip.tqh_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,
+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
+ return _hip
+
+
def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, ncand: int = 1, exact: int = 0,
- scale_tries: int = 1, threads: int = 0):
+ scale_tries: int = 1, threads: int = 0, gpu: bool = False):
"""w: float32 array whose last axis is a multiple of 128. Returns (packed uint8 [nblocks, block_bytes],
per-block error float64 [nblocks], total error, stats dict). Error = sum h (w - decoded)^2 (h = 1 if None)."""
w = np.ascontiguousarray(w, dtype=np.float32)
@@ -75,6 +102,12 @@ def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, n
if h.shape != w.shape:
raise ValueError("importance weights must have the shape of w")
hp = h.ctypes.data
+ if gpu:
+ if exact or t == TQ2_T:
+ raise ValueError("the GPU encoder has no exact search and no TQ2_T")
+ tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,
+ err.ctypes.data, stats.ctypes.data, None, 0)
+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}
tot = lib().tqe_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, exact, scale_tries, threads,
err.ctypes.data, stats.ctypes.data)
return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}
diff --git a/tools/trellis/gyro_error_profile.py b/tools/trellis/gyro_error_profile.py
new file mode 100644
index 000000000..3ead8b027
--- /dev/null
+++ b/tools/trellis/gyro_error_profile.py
@@ -0,0 +1,91 @@
+"""gyro_error_profile.py - how is agentionai's quantization error distributed, compared with a plain-MSE encode?
+
+For a few expert matrices of a Gyro file: W' = rotated original (hf_slice), Wd = the file's decoded weights, Wo = our plain
+encode. Reports, per column of the (rotated) input axis and per row, how uneven the squared error is (a plain-MSE encoder
+spreads it evenly; an activation-weighted one concentrates it where the weights matter less), the scale ratio of their
+blocks to the least-squares scale of their own path, and the correlation of their error with ours.
+"""
+from __future__ import annotations
+
+import argparse
+import mmap
+import pathlib
+import sys
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import gguf_reader # noqa: E402
+import hf_slice # noqa: E402
+import tqenc # noqa: E402
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--gguf", required=True)
+ ap.add_argument("--repo", default="Qwen/Qwen3.8-Flash-Next")
+ ap.add_argument("--layers", default="10")
+ ap.add_argument("--experts", default="0")
+ ap.add_argument("--roles", default="gate,down")
+ a = ap.parse_args()
+ g = gguf_reader.GGUFFile(a.gguf)
+ mm = mmap.mmap(open(a.gguf, "rb").fileno(), 0, access=mmap.ACCESS_READ)
+ by = {t.name: t for t in g.tensors}
+ sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]
+ sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)
+ signs, at = {}, 0
+ for w in sw:
+ signs[w] = sv[at:at + w]
+ at += w
+ repo = hf_slice.Repo(a.repo)
+ for layer in [int(x) for x in a.layers.split(",")]:
+ pre = "model.language_model.layers.%d.mlp.experts." % layer
+ for e in [int(x) for x in a.experts.split(",")]:
+ gu = repo.tensor(pre + "gate_up_proj", e)[0]
+ dn = repo.tensor(pre + "down_proj", e)[0]
+ mats = {"gate": gu[:640], "up": gu[640:], "down": dn}
+ for role in a.roles.split(","):
+ W = mats[role]
+ ti = by["blk.%d.ffn_%s_exps.weight" % (layer, role)]
+ t = tqenc.TYPE_BY_NAME[ti.type_name]
+ n_in, n_out = ti.shape[0], ti.shape[1]
+ bb = tqenc.block_bytes(t)
+ rb = n_in // 128 * bb
+ raw = np.frombuffer(mm, dtype=np.uint8, count=n_out * rb, offset=g.data_start + ti.offset + e * n_out * rb)
+ Wd = tqenc.decode(raw, t).reshape(n_out, n_in)
+ Wr = tqenc.rotate(W, signs[n_in])
+ packed, _, _, _ = tqenc.encode(Wr, t, iters=3, ncand=3)
+ Wo = tqenc.decode(packed, t).reshape(n_out, n_in)
+ Ed, Eo = (Wr - Wd).astype(np.float64), (Wr - Wo).astype(np.float64)
+ print("L%d e%d %s %s" % (layer, e, role, ti.type_name))
+ for name, E in (("agentionai", Ed), ("ours (plain MSE)", Eo)):
+ col = (E ** 2).mean(axis=0)
+ row = (E ** 2).mean(axis=1)
+ blk = (E ** 2).reshape(n_out, n_in // 128, 128).mean(axis=2)
+ pos = (E ** 2).reshape(-1, 128).mean(axis=0)
+ print(" %-18s rel MSE %.4f | per-column error: cv %.3f, max/mean %.2f | per-row: cv %.3f | per-128-block: cv %.3f | "
+ "position-in-block cv %.3f | corr(error, |w|) %.3f" % (
+ name, (E ** 2).sum() / (Wr.astype(np.float64) ** 2).sum(), col.std() / col.mean(), col.max() / col.mean(),
+ row.std() / row.mean(), blk.std() / blk.mean(), pos.std() / pos.mean(),
+ np.corrcoef(np.abs(Wr).ravel()[::7], np.abs(E).ravel()[::7])[0, 1]))
+ # their scale vs the least-squares scale for their own path
+ dv = raw.reshape(-1, bb)[:, :2].copy().view(np.float16).astype(np.float64).ravel()
+ cw = tqenc.decode(np.concatenate([np.zeros((raw.size // bb, 2), np.uint8), raw.reshape(-1, bb)[:, 2:]], axis=1), t) # d = 0 -> zeros
+ # unit-scale path values: decode with d = 1.0 (fp16 0x3c00)
+ one = raw.reshape(-1, bb).copy()
+ one[:, 0] = 0x00
+ one[:, 1] = 0x3c
+ cu = tqenc.decode(one, t).reshape(-1, 128).astype(np.float64)
+ wb = Wr.reshape(-1, 128).astype(np.float64)
+ ls = (wb * cu).sum(axis=1) / (cu * cu).sum(axis=1)
+ print(" their scale / LS-optimal scale for their own path: median %.3f p10 %.3f p90 %.3f" % (
+ np.median(dv / ls), np.percentile(dv / ls, 10), np.percentile(dv / ls, 90)))
+ print(" error correlation (theirs, ours): %.3f energy of their error that is orthogonal to ours: %.3f" % (
+ np.corrcoef(Ed.ravel()[::5], Eo.ravel()[::5])[0, 1], 1 - (Ed * Eo).sum() ** 2 / ((Ed ** 2).sum() * (Eo ** 2).sum())))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.py
index 7d722f07c..9781251a8 100644
--- a/tools/trellis/pilot.py
+++ b/tools/trellis/pilot.py
@@ -150,6 +150,11 @@ def main() -> int:
ap.add_argument("--out", required=True, help="jsonl results")
ap.add_argument("--seed", type=int, default=1)
ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")
+ ap.add_argument("--gpu", action="store_true", help="encode on the GPU (run under the box locks, HIP_VISIBLE_DEVICES)")
+ ap.add_argument("--act-diag-dir", default=None,
+ help="directory with L<layer>_<role>.npy = E[x^2] per input channel of that projection (calibration "
+ "second moments, diagonal). Adds act_diag (the exact diagonal-Hessian error) and an importance-weighted "
+ "encode (h = E[x^2]; in the Hadamard basis h = the block average of it)")
a = ap.parse_args()
if a.remote:
@@ -180,13 +185,19 @@ def main() -> int:
x_cache[n_in] = (np.random.default_rng(7).standard_normal((n_in, a.tokens)).astype(np.float32),
synth_x(n_in, a.tokens, 9))
Xi, Xs = x_cache[n_in]
+ diag = None
+ if a.act_diag_dir:
+ dp = pathlib.Path(a.act_diag_dir) / ("L%d_%s.npy" % (layer, role))
+ if dp.exists():
+ diag = np.load(dp).astype(np.float64)
+ assert diag.shape == (n_in,), (dp, diag.shape)
for had in (False, True):
sg = sign_cache[n_in] if had else None
Wr = tqenc.rotate(W, sg) if had else W
for rname in a.rates.split(","):
t = RATES[rname]
t0 = time.time()
- packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, **cfg)
+ packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, gpu=a.gpu, **cfg)
sec = time.time() - t0
dec = tqenc.decode(packed, t).reshape(Wr.shape)
What = tqenc.unrotate(dec, sg) if had else dec
@@ -195,6 +206,16 @@ def main() -> int:
act_iso=act_err(W, What, Xi), act_syn=act_err(W, What, Xs),
bpw=tqenc.bpw(t), seconds=sec, zero_blocks=st["zero_blocks"], cfg=a.cfg,
x_kind="placeholder: iso gaussian / synthetic outlier channels")
+ if diag is not None:
+ hv = np.maximum(diag, 1e-30).astype(np.float32)
+ rec["act_diag"] = float(np.sum(((W - What).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))
+ hb = hv if not had else np.repeat(hv.reshape(-1, 128).mean(axis=1), 128)
+ hw = np.broadcast_to(hb, Wr.shape).astype(np.float32)
+ packed2, _, _, _ = tqenc.encode(Wr, t, h=hw, threads=a.threads, gpu=a.gpu, **cfg)
+ dec2 = tqenc.decode(packed2, t).reshape(Wr.shape)
+ What2 = tqenc.unrotate(dec2, sg) if had else dec2
+ rec["act_diag_weighted_encode"] = float(np.sum(((W - What2).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))
+ rec["mse_weighted_encode"] = float(np.sum((W.astype(np.float64) - What2) ** 2) / en)
out.write(json.dumps(rec) + "\n")
out.flush()
print("L%-3d %-4s e%-3d %s %-5s mse %.4f (%.2f dB) act_iso %.4f act_syn %.4f %.1fs" % (
diff --git a/tools/trellis/pilot_summary.py b/tools/trellis/pilot_summary.py
new file mode 100644
index 000000000..76c3c80c9
--- /dev/null
+++ b/tools/trellis/pilot_summary.py
@@ -0,0 +1,42 @@
+"""pilot_summary.py - tables from the jsonl of pilot.py.
+ python3 tools/trellis/pilot_summary.py pilot1.jsonl
+"""
+import collections
+import json
+import math
+import sys
+
+rows = [json.loads(l) for l in open(sys.argv[1])]
+print("records:", len(rows), "layers:", sorted({r["layer"] for r in rows}), "experts:", sorted({r["expert"] for r in rows}),
+ "rows per matrix:", sorted({r["rows"] for r in rows}))
+print("activation columns are PLACEHOLDERS (iso gaussian / synthetic outlier channels), not calibration data\n")
+
+
+def db(x):
+ return -10 * math.log10(x)
+
+
+def table(keyf, title, cols=("mse", "act_iso", "act_syn")):
+ g = collections.defaultdict(list)
+ for r in rows:
+ g[keyf(r)].append(r)
+ print(title)
+ for k in sorted(g):
+ v = g[k]
+ print(" %-40s n=%-3d " % (k, len(v)) + " ".join("%s %.4f (%.2f dB)" % (c, sum(x[c] for x in v) / len(v), db(sum(x[c] for x in v) / len(v))) for c in cols))
+ print()
+
+
+rate_bpw = {"K4": 1.125, "K5": 1.375, "K6": 1.625, "K7": 1.875}
+table(lambda r: "%s %.3f bpw %s" % (r["rate"], rate_bpw[r["rate"]], "hadamard" if r["had"] else "plain "), "mean over everything")
+table(lambda r: "%s %s %s" % (r["role"], r["rate"], "had" if r["had"] else "plain"), "by projection")
+table(lambda r: "L%02d %s %s" % (r["layer"], r["rate"], "had" if r["had"] else "plain"), "by layer")
+# spread of the weight mse across experts for K5 plain
+v = [r["mse"] for r in rows if r["rate"] == "K5" and not r["had"]]
+print("K5 plain weight mse over all %d matrices: min %.4f max %.4f" % (len(v), min(v), max(v)))
+# hadamard gain on the synthetic outlier activations
+print("\nHadamard effect on act_syn (relative error, plain / hadamard), per rate")
+for rate in ("K4", "K5", "K6", "K7"):
+ a = [r["act_syn"] for r in rows if r["rate"] == rate and not r["had"]]
+ b = [r["act_syn"] for r in rows if r["rate"] == rate and r["had"]]
+ print(" %s plain %.4f had %.4f ratio %.3f" % (rate, sum(a) / len(a), sum(b) / len(b), (sum(a) / len(a)) / (sum(b) / len(b))))