Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: block-feedback (BlockLDLQ) encoder and the ldl mode of build_swap
Niko1221a7eb99bba92f
diff --git a/tools/trellis/blockldl.py b/tools/trellis/blockldl.pynew file mode 100644index 000000000..e1ff3c2e1--- /dev/null+++ b/tools/trellis/blockldl.py@@ -0,0 +1,73 @@+"""blockldl.py - Hessian-aware trellis encoding by block feedback (the block form of GPTQ / LDLQ, as QTIP does it).++Weights W [m, n] in the rotated basis, H = E[x x^T] [n, n] of the rotated layer input. Columns are cut into 128-blocks (the+trellis blocks). Block b is encoded on its own (m blocks of 128, plain-MSE trellis search), its error E_b = W_b - Q_b is+pushed onto the not yet coded columns, exactly as the OBS update does for a whole block:++ W[:, c1:] -= E_b R_bb^{-1} R[c0:c1, c1:] R upper, H^-1 = R^T R++so the later blocks absorb what the earlier ones could not represent, in the directions that matter for ||(W - Q) X||.+The result is a normal TQK6 / TQK7 / TQK5X / TQK4X tensor; only the encoder changed.+"""+from __future__ import annotations++import numpy as np++import tqenc++try:+ from scipy.linalg import solve_triangular+except Exception: # numpy only: a small back substitution+ solve_triangular = None+++def _solve_upper(A: np.ndarray, B: np.ndarray) -> np.ndarray:+ if solve_triangular is not None:+ return solve_triangular(A, B, lower=False)+ return np.linalg.solve(A, B)+++def hessian_factor(H: np.ndarray, damp: float = 0.01) -> np.ndarray:+ """R (upper) with (H + damp * mean(diag H) I)^-1 = R^T R."""+ H = np.asarray(H, dtype=np.float64).copy()+ d = damp * float(np.mean(np.diag(H)))+ H[np.diag_indices_from(H)] += d+ Hinv = np.linalg.inv(H)+ Hinv = (Hinv + Hinv.T) * 0.5+ L = np.linalg.cholesky(Hinv)+ return L.T+++def feedback_matrices(R: np.ndarray, g: int = 128):+ """For every block b the matrix R_bb^{-1} R[b, b+1:] ([g, n - (b+1) g]); the last block has none."""+ n = R.shape[0]+ out = []+ for c0 in range(0, n, g):+ c1 = c0 + g+ out.append(_solve_upper(R[c0:c1, c0:c1], R[c0:c1, c1:]).astype(np.float32) if c1 < n else None)+ return out+++def encode_ldl(W: np.ndarray, t: int, fb, gpu: bool = False, iters: int = 1, ncand: int = 1, threads: int = 0):+ """W [m, n] float32 (n % 128 == 0), fb from feedback_matrices. Returns (packed [m, n/128, block_bytes], decoded [m, n])."""+ m, n = W.shape+ B = n // 128+ bb = tqenc.block_bytes(t)+ Wc = np.ascontiguousarray(W, dtype=np.float32).copy()+ packed = np.empty((m, B, bb), dtype=np.uint8)+ dec = np.empty((m, n), dtype=np.float32)+ kw = dict(iters=iters, ncand=ncand)+ if gpu:+ kw["gpu"] = True+ else:+ kw["threads"] = threads+ for b in range(B):+ c0, c1 = b * 128, (b + 1) * 128+ blk = np.ascontiguousarray(Wc[:, c0:c1])+ pk, _, _, _ = tqenc.encode(blk, t, **kw)+ q = tqenc.decode(pk, t).reshape(m, 128)+ packed[:, b] = pk+ dec[:, c0:c1] = q+ if fb[b] is not None:+ Wc[:, c1:] -= (blk - q) @ fb[b]+ return packed, decdiff --git a/tools/trellis/build_swap.py b/tools/trellis/build_swap.pyindex 2df3be544..e9f797ff8 100644--- a/tools/trellis/build_swap.py+++ b/tools/trellis/build_swap.py@@ -8,6 +8,7 @@ Modes (all from the ORIGINAL Qwen weights, rotated into the file's basis with thq8 Q8_0 (types 8/8): the near-lossless reference for the swapped layersplain TQK6 gate/up + TQK7 down, plain-MSE trellis encode (tools/tqenc.py)diag the same with per-input-column importance weights h = E[x^2]^power from the calibration dump (STRATA_ACT_DUMP)+ ldl the same with block feedback from the full second-moment matrix of the dump (STRATA_ACT_DUMP_COV=1), blockldl.pypython3 tools/trellis/build_swap.py --gguf SHIPPED --pack PACK --orig DIR --layers 4,16,28,40 --mode plain --var DIR [--gpu]"""@@ -25,6 +26,7 @@ HERE = pathlib.Path(__file__).resolve().parentsys.path.insert(0, str(HERE.parent))sys.path.insert(0, str(HERE))import gguf_reader # noqa: E402+import blockldl # noqa: E402import tqenc # noqa: E402N_EXP, N_EMBD, N_FF = 512, 2560, 640@@ -62,13 +64,24 @@ def load_h(dump_dir, layer, kind, power):return (h / h.mean()).astype(np.float32), int(count)+def load_hessian(dump_dir, layer, kind):+ p = pathlib.Path(dump_dir) / ("act_L%d_%s.bin" % (layer, kind))+ raw = p.read_bytes()+ n, count = np.frombuffer(raw[:16], dtype=np.int64)+ n = int(n)+ if len(raw) < 16 + 8 * n + 8 * n * n:+ raise SystemExit("%s has no covariance (run the calibration with STRATA_ACT_DUMP_COV=1)" % p)+ cov = np.frombuffer(raw[16 + 8 * n:16 + 8 * n + 8 * n * n], dtype=np.float64).reshape(n, n)+ return cov / max(int(count), 1)++def main() -> int:ap = argparse.ArgumentParser()ap.add_argument("--gguf", required=True)ap.add_argument("--pack", required=True)ap.add_argument("--orig", required=True)ap.add_argument("--layers", required=True)- ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag"])+ ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag", "ldl"])ap.add_argument("--var", required=True)ap.add_argument("--dump-dir", default=None)ap.add_argument("--power", type=float, default=1.0)@@ -77,6 +90,7 @@ def main() -> int:ap.add_argument("--iters", type=int, default=2)ap.add_argument("--ncand", type=int, default=2)ap.add_argument("--chunk", type=int, default=16)+ ap.add_argument("--damp", type=float, default=0.01, help="ldl: damping, a fraction of the mean diagonal of H")a = ap.parse_args()layers = [int(x) for x in a.layers.split(",")]var = pathlib.Path(a.var)@@ -126,6 +140,10 @@ def main() -> int:gu_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_gate_up.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, 2 * N_FF, N_EMBD)dn_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_down.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, N_EMBD, N_FF)h_gu = h_dn = None+ fb_gu = fb_dn = None+ if a.mode == "ldl":+ fb_gu = blockldl.feedback_matrices(blockldl.hessian_factor(load_hessian(a.dump_dir, l, "gu"), a.damp))+ fb_dn = blockldl.feedback_matrices(blockldl.hessian_factor(load_hessian(a.dump_dir, l, "dn"), a.damp))if a.mode == "diag":h_gu, c1 = load_h(a.dump_dir, l, "gu", a.power)h_dn, c2 = load_h(a.dump_dir, l, "dn", a.power)@@ -139,12 +157,27 @@ def main() -> int:mats = {"gate": (gu[:, :N_FF], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),"up": (gu[:, N_FF:], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),"down": (dn, signs[N_FF], h_dn, row_d, N_EMBD, t_down, d_t)}+ ldl_done = {}+ if a.mode == "ldl":+ # gate and up share H: one batch of rows+ both = np.concatenate([tqenc.rotate(np.ascontiguousarray(gu[:, :N_FF]), signs[N_EMBD]).reshape(-1, N_EMBD),+ tqenc.rotate(np.ascontiguousarray(gu[:, N_FF:]), signs[N_EMBD]).reshape(-1, N_EMBD)])+ pk, dec = blockldl.encode_ldl(both, gu_t, fb_gu, gpu=a.gpu, iters=a.iters, ncand=a.ncand, threads=a.threads)+ half = pk.shape[0] // 2+ ldl_done["gate"] = (pk[:half].reshape(half, -1), dec[:half])+ ldl_done["up"] = (pk[half:].reshape(half, -1), dec[half:])+ wrd = tqenc.rotate(np.ascontiguousarray(dn), signs[N_FF]).reshape(-1, N_FF)+ pk, dec = blockldl.encode_ldl(wrd, d_t, fb_dn, gpu=a.gpu, iters=a.iters, ncand=a.ncand, threads=a.threads)+ ldl_done["down"] = (pk.reshape(pk.shape[0], -1), dec)for role, (w, sg, h, row, n_out, tb, ty) in mats.items():wr = tqenc.rotate(np.ascontiguousarray(w), sg) # [chunk, n_out, n_in]n_in = wr.shape[-1]if a.mode == "q8":data = q8_0_quantize(wr)dec = None+ elif a.mode == "ldl":+ data, dec = ldl_done[role]+ err_sum[role] += float(np.sum((wr.reshape(-1, n_in).astype(np.float64) - dec) ** 2))else:packed, tot = enc(wr.reshape(-1, n_in), ty, h if a.mode == "diag" else None)data = packeddiff --git a/tools/trellis/test_blockldl.py b/tools/trellis/test_blockldl.pynew file mode 100644index 000000000..85dabee98--- /dev/null+++ b/tools/trellis/test_blockldl.py@@ -0,0 +1,44 @@+"""test_blockldl.py - block feedback against plain encoding on a synthetic anisotropic H (an incoherent random basis with a+spread spectrum) and on real expert weights: the proxy loss tr(E H E^T) must drop, the plain loss rises a little.+ python3 tools/trellis/test_blockldl.py [--orig DIR --layer 4]+"""+import argparse+import pathlib+import sys++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import blockldl # noqa: E402+import tqenc # noqa: E402++ap = argparse.ArgumentParser()+ap.add_argument("--n", type=int, default=1024)+ap.add_argument("--rows", type=int, default=512)+ap.add_argument("--spread", type=float, default=100.0, help="ratio of the largest to the smallest eigenvalue")+ap.add_argument("--type", default="TQK6")+a = ap.parse_args()+t = tqenc.TYPE_BY_NAME[a.type]+rng = np.random.default_rng(1)+n = a.n+Q, _ = np.linalg.qr(rng.standard_normal((n, n)))+lam = np.logspace(0, -np.log10(a.spread), n)+H = (Q * lam) @ Q.T+W = (rng.standard_normal((a.rows, n)) * 0.02).astype(np.float32)+R = blockldl.hessian_factor(H, 0.01)+fb = blockldl.feedback_matrices(R)+pk, _, _, _ = tqenc.encode(W, t, iters=1, threads=8)+plain = tqenc.decode(pk, t).reshape(W.shape)+_, ldl = blockldl.encode_ldl(W, t, fb, threads=8)+++def proxy(Wq):+ E = (W - Wq).astype(np.float64)+ return np.trace(E @ H @ E.T) / np.trace(W.astype(np.float64) @ H @ W.T.astype(np.float64))+++for name, Wq in (("plain", plain), ("block LDL", ldl)):+ E = W - Wq+ print("%-10s unweighted %.2f dB H-weighted %.2f dB" % (name, 10 * np.log10(np.sum(W.astype(np.float64) ** 2) / np.sum(E.astype(np.float64) ** 2)), -10 * np.log10(proxy(Wq))))