Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis: block-feedback (BlockLDLQ) encoder and the ldl mode of build_swap

Niko1221a7eb99bba92f
diff --git a/tools/trellis/blockldl.py b/tools/trellis/blockldl.py
new file mode 100644
index 000000000..e1ff3c2e1
--- /dev/null
+++ b/tools/trellis/blockldl.py
@@ -0,0 +1,73 @@
+"""blockldl.py - Hessian-aware trellis encoding by block feedback (the block form of GPTQ / LDLQ, as QTIP does it).
+
+Weights W [m, n] in the rotated basis, H = E[x x^T] [n, n] of the rotated layer input. Columns are cut into 128-blocks (the
+trellis blocks). Block b is encoded on its own (m blocks of 128, plain-MSE trellis search), its error E_b = W_b - Q_b is
+pushed onto the not yet coded columns, exactly as the OBS update does for a whole block:
+
+ W[:, c1:] -= E_b R_bb^{-1} R[c0:c1, c1:] R upper, H^-1 = R^T R
+
+so the later blocks absorb what the earlier ones could not represent, in the directions that matter for ||(W - Q) X||.
+The result is a normal TQK6 / TQK7 / TQK5X / TQK4X tensor; only the encoder changed.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+import tqenc
+
+try:
+ from scipy.linalg import solve_triangular
+except Exception: # numpy only: a small back substitution
+ solve_triangular = None
+
+
+def _solve_upper(A: np.ndarray, B: np.ndarray) -> np.ndarray:
+ if solve_triangular is not None:
+ return solve_triangular(A, B, lower=False)
+ return np.linalg.solve(A, B)
+
+
+def hessian_factor(H: np.ndarray, damp: float = 0.01) -> np.ndarray:
+ """R (upper) with (H + damp * mean(diag H) I)^-1 = R^T R."""
+ H = np.asarray(H, dtype=np.float64).copy()
+ d = damp * float(np.mean(np.diag(H)))
+ H[np.diag_indices_from(H)] += d
+ Hinv = np.linalg.inv(H)
+ Hinv = (Hinv + Hinv.T) * 0.5
+ L = np.linalg.cholesky(Hinv)
+ return L.T
+
+
+def feedback_matrices(R: np.ndarray, g: int = 128):
+ """For every block b the matrix R_bb^{-1} R[b, b+1:] ([g, n - (b+1) g]); the last block has none."""
+ n = R.shape[0]
+ out = []
+ for c0 in range(0, n, g):
+ c1 = c0 + g
+ out.append(_solve_upper(R[c0:c1, c0:c1], R[c0:c1, c1:]).astype(np.float32) if c1 < n else None)
+ return out
+
+
+def encode_ldl(W: np.ndarray, t: int, fb, gpu: bool = False, iters: int = 1, ncand: int = 1, threads: int = 0):
+ """W [m, n] float32 (n % 128 == 0), fb from feedback_matrices. Returns (packed [m, n/128, block_bytes], decoded [m, n])."""
+ m, n = W.shape
+ B = n // 128
+ bb = tqenc.block_bytes(t)
+ Wc = np.ascontiguousarray(W, dtype=np.float32).copy()
+ packed = np.empty((m, B, bb), dtype=np.uint8)
+ dec = np.empty((m, n), dtype=np.float32)
+ kw = dict(iters=iters, ncand=ncand)
+ if gpu:
+ kw["gpu"] = True
+ else:
+ kw["threads"] = threads
+ for b in range(B):
+ c0, c1 = b * 128, (b + 1) * 128
+ blk = np.ascontiguousarray(Wc[:, c0:c1])
+ pk, _, _, _ = tqenc.encode(blk, t, **kw)
+ q = tqenc.decode(pk, t).reshape(m, 128)
+ packed[:, b] = pk
+ dec[:, c0:c1] = q
+ if fb[b] is not None:
+ Wc[:, c1:] -= (blk - q) @ fb[b]
+ return packed, dec
diff --git a/tools/trellis/build_swap.py b/tools/trellis/build_swap.py
index 2df3be544..e9f797ff8 100644
--- a/tools/trellis/build_swap.py
+++ b/tools/trellis/build_swap.py
@@ -8,6 +8,7 @@ Modes (all from the ORIGINAL Qwen weights, rotated into the file's basis with th
q8 Q8_0 (types 8/8): the near-lossless reference for the swapped layers
plain TQK6 gate/up + TQK7 down, plain-MSE trellis encode (tools/tqenc.py)
diag the same with per-input-column importance weights h = E[x^2]^power from the calibration dump (STRATA_ACT_DUMP)
+ ldl the same with block feedback from the full second-moment matrix of the dump (STRATA_ACT_DUMP_COV=1), blockldl.py
python3 tools/trellis/build_swap.py --gguf SHIPPED --pack PACK --orig DIR --layers 4,16,28,40 --mode plain --var DIR [--gpu]
"""
@@ -25,6 +26,7 @@ HERE = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent))
sys.path.insert(0, str(HERE))
import gguf_reader # noqa: E402
+import blockldl # noqa: E402
import tqenc # noqa: E402
N_EXP, N_EMBD, N_FF = 512, 2560, 640
@@ -62,13 +64,24 @@ def load_h(dump_dir, layer, kind, power):
return (h / h.mean()).astype(np.float32), int(count)
+def load_hessian(dump_dir, layer, kind):
+ p = pathlib.Path(dump_dir) / ("act_L%d_%s.bin" % (layer, kind))
+ raw = p.read_bytes()
+ n, count = np.frombuffer(raw[:16], dtype=np.int64)
+ n = int(n)
+ if len(raw) < 16 + 8 * n + 8 * n * n:
+ raise SystemExit("%s has no covariance (run the calibration with STRATA_ACT_DUMP_COV=1)" % p)
+ cov = np.frombuffer(raw[16 + 8 * n:16 + 8 * n + 8 * n * n], dtype=np.float64).reshape(n, n)
+ return cov / max(int(count), 1)
+
+
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--gguf", required=True)
ap.add_argument("--pack", required=True)
ap.add_argument("--orig", required=True)
ap.add_argument("--layers", required=True)
- ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag"])
+ ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag", "ldl"])
ap.add_argument("--var", required=True)
ap.add_argument("--dump-dir", default=None)
ap.add_argument("--power", type=float, default=1.0)
@@ -77,6 +90,7 @@ def main() -> int:
ap.add_argument("--iters", type=int, default=2)
ap.add_argument("--ncand", type=int, default=2)
ap.add_argument("--chunk", type=int, default=16)
+ ap.add_argument("--damp", type=float, default=0.01, help="ldl: damping, a fraction of the mean diagonal of H")
a = ap.parse_args()
layers = [int(x) for x in a.layers.split(",")]
var = pathlib.Path(a.var)
@@ -126,6 +140,10 @@ def main() -> int:
gu_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_gate_up.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, 2 * N_FF, N_EMBD)
dn_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_down.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, N_EMBD, N_FF)
h_gu = h_dn = None
+ fb_gu = fb_dn = None
+ if a.mode == "ldl":
+ fb_gu = blockldl.feedback_matrices(blockldl.hessian_factor(load_hessian(a.dump_dir, l, "gu"), a.damp))
+ fb_dn = blockldl.feedback_matrices(blockldl.hessian_factor(load_hessian(a.dump_dir, l, "dn"), a.damp))
if a.mode == "diag":
h_gu, c1 = load_h(a.dump_dir, l, "gu", a.power)
h_dn, c2 = load_h(a.dump_dir, l, "dn", a.power)
@@ -139,12 +157,27 @@ def main() -> int:
mats = {"gate": (gu[:, :N_FF], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),
"up": (gu[:, N_FF:], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),
"down": (dn, signs[N_FF], h_dn, row_d, N_EMBD, t_down, d_t)}
+ ldl_done = {}
+ if a.mode == "ldl":
+ # gate and up share H: one batch of rows
+ both = np.concatenate([tqenc.rotate(np.ascontiguousarray(gu[:, :N_FF]), signs[N_EMBD]).reshape(-1, N_EMBD),
+ tqenc.rotate(np.ascontiguousarray(gu[:, N_FF:]), signs[N_EMBD]).reshape(-1, N_EMBD)])
+ pk, dec = blockldl.encode_ldl(both, gu_t, fb_gu, gpu=a.gpu, iters=a.iters, ncand=a.ncand, threads=a.threads)
+ half = pk.shape[0] // 2
+ ldl_done["gate"] = (pk[:half].reshape(half, -1), dec[:half])
+ ldl_done["up"] = (pk[half:].reshape(half, -1), dec[half:])
+ wrd = tqenc.rotate(np.ascontiguousarray(dn), signs[N_FF]).reshape(-1, N_FF)
+ pk, dec = blockldl.encode_ldl(wrd, d_t, fb_dn, gpu=a.gpu, iters=a.iters, ncand=a.ncand, threads=a.threads)
+ ldl_done["down"] = (pk.reshape(pk.shape[0], -1), dec)
for role, (w, sg, h, row, n_out, tb, ty) in mats.items():
wr = tqenc.rotate(np.ascontiguousarray(w), sg) # [chunk, n_out, n_in]
n_in = wr.shape[-1]
if a.mode == "q8":
data = q8_0_quantize(wr)
dec = None
+ elif a.mode == "ldl":
+ data, dec = ldl_done[role]
+ err_sum[role] += float(np.sum((wr.reshape(-1, n_in).astype(np.float64) - dec) ** 2))
else:
packed, tot = enc(wr.reshape(-1, n_in), ty, h if a.mode == "diag" else None)
data = packed
diff --git a/tools/trellis/test_blockldl.py b/tools/trellis/test_blockldl.py
new file mode 100644
index 000000000..85dabee98
--- /dev/null
+++ b/tools/trellis/test_blockldl.py
@@ -0,0 +1,44 @@
+"""test_blockldl.py - block feedback against plain encoding on a synthetic anisotropic H (an incoherent random basis with a
+spread spectrum) and on real expert weights: the proxy loss tr(E H E^T) must drop, the plain loss rises a little.
+ python3 tools/trellis/test_blockldl.py [--orig DIR --layer 4]
+"""
+import argparse
+import pathlib
+import sys
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import blockldl # noqa: E402
+import tqenc # noqa: E402
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--n", type=int, default=1024)
+ap.add_argument("--rows", type=int, default=512)
+ap.add_argument("--spread", type=float, default=100.0, help="ratio of the largest to the smallest eigenvalue")
+ap.add_argument("--type", default="TQK6")
+a = ap.parse_args()
+t = tqenc.TYPE_BY_NAME[a.type]
+rng = np.random.default_rng(1)
+n = a.n
+Q, _ = np.linalg.qr(rng.standard_normal((n, n)))
+lam = np.logspace(0, -np.log10(a.spread), n)
+H = (Q * lam) @ Q.T
+W = (rng.standard_normal((a.rows, n)) * 0.02).astype(np.float32)
+R = blockldl.hessian_factor(H, 0.01)
+fb = blockldl.feedback_matrices(R)
+pk, _, _, _ = tqenc.encode(W, t, iters=1, threads=8)
+plain = tqenc.decode(pk, t).reshape(W.shape)
+_, ldl = blockldl.encode_ldl(W, t, fb, threads=8)
+
+
+def proxy(Wq):
+ E = (W - Wq).astype(np.float64)
+ return np.trace(E @ H @ E.T) / np.trace(W.astype(np.float64) @ H @ W.T.astype(np.float64))
+
+
+for name, Wq in (("plain", plain), ("block LDL", ldl)):
+ E = W - Wq
+ print("%-10s unweighted %.2f dB H-weighted %.2f dB" % (name, 10 * np.log10(np.sum(W.astype(np.float64) ** 2) / np.sum(E.astype(np.float64) ** 2)), -10 * np.log10(proxy(Wq))))