Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
10.3 KB
Raw Blame
"""build_swap.py - Gyro-S with a few layers' experts swapped, without copying the 58 GB file.

Writes <var>/swap.bin (the new gate / up / down tensors of the chosen layers) and a pack <var>/pack whose native_experts.txt
points those layers at swap.bin (the shard column), everything else being the shipped Gyro-S pack and file: <var>/ holds a
symlink to the shipped GGUF, so `--native <var>/<gguf name>` finds swap.bin beside it.

Modes (all from the ORIGINAL Qwen weights, rotated into the file's basis with the file's own sign vectors):
  q8     Q8_0 (types 8/8): the near-lossless reference for the swapped layers
  plain  TQK6 gate/up + TQK7 down, plain-MSE trellis encode (tools/tqenc.py)
  diag   the same with per-input-column importance weights h = E[x^2]^power from the calibration dump (STRATA_ACT_DUMP)
  ldl    the same with block feedback from the full second-moment matrix of the dump (STRATA_ACT_DUMP_COV=1), blockldl.py

  python3 tools/trellis/build_swap.py --gguf SHIPPED --pack PACK --orig DIR --layers 4,16,28,40 --mode plain --var DIR [--gpu]
"""
from __future__ import annotations

import argparse
import os
import pathlib
import sys
import time

import numpy as np

HERE = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent))
sys.path.insert(0, str(HERE))
import gguf_reader  # noqa: E402
import blockldl  # noqa: E402
import tqenc  # noqa: E402

N_EXP, N_EMBD, N_FF = 512, 2560, 640
ALIGN = 4096


def q8_0_quantize(w: np.ndarray) -> np.ndarray:
    """float32 [..., n] (n % 32 == 0) -> uint8 [..., n/32 * 34] block_q8_0 (fp16 scale, 32 int8)."""
    shp = w.shape
    b = w.reshape(-1, 32).astype(np.float32)
    amax = np.abs(b).max(axis=1)
    d = amax / 127.0
    d16 = d.astype(np.float16)
    dq = d16.astype(np.float32)
    with np.errstate(divide="ignore", invalid="ignore"):
        q = np.where(dq[:, None] > 0, b / dq[:, None], 0.0)
    q = np.clip(np.sign(q) * np.floor(np.abs(q) + 0.5), -127, 127).astype(np.int8)
    out = np.empty((b.shape[0], 34), dtype=np.uint8)
    out[:, :2] = d16.view(np.uint8).reshape(-1, 2)
    out[:, 2:] = q.view(np.uint8)
    return out.reshape(*shp[:-1], shp[-1] // 32 * 34)


def bf16(raw: np.ndarray) -> np.ndarray:
    return (raw.astype(np.uint32) << 16).view(np.float32)


def load_h(dump_dir, layer, kind, power):
    p = pathlib.Path(dump_dir) / ("act_L%d_%s.bin" % (layer, kind))
    raw = p.read_bytes()
    n, count = np.frombuffer(raw[:16], dtype=np.int64)
    sq = np.frombuffer(raw[16:16 + 8 * n], dtype=np.float64)
    h = (sq / max(count, 1)) ** power
    h = np.maximum(h, h.mean() * 1e-4)    # no weight below 1e-4 of the mean
    return (h / h.mean()).astype(np.float32), int(count)


def load_hessian(dump_dir, layer, kind):
    p = pathlib.Path(dump_dir) / ("act_L%d_%s.bin" % (layer, kind))
    raw = p.read_bytes()
    n, count = np.frombuffer(raw[:16], dtype=np.int64)
    n = int(n)
    if len(raw) < 16 + 8 * n + 8 * n * n:
        raise SystemExit("%s has no covariance (run the calibration with STRATA_ACT_DUMP_COV=1)" % p)
    cov = np.frombuffer(raw[16 + 8 * n:16 + 8 * n + 8 * n * n], dtype=np.float64).reshape(n, n)
    return cov / max(int(count), 1)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--gguf", required=True)
    ap.add_argument("--pack", required=True)
    ap.add_argument("--orig", required=True)
    ap.add_argument("--layers", required=True)
    ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag", "ldl"])
    ap.add_argument("--var", required=True)
    ap.add_argument("--dump-dir", default=None)
    ap.add_argument("--power", type=float, default=1.0)
    ap.add_argument("--gpu", action="store_true")
    ap.add_argument("--threads", type=int, default=16)
    ap.add_argument("--iters", type=int, default=2)
    ap.add_argument("--ncand", type=int, default=2)
    ap.add_argument("--chunk", type=int, default=16)
    ap.add_argument("--damp", type=float, default=0.3, help="ldl: damping, a fraction of the mean diagonal of H")
    a = ap.parse_args()
    layers = [int(x) for x in a.layers.split(",")]
    var = pathlib.Path(a.var)
    var.mkdir(parents=True, exist_ok=True)
    gg = pathlib.Path(a.gguf)
    link = var / gg.name
    if not link.exists():
        link.symlink_to(gg)
    g = gguf_reader.GGUFFile(a.gguf)
    sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]
    sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)
    signs, at = {}, 0
    for w in sw:
        signs[w] = sv[at:at + w]
        at += w
    if a.mode == "q8":
        gu_t = d_t = 8
        row_gu, row_d = N_EMBD // 32 * 34, N_FF // 32 * 34
    else:
        gu_t, d_t = tqenc.TQK6, tqenc.TQK7
        row_gu, row_d = N_EMBD // 128 * tqenc.block_bytes(gu_t), N_FF // 128 * tqenc.block_bytes(d_t)
    t_gate = N_EXP * N_FF * row_gu          # bytes of the gate (and up) tensor of a layer
    t_down = N_EXP * N_EMBD * row_d
    swap = open(var / "swap.bin", "wb")
    offs = {}
    pos = 0
    for l in layers:
        offs[l] = {}
        for role, nbytes in (("gate", t_gate), ("up", t_gate), ("down", t_down)):
            offs[l][role] = pos
            pos += (nbytes + ALIGN - 1) // ALIGN * ALIGN
    swap.truncate(pos)

    def enc(w2d, t, h):
        kw = dict(iters=a.iters, ncand=a.ncand, gpu=a.gpu)
        if not a.gpu:
            kw["threads"] = a.threads
        hh = None
        if h is not None:
            hh = np.broadcast_to(h, w2d.shape).astype(np.float32)
        packed, err, tot, st = tqenc.encode(w2d, t, h=hh, **kw)
        return packed, tot

    stats = {}
    t_start = time.time()
    for l in layers:
        gu_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_gate_up.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, 2 * N_FF, N_EMBD)
        dn_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_down.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, N_EMBD, N_FF)
        h_gu = h_dn = None
        fb_gu = fb_dn = None
        if a.mode == "ldl":
            fb_gu = blockldl.feedback_matrices(blockldl.hessian_factor(load_hessian(a.dump_dir, l, "gu"), a.damp))
            fb_dn = blockldl.feedback_matrices(blockldl.hessian_factor(load_hessian(a.dump_dir, l, "dn"), a.damp))
        if a.mode == "diag":
            h_gu, c1 = load_h(a.dump_dir, l, "gu", a.power)
            h_dn, c2 = load_h(a.dump_dir, l, "dn", a.power)
            print("layer %d: calibration counts gu %d dn %d; h cv gu %.2f dn %.2f" % (l, c1, c2, h_gu.std() / h_gu.mean(), h_dn.std() / h_dn.mean()))
        err_sum = {"gate": 0.0, "up": 0.0, "down": 0.0}
        en_sum = {"gate": 0.0, "up": 0.0, "down": 0.0}
        for e0 in range(0, N_EXP, a.chunk):
            e1 = min(N_EXP, e0 + a.chunk)
            gu = bf16(np.asarray(gu_raw[e0:e1]))
            dn = bf16(np.asarray(dn_raw[e0:e1]))
            mats = {"gate": (gu[:, :N_FF], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),
                    "up": (gu[:, N_FF:], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),
                    "down": (dn, signs[N_FF], h_dn, row_d, N_EMBD, t_down, d_t)}
            ldl_done = {}
            if a.mode == "ldl":
                # gate and up share H: one batch of rows
                both = np.concatenate([tqenc.rotate(np.ascontiguousarray(gu[:, :N_FF]), signs[N_EMBD]).reshape(-1, N_EMBD),
                                       tqenc.rotate(np.ascontiguousarray(gu[:, N_FF:]), signs[N_EMBD]).reshape(-1, N_EMBD)])
                pk, dec = blockldl.encode_ldl(both, gu_t, fb_gu, gpu=a.gpu, iters=a.iters, ncand=a.ncand, threads=a.threads)
                half = pk.shape[0] // 2
                ldl_done["gate"] = (pk[:half].reshape(half, -1), dec[:half])
                ldl_done["up"] = (pk[half:].reshape(half, -1), dec[half:])
                wrd = tqenc.rotate(np.ascontiguousarray(dn), signs[N_FF]).reshape(-1, N_FF)
                pk, dec = blockldl.encode_ldl(wrd, d_t, fb_dn, gpu=a.gpu, iters=a.iters, ncand=a.ncand, threads=a.threads)
                ldl_done["down"] = (pk.reshape(pk.shape[0], -1), dec)
            for role, (w, sg, h, row, n_out, tb, ty) in mats.items():
                wr = tqenc.rotate(np.ascontiguousarray(w), sg)           # [chunk, n_out, n_in]
                n_in = wr.shape[-1]
                if a.mode == "q8":
                    data = q8_0_quantize(wr)
                    dec = None
                elif a.mode == "ldl":
                    data, dec = ldl_done[role]
                    err_sum[role] += float(np.sum((wr.reshape(-1, n_in).astype(np.float64) - dec) ** 2))
                else:
                    packed, tot = enc(wr.reshape(-1, n_in), ty, h if a.mode == "diag" else None)
                    data = packed
                    err_sum[role] += tot
                en_sum[role] += float(np.sum(wr.astype(np.float64) ** 2))
                swap.seek(offs[l][role] + e0 * n_out * row)
                swap.write(np.ascontiguousarray(data).tobytes())
            if e0 % 128 == 0:
                print("layer %d experts %d-%d done  (%.0f s)" % (l, e0, e1, time.time() - t_start), flush=True)
        stats[l] = {r: (10 * np.log10(en_sum[r] / err_sum[r]) if err_sum[r] > 0 else None) for r in err_sum}
        print("layer %d weight SQNR (dB, unweighted): %s" % (l, {k: (round(v, 3) if v else v) for k, v in stats[l].items()}), flush=True)
    swap.close()
    # the pack: symlinks to the shipped pack's files, a new native_experts.txt
    pk = var / "pack"
    pk.mkdir(exist_ok=True)
    src = pathlib.Path(a.pack)
    for f in src.iterdir():
        if f.name == "native_experts.txt":
            continue
        d = pk / f.name
        if not d.exists():
            d.symlink_to(f.resolve())
    lines = (src / "native_experts.txt").read_text().splitlines()
    head, body = lines[0], lines[1:]
    out_lines, arena = [], 0
    for ln in body:
        f = ln.split()
        l = int(f[0])
        if l in offs:
            blob = row_gu * N_FF * 2 + row_d * N_EMBD
            f = [str(l), str(gu_t), str(d_t), str(arena), str(blob), str(offs[l]["gate"]), str(offs[l]["up"]), str(offs[l]["down"]), "swap.bin"]
        else:
            f[3] = str(arena)
        out_lines.append(" ".join(f))
        arena += N_EXP * int(f[4])
    import re
    head = re.sub(r"total \d+", "total %d" % arena, head)
    (pk / "native_experts.txt").write_text(head + "\n" + "\n".join(out_lines) + "\n")
    print("wrote", var, "swap.bin %.2f GB, pack %s" % (pos / 1e9, pk))
    return 0


if __name__ == "__main__":
    sys.exit(main())