Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: computed codebook on the GPU encoder, README for the K5/K4 versions
Niko1221271312172516
diff --git a/tools/tqenc.py b/tools/tqenc.pyindex 28b3ae2fa..c76a42d67 100644--- a/tools/tqenc.py+++ b/tools/tqenc.py@@ -152,8 +152,8 @@ def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, nraise ValueError("importance weights must have the shape of w")hp = h.ctypes.dataif gpu:- if exact or t == TQ2_T or t in COMPUTED:- raise ValueError("the GPU encoder has no exact search, no TQ2_T and no computed-codebook types (TQK5X / TQK4X)")+ if exact or t == TQ2_T:+ raise ValueError("the GPU encoder has no exact search and no TQ2_T")with _gpu_lock() as gl:if gl.ok:tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,diff --git a/tools/trellis/README.md b/tools/trellis/README.mdindex a76354aa7..cd117d16e 100644--- a/tools/trellis/README.md+++ b/tools/trellis/README.md@@ -1,7 +1,11 @@# Trellis encoder (research, branch f-dsv41-enc)An encoder for the trellis expert formats that Strata already decodes (TQ2_T, TQK6, TQK7 of agentionai's llama.cpp fork)-and for two experimental lower rates, TQK5X (22 bytes per 128 weights, 1.375 bpw) and TQK4X (18 bytes, 1.125 bpw).+and for two experimental lower rates, K5 (22 bytes per 128 weights, 1.375 bpw) and K4 (18 bytes, 1.125 bpw), in two versions:+TQK5X / TQK4X (ids 224 / 225, experimental version 2) use a computed codebook, two LCG words per state masked into four fp16+(`src/kernels/cpu/tq_computed.hpp`, from the z440 lab's "3single"), and TQK5T / TQK4T (226 / 227, version 1) the fork's+hashed point table. On real DeepSeek expert tensors the computed code costs 0.02 to 0.04 dB (K5 7.07 against 7.10 dB, K4 5.61+against 5.63) and needs no table lookups in a GEMV. The fork's types keep their table.Written for the DeepSeek-V4.1-Flash compression plan (gate 2 and the start of gate 3). Nothing here changes an existingtype, layout or default; the engine's CPU decoder only gains K = 5 and 4 (ids 224 / 225, experimental version 1).diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cppindex 0bf465f02..a7c1d39b6 100644--- a/tools/trellis/tqenc.cpp+++ b/tools/trellis/tqenc.cpp@@ -83,6 +83,8 @@ int type_kbits(int type) {bool type_computed(int type) { return type == TQK5X || type == TQK4X; }+double codebook_rms(int type);+namespace {// ---------------------------------------------------------------------------------------------------------------@@ -390,6 +392,7 @@ const float* codebook_state(int type, uint32_t state) {}const int16_t* codebook_table(int type) { return cb(type_computed(type)).tab; }+double codebook_rms(int type) { return cb(type_computed(type)).rms; }const float* point_table() {static float pts[4096];diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hppindex 20c437e7b..86b2c77d2 100644--- a/tools/trellis/tqenc.hpp+++ b/tools/trellis/tqenc.hpp@@ -45,6 +45,8 @@ enum Type : int {};/// Whether the type's codebook is the computed one (TQK5X, TQK4X).bool type_computed(int type);+/// rms of the codebook values (the first scale guess is rms(block) / this): 1.072 for the table, 0.88 for the computed code.+double codebook_rms(int type);/// bits per step (8, 6, 7, 5, 4), 0 for any other type.int type_kbits(int type);diff --git a/tools/trellis/tqenc_hip.cpp b/tools/trellis/tqenc_hip.cppindex 5ec20af04..3c619dec5 100644--- a/tools/trellis/tqenc_hip.cpp+++ b/tools/trellis/tqenc_hip.cpp@@ -44,6 +44,23 @@ __device__ __forceinline__ uint16_t f2h(float f) { return __half_as_ushort(__flo// weight index of component j at processing position p (TQK)__device__ __forceinline__ int widx_d(int p, int j) { return 4 * (31 - p) + j; }+// the computed codebook of TQK5X / TQK4X (src/kernels/cpu/tq_computed.hpp): four fp16 from two LCG words, no table+__device__ __forceinline__ void comp_vals(uint32_t s, float* w) {+ const uint32_t r0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ const uint32_t r1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ w[0] = h2f((uint16_t) (r0 & 0xffffu));+ w[1] = h2f((uint16_t) (r0 >> 16));+ w[2] = h2f((uint16_t) (r1 & 0xffffu));+ w[3] = h2f((uint16_t) (r1 >> 16));+}++__device__ __forceinline__ float comp_cost(uint32_t s, const float* t, const float* h) {+ float w[4];+ comp_vals(s, w);+ const float e0 = t[0] - w[0], e1 = t[1] - w[1], e2 = t[2] - w[2], e3 = t[3] - w[3];+ return h[0] * e0 * e0 + h[1] * e1 * e1 + h[2] * e2 * e2 + h[3] * e3 * e3;+}+struct Pass {const float* w;const float* h; // null: ones@@ -59,7 +76,7 @@ struct Pass {int* c_out; // free: junction group [nb]};-template <int K, int SL>+template <int K, int SL, bool COMP>__global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {constexpr int SPT = 1 << SL; // states per threadconstexpr int NT = 32768 >> SL; // threads per workgroup@@ -101,6 +118,7 @@ __global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {float D[SPT];// tables of step q into parity q & 1auto build = [&](int q) {+ if constexpr (COMP) return;const int p = (P.rot + q) & 31;const float t0 = Tt[p][0], t1 = Tt[p][1], t2 = Tt[p][2], t3 = Tt[p][3];const float h0 = Hh[p][0], h1 = Hh[p][1], h2 = Hh[p][2], h3 = Hh[p][3];@@ -113,12 +131,21 @@ __global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {};build(0);__syncthreads();+ {+ float tc[4], hc[4];+ if constexpr (COMP) {+ const int p0 = P.rot & 31;+ for (int i = 0; i < 4; ++i) { tc[i] = Tt[p0][i]; hc[i] = Hh[p0][i]; }+ }#pragma unroll- for (int j = 0; j < SPT; ++j) {- const uint32_t x = x0 + (uint32_t) j * kC;- float v = tab[0][0][x >> 21] + tab[0][1][(x >> 10) & 2047u];- if (fc >= 0 && (((SPT * tid + j) & (G - 1)) != fc)) v = kInf;- D[j] = v;+ for (int j = 0; j < SPT; ++j) {+ const uint32_t x = x0 + (uint32_t) j * kC;+ float v;+ if constexpr (COMP) v = comp_cost((uint32_t) (SPT * tid + j), tc, hc);+ else v = tab[0][0][x >> 21] + tab[0][1][(x >> 10) & 2047u];+ if (fc >= 0 && (((SPT * tid + j) & (G - 1)) != fc)) v = kInf;+ D[j] = v;+ }}for (int q = 1; q < 32; ++q) {const int par = q & 1;@@ -167,15 +194,28 @@ __global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {// and 128 live registers spill): an empty asm makes x0 "new" every stepuint32_t x0s = x0;asm volatile("" : "+v"(x0s));+ float tc[4], hc[4];+ if constexpr (COMP) {+ const int pq = (P.rot + q) & 31;+ for (int i = 0; i < 4; ++i) { tc[i] = Tt[pq][i]; hc[i] = Hh[pq][i]; }+ }#pragma unrollfor (int j4 = 0; j4 < SPT / 4; ++j4) {const float4 mm = m4[j4];const uint32_t xa = x0s + (uint32_t) (4 * j4) * kC;const uint32_t xb = xa + kC, xc = xb + kC, xd = xc + kC;- D[4 * j4 + 0] = mm.x + tab[par][0][xa >> 21] + tab[par][1][(xa >> 10) & 2047u];- D[4 * j4 + 1] = mm.y + tab[par][0][xb >> 21] + tab[par][1][(xb >> 10) & 2047u];- D[4 * j4 + 2] = mm.z + tab[par][0][xc >> 21] + tab[par][1][(xc >> 10) & 2047u];- D[4 * j4 + 3] = mm.w + tab[par][0][xd >> 21] + tab[par][1][(xd >> 10) & 2047u];+ if constexpr (COMP) {+ const uint32_t sa = (uint32_t) (SPT * tid + 4 * j4);+ D[4 * j4 + 0] = mm.x + comp_cost(sa, tc, hc);+ D[4 * j4 + 1] = mm.y + comp_cost(sa + 1, tc, hc);+ D[4 * j4 + 2] = mm.z + comp_cost(sa + 2, tc, hc);+ D[4 * j4 + 3] = mm.w + comp_cost(sa + 3, tc, hc);+ } else {+ D[4 * j4 + 0] = mm.x + tab[par][0][xa >> 21] + tab[par][1][(xa >> 10) & 2047u];+ D[4 * j4 + 1] = mm.y + tab[par][0][xb >> 21] + tab[par][1][(xb >> 10) & 2047u];+ D[4 * j4 + 2] = mm.z + tab[par][0][xc >> 21] + tab[par][1][(xc >> 10) & 2047u];+ D[4 * j4 + 3] = mm.w + tab[par][0][xd >> 21] + tab[par][1][(xd >> 10) & 2047u];+ }if ((j4 & 3) == 3) __builtin_amdgcn_sched_barrier(0); // 16 states at a time: do not batch all the gathers}}@@ -225,14 +265,15 @@ __global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {// ------------------------------------------------------------------------------------------------ per-block stages-__device__ __forceinline__ void state_cb(const float2* lutq, uint32_t s, float* c) {+__device__ __forceinline__ void state_cb(const float2* lutq, int comp, uint32_t s, float* c) {+ if (comp) { comp_vals(s, c); return; }const uint32_t x = s * 0x9e3779b1u;const float2 a = lutq[x >> 21], b = lutq[(x >> 10) & 2047u];c[0] = a.x; c[1] = a.y; c[2] = b.x; c[3] = b.y;}__global__ void init_kernel(const float* w, const float* h, int64_t nb, uint16_t* dcur, uint8_t* active, uint8_t* status,- double* sumhw2, double mult, int first, uint16_t fixed_scale) {+ double* sumhw2, double mult, int first, uint16_t fixed_scale, double cb_rms) {const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;if (b >= nb) return;double s2 = 0, sh2 = 0, amax = 0;@@ -251,7 +292,7 @@ __global__ void init_kernel(const float* w, const float* h, int64_t nb, uint16_t}if (status[b] != 0) { active[b] = 0; return; }const double rms = sqrt(s2 / 128.0);- double dd = rms / 1.072 * mult;+ double dd = rms / cb_rms * mult;uint16_t dh = f2h((float) fmin(dd, 65504.0));if (dh == 0) dh = 1;if (fixed_scale) dh = fixed_scale;@@ -260,12 +301,12 @@ __global__ void init_kernel(const float* w, const float* h, int64_t nb, uint16_t}// error of (states by position st, fp16 scale dh)-__device__ double fit_d(const float* w, const float* h, int64_t b, const uint16_t* st, uint16_t dh, const float2* lutq) {+__device__ double fit_d(const float* w, const float* h, int64_t b, const uint16_t* st, uint16_t dh, const float2* lutq, int comp) {const float d = h2f(dh);double e = 0;for (int p = 0; p < 32; ++p) {float c[4];- state_cb(lutq, st[p], c);+ state_cb(lutq, comp, st[p], c);for (int j = 0; j < 4; ++j) {const int wi = widx_d(p, j);const float rec = d * c[j];@@ -279,7 +320,7 @@ __device__ double fit_d(const float* w, const float* h, int64_t b, const uint16_// pick the best candidate, then the scale refinement step of encode_block__global__ void refine_kernel(const float* w, const float* h, int64_t nb, int nc, const uint16_t* st_c, const float* cost_c,uint16_t* dcur, uint8_t* active, uint16_t* best_d, double* best_err, uint16_t* best_st,- const float2* lutq, int fixed_scale) {+ const float2* lutq, int fixed_scale, int comp) {const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;if (b >= nb || !active[b]) return;int bc = -1;@@ -292,7 +333,7 @@ __global__ void refine_kernel(const float* w, const float* h, int64_t nb, int ncuint16_t st[32];for (int p = 0; p < 32; ++p) st[p] = st_c[((int64_t) bc * nb + b) * 32 + p];const uint16_t dh = dcur[b];- const double e = fit_d(w, h, b, st, dh, lutq);+ const double e = fit_d(w, h, b, st, dh, lutq, comp);if (e < best_err[b]) {best_err[b] = e;best_d[b] = dh;@@ -302,7 +343,7 @@ __global__ void refine_kernel(const float* w, const float* h, int64_t nb, int ncdouble num = 0, den = 0;for (int p = 0; p < 32; ++p) {float c[4];- state_cb(lutq, st[p], c);+ state_cb(lutq, comp, st[p], c);for (int j = 0; j < 4; ++j) {const int wi = widx_d(p, j);const double hi = h ? (double) h[b * 128 + wi] : 1.0, cc = c[j];@@ -313,7 +354,7 @@ __global__ void refine_kernel(const float* w, const float* h, int64_t nb, int ncif (den <= 0 || num <= 0) { active[b] = 0; return; }const uint16_t nh = f2h((float) fmin(num / den, 65504.0));if (nh == 0 || nh == dh) { active[b] = 0; return; }- const double en = fit_d(w, h, b, st, nh, lutq);+ const double en = fit_d(w, h, b, st, nh, lutq, comp);if (en < best_err[b]) {best_err[b] = en;best_d[b] = nh;@@ -359,17 +400,22 @@ __global__ void init_best_kernel(int64_t nb, double* best_err, uint8_t* have) {best_err[b] = 1e300;}-template <int K>+template <int K, bool COMP>void launch_pass(const Pass& P, int grid) {- hipLaunchKernelGGL((pass_kernel<K, kSL>), dim3(grid), dim3(NT), 0, 0, P);+ hipLaunchKernelGGL((pass_kernel<K, kSL, COMP>), dim3(grid), dim3(NT), 0, 0, P);}-void launch_pass_k(int K, const Pass& P, int grid) {+void launch_pass_k(int K, bool comp, const Pass& P, int grid) {+ if (comp) { // TQK5X / TQK4X+ if (K == 4) launch_pass<4, true>(P, grid);+ else launch_pass<5, true>(P, grid);+ return;+ }switch (K) {- case 4: launch_pass<4>(P, grid); break;- case 5: launch_pass<5>(P, grid); break;- case 6: launch_pass<6>(P, grid); break;- default: launch_pass<7>(P, grid); break;+ case 4: launch_pass<4, false>(P, grid); break;+ case 5: launch_pass<5, false>(P, grid); break;+ case 6: launch_pass<6, false>(P, grid); break;+ default: launch_pass<7, false>(P, grid); break;}}@@ -419,7 +465,9 @@ TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t nint scale_tries, int device, double* err_out, int64_t* stats, double* timing,int fixed_scale_bits) {const int K = tqenc::type_kbits(type);- if (K < 4 || K > 7 || tqenc::type_computed(type)) return -1.0; // the computed codebook of TQK5X / TQK4X has no 2048-point table: CPU only+ if (K < 4 || K > 7) return -1.0;+ const bool comp = tqenc::type_computed(type);+ const double cb_rms = tqenc::codebook_rms(type);ensure_dev(device);const int bb = 2 + 4 * K, G = 1 << (15 - K);const int nc = std::max(1, std::min(ncand, 8));@@ -457,24 +505,24 @@ TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t nconst auto t0 = std::chrono::steady_clock::now();for (int tr = 0; tr < tries; ++tr) {hipLaunchKernelGGL(init_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, dcur, active, status, sumhw2, mult[tr],- tr == 0 ? 1 : 0, (uint16_t) fixed_scale_bits);+ tr == 0 ? 1 : 0, (uint16_t) fixed_scale_bits, cb_rms);for (int it = 0; it <= (fixed_scale_bits ? 0 : iters); ++it) {Pass P{};P.w = dw; P.h = dh; P.dcur = dcur; P.active = active; P.nb = nb; P.lutq = g_dev.lutq; P.bp_scratch = bp;const int grid = (int) std::min<int64_t>(nb, grid_cap);for (int r = 0; r < nc; ++r) {P.rot = rots[r]; P.cin = nullptr; P.c_out = c_all + (int64_t) r * nb;- launch_pass_k(K, P, grid);+ launch_pass_k(K, comp, P, grid);}hipLaunchKernelGGL(dedupe_kernel, dim3(gb), dim3(tb), 0, 0, nb, nc, c_all);HCHECK(hipMemset(cost_c, 0x7f, (size_t) nc * nb * sizeof(float))); // skipped candidates cost "infinity"for (int r = 0; r < nc; ++r) {P.rot = 0; P.cin = c_all + (int64_t) r * nb; P.c_out = nullptr;P.st_out = st_c + (int64_t) r * nb * 32; P.cost_out = cost_c + (int64_t) r * nb;- launch_pass_k(K, P, grid);+ launch_pass_k(K, comp, P, grid);}hipLaunchKernelGGL(refine_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, nc, st_c, cost_c, dcur, active,- best_d, best_err, best_st, g_dev.lutq, fixed_scale_bits);+ best_d, best_err, best_st, g_dev.lutq, fixed_scale_bits, comp ? 1 : 0);npass += 2 * nc;if (std::getenv("TQH_DEBUG")) {HCHECK(hipDeviceSynchronize());diff --git a/tools/trellis/tqenc_hip_test.cpp b/tools/trellis/tqenc_hip_test.cppindex b809ff85c..6bf5b5f64 100644--- a/tools/trellis/tqenc_hip_test.cpp+++ b/tools/trellis/tqenc_hip_test.cpp@@ -18,13 +18,13 @@ using namespace tqenc;static int g_fail = 0;#define CHECK(c, ...) do { if (!(c)) { ++g_fail; std::printf(" FAIL line %d: %s ", __LINE__, #c); std::printf(__VA_ARGS__); std::printf("\n"); } } while (0)static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }-static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5T ? "TQK5T" : "TQK4T"; }+static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5T ? "TQK5T" : t == TQK4T ? "TQK4T" : t == TQK5X ? "TQK5X" : "TQK4X"; }int main(int argc, char** argv) {const long big = argc > 1 ? std::atol(argv[1]) : 131072; // blocks for the speed runsstd::mt19937_64 rng(77);std::normal_distribution<float> nd(0.f, 1.f);- const int types[4] = {TQK4T, TQK5T, TQK6, TQK7};+ const int types[6] = {TQK4T, TQK5T, TQK4X, TQK5X, TQK6, TQK7};for (int type : types) {const int bb = block_bytes(type);std::printf("== %s\n", tname(type));