Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cudalab: computed (LUT-free) codebooks for the 15-bit trellis (1mad, 3inst, 3sum, 3single), encoder hook TQE_CODE, LUT-free GEMV kernel, code SQNR tool, pilot scripts from f-dsv41-enc

Niko12216f703a6dbe72
diff --git a/tools/dsv41_lab/code_sqnr.cpp b/tools/dsv41_lab/code_sqnr.cpp
new file mode 100644
index 000000000..9193fc2e2
--- /dev/null
+++ b/tools/dsv41_lab/code_sqnr.cpp
@@ -0,0 +1,34 @@
+// tools/dsv41_lab/code_sqnr.cpp - weight SQNR of the trellis encoder on N(0,1) blocks for a codebook chosen by TQE_CODE
+// (unset = the shipped table). TQE_CODE=3sum ./code_sqnr [type 224] [blocks 16384] [thorough 0|1]
+// Rate-distortion limit of a Gaussian at K bits per 4 weights: 6.02 K / 4 dB.
+#include "../trellis/tqenc.hpp"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+#include <vector>
+
+int main(int argc, char** argv) {
+ const int type = argc > 1 ? std::atoi(argv[1]) : 224;
+ const int64_t nb = argc > 2 ? std::atoll(argv[2]) : 16384;
+ const bool thorough = argc > 3 && std::atoi(argv[3]);
+ const int K = tqenc::type_kbits(type);
+ std::mt19937_64 rng(12345);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::vector<float> w((size_t) nb * 128);
+ for (auto& v : w) v = nd(rng);
+ std::vector<uint8_t> enc((size_t) nb * tqenc::block_bytes(type));
+ tqenc::Params p;
+ if (thorough) { p.iters = 3; p.ncand = 3; p.scale_tries = 3; }
+ int64_t nonfinite = 0, zero = 0;
+ tqenc::encode_blocks(type, w.data(), nullptr, nb, enc.data(), p, 0, nullptr, &nonfinite, &zero);
+ std::vector<float> dec((size_t) nb * 128);
+ tqenc::decode_blocks(type, enc.data(), nb, dec.data());
+ double sq = 0, ee = 0;
+ for (size_t i = 0; i < w.size(); ++i) { sq += (double) w[i] * w[i]; ee += (double) (dec[i] - w[i]) * (dec[i] - w[i]); }
+ const double sqnr = 10 * std::log10(sq / ee), lim = 6.0206 * K / 4.0;
+ std::printf("TQE_CODE=%-8s type %d K=%d %s: SQNR %.3f dB, limit %.3f dB, gap %.3f dB (rel MSE %.4f)\n", std::getenv("TQE_CODE") ? std::getenv("TQE_CODE") : "table", type, K,
+ thorough ? "thorough" : "fast", sqnr, lim, lim - sqnr, ee / sq);
+ return 0;
+}
diff --git a/tools/dsv41_lab/computed_codes.hpp b/tools/dsv41_lab/computed_codes.hpp
new file mode 100644
index 000000000..2100ef401
--- /dev/null
+++ b/tools/dsv41_lab/computed_codes.hpp
@@ -0,0 +1,84 @@
+// tools/dsv41_lab/computed_codes.hpp - LUT-free ("computed") codebooks for the 15-bit-state, 4-weights-a-step trellis, after
+// QTIP (Tseng et al., Cornell-RelaxML/qtip lib/codebook/bitshift.py): 1MAD and 3INST decode a state with a 32-bit LCG and a few
+// bit tricks instead of a table. QTIP emits one weight per LCG word (V = 1); a step of our format has four weights, so every
+// variant here uses 2 to 4 LCG words per state. Host-side definition (the encoder takes it through the TQE_CODE environment
+// variable, tools/trellis/tqenc.cpp) and the CUDA kernels use the same constants (tqk_gemv_c.cuh).
+#pragma once
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+
+namespace tqcode {
+
+// LCG words of a state: x_i = s * kA[i] + kB[i]
+inline constexpr uint32_t kA[4] = {89226354u, 0x9E3779B1u, 0x85EBCA6Bu, 0xC2B2AE35u};
+inline constexpr uint32_t kB[4] = {64248484u, 0x7F4A7C15u, 0x165667B1u, 0x27D4EB2Fu};
+inline constexpr uint32_t kMask3 = 0x8FFF8FFFu, kXor3 = 0x3B603B60u; // QTIP 3INST: (x & 0x8FFF8FFF) ^ 0x3B603B60 = two fp16
+
+inline float h2f(uint16_t h) {
+ const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;
+ const uint32_t e = (h >> 10) & 0x1fu, m = h & 0x3ffu;
+ float f;
+ if (e == 0) f = std::ldexp((float) m, -24);
+ else if (e == 31) f = INFINITY;
+ else f = std::ldexp(1.0f + m / 1024.0f, (int) e - 15);
+ return sign ? -f : f;
+}
+
+// float -> fp16 -> float, round to nearest even (what HADD2 does to the exact sum of two fp16 values)
+inline float rn16(float x) {
+ if (x == 0.0f) return x;
+ int e;
+ std::frexp(x, &e); // |x| in [2^(e-1), 2^e)
+ const int q = (e - 1 < -14 ? -14 : e - 1) - 10;
+ const float quantum = std::ldexp(1.0f, q);
+ return std::nearbyint(x / quantum) * quantum;
+}
+
+inline uint32_t word3(uint32_t s, int i) { return ((s * kA[i] + kB[i]) & kMask3) ^ kXor3; }
+
+enum Kind { kNone = 0, k1mad, k3inst, k3sum, k3single };
+
+inline Kind kind_of(const char* n) {
+ if (!n || !*n) return kNone;
+ if (!std::strcmp(n, "1mad")) return k1mad;
+ if (!std::strcmp(n, "3inst")) return k3inst;
+ if (!std::strcmp(n, "3sum")) return k3sum;
+ if (!std::strcmp(n, "3single")) return k3single;
+ return kNone;
+}
+
+// the four (unnormalised) weights of state s
+inline void code4(Kind k, uint32_t s, float out[4]) {
+ s &= 0x7fffu;
+ switch (k) {
+ case k1mad:
+ for (int i = 0; i < 4; ++i) {
+ const uint32_t x = s * kA[i] + kB[i];
+ const int y = (int) (x & 255) + (int) ((x >> 8) & 255) + (int) ((x >> 16) & 255) + (int) (x >> 24);
+ out[i] = (float) (y - 510) / 147.800537109375f;
+ }
+ break;
+ case k3inst:
+ for (int i = 0; i < 4; ++i) { const uint32_t r = word3(s, i); out[i] = rn16(h2f((uint16_t) (r >> 16)) + h2f((uint16_t) (r & 0xffffu))); }
+ break;
+ case k3sum: {
+ const uint32_t r0 = word3(s, 0), r1 = word3(s, 1), r2 = word3(s, 2), r3 = word3(s, 3);
+ out[0] = rn16(h2f((uint16_t) (r0 & 0xffffu)) + h2f((uint16_t) (r1 & 0xffffu)));
+ out[1] = rn16(h2f((uint16_t) (r0 >> 16)) + h2f((uint16_t) (r1 >> 16)));
+ out[2] = rn16(h2f((uint16_t) (r2 & 0xffffu)) + h2f((uint16_t) (r3 & 0xffffu)));
+ out[3] = rn16(h2f((uint16_t) (r2 >> 16)) + h2f((uint16_t) (r3 >> 16)));
+ break;
+ }
+ default: {
+ const uint32_t r0 = word3(s, 0), r1 = word3(s, 1);
+ out[0] = h2f((uint16_t) (r0 & 0xffffu));
+ out[1] = h2f((uint16_t) (r0 >> 16));
+ out[2] = h2f((uint16_t) (r1 & 0xffffu));
+ out[3] = h2f((uint16_t) (r1 >> 16));
+ }
+ }
+}
+
+} // namespace tqcode
diff --git a/tools/dsv41_lab/tqk_gemv_c.cuh b/tools/dsv41_lab/tqk_gemv_c.cuh
new file mode 100644
index 000000000..d5ea91eaa
--- /dev/null
+++ b/tools/dsv41_lab/tqk_gemv_c.cuh
@@ -0,0 +1,109 @@
+// tools/dsv41_lab/tqk_gemv_c.cuh - v3 with a COMPUTED codebook instead of the 2048-entry table (LUT-free): the step's four weights
+// come from 2 or 4 LCG words, x_i = s * A_i + B_i, w = (x_i & 0x8FFF8FFF) ^ 0x3B603B60 read as two fp16 (QTIP 3INST's trick;
+// computed_codes.hpp). CODE 0 "3single": the two words are the A fragment (a0, a2) as they are: 2 IMAD + 2 LOP3 a step.
+// CODE 1 "3sum": a0 = HADD2(w0, w1), a2 = HADD2(w2, w3): 4 IMAD + 4 LOP3 + 2 HADD2. No shared-memory table, no __syncthreads.
+// Everything else (TQL layout, prepared B fragments, mma) is gemv_v3.
+#pragma once
+#include "tqk_gemv3.cuh"
+
+namespace tqlab {
+
+__device__ __forceinline__ uint32_t cword(uint32_t s, int i) {
+ constexpr uint32_t A[4] = {89226354u, 0x9E3779B1u, 0x85EBCA6Bu, 0xC2B2AE35u};
+ constexpr uint32_t B[4] = {64248484u, 0x7F4A7C15u, 0x165667B1u, 0x27D4EB2Fu};
+ return ((s * A[i] + B[i]) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+}
+__device__ __forceinline__ uint32_t hadd2_u(uint32_t a, uint32_t b) {
+ __half2 x = *(__half2*) &a, y = *(__half2*) &b;
+ const __half2 r = __hadd2(x, y);
+ return *(const uint32_t*) &r;
+}
+
+template <int K, int NB, int T, int EXACT, int CODE>
+__global__ void __launch_bounds__(128) gemv_v3c(const Job* __restrict__ jobs, const uint32_t* __restrict__ lut_unused, int in_unused) {
+ const Job job = jobs[blockIdx.y];
+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, q = lane & 3;
+ const int tile0 = (blockIdx.x * (blockDim.x >> 5) + warp) * T;
+ if (tile0 * 16 >= job.rows) return;
+ const int nblk = job.in / 128, NP = nblk / 2;
+ float tot[T][4];
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 4; ++i) tot[t][i] = 0.f;
+ const int src_lane = 4 * g + ((q + 3) & 3);
+ for (int P = 0; P < NP; ++P) {
+ uint32_t bh[2][8][2], bl[2][8][2];
+#pragma unroll
+ for (int bk = 0; bk < 2; ++bk) {
+ if (g < NB) {
+ const uint4* xp = ((const uint4*) job.x) + (((size_t) (2 * P + bk) * 8) * NB + g) * 4 + q;
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ const uint4 v = __ldg(xp + (size_t) m * NB * 4);
+ bh[bk][m][0] = v.x;
+ bh[bk][m][1] = v.y;
+ if (EXACT) { bl[bk][m][0] = v.z; bl[bk][m][1] = v.w; }
+ }
+ } else {
+#pragma unroll
+ for (int m = 0; m < 8; ++m) { bh[bk][m][0] = bh[bk][m][1] = bl[bk][m][0] = bl[bk][m][1] = 0u; }
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t) {
+ const size_t base = (size_t) (tile0 + t) * NP + P;
+ uint32_t r[K];
+#pragma unroll
+ for (int j = 0; j < K; ++j) r[j] = __ldcs(((const uint32_t*) job.w) + (base * K + j) * 32 + lane);
+ const uint64_t dq = __ldcs(job.sc + base * 8 + g);
+ uint64_t win[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int a = (8 * K * i) >> 5, sh = (8 * K * i) & 31;
+ const uint32_t w0 = r[a];
+ const uint32_t w1 = a + 1 < K ? r[a + 1 < K ? a + 1 : 0] : 0u;
+ const uint32_t w2 = a + 2 < K ? r[a + 2 < K ? a + 2 : 0] : 0u;
+ const uint32_t lo = __funnelshift_r(w0, w1, sh), hi = __funnelshift_r(w1, w2, sh);
+ const uint64_t ch = (((uint64_t) hi << 32) | lo) & ((1ull << (8 * K)) - 1ull);
+ const uint32_t fw = __shfl_sync(0xffffffffu, lo, src_lane);
+ win[i] = ch | ((uint64_t) (fw & ((1u << (15 - K)) - 1u)) << (8 * K));
+ }
+#pragma unroll
+ for (int bk = 0; bk < 2; ++bk) {
+ float acc[4] = {0.f, 0.f, 0.f, 0.f};
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ uint32_t a0, a1, a2, a3; // (row g, slots 2q..), (row g+8, ..), (row g, slots 2q+8..), (row g+8, ..)
+ {
+ const uint32_t s = (uint32_t) (win[2 * bk + 0] >> ((7 - m) * K)) & kMask;
+ if (CODE == 0) { a0 = cword(s, 0); a2 = cword(s, 1); }
+ else { a0 = hadd2_u(cword(s, 0), cword(s, 1)); a2 = hadd2_u(cword(s, 2), cword(s, 3)); }
+ }
+ {
+ const uint32_t s = (uint32_t) (win[2 * bk + 1] >> ((7 - m) * K)) & kMask;
+ if (CODE == 0) { a1 = cword(s, 0); a3 = cword(s, 1); }
+ else { a1 = hadd2_u(cword(s, 0), cword(s, 1)); a3 = hadd2_u(cword(s, 2), cword(s, 3)); }
+ }
+ mma16816(acc, a0, a1, a2, a3, bh[bk][m][0], bh[bk][m][1]);
+ if (EXACT) mma16816(acc, a0, a1, a2, a3, bl[bk][m][0], bl[bk][m][1]);
+ }
+ const float d0 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 0)))));
+ const float d1 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 1)))));
+ tot[t][0] += d0 * acc[0];
+ tot[t][1] += d0 * acc[1];
+ tot[t][2] += d1 * acc[2];
+ tot[t][3] += d1 * acc[3];
+ }
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int col = 2 * q + (i & 1), row = (tile0 + t) * 16 + g + 8 * (i >> 1);
+ if (col < NB) job.y[(size_t) col * job.ldy + row] = tot[t][i];
+ }
+}
+
+} // namespace tqlab
diff --git a/tools/dsv41_lab/tqk_lab.cu b/tools/dsv41_lab/tqk_lab.cu
index d2938fa8e..9bbc1b2f8 100644
--- a/tools/dsv41_lab/tqk_lab.cu
+++ b/tools/dsv41_lab/tqk_lab.cu
@@ -4,6 +4,7 @@
// Bench: one decode token of a routed layer = top-6 experts x (gate, up: 2304 x 5120; down: 5120 x 2304).
#include "tqk_gemv.cuh"
#include "tqk_gemv3.cuh"
+#include "tqk_gemv_c.cuh"
#include "tqk_repack.hpp"
#include "../trellis/tqenc.hpp"
#include "strata/kernels/cpu/tq_cpu.hpp"
@@ -55,7 +56,14 @@ template <int K, int NB> static KFn pick(int variant) {
case 8: return {(const void*) gemv_v3<K, NB, 1, 1>, 128, 64, 1, 1};
case 9: return {(const void*) gemv_v3<K, NB, 2, 1>, 128, 128, 1, 1};
case 10: return {(const void*) gemv_v3<K, NB, 4, 1>, 128, 256, 1, 1};
- default: return {(const void*) gemv_v3<K, NB, 2, 0>, 128, 128, 1, 1};
+ case 11: return {(const void*) gemv_v3<K, NB, 2, 0>, 128, 128, 1, 1};
+ case 12: return {(const void*) gemv_v3c<K, NB, 4, 1, 0>, 128, 256, 1, 1};
+ case 13: return {(const void*) gemv_v3c<K, NB, 4, 1, 1>, 128, 256, 1, 1};
+ case 14: return {(const void*) gemv_v3c<K, NB, 2, 1, 0>, 128, 128, 1, 1};
+ case 15: return {(const void*) gemv_v3c<K, NB, 2, 1, 1>, 128, 128, 1, 1};
+ case 16: return {(const void*) gemv_v3c<K, NB, 2, 0, 0>, 128, 128, 1, 1};
+ case 17: return {(const void*) gemv_v3c<K, NB, 4, 0, 0>, 128, 256, 1, 1};
+ default: return {(const void*) gemv_v3c<K, NB, 1, 0, 0>, 128, 64, 1, 1};
}
}
template <int K> static KFn pick_nb(int nb, int variant) {
@@ -163,7 +171,7 @@ int main(int argc, char** argv) {
double sq = 0, ee = 0; for (int64_t i = 0; i < nenc * 128; ++i) { const double r = got[(size_t) (nblocks - nenc) * 128 + i]; sq += (double) g[i] * g[i]; ee += (r - g[i]) * (r - g[i]); }
std::printf("dequant: %zu / %zu weights differ from tqenc::decode_blocks (bit exact %s); encoded N(0,1) SQNR %.2f dB (encoder err %.3f)\n",
bad, got.size(), bad ? "NO" : "yes", 10 * std::log10(sq / ee), err / (nenc * 128));
- fails += bad != 0;
+ fails += (bad != 0) && !std::getenv("TQE_CODE"); // (the table kernel is meaningless under a computed code)
// ---- parity 2: GEMV vs double reference on the decoded weights, rows x in small
for (int nb : NBs) {
const int rows = 256, in = 768, nblk = in / 128;
@@ -190,7 +198,7 @@ int main(int argc, char** argv) {
for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); } ref[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }
std::vector<float> y = gpu_run(x);
double worst = 0; for (size_t i = 0; i < y.size(); ++i) worst = std::max(worst, std::fabs(y[i] - ref[i]) / (refabs[i] + 1e-30));
- const bool ok = worst < 2e-6;
+ const bool ok = worst < 2e-6 || (variant >= 16 && variant <= 18 && worst < 1e-4);
// (2) the CPU kernel of the repo (tq_cpu scalar and AVX2, Q8_0 activations): same weights; the GPU runs on the dequantised Q8_0 activations
std::vector<uint8_t> q8((size_t) nb * (in / 32) * 34); std::vector<float> xq((size_t) nb * in);
for (int c = 0; c < nb; ++c) for (int b = 0; b < in / 32; ++b) {
@@ -212,7 +220,7 @@ int main(int argc, char** argv) {
double w_gc = 0, w_cr = 0, w_ac = 0;
for (size_t i = 0; i < yq.size(); ++i) { const double den = refabs[i] + 1e-30; w_gc = std::max(w_gc, std::fabs(yq[i] - ycpu_s[i]) / den); w_cr = std::max(w_cr, std::fabs(ycpu_s[i] - refq[i]) / den); if (have_avx2) w_ac = std::max(w_ac, std::fabs(ycpu_a[i] - ycpu_s[i]) / den); }
std::printf("gemv nb=%d: GPU vs double ref %.2e %s | vs CPU kernel of the repo (Q8_0 acts): GPU-CPU %.2e, CPU-double %.2e, avx2-scalar %.2e (all / sum|w x|)\n", nb, worst, ok ? "ok" : "FAIL", w_gc, w_cr, w_ac);
- fails += !ok || w_gc > 5e-6;
+ fails += !ok || (w_gc > 5e-6 && !std::getenv("TQE_CODE"));
cudaFree(mwd); if (kf.tql) { cudaFree(wd); }
}
cudaFree(dw); cudaFree(dout);
diff --git a/tools/tqenc.py b/tools/tqenc.py
new file mode 100644
index 000000000..a92feb7a6
--- /dev/null
+++ b/tools/tqenc.py
@@ -0,0 +1,221 @@
+"""tqenc.py - Python side of the trellis encoder (tools/trellis/tqenc.cpp, built to libtqenc.so by
+`sh tools/trellis/build.sh lib`): encode / decode blocks of TQ2_T, TQK6, TQK7 and the experimental TQK5X / TQK4X, the
+input-only block Hadamard of the DeepSeek plan, and error metrics.
+
+Rows of a tensor are blocks of 128 consecutive weights along the INPUT dimension (the GGUF fastest axis), exactly as the
+decoders read them. A tensor of shape [rows, n] (n a multiple of 128) is encoded row by row; the output has
+rows * n/128 * block_bytes bytes.
+"""
+from __future__ import annotations
+
+import ctypes
+import os
+import pathlib
+
+import numpy as np
+
+TQ2_T, TQK6, TQK7, TQK5X, TQK4X = 144, 145, 146, 224, 225
+TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X"}
+TYPE_BY_NAME = {v: k for k, v in TYPE_NAMES.items()}
+KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4}
+BLOCK = 128
+
+
+def block_bytes(t: int) -> int:
+ return 2 + 4 * KBITS[t]
+
+
+def bpw(t: int) -> float:
+ return block_bytes(t) * 8 / BLOCK
+
+
+_lib = None
+
+
+def lib() -> ctypes.CDLL:
+ global _lib
+ if _lib is None:
+ cands = [os.environ.get("TQENC_LIB", ""),
+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc.so"),
+ "libtqenc.so"]
+ last = None
+ for c in cands:
+ if not c:
+ continue
+ try:
+ _lib = ctypes.CDLL(c)
+ break
+ except OSError as e:
+ last = e
+ if _lib is None:
+ raise RuntimeError("libtqenc.so not found (sh tools/trellis/build.sh lib): %s" % last)
+ _lib.tqe_encode.restype = ctypes.c_double
+ _lib.tqe_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,
+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,
+ ctypes.c_void_p, ctypes.c_void_p]
+ _lib.tqe_decode.restype = None
+ _lib.tqe_decode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p]
+ return _lib
+
+
+_hip = None
+
+
+def hip_lib() -> ctypes.CDLL:
+ """libtqenc_hip.so (sh tools/trellis/build.sh hip; run under the box GPU locks, HIP_VISIBLE_DEVICES picks the card)."""
+ global _hip
+ if _hip is None:
+ cands = [os.environ.get("TQENC_HIP_LIB", ""),
+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc_hip.so")]
+ last = None
+ for c in cands:
+ if not c:
+ continue
+ try:
+ _hip = ctypes.CDLL(c)
+ break
+ except OSError as e:
+ last = e
+ if _hip is None:
+ raise RuntimeError("libtqenc_hip.so not found: %s" % last)
+ _hip.tqh_encode.restype = ctypes.c_double
+ _hip.tqh_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,
+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
+ return _hip
+
+
+def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, ncand: int = 1, exact: int = 0,
+ scale_tries: int = 1, threads: int = 0, gpu: bool = False):
+ """w: float32 array whose last axis is a multiple of 128. Returns (packed uint8 [nblocks, block_bytes],
+ per-block error float64 [nblocks], total error, stats dict). Error = sum h (w - decoded)^2 (h = 1 if None)."""
+ w = np.ascontiguousarray(w, dtype=np.float32)
+ if w.shape[-1] % BLOCK:
+ raise ValueError("last axis %d is not a multiple of 128" % w.shape[-1])
+ nb = w.size // BLOCK
+ out = np.zeros((nb, block_bytes(t)), dtype=np.uint8)
+ err = np.zeros(nb, dtype=np.float64)
+ stats = np.zeros(2, dtype=np.int64)
+ hp = None
+ if h is not None:
+ h = np.ascontiguousarray(h, dtype=np.float32)
+ if h.shape != w.shape:
+ raise ValueError("importance weights must have the shape of w")
+ hp = h.ctypes.data
+ if gpu:
+ if exact or t == TQ2_T:
+ raise ValueError("the GPU encoder has no exact search and no TQ2_T")
+ tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,
+ err.ctypes.data, stats.ctypes.data, None, 0)
+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}
+ tot = lib().tqe_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, exact, scale_tries, threads,
+ err.ctypes.data, stats.ctypes.data)
+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}
+
+
+def decode(packed: np.ndarray, t: int) -> np.ndarray:
+ """packed: uint8 array of whole blocks. Returns float32 [nblocks * 128] (d * codebook)."""
+ packed = np.ascontiguousarray(packed, dtype=np.uint8)
+ bb = block_bytes(t)
+ if packed.size % bb:
+ raise ValueError("%d bytes is not a whole number of %s blocks" % (packed.size, TYPE_NAMES[t]))
+ nb = packed.size // bb
+ out = np.zeros(nb * BLOCK, dtype=np.float32)
+ lib().tqe_decode(t, packed.ctypes.data, nb, out.ctypes.data)
+ return out
+
+
+# ---------------------------------------------------------------------------------------------------- Hadamard
+
+def hadamard_matrix(n: int = BLOCK) -> np.ndarray:
+ """The normalized Sylvester-Walsh-Hadamard matrix (entry (-1)^popcount(r & c) / sqrt(n)); symmetric, orthonormal."""
+ h = np.array([[1.0]])
+ while h.shape[0] < n:
+ h = np.block([[h, h], [h, -h]])
+ return (h / np.sqrt(n)).astype(np.float32)
+
+
+def make_signs(width: int, seed: int) -> np.ndarray:
+ """A reproducible +1/-1 vector (the plan's D). The vector itself, not the seed, is what a file must store."""
+ r = np.random.default_rng(seed)
+ return (r.integers(0, 2, size=width) * 2 - 1).astype(np.float32)
+
+
+def pack_signs(signs: np.ndarray) -> bytes:
+ return np.packbits((signs > 0).astype(np.uint8), bitorder="little").tobytes()
+
+
+def unpack_signs(b: bytes, width: int) -> np.ndarray:
+ bits = np.unpackbits(np.frombuffer(b, dtype=np.uint8), bitorder="little")[:width]
+ return bits.astype(np.float32) * 2 - 1
+
+
+def rotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:
+ """The input transform U = H D on the last axis, per 128-block: Wrot = W U^T, i.e. each row w -> H (signs * w).
+ The same function applied to an activation x gives U x."""
+ w = np.asarray(w, dtype=np.float32)
+ if signs is not None:
+ w = w * signs
+ shp = w.shape
+ return (w.reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp) # H is symmetric
+
+
+def unrotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:
+ """Inverse of rotate (H is its own inverse: orthonormal and symmetric; then the signs)."""
+ shp = w.shape
+ x = (np.asarray(w, dtype=np.float32).reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp)
+ return x * signs if signs is not None else x
+
+
+# ---------------------------------------------------------------------------------------------------- metrics
+
+def sqnr_db(ref: np.ndarray, approx: np.ndarray) -> float:
+ ref = np.asarray(ref, dtype=np.float64).ravel()
+ e = np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2)
+ return float("inf") if e == 0 else float(10 * np.log10(np.sum(ref ** 2) / e))
+
+
+def rel_mse(ref: np.ndarray, approx: np.ndarray) -> float:
+ ref = np.asarray(ref, dtype=np.float64).ravel()
+ return float(np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2) / max(np.sum(ref ** 2), 1e-300))
+
+
+# ---------------------------------------------------------------------------------------------------- .tqx container
+# A small versioned file for ONE encoded matrix (research artifacts; not a GGUF). Little endian:
+# "TQXP" | u32 version (1) | u32 type id | u32 rows | u32 cols | u32 flags (bit 0: input Hadamard, 128-blocks)
+# | u32 n_sign_bytes | sign bits (1 = +1, LSB first, one per input column) | u32 hadamard block (128) | packed blocks
+# An unknown version or type is refused before any data is read.
+TQX_MAGIC = b"TQXP"
+TQX_VERSION = 1
+
+
+def write_tqx(path, t: int, rows: int, cols: int, packed: np.ndarray, signs: np.ndarray | None) -> None:
+ import struct
+ sb = pack_signs(signs) if signs is not None else b""
+ with open(path, "wb") as f:
+ f.write(TQX_MAGIC + struct.pack("<IIIIII", TQX_VERSION, t, rows, cols, 1 if signs is not None else 0, len(sb)))
+ f.write(sb)
+ f.write(struct.pack("<I", BLOCK))
+ f.write(np.ascontiguousarray(packed, dtype=np.uint8).tobytes())
+
+
+def read_tqx(path):
+ import struct
+ data = open(path, "rb").read()
+ if data[:4] != TQX_MAGIC:
+ raise ValueError("not a TQXP file")
+ ver, t, rows, cols, flags, nsb = struct.unpack_from("<IIIIII", data, 4)
+ if ver != TQX_VERSION:
+ raise ValueError("unsupported TQXP version %d (this reader handles %d)" % (ver, TQX_VERSION))
+ if t not in KBITS:
+ raise ValueError("unknown trellis type %d" % t)
+ at = 4 + 24
+ signs = unpack_signs(data[at:at + nsb], cols) if flags & 1 else None
+ at += nsb
+ (blk,) = struct.unpack_from("<I", data, at)
+ at += 4
+ if blk != BLOCK:
+ raise ValueError("unsupported Hadamard block %d" % blk)
+ nb = rows * (cols // BLOCK)
+ packed = np.frombuffer(data, dtype=np.uint8, count=nb * block_bytes(t), offset=at).reshape(nb, block_bytes(t))
+ return t, rows, cols, packed, signs
diff --git a/tools/trellis/README.md b/tools/trellis/README.md
new file mode 100644
index 000000000..a76354aa7
--- /dev/null
+++ b/tools/trellis/README.md
@@ -0,0 +1,34 @@
+# Trellis encoder (research, branch f-dsv41-enc)
+
+An encoder for the trellis expert formats that Strata already decodes (TQ2_T, TQK6, TQK7 of agentionai's llama.cpp fork)
+and for two experimental lower rates, TQK5X (22 bytes per 128 weights, 1.375 bpw) and TQK4X (18 bytes, 1.125 bpw).
+Written for the DeepSeek-V4.1-Flash compression plan (gate 2 and the start of gate 3). Nothing here changes an existing
+type, layout or default; the engine's CPU decoder only gains K = 5 and 4 (ids 224 / 225, experimental version 1).
+
+| file | what |
+| --- | --- |
+| `tqenc.hpp`, `tqenc.cpp` | the encoder: tail-biting Viterbi on the 2^15-state shift trellis, fp16 scale refinement, importance weights; a C API |
+| `tqenc_test.cpp` | the tests (about 64,000 checks): streams and the circular wrap, Viterbi against a naive DP, planted solutions, bit-exact decode against `tq_states.hpp` and the scalar / AVX2 / AVX-512 kernels, special blocks, thread determinism, quality against the rate-distortion limit |
+| `tqenc_hip.cpp`, `tqenc_hip_test.cpp` | the same search on a GPU (HIP, gfx1201): bit-identical blocks, about 20 M weights/s on one R9700 |
+| `../tqenc.py` | Python wrapper (ctypes): `encode`, `decode`, the input-only block-128 Hadamard, signs, the `.tqx` container |
+| `pilot.py`, `pilot_summary.py` | the block-quality pilot on a source GGUF (local files or HTTP range requests) |
+| `gyro_reencode.py`, `gyro_vs_original.py`, `gyro_error_profile.py`, `hf_slice.py` | checks against agentionai's Gyro files and the original Qwen weights |
+
+Build and test (Linux x86-64, g++ with AVX2; the GPU part needs ROCm):
+
+ sh tools/trellis/build.sh test && build-tqenc/tqenc_test # --quick skips the quality table
+ sh tools/trellis/build.sh lib && python3 tools/trellis/test_tqenc_py.py
+ sh tools/trellis/build.sh hip && HIP_VISIBLE_DEVICES=3 build-tqenc/tqenc_hip_test
+
+`build.sh hip` compiles `tqenc.cpp` with g++ on purpose: under hipcc, `ggml-common-tq.h` turns its codebook into a
+`__device__` variable and the host side reads zeros.
+
+## Format facts the encoder relies on (checked against the decoder, not the comments)
+
+* A block is `{fp16 d; 4K bytes}` for 128 weights: 32 steps, step t gives weights 4t..4t+3.
+* State of step t: 15 bits. TQK: the 16-bit window of the stream at bit `(31 - t) K` (LSB first, circular modulo 32 K bits), masked
+ to 15 bits. TQ2_T: `(qs[(t+31)%32] << 8 | qs[t]) & 0x7fff`.
+* Weights of a step: `x = state * 0x9e3779b1`, the two codebook points `lut[x >> 21]` and `lut[(x >> 10) & 2047]`, times d. The
+ decoder's table is `round(fp16 * 8192)`; the encoder uses that table, so its reconstruction is the decoder's, bit for bit.
+* A legal stream is any byte string (every state is a window of the same circular stream); tail-biting only matters to the search.
+* K5 and K4 are the TQK rule with K = 5 and 4. Nothing else changes (hash, codebook, mask, bit order, scale).
diff --git a/tools/trellis/build.sh b/tools/trellis/build.sh
new file mode 100644
index 000000000..45ab53842
--- /dev/null
+++ b/tools/trellis/build.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+# tools/trellis/build.sh [test|lib|bench] - builds the trellis encoder standalone (g++, Linux x86-64).
+# Run from the repository root. Output in ${TQ_OUT:-build-tqenc}.
+set -e
+OUT=${TQ_OUT:-build-tqenc}
+mkdir -p "$OUT"
+CXX=${CXX:-g++}
+INC="-Itools/trellis -Iinclude -Isrc/kernels/cpu -Ithird_party/ggml"
+OPT="-O3 -std=c++17 -mavx2 -mfma -mf16c -fno-fast-math -pthread"
+case "${1:-test}" in
+lib)
+ $CXX $OPT -fPIC -shared $INC tools/trellis/tqenc.cpp -o "$OUT/libtqenc.so" ;;
+hip)
+ # needs the ROCm of /data/dsv41/rv: export ROCM_PATH=/data/dsv41/rv/lib/python3.14/site-packages/_rocm_sdk_devel PATH=/data/dsv41/rv/bin:$PATH
+ # the host part of the encoder (tqenc.cpp) is built by g++: under hipcc ggml-common-tq.h's table becomes a __device__ variable
+ g++ $OPT -fPIC $INC -c tools/trellis/tqenc.cpp -o "$OUT/tqenc_host.o"
+ HF="-O3 --offload-arch=gfx1201 -mcumode -std=c++17 -mavx2 -mfma -mf16c"
+ hipcc $HF -fPIC $INC -c tools/trellis/tqenc_hip.cpp -o "$OUT/tqenc_hip.o"
+ hipcc $HF $INC -c tools/trellis/tqenc_hip_test.cpp -o "$OUT/tqenc_hip_test.o"
+ hipcc --offload-arch=gfx1201 "$OUT/tqenc_hip_test.o" "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/tqenc_hip_test"
+ hipcc --offload-arch=gfx1201 -shared "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/libtqenc_hip.so" ;;
+test)
+ $CXX $OPT $INC -c src/kernels/cpu/tq_cpu.cpp -o "$OUT/tq_cpu.o"
+ $CXX $OPT $INC -c src/kernels/cpu/tq_avx2.cpp -o "$OUT/tq_avx2.o"
+ $CXX -O3 -std=c++17 -mavx512f -mavx512bw -mavx512vl -mavx512vbmi -mavx512vnni -mfma -mf16c -mavx2 $INC -c src/kernels/cpu/tq_avx512.cpp -o "$OUT/tq_avx512.o"
+ $CXX $OPT $INC tools/trellis/tqenc_test.cpp tools/trellis/tqenc.cpp "$OUT/tq_cpu.o" "$OUT/tq_avx2.o" "$OUT/tq_avx512.o" -o "$OUT/tqenc_test"
+ ;;
+esac
+echo built "$OUT"
diff --git a/tools/trellis/hf_slice.py b/tools/trellis/hf_slice.py
new file mode 100644
index 000000000..6e6186601
--- /dev/null
+++ b/tools/trellis/hf_slice.py
@@ -0,0 +1,94 @@
+"""hf_slice.py - read single tensors (or slices of them) out of a remote safetensors repo with HTTP range requests, so a
+quality test needs megabytes of a model, not the model. No token is read or printed (public repos only; set HF_TOKEN in
+the environment for gated ones - it is sent as a header and never printed).
+
+ from hf_slice import Repo
+ r = Repo("Qwen/Qwen3.8-Flash-Next")
+ w = r.tensor("model.language_model.layers.0.mlp.experts.gate_up_proj", rows=slice(0, 1)) # expert 0 only
+"""
+from __future__ import annotations
+
+import concurrent.futures as cf
+import json
+import os
+import struct
+import urllib.request
+
+import numpy as np
+
+_DT = {"BF16": (2, None), "F16": (2, np.float16), "F32": (4, np.float32), "F8_E4M3": (1, None), "I8": (1, np.int8),
+ "U8": (1, np.uint8), "I32": (4, np.int32), "I64": (8, np.int64)}
+
+
+def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:
+ req = urllib.request.Request(url)
+ tok = os.environ.get("HF_TOKEN")
+ if tok:
+ req.add_header("Authorization", "Bearer " + tok)
+ if start is not None:
+ req.add_header("Range", "bytes=%d-%d" % (start, end))
+ with urllib.request.urlopen(req, timeout=120) as r:
+ return r.read()
+
+
+def get_parallel(url: str, start: int, end: int, workers: int = 16, piece: int = 8 << 20) -> bytes:
+ """bytes start..end (inclusive) of url, in `piece`-byte range requests on `workers` threads (one stream of a CDN
+ object is slow, a dozen are not)."""
+ spans = [(a, min(end, a + piece - 1)) for a in range(start, end + 1, piece)]
+ if len(spans) == 1:
+ return _get(url, start, end)
+
+ def one(sp):
+ for attempt in range(6):
+ try:
+ return _get(url, sp[0], sp[1])
+ except Exception:
+ if attempt == 5:
+ raise
+ return b""
+ with cf.ThreadPoolExecutor(workers) as ex:
+ return b"".join(ex.map(one, spans))
+
+
+def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:
+ u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
+ return u.view(np.float32)
+
+
+class Repo:
+ def __init__(self, repo: str, rev: str = "main"):
+ self.repo, self.rev = repo, rev
+ self.base = "https://huggingface.co/%s/resolve/%s/" % (repo, rev)
+ self.map = json.loads(_get(self.base + "model.safetensors.index.json"))["weight_map"]
+ self._hdr: dict[str, tuple[dict, int]] = {}
+
+ def header(self, shard: str):
+ if shard not in self._hdr:
+ n = struct.unpack("<Q", _get(self.base + shard, 0, 7))[0]
+ h = json.loads(_get(self.base + shard, 8, 8 + n - 1))
+ self._hdr[shard] = (h, 8 + n)
+ return self._hdr[shard]
+
+ def info(self, name: str):
+ shard = self.map[name]
+ h, base = self.header(shard)
+ return shard, h[name], base
+
+ def tensor(self, name: str, lead: slice | int | None = None, workers: int = 16) -> np.ndarray:
+ """The tensor `name` as float32. lead: an index or slice on the first axis (e.g. an expert), fetched alone."""
+ shard, t, base = self.info(name)
+ shape = t["shape"]
+ item = _DT[t["dtype"]][0]
+ s, e = t["data_offsets"]
+ inner = int(np.prod(shape[1:])) * item if len(shape) > 1 else item
+ if lead is None:
+ i0, i1 = 0, shape[0]
+ elif isinstance(lead, int):
+ i0, i1 = lead, lead + 1
+ else:
+ i0, i1 = lead.start or 0, lead.stop
+ raw = get_parallel(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1, workers)
+ out_shape = [i1 - i0] + shape[1:]
+ if t["dtype"] == "BF16":
+ return bf16_to_f32(raw).reshape(out_shape)
+ return np.frombuffer(raw, dtype=_DT[t["dtype"]][1]).astype(np.float32).reshape(out_shape)
diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.py
new file mode 100644
index 000000000..9781251a8
--- /dev/null
+++ b/tools/trellis/pilot.py
@@ -0,0 +1,228 @@
+"""pilot.py - gate-3 block-quality pilot: a few experts from several layers of a source checkpoint, encoded at TQK4X / TQK5X /
+TQK6 / TQK7 with and without the input-only block-128 Hadamard, and measured.
+
+Sources: --gguf <shard1 or shard dir files...> (UD-Q8_K_XL: MXFP4 / Q8_0 / BF16 / F16 tensors, decoded exactly: the decoded
+ source is the teacher, as the plan says) or --hf Qwen/... (range requests; test of the pipeline)
+
+Metrics per (layer, role, expert, rate, transform), over a random row subset of the projection (--rows):
+ mse relative weight MSE ||W - W^||^2 / ||W||^2 in the original basis (the Hadamard is orthonormal: the same in either basis)
+ act_iso ||(W - W^) X||^2 / ||W X||^2 with X i.i.d. N(0,1) columns - PLACEHOLDER; in expectation equal to mse
+ act_syn ||(W - W^) X||^2 / ||W X||^2 with X whose input channels have heavy-tailed scales (a few outlier channels,
+ log-normal rest) - SYNTHETIC PLACEHOLDER for real calibration activations, labelled as such in every output
+Real activation-weighted numbers need calibration activations from the reference runtime (not done here).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import mmap
+import pathlib
+import sys
+import time
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import gguf_reader # noqa: E402
+import tqenc # noqa: E402
+
+MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)
+RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7}
+CFGS = {
+ "fast": dict(iters=0, ncand=1, scale_tries=1),
+ "default": dict(iters=1, ncand=1, scale_tries=1),
+ "rot3": dict(iters=3, ncand=3, scale_tries=1),
+}
+
+
+def decode_mxfp4(raw: np.ndarray, n_in: int) -> np.ndarray:
+ """raw: [rows * n_in/32 * 17] bytes -> float32 [rows, n_in]. ggml block_mxfp4: e8m0 scale, 16 bytes, low nibble =
+ element j, high nibble = element j + 16, value = kvalues[nibble] * 2^(e - 128) (the e8m0 half scale)."""
+ b = raw.reshape(-1, 17)
+ e = b[:, 0].astype(np.int32)
+ d = np.where(e == 0, np.float32(2.0 ** -127), np.ldexp(np.float32(1.0), e - 128)).astype(np.float32)
+ q = b[:, 1:]
+ vals = np.concatenate([MXFP4_LUT[q & 15], MXFP4_LUT[q >> 4]], axis=1) * d[:, None]
+ return vals.reshape(-1, n_in)
+
+
+def decode_q8_0(raw: np.ndarray, n_in: int) -> np.ndarray:
+ b = raw.reshape(-1, 34)
+ d = b[:, :2].copy().view(np.float16).astype(np.float32)
+ q = b[:, 2:].copy().view(np.int8).astype(np.float32)
+ return (q * d).reshape(-1, n_in)
+
+
+class Source:
+ """Yields float32 [rows, n_in] row subsets of one expert's projection. Local GGUF files (mmap) or, with
+ remote=(repo, rev, [shard paths]), the same files read by HTTP range requests (headers and the few experts needed)."""
+
+ def __init__(self, gguf_paths=None, remote=None, tmpdir="/tmp"):
+ self.tensors = {}
+ if gguf_paths:
+ for p in gguf_paths:
+ g = gguf_reader.GGUFFile(p)
+ fh = open(p, "rb")
+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
+ for t in g.tensors:
+ self.tensors[t.name] = (g, (lambda off, n, mm=mm: np.frombuffer(mm, dtype=np.uint8, count=n, offset=off)), t)
+ if remote:
+ import urllib.request
+ repo, rev, shards = remote
+ for sh in shards:
+ url = "https://huggingface.co/%s/resolve/%s/%s" % (repo, rev, sh)
+
+ def rd(off, n, url=url):
+ req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (off, off + n - 1)})
+ with urllib.request.urlopen(req, timeout=300) as r:
+ return np.frombuffer(r.read(), dtype=np.uint8)
+ hdr = rd(0, 6 << 20)
+ tp = pathlib.Path(tmpdir) / ("hdr_%s" % pathlib.Path(sh).name)
+ tp.write_bytes(hdr.tobytes())
+ g = gguf_reader.GGUFFile(tp)
+ for t in g.tensors:
+ self.tensors[t.name] = (g, rd, t)
+
+ def names(self, needle):
+ return sorted(n for n in self.tensors if needle in n)
+
+ def expert_rows(self, name: str, expert: int, rows: np.ndarray) -> np.ndarray:
+ g, rd, ti = self.tensors[name]
+ n_in, n_out = ti.shape[0], ti.shape[1]
+ if ti.type_name == "MXFP4":
+ rb = n_in // 32 * 17
+ elif ti.type_name == "Q8_0":
+ rb = n_in // 32 * 34
+ elif ti.type_name in ("BF16", "F16"):
+ rb = n_in * 2
+ elif ti.type_name == "F32":
+ rb = n_in * 4
+ else:
+ raise ValueError("%s: tensor type %s is not readable by this pilot" % (name, ti.type_name))
+ base = g.data_start + ti.offset + expert * n_out * rb
+ whole = rd(base, n_out * rb) # one expert's projection (6.3 MB for MXFP4 2304 x 5120), then the row subset
+ out = np.empty((len(rows), n_in), dtype=np.float32)
+ for k, r in enumerate(rows):
+ raw = whole[int(r) * rb:(int(r) + 1) * rb]
+ if ti.type_name == "MXFP4":
+ out[k] = decode_mxfp4(raw, n_in)[0]
+ elif ti.type_name == "Q8_0":
+ out[k] = decode_q8_0(raw, n_in)[0]
+ elif ti.type_name == "BF16":
+ out[k] = (raw.view(np.uint16).astype(np.uint32) << 16).view(np.float32)
+ elif ti.type_name == "F16":
+ out[k] = raw.view(np.float16).astype(np.float32)
+ else:
+ out[k] = raw.view(np.float32)
+ return out
+
+
+def synth_x(n_in: int, T: int, seed: int) -> np.ndarray:
+ """SYNTHETIC activations: channel scale exp(N(0, 0.5)), 1% of the channels x (10..40)."""
+ r = np.random.default_rng(seed)
+ s = np.exp(0.5 * r.standard_normal(n_in)).astype(np.float32)
+ out = r.choice(n_in, size=max(1, n_in // 100), replace=False)
+ s[out] *= r.uniform(10, 40, size=out.size).astype(np.float32)
+ return (r.standard_normal((n_in, T)).astype(np.float32) * s[:, None])
+
+
+def act_err(W, What, X):
+ D = (W - What).astype(np.float32) @ X
+ Y = W.astype(np.float32) @ X
+ return float(np.sum(D.astype(np.float64) ** 2) / max(np.sum(Y.astype(np.float64) ** 2), 1e-300))
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--gguf", nargs="+", default=None)
+ ap.add_argument("--remote", default=None, help="REPO@REV: read the shards named in --shards by HTTP range requests")
+ ap.add_argument("--shards", nargs="+", default=[])
+ ap.add_argument("--layers", default="1,7,14,21,30,38")
+ ap.add_argument("--experts", default="0,97,200,383")
+ ap.add_argument("--roles", default="gate,up,down")
+ ap.add_argument("--rates", default="K4,K5,K6,K7")
+ ap.add_argument("--rows", type=int, default=256)
+ ap.add_argument("--tokens", type=int, default=256, help="columns of the placeholder X")
+ ap.add_argument("--cfg", default="default")
+ ap.add_argument("--threads", type=int, default=0)
+ ap.add_argument("--out", required=True, help="jsonl results")
+ ap.add_argument("--seed", type=int, default=1)
+ ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")
+ ap.add_argument("--gpu", action="store_true", help="encode on the GPU (run under the box locks, HIP_VISIBLE_DEVICES)")
+ ap.add_argument("--act-diag-dir", default=None,
+ help="directory with L<layer>_<role>.npy = E[x^2] per input channel of that projection (calibration "
+ "second moments, diagonal). Adds act_diag (the exact diagonal-Hessian error) and an importance-weighted "
+ "encode (h = E[x^2]; in the Hadamard basis h = the block average of it)")
+ a = ap.parse_args()
+
+ if a.remote:
+ repo, rev = a.remote.split("@")
+ src = Source(remote=(repo, rev, a.shards))
+ else:
+ src = Source(a.gguf)
+ rng = np.random.default_rng(a.seed)
+ cfg = CFGS[a.cfg]
+ sign_cache, x_cache = {}, {}
+ out = open(a.out, "a")
+ for layer in [int(x) for x in a.layers.split(",")]:
+ for role in a.roles.split(","):
+ name = a.pattern.format(L=layer, R=role)
+ if name not in src.tensors:
+ print("no tensor", name, "(have e.g. %s)" % src.names("exps")[:3])
+ continue
+ g, rd, ti = src.tensors[name]
+ n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]
+ for e in [int(x) for x in a.experts.split(",")]:
+ if e >= n_exp:
+ continue
+ rows = np.sort(rng.choice(n_out, size=min(a.rows, n_out), replace=False))
+ W = src.expert_rows(name, e, rows)
+ en = float(np.sum(W.astype(np.float64) ** 2))
+ if n_in not in sign_cache:
+ sign_cache[n_in] = tqenc.make_signs(n_in, 1000 + n_in)
+ x_cache[n_in] = (np.random.default_rng(7).standard_normal((n_in, a.tokens)).astype(np.float32),
+ synth_x(n_in, a.tokens, 9))
+ Xi, Xs = x_cache[n_in]
+ diag = None
+ if a.act_diag_dir:
+ dp = pathlib.Path(a.act_diag_dir) / ("L%d_%s.npy" % (layer, role))
+ if dp.exists():
+ diag = np.load(dp).astype(np.float64)
+ assert diag.shape == (n_in,), (dp, diag.shape)
+ for had in (False, True):
+ sg = sign_cache[n_in] if had else None
+ Wr = tqenc.rotate(W, sg) if had else W
+ for rname in a.rates.split(","):
+ t = RATES[rname]
+ t0 = time.time()
+ packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, gpu=a.gpu, **cfg)
+ sec = time.time() - t0
+ dec = tqenc.decode(packed, t).reshape(Wr.shape)
+ What = tqenc.unrotate(dec, sg) if had else dec
+ rec = dict(layer=layer, role=role, expert=e, rate=rname, had=had, n_in=n_in, rows=len(rows),
+ mse=float(np.sum((W.astype(np.float64) - What) ** 2) / en),
+ act_iso=act_err(W, What, Xi), act_syn=act_err(W, What, Xs),
+ bpw=tqenc.bpw(t), seconds=sec, zero_blocks=st["zero_blocks"], cfg=a.cfg,
+ x_kind="placeholder: iso gaussian / synthetic outlier channels")
+ if diag is not None:
+ hv = np.maximum(diag, 1e-30).astype(np.float32)
+ rec["act_diag"] = float(np.sum(((W - What).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))
+ hb = hv if not had else np.repeat(hv.reshape(-1, 128).mean(axis=1), 128)
+ hw = np.broadcast_to(hb, Wr.shape).astype(np.float32)
+ packed2, _, _, _ = tqenc.encode(Wr, t, h=hw, threads=a.threads, gpu=a.gpu, **cfg)
+ dec2 = tqenc.decode(packed2, t).reshape(Wr.shape)
+ What2 = tqenc.unrotate(dec2, sg) if had else dec2
+ rec["act_diag_weighted_encode"] = float(np.sum(((W - What2).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))
+ rec["mse_weighted_encode"] = float(np.sum((W.astype(np.float64) - What2) ** 2) / en)
+ out.write(json.dumps(rec) + "\n")
+ out.flush()
+ print("L%-3d %-4s e%-3d %s %-5s mse %.4f (%.2f dB) act_iso %.4f act_syn %.4f %.1fs" % (
+ layer, role, e, rname, "had" if had else "plain", rec["mse"], -10 * np.log10(rec["mse"]),
+ rec["act_iso"], rec["act_syn"], sec), flush=True)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/pilot_summary.py b/tools/trellis/pilot_summary.py
new file mode 100644
index 000000000..76c3c80c9
--- /dev/null
+++ b/tools/trellis/pilot_summary.py
@@ -0,0 +1,42 @@
+"""pilot_summary.py - tables from the jsonl of pilot.py.
+ python3 tools/trellis/pilot_summary.py pilot1.jsonl
+"""
+import collections
+import json
+import math
+import sys
+
+rows = [json.loads(l) for l in open(sys.argv[1])]
+print("records:", len(rows), "layers:", sorted({r["layer"] for r in rows}), "experts:", sorted({r["expert"] for r in rows}),
+ "rows per matrix:", sorted({r["rows"] for r in rows}))
+print("activation columns are PLACEHOLDERS (iso gaussian / synthetic outlier channels), not calibration data\n")
+
+
+def db(x):
+ return -10 * math.log10(x)
+
+
+def table(keyf, title, cols=("mse", "act_iso", "act_syn")):
+ g = collections.defaultdict(list)
+ for r in rows:
+ g[keyf(r)].append(r)
+ print(title)
+ for k in sorted(g):
+ v = g[k]
+ print(" %-40s n=%-3d " % (k, len(v)) + " ".join("%s %.4f (%.2f dB)" % (c, sum(x[c] for x in v) / len(v), db(sum(x[c] for x in v) / len(v))) for c in cols))
+ print()
+
+
+rate_bpw = {"K4": 1.125, "K5": 1.375, "K6": 1.625, "K7": 1.875}
+table(lambda r: "%s %.3f bpw %s" % (r["rate"], rate_bpw[r["rate"]], "hadamard" if r["had"] else "plain "), "mean over everything")
+table(lambda r: "%s %s %s" % (r["role"], r["rate"], "had" if r["had"] else "plain"), "by projection")
+table(lambda r: "L%02d %s %s" % (r["layer"], r["rate"], "had" if r["had"] else "plain"), "by layer")
+# spread of the weight mse across experts for K5 plain
+v = [r["mse"] for r in rows if r["rate"] == "K5" and not r["had"]]
+print("K5 plain weight mse over all %d matrices: min %.4f max %.4f" % (len(v), min(v), max(v)))
+# hadamard gain on the synthetic outlier activations
+print("\nHadamard effect on act_syn (relative error, plain / hadamard), per rate")
+for rate in ("K4", "K5", "K6", "K7"):
+ a = [r["act_syn"] for r in rows if r["rate"] == rate and not r["had"]]
+ b = [r["act_syn"] for r in rows if r["rate"] == rate and r["had"]]
+ print(" %s plain %.4f had %.4f ratio %.3f" % (rate, sum(a) / len(a), sum(b) / len(b), (sum(a) / len(a)) / (sum(b) / len(b))))
diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cpp
index 67e1e9974..05cef88e7 100644
--- a/tools/trellis/tqenc.cpp
+++ b/tools/trellis/tqenc.cpp
@@ -2,6 +2,7 @@
#include "tqenc.hpp"
#include "ggml-common-tq.h"
+#include "../dsv41_lab/computed_codes.hpp"
#include <algorithm>
#include <atomic>
@@ -88,6 +89,7 @@ namespace {
struct Codebook {
int16_t tab[kStates * 4];
alignas(64) float soa[4][kStates];
+ double cb_rms = 1.072;
Codebook() {
for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {
const uint32_t x = s * 0x9e3779b1u;
@@ -100,6 +102,17 @@ struct Codebook {
soa[2 * h + j][s] = (float) q * (1.0f / 8192.0f);
}
}
+ // research hook (tools/dsv41_lab/computed_codes.hpp): TQE_CODE=1mad|3inst|3sum|3single replaces the table by a computed
+ // LUT-free code (the same shift trellis, the same search); cb_rms then is the code's own rms
+ if (tqcode::Kind k = tqcode::kind_of(std::getenv("TQE_CODE")); k != tqcode::kNone) {
+ double ss = 0;
+ for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {
+ float v[4];
+ tqcode::code4(k, s, v);
+ for (int j = 0; j < 4; ++j) { soa[j][s] = v[j]; ss += (double) v[j] * v[j]; tab[4 * s + j] = (int16_t) std::lround(std::max(-32767.0, std::min(32767.0, v[j] * 8192.0))); }
+ }
+ cb_rms = std::sqrt(ss / (4.0 * kStates));
+ }
}
};
@@ -424,7 +437,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,
if (amax == 0.0) { r.status = 1; return r; }
const double rms = std::sqrt(sumw2 / 128.0);
// initial scale: the codebook has unit-ish rms
- constexpr double kCbRms = 1.072;
+ const double kCbRms = cb().cb_rms;
static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};
Search se;