Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 cudalab: computed (LUT-free) codebooks for the 15-bit trellis (1mad, 3inst, 3sum, 3single), encoder hook TQE_CODE, LUT-free GEMV kernel, code SQNR tool, pilot scripts from f-dsv41-enc
Niko12216f703a6dbe72
diff --git a/tools/dsv41_lab/code_sqnr.cpp b/tools/dsv41_lab/code_sqnr.cppnew file mode 100644index 000000000..9193fc2e2--- /dev/null+++ b/tools/dsv41_lab/code_sqnr.cpp@@ -0,0 +1,34 @@+// tools/dsv41_lab/code_sqnr.cpp - weight SQNR of the trellis encoder on N(0,1) blocks for a codebook chosen by TQE_CODE+// (unset = the shipped table). TQE_CODE=3sum ./code_sqnr [type 224] [blocks 16384] [thorough 0|1]+// Rate-distortion limit of a Gaussian at K bits per 4 weights: 6.02 K / 4 dB.+#include "../trellis/tqenc.hpp"++#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <random>+#include <vector>++int main(int argc, char** argv) {+ const int type = argc > 1 ? std::atoi(argv[1]) : 224;+ const int64_t nb = argc > 2 ? std::atoll(argv[2]) : 16384;+ const bool thorough = argc > 3 && std::atoi(argv[3]);+ const int K = tqenc::type_kbits(type);+ std::mt19937_64 rng(12345);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::vector<float> w((size_t) nb * 128);+ for (auto& v : w) v = nd(rng);+ std::vector<uint8_t> enc((size_t) nb * tqenc::block_bytes(type));+ tqenc::Params p;+ if (thorough) { p.iters = 3; p.ncand = 3; p.scale_tries = 3; }+ int64_t nonfinite = 0, zero = 0;+ tqenc::encode_blocks(type, w.data(), nullptr, nb, enc.data(), p, 0, nullptr, &nonfinite, &zero);+ std::vector<float> dec((size_t) nb * 128);+ tqenc::decode_blocks(type, enc.data(), nb, dec.data());+ double sq = 0, ee = 0;+ for (size_t i = 0; i < w.size(); ++i) { sq += (double) w[i] * w[i]; ee += (double) (dec[i] - w[i]) * (dec[i] - w[i]); }+ const double sqnr = 10 * std::log10(sq / ee), lim = 6.0206 * K / 4.0;+ std::printf("TQE_CODE=%-8s type %d K=%d %s: SQNR %.3f dB, limit %.3f dB, gap %.3f dB (rel MSE %.4f)\n", std::getenv("TQE_CODE") ? std::getenv("TQE_CODE") : "table", type, K,+ thorough ? "thorough" : "fast", sqnr, lim, lim - sqnr, ee / sq);+ return 0;+}diff --git a/tools/dsv41_lab/computed_codes.hpp b/tools/dsv41_lab/computed_codes.hppnew file mode 100644index 000000000..2100ef401--- /dev/null+++ b/tools/dsv41_lab/computed_codes.hpp@@ -0,0 +1,84 @@+// tools/dsv41_lab/computed_codes.hpp - LUT-free ("computed") codebooks for the 15-bit-state, 4-weights-a-step trellis, after+// QTIP (Tseng et al., Cornell-RelaxML/qtip lib/codebook/bitshift.py): 1MAD and 3INST decode a state with a 32-bit LCG and a few+// bit tricks instead of a table. QTIP emits one weight per LCG word (V = 1); a step of our format has four weights, so every+// variant here uses 2 to 4 LCG words per state. Host-side definition (the encoder takes it through the TQE_CODE environment+// variable, tools/trellis/tqenc.cpp) and the CUDA kernels use the same constants (tqk_gemv_c.cuh).+#pragma once+#include <cmath>+#include <cstdint>+#include <cstdlib>+#include <cstring>++namespace tqcode {++// LCG words of a state: x_i = s * kA[i] + kB[i]+inline constexpr uint32_t kA[4] = {89226354u, 0x9E3779B1u, 0x85EBCA6Bu, 0xC2B2AE35u};+inline constexpr uint32_t kB[4] = {64248484u, 0x7F4A7C15u, 0x165667B1u, 0x27D4EB2Fu};+inline constexpr uint32_t kMask3 = 0x8FFF8FFFu, kXor3 = 0x3B603B60u; // QTIP 3INST: (x & 0x8FFF8FFF) ^ 0x3B603B60 = two fp16++inline float h2f(uint16_t h) {+ const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;+ const uint32_t e = (h >> 10) & 0x1fu, m = h & 0x3ffu;+ float f;+ if (e == 0) f = std::ldexp((float) m, -24);+ else if (e == 31) f = INFINITY;+ else f = std::ldexp(1.0f + m / 1024.0f, (int) e - 15);+ return sign ? -f : f;+}++// float -> fp16 -> float, round to nearest even (what HADD2 does to the exact sum of two fp16 values)+inline float rn16(float x) {+ if (x == 0.0f) return x;+ int e;+ std::frexp(x, &e); // |x| in [2^(e-1), 2^e)+ const int q = (e - 1 < -14 ? -14 : e - 1) - 10;+ const float quantum = std::ldexp(1.0f, q);+ return std::nearbyint(x / quantum) * quantum;+}++inline uint32_t word3(uint32_t s, int i) { return ((s * kA[i] + kB[i]) & kMask3) ^ kXor3; }++enum Kind { kNone = 0, k1mad, k3inst, k3sum, k3single };++inline Kind kind_of(const char* n) {+ if (!n || !*n) return kNone;+ if (!std::strcmp(n, "1mad")) return k1mad;+ if (!std::strcmp(n, "3inst")) return k3inst;+ if (!std::strcmp(n, "3sum")) return k3sum;+ if (!std::strcmp(n, "3single")) return k3single;+ return kNone;+}++// the four (unnormalised) weights of state s+inline void code4(Kind k, uint32_t s, float out[4]) {+ s &= 0x7fffu;+ switch (k) {+ case k1mad:+ for (int i = 0; i < 4; ++i) {+ const uint32_t x = s * kA[i] + kB[i];+ const int y = (int) (x & 255) + (int) ((x >> 8) & 255) + (int) ((x >> 16) & 255) + (int) (x >> 24);+ out[i] = (float) (y - 510) / 147.800537109375f;+ }+ break;+ case k3inst:+ for (int i = 0; i < 4; ++i) { const uint32_t r = word3(s, i); out[i] = rn16(h2f((uint16_t) (r >> 16)) + h2f((uint16_t) (r & 0xffffu))); }+ break;+ case k3sum: {+ const uint32_t r0 = word3(s, 0), r1 = word3(s, 1), r2 = word3(s, 2), r3 = word3(s, 3);+ out[0] = rn16(h2f((uint16_t) (r0 & 0xffffu)) + h2f((uint16_t) (r1 & 0xffffu)));+ out[1] = rn16(h2f((uint16_t) (r0 >> 16)) + h2f((uint16_t) (r1 >> 16)));+ out[2] = rn16(h2f((uint16_t) (r2 & 0xffffu)) + h2f((uint16_t) (r3 & 0xffffu)));+ out[3] = rn16(h2f((uint16_t) (r2 >> 16)) + h2f((uint16_t) (r3 >> 16)));+ break;+ }+ default: {+ const uint32_t r0 = word3(s, 0), r1 = word3(s, 1);+ out[0] = h2f((uint16_t) (r0 & 0xffffu));+ out[1] = h2f((uint16_t) (r0 >> 16));+ out[2] = h2f((uint16_t) (r1 & 0xffffu));+ out[3] = h2f((uint16_t) (r1 >> 16));+ }+ }+}++} // namespace tqcodediff --git a/tools/dsv41_lab/tqk_gemv_c.cuh b/tools/dsv41_lab/tqk_gemv_c.cuhnew file mode 100644index 000000000..d5ea91eaa--- /dev/null+++ b/tools/dsv41_lab/tqk_gemv_c.cuh@@ -0,0 +1,109 @@+// tools/dsv41_lab/tqk_gemv_c.cuh - v3 with a COMPUTED codebook instead of the 2048-entry table (LUT-free): the step's four weights+// come from 2 or 4 LCG words, x_i = s * A_i + B_i, w = (x_i & 0x8FFF8FFF) ^ 0x3B603B60 read as two fp16 (QTIP 3INST's trick;+// computed_codes.hpp). CODE 0 "3single": the two words are the A fragment (a0, a2) as they are: 2 IMAD + 2 LOP3 a step.+// CODE 1 "3sum": a0 = HADD2(w0, w1), a2 = HADD2(w2, w3): 4 IMAD + 4 LOP3 + 2 HADD2. No shared-memory table, no __syncthreads.+// Everything else (TQL layout, prepared B fragments, mma) is gemv_v3.+#pragma once+#include "tqk_gemv3.cuh"++namespace tqlab {++__device__ __forceinline__ uint32_t cword(uint32_t s, int i) {+ constexpr uint32_t A[4] = {89226354u, 0x9E3779B1u, 0x85EBCA6Bu, 0xC2B2AE35u};+ constexpr uint32_t B[4] = {64248484u, 0x7F4A7C15u, 0x165667B1u, 0x27D4EB2Fu};+ return ((s * A[i] + B[i]) & 0x8FFF8FFFu) ^ 0x3B603B60u;+}+__device__ __forceinline__ uint32_t hadd2_u(uint32_t a, uint32_t b) {+ __half2 x = *(__half2*) &a, y = *(__half2*) &b;+ const __half2 r = __hadd2(x, y);+ return *(const uint32_t*) &r;+}++template <int K, int NB, int T, int EXACT, int CODE>+__global__ void __launch_bounds__(128) gemv_v3c(const Job* __restrict__ jobs, const uint32_t* __restrict__ lut_unused, int in_unused) {+ const Job job = jobs[blockIdx.y];+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, q = lane & 3;+ const int tile0 = (blockIdx.x * (blockDim.x >> 5) + warp) * T;+ if (tile0 * 16 >= job.rows) return;+ const int nblk = job.in / 128, NP = nblk / 2;+ float tot[T][4];+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 4; ++i) tot[t][i] = 0.f;+ const int src_lane = 4 * g + ((q + 3) & 3);+ for (int P = 0; P < NP; ++P) {+ uint32_t bh[2][8][2], bl[2][8][2];+#pragma unroll+ for (int bk = 0; bk < 2; ++bk) {+ if (g < NB) {+ const uint4* xp = ((const uint4*) job.x) + (((size_t) (2 * P + bk) * 8) * NB + g) * 4 + q;+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ const uint4 v = __ldg(xp + (size_t) m * NB * 4);+ bh[bk][m][0] = v.x;+ bh[bk][m][1] = v.y;+ if (EXACT) { bl[bk][m][0] = v.z; bl[bk][m][1] = v.w; }+ }+ } else {+#pragma unroll+ for (int m = 0; m < 8; ++m) { bh[bk][m][0] = bh[bk][m][1] = bl[bk][m][0] = bl[bk][m][1] = 0u; }+ }+ }+#pragma unroll+ for (int t = 0; t < T; ++t) {+ const size_t base = (size_t) (tile0 + t) * NP + P;+ uint32_t r[K];+#pragma unroll+ for (int j = 0; j < K; ++j) r[j] = __ldcs(((const uint32_t*) job.w) + (base * K + j) * 32 + lane);+ const uint64_t dq = __ldcs(job.sc + base * 8 + g);+ uint64_t win[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int a = (8 * K * i) >> 5, sh = (8 * K * i) & 31;+ const uint32_t w0 = r[a];+ const uint32_t w1 = a + 1 < K ? r[a + 1 < K ? a + 1 : 0] : 0u;+ const uint32_t w2 = a + 2 < K ? r[a + 2 < K ? a + 2 : 0] : 0u;+ const uint32_t lo = __funnelshift_r(w0, w1, sh), hi = __funnelshift_r(w1, w2, sh);+ const uint64_t ch = (((uint64_t) hi << 32) | lo) & ((1ull << (8 * K)) - 1ull);+ const uint32_t fw = __shfl_sync(0xffffffffu, lo, src_lane);+ win[i] = ch | ((uint64_t) (fw & ((1u << (15 - K)) - 1u)) << (8 * K));+ }+#pragma unroll+ for (int bk = 0; bk < 2; ++bk) {+ float acc[4] = {0.f, 0.f, 0.f, 0.f};+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ uint32_t a0, a1, a2, a3; // (row g, slots 2q..), (row g+8, ..), (row g, slots 2q+8..), (row g+8, ..)+ {+ const uint32_t s = (uint32_t) (win[2 * bk + 0] >> ((7 - m) * K)) & kMask;+ if (CODE == 0) { a0 = cword(s, 0); a2 = cword(s, 1); }+ else { a0 = hadd2_u(cword(s, 0), cword(s, 1)); a2 = hadd2_u(cword(s, 2), cword(s, 3)); }+ }+ {+ const uint32_t s = (uint32_t) (win[2 * bk + 1] >> ((7 - m) * K)) & kMask;+ if (CODE == 0) { a1 = cword(s, 0); a3 = cword(s, 1); }+ else { a1 = hadd2_u(cword(s, 0), cword(s, 1)); a3 = hadd2_u(cword(s, 2), cword(s, 3)); }+ }+ mma16816(acc, a0, a1, a2, a3, bh[bk][m][0], bh[bk][m][1]);+ if (EXACT) mma16816(acc, a0, a1, a2, a3, bl[bk][m][0], bl[bk][m][1]);+ }+ const float d0 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 0)))));+ const float d1 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 1)))));+ tot[t][0] += d0 * acc[0];+ tot[t][1] += d0 * acc[1];+ tot[t][2] += d1 * acc[2];+ tot[t][3] += d1 * acc[3];+ }+ }+ }+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int col = 2 * q + (i & 1), row = (tile0 + t) * 16 + g + 8 * (i >> 1);+ if (col < NB) job.y[(size_t) col * job.ldy + row] = tot[t][i];+ }+}++} // namespace tqlabdiff --git a/tools/dsv41_lab/tqk_lab.cu b/tools/dsv41_lab/tqk_lab.cuindex d2938fa8e..9bbc1b2f8 100644--- a/tools/dsv41_lab/tqk_lab.cu+++ b/tools/dsv41_lab/tqk_lab.cu@@ -4,6 +4,7 @@// Bench: one decode token of a routed layer = top-6 experts x (gate, up: 2304 x 5120; down: 5120 x 2304).#include "tqk_gemv.cuh"#include "tqk_gemv3.cuh"+#include "tqk_gemv_c.cuh"#include "tqk_repack.hpp"#include "../trellis/tqenc.hpp"#include "strata/kernels/cpu/tq_cpu.hpp"@@ -55,7 +56,14 @@ template <int K, int NB> static KFn pick(int variant) {case 8: return {(const void*) gemv_v3<K, NB, 1, 1>, 128, 64, 1, 1};case 9: return {(const void*) gemv_v3<K, NB, 2, 1>, 128, 128, 1, 1};case 10: return {(const void*) gemv_v3<K, NB, 4, 1>, 128, 256, 1, 1};- default: return {(const void*) gemv_v3<K, NB, 2, 0>, 128, 128, 1, 1};+ case 11: return {(const void*) gemv_v3<K, NB, 2, 0>, 128, 128, 1, 1};+ case 12: return {(const void*) gemv_v3c<K, NB, 4, 1, 0>, 128, 256, 1, 1};+ case 13: return {(const void*) gemv_v3c<K, NB, 4, 1, 1>, 128, 256, 1, 1};+ case 14: return {(const void*) gemv_v3c<K, NB, 2, 1, 0>, 128, 128, 1, 1};+ case 15: return {(const void*) gemv_v3c<K, NB, 2, 1, 1>, 128, 128, 1, 1};+ case 16: return {(const void*) gemv_v3c<K, NB, 2, 0, 0>, 128, 128, 1, 1};+ case 17: return {(const void*) gemv_v3c<K, NB, 4, 0, 0>, 128, 256, 1, 1};+ default: return {(const void*) gemv_v3c<K, NB, 1, 0, 0>, 128, 64, 1, 1};}}template <int K> static KFn pick_nb(int nb, int variant) {@@ -163,7 +171,7 @@ int main(int argc, char** argv) {double sq = 0, ee = 0; for (int64_t i = 0; i < nenc * 128; ++i) { const double r = got[(size_t) (nblocks - nenc) * 128 + i]; sq += (double) g[i] * g[i]; ee += (r - g[i]) * (r - g[i]); }std::printf("dequant: %zu / %zu weights differ from tqenc::decode_blocks (bit exact %s); encoded N(0,1) SQNR %.2f dB (encoder err %.3f)\n",bad, got.size(), bad ? "NO" : "yes", 10 * std::log10(sq / ee), err / (nenc * 128));- fails += bad != 0;+ fails += (bad != 0) && !std::getenv("TQE_CODE"); // (the table kernel is meaningless under a computed code)// ---- parity 2: GEMV vs double reference on the decoded weights, rows x in smallfor (int nb : NBs) {const int rows = 256, in = 768, nblk = in / 128;@@ -190,7 +198,7 @@ int main(int argc, char** argv) {for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); } ref[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }std::vector<float> y = gpu_run(x);double worst = 0; for (size_t i = 0; i < y.size(); ++i) worst = std::max(worst, std::fabs(y[i] - ref[i]) / (refabs[i] + 1e-30));- const bool ok = worst < 2e-6;+ const bool ok = worst < 2e-6 || (variant >= 16 && variant <= 18 && worst < 1e-4);// (2) the CPU kernel of the repo (tq_cpu scalar and AVX2, Q8_0 activations): same weights; the GPU runs on the dequantised Q8_0 activationsstd::vector<uint8_t> q8((size_t) nb * (in / 32) * 34); std::vector<float> xq((size_t) nb * in);for (int c = 0; c < nb; ++c) for (int b = 0; b < in / 32; ++b) {@@ -212,7 +220,7 @@ int main(int argc, char** argv) {double w_gc = 0, w_cr = 0, w_ac = 0;for (size_t i = 0; i < yq.size(); ++i) { const double den = refabs[i] + 1e-30; w_gc = std::max(w_gc, std::fabs(yq[i] - ycpu_s[i]) / den); w_cr = std::max(w_cr, std::fabs(ycpu_s[i] - refq[i]) / den); if (have_avx2) w_ac = std::max(w_ac, std::fabs(ycpu_a[i] - ycpu_s[i]) / den); }std::printf("gemv nb=%d: GPU vs double ref %.2e %s | vs CPU kernel of the repo (Q8_0 acts): GPU-CPU %.2e, CPU-double %.2e, avx2-scalar %.2e (all / sum|w x|)\n", nb, worst, ok ? "ok" : "FAIL", w_gc, w_cr, w_ac);- fails += !ok || w_gc > 5e-6;+ fails += !ok || (w_gc > 5e-6 && !std::getenv("TQE_CODE"));cudaFree(mwd); if (kf.tql) { cudaFree(wd); }}cudaFree(dw); cudaFree(dout);diff --git a/tools/tqenc.py b/tools/tqenc.pynew file mode 100644index 000000000..a92feb7a6--- /dev/null+++ b/tools/tqenc.py@@ -0,0 +1,221 @@+"""tqenc.py - Python side of the trellis encoder (tools/trellis/tqenc.cpp, built to libtqenc.so by+`sh tools/trellis/build.sh lib`): encode / decode blocks of TQ2_T, TQK6, TQK7 and the experimental TQK5X / TQK4X, the+input-only block Hadamard of the DeepSeek plan, and error metrics.++Rows of a tensor are blocks of 128 consecutive weights along the INPUT dimension (the GGUF fastest axis), exactly as the+decoders read them. A tensor of shape [rows, n] (n a multiple of 128) is encoded row by row; the output has+rows * n/128 * block_bytes bytes.+"""+from __future__ import annotations++import ctypes+import os+import pathlib++import numpy as np++TQ2_T, TQK6, TQK7, TQK5X, TQK4X = 144, 145, 146, 224, 225+TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X"}+TYPE_BY_NAME = {v: k for k, v in TYPE_NAMES.items()}+KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4}+BLOCK = 128+++def block_bytes(t: int) -> int:+ return 2 + 4 * KBITS[t]+++def bpw(t: int) -> float:+ return block_bytes(t) * 8 / BLOCK+++_lib = None+++def lib() -> ctypes.CDLL:+ global _lib+ if _lib is None:+ cands = [os.environ.get("TQENC_LIB", ""),+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc.so"),+ "libtqenc.so"]+ last = None+ for c in cands:+ if not c:+ continue+ try:+ _lib = ctypes.CDLL(c)+ break+ except OSError as e:+ last = e+ if _lib is None:+ raise RuntimeError("libtqenc.so not found (sh tools/trellis/build.sh lib): %s" % last)+ _lib.tqe_encode.restype = ctypes.c_double+ _lib.tqe_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int,+ ctypes.c_void_p, ctypes.c_void_p]+ _lib.tqe_decode.restype = None+ _lib.tqe_decode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p]+ return _lib+++_hip = None+++def hip_lib() -> ctypes.CDLL:+ """libtqenc_hip.so (sh tools/trellis/build.sh hip; run under the box GPU locks, HIP_VISIBLE_DEVICES picks the card)."""+ global _hip+ if _hip is None:+ cands = [os.environ.get("TQENC_HIP_LIB", ""),+ str(pathlib.Path(__file__).resolve().parent.parent / "build-tqenc" / "libtqenc_hip.so")]+ last = None+ for c in cands:+ if not c:+ continue+ try:+ _hip = ctypes.CDLL(c)+ break+ except OSError as e:+ last = e+ if _hip is None:+ raise RuntimeError("libtqenc_hip.so not found: %s" % last)+ _hip.tqh_encode.restype = ctypes.c_double+ _hip.tqh_encode.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p,+ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p,+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]+ return _hip+++def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, ncand: int = 1, exact: int = 0,+ scale_tries: int = 1, threads: int = 0, gpu: bool = False):+ """w: float32 array whose last axis is a multiple of 128. Returns (packed uint8 [nblocks, block_bytes],+ per-block error float64 [nblocks], total error, stats dict). Error = sum h (w - decoded)^2 (h = 1 if None)."""+ w = np.ascontiguousarray(w, dtype=np.float32)+ if w.shape[-1] % BLOCK:+ raise ValueError("last axis %d is not a multiple of 128" % w.shape[-1])+ nb = w.size // BLOCK+ out = np.zeros((nb, block_bytes(t)), dtype=np.uint8)+ err = np.zeros(nb, dtype=np.float64)+ stats = np.zeros(2, dtype=np.int64)+ hp = None+ if h is not None:+ h = np.ascontiguousarray(h, dtype=np.float32)+ if h.shape != w.shape:+ raise ValueError("importance weights must have the shape of w")+ hp = h.ctypes.data+ if gpu:+ if exact or t == TQ2_T:+ raise ValueError("the GPU encoder has no exact search and no TQ2_T")+ tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,+ err.ctypes.data, stats.ctypes.data, None, 0)+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}+ tot = lib().tqe_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, exact, scale_tries, threads,+ err.ctypes.data, stats.ctypes.data)+ return out, err, tot, {"nonfinite_blocks": int(stats[0]), "zero_blocks": int(stats[1])}+++def decode(packed: np.ndarray, t: int) -> np.ndarray:+ """packed: uint8 array of whole blocks. Returns float32 [nblocks * 128] (d * codebook)."""+ packed = np.ascontiguousarray(packed, dtype=np.uint8)+ bb = block_bytes(t)+ if packed.size % bb:+ raise ValueError("%d bytes is not a whole number of %s blocks" % (packed.size, TYPE_NAMES[t]))+ nb = packed.size // bb+ out = np.zeros(nb * BLOCK, dtype=np.float32)+ lib().tqe_decode(t, packed.ctypes.data, nb, out.ctypes.data)+ return out+++# ---------------------------------------------------------------------------------------------------- Hadamard++def hadamard_matrix(n: int = BLOCK) -> np.ndarray:+ """The normalized Sylvester-Walsh-Hadamard matrix (entry (-1)^popcount(r & c) / sqrt(n)); symmetric, orthonormal."""+ h = np.array([[1.0]])+ while h.shape[0] < n:+ h = np.block([[h, h], [h, -h]])+ return (h / np.sqrt(n)).astype(np.float32)+++def make_signs(width: int, seed: int) -> np.ndarray:+ """A reproducible +1/-1 vector (the plan's D). The vector itself, not the seed, is what a file must store."""+ r = np.random.default_rng(seed)+ return (r.integers(0, 2, size=width) * 2 - 1).astype(np.float32)+++def pack_signs(signs: np.ndarray) -> bytes:+ return np.packbits((signs > 0).astype(np.uint8), bitorder="little").tobytes()+++def unpack_signs(b: bytes, width: int) -> np.ndarray:+ bits = np.unpackbits(np.frombuffer(b, dtype=np.uint8), bitorder="little")[:width]+ return bits.astype(np.float32) * 2 - 1+++def rotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:+ """The input transform U = H D on the last axis, per 128-block: Wrot = W U^T, i.e. each row w -> H (signs * w).+ The same function applied to an activation x gives U x."""+ w = np.asarray(w, dtype=np.float32)+ if signs is not None:+ w = w * signs+ shp = w.shape+ return (w.reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp) # H is symmetric+++def unrotate(w: np.ndarray, signs: np.ndarray | None) -> np.ndarray:+ """Inverse of rotate (H is its own inverse: orthonormal and symmetric; then the signs)."""+ shp = w.shape+ x = (np.asarray(w, dtype=np.float32).reshape(-1, BLOCK) @ hadamard_matrix()).reshape(shp)+ return x * signs if signs is not None else x+++# ---------------------------------------------------------------------------------------------------- metrics++def sqnr_db(ref: np.ndarray, approx: np.ndarray) -> float:+ ref = np.asarray(ref, dtype=np.float64).ravel()+ e = np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2)+ return float("inf") if e == 0 else float(10 * np.log10(np.sum(ref ** 2) / e))+++def rel_mse(ref: np.ndarray, approx: np.ndarray) -> float:+ ref = np.asarray(ref, dtype=np.float64).ravel()+ return float(np.sum((ref - np.asarray(approx, dtype=np.float64).ravel()) ** 2) / max(np.sum(ref ** 2), 1e-300))+++# ---------------------------------------------------------------------------------------------------- .tqx container+# A small versioned file for ONE encoded matrix (research artifacts; not a GGUF). Little endian:+# "TQXP" | u32 version (1) | u32 type id | u32 rows | u32 cols | u32 flags (bit 0: input Hadamard, 128-blocks)+# | u32 n_sign_bytes | sign bits (1 = +1, LSB first, one per input column) | u32 hadamard block (128) | packed blocks+# An unknown version or type is refused before any data is read.+TQX_MAGIC = b"TQXP"+TQX_VERSION = 1+++def write_tqx(path, t: int, rows: int, cols: int, packed: np.ndarray, signs: np.ndarray | None) -> None:+ import struct+ sb = pack_signs(signs) if signs is not None else b""+ with open(path, "wb") as f:+ f.write(TQX_MAGIC + struct.pack("<IIIIII", TQX_VERSION, t, rows, cols, 1 if signs is not None else 0, len(sb)))+ f.write(sb)+ f.write(struct.pack("<I", BLOCK))+ f.write(np.ascontiguousarray(packed, dtype=np.uint8).tobytes())+++def read_tqx(path):+ import struct+ data = open(path, "rb").read()+ if data[:4] != TQX_MAGIC:+ raise ValueError("not a TQXP file")+ ver, t, rows, cols, flags, nsb = struct.unpack_from("<IIIIII", data, 4)+ if ver != TQX_VERSION:+ raise ValueError("unsupported TQXP version %d (this reader handles %d)" % (ver, TQX_VERSION))+ if t not in KBITS:+ raise ValueError("unknown trellis type %d" % t)+ at = 4 + 24+ signs = unpack_signs(data[at:at + nsb], cols) if flags & 1 else None+ at += nsb+ (blk,) = struct.unpack_from("<I", data, at)+ at += 4+ if blk != BLOCK:+ raise ValueError("unsupported Hadamard block %d" % blk)+ nb = rows * (cols // BLOCK)+ packed = np.frombuffer(data, dtype=np.uint8, count=nb * block_bytes(t), offset=at).reshape(nb, block_bytes(t))+ return t, rows, cols, packed, signsdiff --git a/tools/trellis/README.md b/tools/trellis/README.mdnew file mode 100644index 000000000..a76354aa7--- /dev/null+++ b/tools/trellis/README.md@@ -0,0 +1,34 @@+# Trellis encoder (research, branch f-dsv41-enc)++An encoder for the trellis expert formats that Strata already decodes (TQ2_T, TQK6, TQK7 of agentionai's llama.cpp fork)+and for two experimental lower rates, TQK5X (22 bytes per 128 weights, 1.375 bpw) and TQK4X (18 bytes, 1.125 bpw).+Written for the DeepSeek-V4.1-Flash compression plan (gate 2 and the start of gate 3). Nothing here changes an existing+type, layout or default; the engine's CPU decoder only gains K = 5 and 4 (ids 224 / 225, experimental version 1).++| file | what |+| --- | --- |+| `tqenc.hpp`, `tqenc.cpp` | the encoder: tail-biting Viterbi on the 2^15-state shift trellis, fp16 scale refinement, importance weights; a C API |+| `tqenc_test.cpp` | the tests (about 64,000 checks): streams and the circular wrap, Viterbi against a naive DP, planted solutions, bit-exact decode against `tq_states.hpp` and the scalar / AVX2 / AVX-512 kernels, special blocks, thread determinism, quality against the rate-distortion limit |+| `tqenc_hip.cpp`, `tqenc_hip_test.cpp` | the same search on a GPU (HIP, gfx1201): bit-identical blocks, about 20 M weights/s on one R9700 |+| `../tqenc.py` | Python wrapper (ctypes): `encode`, `decode`, the input-only block-128 Hadamard, signs, the `.tqx` container |+| `pilot.py`, `pilot_summary.py` | the block-quality pilot on a source GGUF (local files or HTTP range requests) |+| `gyro_reencode.py`, `gyro_vs_original.py`, `gyro_error_profile.py`, `hf_slice.py` | checks against agentionai's Gyro files and the original Qwen weights |++Build and test (Linux x86-64, g++ with AVX2; the GPU part needs ROCm):++ sh tools/trellis/build.sh test && build-tqenc/tqenc_test # --quick skips the quality table+ sh tools/trellis/build.sh lib && python3 tools/trellis/test_tqenc_py.py+ sh tools/trellis/build.sh hip && HIP_VISIBLE_DEVICES=3 build-tqenc/tqenc_hip_test++`build.sh hip` compiles `tqenc.cpp` with g++ on purpose: under hipcc, `ggml-common-tq.h` turns its codebook into a+`__device__` variable and the host side reads zeros.++## Format facts the encoder relies on (checked against the decoder, not the comments)++* A block is `{fp16 d; 4K bytes}` for 128 weights: 32 steps, step t gives weights 4t..4t+3.+* State of step t: 15 bits. TQK: the 16-bit window of the stream at bit `(31 - t) K` (LSB first, circular modulo 32 K bits), masked+ to 15 bits. TQ2_T: `(qs[(t+31)%32] << 8 | qs[t]) & 0x7fff`.+* Weights of a step: `x = state * 0x9e3779b1`, the two codebook points `lut[x >> 21]` and `lut[(x >> 10) & 2047]`, times d. The+ decoder's table is `round(fp16 * 8192)`; the encoder uses that table, so its reconstruction is the decoder's, bit for bit.+* A legal stream is any byte string (every state is a window of the same circular stream); tail-biting only matters to the search.+* K5 and K4 are the TQK rule with K = 5 and 4. Nothing else changes (hash, codebook, mask, bit order, scale).diff --git a/tools/trellis/build.sh b/tools/trellis/build.shnew file mode 100644index 000000000..45ab53842--- /dev/null+++ b/tools/trellis/build.sh@@ -0,0 +1,29 @@+#!/bin/sh+# tools/trellis/build.sh [test|lib|bench] - builds the trellis encoder standalone (g++, Linux x86-64).+# Run from the repository root. Output in ${TQ_OUT:-build-tqenc}.+set -e+OUT=${TQ_OUT:-build-tqenc}+mkdir -p "$OUT"+CXX=${CXX:-g++}+INC="-Itools/trellis -Iinclude -Isrc/kernels/cpu -Ithird_party/ggml"+OPT="-O3 -std=c++17 -mavx2 -mfma -mf16c -fno-fast-math -pthread"+case "${1:-test}" in+lib)+ $CXX $OPT -fPIC -shared $INC tools/trellis/tqenc.cpp -o "$OUT/libtqenc.so" ;;+hip)+ # needs the ROCm of /data/dsv41/rv: export ROCM_PATH=/data/dsv41/rv/lib/python3.14/site-packages/_rocm_sdk_devel PATH=/data/dsv41/rv/bin:$PATH+ # the host part of the encoder (tqenc.cpp) is built by g++: under hipcc ggml-common-tq.h's table becomes a __device__ variable+ g++ $OPT -fPIC $INC -c tools/trellis/tqenc.cpp -o "$OUT/tqenc_host.o"+ HF="-O3 --offload-arch=gfx1201 -mcumode -std=c++17 -mavx2 -mfma -mf16c"+ hipcc $HF -fPIC $INC -c tools/trellis/tqenc_hip.cpp -o "$OUT/tqenc_hip.o"+ hipcc $HF $INC -c tools/trellis/tqenc_hip_test.cpp -o "$OUT/tqenc_hip_test.o"+ hipcc --offload-arch=gfx1201 "$OUT/tqenc_hip_test.o" "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/tqenc_hip_test"+ hipcc --offload-arch=gfx1201 -shared "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/libtqenc_hip.so" ;;+test)+ $CXX $OPT $INC -c src/kernels/cpu/tq_cpu.cpp -o "$OUT/tq_cpu.o"+ $CXX $OPT $INC -c src/kernels/cpu/tq_avx2.cpp -o "$OUT/tq_avx2.o"+ $CXX -O3 -std=c++17 -mavx512f -mavx512bw -mavx512vl -mavx512vbmi -mavx512vnni -mfma -mf16c -mavx2 $INC -c src/kernels/cpu/tq_avx512.cpp -o "$OUT/tq_avx512.o"+ $CXX $OPT $INC tools/trellis/tqenc_test.cpp tools/trellis/tqenc.cpp "$OUT/tq_cpu.o" "$OUT/tq_avx2.o" "$OUT/tq_avx512.o" -o "$OUT/tqenc_test"+ ;;+esac+echo built "$OUT"diff --git a/tools/trellis/hf_slice.py b/tools/trellis/hf_slice.pynew file mode 100644index 000000000..6e6186601--- /dev/null+++ b/tools/trellis/hf_slice.py@@ -0,0 +1,94 @@+"""hf_slice.py - read single tensors (or slices of them) out of a remote safetensors repo with HTTP range requests, so a+quality test needs megabytes of a model, not the model. No token is read or printed (public repos only; set HF_TOKEN in+the environment for gated ones - it is sent as a header and never printed).++ from hf_slice import Repo+ r = Repo("Qwen/Qwen3.8-Flash-Next")+ w = r.tensor("model.language_model.layers.0.mlp.experts.gate_up_proj", rows=slice(0, 1)) # expert 0 only+"""+from __future__ import annotations++import concurrent.futures as cf+import json+import os+import struct+import urllib.request++import numpy as np++_DT = {"BF16": (2, None), "F16": (2, np.float16), "F32": (4, np.float32), "F8_E4M3": (1, None), "I8": (1, np.int8),+ "U8": (1, np.uint8), "I32": (4, np.int32), "I64": (8, np.int64)}+++def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:+ req = urllib.request.Request(url)+ tok = os.environ.get("HF_TOKEN")+ if tok:+ req.add_header("Authorization", "Bearer " + tok)+ if start is not None:+ req.add_header("Range", "bytes=%d-%d" % (start, end))+ with urllib.request.urlopen(req, timeout=120) as r:+ return r.read()+++def get_parallel(url: str, start: int, end: int, workers: int = 16, piece: int = 8 << 20) -> bytes:+ """bytes start..end (inclusive) of url, in `piece`-byte range requests on `workers` threads (one stream of a CDN+ object is slow, a dozen are not)."""+ spans = [(a, min(end, a + piece - 1)) for a in range(start, end + 1, piece)]+ if len(spans) == 1:+ return _get(url, start, end)++ def one(sp):+ for attempt in range(6):+ try:+ return _get(url, sp[0], sp[1])+ except Exception:+ if attempt == 5:+ raise+ return b""+ with cf.ThreadPoolExecutor(workers) as ex:+ return b"".join(ex.map(one, spans))+++def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:+ u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16+ return u.view(np.float32)+++class Repo:+ def __init__(self, repo: str, rev: str = "main"):+ self.repo, self.rev = repo, rev+ self.base = "https://huggingface.co/%s/resolve/%s/" % (repo, rev)+ self.map = json.loads(_get(self.base + "model.safetensors.index.json"))["weight_map"]+ self._hdr: dict[str, tuple[dict, int]] = {}++ def header(self, shard: str):+ if shard not in self._hdr:+ n = struct.unpack("<Q", _get(self.base + shard, 0, 7))[0]+ h = json.loads(_get(self.base + shard, 8, 8 + n - 1))+ self._hdr[shard] = (h, 8 + n)+ return self._hdr[shard]++ def info(self, name: str):+ shard = self.map[name]+ h, base = self.header(shard)+ return shard, h[name], base++ def tensor(self, name: str, lead: slice | int | None = None, workers: int = 16) -> np.ndarray:+ """The tensor `name` as float32. lead: an index or slice on the first axis (e.g. an expert), fetched alone."""+ shard, t, base = self.info(name)+ shape = t["shape"]+ item = _DT[t["dtype"]][0]+ s, e = t["data_offsets"]+ inner = int(np.prod(shape[1:])) * item if len(shape) > 1 else item+ if lead is None:+ i0, i1 = 0, shape[0]+ elif isinstance(lead, int):+ i0, i1 = lead, lead + 1+ else:+ i0, i1 = lead.start or 0, lead.stop+ raw = get_parallel(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1, workers)+ out_shape = [i1 - i0] + shape[1:]+ if t["dtype"] == "BF16":+ return bf16_to_f32(raw).reshape(out_shape)+ return np.frombuffer(raw, dtype=_DT[t["dtype"]][1]).astype(np.float32).reshape(out_shape)diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.pynew file mode 100644index 000000000..9781251a8--- /dev/null+++ b/tools/trellis/pilot.py@@ -0,0 +1,228 @@+"""pilot.py - gate-3 block-quality pilot: a few experts from several layers of a source checkpoint, encoded at TQK4X / TQK5X /+TQK6 / TQK7 with and without the input-only block-128 Hadamard, and measured.++Sources: --gguf <shard1 or shard dir files...> (UD-Q8_K_XL: MXFP4 / Q8_0 / BF16 / F16 tensors, decoded exactly: the decoded+ source is the teacher, as the plan says) or --hf Qwen/... (range requests; test of the pipeline)++Metrics per (layer, role, expert, rate, transform), over a random row subset of the projection (--rows):+ mse relative weight MSE ||W - W^||^2 / ||W||^2 in the original basis (the Hadamard is orthonormal: the same in either basis)+ act_iso ||(W - W^) X||^2 / ||W X||^2 with X i.i.d. N(0,1) columns - PLACEHOLDER; in expectation equal to mse+ act_syn ||(W - W^) X||^2 / ||W X||^2 with X whose input channels have heavy-tailed scales (a few outlier channels,+ log-normal rest) - SYNTHETIC PLACEHOLDER for real calibration activations, labelled as such in every output+Real activation-weighted numbers need calibration activations from the reference runtime (not done here).+"""+from __future__ import annotations++import argparse+import json+import mmap+import pathlib+import sys+import time++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import gguf_reader # noqa: E402+import tqenc # noqa: E402++MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)+RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7}+CFGS = {+ "fast": dict(iters=0, ncand=1, scale_tries=1),+ "default": dict(iters=1, ncand=1, scale_tries=1),+ "rot3": dict(iters=3, ncand=3, scale_tries=1),+}+++def decode_mxfp4(raw: np.ndarray, n_in: int) -> np.ndarray:+ """raw: [rows * n_in/32 * 17] bytes -> float32 [rows, n_in]. ggml block_mxfp4: e8m0 scale, 16 bytes, low nibble =+ element j, high nibble = element j + 16, value = kvalues[nibble] * 2^(e - 128) (the e8m0 half scale)."""+ b = raw.reshape(-1, 17)+ e = b[:, 0].astype(np.int32)+ d = np.where(e == 0, np.float32(2.0 ** -127), np.ldexp(np.float32(1.0), e - 128)).astype(np.float32)+ q = b[:, 1:]+ vals = np.concatenate([MXFP4_LUT[q & 15], MXFP4_LUT[q >> 4]], axis=1) * d[:, None]+ return vals.reshape(-1, n_in)+++def decode_q8_0(raw: np.ndarray, n_in: int) -> np.ndarray:+ b = raw.reshape(-1, 34)+ d = b[:, :2].copy().view(np.float16).astype(np.float32)+ q = b[:, 2:].copy().view(np.int8).astype(np.float32)+ return (q * d).reshape(-1, n_in)+++class Source:+ """Yields float32 [rows, n_in] row subsets of one expert's projection. Local GGUF files (mmap) or, with+ remote=(repo, rev, [shard paths]), the same files read by HTTP range requests (headers and the few experts needed)."""++ def __init__(self, gguf_paths=None, remote=None, tmpdir="/tmp"):+ self.tensors = {}+ if gguf_paths:+ for p in gguf_paths:+ g = gguf_reader.GGUFFile(p)+ fh = open(p, "rb")+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)+ for t in g.tensors:+ self.tensors[t.name] = (g, (lambda off, n, mm=mm: np.frombuffer(mm, dtype=np.uint8, count=n, offset=off)), t)+ if remote:+ import urllib.request+ repo, rev, shards = remote+ for sh in shards:+ url = "https://huggingface.co/%s/resolve/%s/%s" % (repo, rev, sh)++ def rd(off, n, url=url):+ req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (off, off + n - 1)})+ with urllib.request.urlopen(req, timeout=300) as r:+ return np.frombuffer(r.read(), dtype=np.uint8)+ hdr = rd(0, 6 << 20)+ tp = pathlib.Path(tmpdir) / ("hdr_%s" % pathlib.Path(sh).name)+ tp.write_bytes(hdr.tobytes())+ g = gguf_reader.GGUFFile(tp)+ for t in g.tensors:+ self.tensors[t.name] = (g, rd, t)++ def names(self, needle):+ return sorted(n for n in self.tensors if needle in n)++ def expert_rows(self, name: str, expert: int, rows: np.ndarray) -> np.ndarray:+ g, rd, ti = self.tensors[name]+ n_in, n_out = ti.shape[0], ti.shape[1]+ if ti.type_name == "MXFP4":+ rb = n_in // 32 * 17+ elif ti.type_name == "Q8_0":+ rb = n_in // 32 * 34+ elif ti.type_name in ("BF16", "F16"):+ rb = n_in * 2+ elif ti.type_name == "F32":+ rb = n_in * 4+ else:+ raise ValueError("%s: tensor type %s is not readable by this pilot" % (name, ti.type_name))+ base = g.data_start + ti.offset + expert * n_out * rb+ whole = rd(base, n_out * rb) # one expert's projection (6.3 MB for MXFP4 2304 x 5120), then the row subset+ out = np.empty((len(rows), n_in), dtype=np.float32)+ for k, r in enumerate(rows):+ raw = whole[int(r) * rb:(int(r) + 1) * rb]+ if ti.type_name == "MXFP4":+ out[k] = decode_mxfp4(raw, n_in)[0]+ elif ti.type_name == "Q8_0":+ out[k] = decode_q8_0(raw, n_in)[0]+ elif ti.type_name == "BF16":+ out[k] = (raw.view(np.uint16).astype(np.uint32) << 16).view(np.float32)+ elif ti.type_name == "F16":+ out[k] = raw.view(np.float16).astype(np.float32)+ else:+ out[k] = raw.view(np.float32)+ return out+++def synth_x(n_in: int, T: int, seed: int) -> np.ndarray:+ """SYNTHETIC activations: channel scale exp(N(0, 0.5)), 1% of the channels x (10..40)."""+ r = np.random.default_rng(seed)+ s = np.exp(0.5 * r.standard_normal(n_in)).astype(np.float32)+ out = r.choice(n_in, size=max(1, n_in // 100), replace=False)+ s[out] *= r.uniform(10, 40, size=out.size).astype(np.float32)+ return (r.standard_normal((n_in, T)).astype(np.float32) * s[:, None])+++def act_err(W, What, X):+ D = (W - What).astype(np.float32) @ X+ Y = W.astype(np.float32) @ X+ return float(np.sum(D.astype(np.float64) ** 2) / max(np.sum(Y.astype(np.float64) ** 2), 1e-300))+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--gguf", nargs="+", default=None)+ ap.add_argument("--remote", default=None, help="REPO@REV: read the shards named in --shards by HTTP range requests")+ ap.add_argument("--shards", nargs="+", default=[])+ ap.add_argument("--layers", default="1,7,14,21,30,38")+ ap.add_argument("--experts", default="0,97,200,383")+ ap.add_argument("--roles", default="gate,up,down")+ ap.add_argument("--rates", default="K4,K5,K6,K7")+ ap.add_argument("--rows", type=int, default=256)+ ap.add_argument("--tokens", type=int, default=256, help="columns of the placeholder X")+ ap.add_argument("--cfg", default="default")+ ap.add_argument("--threads", type=int, default=0)+ ap.add_argument("--out", required=True, help="jsonl results")+ ap.add_argument("--seed", type=int, default=1)+ ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")+ ap.add_argument("--gpu", action="store_true", help="encode on the GPU (run under the box locks, HIP_VISIBLE_DEVICES)")+ ap.add_argument("--act-diag-dir", default=None,+ help="directory with L<layer>_<role>.npy = E[x^2] per input channel of that projection (calibration "+ "second moments, diagonal). Adds act_diag (the exact diagonal-Hessian error) and an importance-weighted "+ "encode (h = E[x^2]; in the Hadamard basis h = the block average of it)")+ a = ap.parse_args()++ if a.remote:+ repo, rev = a.remote.split("@")+ src = Source(remote=(repo, rev, a.shards))+ else:+ src = Source(a.gguf)+ rng = np.random.default_rng(a.seed)+ cfg = CFGS[a.cfg]+ sign_cache, x_cache = {}, {}+ out = open(a.out, "a")+ for layer in [int(x) for x in a.layers.split(",")]:+ for role in a.roles.split(","):+ name = a.pattern.format(L=layer, R=role)+ if name not in src.tensors:+ print("no tensor", name, "(have e.g. %s)" % src.names("exps")[:3])+ continue+ g, rd, ti = src.tensors[name]+ n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]+ for e in [int(x) for x in a.experts.split(",")]:+ if e >= n_exp:+ continue+ rows = np.sort(rng.choice(n_out, size=min(a.rows, n_out), replace=False))+ W = src.expert_rows(name, e, rows)+ en = float(np.sum(W.astype(np.float64) ** 2))+ if n_in not in sign_cache:+ sign_cache[n_in] = tqenc.make_signs(n_in, 1000 + n_in)+ x_cache[n_in] = (np.random.default_rng(7).standard_normal((n_in, a.tokens)).astype(np.float32),+ synth_x(n_in, a.tokens, 9))+ Xi, Xs = x_cache[n_in]+ diag = None+ if a.act_diag_dir:+ dp = pathlib.Path(a.act_diag_dir) / ("L%d_%s.npy" % (layer, role))+ if dp.exists():+ diag = np.load(dp).astype(np.float64)+ assert diag.shape == (n_in,), (dp, diag.shape)+ for had in (False, True):+ sg = sign_cache[n_in] if had else None+ Wr = tqenc.rotate(W, sg) if had else W+ for rname in a.rates.split(","):+ t = RATES[rname]+ t0 = time.time()+ packed, err, tot, st = tqenc.encode(Wr, t, threads=a.threads, gpu=a.gpu, **cfg)+ sec = time.time() - t0+ dec = tqenc.decode(packed, t).reshape(Wr.shape)+ What = tqenc.unrotate(dec, sg) if had else dec+ rec = dict(layer=layer, role=role, expert=e, rate=rname, had=had, n_in=n_in, rows=len(rows),+ mse=float(np.sum((W.astype(np.float64) - What) ** 2) / en),+ act_iso=act_err(W, What, Xi), act_syn=act_err(W, What, Xs),+ bpw=tqenc.bpw(t), seconds=sec, zero_blocks=st["zero_blocks"], cfg=a.cfg,+ x_kind="placeholder: iso gaussian / synthetic outlier channels")+ if diag is not None:+ hv = np.maximum(diag, 1e-30).astype(np.float32)+ rec["act_diag"] = float(np.sum(((W - What).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))+ hb = hv if not had else np.repeat(hv.reshape(-1, 128).mean(axis=1), 128)+ hw = np.broadcast_to(hb, Wr.shape).astype(np.float32)+ packed2, _, _, _ = tqenc.encode(Wr, t, h=hw, threads=a.threads, gpu=a.gpu, **cfg)+ dec2 = tqenc.decode(packed2, t).reshape(Wr.shape)+ What2 = tqenc.unrotate(dec2, sg) if had else dec2+ rec["act_diag_weighted_encode"] = float(np.sum(((W - What2).astype(np.float64) ** 2) * hv) / np.sum((W.astype(np.float64) ** 2) * hv))+ rec["mse_weighted_encode"] = float(np.sum((W.astype(np.float64) - What2) ** 2) / en)+ out.write(json.dumps(rec) + "\n")+ out.flush()+ print("L%-3d %-4s e%-3d %s %-5s mse %.4f (%.2f dB) act_iso %.4f act_syn %.4f %.1fs" % (+ layer, role, e, rname, "had" if had else "plain", rec["mse"], -10 * np.log10(rec["mse"]),+ rec["act_iso"], rec["act_syn"], sec), flush=True)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/pilot_summary.py b/tools/trellis/pilot_summary.pynew file mode 100644index 000000000..76c3c80c9--- /dev/null+++ b/tools/trellis/pilot_summary.py@@ -0,0 +1,42 @@+"""pilot_summary.py - tables from the jsonl of pilot.py.+ python3 tools/trellis/pilot_summary.py pilot1.jsonl+"""+import collections+import json+import math+import sys++rows = [json.loads(l) for l in open(sys.argv[1])]+print("records:", len(rows), "layers:", sorted({r["layer"] for r in rows}), "experts:", sorted({r["expert"] for r in rows}),+ "rows per matrix:", sorted({r["rows"] for r in rows}))+print("activation columns are PLACEHOLDERS (iso gaussian / synthetic outlier channels), not calibration data\n")+++def db(x):+ return -10 * math.log10(x)+++def table(keyf, title, cols=("mse", "act_iso", "act_syn")):+ g = collections.defaultdict(list)+ for r in rows:+ g[keyf(r)].append(r)+ print(title)+ for k in sorted(g):+ v = g[k]+ print(" %-40s n=%-3d " % (k, len(v)) + " ".join("%s %.4f (%.2f dB)" % (c, sum(x[c] for x in v) / len(v), db(sum(x[c] for x in v) / len(v))) for c in cols))+ print()+++rate_bpw = {"K4": 1.125, "K5": 1.375, "K6": 1.625, "K7": 1.875}+table(lambda r: "%s %.3f bpw %s" % (r["rate"], rate_bpw[r["rate"]], "hadamard" if r["had"] else "plain "), "mean over everything")+table(lambda r: "%s %s %s" % (r["role"], r["rate"], "had" if r["had"] else "plain"), "by projection")+table(lambda r: "L%02d %s %s" % (r["layer"], r["rate"], "had" if r["had"] else "plain"), "by layer")+# spread of the weight mse across experts for K5 plain+v = [r["mse"] for r in rows if r["rate"] == "K5" and not r["had"]]+print("K5 plain weight mse over all %d matrices: min %.4f max %.4f" % (len(v), min(v), max(v)))+# hadamard gain on the synthetic outlier activations+print("\nHadamard effect on act_syn (relative error, plain / hadamard), per rate")+for rate in ("K4", "K5", "K6", "K7"):+ a = [r["act_syn"] for r in rows if r["rate"] == rate and not r["had"]]+ b = [r["act_syn"] for r in rows if r["rate"] == rate and r["had"]]+ print(" %s plain %.4f had %.4f ratio %.3f" % (rate, sum(a) / len(a), sum(b) / len(b), (sum(a) / len(a)) / (sum(b) / len(b))))diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cppindex 67e1e9974..05cef88e7 100644--- a/tools/trellis/tqenc.cpp+++ b/tools/trellis/tqenc.cpp@@ -2,6 +2,7 @@#include "tqenc.hpp"#include "ggml-common-tq.h"+#include "../dsv41_lab/computed_codes.hpp"#include <algorithm>#include <atomic>@@ -88,6 +89,7 @@ namespace {struct Codebook {int16_t tab[kStates * 4];alignas(64) float soa[4][kStates];+ double cb_rms = 1.072;Codebook() {for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {const uint32_t x = s * 0x9e3779b1u;@@ -100,6 +102,17 @@ struct Codebook {soa[2 * h + j][s] = (float) q * (1.0f / 8192.0f);}}+ // research hook (tools/dsv41_lab/computed_codes.hpp): TQE_CODE=1mad|3inst|3sum|3single replaces the table by a computed+ // LUT-free code (the same shift trellis, the same search); cb_rms then is the code's own rms+ if (tqcode::Kind k = tqcode::kind_of(std::getenv("TQE_CODE")); k != tqcode::kNone) {+ double ss = 0;+ for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {+ float v[4];+ tqcode::code4(k, s, v);+ for (int j = 0; j < 4; ++j) { soa[j][s] = v[j]; ss += (double) v[j] * v[j]; tab[4 * s + j] = (int16_t) std::lround(std::max(-32767.0, std::min(32767.0, v[j] * 8192.0))); }+ }+ cb_rms = std::sqrt(ss / (4.0 * kStates));+ }}};@@ -424,7 +437,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,if (amax == 0.0) { r.status = 1; return r; }const double rms = std::sqrt(sumw2 / 128.0);// initial scale: the codebook has unit-ish rms- constexpr double kCbRms = 1.072;+ const double kCbRms = cb().cb_rms;static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};Search se;