Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis encoder: tail-biting Viterbi for TQ2_T/TQK6/TQK7 plus experimental TQK5X/TQK4X (CPU decoder reads them)

Niko1221512e298e6e6d
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hpp
index 132cb99e1..a73e300e8 100644
--- a/include/strata/kernels/cpu/tq_cpu.hpp
+++ b/include/strata/kernels/cpu/tq_cpu.hpp
@@ -31,11 +31,16 @@ namespace strata::kernels::cpu {
enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };
-/// TQ2_T (144), TQK6 (145), TQK7 (146).
+/// EXPERIMENTAL lower-rate versions of the TQK rule (tools/trellis/tqenc.hpp, version 1; Strata-internal ids, not ggml
+/// types, not in any GGUF): TQK5X = 22 bytes / 128 weights (1.375 bpw), TQK4X = 18 bytes (1.125 bpw). Same hash,
+/// codebook, 15-bit mask, LSB-first circular stream and fp16 scale as TQK6/7 with K = 5 / 4.
+inline constexpr int kTqTypeK5X = 224;
+inline constexpr int kTqTypeK4X = 225;
+/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X.
bool tq_supported(int ggml_type) noexcept;
-/// Bits per step: 8, 6, 7 (0 for any other type).
+/// Bits per step: 8, 6, 7, 5, 4 (0 for any other type).
int tq_kbits(int ggml_type) noexcept;
-/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30).
+/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30, 22, 18).
constexpr int kTqBlock = 128;
inline size_t tq_block_bytes(int ggml_type) noexcept { return (size_t) (2 + 4 * tq_kbits(ggml_type)); }
/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).
diff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cpp
index 00918edf6..f03892510 100644
--- a/src/kernels/cpu/tq_avx2.cpp
+++ b/src/kernels/cpu/tq_avx2.cpp
@@ -81,6 +81,8 @@ void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct
float* const* out, int r0, int r1) {
if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
+ else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
}
diff --git a/src/kernels/cpu/tq_avx512.cpp b/src/kernels/cpu/tq_avx512.cpp
index b45118844..9031b8cf5 100644
--- a/src/kernels/cpu/tq_avx512.cpp
+++ b/src/kernels/cpu/tq_avx512.cpp
@@ -89,6 +89,8 @@ void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqA
float* const* out, int r0, int r1) {
if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
+ else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
}
diff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cpp
index 1240d7f74..ad7ea297d 100644
--- a/src/kernels/cpu/tq_cpu.cpp
+++ b/src/kernels/cpu/tq_cpu.cpp
@@ -89,9 +89,11 @@ const int16_t* tq_state_table() {
return g_table;
}
-bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146; }
+bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X; }
-int tq_kbits(int t) noexcept { return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : 0; }
+int tq_kbits(int t) noexcept {
+ return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : t == kTqTypeK5X ? 5 : t == kTqTypeK4X ? 4 : 0;
+}
bool tq_isa_ok(TqIsa isa) {
#if defined(STRATA_TQ_X86)
@@ -144,6 +146,8 @@ void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqA
float* const* out, int r0, int r1) {
if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 5) rows_scalar_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
+ else if (kb == 4) rows_scalar_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
else rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
}
diff --git a/tools/trellis/build.sh b/tools/trellis/build.sh
new file mode 100644
index 000000000..c85354033
--- /dev/null
+++ b/tools/trellis/build.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# tools/trellis/build.sh [test|lib|bench] - builds the trellis encoder standalone (g++, Linux x86-64).
+# Run from the repository root. Output in ${TQ_OUT:-build-tqenc}.
+set -e
+OUT=${TQ_OUT:-build-tqenc}
+mkdir -p "$OUT"
+CXX=${CXX:-g++}
+INC="-Itools/trellis -Iinclude -Isrc/kernels/cpu -Ithird_party/ggml"
+OPT="-O3 -std=c++17 -mavx2 -mfma -mf16c -fno-fast-math -pthread"
+case "${1:-test}" in
+lib)
+ $CXX $OPT -fPIC -shared $INC tools/trellis/tqenc.cpp -o "$OUT/libtqenc.so" ;;
+test)
+ $CXX $OPT $INC -c src/kernels/cpu/tq_cpu.cpp -o "$OUT/tq_cpu.o"
+ $CXX $OPT $INC -c src/kernels/cpu/tq_avx2.cpp -o "$OUT/tq_avx2.o"
+ $CXX -O3 -std=c++17 -mavx512f -mavx512bw -mavx512vl -mavx512vbmi -mavx512vnni -mfma -mf16c -mavx2 $INC -c src/kernels/cpu/tq_avx512.cpp -o "$OUT/tq_avx512.o"
+ $CXX $OPT $INC tools/trellis/tqenc_test.cpp tools/trellis/tqenc.cpp "$OUT/tq_cpu.o" "$OUT/tq_avx2.o" "$OUT/tq_avx512.o" -o "$OUT/tqenc_test"
+ ;;
+esac
+echo built "$OUT"
diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cpp
new file mode 100644
index 000000000..61aca400b
--- /dev/null
+++ b/tools/trellis/tqenc.cpp
@@ -0,0 +1,546 @@
+// tools/trellis/tqenc.cpp - see tqenc.hpp.
+#include "tqenc.hpp"
+
+#include "ggml-common-tq.h"
+
+#include <algorithm>
+#include <atomic>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#if defined(__AVX2__) && defined(__FMA__)
+#include <immintrin.h>
+#define TQENC_AVX2 1
+#endif
+
+namespace tqenc {
+namespace {
+
+constexpr float kInf = 1e30f;
+
+// ---------------------------------------------------------------------------------------------------------------
+// fp16
+
+} // namespace
+
+float half_to_float(uint16_t h) {
+ const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;
+ uint32_t exp = (h >> 10) & 0x1fu, man = h & 0x3ffu, bits;
+ if (exp == 0x1fu) {
+ bits = sign | 0x7f800000u | (man << 13);
+ } else if (exp != 0) {
+ bits = sign | ((exp + 112u) << 23) | (man << 13);
+ } else if (man == 0) {
+ bits = sign;
+ } else {
+ const float m = (float) man * (1.0f / 16777216.0f);
+ return sign ? -m : m;
+ }
+ float f;
+ std::memcpy(&f, &bits, 4);
+ return f;
+}
+
+uint16_t float_to_half(float f) {
+ uint32_t x;
+ std::memcpy(&x, &f, 4);
+ const uint32_t sign = (x >> 16) & 0x8000u;
+ const uint32_t ax = x & 0x7fffffffu;
+ if (ax >= 0x7f800000u) return (uint16_t) (sign | 0x7c00u | (ax > 0x7f800000u ? 0x200u : 0u)); // inf / nan
+ if (ax >= 0x477ff000u) return (uint16_t) (sign | 0x7c00u); // >= 65520 rounds to inf
+ if (ax < 0x33000001u) return (uint16_t) sign; // <= 2^-25: rounds to zero (ties to even)
+ int e = (int) (ax >> 23) - 127;
+ uint32_t man = (ax & 0x7fffffu) | 0x800000u;
+ if (e < -14) { // subnormal half
+ const int shift = -14 - e + 13; // total right shift of the 24-bit mantissa
+ const uint32_t half = 1u << (shift - 1), rest = man & ((1u << shift) - 1);
+ uint32_t q = man >> shift;
+ if (rest > half || (rest == half && (q & 1))) ++q;
+ return (uint16_t) (sign | q);
+ }
+ uint32_t q = ((uint32_t) (e + 15) << 10) | ((man >> 13) & 0x3ffu);
+ const uint32_t rest = man & 0x1fffu;
+ if (rest > 0x1000u || (rest == 0x1000u && (q & 1))) ++q; // a carry into the exponent is correct
+ return (uint16_t) (sign | q);
+}
+
+int type_kbits(int type) {
+ switch (type) {
+ case TQ2_T: return 8;
+ case TQK6: return 6;
+ case TQK7: return 7;
+ case TQK5X: return 5;
+ case TQK4X: return 4;
+ default: return 0;
+ }
+}
+
+namespace {
+
+// ---------------------------------------------------------------------------------------------------------------
+// codebook: the decoder's int16 table (round half away from zero of fp16 * 8192, as tq_cpu.cpp fill_table), SoA floats
+
+struct Codebook {
+ int16_t tab[kStates * 4];
+ alignas(64) float soa[4][kStates];
+ Codebook() {
+ for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {
+ const uint32_t x = s * 0x9e3779b1u;
+ const uint32_t pts[2] = {x >> 21, (x >> 10) & 2047u};
+ for (int h = 0; h < 2; ++h)
+ for (int j = 0; j < 2; ++j) {
+ const float v = half_to_float(tq_lut_f16[2 * pts[h] + j]) * 8192.0f;
+ const int16_t q = (int16_t) (v < 0.0f ? v - 0.5f : v + 0.5f);
+ tab[4 * s + 2 * h + j] = q;
+ soa[2 * h + j][s] = (float) q * (1.0f / 8192.0f);
+ }
+ }
+ }
+};
+
+const Codebook& cb() {
+ static const Codebook c;
+ return c;
+}
+
+struct Geo {
+ int K = 0;
+ bool t2 = false; // TQ2_T (byte stream, the other shift direction)
+ int G = 0; // predecessor groups
+};
+
+Geo geo_for(int type) {
+ Geo g;
+ g.K = type_kbits(type);
+ g.t2 = type == TQ2_T;
+ g.G = g.t2 ? 128 : (1 << (15 - g.K));
+ return g;
+}
+
+// group of a state as a successor / as a predecessor
+inline int gsucc(const Geo& g, uint32_t s) { return g.t2 ? (int) (s >> 8) : (int) (s & (uint32_t) (g.G - 1)); }
+inline int gpred(const Geo& g, uint32_t s) { return g.t2 ? (int) (s & 0x7f) : (int) (s >> g.K); }
+
+// weight index (0..127) of component j of processing position p
+inline int widx(const Geo& g, int p, int j) { return g.t2 ? 4 * p + j : 4 * (31 - p) + j; }
+
+struct Scratch {
+ alignas(64) float hist[32][kStates];
+ alignas(64) float m[2048];
+ alignas(64) float zero[kStates];
+};
+
+Scratch& scratch() {
+ thread_local std::unique_ptr<Scratch> s;
+ if (!s) {
+ s.reset(new Scratch);
+ std::memset(s->zero, 0, sizeof s->zero);
+ std::memset(s->m, 0, sizeof s->m);
+ }
+ return *s;
+}
+
+inline float min_range(const float* p, int n) {
+#if TQENC_AVX2
+ if (n >= 8) {
+ __m256 v = _mm256_loadu_ps(p);
+ for (int i = 8; i + 8 <= n; i += 8) v = _mm256_min_ps(v, _mm256_loadu_ps(p + i));
+ __m128 a = _mm_min_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));
+ a = _mm_min_ps(a, _mm_movehl_ps(a, a));
+ a = _mm_min_ss(a, _mm_shuffle_ps(a, a, 1));
+ return _mm_cvtss_f32(a);
+ }
+#endif
+ float m = p[0];
+ for (int i = 1; i < n; ++i) m = std::min(m, p[i]);
+ return m;
+}
+
+// out[i] = add[i] + sum_j h_j (t_j - cb_j[s0 + i])^2 for i in [0, n)
+inline void step_cost(float* out, const float* add, int s0, int n, const float* t, const float* h) {
+ const float *c0 = cb().soa[0] + s0, *c1 = cb().soa[1] + s0, *c2 = cb().soa[2] + s0, *c3 = cb().soa[3] + s0;
+ int i = 0;
+#if TQENC_AVX2
+ const __m256 t0 = _mm256_set1_ps(t[0]), t1 = _mm256_set1_ps(t[1]), t2 = _mm256_set1_ps(t[2]), t3 = _mm256_set1_ps(t[3]);
+ const __m256 h0 = _mm256_set1_ps(h[0]), h1 = _mm256_set1_ps(h[1]), h2 = _mm256_set1_ps(h[2]), h3 = _mm256_set1_ps(h[3]);
+ for (; i + 8 <= n; i += 8) {
+ const __m256 e0 = _mm256_sub_ps(t0, _mm256_loadu_ps(c0 + i)), e1 = _mm256_sub_ps(t1, _mm256_loadu_ps(c1 + i));
+ const __m256 e2 = _mm256_sub_ps(t2, _mm256_loadu_ps(c2 + i)), e3 = _mm256_sub_ps(t3, _mm256_loadu_ps(c3 + i));
+ __m256 a = _mm256_fmadd_ps(_mm256_mul_ps(h0, e0), e0, _mm256_loadu_ps(add + i));
+ a = _mm256_fmadd_ps(_mm256_mul_ps(h1, e1), e1, a);
+ a = _mm256_fmadd_ps(_mm256_mul_ps(h2, e2), e2, a);
+ a = _mm256_fmadd_ps(_mm256_mul_ps(h3, e3), e3, a);
+ _mm256_storeu_ps(out + i, a);
+ }
+#endif
+ for (; i < n; ++i) {
+ const float e0 = t[0] - c0[i], e1 = t[1] - c1[i], e2 = t[2] - c2[i], e3 = t[3] - c3[i];
+ out[i] = add[i] + h[0] * e0 * e0 + h[1] * e1 * e1 + h[2] * e2 * e2 + h[3] * e3 * e3;
+ }
+}
+
+// the broadcast variant: out[i] = a + cost(s0 + i)
+inline void step_cost_bc(float* out, float a, int s0, int n, const float* t, const float* h) {
+ alignas(32) float tmp[256];
+ for (int i = 0; i < n; ++i) tmp[i] = a;
+ step_cost(out, tmp, s0, n, t, h);
+}
+
+// T[p][4]: targets of processing position p (weights / d), H[p][4]: importance. rot: the sequence is processed from
+// position rot (cyclically); fixed_c >= 0: tail-biting with the successor group of the first state and the predecessor
+// group of the last state both == fixed_c. Writes the states by position into st[32]; returns the path cost.
+float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st) {
+ Scratch& sc = scratch();
+ const int K = g.K, G = g.G;
+ // q = 0
+ {
+ const int p = rot & 31;
+ step_cost(sc.hist[0], sc.zero, 0, kStates, T[p], H[p]);
+ if (fixed_c >= 0) {
+ float* h0 = sc.hist[0];
+ if (g.t2) { // gsucc(s) = s >> 8 == c: the 256 states [c<<8, c<<8 + 256) allowed
+ for (int s = 0; s < kStates; ++s)
+ if ((s >> 8) != fixed_c) h0[s] = kInf;
+ } else { // s & (G-1) == c
+ for (int s = 0; s < kStates; ++s)
+ if ((s & (G - 1)) != fixed_c) h0[s] = kInf;
+ }
+ }
+ }
+ for (int q = 1; q < 32; ++q) {
+ const float* prev = sc.hist[q - 1];
+ float* cur = sc.hist[q];
+ const int p = (rot + q) & 31;
+ float* m = sc.m;
+ if (g.t2) {
+ // m[a] = min over h of prev[a + 128 h]
+ for (int a = 0; a < 128; ++a) m[a] = prev[a];
+ for (int h = 1; h < 256; ++h) {
+ const float* row = prev + 128 * h;
+ int a = 0;
+#if TQENC_AVX2
+ for (; a + 8 <= 128; a += 8) _mm256_storeu_ps(m + a, _mm256_min_ps(_mm256_loadu_ps(m + a), _mm256_loadu_ps(row + a)));
+#endif
+ for (; a < 128; ++a) m[a] = std::min(m[a], row[a]);
+ }
+ for (int b = 0; b < 128; ++b) step_cost_bc(cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);
+ } else {
+ const int span = 1 << K;
+ for (int a = 0; a < G; ++a) m[a] = min_range(prev + (size_t) a * span, span);
+ for (int base = 0; base < kStates; base += G) step_cost(cur + base, m, base, G, T[p], H[p]);
+ }
+ }
+ // final state
+ const float* last = sc.hist[31];
+ int best = -1;
+ float bc = 3e38f;
+ if (fixed_c < 0) {
+ for (int s = 0; s < kStates; ++s)
+ if (last[s] < bc) { bc = last[s]; best = s; }
+ } else if (g.t2) {
+ for (int h = 0; h < 256; ++h) {
+ const int s = fixed_c + 128 * h;
+ if (last[s] < bc) { bc = last[s]; best = s; }
+ }
+ } else {
+ for (int i = 0; i < (1 << K); ++i) {
+ const int s = (fixed_c << K) + i;
+ if (last[s] < bc) { bc = last[s]; best = s; }
+ }
+ }
+ if (best < 0 || bc >= kInf * 0.5f) return kInf;
+ uint32_t s = (uint32_t) best;
+ st[(rot + 31) & 31] = (uint16_t) s;
+ for (int q = 31; q >= 1; --q) {
+ const int gv = gsucc(g, s);
+ const float* prev = sc.hist[q - 1];
+ int arg = -1;
+ float mv = 3e38f;
+ if (g.t2) {
+ for (int h = 0; h < 256; ++h) {
+ const int x = gv + 128 * h;
+ if (prev[x] < mv) { mv = prev[x]; arg = x; }
+ }
+ } else {
+ const int span = 1 << K;
+ for (int i = 0; i < span; ++i) {
+ const int x = (gv << K) + i;
+ if (prev[x] < mv) { mv = prev[x]; arg = x; }
+ }
+ }
+ s = (uint32_t) arg;
+ st[(rot + q - 1) & 31] = (uint16_t) s;
+ }
+ return bc;
+}
+
+// junction group of a free pass: gpred of the state at orig position 31 (== gsucc of the state at position 0)
+inline int junction_group(const Geo& g, const uint16_t* st) { return gpred(g, st[31]); }
+
+struct Search {
+ const Geo* g;
+ const Params* p;
+ float T[32][4];
+ float H[32][4];
+ uint16_t st[32];
+ int passes = 0;
+
+ // one tail-biting search at scale d (weights already divided: set_targets); returns the path cost in scaled units
+ float run() {
+ static const int rots[8] = {16, 8, 24, 12, 20, 4, 28, 0};
+ float best = kInf;
+ uint16_t tmp[32], free_st[32];
+ if (p->exact) {
+ for (int c = 0; c < g->G; ++c) {
+ const float v = viterbi(*g, T, H, 0, c, tmp);
+ ++passes;
+ if (v < best) { best = v; std::memcpy(st, tmp, sizeof tmp); }
+ }
+ return best;
+ }
+ int seen[8], ns = 0;
+ const int nc = std::max(1, std::min(p->ncand, 8));
+ for (int r = 0; r < nc; ++r) {
+ viterbi(*g, T, H, rots[r], -1, free_st);
+ ++passes;
+ const int c = junction_group(*g, free_st);
+ bool dup = false;
+ for (int i = 0; i < ns; ++i) dup |= seen[i] == c;
+ if (dup) continue;
+ seen[ns++] = c;
+ const float v = viterbi(*g, T, H, 0, c, tmp);
+ ++passes;
+ if (v < best) { best = v; std::memcpy(st, tmp, sizeof tmp); }
+ }
+ return best;
+ }
+};
+
+void pack_stream(const Geo& g, const uint16_t* st, uint8_t* qs) {
+ const int K = g.K;
+ std::memset(qs, 0, (size_t) 4 * K);
+ if (g.t2) {
+ for (int t = 0; t < 32; ++t) qs[t] = (uint8_t) (st[t] & 0xff);
+ return;
+ }
+ for (int u = 0; u < 32; ++u) {
+ const uint32_t x = st[u] & ((1u << K) - 1u);
+ for (int b = 0; b < K; ++b)
+ if ((x >> b) & 1u) qs[(u * K + b) >> 3] |= (uint8_t) (1u << ((u * K + b) & 7));
+ }
+}
+
+} // namespace
+
+float debug_viterbi(int type, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st) {
+ return viterbi(geo_for(type), T, H, rot, fixed_c, st);
+}
+int debug_groups(int type) { return geo_for(type).G; }
+int debug_widx(int type, int p, int j) { return widx(geo_for(type), p, j); }
+void debug_pack(int type, const uint16_t* st, uint8_t* qs) { pack_stream(geo_for(type), st, qs); }
+
+const float* codebook_state(uint32_t state) {
+ static thread_local float v[4];
+ for (int j = 0; j < 4; ++j) v[j] = cb().soa[j][state & 0x7fff];
+ return v;
+}
+
+const int16_t* codebook_table() { return cb().tab; }
+
+uint32_t block_state(int type, const uint8_t* qs, int t) {
+ const int K = type_kbits(type);
+ if (type == TQ2_T) return (((uint32_t) qs[(t + 31) & 31] << 8) | qs[t]) & 0x7fffu;
+ const int total = 32 * K;
+ uint32_t s = 0;
+ for (int i = 0; i < 15; ++i) {
+ const int b = ((31 - t) * K + i) % total;
+ s |= (uint32_t) ((qs[b >> 3] >> (b & 7)) & 1u) << i;
+ }
+ return s;
+}
+
+void decode_blocks(int type, const uint8_t* src, int64_t nblocks, float* out) {
+ const int K = type_kbits(type), bb = 2 + 4 * K;
+ for (int64_t b = 0; b < nblocks; ++b) {
+ const uint8_t* blk = src + (size_t) b * bb;
+ const uint16_t dh = (uint16_t) (blk[0] | (blk[1] << 8));
+ const float d = half_to_float(dh);
+ for (int t = 0; t < 32; ++t) {
+ const uint32_t s = block_state(type, blk + 2, t);
+ for (int j = 0; j < 4; ++j) out[b * 128 + 4 * t + j] = d * cb().soa[j][s];
+ }
+ }
+}
+
+BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst, const Params& prm) {
+ BlockResult r;
+ const Geo g = geo_for(type);
+ const int K = g.K, bb = 2 + 4 * K;
+ std::memset(dst, 0, (size_t) bb);
+ double sumw2 = 0, sumhw2 = 0, amax = 0;
+ for (int i = 0; i < 128; ++i) {
+ if (!std::isfinite(w[i]) || (h && !std::isfinite(h[i]))) { r.status = 2; r.err = NAN; return r; }
+ const double hi = h ? h[i] : 1.0;
+ sumw2 += (double) w[i] * w[i];
+ sumhw2 += hi * (double) w[i] * w[i];
+ amax = std::max(amax, (double) std::fabs(w[i]));
+ }
+ if (amax == 0.0) { r.status = 1; return r; }
+ const double rms = std::sqrt(sumw2 / 128.0);
+ // initial scale: the codebook has unit-ish rms
+ constexpr double kCbRms = 1.072;
+ static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};
+
+ Search se;
+ se.g = &g;
+ se.p = &prm;
+ for (int p = 0; p < 32; ++p)
+ for (int j = 0; j < 4; ++j) se.H[p][j] = h ? h[widx(g, p, j)] : 1.0f;
+
+ double best_err = 1e300;
+ uint16_t best_st[32] = {};
+ uint16_t best_d = 0;
+ bool have = false;
+
+ auto fit = [&](const uint16_t* st, uint16_t dh, double& err) {
+ // err of (st, dh): sum h (w - d cv)^2
+ const float d = half_to_float(dh);
+ double e = 0;
+ for (int p = 0; p < 32; ++p)
+ for (int j = 0; j < 4; ++j) {
+ const int wi = widx(g, p, j);
+ const float rec = d * cb().soa[j][st[p]];
+ const double df = (double) w[wi] - rec;
+ e += (h ? h[wi] : 1.0) * df * df;
+ }
+ err = e;
+ };
+
+ const int tries = prm.fixed_scale ? 1 : std::max(1, std::min(prm.scale_tries, 7));
+ for (int tr = 0; tr < tries; ++tr) {
+ double dd = rms / kCbRms * mult[tr];
+ uint16_t dh = float_to_half((float) std::min(dd, 65504.0));
+ if (dh == 0) dh = 1; // the smallest subnormal: weights below ~3e-8 are noise at this scale
+ if (prm.fixed_scale) dh = prm.fixed_scale;
+ for (int it = 0; it <= (prm.fixed_scale ? 0 : prm.iters); ++it) {
+ const float d = half_to_float(dh);
+ for (int p = 0; p < 32; ++p)
+ for (int j = 0; j < 4; ++j) se.T[p][j] = w[widx(g, p, j)] / d;
+ if (se.run() >= kInf * 0.5f) break;
+ double e;
+ fit(se.st, dh, e);
+ if (e < best_err) {
+ best_err = e;
+ best_d = dh;
+ std::memcpy(best_st, se.st, sizeof best_st);
+ have = true;
+ }
+ if (e == 0.0 || prm.fixed_scale) break;
+ // least-squares scale for this path
+ double num = 0, den = 0;
+ for (int p = 0; p < 32; ++p)
+ for (int j = 0; j < 4; ++j) {
+ const int wi = widx(g, p, j);
+ const double hi = h ? h[wi] : 1.0, c = cb().soa[j][se.st[p]];
+ num += hi * w[wi] * c;
+ den += hi * c * c;
+ }
+ if (den <= 0 || num <= 0) break;
+ uint16_t nh = float_to_half((float) std::min(num / den, 65504.0));
+ if (nh == 0 || nh == dh) break;
+ double en;
+ fit(se.st, nh, en);
+ if (en < best_err) {
+ best_err = en;
+ best_d = nh;
+ std::memcpy(best_st, se.st, sizeof best_st);
+ }
+ if (en >= e) break; // the scale step did not help this path: stop refining this start
+ dh = nh;
+ }
+ r.searches = se.passes;
+ }
+ if (!have || best_err >= sumhw2) { // nothing beats a zero block (weights below the fp16 scale resolution)
+ std::memset(dst, 0, (size_t) bb);
+ r.status = 1;
+ r.err = sumhw2;
+ return r;
+ }
+ dst[0] = (uint8_t) (best_d & 0xff);
+ dst[1] = (uint8_t) (best_d >> 8);
+ pack_stream(g, best_st, dst + 2);
+ r.err = best_err;
+ r.scale = half_to_float(best_d);
+ r.searches = se.passes;
+ return r;
+}
+
+double encode_blocks(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, const Params& p,
+ int nthreads, double* err_out, int64_t* nonfinite, int64_t* zero_blocks) {
+ const int bb = block_bytes(type);
+ if (nthreads <= 0) nthreads = (int) std::max(1u, std::thread::hardware_concurrency());
+ nthreads = (int) std::min<int64_t>(nthreads, std::max<int64_t>(1, nblocks));
+ std::atomic<int64_t> next{0}, nf{0}, zb{0};
+ std::vector<double> sums((size_t) nthreads, 0.0);
+ auto work = [&](int ti) {
+ double s = 0;
+ for (;;) {
+ const int64_t b0 = next.fetch_add(4);
+ if (b0 >= nblocks) break;
+ const int64_t b1 = std::min(nblocks, b0 + 4);
+ for (int64_t b = b0; b < b1; ++b) {
+ const BlockResult r = encode_block(type, w + b * 128, h ? h + b * 128 : nullptr, dst + (size_t) b * bb, p);
+ if (r.status == 2) { nf.fetch_add(1); }
+ else s += r.err;
+ if (r.status == 1) zb.fetch_add(1);
+ if (err_out) err_out[b] = r.err;
+ }
+ }
+ sums[ti] = s;
+ };
+ std::vector<std::thread> th;
+ for (int i = 1; i < nthreads; ++i) th.emplace_back(work, i);
+ work(0);
+ for (auto& t : th) t.join();
+ if (nonfinite) *nonfinite = nf.load();
+ if (zero_blocks) *zero_blocks = zb.load();
+ double tot = 0;
+ for (double s : sums) tot += s;
+ return tot;
+}
+
+} // namespace tqenc
+
+// ---------------------------------------------------------------------------------------------------------------
+// C API for the Python wrapper (tools/tqenc.py, ctypes)
+extern "C" {
+#if defined(_WIN32)
+#define TQE_EXPORT __declspec(dllexport)
+#else
+#define TQE_EXPORT __attribute__((visibility("default")))
+#endif
+TQE_EXPORT int tqe_block_bytes(int type) { return tqenc::block_bytes(type); }
+TQE_EXPORT int tqe_kbits(int type) { return tqenc::type_kbits(type); }
+TQE_EXPORT void tqe_decode(int type, const uint8_t* src, int64_t nblocks, float* out) {
+ tqenc::decode_blocks(type, src, nblocks, out);
+}
+// returns the summed block error; stats[0] = nonfinite blocks, stats[1] = zero blocks
+TQE_EXPORT double tqe_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters,
+ int ncand, int exact, int scale_tries, int nthreads, double* err_out, int64_t* stats) {
+ tqenc::Params p;
+ p.iters = iters;
+ p.ncand = ncand;
+ p.exact = exact;
+ p.scale_tries = scale_tries;
+ int64_t nf = 0, zb = 0;
+ const double e = tqenc::encode_blocks(type, w, h, nblocks, dst, p, nthreads, err_out, &nf, &zb);
+ if (stats) { stats[0] = nf; stats[1] = zb; }
+ return e;
+}
+TQE_EXPORT const int16_t* tqe_table() { return tqenc::codebook_table(); }
+}
diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hpp
new file mode 100644
index 000000000..0ed36821d
--- /dev/null
+++ b/tools/trellis/tqenc.hpp
@@ -0,0 +1,93 @@
+// tools/trellis/tqenc.hpp - tail-biting Viterbi ENCODER for the trellis expert formats (TQ2_T, TQK6, TQK7 of the
+// agentionai llama.cpp fork, ggml types 144-146) and two experimental lower-rate versions (TQK5X, TQK4X).
+//
+// Standalone (no ggml, no Strata engine): it only needs third_party/ggml/ggml-common-tq.h (the 2048-point codebook).
+// The decoders it must match are src/kernels/cpu/tq_states.hpp / tq_cpu.cpp (tools/trellis/tqenc_test.cpp checks
+// bit for bit against them, and the scalar / AVX2 / AVX-512 kernels against the encoder's own reconstruction).
+//
+// THE FORMAT (see tq_states.hpp, include/strata/kernels/cpu/tq_cpu.hpp, docs/GYRO.md):
+// block = 128 weights = {fp16 d (little endian); qs[4*K]}, 32 steps t of 4 weights (weights 4t..4t+3).
+// State of step t: 15 bits. weights = d * codebook(state); the state's hash x = state * 0x9e3779b1 picks two
+// points lut[x >> 21] and lut[(x >> 10) & 2047] (fp16 pairs) = four values. The decoder works on the int16 table
+// round(fp16 * 8192) (tq_state_table), so that table, divided by 8192, is the codebook here.
+// TQK (K = 4,5,6,7): state(t) = bits [(31-t)K, (31-t)K + 15) of the qs bit stream (LSB first), circular modulo
+// 32K bits. With u = 31 - t the states form a shift trellis: state(u+1) = (state(u) >> K) | (new K bits <<
+// (15-K)); the K low bits of state(u) are stream symbol u. Tail-biting: state(0) closes the cycle.
+// TQ2_T (K = 8): state(t) = ((qs[(t+31)%32] << 8) | qs[t]) & 0x7fff: a shift trellis in the other direction
+// (state(t) = ((state(t-1) & 0x7f) << 8) | qs[t]).
+// Both are a "de Bruijn" trellis of 2^15 states with 2^K branches per state, so one Viterbi step is a min over G
+// groups of predecessors followed by a broadcast add (G = 2^(15-K) groups for TQK, 128 for TQ2_T), not 2^15 * 2^K
+// additions. Tail-biting = a cycle of 32 states: the group of state(0) as successor equals the group of state(31)
+// as predecessor. The search fixes that group c (exact: all G values; approximate: c taken from a free Viterbi pass
+// on the cyclically rotated sequence, QTIP's approach, optionally the best of several rotations).
+//
+// New experimental types: TQK5X (id 224, 22 bytes / 128 weights, 1.375 bpw) and TQK4X (id 225, 18 bytes, 1.125 bpw):
+// the TQK rule with K = 5 / 4, nothing else changed (same hash, codebook, mask, bit order, circular stream, fp16
+// scale). EXPERIMENTAL_VERSION = 1: not in any ggml enum, not in any GGUF reader.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace tqenc {
+
+inline constexpr int kBlock = 128;
+inline constexpr int kStates = 1 << 15;
+inline constexpr int kExperimentalVersion = 1;
+
+// ggml type ids of the fork (decoders exist) and the experimental ones (encoder + Strata CPU decoder only)
+enum Type : int { TQ2_T = 144, TQK6 = 145, TQK7 = 146, TQK5X = 224, TQK4X = 225 };
+
+/// bits per step (8, 6, 7, 5, 4), 0 for any other type.
+int type_kbits(int type);
+inline int block_bytes(int type) { return type_kbits(type) ? 2 + 4 * type_kbits(type) : 0; }
+
+struct Params {
+ int iters = 3; ///< scale refinement rounds after the first search (path given d -> d given path)
+ int ncand = 1; ///< tail-biting start-group candidates (rotations 16, 8, 24, 12, 20, 4, 28 ...); 1 = QTIP's
+ int exact = 0; ///< 1: try every start group (the exact tail-biting optimum at the current scale; slow)
+ int scale_tries = 1; ///< initial scales d0 * {1, 0.9, 1.1, 0.8, 1.2, ...}: the best of the first n searches
+ uint16_t fixed_scale = 0; ///< nonzero: this fp16 scale is used as is (no refinement); tests, planted solutions
+};
+
+struct BlockResult {
+ double err = 0; ///< sum_i h_i (w_i - rec_i)^2 for the stored block (h = 1 without importance weights)
+ float scale = 0; ///< the stored fp16 scale
+ int searches = 0; ///< Viterbi passes run
+ int status = 0; ///< 0 ok, 1 zero block (all weights zero or below fp16 resolution), 2 nonfinite input
+};
+
+/// The codebook entry of a state as the decoder sees it: four floats (int16 table / 8192, exact).
+const float* codebook_state(uint32_t state); // 4 floats
+const int16_t* codebook_table(); // 32768 * 4 int16, the decoder's tq_state_table()
+
+/// The state of step t (0..31) of a block's stream, by the decoder's rule, written bit by bit (independent of
+/// tq_states.hpp). qs = the 4K stream bytes.
+uint32_t block_state(int type, const uint8_t* qs, int t);
+
+/// Decodes `nblocks` blocks (block_bytes(type) each) to 128 floats each: d * codebook, d the fp16 scale.
+void decode_blocks(int type, const uint8_t* src, int64_t nblocks, float* out);
+
+/// Encodes one block of 128 weights (importance weights h[128] or null) into block_bytes(type) bytes.
+BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst, const Params& p);
+
+/// Encodes nblocks consecutive blocks on `nthreads` threads (0: hardware concurrency). err_out (nblocks doubles) and
+/// the returned sum of block errors are the objective sum h (w - rec)^2. Sets *nonfinite to the number of blocks
+/// with a NaN/Inf input weight (those are written as zero blocks).
+double encode_blocks(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, const Params& p,
+ int nthreads, double* err_out, int64_t* nonfinite, int64_t* zero_blocks);
+
+/// TEST HOOK: one Viterbi pass over a scaled target (T[p][4] = weights / d of processing position p, H[p][4]) and the
+/// legal-path states by position. rot = first position processed, fixed_c = tail-biting group (-1 free).
+float debug_viterbi(int type, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st);
+/// Group sizes: predecessor groups G of a type; processing position p -> weight index of component j.
+int debug_groups(int type);
+int debug_widx(int type, int p, int j);
+/// the stream bytes (4K) of a state path by position, as the encoder packs them
+void debug_pack(int type, const uint16_t* st, uint8_t* qs);
+
+// fp16 helpers (round to nearest even, like the hardware; exposed for the tests)
+uint16_t float_to_half(float f);
+float half_to_float(uint16_t h);
+
+} // namespace tqenc
diff --git a/tools/trellis/tqenc_test.cpp b/tools/trellis/tqenc_test.cpp
new file mode 100644
index 000000000..38596d0b1
--- /dev/null
+++ b/tools/trellis/tqenc_test.cpp
@@ -0,0 +1,540 @@
+// tools/trellis/tqenc_test.cpp - tests of the trellis encoder (tqenc.hpp) against Strata's own CPU decoder
+// (src/kernels/cpu/tq_states.hpp, tq_cpu.cpp: scalar / AVX2 / AVX-512 kernels). Build and run: tools/trellis/build.sh test.
+//
+// T1 fp16 conversion (all 65536 halves, random floats vs the hardware where F16C exists)
+// T2 the encoder's codebook == the decoder's state table (bit for bit, 32768 x 4)
+// T3 random streams: block_state (bit by bit) == tq_states.hpp; every window obeys the shift-trellis relation,
+// circularly (so a legal path = the chain relation + the closure state(31) -> state(0)); pack(unpack(s)) == s
+// T4 the Viterbi pass == a naive DP written from the legality relation (every state, every predecessor), for a fixed
+// start group (tail-biting), a free start and a rotated start, with importance weights
+// T5 exhaustive: every symbol value at every stream position round-trips through pack/decode
+// T6 planted solutions: weights decoded from a random legal stream and scale; exact search at that scale must find
+// error 0 and bit-identical weights; the approximate search is reported (recovery rate)
+// T7 encode -> decode: the encoder's reconstruction == the decoder's weights bit for bit (tq_states + state table),
+// and the CPU kernels' dot products against Q8_0 activations == the dense dot of those weights
+// T8 specials: zero blocks, tiny / huge / extreme scales, constants, spikes, NaN / Inf, determinism over threads
+// T9 quality on Gaussian blocks: SQNR against the rate-distortion bound 6.02 * K/4 dB; approximate vs exact search
+#include "tqenc.hpp"
+
+#include "strata/kernels/cpu/tq_cpu.hpp"
+#include "tq_states.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+#if defined(__F16C__)
+#include <immintrin.h>
+#endif
+
+namespace strata::kernels::cpu {
+bool cpu_avx512_ok() { return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("avx512bw") && __builtin_cpu_supports("avx512vl") && __builtin_cpu_supports("avx512vbmi") && __builtin_cpu_supports("avx512vnni"); }
+bool cpu_avx2_ok() { return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); }
+} // namespace strata::kernels::cpu
+
+using namespace tqenc;
+namespace kc = strata::kernels::cpu;
+
+static int g_fail = 0, g_checks = 0;
+#define CHECK(cond, ...) \
+ do { \
+ ++g_checks; \
+ if (!(cond)) { \
+ ++g_fail; \
+ std::printf(" FAIL %s:%d %s ", __FILE__, __LINE__, #cond); \
+ std::printf(__VA_ARGS__); \
+ std::printf("\n"); \
+ } \
+ } while (0)
+
+static const int kTypes[5] = {TQ2_T, TQK6, TQK7, TQK5X, TQK4X};
+static const char* tname(int t) {
+ return t == TQ2_T ? "TQ2_T" : t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X";
+}
+
+template <int KB>
+static void states_repo(const uint8_t* qs, uint32_t* st) { kc::tq_detail::tq_states<KB>(qs, st); }
+static void states_repo_k(int K, const uint8_t* qs, uint32_t* st) {
+ switch (K) {
+ case 8: states_repo<8>(qs, st); break;
+ case 7: states_repo<7>(qs, st); break;
+ case 6: states_repo<6>(qs, st); break;
+ case 5: states_repo<5>(qs, st); break;
+ default: states_repo<4>(qs, st); break;
+ }
+}
+
+static std::mt19937_64 rng(12345);
+static float gauss() { static std::normal_distribution<float> n(0.f, 1.f); return n(rng); }
+
+// chain relation between consecutive steps, circularly: returns true when state(t) and state(t') obey the shift trellis
+static bool chain_ok(int type, const uint32_t* st) {
+ const int K = type_kbits(type);
+ for (int t = 0; t < 32; ++t) {
+ if (type == TQ2_T) { // state(t) = ((state(t-1) & 0x7f) << 8) | byte
+ const int tp = (t + 31) & 31;
+ if ((st[t] >> 8) != (st[tp] & 0x7f)) return false;
+ } else { // u = 31 - t; state(u+1) low (15-K) bits = state(u) >> K, i.e. step t-1 follows step t
+ const int tp = (t + 31) & 31;
+ if ((st[tp] & ((1u << (15 - K)) - 1u)) != (st[t] >> K)) return false;
+ }
+ }
+ return true;
+}
+
+static void t1() {
+ std::printf("T1 fp16\n");
+ for (uint32_t h = 0; h < 65536; ++h) {
+ const float f = half_to_float((uint16_t) h);
+ if (std::isnan(f)) continue;
+ CHECK(float_to_half(f) == (uint16_t) h || (h == 0x8000 && float_to_half(f) == 0x8000), "half %04x", h);
+ }
+#if defined(__F16C__)
+ int bad = 0;
+ std::uniform_real_distribution<float> u(-1.f, 1.f);
+ for (int i = 0; i < 4000000; ++i) {
+ float f = u(rng) * std::pow(2.f, (float) (int) (rng() % 50) - 30.f);
+ if (i % 7 == 0) f = std::ldexp(u(rng), -25 + (int) (rng() % 6)); // subnormal range and its ties
+ if (i % 11 == 0) f = std::ldexp(u(rng), 14 + (int) (rng() % 3)); // the overflow edge
+ const uint16_t hw = _cvtss_sh(f, _MM_FROUND_TO_NEAREST_INT);
+ bad += float_to_half(f) != hw;
+ }
+ CHECK(bad == 0, "float_to_half vs F16C: %d differ", bad);
+#endif
+}
+
+static void t2() {
+ std::printf("T2 codebook == decoder table\n");
+ const int16_t* a = codebook_table();
+ const int16_t* b = kc::tq_state_table();
+ CHECK(std::memcmp(a, b, 32768 * 4 * sizeof(int16_t)) == 0, "tables differ");
+}
+
+static void t3() {
+ std::printf("T3 states of random streams\n");
+ for (int type : kTypes) {
+ const int K = type_kbits(type), nb = 4 * K;
+ int bad_states = 0, bad_chain = 0, bad_pack = 0;
+ for (int it = 0; it < 20000; ++it) {
+ uint8_t qs[32];
+ for (int i = 0; i < nb; ++i) qs[i] = (uint8_t) rng();
+ uint32_t sr[32];
+ states_repo_k(K, qs, sr);
+ uint32_t sm[32];
+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);
+ bad_states += std::memcmp(sr, sm, sizeof sr) != 0;
+ bad_chain += !chain_ok(type, sm);
+ // pack(unpack): processing order p -> step t
+ uint16_t by_pos[32];
+ for (int p = 0; p < 32; ++p) by_pos[p] = (uint16_t) sm[type == TQ2_T ? p : 31 - p];
+ uint8_t q2[32];
+ debug_pack(type, by_pos, q2);
+ // TQ2_T's bit 7 of each byte is part of the stream but the state uses only 15 bits of two bytes: bit 7 of
+ // qs[t] is in state(t) (bit 7), so it survives; every stream bit is in some state
+ bad_pack += std::memcmp(qs, q2, nb) != 0;
+ }
+ std::printf(" %-6s states vs tq_states.hpp: %d bad, chain relation: %d bad, pack(unpack): %d bad\n", tname(type),
+ bad_states, bad_chain, bad_pack);
+ CHECK(bad_states == 0 && bad_chain == 0 && bad_pack == 0, "%s", tname(type));
+ }
+}
+
+// naive DP from the legality relation, one pass; same conventions as debug_viterbi
+static float naive(int type, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st_out) {
+ const int K = type_kbits(type);
+ const bool t2 = type == TQ2_T;
+ const int G = debug_groups(type);
+ std::vector<std::vector<float>> D(32, std::vector<float>(32768, 1e30f));
+ std::vector<std::vector<int>> B(32, std::vector<int>(32768, -1));
+ auto cost = [&](int p, uint32_t s) {
+ const float* c = codebook_state(s);
+ float e = 0;
+ for (int j = 0; j < 4; ++j) { const float d = T[p][j] - c[j]; e += H[p][j] * d * d; }
+ return e;
+ };
+ auto legal = [&](uint32_t prev, uint32_t next) { // next may follow prev
+ return t2 ? (next >> 8) == (prev & 0x7f) : (next & (uint32_t) (G - 1)) == (prev >> K);
+ };
+ for (uint32_t s = 0; s < 32768; ++s) {
+ const bool ok = fixed_c < 0 || (t2 ? (int) (s >> 8) == fixed_c : (int) (s & (G - 1)) == fixed_c);
+ if (ok) D[0][s] = cost(rot & 31, s);
+ }
+ for (int q = 1; q < 32; ++q)
+ for (uint32_t s = 0; s < 32768; ++s) {
+ float best = 1e30f;
+ int arg = -1;
+ // enumerate the predecessors from the relation (all prev with legal(prev, s)): scan the 2^K candidates
+ const int n = 1 << K;
+ for (int y = 0; y < (t2 ? 256 : n); ++y) {
+ const uint32_t prev = t2 ? ((uint32_t) y << 7) | (s >> 8) : (((s & (uint32_t) (G - 1)) << K) | (uint32_t) y);
+ if (!legal(prev, s)) continue;
+ if (D[q - 1][prev] < best) { best = D[q - 1][prev]; arg = (int) prev; }
+ }
+ if (arg >= 0) { D[q][s] = best + cost((rot + q) & 31, s); B[q][s] = arg; }
+ }
+ float best = 1e30f;
+ int bs = -1;
+ for (uint32_t s = 0; s < 32768; ++s) {
+ const bool ok = fixed_c < 0 || (t2 ? (int) (s & 0x7f) == fixed_c : (int) (s >> K) == fixed_c);
+ if (ok && D[31][s] < best) { best = D[31][s]; bs = (int) s; }
+ }
+ if (bs < 0) return 1e30f;
+ int s = bs;
+ for (int q = 31; q >= 0; --q) {
+ st_out[(rot + q) & 31] = (uint16_t) s;
+ if (q > 0) s = B[q][s];
+ }
+ return best;
+}
+
+static void t4() {
+ std::printf("T4 Viterbi vs naive DP\n");
+ for (int type : kTypes) {
+ const int G = debug_groups(type);
+ double worst = 0;
+ int cases = 0;
+ for (int trial = 0; trial < 3; ++trial) {
+ float T[32][4], H[32][4];
+ for (int p = 0; p < 32; ++p)
+ for (int j = 0; j < 4; ++j) {
+ T[p][j] = gauss() * (trial == 2 ? 2.5f : 1.0f);
+ H[p][j] = trial == 0 ? 1.f : 0.1f + std::fabs(gauss());
+ }
+ struct Case { int rot, c; };
+ Case cs[3] = {{0, (int) (rng() % G)}, {0, -1}, {13, -1}};
+ for (const Case& cc : cs) {
+ uint16_t sa[32], sb[32];
+ const float a = debug_viterbi(type, T, H, cc.rot, cc.c, sa);
+ const float b = naive(type, T, H, cc.rot, cc.c, sb);
+ const double rel = std::fabs(a - b) / std::max(1e-6f, b);
+ worst = std::max(worst, rel);
+ CHECK(rel < 2e-5, "%s trial %d rot %d c %d: viterbi %.6f naive %.6f", tname(type), trial, cc.rot, cc.c, a, b);
+ // the returned path is legal and its own cost is the reported cost
+ uint32_t sm[32];
+ uint8_t qs[32];
+ debug_pack(type, sa, qs);
+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);
+ bool same = true;
+ for (int p = 0; p < 32; ++p) {
+ const uint32_t d = sm[type == TQ2_T ? p : 31 - p];
+ if (cc.rot == 0 || cc.c < 0) {
+ // a free rotated pass is a legal chain only if the stream closes; compare only closed ones
+ }
+ same = same && (cc.c < 0 ? true : d == sa[p]);
+ }
+ if (cc.c >= 0) CHECK(same, "%s tail-biting path does not decode to its own states", tname(type));
+ double sum = 0;
+ for (int q = 0; q < 32; ++q) {
+ const int p = (cc.rot + q) & 31;
+ const float* c = codebook_state(sa[p]);
+ for (int j = 0; j < 4; ++j) { const double d = T[p][j] - c[j]; sum += H[p][j] * d * d; }
+ }
+ CHECK(std::fabs(sum - a) < 1e-3 * std::max(1.0, (double) a), "path cost %.6f vs reported %.6f", sum, a);
+ ++cases;
+ }
+ }
+ std::printf(" %-6s %d cases, worst relative cost difference %.2e\n", tname(type), cases, worst);
+ }
+}
+
+static void t5() {
+ std::printf("T5 every symbol value at every position\n");
+ for (int type : kTypes) {
+ const int K = type_kbits(type), nb = 4 * K;
+ long bad = 0, n = 0;
+ if (type == TQ2_T) {
+ for (int pos = 0; pos < 32; ++pos)
+ for (int x = 0; x < 256; ++x) {
+ uint8_t qs[32];
+ for (int i = 0; i < nb; ++i) qs[i] = (uint8_t) rng();
+ qs[pos] = (uint8_t) x;
+ uint32_t sm[32];
+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);
+ uint16_t bp[32];
+ for (int p = 0; p < 32; ++p) bp[p] = (uint16_t) sm[p];
+ uint8_t q2[32];
+ debug_pack(type, bp, q2);
+ bad += std::memcmp(qs, q2, nb) != 0 || !chain_ok(type, sm);
+ ++n;
+ }
+ } else {
+ for (int u = 0; u < 32; ++u)
+ for (int x = 0; x < (1 << K); ++x) {
+ uint8_t qs[32];
+ for (int i = 0; i < nb; ++i) qs[i] = (uint8_t) rng();
+ // overwrite symbol u
+ for (int b = 0; b < K; ++b) {
+ const int bit = u * K + b;
+ qs[bit >> 3] = (uint8_t) ((qs[bit >> 3] & ~(1u << (bit & 7))) | (((x >> b) & 1u) << (bit & 7)));
+ }
+ uint32_t sm[32];
+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);
+ // symbol u is the low K bits of the state of step 31 - u
+ bad += (int) (sm[31 - u] & ((1u << K) - 1u)) != x;
+ uint16_t bp[32];
+ for (int p = 0; p < 32; ++p) bp[p] = (uint16_t) sm[31 - p];
+ uint8_t q2[32];
+ debug_pack(type, bp, q2);
+ bad += std::memcmp(qs, q2, nb) != 0 || !chain_ok(type, sm);
+ ++n;
+ }
+ }
+ std::printf(" %-6s %ld cases, %ld bad\n", tname(type), n, bad);
+ CHECK(bad == 0, "%s", tname(type));
+ }
+}
+
+static void t6() {
+ std::printf("T6 planted solutions\n");
+ for (int type : kTypes) {
+ const int K = type_kbits(type), nb = 4 * K, bb = block_bytes(type);
+ const int N = (type == TQK4X ? 12 : 30);
+ int exact_ok = 0, approx_ok1 = 0, approx_ok3 = 0;
+ double approx_err_sum = 0, wsum = 0;
+ for (int it = 0; it < N; ++it) {
+ std::vector<uint8_t> blk(bb);
+ const float dsc = std::ldexp(0.5f + (float) (rng() % 1000) / 1000.f, -6 + (int) (rng() % 10));
+ const uint16_t dh = float_to_half(dsc);
+ blk[0] = (uint8_t) (dh & 255);
+ blk[1] = (uint8_t) (dh >> 8);
+ // a legal stream: any bytes are legal (every stream is a closed cycle by construction); for TQ2_T too
+ for (int i = 0; i < nb; ++i) blk[2 + i] = (uint8_t) rng();
+ std::vector<float> w(128);
+ decode_blocks(type, blk.data(), 1, w.data());
+ for (float x : w) wsum += (double) x * x;
+ Params p;
+ p.fixed_scale = dh;
+ std::vector<uint8_t> out(bb);
+ p.exact = 1;
+ BlockResult r = encode_block(type, w.data(), nullptr, out.data(), p);
+ std::vector<float> w2(128);
+ decode_blocks(type, out.data(), 1, w2.data());
+ const bool same = std::memcmp(w.data(), w2.data(), 512) == 0;
+ exact_ok += (r.err == 0.0 && same);
+ CHECK(r.err == 0.0 && same, "%s exact search missed a planted solution (err %g)", tname(type), r.err);
+ p.exact = 0;
+ p.ncand = 1;
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ approx_ok1 += r.err == 0.0;
+ approx_err_sum += r.err;
+ p.ncand = 3;
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ approx_ok3 += r.err == 0.0;
+ }
+ std::printf(" %-6s exact: %d/%d recovered; approximate (rotation 16): %d/%d (err/energy %.2e); best of 3 rotations: %d/%d\n",
+ tname(type), exact_ok, N, approx_ok1, N, approx_err_sum / wsum, approx_ok3, N);
+ }
+}
+
+static uint16_t q8_scale(const std::vector<float>& x, int b) {
+ float amax = 0;
+ for (int j = 0; j < 32; ++j) amax = std::max(amax, std::fabs(x[32 * b + j]));
+ return float_to_half(amax / 127.f);
+}
+
+static void t7() {
+ std::printf("T7 encoder reconstruction == decoder, kernels vs dense\n");
+ for (int type : kTypes) {
+ const int K = type_kbits(type), bb = block_bytes(type);
+ const int rows = 24, n = 512, nbr = n / 128;
+ std::vector<float> w((size_t) rows * n);
+ for (int r = 0; r < rows; ++r)
+ for (int i = 0; i < n; ++i) {
+ float v = gauss() * 0.02f * (1.f + (r % 5));
+ if (r == 3) v = (i % 97 == 0) ? 1.5f : 0.001f * gauss(); // spiky row
+ if (r == 4) v = 0.f; // zero row
+ if (r == 5) v *= 300.f; // big
+ w[(size_t) r * n + i] = v;
+ }
+ std::vector<uint8_t> enc((size_t) rows * nbr * bb);
+ int64_t nf = 0, zb = 0;
+ std::vector<double> errs((size_t) rows * nbr);
+ Params p;
+ const double tot = encode_blocks(type, w.data(), nullptr, (int64_t) rows * nbr, enc.data(), p, 8, errs.data(), &nf, &zb);
+ (void) tot;
+ // decoder reference: tq_states.hpp + the state table
+ const int16_t* tab = kc::tq_state_table();
+ std::vector<float> ref((size_t) rows * n), ours((size_t) rows * n);
+ for (int64_t b = 0; b < (int64_t) rows * nbr; ++b) {
+ const uint8_t* blk = enc.data() + b * bb;
+ const float d = kc::tq_detail::tq_half_to_float(kc::tq_detail::tq_ld16(blk));
+ uint32_t st[32];
+ states_repo_k(K, blk + 2, st);
+ for (int t = 0; t < 32; ++t)
+ for (int j = 0; j < 4; ++j) ref[b * 128 + 4 * t + j] = d * ((float) tab[4 * st[t] + j] * (1.0f / 8192.0f));
+ }
+ decode_blocks(type, enc.data(), (int64_t) rows * nbr, ours.data());
+ CHECK(std::memcmp(ref.data(), ours.data(), ref.size() * 4) == 0, "%s decode != tq_states + state table", tname(type));
+ // the encoder's own error equals the error of the decoded weights
+ double worst = 0;
+ for (int64_t b = 0; b < (int64_t) rows * nbr; ++b) {
+ double e = 0;
+ for (int i = 0; i < 128; ++i) { const double d = (double) w[b * 128 + i] - ref[b * 128 + i]; e += d * d; }
+ worst = std::max(worst, std::fabs(e - errs[b]) / std::max(1e-30, e + errs[b]));
+ }
+ CHECK(worst < 1e-9, "%s encoder error != decoded error (%.3e)", tname(type), worst);
+ // kernels (scalar, AVX2, AVX-512 where present) vs the dense dot with the dequantized Q8_0 activation
+ std::vector<float> x(n);
+ for (auto& v : x) v = gauss();
+ std::vector<uint8_t> q8((size_t) (n / 32) * 34);
+ std::vector<float> xd(n);
+ for (int b = 0; b < n / 32; ++b) {
+ const uint16_t dh = q8_scale(x, b);
+ const float d = half_to_float(dh);
+ q8[34 * b] = (uint8_t) (dh & 255);
+ q8[34 * b + 1] = (uint8_t) (dh >> 8);
+ for (int j = 0; j < 32; ++j) {
+ const int q = d == 0.f ? 0 : (int) std::lround(x[32 * b + j] / d);
+ q8[34 * b + 2 + j] = (uint8_t) (int8_t) std::max(-127, std::min(127, q));
+ xd[32 * b + j] = d * (float) (int8_t) q8[34 * b + 2 + j];
+ }
+ }
+ const void* ap[1] = {q8.data()};
+ for (kc::TqIsa isa : {kc::TqIsa::scalar, kc::TqIsa::avx2, kc::TqIsa::avx512}) {
+ if (!kc::tq_isa_ok(isa)) continue;
+ std::vector<float> out(rows);
+ float* op[1] = {out.data()};
+ kc::tq_rows(isa, type, enc.data(), (size_t) nbr * bb, n, ap, 1, op, 0, rows);
+ double worst_rel = 0;
+ for (int r = 0; r < rows; ++r) {
+ double dsum = 0, mag = 0;
+ for (int i = 0; i < n; ++i) {
+ dsum += (double) ref[(size_t) r * n + i] * xd[i];
+ mag += std::fabs((double) ref[(size_t) r * n + i] * xd[i]);
+ }
+ worst_rel = std::max(worst_rel, std::fabs(dsum - out[r]) / (mag + 1e-30));
+ }
+ std::printf(" %-6s %-6s kernel vs dense dot of the decoded weights: worst error %.2e of sum|w a|\n", tname(type),
+ kc::tq_isa_name(isa), worst_rel);
+ CHECK(worst_rel < 1e-5, "%s %s", tname(type), kc::tq_isa_name(isa));
+ }
+ std::printf(" %-6s decode == tq_states+table: bit-exact, %zu weights; zero blocks %lld\n", tname(type), ref.size(), (long long) zb);
+ }
+}
+
+static void t8() {
+ std::printf("T8 specials\n");
+ for (int type : kTypes) {
+ const int bb = block_bytes(type);
+ std::vector<uint8_t> out(bb), z(bb, 0);
+ std::vector<float> w(128), dec(128);
+ Params p;
+ // zero block
+ std::fill(w.begin(), w.end(), 0.f);
+ BlockResult r = encode_block(type, w.data(), nullptr, out.data(), p);
+ CHECK(r.status == 1 && out == z, "%s zero block must be canonical all-zero bytes", tname(type));
+ w[5] = -0.f;
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ CHECK(r.status == 1 && out == z, "%s negative zero", tname(type));
+ decode_blocks(type, out.data(), 1, dec.data());
+ for (float v : dec) CHECK(v == 0.f, "zero block decodes to 0");
+ // scales: from the smallest subnormal weights to beyond fp16
+ const float mags[] = {1e-9f, 3e-8f, 1e-7f, 6e-5f, 1e-3f, 1.f, 100.f, 3000.f, 30000.f, 60000.f, 65000.f, 1e5f, 1e7f};
+ for (float mag : mags) {
+ for (int i = 0; i < 128; ++i) w[i] = mag * gauss();
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ const uint16_t dh = (uint16_t) (out[0] | (out[1] << 8));
+ const float d = half_to_float(dh);
+ CHECK(std::isfinite(d) && d >= 0.f, "%s scale %g not finite for magnitude %g", tname(type), d, mag);
+ decode_blocks(type, out.data(), 1, dec.data());
+ double e = 0, en = 0;
+ bool fin = true;
+ for (int i = 0; i < 128; ++i) { fin &= std::isfinite(dec[i]); e += (w[i] - dec[i]) * (double) (w[i] - dec[i]); en += (double) w[i] * w[i]; }
+ CHECK(fin, "decoded nonfinite");
+ std::printf(" %-6s magnitude %-8g d=%-12g rel err %.3f status %d\n", tname(type), mag, d, e / std::max(en, 1e-300), r.status);
+ if (mag >= 1e-3f && mag <= 60000.f) CHECK(e / en < 0.5, "%s error too large at magnitude %g (%.3f)", tname(type), mag, e / en);
+ }
+ // constant, spike, alternating, power of two
+ for (int variant = 0; variant < 4; ++variant) {
+ for (int i = 0; i < 128; ++i)
+ w[i] = variant == 0 ? 0.37f : variant == 1 ? (i == 77 ? 5.f : 0.f) : variant == 2 ? (i & 1 ? 1.f : -1.f) : std::ldexp(1.f, -3);
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ decode_blocks(type, out.data(), 1, dec.data());
+ double e = 0, en = 0;
+ for (int i = 0; i < 128; ++i) { e += (w[i] - dec[i]) * (double) (w[i] - dec[i]); en += (double) w[i] * w[i]; }
+ std::printf(" %-6s pattern %d rel err %.4f\n", tname(type), variant, e / en);
+ CHECK(e / en < 0.8, "%s pattern %d", tname(type), variant);
+ }
+ // non-finite input
+ for (int i = 0; i < 128; ++i) w[i] = gauss();
+ w[17] = INFINITY;
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ CHECK(r.status == 2 && out == z, "%s inf input", tname(type));
+ w[17] = NAN;
+ r = encode_block(type, w.data(), nullptr, out.data(), p);
+ CHECK(r.status == 2 && out == z, "%s nan input", tname(type));
+ }
+ // determinism over thread counts
+ {
+ const int type = TQK6, nbl = 64, bb = block_bytes(type);
+ std::vector<float> w((size_t) nbl * 128);
+ for (auto& v : w) v = gauss();
+ std::vector<uint8_t> a((size_t) nbl * bb), b(a.size());
+ Params p;
+ encode_blocks(type, w.data(), nullptr, nbl, a.data(), p, 1, nullptr, nullptr, nullptr);
+ encode_blocks(type, w.data(), nullptr, nbl, b.data(), p, 7, nullptr, nullptr, nullptr);
+ CHECK(a == b, "1 thread vs 7 threads differ");
+ }
+}
+
+static void t9() {
+ std::printf("T9 quality on N(0,1) blocks (SQNR in dB; the rate-distortion limit of the K/4 bits per weight is 6.02 * K/4)\n");
+ for (int type : kTypes) {
+ const int K = type_kbits(type), bb = block_bytes(type);
+ const int nbl = (type == TQ2_T || type == TQK7) ? 256 : 384;
+ std::vector<float> w((size_t) nbl * 128);
+ for (auto& v : w) v = gauss();
+ std::vector<uint8_t> out((size_t) nbl * bb);
+ double en = 0;
+ for (float v : w) en += (double) v * v;
+ struct Cfg { const char* name; Params p; int th; };
+ Cfg cfgs[5];
+ cfgs[0] = {"1 pass, scale from rms", Params{0, 1, 0, 1, 0}, 8};
+ cfgs[1] = {"default (rot 16, 3 scale rounds)", Params{}, 8};
+ cfgs[2] = {"3 rotations, 3 scale rounds", Params{3, 3, 0, 1, 0}, 8};
+ cfgs[3] = {"3 rotations, 3 starting scales", Params{3, 3, 0, 3, 0}, 8};
+ int ncfg = 4;
+ for (int i = 0; i < ncfg; ++i) {
+ const auto t0 = std::chrono::steady_clock::now();
+ const double e = encode_blocks(type, w.data(), nullptr, nbl, out.data(), cfgs[i].p, cfgs[i].th, nullptr, nullptr, nullptr);
+ const double sec = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+ const double sq = 10.0 * std::log10(en / e);
+ std::printf(" %-6s %-34s SQNR %.3f dB (limit %.2f, gap %.2f dB) %.0f weights/s (%d threads)\n", tname(type),
+ cfgs[i].name, sq, 6.0206 * K / 4.0, 6.0206 * K / 4.0 - sq, (double) nbl * 128 / sec, cfgs[i].th);
+ }
+ // approximate vs exact start group at the default scale rounds on a few blocks
+ {
+ const int nb2 = type == TQK4X ? 2 : 8;
+ Params pe;
+ pe.exact = 1;
+ pe.iters = 0;
+ Params pa;
+ pa.iters = 0;
+ const double ee = encode_blocks(type, w.data(), nullptr, nb2, out.data(), pe, 8, nullptr, nullptr, nullptr);
+ const double ea = encode_blocks(type, w.data(), nullptr, nb2, out.data(), pa, 8, nullptr, nullptr, nullptr);
+ std::printf(" %-6s tail-biting search, %d blocks, one scale: approximate error / exact error = %.4f\n", tname(type), nb2, ea / ee);
+ CHECK(ea >= ee * (1 - 1e-6), "approximate beat exact?");
+ }
+ }
+}
+
+int main(int argc, char** argv) {
+ const bool quick = argc > 1 && std::strcmp(argv[1], "--quick") == 0;
+ t1();
+ t2();
+ t3();
+ t5();
+ t4();
+ t6();
+ t7();
+ t8();
+ if (!quick) t9();
+ std::printf("\n%d checks, %d failures\n", g_checks, g_fail);
+ return g_fail ? 1 : 0;
+}