Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis encoder: tail-biting Viterbi for TQ2_T/TQK6/TQK7 plus experimental TQK5X/TQK4X (CPU decoder reads them)
Niko1221512e298e6e6d
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hppindex 132cb99e1..a73e300e8 100644--- a/include/strata/kernels/cpu/tq_cpu.hpp+++ b/include/strata/kernels/cpu/tq_cpu.hpp@@ -31,11 +31,16 @@ namespace strata::kernels::cpu {enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };-/// TQ2_T (144), TQK6 (145), TQK7 (146).+/// EXPERIMENTAL lower-rate versions of the TQK rule (tools/trellis/tqenc.hpp, version 1; Strata-internal ids, not ggml+/// types, not in any GGUF): TQK5X = 22 bytes / 128 weights (1.375 bpw), TQK4X = 18 bytes (1.125 bpw). Same hash,+/// codebook, 15-bit mask, LSB-first circular stream and fp16 scale as TQK6/7 with K = 5 / 4.+inline constexpr int kTqTypeK5X = 224;+inline constexpr int kTqTypeK4X = 225;+/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X.bool tq_supported(int ggml_type) noexcept;-/// Bits per step: 8, 6, 7 (0 for any other type).+/// Bits per step: 8, 6, 7, 5, 4 (0 for any other type).int tq_kbits(int ggml_type) noexcept;-/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30).+/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30, 22, 18).constexpr int kTqBlock = 128;inline size_t tq_block_bytes(int ggml_type) noexcept { return (size_t) (2 + 4 * tq_kbits(ggml_type)); }/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).diff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cppindex 00918edf6..f03892510 100644--- a/src/kernels/cpu/tq_avx2.cpp+++ b/src/kernels/cpu/tq_avx2.cpp@@ -81,6 +81,8 @@ void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqActfloat* const* out, int r0, int r1) {if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X+ else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4Xelse rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);}diff --git a/src/kernels/cpu/tq_avx512.cpp b/src/kernels/cpu/tq_avx512.cppindex b45118844..9031b8cf5 100644--- a/src/kernels/cpu/tq_avx512.cpp+++ b/src/kernels/cpu/tq_avx512.cpp@@ -89,6 +89,8 @@ void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAfloat* const* out, int r0, int r1) {if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X+ else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4Xelse rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);}diff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cppindex 1240d7f74..ad7ea297d 100644--- a/src/kernels/cpu/tq_cpu.cpp+++ b/src/kernels/cpu/tq_cpu.cpp@@ -89,9 +89,11 @@ const int16_t* tq_state_table() {return g_table;}-bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146; }+bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X; }-int tq_kbits(int t) noexcept { return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : 0; }+int tq_kbits(int t) noexcept {+ return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : t == kTqTypeK5X ? 5 : t == kTqTypeK4X ? 4 : 0;+}bool tq_isa_ok(TqIsa isa) {#if defined(STRATA_TQ_X86)@@ -144,6 +146,8 @@ void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAfloat* const* out, int r0, int r1) {if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 5) rows_scalar_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X+ else if (kb == 4) rows_scalar_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4Xelse rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);}diff --git a/tools/trellis/build.sh b/tools/trellis/build.shnew file mode 100644index 000000000..c85354033--- /dev/null+++ b/tools/trellis/build.sh@@ -0,0 +1,20 @@+#!/bin/sh+# tools/trellis/build.sh [test|lib|bench] - builds the trellis encoder standalone (g++, Linux x86-64).+# Run from the repository root. Output in ${TQ_OUT:-build-tqenc}.+set -e+OUT=${TQ_OUT:-build-tqenc}+mkdir -p "$OUT"+CXX=${CXX:-g++}+INC="-Itools/trellis -Iinclude -Isrc/kernels/cpu -Ithird_party/ggml"+OPT="-O3 -std=c++17 -mavx2 -mfma -mf16c -fno-fast-math -pthread"+case "${1:-test}" in+lib)+ $CXX $OPT -fPIC -shared $INC tools/trellis/tqenc.cpp -o "$OUT/libtqenc.so" ;;+test)+ $CXX $OPT $INC -c src/kernels/cpu/tq_cpu.cpp -o "$OUT/tq_cpu.o"+ $CXX $OPT $INC -c src/kernels/cpu/tq_avx2.cpp -o "$OUT/tq_avx2.o"+ $CXX -O3 -std=c++17 -mavx512f -mavx512bw -mavx512vl -mavx512vbmi -mavx512vnni -mfma -mf16c -mavx2 $INC -c src/kernels/cpu/tq_avx512.cpp -o "$OUT/tq_avx512.o"+ $CXX $OPT $INC tools/trellis/tqenc_test.cpp tools/trellis/tqenc.cpp "$OUT/tq_cpu.o" "$OUT/tq_avx2.o" "$OUT/tq_avx512.o" -o "$OUT/tqenc_test"+ ;;+esac+echo built "$OUT"diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cppnew file mode 100644index 000000000..61aca400b--- /dev/null+++ b/tools/trellis/tqenc.cpp@@ -0,0 +1,546 @@+// tools/trellis/tqenc.cpp - see tqenc.hpp.+#include "tqenc.hpp"++#include "ggml-common-tq.h"++#include <algorithm>+#include <atomic>+#include <cmath>+#include <cstdlib>+#include <cstring>+#include <memory>+#include <mutex>+#include <thread>+#include <vector>++#if defined(__AVX2__) && defined(__FMA__)+#include <immintrin.h>+#define TQENC_AVX2 1+#endif++namespace tqenc {+namespace {++constexpr float kInf = 1e30f;++// ---------------------------------------------------------------------------------------------------------------+// fp16++} // namespace++float half_to_float(uint16_t h) {+ const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;+ uint32_t exp = (h >> 10) & 0x1fu, man = h & 0x3ffu, bits;+ if (exp == 0x1fu) {+ bits = sign | 0x7f800000u | (man << 13);+ } else if (exp != 0) {+ bits = sign | ((exp + 112u) << 23) | (man << 13);+ } else if (man == 0) {+ bits = sign;+ } else {+ const float m = (float) man * (1.0f / 16777216.0f);+ return sign ? -m : m;+ }+ float f;+ std::memcpy(&f, &bits, 4);+ return f;+}++uint16_t float_to_half(float f) {+ uint32_t x;+ std::memcpy(&x, &f, 4);+ const uint32_t sign = (x >> 16) & 0x8000u;+ const uint32_t ax = x & 0x7fffffffu;+ if (ax >= 0x7f800000u) return (uint16_t) (sign | 0x7c00u | (ax > 0x7f800000u ? 0x200u : 0u)); // inf / nan+ if (ax >= 0x477ff000u) return (uint16_t) (sign | 0x7c00u); // >= 65520 rounds to inf+ if (ax < 0x33000001u) return (uint16_t) sign; // <= 2^-25: rounds to zero (ties to even)+ int e = (int) (ax >> 23) - 127;+ uint32_t man = (ax & 0x7fffffu) | 0x800000u;+ if (e < -14) { // subnormal half+ const int shift = -14 - e + 13; // total right shift of the 24-bit mantissa+ const uint32_t half = 1u << (shift - 1), rest = man & ((1u << shift) - 1);+ uint32_t q = man >> shift;+ if (rest > half || (rest == half && (q & 1))) ++q;+ return (uint16_t) (sign | q);+ }+ uint32_t q = ((uint32_t) (e + 15) << 10) | ((man >> 13) & 0x3ffu);+ const uint32_t rest = man & 0x1fffu;+ if (rest > 0x1000u || (rest == 0x1000u && (q & 1))) ++q; // a carry into the exponent is correct+ return (uint16_t) (sign | q);+}++int type_kbits(int type) {+ switch (type) {+ case TQ2_T: return 8;+ case TQK6: return 6;+ case TQK7: return 7;+ case TQK5X: return 5;+ case TQK4X: return 4;+ default: return 0;+ }+}++namespace {++// ---------------------------------------------------------------------------------------------------------------+// codebook: the decoder's int16 table (round half away from zero of fp16 * 8192, as tq_cpu.cpp fill_table), SoA floats++struct Codebook {+ int16_t tab[kStates * 4];+ alignas(64) float soa[4][kStates];+ Codebook() {+ for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {+ const uint32_t x = s * 0x9e3779b1u;+ const uint32_t pts[2] = {x >> 21, (x >> 10) & 2047u};+ for (int h = 0; h < 2; ++h)+ for (int j = 0; j < 2; ++j) {+ const float v = half_to_float(tq_lut_f16[2 * pts[h] + j]) * 8192.0f;+ const int16_t q = (int16_t) (v < 0.0f ? v - 0.5f : v + 0.5f);+ tab[4 * s + 2 * h + j] = q;+ soa[2 * h + j][s] = (float) q * (1.0f / 8192.0f);+ }+ }+ }+};++const Codebook& cb() {+ static const Codebook c;+ return c;+}++struct Geo {+ int K = 0;+ bool t2 = false; // TQ2_T (byte stream, the other shift direction)+ int G = 0; // predecessor groups+};++Geo geo_for(int type) {+ Geo g;+ g.K = type_kbits(type);+ g.t2 = type == TQ2_T;+ g.G = g.t2 ? 128 : (1 << (15 - g.K));+ return g;+}++// group of a state as a successor / as a predecessor+inline int gsucc(const Geo& g, uint32_t s) { return g.t2 ? (int) (s >> 8) : (int) (s & (uint32_t) (g.G - 1)); }+inline int gpred(const Geo& g, uint32_t s) { return g.t2 ? (int) (s & 0x7f) : (int) (s >> g.K); }++// weight index (0..127) of component j of processing position p+inline int widx(const Geo& g, int p, int j) { return g.t2 ? 4 * p + j : 4 * (31 - p) + j; }++struct Scratch {+ alignas(64) float hist[32][kStates];+ alignas(64) float m[2048];+ alignas(64) float zero[kStates];+};++Scratch& scratch() {+ thread_local std::unique_ptr<Scratch> s;+ if (!s) {+ s.reset(new Scratch);+ std::memset(s->zero, 0, sizeof s->zero);+ std::memset(s->m, 0, sizeof s->m);+ }+ return *s;+}++inline float min_range(const float* p, int n) {+#if TQENC_AVX2+ if (n >= 8) {+ __m256 v = _mm256_loadu_ps(p);+ for (int i = 8; i + 8 <= n; i += 8) v = _mm256_min_ps(v, _mm256_loadu_ps(p + i));+ __m128 a = _mm_min_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));+ a = _mm_min_ps(a, _mm_movehl_ps(a, a));+ a = _mm_min_ss(a, _mm_shuffle_ps(a, a, 1));+ return _mm_cvtss_f32(a);+ }+#endif+ float m = p[0];+ for (int i = 1; i < n; ++i) m = std::min(m, p[i]);+ return m;+}++// out[i] = add[i] + sum_j h_j (t_j - cb_j[s0 + i])^2 for i in [0, n)+inline void step_cost(float* out, const float* add, int s0, int n, const float* t, const float* h) {+ const float *c0 = cb().soa[0] + s0, *c1 = cb().soa[1] + s0, *c2 = cb().soa[2] + s0, *c3 = cb().soa[3] + s0;+ int i = 0;+#if TQENC_AVX2+ const __m256 t0 = _mm256_set1_ps(t[0]), t1 = _mm256_set1_ps(t[1]), t2 = _mm256_set1_ps(t[2]), t3 = _mm256_set1_ps(t[3]);+ const __m256 h0 = _mm256_set1_ps(h[0]), h1 = _mm256_set1_ps(h[1]), h2 = _mm256_set1_ps(h[2]), h3 = _mm256_set1_ps(h[3]);+ for (; i + 8 <= n; i += 8) {+ const __m256 e0 = _mm256_sub_ps(t0, _mm256_loadu_ps(c0 + i)), e1 = _mm256_sub_ps(t1, _mm256_loadu_ps(c1 + i));+ const __m256 e2 = _mm256_sub_ps(t2, _mm256_loadu_ps(c2 + i)), e3 = _mm256_sub_ps(t3, _mm256_loadu_ps(c3 + i));+ __m256 a = _mm256_fmadd_ps(_mm256_mul_ps(h0, e0), e0, _mm256_loadu_ps(add + i));+ a = _mm256_fmadd_ps(_mm256_mul_ps(h1, e1), e1, a);+ a = _mm256_fmadd_ps(_mm256_mul_ps(h2, e2), e2, a);+ a = _mm256_fmadd_ps(_mm256_mul_ps(h3, e3), e3, a);+ _mm256_storeu_ps(out + i, a);+ }+#endif+ for (; i < n; ++i) {+ const float e0 = t[0] - c0[i], e1 = t[1] - c1[i], e2 = t[2] - c2[i], e3 = t[3] - c3[i];+ out[i] = add[i] + h[0] * e0 * e0 + h[1] * e1 * e1 + h[2] * e2 * e2 + h[3] * e3 * e3;+ }+}++// the broadcast variant: out[i] = a + cost(s0 + i)+inline void step_cost_bc(float* out, float a, int s0, int n, const float* t, const float* h) {+ alignas(32) float tmp[256];+ for (int i = 0; i < n; ++i) tmp[i] = a;+ step_cost(out, tmp, s0, n, t, h);+}++// T[p][4]: targets of processing position p (weights / d), H[p][4]: importance. rot: the sequence is processed from+// position rot (cyclically); fixed_c >= 0: tail-biting with the successor group of the first state and the predecessor+// group of the last state both == fixed_c. Writes the states by position into st[32]; returns the path cost.+float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st) {+ Scratch& sc = scratch();+ const int K = g.K, G = g.G;+ // q = 0+ {+ const int p = rot & 31;+ step_cost(sc.hist[0], sc.zero, 0, kStates, T[p], H[p]);+ if (fixed_c >= 0) {+ float* h0 = sc.hist[0];+ if (g.t2) { // gsucc(s) = s >> 8 == c: the 256 states [c<<8, c<<8 + 256) allowed+ for (int s = 0; s < kStates; ++s)+ if ((s >> 8) != fixed_c) h0[s] = kInf;+ } else { // s & (G-1) == c+ for (int s = 0; s < kStates; ++s)+ if ((s & (G - 1)) != fixed_c) h0[s] = kInf;+ }+ }+ }+ for (int q = 1; q < 32; ++q) {+ const float* prev = sc.hist[q - 1];+ float* cur = sc.hist[q];+ const int p = (rot + q) & 31;+ float* m = sc.m;+ if (g.t2) {+ // m[a] = min over h of prev[a + 128 h]+ for (int a = 0; a < 128; ++a) m[a] = prev[a];+ for (int h = 1; h < 256; ++h) {+ const float* row = prev + 128 * h;+ int a = 0;+#if TQENC_AVX2+ for (; a + 8 <= 128; a += 8) _mm256_storeu_ps(m + a, _mm256_min_ps(_mm256_loadu_ps(m + a), _mm256_loadu_ps(row + a)));+#endif+ for (; a < 128; ++a) m[a] = std::min(m[a], row[a]);+ }+ for (int b = 0; b < 128; ++b) step_cost_bc(cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);+ } else {+ const int span = 1 << K;+ for (int a = 0; a < G; ++a) m[a] = min_range(prev + (size_t) a * span, span);+ for (int base = 0; base < kStates; base += G) step_cost(cur + base, m, base, G, T[p], H[p]);+ }+ }+ // final state+ const float* last = sc.hist[31];+ int best = -1;+ float bc = 3e38f;+ if (fixed_c < 0) {+ for (int s = 0; s < kStates; ++s)+ if (last[s] < bc) { bc = last[s]; best = s; }+ } else if (g.t2) {+ for (int h = 0; h < 256; ++h) {+ const int s = fixed_c + 128 * h;+ if (last[s] < bc) { bc = last[s]; best = s; }+ }+ } else {+ for (int i = 0; i < (1 << K); ++i) {+ const int s = (fixed_c << K) + i;+ if (last[s] < bc) { bc = last[s]; best = s; }+ }+ }+ if (best < 0 || bc >= kInf * 0.5f) return kInf;+ uint32_t s = (uint32_t) best;+ st[(rot + 31) & 31] = (uint16_t) s;+ for (int q = 31; q >= 1; --q) {+ const int gv = gsucc(g, s);+ const float* prev = sc.hist[q - 1];+ int arg = -1;+ float mv = 3e38f;+ if (g.t2) {+ for (int h = 0; h < 256; ++h) {+ const int x = gv + 128 * h;+ if (prev[x] < mv) { mv = prev[x]; arg = x; }+ }+ } else {+ const int span = 1 << K;+ for (int i = 0; i < span; ++i) {+ const int x = (gv << K) + i;+ if (prev[x] < mv) { mv = prev[x]; arg = x; }+ }+ }+ s = (uint32_t) arg;+ st[(rot + q - 1) & 31] = (uint16_t) s;+ }+ return bc;+}++// junction group of a free pass: gpred of the state at orig position 31 (== gsucc of the state at position 0)+inline int junction_group(const Geo& g, const uint16_t* st) { return gpred(g, st[31]); }++struct Search {+ const Geo* g;+ const Params* p;+ float T[32][4];+ float H[32][4];+ uint16_t st[32];+ int passes = 0;++ // one tail-biting search at scale d (weights already divided: set_targets); returns the path cost in scaled units+ float run() {+ static const int rots[8] = {16, 8, 24, 12, 20, 4, 28, 0};+ float best = kInf;+ uint16_t tmp[32], free_st[32];+ if (p->exact) {+ for (int c = 0; c < g->G; ++c) {+ const float v = viterbi(*g, T, H, 0, c, tmp);+ ++passes;+ if (v < best) { best = v; std::memcpy(st, tmp, sizeof tmp); }+ }+ return best;+ }+ int seen[8], ns = 0;+ const int nc = std::max(1, std::min(p->ncand, 8));+ for (int r = 0; r < nc; ++r) {+ viterbi(*g, T, H, rots[r], -1, free_st);+ ++passes;+ const int c = junction_group(*g, free_st);+ bool dup = false;+ for (int i = 0; i < ns; ++i) dup |= seen[i] == c;+ if (dup) continue;+ seen[ns++] = c;+ const float v = viterbi(*g, T, H, 0, c, tmp);+ ++passes;+ if (v < best) { best = v; std::memcpy(st, tmp, sizeof tmp); }+ }+ return best;+ }+};++void pack_stream(const Geo& g, const uint16_t* st, uint8_t* qs) {+ const int K = g.K;+ std::memset(qs, 0, (size_t) 4 * K);+ if (g.t2) {+ for (int t = 0; t < 32; ++t) qs[t] = (uint8_t) (st[t] & 0xff);+ return;+ }+ for (int u = 0; u < 32; ++u) {+ const uint32_t x = st[u] & ((1u << K) - 1u);+ for (int b = 0; b < K; ++b)+ if ((x >> b) & 1u) qs[(u * K + b) >> 3] |= (uint8_t) (1u << ((u * K + b) & 7));+ }+}++} // namespace++float debug_viterbi(int type, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st) {+ return viterbi(geo_for(type), T, H, rot, fixed_c, st);+}+int debug_groups(int type) { return geo_for(type).G; }+int debug_widx(int type, int p, int j) { return widx(geo_for(type), p, j); }+void debug_pack(int type, const uint16_t* st, uint8_t* qs) { pack_stream(geo_for(type), st, qs); }++const float* codebook_state(uint32_t state) {+ static thread_local float v[4];+ for (int j = 0; j < 4; ++j) v[j] = cb().soa[j][state & 0x7fff];+ return v;+}++const int16_t* codebook_table() { return cb().tab; }++uint32_t block_state(int type, const uint8_t* qs, int t) {+ const int K = type_kbits(type);+ if (type == TQ2_T) return (((uint32_t) qs[(t + 31) & 31] << 8) | qs[t]) & 0x7fffu;+ const int total = 32 * K;+ uint32_t s = 0;+ for (int i = 0; i < 15; ++i) {+ const int b = ((31 - t) * K + i) % total;+ s |= (uint32_t) ((qs[b >> 3] >> (b & 7)) & 1u) << i;+ }+ return s;+}++void decode_blocks(int type, const uint8_t* src, int64_t nblocks, float* out) {+ const int K = type_kbits(type), bb = 2 + 4 * K;+ for (int64_t b = 0; b < nblocks; ++b) {+ const uint8_t* blk = src + (size_t) b * bb;+ const uint16_t dh = (uint16_t) (blk[0] | (blk[1] << 8));+ const float d = half_to_float(dh);+ for (int t = 0; t < 32; ++t) {+ const uint32_t s = block_state(type, blk + 2, t);+ for (int j = 0; j < 4; ++j) out[b * 128 + 4 * t + j] = d * cb().soa[j][s];+ }+ }+}++BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst, const Params& prm) {+ BlockResult r;+ const Geo g = geo_for(type);+ const int K = g.K, bb = 2 + 4 * K;+ std::memset(dst, 0, (size_t) bb);+ double sumw2 = 0, sumhw2 = 0, amax = 0;+ for (int i = 0; i < 128; ++i) {+ if (!std::isfinite(w[i]) || (h && !std::isfinite(h[i]))) { r.status = 2; r.err = NAN; return r; }+ const double hi = h ? h[i] : 1.0;+ sumw2 += (double) w[i] * w[i];+ sumhw2 += hi * (double) w[i] * w[i];+ amax = std::max(amax, (double) std::fabs(w[i]));+ }+ if (amax == 0.0) { r.status = 1; return r; }+ const double rms = std::sqrt(sumw2 / 128.0);+ // initial scale: the codebook has unit-ish rms+ constexpr double kCbRms = 1.072;+ static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};++ Search se;+ se.g = &g;+ se.p = &prm;+ for (int p = 0; p < 32; ++p)+ for (int j = 0; j < 4; ++j) se.H[p][j] = h ? h[widx(g, p, j)] : 1.0f;++ double best_err = 1e300;+ uint16_t best_st[32] = {};+ uint16_t best_d = 0;+ bool have = false;++ auto fit = [&](const uint16_t* st, uint16_t dh, double& err) {+ // err of (st, dh): sum h (w - d cv)^2+ const float d = half_to_float(dh);+ double e = 0;+ for (int p = 0; p < 32; ++p)+ for (int j = 0; j < 4; ++j) {+ const int wi = widx(g, p, j);+ const float rec = d * cb().soa[j][st[p]];+ const double df = (double) w[wi] - rec;+ e += (h ? h[wi] : 1.0) * df * df;+ }+ err = e;+ };++ const int tries = prm.fixed_scale ? 1 : std::max(1, std::min(prm.scale_tries, 7));+ for (int tr = 0; tr < tries; ++tr) {+ double dd = rms / kCbRms * mult[tr];+ uint16_t dh = float_to_half((float) std::min(dd, 65504.0));+ if (dh == 0) dh = 1; // the smallest subnormal: weights below ~3e-8 are noise at this scale+ if (prm.fixed_scale) dh = prm.fixed_scale;+ for (int it = 0; it <= (prm.fixed_scale ? 0 : prm.iters); ++it) {+ const float d = half_to_float(dh);+ for (int p = 0; p < 32; ++p)+ for (int j = 0; j < 4; ++j) se.T[p][j] = w[widx(g, p, j)] / d;+ if (se.run() >= kInf * 0.5f) break;+ double e;+ fit(se.st, dh, e);+ if (e < best_err) {+ best_err = e;+ best_d = dh;+ std::memcpy(best_st, se.st, sizeof best_st);+ have = true;+ }+ if (e == 0.0 || prm.fixed_scale) break;+ // least-squares scale for this path+ double num = 0, den = 0;+ for (int p = 0; p < 32; ++p)+ for (int j = 0; j < 4; ++j) {+ const int wi = widx(g, p, j);+ const double hi = h ? h[wi] : 1.0, c = cb().soa[j][se.st[p]];+ num += hi * w[wi] * c;+ den += hi * c * c;+ }+ if (den <= 0 || num <= 0) break;+ uint16_t nh = float_to_half((float) std::min(num / den, 65504.0));+ if (nh == 0 || nh == dh) break;+ double en;+ fit(se.st, nh, en);+ if (en < best_err) {+ best_err = en;+ best_d = nh;+ std::memcpy(best_st, se.st, sizeof best_st);+ }+ if (en >= e) break; // the scale step did not help this path: stop refining this start+ dh = nh;+ }+ r.searches = se.passes;+ }+ if (!have || best_err >= sumhw2) { // nothing beats a zero block (weights below the fp16 scale resolution)+ std::memset(dst, 0, (size_t) bb);+ r.status = 1;+ r.err = sumhw2;+ return r;+ }+ dst[0] = (uint8_t) (best_d & 0xff);+ dst[1] = (uint8_t) (best_d >> 8);+ pack_stream(g, best_st, dst + 2);+ r.err = best_err;+ r.scale = half_to_float(best_d);+ r.searches = se.passes;+ return r;+}++double encode_blocks(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, const Params& p,+ int nthreads, double* err_out, int64_t* nonfinite, int64_t* zero_blocks) {+ const int bb = block_bytes(type);+ if (nthreads <= 0) nthreads = (int) std::max(1u, std::thread::hardware_concurrency());+ nthreads = (int) std::min<int64_t>(nthreads, std::max<int64_t>(1, nblocks));+ std::atomic<int64_t> next{0}, nf{0}, zb{0};+ std::vector<double> sums((size_t) nthreads, 0.0);+ auto work = [&](int ti) {+ double s = 0;+ for (;;) {+ const int64_t b0 = next.fetch_add(4);+ if (b0 >= nblocks) break;+ const int64_t b1 = std::min(nblocks, b0 + 4);+ for (int64_t b = b0; b < b1; ++b) {+ const BlockResult r = encode_block(type, w + b * 128, h ? h + b * 128 : nullptr, dst + (size_t) b * bb, p);+ if (r.status == 2) { nf.fetch_add(1); }+ else s += r.err;+ if (r.status == 1) zb.fetch_add(1);+ if (err_out) err_out[b] = r.err;+ }+ }+ sums[ti] = s;+ };+ std::vector<std::thread> th;+ for (int i = 1; i < nthreads; ++i) th.emplace_back(work, i);+ work(0);+ for (auto& t : th) t.join();+ if (nonfinite) *nonfinite = nf.load();+ if (zero_blocks) *zero_blocks = zb.load();+ double tot = 0;+ for (double s : sums) tot += s;+ return tot;+}++} // namespace tqenc++// ---------------------------------------------------------------------------------------------------------------+// C API for the Python wrapper (tools/tqenc.py, ctypes)+extern "C" {+#if defined(_WIN32)+#define TQE_EXPORT __declspec(dllexport)+#else+#define TQE_EXPORT __attribute__((visibility("default")))+#endif+TQE_EXPORT int tqe_block_bytes(int type) { return tqenc::block_bytes(type); }+TQE_EXPORT int tqe_kbits(int type) { return tqenc::type_kbits(type); }+TQE_EXPORT void tqe_decode(int type, const uint8_t* src, int64_t nblocks, float* out) {+ tqenc::decode_blocks(type, src, nblocks, out);+}+// returns the summed block error; stats[0] = nonfinite blocks, stats[1] = zero blocks+TQE_EXPORT double tqe_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters,+ int ncand, int exact, int scale_tries, int nthreads, double* err_out, int64_t* stats) {+ tqenc::Params p;+ p.iters = iters;+ p.ncand = ncand;+ p.exact = exact;+ p.scale_tries = scale_tries;+ int64_t nf = 0, zb = 0;+ const double e = tqenc::encode_blocks(type, w, h, nblocks, dst, p, nthreads, err_out, &nf, &zb);+ if (stats) { stats[0] = nf; stats[1] = zb; }+ return e;+}+TQE_EXPORT const int16_t* tqe_table() { return tqenc::codebook_table(); }+}diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hppnew file mode 100644index 000000000..0ed36821d--- /dev/null+++ b/tools/trellis/tqenc.hpp@@ -0,0 +1,93 @@+// tools/trellis/tqenc.hpp - tail-biting Viterbi ENCODER for the trellis expert formats (TQ2_T, TQK6, TQK7 of the+// agentionai llama.cpp fork, ggml types 144-146) and two experimental lower-rate versions (TQK5X, TQK4X).+//+// Standalone (no ggml, no Strata engine): it only needs third_party/ggml/ggml-common-tq.h (the 2048-point codebook).+// The decoders it must match are src/kernels/cpu/tq_states.hpp / tq_cpu.cpp (tools/trellis/tqenc_test.cpp checks+// bit for bit against them, and the scalar / AVX2 / AVX-512 kernels against the encoder's own reconstruction).+//+// THE FORMAT (see tq_states.hpp, include/strata/kernels/cpu/tq_cpu.hpp, docs/GYRO.md):+// block = 128 weights = {fp16 d (little endian); qs[4*K]}, 32 steps t of 4 weights (weights 4t..4t+3).+// State of step t: 15 bits. weights = d * codebook(state); the state's hash x = state * 0x9e3779b1 picks two+// points lut[x >> 21] and lut[(x >> 10) & 2047] (fp16 pairs) = four values. The decoder works on the int16 table+// round(fp16 * 8192) (tq_state_table), so that table, divided by 8192, is the codebook here.+// TQK (K = 4,5,6,7): state(t) = bits [(31-t)K, (31-t)K + 15) of the qs bit stream (LSB first), circular modulo+// 32K bits. With u = 31 - t the states form a shift trellis: state(u+1) = (state(u) >> K) | (new K bits <<+// (15-K)); the K low bits of state(u) are stream symbol u. Tail-biting: state(0) closes the cycle.+// TQ2_T (K = 8): state(t) = ((qs[(t+31)%32] << 8) | qs[t]) & 0x7fff: a shift trellis in the other direction+// (state(t) = ((state(t-1) & 0x7f) << 8) | qs[t]).+// Both are a "de Bruijn" trellis of 2^15 states with 2^K branches per state, so one Viterbi step is a min over G+// groups of predecessors followed by a broadcast add (G = 2^(15-K) groups for TQK, 128 for TQ2_T), not 2^15 * 2^K+// additions. Tail-biting = a cycle of 32 states: the group of state(0) as successor equals the group of state(31)+// as predecessor. The search fixes that group c (exact: all G values; approximate: c taken from a free Viterbi pass+// on the cyclically rotated sequence, QTIP's approach, optionally the best of several rotations).+//+// New experimental types: TQK5X (id 224, 22 bytes / 128 weights, 1.375 bpw) and TQK4X (id 225, 18 bytes, 1.125 bpw):+// the TQK rule with K = 5 / 4, nothing else changed (same hash, codebook, mask, bit order, circular stream, fp16+// scale). EXPERIMENTAL_VERSION = 1: not in any ggml enum, not in any GGUF reader.+#pragma once++#include <cstddef>+#include <cstdint>++namespace tqenc {++inline constexpr int kBlock = 128;+inline constexpr int kStates = 1 << 15;+inline constexpr int kExperimentalVersion = 1;++// ggml type ids of the fork (decoders exist) and the experimental ones (encoder + Strata CPU decoder only)+enum Type : int { TQ2_T = 144, TQK6 = 145, TQK7 = 146, TQK5X = 224, TQK4X = 225 };++/// bits per step (8, 6, 7, 5, 4), 0 for any other type.+int type_kbits(int type);+inline int block_bytes(int type) { return type_kbits(type) ? 2 + 4 * type_kbits(type) : 0; }++struct Params {+ int iters = 3; ///< scale refinement rounds after the first search (path given d -> d given path)+ int ncand = 1; ///< tail-biting start-group candidates (rotations 16, 8, 24, 12, 20, 4, 28 ...); 1 = QTIP's+ int exact = 0; ///< 1: try every start group (the exact tail-biting optimum at the current scale; slow)+ int scale_tries = 1; ///< initial scales d0 * {1, 0.9, 1.1, 0.8, 1.2, ...}: the best of the first n searches+ uint16_t fixed_scale = 0; ///< nonzero: this fp16 scale is used as is (no refinement); tests, planted solutions+};++struct BlockResult {+ double err = 0; ///< sum_i h_i (w_i - rec_i)^2 for the stored block (h = 1 without importance weights)+ float scale = 0; ///< the stored fp16 scale+ int searches = 0; ///< Viterbi passes run+ int status = 0; ///< 0 ok, 1 zero block (all weights zero or below fp16 resolution), 2 nonfinite input+};++/// The codebook entry of a state as the decoder sees it: four floats (int16 table / 8192, exact).+const float* codebook_state(uint32_t state); // 4 floats+const int16_t* codebook_table(); // 32768 * 4 int16, the decoder's tq_state_table()++/// The state of step t (0..31) of a block's stream, by the decoder's rule, written bit by bit (independent of+/// tq_states.hpp). qs = the 4K stream bytes.+uint32_t block_state(int type, const uint8_t* qs, int t);++/// Decodes `nblocks` blocks (block_bytes(type) each) to 128 floats each: d * codebook, d the fp16 scale.+void decode_blocks(int type, const uint8_t* src, int64_t nblocks, float* out);++/// Encodes one block of 128 weights (importance weights h[128] or null) into block_bytes(type) bytes.+BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst, const Params& p);++/// Encodes nblocks consecutive blocks on `nthreads` threads (0: hardware concurrency). err_out (nblocks doubles) and+/// the returned sum of block errors are the objective sum h (w - rec)^2. Sets *nonfinite to the number of blocks+/// with a NaN/Inf input weight (those are written as zero blocks).+double encode_blocks(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, const Params& p,+ int nthreads, double* err_out, int64_t* nonfinite, int64_t* zero_blocks);++/// TEST HOOK: one Viterbi pass over a scaled target (T[p][4] = weights / d of processing position p, H[p][4]) and the+/// legal-path states by position. rot = first position processed, fixed_c = tail-biting group (-1 free).+float debug_viterbi(int type, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st);+/// Group sizes: predecessor groups G of a type; processing position p -> weight index of component j.+int debug_groups(int type);+int debug_widx(int type, int p, int j);+/// the stream bytes (4K) of a state path by position, as the encoder packs them+void debug_pack(int type, const uint16_t* st, uint8_t* qs);++// fp16 helpers (round to nearest even, like the hardware; exposed for the tests)+uint16_t float_to_half(float f);+float half_to_float(uint16_t h);++} // namespace tqencdiff --git a/tools/trellis/tqenc_test.cpp b/tools/trellis/tqenc_test.cppnew file mode 100644index 000000000..38596d0b1--- /dev/null+++ b/tools/trellis/tqenc_test.cpp@@ -0,0 +1,540 @@+// tools/trellis/tqenc_test.cpp - tests of the trellis encoder (tqenc.hpp) against Strata's own CPU decoder+// (src/kernels/cpu/tq_states.hpp, tq_cpu.cpp: scalar / AVX2 / AVX-512 kernels). Build and run: tools/trellis/build.sh test.+//+// T1 fp16 conversion (all 65536 halves, random floats vs the hardware where F16C exists)+// T2 the encoder's codebook == the decoder's state table (bit for bit, 32768 x 4)+// T3 random streams: block_state (bit by bit) == tq_states.hpp; every window obeys the shift-trellis relation,+// circularly (so a legal path = the chain relation + the closure state(31) -> state(0)); pack(unpack(s)) == s+// T4 the Viterbi pass == a naive DP written from the legality relation (every state, every predecessor), for a fixed+// start group (tail-biting), a free start and a rotated start, with importance weights+// T5 exhaustive: every symbol value at every stream position round-trips through pack/decode+// T6 planted solutions: weights decoded from a random legal stream and scale; exact search at that scale must find+// error 0 and bit-identical weights; the approximate search is reported (recovery rate)+// T7 encode -> decode: the encoder's reconstruction == the decoder's weights bit for bit (tq_states + state table),+// and the CPU kernels' dot products against Q8_0 activations == the dense dot of those weights+// T8 specials: zero blocks, tiny / huge / extreme scales, constants, spikes, NaN / Inf, determinism over threads+// T9 quality on Gaussian blocks: SQNR against the rate-distortion bound 6.02 * K/4 dB; approximate vs exact search+#include "tqenc.hpp"++#include "strata/kernels/cpu/tq_cpu.hpp"+#include "tq_states.hpp"++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++#if defined(__F16C__)+#include <immintrin.h>+#endif++namespace strata::kernels::cpu {+bool cpu_avx512_ok() { return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("avx512bw") && __builtin_cpu_supports("avx512vl") && __builtin_cpu_supports("avx512vbmi") && __builtin_cpu_supports("avx512vnni"); }+bool cpu_avx2_ok() { return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); }+} // namespace strata::kernels::cpu++using namespace tqenc;+namespace kc = strata::kernels::cpu;++static int g_fail = 0, g_checks = 0;+#define CHECK(cond, ...) \+ do { \+ ++g_checks; \+ if (!(cond)) { \+ ++g_fail; \+ std::printf(" FAIL %s:%d %s ", __FILE__, __LINE__, #cond); \+ std::printf(__VA_ARGS__); \+ std::printf("\n"); \+ } \+ } while (0)++static const int kTypes[5] = {TQ2_T, TQK6, TQK7, TQK5X, TQK4X};+static const char* tname(int t) {+ return t == TQ2_T ? "TQ2_T" : t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X";+}++template <int KB>+static void states_repo(const uint8_t* qs, uint32_t* st) { kc::tq_detail::tq_states<KB>(qs, st); }+static void states_repo_k(int K, const uint8_t* qs, uint32_t* st) {+ switch (K) {+ case 8: states_repo<8>(qs, st); break;+ case 7: states_repo<7>(qs, st); break;+ case 6: states_repo<6>(qs, st); break;+ case 5: states_repo<5>(qs, st); break;+ default: states_repo<4>(qs, st); break;+ }+}++static std::mt19937_64 rng(12345);+static float gauss() { static std::normal_distribution<float> n(0.f, 1.f); return n(rng); }++// chain relation between consecutive steps, circularly: returns true when state(t) and state(t') obey the shift trellis+static bool chain_ok(int type, const uint32_t* st) {+ const int K = type_kbits(type);+ for (int t = 0; t < 32; ++t) {+ if (type == TQ2_T) { // state(t) = ((state(t-1) & 0x7f) << 8) | byte+ const int tp = (t + 31) & 31;+ if ((st[t] >> 8) != (st[tp] & 0x7f)) return false;+ } else { // u = 31 - t; state(u+1) low (15-K) bits = state(u) >> K, i.e. step t-1 follows step t+ const int tp = (t + 31) & 31;+ if ((st[tp] & ((1u << (15 - K)) - 1u)) != (st[t] >> K)) return false;+ }+ }+ return true;+}++static void t1() {+ std::printf("T1 fp16\n");+ for (uint32_t h = 0; h < 65536; ++h) {+ const float f = half_to_float((uint16_t) h);+ if (std::isnan(f)) continue;+ CHECK(float_to_half(f) == (uint16_t) h || (h == 0x8000 && float_to_half(f) == 0x8000), "half %04x", h);+ }+#if defined(__F16C__)+ int bad = 0;+ std::uniform_real_distribution<float> u(-1.f, 1.f);+ for (int i = 0; i < 4000000; ++i) {+ float f = u(rng) * std::pow(2.f, (float) (int) (rng() % 50) - 30.f);+ if (i % 7 == 0) f = std::ldexp(u(rng), -25 + (int) (rng() % 6)); // subnormal range and its ties+ if (i % 11 == 0) f = std::ldexp(u(rng), 14 + (int) (rng() % 3)); // the overflow edge+ const uint16_t hw = _cvtss_sh(f, _MM_FROUND_TO_NEAREST_INT);+ bad += float_to_half(f) != hw;+ }+ CHECK(bad == 0, "float_to_half vs F16C: %d differ", bad);+#endif+}++static void t2() {+ std::printf("T2 codebook == decoder table\n");+ const int16_t* a = codebook_table();+ const int16_t* b = kc::tq_state_table();+ CHECK(std::memcmp(a, b, 32768 * 4 * sizeof(int16_t)) == 0, "tables differ");+}++static void t3() {+ std::printf("T3 states of random streams\n");+ for (int type : kTypes) {+ const int K = type_kbits(type), nb = 4 * K;+ int bad_states = 0, bad_chain = 0, bad_pack = 0;+ for (int it = 0; it < 20000; ++it) {+ uint8_t qs[32];+ for (int i = 0; i < nb; ++i) qs[i] = (uint8_t) rng();+ uint32_t sr[32];+ states_repo_k(K, qs, sr);+ uint32_t sm[32];+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);+ bad_states += std::memcmp(sr, sm, sizeof sr) != 0;+ bad_chain += !chain_ok(type, sm);+ // pack(unpack): processing order p -> step t+ uint16_t by_pos[32];+ for (int p = 0; p < 32; ++p) by_pos[p] = (uint16_t) sm[type == TQ2_T ? p : 31 - p];+ uint8_t q2[32];+ debug_pack(type, by_pos, q2);+ // TQ2_T's bit 7 of each byte is part of the stream but the state uses only 15 bits of two bytes: bit 7 of+ // qs[t] is in state(t) (bit 7), so it survives; every stream bit is in some state+ bad_pack += std::memcmp(qs, q2, nb) != 0;+ }+ std::printf(" %-6s states vs tq_states.hpp: %d bad, chain relation: %d bad, pack(unpack): %d bad\n", tname(type),+ bad_states, bad_chain, bad_pack);+ CHECK(bad_states == 0 && bad_chain == 0 && bad_pack == 0, "%s", tname(type));+ }+}++// naive DP from the legality relation, one pass; same conventions as debug_viterbi+static float naive(int type, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st_out) {+ const int K = type_kbits(type);+ const bool t2 = type == TQ2_T;+ const int G = debug_groups(type);+ std::vector<std::vector<float>> D(32, std::vector<float>(32768, 1e30f));+ std::vector<std::vector<int>> B(32, std::vector<int>(32768, -1));+ auto cost = [&](int p, uint32_t s) {+ const float* c = codebook_state(s);+ float e = 0;+ for (int j = 0; j < 4; ++j) { const float d = T[p][j] - c[j]; e += H[p][j] * d * d; }+ return e;+ };+ auto legal = [&](uint32_t prev, uint32_t next) { // next may follow prev+ return t2 ? (next >> 8) == (prev & 0x7f) : (next & (uint32_t) (G - 1)) == (prev >> K);+ };+ for (uint32_t s = 0; s < 32768; ++s) {+ const bool ok = fixed_c < 0 || (t2 ? (int) (s >> 8) == fixed_c : (int) (s & (G - 1)) == fixed_c);+ if (ok) D[0][s] = cost(rot & 31, s);+ }+ for (int q = 1; q < 32; ++q)+ for (uint32_t s = 0; s < 32768; ++s) {+ float best = 1e30f;+ int arg = -1;+ // enumerate the predecessors from the relation (all prev with legal(prev, s)): scan the 2^K candidates+ const int n = 1 << K;+ for (int y = 0; y < (t2 ? 256 : n); ++y) {+ const uint32_t prev = t2 ? ((uint32_t) y << 7) | (s >> 8) : (((s & (uint32_t) (G - 1)) << K) | (uint32_t) y);+ if (!legal(prev, s)) continue;+ if (D[q - 1][prev] < best) { best = D[q - 1][prev]; arg = (int) prev; }+ }+ if (arg >= 0) { D[q][s] = best + cost((rot + q) & 31, s); B[q][s] = arg; }+ }+ float best = 1e30f;+ int bs = -1;+ for (uint32_t s = 0; s < 32768; ++s) {+ const bool ok = fixed_c < 0 || (t2 ? (int) (s & 0x7f) == fixed_c : (int) (s >> K) == fixed_c);+ if (ok && D[31][s] < best) { best = D[31][s]; bs = (int) s; }+ }+ if (bs < 0) return 1e30f;+ int s = bs;+ for (int q = 31; q >= 0; --q) {+ st_out[(rot + q) & 31] = (uint16_t) s;+ if (q > 0) s = B[q][s];+ }+ return best;+}++static void t4() {+ std::printf("T4 Viterbi vs naive DP\n");+ for (int type : kTypes) {+ const int G = debug_groups(type);+ double worst = 0;+ int cases = 0;+ for (int trial = 0; trial < 3; ++trial) {+ float T[32][4], H[32][4];+ for (int p = 0; p < 32; ++p)+ for (int j = 0; j < 4; ++j) {+ T[p][j] = gauss() * (trial == 2 ? 2.5f : 1.0f);+ H[p][j] = trial == 0 ? 1.f : 0.1f + std::fabs(gauss());+ }+ struct Case { int rot, c; };+ Case cs[3] = {{0, (int) (rng() % G)}, {0, -1}, {13, -1}};+ for (const Case& cc : cs) {+ uint16_t sa[32], sb[32];+ const float a = debug_viterbi(type, T, H, cc.rot, cc.c, sa);+ const float b = naive(type, T, H, cc.rot, cc.c, sb);+ const double rel = std::fabs(a - b) / std::max(1e-6f, b);+ worst = std::max(worst, rel);+ CHECK(rel < 2e-5, "%s trial %d rot %d c %d: viterbi %.6f naive %.6f", tname(type), trial, cc.rot, cc.c, a, b);+ // the returned path is legal and its own cost is the reported cost+ uint32_t sm[32];+ uint8_t qs[32];+ debug_pack(type, sa, qs);+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);+ bool same = true;+ for (int p = 0; p < 32; ++p) {+ const uint32_t d = sm[type == TQ2_T ? p : 31 - p];+ if (cc.rot == 0 || cc.c < 0) {+ // a free rotated pass is a legal chain only if the stream closes; compare only closed ones+ }+ same = same && (cc.c < 0 ? true : d == sa[p]);+ }+ if (cc.c >= 0) CHECK(same, "%s tail-biting path does not decode to its own states", tname(type));+ double sum = 0;+ for (int q = 0; q < 32; ++q) {+ const int p = (cc.rot + q) & 31;+ const float* c = codebook_state(sa[p]);+ for (int j = 0; j < 4; ++j) { const double d = T[p][j] - c[j]; sum += H[p][j] * d * d; }+ }+ CHECK(std::fabs(sum - a) < 1e-3 * std::max(1.0, (double) a), "path cost %.6f vs reported %.6f", sum, a);+ ++cases;+ }+ }+ std::printf(" %-6s %d cases, worst relative cost difference %.2e\n", tname(type), cases, worst);+ }+}++static void t5() {+ std::printf("T5 every symbol value at every position\n");+ for (int type : kTypes) {+ const int K = type_kbits(type), nb = 4 * K;+ long bad = 0, n = 0;+ if (type == TQ2_T) {+ for (int pos = 0; pos < 32; ++pos)+ for (int x = 0; x < 256; ++x) {+ uint8_t qs[32];+ for (int i = 0; i < nb; ++i) qs[i] = (uint8_t) rng();+ qs[pos] = (uint8_t) x;+ uint32_t sm[32];+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);+ uint16_t bp[32];+ for (int p = 0; p < 32; ++p) bp[p] = (uint16_t) sm[p];+ uint8_t q2[32];+ debug_pack(type, bp, q2);+ bad += std::memcmp(qs, q2, nb) != 0 || !chain_ok(type, sm);+ ++n;+ }+ } else {+ for (int u = 0; u < 32; ++u)+ for (int x = 0; x < (1 << K); ++x) {+ uint8_t qs[32];+ for (int i = 0; i < nb; ++i) qs[i] = (uint8_t) rng();+ // overwrite symbol u+ for (int b = 0; b < K; ++b) {+ const int bit = u * K + b;+ qs[bit >> 3] = (uint8_t) ((qs[bit >> 3] & ~(1u << (bit & 7))) | (((x >> b) & 1u) << (bit & 7)));+ }+ uint32_t sm[32];+ for (int t = 0; t < 32; ++t) sm[t] = block_state(type, qs, t);+ // symbol u is the low K bits of the state of step 31 - u+ bad += (int) (sm[31 - u] & ((1u << K) - 1u)) != x;+ uint16_t bp[32];+ for (int p = 0; p < 32; ++p) bp[p] = (uint16_t) sm[31 - p];+ uint8_t q2[32];+ debug_pack(type, bp, q2);+ bad += std::memcmp(qs, q2, nb) != 0 || !chain_ok(type, sm);+ ++n;+ }+ }+ std::printf(" %-6s %ld cases, %ld bad\n", tname(type), n, bad);+ CHECK(bad == 0, "%s", tname(type));+ }+}++static void t6() {+ std::printf("T6 planted solutions\n");+ for (int type : kTypes) {+ const int K = type_kbits(type), nb = 4 * K, bb = block_bytes(type);+ const int N = (type == TQK4X ? 12 : 30);+ int exact_ok = 0, approx_ok1 = 0, approx_ok3 = 0;+ double approx_err_sum = 0, wsum = 0;+ for (int it = 0; it < N; ++it) {+ std::vector<uint8_t> blk(bb);+ const float dsc = std::ldexp(0.5f + (float) (rng() % 1000) / 1000.f, -6 + (int) (rng() % 10));+ const uint16_t dh = float_to_half(dsc);+ blk[0] = (uint8_t) (dh & 255);+ blk[1] = (uint8_t) (dh >> 8);+ // a legal stream: any bytes are legal (every stream is a closed cycle by construction); for TQ2_T too+ for (int i = 0; i < nb; ++i) blk[2 + i] = (uint8_t) rng();+ std::vector<float> w(128);+ decode_blocks(type, blk.data(), 1, w.data());+ for (float x : w) wsum += (double) x * x;+ Params p;+ p.fixed_scale = dh;+ std::vector<uint8_t> out(bb);+ p.exact = 1;+ BlockResult r = encode_block(type, w.data(), nullptr, out.data(), p);+ std::vector<float> w2(128);+ decode_blocks(type, out.data(), 1, w2.data());+ const bool same = std::memcmp(w.data(), w2.data(), 512) == 0;+ exact_ok += (r.err == 0.0 && same);+ CHECK(r.err == 0.0 && same, "%s exact search missed a planted solution (err %g)", tname(type), r.err);+ p.exact = 0;+ p.ncand = 1;+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ approx_ok1 += r.err == 0.0;+ approx_err_sum += r.err;+ p.ncand = 3;+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ approx_ok3 += r.err == 0.0;+ }+ std::printf(" %-6s exact: %d/%d recovered; approximate (rotation 16): %d/%d (err/energy %.2e); best of 3 rotations: %d/%d\n",+ tname(type), exact_ok, N, approx_ok1, N, approx_err_sum / wsum, approx_ok3, N);+ }+}++static uint16_t q8_scale(const std::vector<float>& x, int b) {+ float amax = 0;+ for (int j = 0; j < 32; ++j) amax = std::max(amax, std::fabs(x[32 * b + j]));+ return float_to_half(amax / 127.f);+}++static void t7() {+ std::printf("T7 encoder reconstruction == decoder, kernels vs dense\n");+ for (int type : kTypes) {+ const int K = type_kbits(type), bb = block_bytes(type);+ const int rows = 24, n = 512, nbr = n / 128;+ std::vector<float> w((size_t) rows * n);+ for (int r = 0; r < rows; ++r)+ for (int i = 0; i < n; ++i) {+ float v = gauss() * 0.02f * (1.f + (r % 5));+ if (r == 3) v = (i % 97 == 0) ? 1.5f : 0.001f * gauss(); // spiky row+ if (r == 4) v = 0.f; // zero row+ if (r == 5) v *= 300.f; // big+ w[(size_t) r * n + i] = v;+ }+ std::vector<uint8_t> enc((size_t) rows * nbr * bb);+ int64_t nf = 0, zb = 0;+ std::vector<double> errs((size_t) rows * nbr);+ Params p;+ const double tot = encode_blocks(type, w.data(), nullptr, (int64_t) rows * nbr, enc.data(), p, 8, errs.data(), &nf, &zb);+ (void) tot;+ // decoder reference: tq_states.hpp + the state table+ const int16_t* tab = kc::tq_state_table();+ std::vector<float> ref((size_t) rows * n), ours((size_t) rows * n);+ for (int64_t b = 0; b < (int64_t) rows * nbr; ++b) {+ const uint8_t* blk = enc.data() + b * bb;+ const float d = kc::tq_detail::tq_half_to_float(kc::tq_detail::tq_ld16(blk));+ uint32_t st[32];+ states_repo_k(K, blk + 2, st);+ for (int t = 0; t < 32; ++t)+ for (int j = 0; j < 4; ++j) ref[b * 128 + 4 * t + j] = d * ((float) tab[4 * st[t] + j] * (1.0f / 8192.0f));+ }+ decode_blocks(type, enc.data(), (int64_t) rows * nbr, ours.data());+ CHECK(std::memcmp(ref.data(), ours.data(), ref.size() * 4) == 0, "%s decode != tq_states + state table", tname(type));+ // the encoder's own error equals the error of the decoded weights+ double worst = 0;+ for (int64_t b = 0; b < (int64_t) rows * nbr; ++b) {+ double e = 0;+ for (int i = 0; i < 128; ++i) { const double d = (double) w[b * 128 + i] - ref[b * 128 + i]; e += d * d; }+ worst = std::max(worst, std::fabs(e - errs[b]) / std::max(1e-30, e + errs[b]));+ }+ CHECK(worst < 1e-9, "%s encoder error != decoded error (%.3e)", tname(type), worst);+ // kernels (scalar, AVX2, AVX-512 where present) vs the dense dot with the dequantized Q8_0 activation+ std::vector<float> x(n);+ for (auto& v : x) v = gauss();+ std::vector<uint8_t> q8((size_t) (n / 32) * 34);+ std::vector<float> xd(n);+ for (int b = 0; b < n / 32; ++b) {+ const uint16_t dh = q8_scale(x, b);+ const float d = half_to_float(dh);+ q8[34 * b] = (uint8_t) (dh & 255);+ q8[34 * b + 1] = (uint8_t) (dh >> 8);+ for (int j = 0; j < 32; ++j) {+ const int q = d == 0.f ? 0 : (int) std::lround(x[32 * b + j] / d);+ q8[34 * b + 2 + j] = (uint8_t) (int8_t) std::max(-127, std::min(127, q));+ xd[32 * b + j] = d * (float) (int8_t) q8[34 * b + 2 + j];+ }+ }+ const void* ap[1] = {q8.data()};+ for (kc::TqIsa isa : {kc::TqIsa::scalar, kc::TqIsa::avx2, kc::TqIsa::avx512}) {+ if (!kc::tq_isa_ok(isa)) continue;+ std::vector<float> out(rows);+ float* op[1] = {out.data()};+ kc::tq_rows(isa, type, enc.data(), (size_t) nbr * bb, n, ap, 1, op, 0, rows);+ double worst_rel = 0;+ for (int r = 0; r < rows; ++r) {+ double dsum = 0, mag = 0;+ for (int i = 0; i < n; ++i) {+ dsum += (double) ref[(size_t) r * n + i] * xd[i];+ mag += std::fabs((double) ref[(size_t) r * n + i] * xd[i]);+ }+ worst_rel = std::max(worst_rel, std::fabs(dsum - out[r]) / (mag + 1e-30));+ }+ std::printf(" %-6s %-6s kernel vs dense dot of the decoded weights: worst error %.2e of sum|w a|\n", tname(type),+ kc::tq_isa_name(isa), worst_rel);+ CHECK(worst_rel < 1e-5, "%s %s", tname(type), kc::tq_isa_name(isa));+ }+ std::printf(" %-6s decode == tq_states+table: bit-exact, %zu weights; zero blocks %lld\n", tname(type), ref.size(), (long long) zb);+ }+}++static void t8() {+ std::printf("T8 specials\n");+ for (int type : kTypes) {+ const int bb = block_bytes(type);+ std::vector<uint8_t> out(bb), z(bb, 0);+ std::vector<float> w(128), dec(128);+ Params p;+ // zero block+ std::fill(w.begin(), w.end(), 0.f);+ BlockResult r = encode_block(type, w.data(), nullptr, out.data(), p);+ CHECK(r.status == 1 && out == z, "%s zero block must be canonical all-zero bytes", tname(type));+ w[5] = -0.f;+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ CHECK(r.status == 1 && out == z, "%s negative zero", tname(type));+ decode_blocks(type, out.data(), 1, dec.data());+ for (float v : dec) CHECK(v == 0.f, "zero block decodes to 0");+ // scales: from the smallest subnormal weights to beyond fp16+ const float mags[] = {1e-9f, 3e-8f, 1e-7f, 6e-5f, 1e-3f, 1.f, 100.f, 3000.f, 30000.f, 60000.f, 65000.f, 1e5f, 1e7f};+ for (float mag : mags) {+ for (int i = 0; i < 128; ++i) w[i] = mag * gauss();+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ const uint16_t dh = (uint16_t) (out[0] | (out[1] << 8));+ const float d = half_to_float(dh);+ CHECK(std::isfinite(d) && d >= 0.f, "%s scale %g not finite for magnitude %g", tname(type), d, mag);+ decode_blocks(type, out.data(), 1, dec.data());+ double e = 0, en = 0;+ bool fin = true;+ for (int i = 0; i < 128; ++i) { fin &= std::isfinite(dec[i]); e += (w[i] - dec[i]) * (double) (w[i] - dec[i]); en += (double) w[i] * w[i]; }+ CHECK(fin, "decoded nonfinite");+ std::printf(" %-6s magnitude %-8g d=%-12g rel err %.3f status %d\n", tname(type), mag, d, e / std::max(en, 1e-300), r.status);+ if (mag >= 1e-3f && mag <= 60000.f) CHECK(e / en < 0.5, "%s error too large at magnitude %g (%.3f)", tname(type), mag, e / en);+ }+ // constant, spike, alternating, power of two+ for (int variant = 0; variant < 4; ++variant) {+ for (int i = 0; i < 128; ++i)+ w[i] = variant == 0 ? 0.37f : variant == 1 ? (i == 77 ? 5.f : 0.f) : variant == 2 ? (i & 1 ? 1.f : -1.f) : std::ldexp(1.f, -3);+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ decode_blocks(type, out.data(), 1, dec.data());+ double e = 0, en = 0;+ for (int i = 0; i < 128; ++i) { e += (w[i] - dec[i]) * (double) (w[i] - dec[i]); en += (double) w[i] * w[i]; }+ std::printf(" %-6s pattern %d rel err %.4f\n", tname(type), variant, e / en);+ CHECK(e / en < 0.8, "%s pattern %d", tname(type), variant);+ }+ // non-finite input+ for (int i = 0; i < 128; ++i) w[i] = gauss();+ w[17] = INFINITY;+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ CHECK(r.status == 2 && out == z, "%s inf input", tname(type));+ w[17] = NAN;+ r = encode_block(type, w.data(), nullptr, out.data(), p);+ CHECK(r.status == 2 && out == z, "%s nan input", tname(type));+ }+ // determinism over thread counts+ {+ const int type = TQK6, nbl = 64, bb = block_bytes(type);+ std::vector<float> w((size_t) nbl * 128);+ for (auto& v : w) v = gauss();+ std::vector<uint8_t> a((size_t) nbl * bb), b(a.size());+ Params p;+ encode_blocks(type, w.data(), nullptr, nbl, a.data(), p, 1, nullptr, nullptr, nullptr);+ encode_blocks(type, w.data(), nullptr, nbl, b.data(), p, 7, nullptr, nullptr, nullptr);+ CHECK(a == b, "1 thread vs 7 threads differ");+ }+}++static void t9() {+ std::printf("T9 quality on N(0,1) blocks (SQNR in dB; the rate-distortion limit of the K/4 bits per weight is 6.02 * K/4)\n");+ for (int type : kTypes) {+ const int K = type_kbits(type), bb = block_bytes(type);+ const int nbl = (type == TQ2_T || type == TQK7) ? 256 : 384;+ std::vector<float> w((size_t) nbl * 128);+ for (auto& v : w) v = gauss();+ std::vector<uint8_t> out((size_t) nbl * bb);+ double en = 0;+ for (float v : w) en += (double) v * v;+ struct Cfg { const char* name; Params p; int th; };+ Cfg cfgs[5];+ cfgs[0] = {"1 pass, scale from rms", Params{0, 1, 0, 1, 0}, 8};+ cfgs[1] = {"default (rot 16, 3 scale rounds)", Params{}, 8};+ cfgs[2] = {"3 rotations, 3 scale rounds", Params{3, 3, 0, 1, 0}, 8};+ cfgs[3] = {"3 rotations, 3 starting scales", Params{3, 3, 0, 3, 0}, 8};+ int ncfg = 4;+ for (int i = 0; i < ncfg; ++i) {+ const auto t0 = std::chrono::steady_clock::now();+ const double e = encode_blocks(type, w.data(), nullptr, nbl, out.data(), cfgs[i].p, cfgs[i].th, nullptr, nullptr, nullptr);+ const double sec = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();+ const double sq = 10.0 * std::log10(en / e);+ std::printf(" %-6s %-34s SQNR %.3f dB (limit %.2f, gap %.2f dB) %.0f weights/s (%d threads)\n", tname(type),+ cfgs[i].name, sq, 6.0206 * K / 4.0, 6.0206 * K / 4.0 - sq, (double) nbl * 128 / sec, cfgs[i].th);+ }+ // approximate vs exact start group at the default scale rounds on a few blocks+ {+ const int nb2 = type == TQK4X ? 2 : 8;+ Params pe;+ pe.exact = 1;+ pe.iters = 0;+ Params pa;+ pa.iters = 0;+ const double ee = encode_blocks(type, w.data(), nullptr, nb2, out.data(), pe, 8, nullptr, nullptr, nullptr);+ const double ea = encode_blocks(type, w.data(), nullptr, nb2, out.data(), pa, 8, nullptr, nullptr, nullptr);+ std::printf(" %-6s tail-biting search, %d blocks, one scale: approximate error / exact error = %.4f\n", tname(type), nb2, ea / ee);+ CHECK(ea >= ee * (1 - 1e-6), "approximate beat exact?");+ }+ }+}++int main(int argc, char** argv) {+ const bool quick = argc > 1 && std::strcmp(argv[1], "--quick") == 0;+ t1();+ t2();+ t3();+ t5();+ t4();+ t6();+ t7();+ t8();+ if (!quick) t9();+ std::printf("\n%d checks, %d failures\n", g_checks, g_fail);+ return g_fail ? 1 : 0;+}