Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
K5/K4 version 2: computed (LUT-free) codebook for TQK5X/TQK4X, point-table versions kept as TQK5T/TQK4T; CPU decoder tables per type; tests
Niko1221a7c50df0029e
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hppindex a73e300e8..ac6d6f176 100644--- a/include/strata/kernels/cpu/tq_cpu.hpp+++ b/include/strata/kernels/cpu/tq_cpu.hpp@@ -34,9 +34,11 @@ enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };/// EXPERIMENTAL lower-rate versions of the TQK rule (tools/trellis/tqenc.hpp, version 1; Strata-internal ids, not ggml/// types, not in any GGUF): TQK5X = 22 bytes / 128 weights (1.375 bpw), TQK4X = 18 bytes (1.125 bpw). Same hash,/// codebook, 15-bit mask, LSB-first circular stream and fp16 scale as TQK6/7 with K = 5 / 4.-inline constexpr int kTqTypeK5X = 224;-inline constexpr int kTqTypeK4X = 225;-/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X.+inline constexpr int kTqTypeK5X = 224; ///< version 2: K = 5 with the computed codebook (tq_computed.hpp)+inline constexpr int kTqTypeK4X = 225; ///< version 2: K = 4, computed codebook+inline constexpr int kTqTypeK5T = 226; ///< version 1: K = 5 with the fork's point table (the first pilots)+inline constexpr int kTqTypeK4T = 227; ///< version 1: K = 4, point table+/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X / TQK5T / TQK4T.bool tq_supported(int ggml_type) noexcept;/// Bits per step: 8, 6, 7, 5, 4 (0 for any other type).int tq_kbits(int ggml_type) noexcept;@@ -54,8 +56,12 @@ bool tq_isa_ok(TqIsa isa);bool tq_kernels_enabled();const char* tq_isa_name(TqIsa isa);-/// The 32768 x 4 int16 state table (filled on first use; thread-safe).+/// The 32768 x 4 int16 state table of the fork's types (filled on first use; thread-safe).const int16_t* tq_state_table();+/// The same for the computed codebook of TQK5X / TQK4X (tq_computed.hpp).+const int16_t* tq_state_table_computed();+/// The table a type decodes with.+const int16_t* tq_state_table_for(int ggml_type);/// Rows [r0, r1) of a trellis matrix (`row_bytes` per row, `n` weights per row) against `nt` (<= 8) Q8_0/// activations of n values: out[t][r] = w_r . a[t].@@ -78,11 +84,11 @@ struct TqAct {};void unpack_act(const void* q8_0, int n, TqAct& a);// the per-ISA row kernels (dot products only; `out[t][r]` for rows [r0, r1))-void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1);-void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+void tq_rows_avx2(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1);-void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+void tq_rows_avx512(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1);} // namespace tq_detaildiff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cppindex f03892510..9a43393fe 100644--- a/src/kernels/cpu/tq_avx2.cpp+++ b/src/kernels/cpu/tq_avx2.cpp@@ -29,11 +29,11 @@ inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2}template <int KB>-void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,+void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,int r1) {const int nb = n / 128;constexpr size_t bsz = 2 + 4 * KB;- const long long* tab = (const long long*) tq_state_table();+ const long long* tab = (const long long*) table;for (int r = r0; r < r1; ++r) {const uint8_t* x = w + (size_t) r * row_bytes;__m128 acc[kTqMaxTokens];@@ -77,13 +77,13 @@ void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, in} // namespace-void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+void tq_rows_avx2(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1) {- if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);- else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);- else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X- else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X- else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+ if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_t<6>(table, w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 5) rows_t<5>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X+ else if (kb == 4) rows_t<4>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X+ else rows_t<7>(table, w, row_bytes, n, a, nt, out, r0, r1);}} // namespace strata::kernels::cpu::tq_detaildiff --git a/src/kernels/cpu/tq_avx512.cpp b/src/kernels/cpu/tq_avx512.cppindex 9031b8cf5..49ec0726f 100644--- a/src/kernels/cpu/tq_avx512.cpp+++ b/src/kernels/cpu/tq_avx512.cpp@@ -47,11 +47,11 @@ inline __m512i rows8(const long long* tab, const uint8_t* qs, uint64_t wr, int t}template <int KB>-void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,+void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,int r1) {const int nb = n / 128;constexpr size_t bsz = 2 + 4 * KB;- const long long* tab = (const long long*) tq_state_table();+ const long long* tab = (const long long*) table;for (int r = r0; r < r1; ++r) {const uint8_t* x = w + (size_t) r * row_bytes;__m128 acc[kTqMaxTokens];@@ -85,13 +85,13 @@ void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, in} // namespace-void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+void tq_rows_avx512(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1) {- if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);- else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);- else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X- else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X- else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+ if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_t<6>(table, w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 5) rows_t<5>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X+ else if (kb == 4) rows_t<4>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X+ else rows_t<7>(table, w, row_bytes, n, a, nt, out, r0, r1);}} // namespace strata::kernels::cpu::tq_detaildiff --git a/src/kernels/cpu/tq_computed.hpp b/src/kernels/cpu/tq_computed.hppnew file mode 100644index 000000000..3222239fa--- /dev/null+++ b/src/kernels/cpu/tq_computed.hpp@@ -0,0 +1,33 @@+// src/kernels/cpu/tq_computed.hpp - the LUT-free ("computed") codebook of the experimental trellis types TQK5X / TQK4X+// (version 2): a state's four weights come from two 32-bit LCG words instead of the hashed 2048-point table.+//+// r_i = ((s * A_i + B_i) & 0x8FFF8FFF) ^ 0x3B603B60 i = 0, 1 (s = the 15-bit state)+// weights 0, 1 = the low and high fp16 of r_0, weights 2, 3 = those of r_1+//+// (QTIP's 3INST word, Tseng et al., Cornell-RelaxML/qtip lib/codebook/bitshift.py; the variant "3single" of the+// f-dsv41-cudalab study, which measured it at the SQNR of the table on real DeepSeek experts and a GEMV without+// shared-memory lookups.) The mask fixes the top exponent bits, so |w| lies in [0.125, 2) and the int16 table value+// round(w * 2^13) used by the CPU kernels is an exact integer (1024..16383): decoder and encoder agree to the bit.+// The existing types (TQ2_T, TQK6, TQK7) and the table-based K5 / K4 (ids 226 / 227, "version 1") do not use it.+#pragma once++#include <cstdint>++#include "tq_states.hpp"++namespace strata::kernels::cpu::tq_detail {++inline constexpr uint32_t kCompA[2] = {89226354u, 0x9E3779B1u};+inline constexpr uint32_t kCompB[2] = {64248484u, 0x7F4A7C15u};+inline constexpr uint32_t kCompMask = 0x8FFF8FFFu, kCompXor = 0x3B603B60u;++inline uint32_t comp_word(uint32_t s, int i) { return ((s * kCompA[i] + kCompB[i]) & kCompMask) ^ kCompXor; }++/// The four weights of state s at scale 2^13 (int16, exact).+inline void comp_state4(uint32_t s, int16_t out[4]) {+ const uint32_t r0 = comp_word(s & 0x7fffu, 0), r1 = comp_word(s & 0x7fffu, 1);+ const uint16_t h[4] = {(uint16_t) (r0 & 0xffffu), (uint16_t) (r0 >> 16), (uint16_t) (r1 & 0xffffu), (uint16_t) (r1 >> 16)};+ for (int j = 0; j < 4; ++j) out[j] = (int16_t) (tq_half_to_float(h[j]) * 8192.0f);+}++} // namespace strata::kernels::cpu::tq_detaildiff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cppindex ad7ea297d..22a874bcb 100644--- a/src/kernels/cpu/tq_cpu.cpp+++ b/src/kernels/cpu/tq_cpu.cpp@@ -5,6 +5,7 @@#include "strata/kernels/cpu/expert_layout.hpp"#include "tq_states.hpp"+#include "tq_computed.hpp"#include "ggml-common-tq.h"#include <cmath>@@ -25,6 +26,7 @@ namespace strata::kernels::cpu {namespace {alignas(64) int16_t g_table[(tq_detail::kTqStateMask + 1) * 4];+alignas(64) int16_t g_table_c[(tq_detail::kTqStateMask + 1) * 4];void fill_table() {for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) {@@ -39,6 +41,10 @@ void fill_table() {}}+void fill_table_computed() {+ for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) tq_detail::comp_state4(s, g_table_c + 4 * s);+}+int env_isa_cap() {const char* e = std::getenv("STRATA_TQ_ISA");if (e == nullptr || *e == 0) return 2;@@ -49,11 +55,10 @@ int env_isa_cap() {}template <int KB>-void rows_scalar_t(const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,+void rows_scalar_t(const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,float* const* out, int r0, int r1) {const int nb = n / 128;constexpr size_t bsz = 2 + 4 * KB;- const int16_t* tab = tq_state_table();for (int r = r0; r < r1; ++r) {const uint8_t* x = w + (size_t) r * row_bytes;float acc[kTqMaxTokens][4] = {};@@ -89,10 +94,20 @@ const int16_t* tq_state_table() {return g_table;}-bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X; }+const int16_t* tq_state_table_computed() {+ static std::once_flag once;+ std::call_once(once, fill_table_computed);+ return g_table_c;+}++const int16_t* tq_state_table_for(int t) { return (t == kTqTypeK5X || t == kTqTypeK4X) ? tq_state_table_computed() : tq_state_table(); }++bool tq_supported(int t) noexcept {+ return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X || t == kTqTypeK5T || t == kTqTypeK4T;+}int tq_kbits(int t) noexcept {- return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : t == kTqTypeK5X ? 5 : t == kTqTypeK4X ? 4 : 0;+ return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : (t == kTqTypeK5X || t == kTqTypeK5T) ? 5 : (t == kTqTypeK4X || t == kTqTypeK4T) ? 4 : 0;}bool tq_isa_ok(TqIsa isa) {@@ -142,13 +157,13 @@ void unpack_act(const void* q8_0, int n, TqAct& a) {}}-void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1) {- if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);- else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);- else if (kb == 5) rows_scalar_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X- else if (kb == 4) rows_scalar_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X- else rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+ if (kb == 8) rows_scalar_t<8>(tab, w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_scalar_t<6>(tab, w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 5) rows_scalar_t<5>(tab, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X / TQK5T+ else if (kb == 4) rows_scalar_t<4>(tab, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X / TQK4T+ else rows_scalar_t<7>(tab, w, row_bytes, n, a, nt, out, r0, r1);}} // namespace tq_detail@@ -171,14 +186,14 @@ ActSet& unpack_all(const void* const* act, int n, int nt) {return *set;}-void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,- float* const* out, int r0, int r1) {+void rows_on(TqIsa isa, int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n,+ const tq_detail::TqAct* const* a, int nt, float* const* out, int r0, int r1) {switch (isa) {#if defined(STRATA_TQ_X86)- case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, w, row_bytes, n, a, nt, out, r0, r1); break;- case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, w, row_bytes, n, a, nt, out, r0, r1); break;+ case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;+ case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;#endif- default: tq_detail::tq_rows_scalar(kb, w, row_bytes, n, a, nt, out, r0, r1); break;+ default: tq_detail::tq_rows_scalar(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;}}@@ -187,15 +202,15 @@ void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, constvoid tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,float* const* out, int r0, int r1) {if (!tq_isa_ok(isa)) std::abort(); // a caller bug: asked for an ISA this CPU does not have- tq_state_table();+ const int16_t* tab = tq_state_table_for(ggml_type);const ActSet& s = unpack_all(act, n, nt);- rows_on(isa, tq_kbits(ggml_type), w, row_bytes, n, s.ptr, nt, out, r0, r1);+ rows_on(isa, tq_kbits(ggml_type), tab, w, row_bytes, n, s.ptr, nt, out, r0, r1);}void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,const void* const* act, int nt, float* const* ff, int r0, int r1) {if (!tq_isa_ok(isa)) std::abort();- tq_state_table();+ const int16_t* tab = tq_state_table_for(ggml_type);const ActSet& s = unpack_all(act, n, nt);thread_local std::vector<float> gbuf, ubuf;const size_t nr = (size_t) r1; // row r of token t at [t * r1 + r]@@ -207,8 +222,8 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, siup[t] = ubuf.data() + (size_t) t * nr;}const int kb = tq_kbits(ggml_type);- rows_on(isa, kb, blob, gu_row, n, s.ptr, nt, gp, r0, r1);- rows_on(isa, kb, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);+ rows_on(isa, kb, tab, blob, gu_row, n, s.ptr, nt, gp, r0, r1);+ rows_on(isa, kb, tab, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);for (int t = 0; t < nt; ++t)for (int r = r0; r < r1; ++r) {const float g = gp[t][r], u = up[t][r];diff --git a/tools/tqenc.py b/tools/tqenc.pyindex b1b9ff598..28b3ae2fa 100644--- a/tools/tqenc.py+++ b/tools/tqenc.py@@ -14,10 +14,13 @@ import pathlibimport numpy as np-TQ2_T, TQK6, TQK7, TQK5X, TQK4X = 144, 145, 146, 224, 225-TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X"}+TQ2_T, TQK6, TQK7, TQK5X, TQK4X, TQK5T, TQK4T = 144, 145, 146, 224, 225, 226, 227+# TQK5X / TQK4X: experimental version 2, the computed (LUT-free) codebook (src/kernels/cpu/tq_computed.hpp);+# TQK5T / TQK4T: version 1, the fork's point table (what the first pilots and the Gyro-S checks used for K5 / K4)+TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X", TQK5T: "TQK5T", TQK4T: "TQK4T"}TYPE_BY_NAME = {v: k for k, v in TYPE_NAMES.items()}-KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4}+KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4, TQK5T: 5, TQK4T: 4}+COMPUTED = (TQK5X, TQK4X)BLOCK = 128@@ -149,8 +152,8 @@ def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, nraise ValueError("importance weights must have the shape of w")hp = h.ctypes.dataif gpu:- if exact or t == TQ2_T:- raise ValueError("the GPU encoder has no exact search and no TQ2_T")+ if exact or t == TQ2_T or t in COMPUTED:+ raise ValueError("the GPU encoder has no exact search, no TQ2_T and no computed-codebook types (TQK5X / TQK4X)")with _gpu_lock() as gl:if gl.ok:tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,@@ -235,7 +238,7 @@ def rel_mse(ref: np.ndarray, approx: np.ndarray) -> float:# | u32 n_sign_bytes | sign bits (1 = +1, LSB first, one per input column) | u32 hadamard block (128) | packed blocks# An unknown version or type is refused before any data is read.TQX_MAGIC = b"TQXP"-TQX_VERSION = 1+TQX_VERSION = 2 # 2: types 224 / 225 are the computed codebook; a version 1 file means the point table (226 / 227)def write_tqx(path, t: int, rows: int, cols: int, packed: np.ndarray, signs: np.ndarray | None) -> None:@@ -254,8 +257,10 @@ def read_tqx(path):if data[:4] != TQX_MAGIC:raise ValueError("not a TQXP file")ver, t, rows, cols, flags, nsb = struct.unpack_from("<IIIIII", data, 4)- if ver != TQX_VERSION:- raise ValueError("unsupported TQXP version %d (this reader handles %d)" % (ver, TQX_VERSION))+ if ver == 1 and t in (TQK5X, TQK4X):+ t += 2 # a version 1 file: the point table+ elif ver != TQX_VERSION:+ raise ValueError("unsupported TQXP version %d (this reader handles %d, and 1 for the point-table K5 / K4)" % (ver, TQX_VERSION))if t not in KBITS:raise ValueError("unknown trellis type %d" % t)at = 4 + 24diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.pyindex 9781251a8..f28b2ce72 100644--- a/tools/trellis/pilot.py+++ b/tools/trellis/pilot.py@@ -29,7 +29,7 @@ import gguf_reader # noqa: E402import tqenc # noqa: E402MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)-RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7}+RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7, "K4t": tqenc.TQK4T, "K5t": tqenc.TQK5T} # K4 / K5: computed codebook (v2); K4t / K5t: point table (v1)CFGS = {"fast": dict(iters=0, ncand=1, scale_tries=1),"default": dict(iters=1, ncand=1, scale_tries=1),diff --git a/tools/trellis/test_tqenc_py.py b/tools/trellis/test_tqenc_py.pyindex 28f8bfb43..79d55585b 100644--- a/tools/trellis/test_tqenc_py.py+++ b/tools/trellis/test_tqenc_py.py@@ -33,7 +33,7 @@ for n in (2304, 5120):check(np.abs(tqenc.unrotate(Wr, s) - W).max() < 1e-5, "unrotate(rotate(W)) == W")Ux = tqenc.rotate(x[None, :], s)[0]check(np.abs(Wr @ Ux - W @ x).max() < 1e-3, "Wrot (U x) == W x before quantization")- for t in (tqenc.TQK4X, tqenc.TQK5X, tqenc.TQK6, tqenc.TQK7, tqenc.TQ2_T):+ for t in (tqenc.TQK4X, tqenc.TQK5X, tqenc.TQK4T, tqenc.TQK5T, tqenc.TQK6, tqenc.TQK7, tqenc.TQ2_T):packed, err, tot, st = tqenc.encode(Wr, t, iters=0)check(packed.shape == (8 * n // 128, tqenc.block_bytes(t)), "packed shape")Q = tqenc.decode(packed, t).reshape(8, n)diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cppindex 67e1e9974..9aeedfa4d 100644--- a/tools/trellis/tqenc.cpp+++ b/tools/trellis/tqenc.cpp@@ -2,6 +2,7 @@#include "tqenc.hpp"#include "ggml-common-tq.h"+#include "tq_computed.hpp"#include <algorithm>#include <atomic>@@ -74,12 +75,14 @@ int type_kbits(int type) {case TQ2_T: return 8;case TQK6: return 6;case TQK7: return 7;- case TQK5X: return 5;- case TQK4X: return 4;+ case TQK5X: case TQK5T: return 5;+ case TQK4X: case TQK4T: return 4;default: return 0;}}+bool type_computed(int type) { return type == TQK5X || type == TQK4X; }+namespace {// ---------------------------------------------------------------------------------------------------------------@@ -88,7 +91,21 @@ namespace {struct Codebook {int16_t tab[kStates * 4];alignas(64) float soa[4][kStates];- Codebook() {+ double rms = 1.072; // of the weights: the first scale guess is rms(block) / this+ explicit Codebook(bool computed) {+ if (computed) { // tq_computed.hpp+ double ss = 0;+ for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {+ strata::kernels::cpu::tq_detail::comp_state4(s, tab + 4 * s);+ for (int j = 0; j < 4; ++j) {+ soa[j][s] = (float) tab[4 * s + j] * (1.0f / 8192.0f);+ ss += (double) soa[j][s] * soa[j][s];+ }+ }+ rms = std::sqrt(ss / (4.0 * kStates));+ return;+ }+ double ss = 0;for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {const uint32_t x = s * 0x9e3779b1u;const uint32_t pts[2] = {x >> 21, (x >> 10) & 2047u};@@ -98,17 +115,20 @@ struct Codebook {const int16_t q = (int16_t) (v < 0.0f ? v - 0.5f : v + 0.5f);tab[4 * s + 2 * h + j] = q;soa[2 * h + j][s] = (float) q * (1.0f / 8192.0f);+ ss += (double) soa[2 * h + j][s] * soa[2 * h + j][s];}}+ rms = std::sqrt(ss / (4.0 * kStates));}};-const Codebook& cb() {- static const Codebook c;- return c;+const Codebook& cb(bool computed) {+ static const Codebook table(false), comp(true);+ return computed ? comp : table;}struct Geo {+ const Codebook* cbk = nullptr;int K = 0;bool t2 = false; // TQ2_T (byte stream, the other shift direction)int G = 0; // predecessor groups@@ -116,6 +136,7 @@ struct Geo {Geo geo_for(int type) {Geo g;+ g.cbk = &cb(type_computed(type));g.K = type_kbits(type);g.t2 = type == TQ2_T;g.G = g.t2 ? 128 : (1 << (15 - g.K));@@ -164,8 +185,8 @@ inline float min_range(const float* p, int n) {}// out[i] = add[i] + sum_j h_j (t_j - cb_j[s0 + i])^2 for i in [0, n)-inline void step_cost(float* out, const float* add, int s0, int n, const float* t, const float* h) {- const float *c0 = cb().soa[0] + s0, *c1 = cb().soa[1] + s0, *c2 = cb().soa[2] + s0, *c3 = cb().soa[3] + s0;+inline void step_cost(const Codebook& C, float* out, const float* add, int s0, int n, const float* t, const float* h) {+ const float *c0 = C.soa[0] + s0, *c1 = C.soa[1] + s0, *c2 = C.soa[2] + s0, *c3 = C.soa[3] + s0;int i = 0;#if TQENC_AVX2const __m256 t0 = _mm256_set1_ps(t[0]), t1 = _mm256_set1_ps(t[1]), t2 = _mm256_set1_ps(t[2]), t3 = _mm256_set1_ps(t[3]);@@ -187,10 +208,10 @@ inline void step_cost(float* out, const float* add, int s0, int n, const float*}// the broadcast variant: out[i] = a + cost(s0 + i)-inline void step_cost_bc(float* out, float a, int s0, int n, const float* t, const float* h) {+inline void step_cost_bc(const Codebook& C, float* out, float a, int s0, int n, const float* t, const float* h) {alignas(32) float tmp[256];for (int i = 0; i < n; ++i) tmp[i] = a;- step_cost(out, tmp, s0, n, t, h);+ step_cost(C, out, tmp, s0, n, t, h);}// min and first index of the minimum over p[0..n)@@ -220,7 +241,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, i// q = 0{const int p = rot & 31;- step_cost(cur, sc.zero, 0, kStates, T[p], H[p]);+ step_cost(*g.cbk, cur, sc.zero, 0, kStates, T[p], H[p]);if (fixed_c >= 0) {if (g.t2) { // gsucc(s) = s >> 8 == c: the 256 states [c<<8, c<<8 + 256) allowedfor (int s = 0; s < kStates; ++s)@@ -258,7 +279,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, iif (row[a] < m[a]) { m[a] = row[a]; sc.am[a] = h; }}for (int a = 0; a < 128; ++a) bp[a] = (uint16_t) (a + 128 * sc.am[a]);- for (int b = 0; b < 128; ++b) step_cost_bc(cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);+ for (int b = 0; b < 128; ++b) step_cost_bc(*g.cbk, cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);} else {const int span = 1 << K;for (int a = 0; a < G; ++a) {@@ -266,7 +287,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, im[a] = min_arg_range(prev + (size_t) a * span, span, &arg);bp[a] = (uint16_t) ((a << K) + arg);}- for (int base = 0; base < kStates; base += G) step_cost(cur + base, m, base, G, T[p], H[p]);+ for (int base = 0; base < kStates; base += G) step_cost(*g.cbk, cur + base, m, base, G, T[p], H[p]);}}// final state@@ -362,13 +383,13 @@ int debug_groups(int type) { return geo_for(type).G; }int debug_widx(int type, int p, int j) { return widx(geo_for(type), p, j); }void debug_pack(int type, const uint16_t* st, uint8_t* qs) { pack_stream(geo_for(type), st, qs); }-const float* codebook_state(uint32_t state) {+const float* codebook_state(int type, uint32_t state) {static thread_local float v[4];- for (int j = 0; j < 4; ++j) v[j] = cb().soa[j][state & 0x7fff];+ for (int j = 0; j < 4; ++j) v[j] = cb(type_computed(type)).soa[j][state & 0x7fff];return v;}-const int16_t* codebook_table() { return cb().tab; }+const int16_t* codebook_table(int type) { return cb(type_computed(type)).tab; }const float* point_table() {static float pts[4096];@@ -403,7 +424,7 @@ void decode_blocks(int type, const uint8_t* src, int64_t nblocks, float* out) {const float d = half_to_float(dh);for (int t = 0; t < 32; ++t) {const uint32_t s = block_state(type, blk + 2, t);- for (int j = 0; j < 4; ++j) out[b * 128 + 4 * t + j] = d * cb().soa[j][s];+ for (int j = 0; j < 4; ++j) out[b * 128 + 4 * t + j] = d * cb(type_computed(type)).soa[j][s];}}}@@ -424,7 +445,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,if (amax == 0.0) { r.status = 1; return r; }const double rms = std::sqrt(sumw2 / 128.0);// initial scale: the codebook has unit-ish rms- constexpr double kCbRms = 1.072;+ const double kCbRms = g.cbk->rms;static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};Search se;@@ -445,7 +466,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,for (int p = 0; p < 32; ++p)for (int j = 0; j < 4; ++j) {const int wi = widx(g, p, j);- const float rec = d * cb().soa[j][st[p]];+ const float rec = d * g.cbk->soa[j][st[p]];const double df = (double) w[wi] - rec;e += (h ? h[wi] : 1.0) * df * df;}@@ -477,7 +498,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,for (int p = 0; p < 32; ++p)for (int j = 0; j < 4; ++j) {const int wi = widx(g, p, j);- const double hi = h ? h[wi] : 1.0, c = cb().soa[j][se.st[p]];+ const double hi = h ? h[wi] : 1.0, c = g.cbk->soa[j][se.st[p]];num += hi * w[wi] * c;den += hi * c * c;}diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hppindex 241fb6cde..20c437e7b 100644--- a/tools/trellis/tqenc.hpp+++ b/tools/trellis/tqenc.hpp@@ -1,5 +1,5 @@// tools/trellis/tqenc.hpp - tail-biting Viterbi ENCODER for the trellis expert formats (TQ2_T, TQK6, TQK7 of the-// agentionai llama.cpp fork, ggml types 144-146) and two experimental lower-rate versions (TQK5X, TQK4X).+// agentionai llama.cpp fork, ggml types 144-146) and two experimental lower-rate versions (TQK5X, TQK4X, and the table-based TQK5T, TQK4T).//// Standalone (no ggml, no Strata engine): it only needs third_party/ggml/ggml-common-tq.h (the 2048-point codebook).// The decoders it must match are src/kernels/cpu/tq_states.hpp / tq_cpu.cpp (tools/trellis/tqenc_test.cpp checks@@ -21,9 +21,11 @@// as predecessor. The search fixes that group c (exact: all G values; approximate: c taken from a free Viterbi pass// on the cyclically rotated sequence, QTIP's approach, optionally the best of several rotations).//-// New experimental types: TQK5X (id 224, 22 bytes / 128 weights, 1.375 bpw) and TQK4X (id 225, 18 bytes, 1.125 bpw):-// the TQK rule with K = 5 / 4, nothing else changed (same hash, codebook, mask, bit order, circular stream, fp16-// scale). EXPERIMENTAL_VERSION = 1: not in any ggml enum, not in any GGUF reader.+// New experimental types (EXPERIMENTAL_VERSION 2; not in any ggml enum or GGUF reader; ids away from the fork's 143-149):+// TQK5X (224, 22 bytes / 128 weights, 1.375 bpw) and TQK4X (225, 18 bytes, 1.125 bpw): the TQK rule with K = 5 / 4 and the+// COMPUTED codebook (src/kernels/cpu/tq_computed.hpp: two LCG words per state masked into four fp16, no table);+// TQK5T (226) and TQK4T (227): version 1, the same K = 5 / 4 layout with the fork's hashed point table (what the first+// pilots used). Same bit order, circular stream, 15-bit mask and fp16 scale in all of them.#pragma once#include <cstddef>@@ -33,10 +35,16 @@ namespace tqenc {inline constexpr int kBlock = 128;inline constexpr int kStates = 1 << 15;-inline constexpr int kExperimentalVersion = 1;+inline constexpr int kExperimentalVersion = 2;// ggml type ids of the fork (decoders exist) and the experimental ones (encoder + Strata CPU decoder only)-enum Type : int { TQ2_T = 144, TQK6 = 145, TQK7 = 146, TQK5X = 224, TQK4X = 225 };+enum Type : int {+ TQ2_T = 144, TQK6 = 145, TQK7 = 146, // the fork's types (hashed point table)+ TQK5X = 224, TQK4X = 225, // experimental version 2: K = 5 / 4 with the computed (LUT-free) codebook, tq_computed.hpp+ TQK5T = 226, TQK4T = 227 // experimental version 1: K = 5 / 4 with the fork's point table (the first pilots)+};+/// Whether the type's codebook is the computed one (TQK5X, TQK4X).+bool type_computed(int type);/// bits per step (8, 6, 7, 5, 4), 0 for any other type.int type_kbits(int type);@@ -58,8 +66,8 @@ struct BlockResult {};/// The codebook entry of a state as the decoder sees it: four floats (int16 table / 8192, exact).-const float* codebook_state(uint32_t state); // 4 floats-const int16_t* codebook_table(); // 32768 * 4 int16, the decoder's tq_state_table()+const float* codebook_state(int type, uint32_t state); // 4 floats+const int16_t* codebook_table(int type); // 32768 * 4 int16, the decoder's tq_state_table()/// The 2048 codebook points as the decoder sees them (x, y interleaved): round(fp16 * 8192) / 8192. The GPU encoder/// uploads this table; it must be built by the host compiler (ggml-common-tq.h makes its table __device__ under hipcc).const float* point_table(); // 4096 floatsdiff --git a/tools/trellis/tqenc_hip.cpp b/tools/trellis/tqenc_hip.cppindex a4f3d4a61..5ec20af04 100644--- a/tools/trellis/tqenc_hip.cpp+++ b/tools/trellis/tqenc_hip.cpp@@ -419,7 +419,7 @@ TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t nint scale_tries, int device, double* err_out, int64_t* stats, double* timing,int fixed_scale_bits) {const int K = tqenc::type_kbits(type);- if (K < 4 || K > 7) return -1.0;+ if (K < 4 || K > 7 || tqenc::type_computed(type)) return -1.0; // the computed codebook of TQK5X / TQK4X has no 2048-point table: CPU onlyensure_dev(device);const int bb = 2 + 4 * K, G = 1 << (15 - K);const int nc = std::max(1, std::min(ncand, 8));diff --git a/tools/trellis/tqenc_hip_test.cpp b/tools/trellis/tqenc_hip_test.cppindex c416fc8d4..b809ff85c 100644--- a/tools/trellis/tqenc_hip_test.cpp+++ b/tools/trellis/tqenc_hip_test.cpp@@ -18,13 +18,13 @@ using namespace tqenc;static int g_fail = 0;#define CHECK(c, ...) do { if (!(c)) { ++g_fail; std::printf(" FAIL line %d: %s ", __LINE__, #c); std::printf(__VA_ARGS__); std::printf("\n"); } } while (0)static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }-static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X"; }+static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5T ? "TQK5T" : "TQK4T"; }int main(int argc, char** argv) {const long big = argc > 1 ? std::atol(argv[1]) : 131072; // blocks for the speed runsstd::mt19937_64 rng(77);std::normal_distribution<float> nd(0.f, 1.f);- const int types[4] = {TQK4X, TQK5X, TQK6, TQK7};+ const int types[4] = {TQK4T, TQK5T, TQK6, TQK7};for (int type : types) {const int bb = block_bytes(type);std::printf("== %s\n", tname(type));diff --git a/tools/trellis/tqenc_test.cpp b/tools/trellis/tqenc_test.cppindex 38596d0b1..8dd61bd75 100644--- a/tools/trellis/tqenc_test.cpp+++ b/tools/trellis/tqenc_test.cpp@@ -53,9 +53,9 @@ static int g_fail = 0, g_checks = 0;} \} while (0)-static const int kTypes[5] = {TQ2_T, TQK6, TQK7, TQK5X, TQK4X};+static const int kTypes[7] = {TQ2_T, TQK6, TQK7, TQK5X, TQK4X, TQK5T, TQK4T};static const char* tname(int t) {- return t == TQ2_T ? "TQ2_T" : t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X";+ return t == TQ2_T ? "TQ2_T" : t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : t == TQK4X ? "TQK4X" : t == TQK5T ? "TQK5T" : "TQK4T";}template <int KB>@@ -111,9 +111,33 @@ static void t1() {static void t2() {std::printf("T2 codebook == decoder table\n");- const int16_t* a = codebook_table();- const int16_t* b = kc::tq_state_table();- CHECK(std::memcmp(a, b, 32768 * 4 * sizeof(int16_t)) == 0, "tables differ");+ for (int type : kTypes) {+ const int16_t* a = codebook_table(type);+ const int16_t* b = kc::tq_state_table_for(type);+ CHECK(std::memcmp(a, b, 32768 * 4 * sizeof(int16_t)) == 0, "%s: encoder and decoder tables differ", tname(type));+ }+ // the computed codebook: its own definition, evaluated bit by bit here (not through tq_computed.hpp), is what both use+ {+ const int16_t* c = codebook_table(TQK5X);+ int bad = 0, big = 0;+ for (uint32_t s = 0; s < 32768; ++s)+ for (int i = 0; i < 2; ++i) {+ const uint32_t A[2] = {89226354u, 0x9E3779B1u}, B[2] = {64248484u, 0x7F4A7C15u};+ const uint32_t r = ((s * A[i] + B[i]) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ for (int hw = 0; hw < 2; ++hw) {+ const uint16_t h = (uint16_t) (hw ? r >> 16 : r & 0xffff);+ const float v = half_to_float(h) * 8192.0f;+ bad += c[4 * s + 2 * i + hw] != (int16_t) v || v != std::floor(v);+ big += std::fabs(v) >= 16384.0f || std::fabs(v) < 1023.0f;+ }+ }+ CHECK(bad == 0, "computed codebook differs from its definition (%d)", bad);+ CHECK(big == 0, "computed codebook value out of [0.125, 2) (%d)", big);+ // the distribution: rms and the share of the weights below / above one+ double ss = 0;+ for (int i = 0; i < 32768 * 4; ++i) ss += (double) c[i] * c[i];+ std::printf(" computed codebook: rms %.4f (table: %.4f), all values exact integers at 2^13\n", std::sqrt(ss / (32768.0 * 4)) / 8192.0, 1.072);+ }}static void t3() {@@ -153,7 +177,7 @@ static float naive(int type, const float (*T)[4], const float (*H)[4], int rot,std::vector<std::vector<float>> D(32, std::vector<float>(32768, 1e30f));std::vector<std::vector<int>> B(32, std::vector<int>(32768, -1));auto cost = [&](int p, uint32_t s) {- const float* c = codebook_state(s);+ const float* c = codebook_state(type, s);float e = 0;for (int j = 0; j < 4; ++j) { const float d = T[p][j] - c[j]; e += H[p][j] * d * d; }return e;@@ -232,7 +256,7 @@ static void t4() {double sum = 0;for (int q = 0; q < 32; ++q) {const int p = (cc.rot + q) & 31;- const float* c = codebook_state(sa[p]);+ const float* c = codebook_state(type, sa[p]);for (int j = 0; j < 4; ++j) { const double d = T[p][j] - c[j]; sum += H[p][j] * d * d; }}CHECK(std::fabs(sum - a) < 1e-3 * std::max(1.0, (double) a), "path cost %.6f vs reported %.6f", sum, a);@@ -359,7 +383,7 @@ static void t7() {const double tot = encode_blocks(type, w.data(), nullptr, (int64_t) rows * nbr, enc.data(), p, 8, errs.data(), &nf, &zb);(void) tot;// decoder reference: tq_states.hpp + the state table- const int16_t* tab = kc::tq_state_table();+ const int16_t* tab = kc::tq_state_table_for(type);std::vector<float> ref((size_t) rows * n), ours((size_t) rows * n);for (int64_t b = 0; b < (int64_t) rows * nbr; ++b) {const uint8_t* blk = enc.data() + b * bb;