Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

K5/K4 version 2: computed (LUT-free) codebook for TQK5X/TQK4X, point-table versions kept as TQK5T/TQK4T; CPU decoder tables per type; tests

Niko1221a7c50df0029e
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hpp
index a73e300e8..ac6d6f176 100644
--- a/include/strata/kernels/cpu/tq_cpu.hpp
+++ b/include/strata/kernels/cpu/tq_cpu.hpp
@@ -34,9 +34,11 @@ enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };
/// EXPERIMENTAL lower-rate versions of the TQK rule (tools/trellis/tqenc.hpp, version 1; Strata-internal ids, not ggml
/// types, not in any GGUF): TQK5X = 22 bytes / 128 weights (1.375 bpw), TQK4X = 18 bytes (1.125 bpw). Same hash,
/// codebook, 15-bit mask, LSB-first circular stream and fp16 scale as TQK6/7 with K = 5 / 4.
-inline constexpr int kTqTypeK5X = 224;
-inline constexpr int kTqTypeK4X = 225;
-/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X.
+inline constexpr int kTqTypeK5X = 224; ///< version 2: K = 5 with the computed codebook (tq_computed.hpp)
+inline constexpr int kTqTypeK4X = 225; ///< version 2: K = 4, computed codebook
+inline constexpr int kTqTypeK5T = 226; ///< version 1: K = 5 with the fork's point table (the first pilots)
+inline constexpr int kTqTypeK4T = 227; ///< version 1: K = 4, point table
+/// TQ2_T (144), TQK6 (145), TQK7 (146) and the experimental TQK5X / TQK4X / TQK5T / TQK4T.
bool tq_supported(int ggml_type) noexcept;
/// Bits per step: 8, 6, 7, 5, 4 (0 for any other type).
int tq_kbits(int ggml_type) noexcept;
@@ -54,8 +56,12 @@ bool tq_isa_ok(TqIsa isa);
bool tq_kernels_enabled();
const char* tq_isa_name(TqIsa isa);
-/// The 32768 x 4 int16 state table (filled on first use; thread-safe).
+/// The 32768 x 4 int16 state table of the fork's types (filled on first use; thread-safe).
const int16_t* tq_state_table();
+/// The same for the computed codebook of TQK5X / TQK4X (tq_computed.hpp).
+const int16_t* tq_state_table_computed();
+/// The table a type decodes with.
+const int16_t* tq_state_table_for(int ggml_type);
/// Rows [r0, r1) of a trellis matrix (`row_bytes` per row, `n` weights per row) against `nt` (<= 8) Q8_0
/// activations of n values: out[t][r] = w_r . a[t].
@@ -78,11 +84,11 @@ struct TqAct {
};
void unpack_act(const void* q8_0, int n, TqAct& a);
// the per-ISA row kernels (dot products only; `out[t][r]` for rows [r0, r1))
-void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
float* const* out, int r0, int r1);
-void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx2(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
float* const* out, int r0, int r1);
-void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx512(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
float* const* out, int r0, int r1);
} // namespace tq_detail
diff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cpp
index f03892510..9a43393fe 100644
--- a/src/kernels/cpu/tq_avx2.cpp
+++ b/src/kernels/cpu/tq_avx2.cpp
@@ -29,11 +29,11 @@ inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2
}
template <int KB>
-void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
+void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
int r1) {
const int nb = n / 128;
constexpr size_t bsz = 2 + 4 * KB;
- const long long* tab = (const long long*) tq_state_table();
+ const long long* tab = (const long long*) table;
for (int r = r0; r < r1; ++r) {
const uint8_t* x = w + (size_t) r * row_bytes;
__m128 acc[kTqMaxTokens];
@@ -77,13 +77,13 @@ void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, in
} // namespace
-void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx2(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
float* const* out, int r0, int r1) {
- if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
- else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
- else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
- else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
- else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+ if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 6) rows_t<6>(table, w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 5) rows_t<5>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
+ else if (kb == 4) rows_t<4>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
+ else rows_t<7>(table, w, row_bytes, n, a, nt, out, r0, r1);
}
} // namespace strata::kernels::cpu::tq_detail
diff --git a/src/kernels/cpu/tq_avx512.cpp b/src/kernels/cpu/tq_avx512.cpp
index 9031b8cf5..49ec0726f 100644
--- a/src/kernels/cpu/tq_avx512.cpp
+++ b/src/kernels/cpu/tq_avx512.cpp
@@ -47,11 +47,11 @@ inline __m512i rows8(const long long* tab, const uint8_t* qs, uint64_t wr, int t
}
template <int KB>
-void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
+void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
int r1) {
const int nb = n / 128;
constexpr size_t bsz = 2 + 4 * KB;
- const long long* tab = (const long long*) tq_state_table();
+ const long long* tab = (const long long*) table;
for (int r = r0; r < r1; ++r) {
const uint8_t* x = w + (size_t) r * row_bytes;
__m128 acc[kTqMaxTokens];
@@ -85,13 +85,13 @@ void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, in
} // namespace
-void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_avx512(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
float* const* out, int r0, int r1) {
- if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
- else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
- else if (kb == 5) rows_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
- else if (kb == 4) rows_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
- else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+ if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 6) rows_t<6>(table, w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 5) rows_t<5>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
+ else if (kb == 4) rows_t<4>(table, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
+ else rows_t<7>(table, w, row_bytes, n, a, nt, out, r0, r1);
}
} // namespace strata::kernels::cpu::tq_detail
diff --git a/src/kernels/cpu/tq_computed.hpp b/src/kernels/cpu/tq_computed.hpp
new file mode 100644
index 000000000..3222239fa
--- /dev/null
+++ b/src/kernels/cpu/tq_computed.hpp
@@ -0,0 +1,33 @@
+// src/kernels/cpu/tq_computed.hpp - the LUT-free ("computed") codebook of the experimental trellis types TQK5X / TQK4X
+// (version 2): a state's four weights come from two 32-bit LCG words instead of the hashed 2048-point table.
+//
+// r_i = ((s * A_i + B_i) & 0x8FFF8FFF) ^ 0x3B603B60 i = 0, 1 (s = the 15-bit state)
+// weights 0, 1 = the low and high fp16 of r_0, weights 2, 3 = those of r_1
+//
+// (QTIP's 3INST word, Tseng et al., Cornell-RelaxML/qtip lib/codebook/bitshift.py; the variant "3single" of the
+// f-dsv41-cudalab study, which measured it at the SQNR of the table on real DeepSeek experts and a GEMV without
+// shared-memory lookups.) The mask fixes the top exponent bits, so |w| lies in [0.125, 2) and the int16 table value
+// round(w * 2^13) used by the CPU kernels is an exact integer (1024..16383): decoder and encoder agree to the bit.
+// The existing types (TQ2_T, TQK6, TQK7) and the table-based K5 / K4 (ids 226 / 227, "version 1") do not use it.
+#pragma once
+
+#include <cstdint>
+
+#include "tq_states.hpp"
+
+namespace strata::kernels::cpu::tq_detail {
+
+inline constexpr uint32_t kCompA[2] = {89226354u, 0x9E3779B1u};
+inline constexpr uint32_t kCompB[2] = {64248484u, 0x7F4A7C15u};
+inline constexpr uint32_t kCompMask = 0x8FFF8FFFu, kCompXor = 0x3B603B60u;
+
+inline uint32_t comp_word(uint32_t s, int i) { return ((s * kCompA[i] + kCompB[i]) & kCompMask) ^ kCompXor; }
+
+/// The four weights of state s at scale 2^13 (int16, exact).
+inline void comp_state4(uint32_t s, int16_t out[4]) {
+ const uint32_t r0 = comp_word(s & 0x7fffu, 0), r1 = comp_word(s & 0x7fffu, 1);
+ const uint16_t h[4] = {(uint16_t) (r0 & 0xffffu), (uint16_t) (r0 >> 16), (uint16_t) (r1 & 0xffffu), (uint16_t) (r1 >> 16)};
+ for (int j = 0; j < 4; ++j) out[j] = (int16_t) (tq_half_to_float(h[j]) * 8192.0f);
+}
+
+} // namespace strata::kernels::cpu::tq_detail
diff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cpp
index ad7ea297d..22a874bcb 100644
--- a/src/kernels/cpu/tq_cpu.cpp
+++ b/src/kernels/cpu/tq_cpu.cpp
@@ -5,6 +5,7 @@
#include "strata/kernels/cpu/expert_layout.hpp"
#include "tq_states.hpp"
+#include "tq_computed.hpp"
#include "ggml-common-tq.h"
#include <cmath>
@@ -25,6 +26,7 @@ namespace strata::kernels::cpu {
namespace {
alignas(64) int16_t g_table[(tq_detail::kTqStateMask + 1) * 4];
+alignas(64) int16_t g_table_c[(tq_detail::kTqStateMask + 1) * 4];
void fill_table() {
for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) {
@@ -39,6 +41,10 @@ void fill_table() {
}
}
+void fill_table_computed() {
+ for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) tq_detail::comp_state4(s, g_table_c + 4 * s);
+}
+
int env_isa_cap() {
const char* e = std::getenv("STRATA_TQ_ISA");
if (e == nullptr || *e == 0) return 2;
@@ -49,11 +55,10 @@ int env_isa_cap() {
}
template <int KB>
-void rows_scalar_t(const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
+void rows_scalar_t(const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
float* const* out, int r0, int r1) {
const int nb = n / 128;
constexpr size_t bsz = 2 + 4 * KB;
- const int16_t* tab = tq_state_table();
for (int r = r0; r < r1; ++r) {
const uint8_t* x = w + (size_t) r * row_bytes;
float acc[kTqMaxTokens][4] = {};
@@ -89,10 +94,20 @@ const int16_t* tq_state_table() {
return g_table;
}
-bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X; }
+const int16_t* tq_state_table_computed() {
+ static std::once_flag once;
+ std::call_once(once, fill_table_computed);
+ return g_table_c;
+}
+
+const int16_t* tq_state_table_for(int t) { return (t == kTqTypeK5X || t == kTqTypeK4X) ? tq_state_table_computed() : tq_state_table(); }
+
+bool tq_supported(int t) noexcept {
+ return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X || t == kTqTypeK5T || t == kTqTypeK4T;
+}
int tq_kbits(int t) noexcept {
- return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : t == kTqTypeK5X ? 5 : t == kTqTypeK4X ? 4 : 0;
+ return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : (t == kTqTypeK5X || t == kTqTypeK5T) ? 5 : (t == kTqTypeK4X || t == kTqTypeK4T) ? 4 : 0;
}
bool tq_isa_ok(TqIsa isa) {
@@ -142,13 +157,13 @@ void unpack_act(const void* q8_0, int n, TqAct& a) {
}
}
-void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
float* const* out, int r0, int r1) {
- if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
- else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
- else if (kb == 5) rows_scalar_t<5>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X
- else if (kb == 4) rows_scalar_t<4>(w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X
- else rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+ if (kb == 8) rows_scalar_t<8>(tab, w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 6) rows_scalar_t<6>(tab, w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 5) rows_scalar_t<5>(tab, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK5X / TQK5T
+ else if (kb == 4) rows_scalar_t<4>(tab, w, row_bytes, n, a, nt, out, r0, r1); // experimental TQK4X / TQK4T
+ else rows_scalar_t<7>(tab, w, row_bytes, n, a, nt, out, r0, r1);
}
} // namespace tq_detail
@@ -171,14 +186,14 @@ ActSet& unpack_all(const void* const* act, int n, int nt) {
return *set;
}
-void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
- float* const* out, int r0, int r1) {
+void rows_on(TqIsa isa, int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n,
+ const tq_detail::TqAct* const* a, int nt, float* const* out, int r0, int r1) {
switch (isa) {
#if defined(STRATA_TQ_X86)
- case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
- case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+ case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;
+ case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;
#endif
- default: tq_detail::tq_rows_scalar(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+ default: tq_detail::tq_rows_scalar(kb, tab, w, row_bytes, n, a, nt, out, r0, r1); break;
}
}
@@ -187,15 +202,15 @@ void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const
void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
float* const* out, int r0, int r1) {
if (!tq_isa_ok(isa)) std::abort(); // a caller bug: asked for an ISA this CPU does not have
- tq_state_table();
+ const int16_t* tab = tq_state_table_for(ggml_type);
const ActSet& s = unpack_all(act, n, nt);
- rows_on(isa, tq_kbits(ggml_type), w, row_bytes, n, s.ptr, nt, out, r0, r1);
+ rows_on(isa, tq_kbits(ggml_type), tab, w, row_bytes, n, s.ptr, nt, out, r0, r1);
}
void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,
const void* const* act, int nt, float* const* ff, int r0, int r1) {
if (!tq_isa_ok(isa)) std::abort();
- tq_state_table();
+ const int16_t* tab = tq_state_table_for(ggml_type);
const ActSet& s = unpack_all(act, n, nt);
thread_local std::vector<float> gbuf, ubuf;
const size_t nr = (size_t) r1; // row r of token t at [t * r1 + r]
@@ -207,8 +222,8 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, si
up[t] = ubuf.data() + (size_t) t * nr;
}
const int kb = tq_kbits(ggml_type);
- rows_on(isa, kb, blob, gu_row, n, s.ptr, nt, gp, r0, r1);
- rows_on(isa, kb, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);
+ rows_on(isa, kb, tab, blob, gu_row, n, s.ptr, nt, gp, r0, r1);
+ rows_on(isa, kb, tab, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);
for (int t = 0; t < nt; ++t)
for (int r = r0; r < r1; ++r) {
const float g = gp[t][r], u = up[t][r];
diff --git a/tools/tqenc.py b/tools/tqenc.py
index b1b9ff598..28b3ae2fa 100644
--- a/tools/tqenc.py
+++ b/tools/tqenc.py
@@ -14,10 +14,13 @@ import pathlib
import numpy as np
-TQ2_T, TQK6, TQK7, TQK5X, TQK4X = 144, 145, 146, 224, 225
-TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X"}
+TQ2_T, TQK6, TQK7, TQK5X, TQK4X, TQK5T, TQK4T = 144, 145, 146, 224, 225, 226, 227
+# TQK5X / TQK4X: experimental version 2, the computed (LUT-free) codebook (src/kernels/cpu/tq_computed.hpp);
+# TQK5T / TQK4T: version 1, the fork's point table (what the first pilots and the Gyro-S checks used for K5 / K4)
+TYPE_NAMES = {TQ2_T: "TQ2_T", TQK6: "TQK6", TQK7: "TQK7", TQK5X: "TQK5X", TQK4X: "TQK4X", TQK5T: "TQK5T", TQK4T: "TQK4T"}
TYPE_BY_NAME = {v: k for k, v in TYPE_NAMES.items()}
-KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4}
+KBITS = {TQ2_T: 8, TQK6: 6, TQK7: 7, TQK5X: 5, TQK4X: 4, TQK5T: 5, TQK4T: 4}
+COMPUTED = (TQK5X, TQK4X)
BLOCK = 128
@@ -149,8 +152,8 @@ def encode(w: np.ndarray, t: int, h: np.ndarray | None = None, iters: int = 1, n
raise ValueError("importance weights must have the shape of w")
hp = h.ctypes.data
if gpu:
- if exact or t == TQ2_T:
- raise ValueError("the GPU encoder has no exact search and no TQ2_T")
+ if exact or t == TQ2_T or t in COMPUTED:
+ raise ValueError("the GPU encoder has no exact search, no TQ2_T and no computed-codebook types (TQK5X / TQK4X)")
with _gpu_lock() as gl:
if gl.ok:
tot = hip_lib().tqh_encode(t, w.ctypes.data, hp, nb, out.ctypes.data, iters, ncand, scale_tries, 0,
@@ -235,7 +238,7 @@ def rel_mse(ref: np.ndarray, approx: np.ndarray) -> float:
# | u32 n_sign_bytes | sign bits (1 = +1, LSB first, one per input column) | u32 hadamard block (128) | packed blocks
# An unknown version or type is refused before any data is read.
TQX_MAGIC = b"TQXP"
-TQX_VERSION = 1
+TQX_VERSION = 2 # 2: types 224 / 225 are the computed codebook; a version 1 file means the point table (226 / 227)
def write_tqx(path, t: int, rows: int, cols: int, packed: np.ndarray, signs: np.ndarray | None) -> None:
@@ -254,8 +257,10 @@ def read_tqx(path):
if data[:4] != TQX_MAGIC:
raise ValueError("not a TQXP file")
ver, t, rows, cols, flags, nsb = struct.unpack_from("<IIIIII", data, 4)
- if ver != TQX_VERSION:
- raise ValueError("unsupported TQXP version %d (this reader handles %d)" % (ver, TQX_VERSION))
+ if ver == 1 and t in (TQK5X, TQK4X):
+ t += 2 # a version 1 file: the point table
+ elif ver != TQX_VERSION:
+ raise ValueError("unsupported TQXP version %d (this reader handles %d, and 1 for the point-table K5 / K4)" % (ver, TQX_VERSION))
if t not in KBITS:
raise ValueError("unknown trellis type %d" % t)
at = 4 + 24
diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.py
index 9781251a8..f28b2ce72 100644
--- a/tools/trellis/pilot.py
+++ b/tools/trellis/pilot.py
@@ -29,7 +29,7 @@ import gguf_reader # noqa: E402
import tqenc # noqa: E402
MXFP4_LUT = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)
-RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7}
+RATES = {"K4": tqenc.TQK4X, "K5": tqenc.TQK5X, "K6": tqenc.TQK6, "K7": tqenc.TQK7, "K4t": tqenc.TQK4T, "K5t": tqenc.TQK5T} # K4 / K5: computed codebook (v2); K4t / K5t: point table (v1)
CFGS = {
"fast": dict(iters=0, ncand=1, scale_tries=1),
"default": dict(iters=1, ncand=1, scale_tries=1),
diff --git a/tools/trellis/test_tqenc_py.py b/tools/trellis/test_tqenc_py.py
index 28f8bfb43..79d55585b 100644
--- a/tools/trellis/test_tqenc_py.py
+++ b/tools/trellis/test_tqenc_py.py
@@ -33,7 +33,7 @@ for n in (2304, 5120):
check(np.abs(tqenc.unrotate(Wr, s) - W).max() < 1e-5, "unrotate(rotate(W)) == W")
Ux = tqenc.rotate(x[None, :], s)[0]
check(np.abs(Wr @ Ux - W @ x).max() < 1e-3, "Wrot (U x) == W x before quantization")
- for t in (tqenc.TQK4X, tqenc.TQK5X, tqenc.TQK6, tqenc.TQK7, tqenc.TQ2_T):
+ for t in (tqenc.TQK4X, tqenc.TQK5X, tqenc.TQK4T, tqenc.TQK5T, tqenc.TQK6, tqenc.TQK7, tqenc.TQ2_T):
packed, err, tot, st = tqenc.encode(Wr, t, iters=0)
check(packed.shape == (8 * n // 128, tqenc.block_bytes(t)), "packed shape")
Q = tqenc.decode(packed, t).reshape(8, n)
diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cpp
index 67e1e9974..9aeedfa4d 100644
--- a/tools/trellis/tqenc.cpp
+++ b/tools/trellis/tqenc.cpp
@@ -2,6 +2,7 @@
#include "tqenc.hpp"
#include "ggml-common-tq.h"
+#include "tq_computed.hpp"
#include <algorithm>
#include <atomic>
@@ -74,12 +75,14 @@ int type_kbits(int type) {
case TQ2_T: return 8;
case TQK6: return 6;
case TQK7: return 7;
- case TQK5X: return 5;
- case TQK4X: return 4;
+ case TQK5X: case TQK5T: return 5;
+ case TQK4X: case TQK4T: return 4;
default: return 0;
}
}
+bool type_computed(int type) { return type == TQK5X || type == TQK4X; }
+
namespace {
// ---------------------------------------------------------------------------------------------------------------
@@ -88,7 +91,21 @@ namespace {
struct Codebook {
int16_t tab[kStates * 4];
alignas(64) float soa[4][kStates];
- Codebook() {
+ double rms = 1.072; // of the weights: the first scale guess is rms(block) / this
+ explicit Codebook(bool computed) {
+ if (computed) { // tq_computed.hpp
+ double ss = 0;
+ for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {
+ strata::kernels::cpu::tq_detail::comp_state4(s, tab + 4 * s);
+ for (int j = 0; j < 4; ++j) {
+ soa[j][s] = (float) tab[4 * s + j] * (1.0f / 8192.0f);
+ ss += (double) soa[j][s] * soa[j][s];
+ }
+ }
+ rms = std::sqrt(ss / (4.0 * kStates));
+ return;
+ }
+ double ss = 0;
for (uint32_t s = 0; s < (uint32_t) kStates; ++s) {
const uint32_t x = s * 0x9e3779b1u;
const uint32_t pts[2] = {x >> 21, (x >> 10) & 2047u};
@@ -98,17 +115,20 @@ struct Codebook {
const int16_t q = (int16_t) (v < 0.0f ? v - 0.5f : v + 0.5f);
tab[4 * s + 2 * h + j] = q;
soa[2 * h + j][s] = (float) q * (1.0f / 8192.0f);
+ ss += (double) soa[2 * h + j][s] * soa[2 * h + j][s];
}
}
+ rms = std::sqrt(ss / (4.0 * kStates));
}
};
-const Codebook& cb() {
- static const Codebook c;
- return c;
+const Codebook& cb(bool computed) {
+ static const Codebook table(false), comp(true);
+ return computed ? comp : table;
}
struct Geo {
+ const Codebook* cbk = nullptr;
int K = 0;
bool t2 = false; // TQ2_T (byte stream, the other shift direction)
int G = 0; // predecessor groups
@@ -116,6 +136,7 @@ struct Geo {
Geo geo_for(int type) {
Geo g;
+ g.cbk = &cb(type_computed(type));
g.K = type_kbits(type);
g.t2 = type == TQ2_T;
g.G = g.t2 ? 128 : (1 << (15 - g.K));
@@ -164,8 +185,8 @@ inline float min_range(const float* p, int n) {
}
// out[i] = add[i] + sum_j h_j (t_j - cb_j[s0 + i])^2 for i in [0, n)
-inline void step_cost(float* out, const float* add, int s0, int n, const float* t, const float* h) {
- const float *c0 = cb().soa[0] + s0, *c1 = cb().soa[1] + s0, *c2 = cb().soa[2] + s0, *c3 = cb().soa[3] + s0;
+inline void step_cost(const Codebook& C, float* out, const float* add, int s0, int n, const float* t, const float* h) {
+ const float *c0 = C.soa[0] + s0, *c1 = C.soa[1] + s0, *c2 = C.soa[2] + s0, *c3 = C.soa[3] + s0;
int i = 0;
#if TQENC_AVX2
const __m256 t0 = _mm256_set1_ps(t[0]), t1 = _mm256_set1_ps(t[1]), t2 = _mm256_set1_ps(t[2]), t3 = _mm256_set1_ps(t[3]);
@@ -187,10 +208,10 @@ inline void step_cost(float* out, const float* add, int s0, int n, const float*
}
// the broadcast variant: out[i] = a + cost(s0 + i)
-inline void step_cost_bc(float* out, float a, int s0, int n, const float* t, const float* h) {
+inline void step_cost_bc(const Codebook& C, float* out, float a, int s0, int n, const float* t, const float* h) {
alignas(32) float tmp[256];
for (int i = 0; i < n; ++i) tmp[i] = a;
- step_cost(out, tmp, s0, n, t, h);
+ step_cost(C, out, tmp, s0, n, t, h);
}
// min and first index of the minimum over p[0..n)
@@ -220,7 +241,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, i
// q = 0
{
const int p = rot & 31;
- step_cost(cur, sc.zero, 0, kStates, T[p], H[p]);
+ step_cost(*g.cbk, cur, sc.zero, 0, kStates, T[p], H[p]);
if (fixed_c >= 0) {
if (g.t2) { // gsucc(s) = s >> 8 == c: the 256 states [c<<8, c<<8 + 256) allowed
for (int s = 0; s < kStates; ++s)
@@ -258,7 +279,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, i
if (row[a] < m[a]) { m[a] = row[a]; sc.am[a] = h; }
}
for (int a = 0; a < 128; ++a) bp[a] = (uint16_t) (a + 128 * sc.am[a]);
- for (int b = 0; b < 128; ++b) step_cost_bc(cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);
+ for (int b = 0; b < 128; ++b) step_cost_bc(*g.cbk, cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);
} else {
const int span = 1 << K;
for (int a = 0; a < G; ++a) {
@@ -266,7 +287,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, i
m[a] = min_arg_range(prev + (size_t) a * span, span, &arg);
bp[a] = (uint16_t) ((a << K) + arg);
}
- for (int base = 0; base < kStates; base += G) step_cost(cur + base, m, base, G, T[p], H[p]);
+ for (int base = 0; base < kStates; base += G) step_cost(*g.cbk, cur + base, m, base, G, T[p], H[p]);
}
}
// final state
@@ -362,13 +383,13 @@ int debug_groups(int type) { return geo_for(type).G; }
int debug_widx(int type, int p, int j) { return widx(geo_for(type), p, j); }
void debug_pack(int type, const uint16_t* st, uint8_t* qs) { pack_stream(geo_for(type), st, qs); }
-const float* codebook_state(uint32_t state) {
+const float* codebook_state(int type, uint32_t state) {
static thread_local float v[4];
- for (int j = 0; j < 4; ++j) v[j] = cb().soa[j][state & 0x7fff];
+ for (int j = 0; j < 4; ++j) v[j] = cb(type_computed(type)).soa[j][state & 0x7fff];
return v;
}
-const int16_t* codebook_table() { return cb().tab; }
+const int16_t* codebook_table(int type) { return cb(type_computed(type)).tab; }
const float* point_table() {
static float pts[4096];
@@ -403,7 +424,7 @@ void decode_blocks(int type, const uint8_t* src, int64_t nblocks, float* out) {
const float d = half_to_float(dh);
for (int t = 0; t < 32; ++t) {
const uint32_t s = block_state(type, blk + 2, t);
- for (int j = 0; j < 4; ++j) out[b * 128 + 4 * t + j] = d * cb().soa[j][s];
+ for (int j = 0; j < 4; ++j) out[b * 128 + 4 * t + j] = d * cb(type_computed(type)).soa[j][s];
}
}
}
@@ -424,7 +445,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,
if (amax == 0.0) { r.status = 1; return r; }
const double rms = std::sqrt(sumw2 / 128.0);
// initial scale: the codebook has unit-ish rms
- constexpr double kCbRms = 1.072;
+ const double kCbRms = g.cbk->rms;
static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};
Search se;
@@ -445,7 +466,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,
for (int p = 0; p < 32; ++p)
for (int j = 0; j < 4; ++j) {
const int wi = widx(g, p, j);
- const float rec = d * cb().soa[j][st[p]];
+ const float rec = d * g.cbk->soa[j][st[p]];
const double df = (double) w[wi] - rec;
e += (h ? h[wi] : 1.0) * df * df;
}
@@ -477,7 +498,7 @@ BlockResult encode_block(int type, const float* w, const float* h, uint8_t* dst,
for (int p = 0; p < 32; ++p)
for (int j = 0; j < 4; ++j) {
const int wi = widx(g, p, j);
- const double hi = h ? h[wi] : 1.0, c = cb().soa[j][se.st[p]];
+ const double hi = h ? h[wi] : 1.0, c = g.cbk->soa[j][se.st[p]];
num += hi * w[wi] * c;
den += hi * c * c;
}
diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hpp
index 241fb6cde..20c437e7b 100644
--- a/tools/trellis/tqenc.hpp
+++ b/tools/trellis/tqenc.hpp
@@ -1,5 +1,5 @@
// tools/trellis/tqenc.hpp - tail-biting Viterbi ENCODER for the trellis expert formats (TQ2_T, TQK6, TQK7 of the
-// agentionai llama.cpp fork, ggml types 144-146) and two experimental lower-rate versions (TQK5X, TQK4X).
+// agentionai llama.cpp fork, ggml types 144-146) and two experimental lower-rate versions (TQK5X, TQK4X, and the table-based TQK5T, TQK4T).
//
// Standalone (no ggml, no Strata engine): it only needs third_party/ggml/ggml-common-tq.h (the 2048-point codebook).
// The decoders it must match are src/kernels/cpu/tq_states.hpp / tq_cpu.cpp (tools/trellis/tqenc_test.cpp checks
@@ -21,9 +21,11 @@
// as predecessor. The search fixes that group c (exact: all G values; approximate: c taken from a free Viterbi pass
// on the cyclically rotated sequence, QTIP's approach, optionally the best of several rotations).
//
-// New experimental types: TQK5X (id 224, 22 bytes / 128 weights, 1.375 bpw) and TQK4X (id 225, 18 bytes, 1.125 bpw):
-// the TQK rule with K = 5 / 4, nothing else changed (same hash, codebook, mask, bit order, circular stream, fp16
-// scale). EXPERIMENTAL_VERSION = 1: not in any ggml enum, not in any GGUF reader.
+// New experimental types (EXPERIMENTAL_VERSION 2; not in any ggml enum or GGUF reader; ids away from the fork's 143-149):
+// TQK5X (224, 22 bytes / 128 weights, 1.375 bpw) and TQK4X (225, 18 bytes, 1.125 bpw): the TQK rule with K = 5 / 4 and the
+// COMPUTED codebook (src/kernels/cpu/tq_computed.hpp: two LCG words per state masked into four fp16, no table);
+// TQK5T (226) and TQK4T (227): version 1, the same K = 5 / 4 layout with the fork's hashed point table (what the first
+// pilots used). Same bit order, circular stream, 15-bit mask and fp16 scale in all of them.
#pragma once
#include <cstddef>
@@ -33,10 +35,16 @@ namespace tqenc {
inline constexpr int kBlock = 128;
inline constexpr int kStates = 1 << 15;
-inline constexpr int kExperimentalVersion = 1;
+inline constexpr int kExperimentalVersion = 2;
// ggml type ids of the fork (decoders exist) and the experimental ones (encoder + Strata CPU decoder only)
-enum Type : int { TQ2_T = 144, TQK6 = 145, TQK7 = 146, TQK5X = 224, TQK4X = 225 };
+enum Type : int {
+ TQ2_T = 144, TQK6 = 145, TQK7 = 146, // the fork's types (hashed point table)
+ TQK5X = 224, TQK4X = 225, // experimental version 2: K = 5 / 4 with the computed (LUT-free) codebook, tq_computed.hpp
+ TQK5T = 226, TQK4T = 227 // experimental version 1: K = 5 / 4 with the fork's point table (the first pilots)
+};
+/// Whether the type's codebook is the computed one (TQK5X, TQK4X).
+bool type_computed(int type);
/// bits per step (8, 6, 7, 5, 4), 0 for any other type.
int type_kbits(int type);
@@ -58,8 +66,8 @@ struct BlockResult {
};
/// The codebook entry of a state as the decoder sees it: four floats (int16 table / 8192, exact).
-const float* codebook_state(uint32_t state); // 4 floats
-const int16_t* codebook_table(); // 32768 * 4 int16, the decoder's tq_state_table()
+const float* codebook_state(int type, uint32_t state); // 4 floats
+const int16_t* codebook_table(int type); // 32768 * 4 int16, the decoder's tq_state_table()
/// The 2048 codebook points as the decoder sees them (x, y interleaved): round(fp16 * 8192) / 8192. The GPU encoder
/// uploads this table; it must be built by the host compiler (ggml-common-tq.h makes its table __device__ under hipcc).
const float* point_table(); // 4096 floats
diff --git a/tools/trellis/tqenc_hip.cpp b/tools/trellis/tqenc_hip.cpp
index a4f3d4a61..5ec20af04 100644
--- a/tools/trellis/tqenc_hip.cpp
+++ b/tools/trellis/tqenc_hip.cpp
@@ -419,7 +419,7 @@ TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t n
int scale_tries, int device, double* err_out, int64_t* stats, double* timing,
int fixed_scale_bits) {
const int K = tqenc::type_kbits(type);
- if (K < 4 || K > 7) return -1.0;
+ if (K < 4 || K > 7 || tqenc::type_computed(type)) return -1.0; // the computed codebook of TQK5X / TQK4X has no 2048-point table: CPU only
ensure_dev(device);
const int bb = 2 + 4 * K, G = 1 << (15 - K);
const int nc = std::max(1, std::min(ncand, 8));
diff --git a/tools/trellis/tqenc_hip_test.cpp b/tools/trellis/tqenc_hip_test.cpp
index c416fc8d4..b809ff85c 100644
--- a/tools/trellis/tqenc_hip_test.cpp
+++ b/tools/trellis/tqenc_hip_test.cpp
@@ -18,13 +18,13 @@ using namespace tqenc;
static int g_fail = 0;
#define CHECK(c, ...) do { if (!(c)) { ++g_fail; std::printf(" FAIL line %d: %s ", __LINE__, #c); std::printf(__VA_ARGS__); std::printf("\n"); } } while (0)
static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
-static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X"; }
+static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5T ? "TQK5T" : "TQK4T"; }
int main(int argc, char** argv) {
const long big = argc > 1 ? std::atol(argv[1]) : 131072; // blocks for the speed runs
std::mt19937_64 rng(77);
std::normal_distribution<float> nd(0.f, 1.f);
- const int types[4] = {TQK4X, TQK5X, TQK6, TQK7};
+ const int types[4] = {TQK4T, TQK5T, TQK6, TQK7};
for (int type : types) {
const int bb = block_bytes(type);
std::printf("== %s\n", tname(type));
diff --git a/tools/trellis/tqenc_test.cpp b/tools/trellis/tqenc_test.cpp
index 38596d0b1..8dd61bd75 100644
--- a/tools/trellis/tqenc_test.cpp
+++ b/tools/trellis/tqenc_test.cpp
@@ -53,9 +53,9 @@ static int g_fail = 0, g_checks = 0;
} \
} while (0)
-static const int kTypes[5] = {TQ2_T, TQK6, TQK7, TQK5X, TQK4X};
+static const int kTypes[7] = {TQ2_T, TQK6, TQK7, TQK5X, TQK4X, TQK5T, TQK4T};
static const char* tname(int t) {
- return t == TQ2_T ? "TQ2_T" : t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X";
+ return t == TQ2_T ? "TQ2_T" : t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : t == TQK4X ? "TQK4X" : t == TQK5T ? "TQK5T" : "TQK4T";
}
template <int KB>
@@ -111,9 +111,33 @@ static void t1() {
static void t2() {
std::printf("T2 codebook == decoder table\n");
- const int16_t* a = codebook_table();
- const int16_t* b = kc::tq_state_table();
- CHECK(std::memcmp(a, b, 32768 * 4 * sizeof(int16_t)) == 0, "tables differ");
+ for (int type : kTypes) {
+ const int16_t* a = codebook_table(type);
+ const int16_t* b = kc::tq_state_table_for(type);
+ CHECK(std::memcmp(a, b, 32768 * 4 * sizeof(int16_t)) == 0, "%s: encoder and decoder tables differ", tname(type));
+ }
+ // the computed codebook: its own definition, evaluated bit by bit here (not through tq_computed.hpp), is what both use
+ {
+ const int16_t* c = codebook_table(TQK5X);
+ int bad = 0, big = 0;
+ for (uint32_t s = 0; s < 32768; ++s)
+ for (int i = 0; i < 2; ++i) {
+ const uint32_t A[2] = {89226354u, 0x9E3779B1u}, B[2] = {64248484u, 0x7F4A7C15u};
+ const uint32_t r = ((s * A[i] + B[i]) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+ for (int hw = 0; hw < 2; ++hw) {
+ const uint16_t h = (uint16_t) (hw ? r >> 16 : r & 0xffff);
+ const float v = half_to_float(h) * 8192.0f;
+ bad += c[4 * s + 2 * i + hw] != (int16_t) v || v != std::floor(v);
+ big += std::fabs(v) >= 16384.0f || std::fabs(v) < 1023.0f;
+ }
+ }
+ CHECK(bad == 0, "computed codebook differs from its definition (%d)", bad);
+ CHECK(big == 0, "computed codebook value out of [0.125, 2) (%d)", big);
+ // the distribution: rms and the share of the weights below / above one
+ double ss = 0;
+ for (int i = 0; i < 32768 * 4; ++i) ss += (double) c[i] * c[i];
+ std::printf(" computed codebook: rms %.4f (table: %.4f), all values exact integers at 2^13\n", std::sqrt(ss / (32768.0 * 4)) / 8192.0, 1.072);
+ }
}
static void t3() {
@@ -153,7 +177,7 @@ static float naive(int type, const float (*T)[4], const float (*H)[4], int rot,
std::vector<std::vector<float>> D(32, std::vector<float>(32768, 1e30f));
std::vector<std::vector<int>> B(32, std::vector<int>(32768, -1));
auto cost = [&](int p, uint32_t s) {
- const float* c = codebook_state(s);
+ const float* c = codebook_state(type, s);
float e = 0;
for (int j = 0; j < 4; ++j) { const float d = T[p][j] - c[j]; e += H[p][j] * d * d; }
return e;
@@ -232,7 +256,7 @@ static void t4() {
double sum = 0;
for (int q = 0; q < 32; ++q) {
const int p = (cc.rot + q) & 31;
- const float* c = codebook_state(sa[p]);
+ const float* c = codebook_state(type, sa[p]);
for (int j = 0; j < 4; ++j) { const double d = T[p][j] - c[j]; sum += H[p][j] * d * d; }
}
CHECK(std::fabs(sum - a) < 1e-3 * std::max(1.0, (double) a), "path cost %.6f vs reported %.6f", sum, a);
@@ -359,7 +383,7 @@ static void t7() {
const double tot = encode_blocks(type, w.data(), nullptr, (int64_t) rows * nbr, enc.data(), p, 8, errs.data(), &nf, &zb);
(void) tot;
// decoder reference: tq_states.hpp + the state table
- const int16_t* tab = kc::tq_state_table();
+ const int16_t* tab = kc::tq_state_table_for(type);
std::vector<float> ref((size_t) rows * n), ours((size_t) rows * n);
for (int64_t b = 0; b < (int64_t) rows * nbr; ++b) {
const uint8_t* blk = enc.data() + b * bb;