Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis: HIP encoder (gfx1201), bit-identical to the CPU encoder; host helpers; pilot driver reads remote GGUF shards

Niko1221c23936be7873
diff --git a/tools/trellis/build.sh b/tools/trellis/build.sh
index c85354033..45ab53842 100644
--- a/tools/trellis/build.sh
+++ b/tools/trellis/build.sh
@@ -10,6 +10,15 @@ OPT="-O3 -std=c++17 -mavx2 -mfma -mf16c -fno-fast-math -pthread"
case "${1:-test}" in
lib)
$CXX $OPT -fPIC -shared $INC tools/trellis/tqenc.cpp -o "$OUT/libtqenc.so" ;;
+hip)
+ # needs the ROCm of /data/dsv41/rv: export ROCM_PATH=/data/dsv41/rv/lib/python3.14/site-packages/_rocm_sdk_devel PATH=/data/dsv41/rv/bin:$PATH
+ # the host part of the encoder (tqenc.cpp) is built by g++: under hipcc ggml-common-tq.h's table becomes a __device__ variable
+ g++ $OPT -fPIC $INC -c tools/trellis/tqenc.cpp -o "$OUT/tqenc_host.o"
+ HF="-O3 --offload-arch=gfx1201 -mcumode -std=c++17 -mavx2 -mfma -mf16c"
+ hipcc $HF -fPIC $INC -c tools/trellis/tqenc_hip.cpp -o "$OUT/tqenc_hip.o"
+ hipcc $HF $INC -c tools/trellis/tqenc_hip_test.cpp -o "$OUT/tqenc_hip_test.o"
+ hipcc --offload-arch=gfx1201 "$OUT/tqenc_hip_test.o" "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/tqenc_hip_test"
+ hipcc --offload-arch=gfx1201 -shared "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/libtqenc_hip.so" ;;
test)
$CXX $OPT $INC -c src/kernels/cpu/tq_cpu.cpp -o "$OUT/tq_cpu.o"
$CXX $OPT $INC -c src/kernels/cpu/tq_avx2.cpp -o "$OUT/tq_avx2.o"
diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.py
index cd21dfb86..7d722f07c 100644
--- a/tools/trellis/pilot.py
+++ b/tools/trellis/pilot.py
@@ -56,24 +56,40 @@ def decode_q8_0(raw: np.ndarray, n_in: int) -> np.ndarray:
class Source:
- """Yields float32 [rows, n_in] row subsets of one expert's projection."""
+ """Yields float32 [rows, n_in] row subsets of one expert's projection. Local GGUF files (mmap) or, with
+ remote=(repo, rev, [shard paths]), the same files read by HTTP range requests (headers and the few experts needed)."""
- def __init__(self, gguf_paths):
- self.files = []
+ def __init__(self, gguf_paths=None, remote=None, tmpdir="/tmp"):
self.tensors = {}
- for p in gguf_paths:
- g = gguf_reader.GGUFFile(p)
- fh = open(p, "rb")
- mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
- self.files.append((g, fh, mm))
- for t in g.tensors:
- self.tensors[t.name] = (g, mm, t)
+ if gguf_paths:
+ for p in gguf_paths:
+ g = gguf_reader.GGUFFile(p)
+ fh = open(p, "rb")
+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
+ for t in g.tensors:
+ self.tensors[t.name] = (g, (lambda off, n, mm=mm: np.frombuffer(mm, dtype=np.uint8, count=n, offset=off)), t)
+ if remote:
+ import urllib.request
+ repo, rev, shards = remote
+ for sh in shards:
+ url = "https://huggingface.co/%s/resolve/%s/%s" % (repo, rev, sh)
+
+ def rd(off, n, url=url):
+ req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (off, off + n - 1)})
+ with urllib.request.urlopen(req, timeout=300) as r:
+ return np.frombuffer(r.read(), dtype=np.uint8)
+ hdr = rd(0, 6 << 20)
+ tp = pathlib.Path(tmpdir) / ("hdr_%s" % pathlib.Path(sh).name)
+ tp.write_bytes(hdr.tobytes())
+ g = gguf_reader.GGUFFile(tp)
+ for t in g.tensors:
+ self.tensors[t.name] = (g, rd, t)
def names(self, needle):
return sorted(n for n in self.tensors if needle in n)
def expert_rows(self, name: str, expert: int, rows: np.ndarray) -> np.ndarray:
- g, mm, ti = self.tensors[name]
+ g, rd, ti = self.tensors[name]
n_in, n_out = ti.shape[0], ti.shape[1]
if ti.type_name == "MXFP4":
rb = n_in // 32 * 17
@@ -86,9 +102,10 @@ class Source:
else:
raise ValueError("%s: tensor type %s is not readable by this pilot" % (name, ti.type_name))
base = g.data_start + ti.offset + expert * n_out * rb
+ whole = rd(base, n_out * rb) # one expert's projection (6.3 MB for MXFP4 2304 x 5120), then the row subset
out = np.empty((len(rows), n_in), dtype=np.float32)
for k, r in enumerate(rows):
- raw = np.frombuffer(mm, dtype=np.uint8, count=rb, offset=base + int(r) * rb)
+ raw = whole[int(r) * rb:(int(r) + 1) * rb]
if ti.type_name == "MXFP4":
out[k] = decode_mxfp4(raw, n_in)[0]
elif ti.type_name == "Q8_0":
@@ -119,9 +136,11 @@ def act_err(W, What, X):
def main() -> int:
ap = argparse.ArgumentParser()
- ap.add_argument("--gguf", nargs="+", required=True)
- ap.add_argument("--layers", default="0,7,15,23,31,39")
- ap.add_argument("--experts", default="0,1,128,255")
+ ap.add_argument("--gguf", nargs="+", default=None)
+ ap.add_argument("--remote", default=None, help="REPO@REV: read the shards named in --shards by HTTP range requests")
+ ap.add_argument("--shards", nargs="+", default=[])
+ ap.add_argument("--layers", default="1,7,14,21,30,38")
+ ap.add_argument("--experts", default="0,97,200,383")
ap.add_argument("--roles", default="gate,up,down")
ap.add_argument("--rates", default="K4,K5,K6,K7")
ap.add_argument("--rows", type=int, default=256)
@@ -133,7 +152,11 @@ def main() -> int:
ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")
a = ap.parse_args()
- src = Source(a.gguf)
+ if a.remote:
+ repo, rev = a.remote.split("@")
+ src = Source(remote=(repo, rev, a.shards))
+ else:
+ src = Source(a.gguf)
rng = np.random.default_rng(a.seed)
cfg = CFGS[a.cfg]
sign_cache, x_cache = {}, {}
@@ -144,7 +167,7 @@ def main() -> int:
if name not in src.tensors:
print("no tensor", name, "(have e.g. %s)" % src.names("exps")[:3])
continue
- g, mm, ti = src.tensors[name]
+ g, rd, ti = src.tensors[name]
n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]
for e in [int(x) for x in a.experts.split(",")]:
if e >= n_exp:
diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cpp
index 61aca400b..67e1e9974 100644
--- a/tools/trellis/tqenc.cpp
+++ b/tools/trellis/tqenc.cpp
@@ -130,8 +130,10 @@ inline int gpred(const Geo& g, uint32_t s) { return g.t2 ? (int) (s & 0x7f) : (i
inline int widx(const Geo& g, int p, int j) { return g.t2 ? 4 * p + j : 4 * (31 - p) + j; }
struct Scratch {
- alignas(64) float hist[32][kStates];
+ alignas(64) float D[2][kStates]; // the two live columns of the DP
+ alignas(64) uint16_t bp[32][2048]; // per step and predecessor group: the best predecessor state
alignas(64) float m[2048];
+ alignas(64) int32_t am[2048];
alignas(64) float zero[kStates];
};
@@ -191,52 +193,84 @@ inline void step_cost_bc(float* out, float a, int s0, int n, const float* t, con
step_cost(out, tmp, s0, n, t, h);
}
+// min and first index of the minimum over p[0..n)
+inline float min_arg_range(const float* p, int n, int* arg) {
+ const float m = min_range(p, n);
+ int i = 0;
+#if TQENC_AVX2
+ const __m256 vm = _mm256_set1_ps(m);
+ for (; i + 8 <= n; i += 8) {
+ const int mask = _mm256_movemask_ps(_mm256_cmp_ps(_mm256_loadu_ps(p + i), vm, _CMP_EQ_OQ));
+ if (mask) { *arg = i + __builtin_ctz((unsigned) mask); return m; }
+ }
+#endif
+ for (; i < n; ++i)
+ if (p[i] == m) { *arg = i; return m; }
+ *arg = 0;
+ return m;
+}
+
// T[p][4]: targets of processing position p (weights / d), H[p][4]: importance. rot: the sequence is processed from
// position rot (cyclically); fixed_c >= 0: tail-biting with the successor group of the first state and the predecessor
// group of the last state both == fixed_c. Writes the states by position into st[32]; returns the path cost.
float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st) {
Scratch& sc = scratch();
const int K = g.K, G = g.G;
+ float* cur = sc.D[0];
// q = 0
{
const int p = rot & 31;
- step_cost(sc.hist[0], sc.zero, 0, kStates, T[p], H[p]);
+ step_cost(cur, sc.zero, 0, kStates, T[p], H[p]);
if (fixed_c >= 0) {
- float* h0 = sc.hist[0];
if (g.t2) { // gsucc(s) = s >> 8 == c: the 256 states [c<<8, c<<8 + 256) allowed
for (int s = 0; s < kStates; ++s)
- if ((s >> 8) != fixed_c) h0[s] = kInf;
+ if ((s >> 8) != fixed_c) cur[s] = kInf;
} else { // s & (G-1) == c
for (int s = 0; s < kStates; ++s)
- if ((s & (G - 1)) != fixed_c) h0[s] = kInf;
+ if ((s & (G - 1)) != fixed_c) cur[s] = kInf;
}
}
}
for (int q = 1; q < 32; ++q) {
- const float* prev = sc.hist[q - 1];
- float* cur = sc.hist[q];
+ const float* prev = cur;
+ cur = sc.D[q & 1];
const int p = (rot + q) & 31;
float* m = sc.m;
+ uint16_t* bp = sc.bp[q];
if (g.t2) {
- // m[a] = min over h of prev[a + 128 h]
- for (int a = 0; a < 128; ++a) m[a] = prev[a];
+ // m[a] = min over h of prev[a + 128 h], argmin h
+ for (int a = 0; a < 128; ++a) { m[a] = prev[a]; sc.am[a] = 0; }
for (int h = 1; h < 256; ++h) {
const float* row = prev + 128 * h;
int a = 0;
#if TQENC_AVX2
- for (; a + 8 <= 128; a += 8) _mm256_storeu_ps(m + a, _mm256_min_ps(_mm256_loadu_ps(m + a), _mm256_loadu_ps(row + a)));
+ const __m256i vh = _mm256_set1_epi32(h);
+ for (; a + 8 <= 128; a += 8) {
+ const __m256 r = _mm256_loadu_ps(row + a), mm = _mm256_loadu_ps(m + a);
+ const __m256 lt = _mm256_cmp_ps(r, mm, _CMP_LT_OQ); // strictly smaller: the first minimum stays
+ _mm256_storeu_ps(m + a, _mm256_blendv_ps(mm, r, lt));
+ const __m256i old = _mm256_loadu_si256((const __m256i*) (sc.am + a));
+ _mm256_storeu_si256((__m256i*) (sc.am + a), _mm256_castps_si256(_mm256_blendv_ps(
+ _mm256_castsi256_ps(old), _mm256_castsi256_ps(vh), lt)));
+ }
#endif
- for (; a < 128; ++a) m[a] = std::min(m[a], row[a]);
+ for (; a < 128; ++a)
+ if (row[a] < m[a]) { m[a] = row[a]; sc.am[a] = h; }
}
+ for (int a = 0; a < 128; ++a) bp[a] = (uint16_t) (a + 128 * sc.am[a]);
for (int b = 0; b < 128; ++b) step_cost_bc(cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);
} else {
const int span = 1 << K;
- for (int a = 0; a < G; ++a) m[a] = min_range(prev + (size_t) a * span, span);
+ for (int a = 0; a < G; ++a) {
+ int arg;
+ m[a] = min_arg_range(prev + (size_t) a * span, span, &arg);
+ bp[a] = (uint16_t) ((a << K) + arg);
+ }
for (int base = 0; base < kStates; base += G) step_cost(cur + base, m, base, G, T[p], H[p]);
}
}
// final state
- const float* last = sc.hist[31];
+ const float* last = cur;
int best = -1;
float bc = 3e38f;
if (fixed_c < 0) {
@@ -257,23 +291,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, i
uint32_t s = (uint32_t) best;
st[(rot + 31) & 31] = (uint16_t) s;
for (int q = 31; q >= 1; --q) {
- const int gv = gsucc(g, s);
- const float* prev = sc.hist[q - 1];
- int arg = -1;
- float mv = 3e38f;
- if (g.t2) {
- for (int h = 0; h < 256; ++h) {
- const int x = gv + 128 * h;
- if (prev[x] < mv) { mv = prev[x]; arg = x; }
- }
- } else {
- const int span = 1 << K;
- for (int i = 0; i < span; ++i) {
- const int x = (gv << K) + i;
- if (prev[x] < mv) { mv = prev[x]; arg = x; }
- }
- }
- s = (uint32_t) arg;
+ s = sc.bp[q][gsucc(g, s)];
st[(rot + q - 1) & 31] = (uint16_t) s;
}
return bc;
@@ -352,6 +370,19 @@ const float* codebook_state(uint32_t state) {
const int16_t* codebook_table() { return cb().tab; }
+const float* point_table() {
+ static float pts[4096];
+ static std::once_flag once;
+ std::call_once(once, [] {
+ for (int i = 0; i < 4096; ++i) {
+ const float x = half_to_float(tq_lut_f16[i]) * 8192.0f;
+ const int16_t q = (int16_t) (x < 0.0f ? x - 0.5f : x + 0.5f);
+ pts[i] = (float) q * (1.0f / 8192.0f);
+ }
+ });
+ return pts;
+}
+
uint32_t block_state(int type, const uint8_t* qs, int t) {
const int K = type_kbits(type);
if (type == TQ2_T) return (((uint32_t) qs[(t + 31) & 31] << 8) | qs[t]) & 0x7fffu;
diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hpp
index 0ed36821d..241fb6cde 100644
--- a/tools/trellis/tqenc.hpp
+++ b/tools/trellis/tqenc.hpp
@@ -60,6 +60,9 @@ struct BlockResult {
/// The codebook entry of a state as the decoder sees it: four floats (int16 table / 8192, exact).
const float* codebook_state(uint32_t state); // 4 floats
const int16_t* codebook_table(); // 32768 * 4 int16, the decoder's tq_state_table()
+/// The 2048 codebook points as the decoder sees them (x, y interleaved): round(fp16 * 8192) / 8192. The GPU encoder
+/// uploads this table; it must be built by the host compiler (ggml-common-tq.h makes its table __device__ under hipcc).
+const float* point_table(); // 4096 floats
/// The state of step t (0..31) of a block's stream, by the decoder's rule, written bit by bit (independent of
/// tq_states.hpp). qs = the 4K stream bytes.
diff --git a/tools/trellis/tqenc_hip.cpp b/tools/trellis/tqenc_hip.cpp
new file mode 100644
index 000000000..ceed40734
--- /dev/null
+++ b/tools/trellis/tqenc_hip.cpp
@@ -0,0 +1,512 @@
+// tools/trellis/tqenc_hip.cpp - the trellis encoder (tqenc.hpp) on a GPU (HIP, gfx1201 / any wave32 RDNA): TQK4X / TQK5X /
+// TQK6 / TQK7 (TQ2_T stays CPU-only: its predecessor groups are strided across threads).
+//
+// One workgroup (512 threads) runs one Viterbi pass over one 128-weight block: thread t keeps the 64 path costs of the states
+// 64 t .. 64 t + 63 in registers. A step is (a) every thread takes the min/argmin of its predecessor groups (a group
+// is 2^K contiguous states, so never crosses a thread for K <= 7) and writes it to LDS plus the argmin to a global
+// back-pointer row, (b) after a barrier every thread sets its new costs D[s] = m[s & (G-1)] + c01[p0(s)] + c23[p1(s)]:
+// the cost of a state is the sum of the costs of its two hashed codebook points (2048 each), tabulated per step in LDS.
+// The same search as the CPU encoder (rotated free pass for the tail-biting start group, constrained pass, fp16 scale
+// refinement by least squares), batched over all blocks: every stage is one kernel over the whole batch.
+//
+// Build: hipcc -O3 --offload-arch=gfx1201 -mavx2 -mfma -mf16c tools/trellis/tqenc_hip.cpp tools/trellis/tqenc.cpp ...
+#include "tqenc.hpp"
+
+#include <hip/hip_runtime.h>
+#include <hip/hip_fp16.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <vector>
+
+#define HCHECK(x) \
+ do { \
+ hipError_t e_ = (x); \
+ if (e_ != hipSuccess) { \
+ std::fprintf(stderr, "HIP error %s at %s:%d: %s\n", #x, __FILE__, __LINE__, hipGetErrorString(e_)); \
+ std::abort(); \
+ } \
+ } while (0)
+
+namespace {
+
+constexpr int kSL = 6; // log2 of the states per thread (64: 512 threads per workgroup)
+constexpr int NT = 32768 >> kSL;
+constexpr float kInf = 1e30f;
+
+__device__ __forceinline__ float h2f(uint16_t h) { return __half2float(__ushort_as_half(h)); }
+__device__ __forceinline__ uint16_t f2h(float f) { return __half_as_ushort(__float2half_rn(f)); }
+
+// weight index of component j at processing position p (TQK)
+__device__ __forceinline__ int widx_d(int p, int j) { return 4 * (31 - p) + j; }
+
+struct Pass {
+ const float* w;
+ const float* h; // null: ones
+ const uint16_t* dcur; // fp16 scale per block
+ const uint8_t* active;
+ int64_t nb;
+ int rot;
+ const int* cin; // null: free pass, else the tail-biting group per block (<0: skip)
+ const float2* lutq; // 2048 points
+ uint16_t* bp_scratch;
+ uint16_t* st_out; // constrained: [nb][32] by position
+ float* cost_out; // constrained: [nb]
+ int* c_out; // free: junction group [nb]
+};
+
+template <int K, int SL>
+__global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {
+ constexpr int SPT = 1 << SL; // states per thread
+ constexpr int NT = 32768 >> SL; // threads per workgroup
+ constexpr int G = 1 << (15 - K);
+ constexpr int SPAN = 1 << K;
+ constexpr int GPT = SPT >= SPAN ? SPT / SPAN : 1; // groups per thread (SPT >= SPAN)
+ constexpr int PC = SPT >= SPAN ? 1 : SPAN / SPT; // threads sharing one group (SPAN > SPT)
+ constexpr int NP = 2048 / NT; // codebook points per thread
+ __shared__ float tab[2][2][2048];
+ __shared__ float mbuf[2][G];
+ __shared__ float Tt[32][4], Hh[32][4];
+ __shared__ float redv[NT / 32];
+ __shared__ int redi[NT / 32];
+
+ const int tid = threadIdx.x;
+ uint16_t* bp = P.bp_scratch + (size_t) blockIdx.x * 32 * G;
+ float2 L[NP];
+#pragma unroll
+ for (int k = 0; k < NP; ++k) L[k] = P.lutq[tid + NT * k];
+ const uint32_t kC = 0x9e3779b1u;
+ const uint32_t x0 = (uint32_t) (SPT * tid) * kC;
+
+ for (int64_t b = blockIdx.x; b < P.nb; b += gridDim.x) {
+ if (!P.active[b]) continue;
+ int fc = -1;
+ if (P.cin) {
+ fc = P.cin[b];
+ if (fc < 0) continue;
+ }
+ if (tid < 128) {
+ const int p = tid >> 2, j = tid & 3;
+ const int wi = widx_d(p, j);
+ const float dd = h2f(P.dcur[b]);
+ Tt[p][j] = P.w[b * 128 + wi] / dd;
+ Hh[p][j] = P.h ? P.h[b * 128 + wi] : 1.0f;
+ }
+ __syncthreads();
+
+ float D[SPT];
+ // tables of step q into parity q & 1
+ auto build = [&](int q) {
+ const int p = (P.rot + q) & 31;
+ const float t0 = Tt[p][0], t1 = Tt[p][1], t2 = Tt[p][2], t3 = Tt[p][3];
+ const float h0 = Hh[p][0], h1 = Hh[p][1], h2 = Hh[p][2], h3 = Hh[p][3];
+#pragma unroll
+ for (int k = 0; k < NP; ++k) {
+ const float a0 = t0 - L[k].x, a1 = t1 - L[k].y, a2 = t2 - L[k].x, a3 = t3 - L[k].y;
+ tab[q & 1][0][tid + NT * k] = h0 * a0 * a0 + h1 * a1 * a1;
+ tab[q & 1][1][tid + NT * k] = h2 * a2 * a2 + h3 * a3 * a3;
+ }
+ };
+ build(0);
+ __syncthreads();
+#pragma unroll
+ for (int j = 0; j < SPT; ++j) {
+ const uint32_t x = x0 + (uint32_t) j * kC;
+ float v = tab[0][0][x >> 21] + tab[0][1][(x >> 10) & 2047u];
+ if (fc >= 0 && (((SPT * tid + j) & (G - 1)) != fc)) v = kInf;
+ D[j] = v;
+ }
+ for (int q = 1; q < 32; ++q) {
+ const int par = q & 1;
+ build(q);
+ // group minima of the previous column
+ uint16_t* bpq = bp + (size_t) q * G;
+ if constexpr (SPT >= SPAN) {
+#pragma unroll
+ for (int g = 0; g < GPT; ++g) {
+ float mv = D[g * SPAN];
+ int ma = 0;
+#pragma unroll
+ for (int e = 1; e < SPAN; ++e) {
+ const float v = D[g * SPAN + e];
+ if (v < mv) { mv = v; ma = e; }
+ }
+ const int a = ((SPT * tid) >> K) + g;
+ mbuf[par][a] = mv;
+ bpq[a] = (uint16_t) (SPT * tid + g * SPAN + ma);
+ }
+ } else {
+ // a group spans PC consecutive threads (the same wave): min/argmin of the thread, then across the PC
+ float mv = D[0];
+ int ma = 0;
+#pragma unroll
+ for (int e = 1; e < SPT; ++e) {
+ const float v = D[e];
+ if (v < mv) { mv = v; ma = e; }
+ }
+ int st = SPT * tid + ma;
+#pragma unroll
+ for (int off = 1; off < PC; off <<= 1) {
+ const float ov = __shfl_xor(mv, off);
+ const int os = __shfl_xor(st, off);
+ if (ov < mv || (ov == mv && os < st)) { mv = ov; st = os; }
+ }
+ if ((tid & (PC - 1)) == 0) {
+ const int a = (SPT * tid) >> K;
+ mbuf[par][a] = mv;
+ bpq[a] = (uint16_t) st;
+ }
+ }
+ __syncthreads();
+ const float4* m4 = reinterpret_cast<const float4*>(&mbuf[par][(SPT * tid) & (G - 1)]);
+#pragma unroll
+ for (int j4 = 0; j4 < SPT / 4; ++j4) {
+ const float4 mm = m4[j4];
+ const uint32_t xa = x0 + (uint32_t) (4 * j4) * kC;
+ const uint32_t xb = xa + kC, xc = xb + kC, xd = xc + kC;
+ D[4 * j4 + 0] = mm.x + tab[par][0][xa >> 21] + tab[par][1][(xa >> 10) & 2047u];
+ D[4 * j4 + 1] = mm.y + tab[par][0][xb >> 21] + tab[par][1][(xb >> 10) & 2047u];
+ D[4 * j4 + 2] = mm.z + tab[par][0][xc >> 21] + tab[par][1][(xc >> 10) & 2047u];
+ D[4 * j4 + 3] = mm.w + tab[par][0][xd >> 21] + tab[par][1][(xd >> 10) & 2047u];
+ }
+ }
+ // final state
+ const int lo = fc < 0 ? 0 : (fc << K), hi = fc < 0 ? 32768 : lo + SPAN;
+ float bv = 3e38f;
+ int bi = -1;
+#pragma unroll
+ for (int j = 0; j < SPT; ++j) {
+ const int s = SPT * tid + j;
+ if (s >= lo && s < hi && D[j] < bv) { bv = D[j]; bi = s; }
+ }
+#pragma unroll
+ for (int off = 16; off >= 1; off >>= 1) {
+ const float ov = __shfl_xor(bv, off);
+ const int oi = __shfl_xor(bi, off);
+ if (ov < bv || (ov == bv && oi >= 0 && (bi < 0 || oi < bi))) { bv = ov; bi = oi; }
+ }
+ if ((tid & 31) == 0) { redv[tid >> 5] = bv; redi[tid >> 5] = bi; }
+ __syncthreads();
+ if (tid == 0) {
+ float v = redv[0];
+ int i = redi[0];
+ for (int w2 = 1; w2 < NT / 32; ++w2)
+ if (redv[w2] < v || (redv[w2] == v && redi[w2] >= 0 && (i < 0 || redi[w2] < i))) { v = redv[w2]; i = redi[w2]; }
+ // traceback
+ if (i < 0 || v >= kInf * 0.5f) {
+ if (P.cin) P.cost_out[b] = kInf; else P.c_out[b] = 0;
+ } else if (!P.cin) {
+ const int qj = (31 - P.rot) & 31; // the step that processes position 31
+ int s = i;
+ for (int q = 31; q > qj; --q) s = bp[(size_t) q * G + (s & (G - 1))];
+ P.c_out[b] = s >> K;
+ } else {
+ int s = i;
+ P.st_out[b * 32 + ((P.rot + 31) & 31)] = (uint16_t) s;
+ for (int q = 31; q >= 1; --q) {
+ s = bp[(size_t) q * G + (s & (G - 1))];
+ P.st_out[b * 32 + ((P.rot + q - 1) & 31)] = (uint16_t) s;
+ }
+ P.cost_out[b] = v;
+ }
+ }
+ __syncthreads();
+ }
+}
+
+// ------------------------------------------------------------------------------------------------ per-block stages
+
+__device__ __forceinline__ void state_cb(const float2* lutq, uint32_t s, float* c) {
+ const uint32_t x = s * 0x9e3779b1u;
+ const float2 a = lutq[x >> 21], b = lutq[(x >> 10) & 2047u];
+ c[0] = a.x; c[1] = a.y; c[2] = b.x; c[3] = b.y;
+}
+
+__global__ void init_kernel(const float* w, const float* h, int64_t nb, uint16_t* dcur, uint8_t* active, uint8_t* status,
+ double* sumhw2, double mult, int first, uint16_t fixed_scale) {
+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
+ if (b >= nb) return;
+ double s2 = 0, sh2 = 0, amax = 0;
+ bool fin = true;
+ for (int i = 0; i < 128; ++i) {
+ const float x = w[b * 128 + i];
+ const float hi = h ? h[b * 128 + i] : 1.0f;
+ if (!isfinite(x) || !isfinite(hi)) fin = false;
+ s2 += (double) x * x;
+ sh2 += (double) hi * x * x;
+ amax = fmax(amax, fabs((double) x));
+ }
+ if (first) {
+ status[b] = !fin ? 2 : (amax == 0.0 ? 1 : 0);
+ sumhw2[b] = sh2;
+ }
+ if (status[b] != 0) { active[b] = 0; return; }
+ const double rms = sqrt(s2 / 128.0);
+ double dd = rms / 1.072 * mult;
+ uint16_t dh = f2h((float) fmin(dd, 65504.0));
+ if (dh == 0) dh = 1;
+ if (fixed_scale) dh = fixed_scale;
+ dcur[b] = dh;
+ active[b] = 1;
+}
+
+// error of (states by position st, fp16 scale dh)
+__device__ double fit_d(const float* w, const float* h, int64_t b, const uint16_t* st, uint16_t dh, const float2* lutq) {
+ const float d = h2f(dh);
+ double e = 0;
+ for (int p = 0; p < 32; ++p) {
+ float c[4];
+ state_cb(lutq, st[p], c);
+ for (int j = 0; j < 4; ++j) {
+ const int wi = widx_d(p, j);
+ const float rec = d * c[j];
+ const double df = (double) w[b * 128 + wi] - rec;
+ e += (h ? (double) h[b * 128 + wi] : 1.0) * df * df;
+ }
+ }
+ return e;
+}
+
+// pick the best candidate, then the scale refinement step of encode_block
+__global__ void refine_kernel(const float* w, const float* h, int64_t nb, int nc, const uint16_t* st_c, const float* cost_c,
+ uint16_t* dcur, uint8_t* active, uint16_t* best_d, double* best_err, uint16_t* best_st,
+ const float2* lutq, int fixed_scale) {
+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
+ if (b >= nb || !active[b]) return;
+ int bc = -1;
+ float bcost = kInf * 0.5f;
+ for (int r = 0; r < nc; ++r) {
+ const float c = cost_c[(int64_t) r * nb + b];
+ if (c < bcost) { bcost = c; bc = r; }
+ }
+ if (bc < 0) { active[b] = 0; return; }
+ uint16_t st[32];
+ for (int p = 0; p < 32; ++p) st[p] = st_c[((int64_t) bc * nb + b) * 32 + p];
+ const uint16_t dh = dcur[b];
+ const double e = fit_d(w, h, b, st, dh, lutq);
+ if (e < best_err[b]) {
+ best_err[b] = e;
+ best_d[b] = dh;
+ for (int p = 0; p < 32; ++p) best_st[b * 32 + p] = st[p];
+ }
+ if (e == 0.0 || fixed_scale) { active[b] = 0; return; }
+ double num = 0, den = 0;
+ for (int p = 0; p < 32; ++p) {
+ float c[4];
+ state_cb(lutq, st[p], c);
+ for (int j = 0; j < 4; ++j) {
+ const int wi = widx_d(p, j);
+ const double hi = h ? (double) h[b * 128 + wi] : 1.0, cc = c[j];
+ num += hi * w[b * 128 + wi] * cc;
+ den += hi * cc * cc;
+ }
+ }
+ if (den <= 0 || num <= 0) { active[b] = 0; return; }
+ const uint16_t nh = f2h((float) fmin(num / den, 65504.0));
+ if (nh == 0 || nh == dh) { active[b] = 0; return; }
+ const double en = fit_d(w, h, b, st, nh, lutq);
+ if (en < best_err[b]) {
+ best_err[b] = en;
+ best_d[b] = nh;
+ for (int p = 0; p < 32; ++p) best_st[b * 32 + p] = st[p];
+ }
+ if (en >= e) { active[b] = 0; return; }
+ dcur[b] = nh;
+}
+
+__global__ void dedupe_kernel(int64_t nb, int nc, int* c_all) {
+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
+ if (b >= nb) return;
+ for (int r = 1; r < nc; ++r) {
+ const int c = c_all[(int64_t) r * nb + b];
+ for (int q = 0; q < r; ++q)
+ if (c_all[(int64_t) q * nb + b] == c) { c_all[(int64_t) r * nb + b] = -1; break; }
+ }
+}
+
+__global__ void pack_kernel(int K, int64_t nb, const uint8_t* status, const double* sumhw2, const uint16_t* best_d,
+ const double* best_err, const uint16_t* best_st, uint8_t* dst, double* err_out) {
+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
+ if (b >= nb) return;
+ const int bb = 2 + 4 * K;
+ uint8_t* o = dst + b * bb;
+ for (int i = 0; i < bb; ++i) o[i] = 0;
+ if (status[b] == 2) { err_out[b] = NAN; return; }
+ if (status[b] == 1 || best_err[b] >= sumhw2[b]) { err_out[b] = status[b] == 1 ? 0.0 : sumhw2[b]; return; }
+ o[0] = (uint8_t) (best_d[b] & 255);
+ o[1] = (uint8_t) (best_d[b] >> 8);
+ uint8_t* qs = o + 2;
+ for (int u = 0; u < 32; ++u) {
+ const uint32_t x = best_st[b * 32 + u] & ((1u << K) - 1u);
+ for (int bit = 0; bit < K; ++bit)
+ if ((x >> bit) & 1u) qs[(u * K + bit) >> 3] |= (uint8_t) (1u << ((u * K + bit) & 7));
+ }
+ err_out[b] = best_err[b];
+}
+
+__global__ void init_best_kernel(int64_t nb, double* best_err, uint8_t* have) {
+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
+ if (b >= nb) return;
+ best_err[b] = 1e300;
+}
+
+template <int K>
+void launch_pass(const Pass& P, int grid) {
+ hipLaunchKernelGGL((pass_kernel<K, kSL>), dim3(grid), dim3(NT), 0, 0, P);
+}
+
+void launch_pass_k(int K, const Pass& P, int grid) {
+ switch (K) {
+ case 4: launch_pass<4>(P, grid); break;
+ case 5: launch_pass<5>(P, grid); break;
+ case 6: launch_pass<6>(P, grid); break;
+ default: launch_pass<7>(P, grid); break;
+ }
+}
+
+struct Dev {
+ int device = 0;
+ int nsm = 0;
+ float2* lutq = nullptr;
+ bool ready = false;
+} g_dev;
+
+void ensure_dev(int device) {
+ if (g_dev.ready && g_dev.device == device) return;
+ HCHECK(hipSetDevice(device));
+ hipDeviceProp_t pr;
+ HCHECK(hipGetDeviceProperties(&pr, device));
+ g_dev.device = device;
+ g_dev.nsm = pr.multiProcessorCount;
+ // the 2048 codebook points as the decoder sees them: round(fp16 * 8192) / 8192 (tq_cpu.cpp fill_table)
+ std::vector<float2> pts(2048);
+ const float* pt = tqenc::point_table();
+ for (int i = 0; i < 2048; ++i) pts[i] = make_float2(pt[2 * i], pt[2 * i + 1]);
+ if (g_dev.lutq) HCHECK(hipFree(g_dev.lutq));
+ HCHECK(hipMalloc(&g_dev.lutq, 2048 * sizeof(float2)));
+ HCHECK(hipMemcpy(g_dev.lutq, pts.data(), 2048 * sizeof(float2), hipMemcpyHostToDevice));
+ g_dev.ready = true;
+}
+
+template <typename T>
+T* dalloc(size_t n) {
+ T* p = nullptr;
+ HCHECK(hipMalloc(&p, n * sizeof(T)));
+ return p;
+}
+
+} // namespace
+
+extern "C" {
+#if defined(_WIN32)
+#define TQH_EXPORT __declspec(dllexport)
+#else
+#define TQH_EXPORT __attribute__((visibility("default")))
+#endif
+
+// GPU encode of nblocks blocks of TQK4X / 5X / 6 / 7. Same arguments as tqe_encode (iters, ncand, scale_tries; exact
+// search is CPU only). stats[0] = nonfinite blocks, stats[1] = zero blocks; timing[0] = kernel seconds, [1] = passes.
+TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters, int ncand,
+ int scale_tries, int device, double* err_out, int64_t* stats, double* timing,
+ int fixed_scale_bits) {
+ const int K = tqenc::type_kbits(type);
+ if (K < 4 || K > 7) return -1.0;
+ ensure_dev(device);
+ const int bb = 2 + 4 * K, G = 1 << (15 - K);
+ const int nc = std::max(1, std::min(ncand, 8));
+ static const int rots[8] = {16, 8, 24, 12, 20, 4, 28, 0};
+ static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};
+ const int tries = fixed_scale_bits ? 1 : std::max(1, std::min(scale_tries, 7));
+ const int64_t CH = 1 << 16;
+ const int grid_cap = g_dev.nsm * 4;
+ uint16_t* bp = dalloc<uint16_t>((size_t) grid_cap * 32 * G);
+ double tot = 0;
+ int64_t nf = 0, zb = 0;
+ double ksec = 0;
+ int64_t npass = 0;
+ for (int64_t c0 = 0; c0 < nblocks; c0 += CH) {
+ const int64_t nb = std::min(CH, nblocks - c0);
+ float* dw = dalloc<float>((size_t) nb * 128);
+ float* dh = h ? dalloc<float>((size_t) nb * 128) : nullptr;
+ HCHECK(hipMemcpy(dw, w + c0 * 128, (size_t) nb * 128 * 4, hipMemcpyHostToDevice));
+ if (h) HCHECK(hipMemcpy(dh, h + c0 * 128, (size_t) nb * 128 * 4, hipMemcpyHostToDevice));
+ uint16_t* dcur = dalloc<uint16_t>(nb);
+ uint8_t* active = dalloc<uint8_t>(nb);
+ uint8_t* status = dalloc<uint8_t>(nb);
+ double* sumhw2 = dalloc<double>(nb);
+ uint16_t* best_d = dalloc<uint16_t>(nb);
+ double* best_err = dalloc<double>(nb);
+ uint16_t* best_st = dalloc<uint16_t>((size_t) nb * 32);
+ uint16_t* st_c = dalloc<uint16_t>((size_t) nc * nb * 32);
+ float* cost_c = dalloc<float>((size_t) nc * nb);
+ int* c_all = dalloc<int>((size_t) nc * nb);
+ uint8_t* d_dst = dalloc<uint8_t>((size_t) nb * bb);
+ double* d_err = dalloc<double>(nb);
+ uint8_t* have = nullptr;
+ const int tb = 128, gb = (int) ((nb + tb - 1) / tb);
+ hipLaunchKernelGGL(init_best_kernel, dim3(gb), dim3(tb), 0, 0, nb, best_err, have);
+ const auto t0 = std::chrono::steady_clock::now();
+ for (int tr = 0; tr < tries; ++tr) {
+ hipLaunchKernelGGL(init_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, dcur, active, status, sumhw2, mult[tr],
+ tr == 0 ? 1 : 0, (uint16_t) fixed_scale_bits);
+ for (int it = 0; it <= (fixed_scale_bits ? 0 : iters); ++it) {
+ Pass P{};
+ P.w = dw; P.h = dh; P.dcur = dcur; P.active = active; P.nb = nb; P.lutq = g_dev.lutq; P.bp_scratch = bp;
+ const int grid = (int) std::min<int64_t>(nb, grid_cap);
+ for (int r = 0; r < nc; ++r) {
+ P.rot = rots[r]; P.cin = nullptr; P.c_out = c_all + (int64_t) r * nb;
+ launch_pass_k(K, P, grid);
+ }
+ hipLaunchKernelGGL(dedupe_kernel, dim3(gb), dim3(tb), 0, 0, nb, nc, c_all);
+ HCHECK(hipMemset(cost_c, 0x7f, (size_t) nc * nb * sizeof(float))); // skipped candidates cost "infinity"
+ for (int r = 0; r < nc; ++r) {
+ P.rot = 0; P.cin = c_all + (int64_t) r * nb; P.c_out = nullptr;
+ P.st_out = st_c + (int64_t) r * nb * 32; P.cost_out = cost_c + (int64_t) r * nb;
+ launch_pass_k(K, P, grid);
+ }
+ hipLaunchKernelGGL(refine_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, nc, st_c, cost_c, dcur, active,
+ best_d, best_err, best_st, g_dev.lutq, fixed_scale_bits);
+ npass += 2 * nc;
+ if (std::getenv("TQH_DEBUG")) {
+ HCHECK(hipDeviceSynchronize());
+ int hc[4] = {}; float hcost[4] = {}; double he[4] = {}; uint16_t hd[4] = {}; uint8_t ha[4] = {};
+ HCHECK(hipMemcpy(hc, c_all, 16, hipMemcpyDeviceToHost));
+ HCHECK(hipMemcpy(hcost, cost_c, 16, hipMemcpyDeviceToHost));
+ HCHECK(hipMemcpy(he, best_err, 32, hipMemcpyDeviceToHost));
+ HCHECK(hipMemcpy(hd, dcur, 8, hipMemcpyDeviceToHost));
+ HCHECK(hipMemcpy(ha, active, 4, hipMemcpyDeviceToHost));
+ for (int i = 0; i < 4; ++i)
+ std::fprintf(stderr, "[dbg try %d it %d] block %d: c %d cost %g best_err %g dcur %g active %d\n", tr, it, i, hc[i], hcost[i], he[i],
+ tqenc::half_to_float(hd[i]), ha[i]);
+ }
+ }
+ }
+ hipLaunchKernelGGL(pack_kernel, dim3(gb), dim3(tb), 0, 0, K, nb, status, sumhw2, best_d, best_err, best_st, d_dst, d_err);
+ HCHECK(hipDeviceSynchronize());
+ ksec += std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+ HCHECK(hipMemcpy(dst + (size_t) c0 * bb, d_dst, (size_t) nb * bb, hipMemcpyDeviceToHost));
+ std::vector<double> herr(nb);
+ std::vector<uint8_t> hst(nb);
+ HCHECK(hipMemcpy(herr.data(), d_err, nb * 8, hipMemcpyDeviceToHost));
+ HCHECK(hipMemcpy(hst.data(), status, nb, hipMemcpyDeviceToHost));
+ for (int64_t i = 0; i < nb; ++i) {
+ if (hst[i] == 2) ++nf;
+ else tot += herr[i];
+ if (hst[i] == 1) ++zb;
+ if (err_out) err_out[c0 + i] = herr[i];
+ }
+ for (void* p : {(void*) dw, (void*) dh, (void*) dcur, (void*) active, (void*) status, (void*) sumhw2, (void*) best_d,
+ (void*) best_err, (void*) best_st, (void*) st_c, (void*) cost_c, (void*) c_all, (void*) d_dst,
+ (void*) d_err})
+ if (p) HCHECK(hipFree(p));
+ }
+ HCHECK(hipFree(bp));
+ if (stats) { stats[0] = nf; stats[1] = zb; }
+ if (timing) { timing[0] = ksec; timing[1] = (double) npass; }
+ return tot;
+}
+}
diff --git a/tools/trellis/tqenc_hip_test.cpp b/tools/trellis/tqenc_hip_test.cpp
new file mode 100644
index 000000000..c416fc8d4
--- /dev/null
+++ b/tools/trellis/tqenc_hip_test.cpp
@@ -0,0 +1,117 @@
+// tools/trellis/tqenc_hip_test.cpp - GPU encoder vs the CPU encoder: correctness (GPU bytes decode, by the CPU decoder,
+// to exactly the weights whose error the GPU reported), quality (SQNR, planted recovery) and speed (weights/s).
+// Run under the box locks: flock -s /data/gpu.lock flock -x ~/locks/gpu3.lock env HIP_VISIBLE_DEVICES=3 ./tqenc_hip_test
+#include "tqenc.hpp"
+
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+extern "C" double tqh_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters, int ncand,
+ int scale_tries, int device, double* err_out, int64_t* stats, double* timing, int fixed_scale_bits);
+
+using namespace tqenc;
+static int g_fail = 0;
+#define CHECK(c, ...) do { if (!(c)) { ++g_fail; std::printf(" FAIL line %d: %s ", __LINE__, #c); std::printf(__VA_ARGS__); std::printf("\n"); } } while (0)
+static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
+static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X"; }
+
+int main(int argc, char** argv) {
+ const long big = argc > 1 ? std::atol(argv[1]) : 131072; // blocks for the speed runs
+ std::mt19937_64 rng(77);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ const int types[4] = {TQK4X, TQK5X, TQK6, TQK7};
+ for (int type : types) {
+ const int bb = block_bytes(type);
+ std::printf("== %s\n", tname(type));
+ // 1. correctness + CPU comparison on 2048 Gaussian blocks, without and with importance weights
+ {
+ const int nb = 2048;
+ std::vector<float> w((size_t) nb * 128), h((size_t) nb * 128);
+ for (auto& v : w) v = 0.02f * nd(rng);
+ for (auto& v : h) v = 0.2f + std::fabs(nd(rng));
+ for (int wi = 0; wi < 2; ++wi) {
+ const float* hp = wi ? h.data() : nullptr;
+ std::vector<uint8_t> g((size_t) nb * bb), c((size_t) nb * bb);
+ std::vector<double> ge(nb), ce(nb);
+ int64_t st[2];
+ double tm[2];
+ const double gt = tqh_encode(type, w.data(), hp, nb, g.data(), 1, 1, 1, 0, ge.data(), st, tm, 0);
+ Params p;
+ p.iters = 1;
+ int64_t nf, zb;
+ const double ct = encode_blocks(type, w.data(), hp, nb, c.data(), p, 8, ce.data(), &nf, &zb);
+ std::vector<float> dec((size_t) nb * 128);
+ decode_blocks(type, g.data(), nb, dec.data());
+ double worst = 0;
+ for (int b = 0; b < nb; ++b) {
+ double e2 = 0;
+ for (int i = 0; i < 128; ++i) {
+ const double d = (double) w[b * 128 + i] - dec[b * 128 + i];
+ e2 += (hp ? hp[b * 128 + i] : 1.0) * d * d;
+ }
+ worst = std::max(worst, std::fabs(e2 - ge[b]) / std::max(1e-30, e2 + ge[b]));
+ }
+ CHECK(worst < 1e-5, "GPU reported error differs from the decoded error: %.2e", worst);
+ int same = 0;
+ for (int b = 0; b < nb; ++b) same += std::memcmp(&g[(size_t) b * bb], &c[(size_t) b * bb], bb) == 0;
+ std::printf(" importance %-3s GPU total error %.6e, CPU %.6e (GPU/CPU %.5f); identical blocks %d/%d; decoded-vs-reported worst %.1e\n",
+ wi ? "yes" : "no", gt, ct, gt / ct, same, nb, worst);
+ CHECK(gt <= ct * 1.002, "GPU error is more than 0.2%% above the CPU's");
+ }
+ }
+ // 2. planted solutions: weights decoded from random streams must be recovered (the scale is found by the refinement)
+ {
+ const int nb = 2048;
+ std::vector<uint8_t> blk((size_t) nb * bb);
+ for (int b = 0; b < nb; ++b) {
+ const float d = std::ldexp(0.5f + (float) (rng() % 1000) / 1000.f, -8 + (int) (rng() % 6));
+ const uint16_t dh = float_to_half(d);
+ blk[(size_t) b * bb] = dh & 255;
+ blk[(size_t) b * bb + 1] = dh >> 8;
+ for (int i = 2; i < bb; ++i) blk[(size_t) b * bb + i] = (uint8_t) rng();
+ }
+ std::vector<float> w((size_t) nb * 128);
+ decode_blocks(type, blk.data(), nb, w.data());
+ std::vector<uint8_t> g((size_t) nb * bb);
+ std::vector<double> ge(nb);
+ tqh_encode(type, w.data(), nullptr, nb, g.data(), 3, 1, 1, 0, ge.data(), nullptr, nullptr, 0);
+ int ok = 0;
+ for (int b = 0; b < nb; ++b) ok += ge[b] == 0.0;
+ std::printf(" planted: %d/%d blocks recovered exactly (scale unknown)\n", ok, nb);
+ CHECK(ok > nb * 0.9, "planted recovery low");
+ }
+ // 3. speed on big Gaussian batches
+ {
+ std::vector<float> w((size_t) big * 128);
+ for (auto& v : w) v = nd(rng);
+ std::vector<uint8_t> g((size_t) big * bb);
+ double en = 0;
+ for (float v : w) en += (double) v * v;
+ struct Cfg { const char* n; int iters, ncand, tries; } cfgs[3] = {{"fast (1 pass + 1)", 0, 1, 1}, {"default (iters 1)", 1, 1, 1}, {"rot3 iters3", 3, 3, 1}};
+ for (auto& c : cfgs) {
+ double tm[2];
+ int64_t stt[2];
+ const double t0 = now();
+ const double e = tqh_encode(type, w.data(), nullptr, big, g.data(), c.iters, c.ncand, c.tries, 0, nullptr, stt, tm, 0);
+ const double s = now() - t0;
+ {
+ std::vector<float> dec((size_t) big * 128);
+ decode_blocks(type, g.data(), big, dec.data());
+ double e2 = 0;
+ for (size_t i = 0; i < dec.size(); ++i) e2 += ((double) w[i] - dec[i]) * ((double) w[i] - dec[i]);
+ std::printf(" [check: energy %.6e, reported error %.6e, decoded error %.6e -> SQNR %.3f dB]\n", en, e, e2, 10 * std::log10(en / e2));
+ }
+ std::printf(" %-18s SQNR %.3f dB %.2f M weights/s (kernels only %.2f M/s, %.0f passes) total %.2fs [zero blocks %lld, nonfinite %lld]\n", c.n,
+ 10 * std::log10(en / e), (double) big * 128 / s / 1e6, (double) big * 128 / tm[0] / 1e6, tm[1], s,
+ (long long) stt[1], (long long) stt[0]);
+ }
+ }
+ }
+ std::printf("\n%s (%d failures)\n", g_fail ? "FAILED" : "OK", g_fail);
+ return g_fail ? 1 : 0;
+}