Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: HIP encoder (gfx1201), bit-identical to the CPU encoder; host helpers; pilot driver reads remote GGUF shards
Niko1221c23936be7873
diff --git a/tools/trellis/build.sh b/tools/trellis/build.shindex c85354033..45ab53842 100644--- a/tools/trellis/build.sh+++ b/tools/trellis/build.sh@@ -10,6 +10,15 @@ OPT="-O3 -std=c++17 -mavx2 -mfma -mf16c -fno-fast-math -pthread"case "${1:-test}" inlib)$CXX $OPT -fPIC -shared $INC tools/trellis/tqenc.cpp -o "$OUT/libtqenc.so" ;;+hip)+ # needs the ROCm of /data/dsv41/rv: export ROCM_PATH=/data/dsv41/rv/lib/python3.14/site-packages/_rocm_sdk_devel PATH=/data/dsv41/rv/bin:$PATH+ # the host part of the encoder (tqenc.cpp) is built by g++: under hipcc ggml-common-tq.h's table becomes a __device__ variable+ g++ $OPT -fPIC $INC -c tools/trellis/tqenc.cpp -o "$OUT/tqenc_host.o"+ HF="-O3 --offload-arch=gfx1201 -mcumode -std=c++17 -mavx2 -mfma -mf16c"+ hipcc $HF -fPIC $INC -c tools/trellis/tqenc_hip.cpp -o "$OUT/tqenc_hip.o"+ hipcc $HF $INC -c tools/trellis/tqenc_hip_test.cpp -o "$OUT/tqenc_hip_test.o"+ hipcc --offload-arch=gfx1201 "$OUT/tqenc_hip_test.o" "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/tqenc_hip_test"+ hipcc --offload-arch=gfx1201 -shared "$OUT/tqenc_hip.o" "$OUT/tqenc_host.o" -o "$OUT/libtqenc_hip.so" ;;test)$CXX $OPT $INC -c src/kernels/cpu/tq_cpu.cpp -o "$OUT/tq_cpu.o"$CXX $OPT $INC -c src/kernels/cpu/tq_avx2.cpp -o "$OUT/tq_avx2.o"diff --git a/tools/trellis/pilot.py b/tools/trellis/pilot.pyindex cd21dfb86..7d722f07c 100644--- a/tools/trellis/pilot.py+++ b/tools/trellis/pilot.py@@ -56,24 +56,40 @@ def decode_q8_0(raw: np.ndarray, n_in: int) -> np.ndarray:class Source:- """Yields float32 [rows, n_in] row subsets of one expert's projection."""+ """Yields float32 [rows, n_in] row subsets of one expert's projection. Local GGUF files (mmap) or, with+ remote=(repo, rev, [shard paths]), the same files read by HTTP range requests (headers and the few experts needed)."""- def __init__(self, gguf_paths):- self.files = []+ def __init__(self, gguf_paths=None, remote=None, tmpdir="/tmp"):self.tensors = {}- for p in gguf_paths:- g = gguf_reader.GGUFFile(p)- fh = open(p, "rb")- mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)- self.files.append((g, fh, mm))- for t in g.tensors:- self.tensors[t.name] = (g, mm, t)+ if gguf_paths:+ for p in gguf_paths:+ g = gguf_reader.GGUFFile(p)+ fh = open(p, "rb")+ mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)+ for t in g.tensors:+ self.tensors[t.name] = (g, (lambda off, n, mm=mm: np.frombuffer(mm, dtype=np.uint8, count=n, offset=off)), t)+ if remote:+ import urllib.request+ repo, rev, shards = remote+ for sh in shards:+ url = "https://huggingface.co/%s/resolve/%s/%s" % (repo, rev, sh)++ def rd(off, n, url=url):+ req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (off, off + n - 1)})+ with urllib.request.urlopen(req, timeout=300) as r:+ return np.frombuffer(r.read(), dtype=np.uint8)+ hdr = rd(0, 6 << 20)+ tp = pathlib.Path(tmpdir) / ("hdr_%s" % pathlib.Path(sh).name)+ tp.write_bytes(hdr.tobytes())+ g = gguf_reader.GGUFFile(tp)+ for t in g.tensors:+ self.tensors[t.name] = (g, rd, t)def names(self, needle):return sorted(n for n in self.tensors if needle in n)def expert_rows(self, name: str, expert: int, rows: np.ndarray) -> np.ndarray:- g, mm, ti = self.tensors[name]+ g, rd, ti = self.tensors[name]n_in, n_out = ti.shape[0], ti.shape[1]if ti.type_name == "MXFP4":rb = n_in // 32 * 17@@ -86,9 +102,10 @@ class Source:else:raise ValueError("%s: tensor type %s is not readable by this pilot" % (name, ti.type_name))base = g.data_start + ti.offset + expert * n_out * rb+ whole = rd(base, n_out * rb) # one expert's projection (6.3 MB for MXFP4 2304 x 5120), then the row subsetout = np.empty((len(rows), n_in), dtype=np.float32)for k, r in enumerate(rows):- raw = np.frombuffer(mm, dtype=np.uint8, count=rb, offset=base + int(r) * rb)+ raw = whole[int(r) * rb:(int(r) + 1) * rb]if ti.type_name == "MXFP4":out[k] = decode_mxfp4(raw, n_in)[0]elif ti.type_name == "Q8_0":@@ -119,9 +136,11 @@ def act_err(W, What, X):def main() -> int:ap = argparse.ArgumentParser()- ap.add_argument("--gguf", nargs="+", required=True)- ap.add_argument("--layers", default="0,7,15,23,31,39")- ap.add_argument("--experts", default="0,1,128,255")+ ap.add_argument("--gguf", nargs="+", default=None)+ ap.add_argument("--remote", default=None, help="REPO@REV: read the shards named in --shards by HTTP range requests")+ ap.add_argument("--shards", nargs="+", default=[])+ ap.add_argument("--layers", default="1,7,14,21,30,38")+ ap.add_argument("--experts", default="0,97,200,383")ap.add_argument("--roles", default="gate,up,down")ap.add_argument("--rates", default="K4,K5,K6,K7")ap.add_argument("--rows", type=int, default=256)@@ -133,7 +152,11 @@ def main() -> int:ap.add_argument("--pattern", default="blk.{L}.ffn_{R}_exps.weight")a = ap.parse_args()- src = Source(a.gguf)+ if a.remote:+ repo, rev = a.remote.split("@")+ src = Source(remote=(repo, rev, a.shards))+ else:+ src = Source(a.gguf)rng = np.random.default_rng(a.seed)cfg = CFGS[a.cfg]sign_cache, x_cache = {}, {}@@ -144,7 +167,7 @@ def main() -> int:if name not in src.tensors:print("no tensor", name, "(have e.g. %s)" % src.names("exps")[:3])continue- g, mm, ti = src.tensors[name]+ g, rd, ti = src.tensors[name]n_in, n_out, n_exp = ti.shape[0], ti.shape[1], ti.shape[2]for e in [int(x) for x in a.experts.split(",")]:if e >= n_exp:diff --git a/tools/trellis/tqenc.cpp b/tools/trellis/tqenc.cppindex 61aca400b..67e1e9974 100644--- a/tools/trellis/tqenc.cpp+++ b/tools/trellis/tqenc.cpp@@ -130,8 +130,10 @@ inline int gpred(const Geo& g, uint32_t s) { return g.t2 ? (int) (s & 0x7f) : (iinline int widx(const Geo& g, int p, int j) { return g.t2 ? 4 * p + j : 4 * (31 - p) + j; }struct Scratch {- alignas(64) float hist[32][kStates];+ alignas(64) float D[2][kStates]; // the two live columns of the DP+ alignas(64) uint16_t bp[32][2048]; // per step and predecessor group: the best predecessor statealignas(64) float m[2048];+ alignas(64) int32_t am[2048];alignas(64) float zero[kStates];};@@ -191,52 +193,84 @@ inline void step_cost_bc(float* out, float a, int s0, int n, const float* t, constep_cost(out, tmp, s0, n, t, h);}+// min and first index of the minimum over p[0..n)+inline float min_arg_range(const float* p, int n, int* arg) {+ const float m = min_range(p, n);+ int i = 0;+#if TQENC_AVX2+ const __m256 vm = _mm256_set1_ps(m);+ for (; i + 8 <= n; i += 8) {+ const int mask = _mm256_movemask_ps(_mm256_cmp_ps(_mm256_loadu_ps(p + i), vm, _CMP_EQ_OQ));+ if (mask) { *arg = i + __builtin_ctz((unsigned) mask); return m; }+ }+#endif+ for (; i < n; ++i)+ if (p[i] == m) { *arg = i; return m; }+ *arg = 0;+ return m;+}+// T[p][4]: targets of processing position p (weights / d), H[p][4]: importance. rot: the sequence is processed from// position rot (cyclically); fixed_c >= 0: tail-biting with the successor group of the first state and the predecessor// group of the last state both == fixed_c. Writes the states by position into st[32]; returns the path cost.float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, int fixed_c, uint16_t* st) {Scratch& sc = scratch();const int K = g.K, G = g.G;+ float* cur = sc.D[0];// q = 0{const int p = rot & 31;- step_cost(sc.hist[0], sc.zero, 0, kStates, T[p], H[p]);+ step_cost(cur, sc.zero, 0, kStates, T[p], H[p]);if (fixed_c >= 0) {- float* h0 = sc.hist[0];if (g.t2) { // gsucc(s) = s >> 8 == c: the 256 states [c<<8, c<<8 + 256) allowedfor (int s = 0; s < kStates; ++s)- if ((s >> 8) != fixed_c) h0[s] = kInf;+ if ((s >> 8) != fixed_c) cur[s] = kInf;} else { // s & (G-1) == cfor (int s = 0; s < kStates; ++s)- if ((s & (G - 1)) != fixed_c) h0[s] = kInf;+ if ((s & (G - 1)) != fixed_c) cur[s] = kInf;}}}for (int q = 1; q < 32; ++q) {- const float* prev = sc.hist[q - 1];- float* cur = sc.hist[q];+ const float* prev = cur;+ cur = sc.D[q & 1];const int p = (rot + q) & 31;float* m = sc.m;+ uint16_t* bp = sc.bp[q];if (g.t2) {- // m[a] = min over h of prev[a + 128 h]- for (int a = 0; a < 128; ++a) m[a] = prev[a];+ // m[a] = min over h of prev[a + 128 h], argmin h+ for (int a = 0; a < 128; ++a) { m[a] = prev[a]; sc.am[a] = 0; }for (int h = 1; h < 256; ++h) {const float* row = prev + 128 * h;int a = 0;#if TQENC_AVX2- for (; a + 8 <= 128; a += 8) _mm256_storeu_ps(m + a, _mm256_min_ps(_mm256_loadu_ps(m + a), _mm256_loadu_ps(row + a)));+ const __m256i vh = _mm256_set1_epi32(h);+ for (; a + 8 <= 128; a += 8) {+ const __m256 r = _mm256_loadu_ps(row + a), mm = _mm256_loadu_ps(m + a);+ const __m256 lt = _mm256_cmp_ps(r, mm, _CMP_LT_OQ); // strictly smaller: the first minimum stays+ _mm256_storeu_ps(m + a, _mm256_blendv_ps(mm, r, lt));+ const __m256i old = _mm256_loadu_si256((const __m256i*) (sc.am + a));+ _mm256_storeu_si256((__m256i*) (sc.am + a), _mm256_castps_si256(_mm256_blendv_ps(+ _mm256_castsi256_ps(old), _mm256_castsi256_ps(vh), lt)));+ }#endif- for (; a < 128; ++a) m[a] = std::min(m[a], row[a]);+ for (; a < 128; ++a)+ if (row[a] < m[a]) { m[a] = row[a]; sc.am[a] = h; }}+ for (int a = 0; a < 128; ++a) bp[a] = (uint16_t) (a + 128 * sc.am[a]);for (int b = 0; b < 128; ++b) step_cost_bc(cur + 256 * b, m[b], 256 * b, 256, T[p], H[p]);} else {const int span = 1 << K;- for (int a = 0; a < G; ++a) m[a] = min_range(prev + (size_t) a * span, span);+ for (int a = 0; a < G; ++a) {+ int arg;+ m[a] = min_arg_range(prev + (size_t) a * span, span, &arg);+ bp[a] = (uint16_t) ((a << K) + arg);+ }for (int base = 0; base < kStates; base += G) step_cost(cur + base, m, base, G, T[p], H[p]);}}// final state- const float* last = sc.hist[31];+ const float* last = cur;int best = -1;float bc = 3e38f;if (fixed_c < 0) {@@ -257,23 +291,7 @@ float viterbi(const Geo& g, const float (*T)[4], const float (*H)[4], int rot, iuint32_t s = (uint32_t) best;st[(rot + 31) & 31] = (uint16_t) s;for (int q = 31; q >= 1; --q) {- const int gv = gsucc(g, s);- const float* prev = sc.hist[q - 1];- int arg = -1;- float mv = 3e38f;- if (g.t2) {- for (int h = 0; h < 256; ++h) {- const int x = gv + 128 * h;- if (prev[x] < mv) { mv = prev[x]; arg = x; }- }- } else {- const int span = 1 << K;- for (int i = 0; i < span; ++i) {- const int x = (gv << K) + i;- if (prev[x] < mv) { mv = prev[x]; arg = x; }- }- }- s = (uint32_t) arg;+ s = sc.bp[q][gsucc(g, s)];st[(rot + q - 1) & 31] = (uint16_t) s;}return bc;@@ -352,6 +370,19 @@ const float* codebook_state(uint32_t state) {const int16_t* codebook_table() { return cb().tab; }+const float* point_table() {+ static float pts[4096];+ static std::once_flag once;+ std::call_once(once, [] {+ for (int i = 0; i < 4096; ++i) {+ const float x = half_to_float(tq_lut_f16[i]) * 8192.0f;+ const int16_t q = (int16_t) (x < 0.0f ? x - 0.5f : x + 0.5f);+ pts[i] = (float) q * (1.0f / 8192.0f);+ }+ });+ return pts;+}+uint32_t block_state(int type, const uint8_t* qs, int t) {const int K = type_kbits(type);if (type == TQ2_T) return (((uint32_t) qs[(t + 31) & 31] << 8) | qs[t]) & 0x7fffu;diff --git a/tools/trellis/tqenc.hpp b/tools/trellis/tqenc.hppindex 0ed36821d..241fb6cde 100644--- a/tools/trellis/tqenc.hpp+++ b/tools/trellis/tqenc.hpp@@ -60,6 +60,9 @@ struct BlockResult {/// The codebook entry of a state as the decoder sees it: four floats (int16 table / 8192, exact).const float* codebook_state(uint32_t state); // 4 floatsconst int16_t* codebook_table(); // 32768 * 4 int16, the decoder's tq_state_table()+/// The 2048 codebook points as the decoder sees them (x, y interleaved): round(fp16 * 8192) / 8192. The GPU encoder+/// uploads this table; it must be built by the host compiler (ggml-common-tq.h makes its table __device__ under hipcc).+const float* point_table(); // 4096 floats/// The state of step t (0..31) of a block's stream, by the decoder's rule, written bit by bit (independent of/// tq_states.hpp). qs = the 4K stream bytes.diff --git a/tools/trellis/tqenc_hip.cpp b/tools/trellis/tqenc_hip.cppnew file mode 100644index 000000000..ceed40734--- /dev/null+++ b/tools/trellis/tqenc_hip.cpp@@ -0,0 +1,512 @@+// tools/trellis/tqenc_hip.cpp - the trellis encoder (tqenc.hpp) on a GPU (HIP, gfx1201 / any wave32 RDNA): TQK4X / TQK5X /+// TQK6 / TQK7 (TQ2_T stays CPU-only: its predecessor groups are strided across threads).+//+// One workgroup (512 threads) runs one Viterbi pass over one 128-weight block: thread t keeps the 64 path costs of the states+// 64 t .. 64 t + 63 in registers. A step is (a) every thread takes the min/argmin of its predecessor groups (a group+// is 2^K contiguous states, so never crosses a thread for K <= 7) and writes it to LDS plus the argmin to a global+// back-pointer row, (b) after a barrier every thread sets its new costs D[s] = m[s & (G-1)] + c01[p0(s)] + c23[p1(s)]:+// the cost of a state is the sum of the costs of its two hashed codebook points (2048 each), tabulated per step in LDS.+// The same search as the CPU encoder (rotated free pass for the tail-biting start group, constrained pass, fp16 scale+// refinement by least squares), batched over all blocks: every stage is one kernel over the whole batch.+//+// Build: hipcc -O3 --offload-arch=gfx1201 -mavx2 -mfma -mf16c tools/trellis/tqenc_hip.cpp tools/trellis/tqenc.cpp ...+#include "tqenc.hpp"++#include <hip/hip_runtime.h>+#include <hip/hip_fp16.h>++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <vector>++#define HCHECK(x) \+ do { \+ hipError_t e_ = (x); \+ if (e_ != hipSuccess) { \+ std::fprintf(stderr, "HIP error %s at %s:%d: %s\n", #x, __FILE__, __LINE__, hipGetErrorString(e_)); \+ std::abort(); \+ } \+ } while (0)++namespace {++constexpr int kSL = 6; // log2 of the states per thread (64: 512 threads per workgroup)+constexpr int NT = 32768 >> kSL;+constexpr float kInf = 1e30f;++__device__ __forceinline__ float h2f(uint16_t h) { return __half2float(__ushort_as_half(h)); }+__device__ __forceinline__ uint16_t f2h(float f) { return __half_as_ushort(__float2half_rn(f)); }++// weight index of component j at processing position p (TQK)+__device__ __forceinline__ int widx_d(int p, int j) { return 4 * (31 - p) + j; }++struct Pass {+ const float* w;+ const float* h; // null: ones+ const uint16_t* dcur; // fp16 scale per block+ const uint8_t* active;+ int64_t nb;+ int rot;+ const int* cin; // null: free pass, else the tail-biting group per block (<0: skip)+ const float2* lutq; // 2048 points+ uint16_t* bp_scratch;+ uint16_t* st_out; // constrained: [nb][32] by position+ float* cost_out; // constrained: [nb]+ int* c_out; // free: junction group [nb]+};++template <int K, int SL>+__global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {+ constexpr int SPT = 1 << SL; // states per thread+ constexpr int NT = 32768 >> SL; // threads per workgroup+ constexpr int G = 1 << (15 - K);+ constexpr int SPAN = 1 << K;+ constexpr int GPT = SPT >= SPAN ? SPT / SPAN : 1; // groups per thread (SPT >= SPAN)+ constexpr int PC = SPT >= SPAN ? 1 : SPAN / SPT; // threads sharing one group (SPAN > SPT)+ constexpr int NP = 2048 / NT; // codebook points per thread+ __shared__ float tab[2][2][2048];+ __shared__ float mbuf[2][G];+ __shared__ float Tt[32][4], Hh[32][4];+ __shared__ float redv[NT / 32];+ __shared__ int redi[NT / 32];++ const int tid = threadIdx.x;+ uint16_t* bp = P.bp_scratch + (size_t) blockIdx.x * 32 * G;+ float2 L[NP];+#pragma unroll+ for (int k = 0; k < NP; ++k) L[k] = P.lutq[tid + NT * k];+ const uint32_t kC = 0x9e3779b1u;+ const uint32_t x0 = (uint32_t) (SPT * tid) * kC;++ for (int64_t b = blockIdx.x; b < P.nb; b += gridDim.x) {+ if (!P.active[b]) continue;+ int fc = -1;+ if (P.cin) {+ fc = P.cin[b];+ if (fc < 0) continue;+ }+ if (tid < 128) {+ const int p = tid >> 2, j = tid & 3;+ const int wi = widx_d(p, j);+ const float dd = h2f(P.dcur[b]);+ Tt[p][j] = P.w[b * 128 + wi] / dd;+ Hh[p][j] = P.h ? P.h[b * 128 + wi] : 1.0f;+ }+ __syncthreads();++ float D[SPT];+ // tables of step q into parity q & 1+ auto build = [&](int q) {+ const int p = (P.rot + q) & 31;+ const float t0 = Tt[p][0], t1 = Tt[p][1], t2 = Tt[p][2], t3 = Tt[p][3];+ const float h0 = Hh[p][0], h1 = Hh[p][1], h2 = Hh[p][2], h3 = Hh[p][3];+#pragma unroll+ for (int k = 0; k < NP; ++k) {+ const float a0 = t0 - L[k].x, a1 = t1 - L[k].y, a2 = t2 - L[k].x, a3 = t3 - L[k].y;+ tab[q & 1][0][tid + NT * k] = h0 * a0 * a0 + h1 * a1 * a1;+ tab[q & 1][1][tid + NT * k] = h2 * a2 * a2 + h3 * a3 * a3;+ }+ };+ build(0);+ __syncthreads();+#pragma unroll+ for (int j = 0; j < SPT; ++j) {+ const uint32_t x = x0 + (uint32_t) j * kC;+ float v = tab[0][0][x >> 21] + tab[0][1][(x >> 10) & 2047u];+ if (fc >= 0 && (((SPT * tid + j) & (G - 1)) != fc)) v = kInf;+ D[j] = v;+ }+ for (int q = 1; q < 32; ++q) {+ const int par = q & 1;+ build(q);+ // group minima of the previous column+ uint16_t* bpq = bp + (size_t) q * G;+ if constexpr (SPT >= SPAN) {+#pragma unroll+ for (int g = 0; g < GPT; ++g) {+ float mv = D[g * SPAN];+ int ma = 0;+#pragma unroll+ for (int e = 1; e < SPAN; ++e) {+ const float v = D[g * SPAN + e];+ if (v < mv) { mv = v; ma = e; }+ }+ const int a = ((SPT * tid) >> K) + g;+ mbuf[par][a] = mv;+ bpq[a] = (uint16_t) (SPT * tid + g * SPAN + ma);+ }+ } else {+ // a group spans PC consecutive threads (the same wave): min/argmin of the thread, then across the PC+ float mv = D[0];+ int ma = 0;+#pragma unroll+ for (int e = 1; e < SPT; ++e) {+ const float v = D[e];+ if (v < mv) { mv = v; ma = e; }+ }+ int st = SPT * tid + ma;+#pragma unroll+ for (int off = 1; off < PC; off <<= 1) {+ const float ov = __shfl_xor(mv, off);+ const int os = __shfl_xor(st, off);+ if (ov < mv || (ov == mv && os < st)) { mv = ov; st = os; }+ }+ if ((tid & (PC - 1)) == 0) {+ const int a = (SPT * tid) >> K;+ mbuf[par][a] = mv;+ bpq[a] = (uint16_t) st;+ }+ }+ __syncthreads();+ const float4* m4 = reinterpret_cast<const float4*>(&mbuf[par][(SPT * tid) & (G - 1)]);+#pragma unroll+ for (int j4 = 0; j4 < SPT / 4; ++j4) {+ const float4 mm = m4[j4];+ const uint32_t xa = x0 + (uint32_t) (4 * j4) * kC;+ const uint32_t xb = xa + kC, xc = xb + kC, xd = xc + kC;+ D[4 * j4 + 0] = mm.x + tab[par][0][xa >> 21] + tab[par][1][(xa >> 10) & 2047u];+ D[4 * j4 + 1] = mm.y + tab[par][0][xb >> 21] + tab[par][1][(xb >> 10) & 2047u];+ D[4 * j4 + 2] = mm.z + tab[par][0][xc >> 21] + tab[par][1][(xc >> 10) & 2047u];+ D[4 * j4 + 3] = mm.w + tab[par][0][xd >> 21] + tab[par][1][(xd >> 10) & 2047u];+ }+ }+ // final state+ const int lo = fc < 0 ? 0 : (fc << K), hi = fc < 0 ? 32768 : lo + SPAN;+ float bv = 3e38f;+ int bi = -1;+#pragma unroll+ for (int j = 0; j < SPT; ++j) {+ const int s = SPT * tid + j;+ if (s >= lo && s < hi && D[j] < bv) { bv = D[j]; bi = s; }+ }+#pragma unroll+ for (int off = 16; off >= 1; off >>= 1) {+ const float ov = __shfl_xor(bv, off);+ const int oi = __shfl_xor(bi, off);+ if (ov < bv || (ov == bv && oi >= 0 && (bi < 0 || oi < bi))) { bv = ov; bi = oi; }+ }+ if ((tid & 31) == 0) { redv[tid >> 5] = bv; redi[tid >> 5] = bi; }+ __syncthreads();+ if (tid == 0) {+ float v = redv[0];+ int i = redi[0];+ for (int w2 = 1; w2 < NT / 32; ++w2)+ if (redv[w2] < v || (redv[w2] == v && redi[w2] >= 0 && (i < 0 || redi[w2] < i))) { v = redv[w2]; i = redi[w2]; }+ // traceback+ if (i < 0 || v >= kInf * 0.5f) {+ if (P.cin) P.cost_out[b] = kInf; else P.c_out[b] = 0;+ } else if (!P.cin) {+ const int qj = (31 - P.rot) & 31; // the step that processes position 31+ int s = i;+ for (int q = 31; q > qj; --q) s = bp[(size_t) q * G + (s & (G - 1))];+ P.c_out[b] = s >> K;+ } else {+ int s = i;+ P.st_out[b * 32 + ((P.rot + 31) & 31)] = (uint16_t) s;+ for (int q = 31; q >= 1; --q) {+ s = bp[(size_t) q * G + (s & (G - 1))];+ P.st_out[b * 32 + ((P.rot + q - 1) & 31)] = (uint16_t) s;+ }+ P.cost_out[b] = v;+ }+ }+ __syncthreads();+ }+}++// ------------------------------------------------------------------------------------------------ per-block stages++__device__ __forceinline__ void state_cb(const float2* lutq, uint32_t s, float* c) {+ const uint32_t x = s * 0x9e3779b1u;+ const float2 a = lutq[x >> 21], b = lutq[(x >> 10) & 2047u];+ c[0] = a.x; c[1] = a.y; c[2] = b.x; c[3] = b.y;+}++__global__ void init_kernel(const float* w, const float* h, int64_t nb, uint16_t* dcur, uint8_t* active, uint8_t* status,+ double* sumhw2, double mult, int first, uint16_t fixed_scale) {+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;+ if (b >= nb) return;+ double s2 = 0, sh2 = 0, amax = 0;+ bool fin = true;+ for (int i = 0; i < 128; ++i) {+ const float x = w[b * 128 + i];+ const float hi = h ? h[b * 128 + i] : 1.0f;+ if (!isfinite(x) || !isfinite(hi)) fin = false;+ s2 += (double) x * x;+ sh2 += (double) hi * x * x;+ amax = fmax(amax, fabs((double) x));+ }+ if (first) {+ status[b] = !fin ? 2 : (amax == 0.0 ? 1 : 0);+ sumhw2[b] = sh2;+ }+ if (status[b] != 0) { active[b] = 0; return; }+ const double rms = sqrt(s2 / 128.0);+ double dd = rms / 1.072 * mult;+ uint16_t dh = f2h((float) fmin(dd, 65504.0));+ if (dh == 0) dh = 1;+ if (fixed_scale) dh = fixed_scale;+ dcur[b] = dh;+ active[b] = 1;+}++// error of (states by position st, fp16 scale dh)+__device__ double fit_d(const float* w, const float* h, int64_t b, const uint16_t* st, uint16_t dh, const float2* lutq) {+ const float d = h2f(dh);+ double e = 0;+ for (int p = 0; p < 32; ++p) {+ float c[4];+ state_cb(lutq, st[p], c);+ for (int j = 0; j < 4; ++j) {+ const int wi = widx_d(p, j);+ const float rec = d * c[j];+ const double df = (double) w[b * 128 + wi] - rec;+ e += (h ? (double) h[b * 128 + wi] : 1.0) * df * df;+ }+ }+ return e;+}++// pick the best candidate, then the scale refinement step of encode_block+__global__ void refine_kernel(const float* w, const float* h, int64_t nb, int nc, const uint16_t* st_c, const float* cost_c,+ uint16_t* dcur, uint8_t* active, uint16_t* best_d, double* best_err, uint16_t* best_st,+ const float2* lutq, int fixed_scale) {+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;+ if (b >= nb || !active[b]) return;+ int bc = -1;+ float bcost = kInf * 0.5f;+ for (int r = 0; r < nc; ++r) {+ const float c = cost_c[(int64_t) r * nb + b];+ if (c < bcost) { bcost = c; bc = r; }+ }+ if (bc < 0) { active[b] = 0; return; }+ uint16_t st[32];+ for (int p = 0; p < 32; ++p) st[p] = st_c[((int64_t) bc * nb + b) * 32 + p];+ const uint16_t dh = dcur[b];+ const double e = fit_d(w, h, b, st, dh, lutq);+ if (e < best_err[b]) {+ best_err[b] = e;+ best_d[b] = dh;+ for (int p = 0; p < 32; ++p) best_st[b * 32 + p] = st[p];+ }+ if (e == 0.0 || fixed_scale) { active[b] = 0; return; }+ double num = 0, den = 0;+ for (int p = 0; p < 32; ++p) {+ float c[4];+ state_cb(lutq, st[p], c);+ for (int j = 0; j < 4; ++j) {+ const int wi = widx_d(p, j);+ const double hi = h ? (double) h[b * 128 + wi] : 1.0, cc = c[j];+ num += hi * w[b * 128 + wi] * cc;+ den += hi * cc * cc;+ }+ }+ if (den <= 0 || num <= 0) { active[b] = 0; return; }+ const uint16_t nh = f2h((float) fmin(num / den, 65504.0));+ if (nh == 0 || nh == dh) { active[b] = 0; return; }+ const double en = fit_d(w, h, b, st, nh, lutq);+ if (en < best_err[b]) {+ best_err[b] = en;+ best_d[b] = nh;+ for (int p = 0; p < 32; ++p) best_st[b * 32 + p] = st[p];+ }+ if (en >= e) { active[b] = 0; return; }+ dcur[b] = nh;+}++__global__ void dedupe_kernel(int64_t nb, int nc, int* c_all) {+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;+ if (b >= nb) return;+ for (int r = 1; r < nc; ++r) {+ const int c = c_all[(int64_t) r * nb + b];+ for (int q = 0; q < r; ++q)+ if (c_all[(int64_t) q * nb + b] == c) { c_all[(int64_t) r * nb + b] = -1; break; }+ }+}++__global__ void pack_kernel(int K, int64_t nb, const uint8_t* status, const double* sumhw2, const uint16_t* best_d,+ const double* best_err, const uint16_t* best_st, uint8_t* dst, double* err_out) {+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;+ if (b >= nb) return;+ const int bb = 2 + 4 * K;+ uint8_t* o = dst + b * bb;+ for (int i = 0; i < bb; ++i) o[i] = 0;+ if (status[b] == 2) { err_out[b] = NAN; return; }+ if (status[b] == 1 || best_err[b] >= sumhw2[b]) { err_out[b] = status[b] == 1 ? 0.0 : sumhw2[b]; return; }+ o[0] = (uint8_t) (best_d[b] & 255);+ o[1] = (uint8_t) (best_d[b] >> 8);+ uint8_t* qs = o + 2;+ for (int u = 0; u < 32; ++u) {+ const uint32_t x = best_st[b * 32 + u] & ((1u << K) - 1u);+ for (int bit = 0; bit < K; ++bit)+ if ((x >> bit) & 1u) qs[(u * K + bit) >> 3] |= (uint8_t) (1u << ((u * K + bit) & 7));+ }+ err_out[b] = best_err[b];+}++__global__ void init_best_kernel(int64_t nb, double* best_err, uint8_t* have) {+ const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;+ if (b >= nb) return;+ best_err[b] = 1e300;+}++template <int K>+void launch_pass(const Pass& P, int grid) {+ hipLaunchKernelGGL((pass_kernel<K, kSL>), dim3(grid), dim3(NT), 0, 0, P);+}++void launch_pass_k(int K, const Pass& P, int grid) {+ switch (K) {+ case 4: launch_pass<4>(P, grid); break;+ case 5: launch_pass<5>(P, grid); break;+ case 6: launch_pass<6>(P, grid); break;+ default: launch_pass<7>(P, grid); break;+ }+}++struct Dev {+ int device = 0;+ int nsm = 0;+ float2* lutq = nullptr;+ bool ready = false;+} g_dev;++void ensure_dev(int device) {+ if (g_dev.ready && g_dev.device == device) return;+ HCHECK(hipSetDevice(device));+ hipDeviceProp_t pr;+ HCHECK(hipGetDeviceProperties(&pr, device));+ g_dev.device = device;+ g_dev.nsm = pr.multiProcessorCount;+ // the 2048 codebook points as the decoder sees them: round(fp16 * 8192) / 8192 (tq_cpu.cpp fill_table)+ std::vector<float2> pts(2048);+ const float* pt = tqenc::point_table();+ for (int i = 0; i < 2048; ++i) pts[i] = make_float2(pt[2 * i], pt[2 * i + 1]);+ if (g_dev.lutq) HCHECK(hipFree(g_dev.lutq));+ HCHECK(hipMalloc(&g_dev.lutq, 2048 * sizeof(float2)));+ HCHECK(hipMemcpy(g_dev.lutq, pts.data(), 2048 * sizeof(float2), hipMemcpyHostToDevice));+ g_dev.ready = true;+}++template <typename T>+T* dalloc(size_t n) {+ T* p = nullptr;+ HCHECK(hipMalloc(&p, n * sizeof(T)));+ return p;+}++} // namespace++extern "C" {+#if defined(_WIN32)+#define TQH_EXPORT __declspec(dllexport)+#else+#define TQH_EXPORT __attribute__((visibility("default")))+#endif++// GPU encode of nblocks blocks of TQK4X / 5X / 6 / 7. Same arguments as tqe_encode (iters, ncand, scale_tries; exact+// search is CPU only). stats[0] = nonfinite blocks, stats[1] = zero blocks; timing[0] = kernel seconds, [1] = passes.+TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters, int ncand,+ int scale_tries, int device, double* err_out, int64_t* stats, double* timing,+ int fixed_scale_bits) {+ const int K = tqenc::type_kbits(type);+ if (K < 4 || K > 7) return -1.0;+ ensure_dev(device);+ const int bb = 2 + 4 * K, G = 1 << (15 - K);+ const int nc = std::max(1, std::min(ncand, 8));+ static const int rots[8] = {16, 8, 24, 12, 20, 4, 28, 0};+ static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};+ const int tries = fixed_scale_bits ? 1 : std::max(1, std::min(scale_tries, 7));+ const int64_t CH = 1 << 16;+ const int grid_cap = g_dev.nsm * 4;+ uint16_t* bp = dalloc<uint16_t>((size_t) grid_cap * 32 * G);+ double tot = 0;+ int64_t nf = 0, zb = 0;+ double ksec = 0;+ int64_t npass = 0;+ for (int64_t c0 = 0; c0 < nblocks; c0 += CH) {+ const int64_t nb = std::min(CH, nblocks - c0);+ float* dw = dalloc<float>((size_t) nb * 128);+ float* dh = h ? dalloc<float>((size_t) nb * 128) : nullptr;+ HCHECK(hipMemcpy(dw, w + c0 * 128, (size_t) nb * 128 * 4, hipMemcpyHostToDevice));+ if (h) HCHECK(hipMemcpy(dh, h + c0 * 128, (size_t) nb * 128 * 4, hipMemcpyHostToDevice));+ uint16_t* dcur = dalloc<uint16_t>(nb);+ uint8_t* active = dalloc<uint8_t>(nb);+ uint8_t* status = dalloc<uint8_t>(nb);+ double* sumhw2 = dalloc<double>(nb);+ uint16_t* best_d = dalloc<uint16_t>(nb);+ double* best_err = dalloc<double>(nb);+ uint16_t* best_st = dalloc<uint16_t>((size_t) nb * 32);+ uint16_t* st_c = dalloc<uint16_t>((size_t) nc * nb * 32);+ float* cost_c = dalloc<float>((size_t) nc * nb);+ int* c_all = dalloc<int>((size_t) nc * nb);+ uint8_t* d_dst = dalloc<uint8_t>((size_t) nb * bb);+ double* d_err = dalloc<double>(nb);+ uint8_t* have = nullptr;+ const int tb = 128, gb = (int) ((nb + tb - 1) / tb);+ hipLaunchKernelGGL(init_best_kernel, dim3(gb), dim3(tb), 0, 0, nb, best_err, have);+ const auto t0 = std::chrono::steady_clock::now();+ for (int tr = 0; tr < tries; ++tr) {+ hipLaunchKernelGGL(init_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, dcur, active, status, sumhw2, mult[tr],+ tr == 0 ? 1 : 0, (uint16_t) fixed_scale_bits);+ for (int it = 0; it <= (fixed_scale_bits ? 0 : iters); ++it) {+ Pass P{};+ P.w = dw; P.h = dh; P.dcur = dcur; P.active = active; P.nb = nb; P.lutq = g_dev.lutq; P.bp_scratch = bp;+ const int grid = (int) std::min<int64_t>(nb, grid_cap);+ for (int r = 0; r < nc; ++r) {+ P.rot = rots[r]; P.cin = nullptr; P.c_out = c_all + (int64_t) r * nb;+ launch_pass_k(K, P, grid);+ }+ hipLaunchKernelGGL(dedupe_kernel, dim3(gb), dim3(tb), 0, 0, nb, nc, c_all);+ HCHECK(hipMemset(cost_c, 0x7f, (size_t) nc * nb * sizeof(float))); // skipped candidates cost "infinity"+ for (int r = 0; r < nc; ++r) {+ P.rot = 0; P.cin = c_all + (int64_t) r * nb; P.c_out = nullptr;+ P.st_out = st_c + (int64_t) r * nb * 32; P.cost_out = cost_c + (int64_t) r * nb;+ launch_pass_k(K, P, grid);+ }+ hipLaunchKernelGGL(refine_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, nc, st_c, cost_c, dcur, active,+ best_d, best_err, best_st, g_dev.lutq, fixed_scale_bits);+ npass += 2 * nc;+ if (std::getenv("TQH_DEBUG")) {+ HCHECK(hipDeviceSynchronize());+ int hc[4] = {}; float hcost[4] = {}; double he[4] = {}; uint16_t hd[4] = {}; uint8_t ha[4] = {};+ HCHECK(hipMemcpy(hc, c_all, 16, hipMemcpyDeviceToHost));+ HCHECK(hipMemcpy(hcost, cost_c, 16, hipMemcpyDeviceToHost));+ HCHECK(hipMemcpy(he, best_err, 32, hipMemcpyDeviceToHost));+ HCHECK(hipMemcpy(hd, dcur, 8, hipMemcpyDeviceToHost));+ HCHECK(hipMemcpy(ha, active, 4, hipMemcpyDeviceToHost));+ for (int i = 0; i < 4; ++i)+ std::fprintf(stderr, "[dbg try %d it %d] block %d: c %d cost %g best_err %g dcur %g active %d\n", tr, it, i, hc[i], hcost[i], he[i],+ tqenc::half_to_float(hd[i]), ha[i]);+ }+ }+ }+ hipLaunchKernelGGL(pack_kernel, dim3(gb), dim3(tb), 0, 0, K, nb, status, sumhw2, best_d, best_err, best_st, d_dst, d_err);+ HCHECK(hipDeviceSynchronize());+ ksec += std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();+ HCHECK(hipMemcpy(dst + (size_t) c0 * bb, d_dst, (size_t) nb * bb, hipMemcpyDeviceToHost));+ std::vector<double> herr(nb);+ std::vector<uint8_t> hst(nb);+ HCHECK(hipMemcpy(herr.data(), d_err, nb * 8, hipMemcpyDeviceToHost));+ HCHECK(hipMemcpy(hst.data(), status, nb, hipMemcpyDeviceToHost));+ for (int64_t i = 0; i < nb; ++i) {+ if (hst[i] == 2) ++nf;+ else tot += herr[i];+ if (hst[i] == 1) ++zb;+ if (err_out) err_out[c0 + i] = herr[i];+ }+ for (void* p : {(void*) dw, (void*) dh, (void*) dcur, (void*) active, (void*) status, (void*) sumhw2, (void*) best_d,+ (void*) best_err, (void*) best_st, (void*) st_c, (void*) cost_c, (void*) c_all, (void*) d_dst,+ (void*) d_err})+ if (p) HCHECK(hipFree(p));+ }+ HCHECK(hipFree(bp));+ if (stats) { stats[0] = nf; stats[1] = zb; }+ if (timing) { timing[0] = ksec; timing[1] = (double) npass; }+ return tot;+}+}diff --git a/tools/trellis/tqenc_hip_test.cpp b/tools/trellis/tqenc_hip_test.cppnew file mode 100644index 000000000..c416fc8d4--- /dev/null+++ b/tools/trellis/tqenc_hip_test.cpp@@ -0,0 +1,117 @@+// tools/trellis/tqenc_hip_test.cpp - GPU encoder vs the CPU encoder: correctness (GPU bytes decode, by the CPU decoder,+// to exactly the weights whose error the GPU reported), quality (SQNR, planted recovery) and speed (weights/s).+// Run under the box locks: flock -s /data/gpu.lock flock -x ~/locks/gpu3.lock env HIP_VISIBLE_DEVICES=3 ./tqenc_hip_test+#include "tqenc.hpp"++#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <vector>++extern "C" double tqh_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters, int ncand,+ int scale_tries, int device, double* err_out, int64_t* stats, double* timing, int fixed_scale_bits);++using namespace tqenc;+static int g_fail = 0;+#define CHECK(c, ...) do { if (!(c)) { ++g_fail; std::printf(" FAIL line %d: %s ", __LINE__, #c); std::printf(__VA_ARGS__); std::printf("\n"); } } while (0)+static double now() { return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count(); }+static const char* tname(int t) { return t == TQK6 ? "TQK6" : t == TQK7 ? "TQK7" : t == TQK5X ? "TQK5X" : "TQK4X"; }++int main(int argc, char** argv) {+ const long big = argc > 1 ? std::atol(argv[1]) : 131072; // blocks for the speed runs+ std::mt19937_64 rng(77);+ std::normal_distribution<float> nd(0.f, 1.f);+ const int types[4] = {TQK4X, TQK5X, TQK6, TQK7};+ for (int type : types) {+ const int bb = block_bytes(type);+ std::printf("== %s\n", tname(type));+ // 1. correctness + CPU comparison on 2048 Gaussian blocks, without and with importance weights+ {+ const int nb = 2048;+ std::vector<float> w((size_t) nb * 128), h((size_t) nb * 128);+ for (auto& v : w) v = 0.02f * nd(rng);+ for (auto& v : h) v = 0.2f + std::fabs(nd(rng));+ for (int wi = 0; wi < 2; ++wi) {+ const float* hp = wi ? h.data() : nullptr;+ std::vector<uint8_t> g((size_t) nb * bb), c((size_t) nb * bb);+ std::vector<double> ge(nb), ce(nb);+ int64_t st[2];+ double tm[2];+ const double gt = tqh_encode(type, w.data(), hp, nb, g.data(), 1, 1, 1, 0, ge.data(), st, tm, 0);+ Params p;+ p.iters = 1;+ int64_t nf, zb;+ const double ct = encode_blocks(type, w.data(), hp, nb, c.data(), p, 8, ce.data(), &nf, &zb);+ std::vector<float> dec((size_t) nb * 128);+ decode_blocks(type, g.data(), nb, dec.data());+ double worst = 0;+ for (int b = 0; b < nb; ++b) {+ double e2 = 0;+ for (int i = 0; i < 128; ++i) {+ const double d = (double) w[b * 128 + i] - dec[b * 128 + i];+ e2 += (hp ? hp[b * 128 + i] : 1.0) * d * d;+ }+ worst = std::max(worst, std::fabs(e2 - ge[b]) / std::max(1e-30, e2 + ge[b]));+ }+ CHECK(worst < 1e-5, "GPU reported error differs from the decoded error: %.2e", worst);+ int same = 0;+ for (int b = 0; b < nb; ++b) same += std::memcmp(&g[(size_t) b * bb], &c[(size_t) b * bb], bb) == 0;+ std::printf(" importance %-3s GPU total error %.6e, CPU %.6e (GPU/CPU %.5f); identical blocks %d/%d; decoded-vs-reported worst %.1e\n",+ wi ? "yes" : "no", gt, ct, gt / ct, same, nb, worst);+ CHECK(gt <= ct * 1.002, "GPU error is more than 0.2%% above the CPU's");+ }+ }+ // 2. planted solutions: weights decoded from random streams must be recovered (the scale is found by the refinement)+ {+ const int nb = 2048;+ std::vector<uint8_t> blk((size_t) nb * bb);+ for (int b = 0; b < nb; ++b) {+ const float d = std::ldexp(0.5f + (float) (rng() % 1000) / 1000.f, -8 + (int) (rng() % 6));+ const uint16_t dh = float_to_half(d);+ blk[(size_t) b * bb] = dh & 255;+ blk[(size_t) b * bb + 1] = dh >> 8;+ for (int i = 2; i < bb; ++i) blk[(size_t) b * bb + i] = (uint8_t) rng();+ }+ std::vector<float> w((size_t) nb * 128);+ decode_blocks(type, blk.data(), nb, w.data());+ std::vector<uint8_t> g((size_t) nb * bb);+ std::vector<double> ge(nb);+ tqh_encode(type, w.data(), nullptr, nb, g.data(), 3, 1, 1, 0, ge.data(), nullptr, nullptr, 0);+ int ok = 0;+ for (int b = 0; b < nb; ++b) ok += ge[b] == 0.0;+ std::printf(" planted: %d/%d blocks recovered exactly (scale unknown)\n", ok, nb);+ CHECK(ok > nb * 0.9, "planted recovery low");+ }+ // 3. speed on big Gaussian batches+ {+ std::vector<float> w((size_t) big * 128);+ for (auto& v : w) v = nd(rng);+ std::vector<uint8_t> g((size_t) big * bb);+ double en = 0;+ for (float v : w) en += (double) v * v;+ struct Cfg { const char* n; int iters, ncand, tries; } cfgs[3] = {{"fast (1 pass + 1)", 0, 1, 1}, {"default (iters 1)", 1, 1, 1}, {"rot3 iters3", 3, 3, 1}};+ for (auto& c : cfgs) {+ double tm[2];+ int64_t stt[2];+ const double t0 = now();+ const double e = tqh_encode(type, w.data(), nullptr, big, g.data(), c.iters, c.ncand, c.tries, 0, nullptr, stt, tm, 0);+ const double s = now() - t0;+ {+ std::vector<float> dec((size_t) big * 128);+ decode_blocks(type, g.data(), big, dec.data());+ double e2 = 0;+ for (size_t i = 0; i < dec.size(); ++i) e2 += ((double) w[i] - dec[i]) * ((double) w[i] - dec[i]);+ std::printf(" [check: energy %.6e, reported error %.6e, decoded error %.6e -> SQNR %.3f dB]\n", en, e, e2, 10 * std::log10(en / e2));+ }+ std::printf(" %-18s SQNR %.3f dB %.2f M weights/s (kernels only %.2f M/s, %.0f passes) total %.2fs [zero blocks %lld, nonfinite %lld]\n", c.n,+ 10 * std::log10(en / e), (double) big * 128 / s / 1e6, (double) big * 128 / tm[0] / 1e6, tm[1], s,+ (long long) stt[1], (long long) stt[0]);+ }+ }+ }+ std::printf("\n%s (%d failures)\n", g_fail ? "FAILED" : "OK", g_fail);+ return g_fail ? 1 : 0;+}