Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
25.3 KB
Raw Blame
// tools/trellis/tqenc_hip.cpp - the trellis encoder (tqenc.hpp) on a GPU (HIP, gfx1201 / any wave32 RDNA): TQK4X / TQK5X /
// TQK6 / TQK7 (TQ2_T stays CPU-only: its predecessor groups are strided across threads).
//
// One workgroup (512 threads) runs one Viterbi pass over one 128-weight block: thread t keeps the 64 path costs of the states
// 64 t .. 64 t + 63 in registers.  A step is (a) every thread takes the min/argmin of its predecessor groups (a group
// is 2^K contiguous states, so never crosses a thread for K <= 7) and writes it to LDS plus the argmin to a global
// back-pointer row, (b) after a barrier every thread sets its new costs D[s] = m[s & (G-1)] + c01[p0(s)] + c23[p1(s)]:
// the cost of a state is the sum of the costs of its two hashed codebook points (2048 each), tabulated per step in LDS.
// The same search as the CPU encoder (rotated free pass for the tail-biting start group, constrained pass, fp16 scale
// refinement by least squares), batched over all blocks: every stage is one kernel over the whole batch.
//
// Build: hipcc -O3 --offload-arch=gfx1201 -mavx2 -mfma -mf16c tools/trellis/tqenc_hip.cpp tools/trellis/tqenc.cpp ...
#include "tqenc.hpp"

#include <hip/hip_runtime.h>
#include <hip/hip_fp16.h>

#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>

#define HCHECK(x)                                                                                  \
    do {                                                                                           \
        hipError_t e_ = (x);                                                                       \
        if (e_ != hipSuccess) {                                                                    \
            std::fprintf(stderr, "HIP error %s at %s:%d: %s\n", #x, __FILE__, __LINE__, hipGetErrorString(e_)); \
            std::abort();                                                                          \
        }                                                                                          \
    } while (0)

namespace {

constexpr int kSL = 6;        // log2 of the states per thread (64: 512 threads per workgroup)
constexpr int NT = 32768 >> kSL;
constexpr float kInf = 1e30f;

__device__ __forceinline__ float h2f(uint16_t h) { return __half2float(__ushort_as_half(h)); }
__device__ __forceinline__ uint16_t f2h(float f) { return __half_as_ushort(__float2half_rn(f)); }

// weight index of component j at processing position p (TQK)
__device__ __forceinline__ int widx_d(int p, int j) { return 4 * (31 - p) + j; }

// the computed codebook of TQK5X / TQK4X (src/kernels/cpu/tq_computed.hpp): four fp16 from two LCG words, no table
__device__ __forceinline__ void comp_vals(uint32_t s, float* w) {
    const uint32_t r0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
    const uint32_t r1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
    w[0] = h2f((uint16_t) (r0 & 0xffffu));
    w[1] = h2f((uint16_t) (r0 >> 16));
    w[2] = h2f((uint16_t) (r1 & 0xffffu));
    w[3] = h2f((uint16_t) (r1 >> 16));
}

__device__ __forceinline__ float comp_cost(uint32_t s, const float* t, const float* h) {
    float w[4];
    comp_vals(s, w);
    const float e0 = t[0] - w[0], e1 = t[1] - w[1], e2 = t[2] - w[2], e3 = t[3] - w[3];
    return h[0] * e0 * e0 + h[1] * e1 * e1 + h[2] * e2 * e2 + h[3] * e3 * e3;
}

struct Pass {
    const float* w;
    const float* h;            // null: ones
    const uint16_t* dcur;      // fp16 scale per block
    const uint8_t* active;
    int64_t nb;
    int rot;
    const int* cin;            // null: free pass, else the tail-biting group per block (<0: skip)
    const float2* lutq;        // 2048 points
    uint16_t* bp_scratch;
    uint16_t* st_out;          // constrained: [nb][32] by position
    float* cost_out;           // constrained: [nb]
    int* c_out;                // free: junction group [nb]
};

template <int K, int SL, bool COMP>
__global__ void __launch_bounds__(32768 >> SL) pass_kernel(Pass P) {
    constexpr int SPT = 1 << SL;          // states per thread
    constexpr int NT = 32768 >> SL;       // threads per workgroup
    constexpr int G = 1 << (15 - K);
    constexpr int SPAN = 1 << K;
    constexpr int GPT = SPT >= SPAN ? SPT / SPAN : 1;   // groups per thread (SPT >= SPAN)
    constexpr int PC = SPT >= SPAN ? 1 : SPAN / SPT;    // threads sharing one group (SPAN > SPT)
    constexpr int NP = 2048 / NT;                       // codebook points per thread
    __shared__ float tab[2][2][2048];
    __shared__ float mbuf[2][G];
    __shared__ float Tt[32][4], Hh[32][4];
    __shared__ float redv[NT / 32];
    __shared__ int redi[NT / 32];

    const int tid = threadIdx.x;
    uint16_t* bp = P.bp_scratch + (size_t) blockIdx.x * 32 * G;
    float2 L[NP];
#pragma unroll
    for (int k = 0; k < NP; ++k) L[k] = P.lutq[tid + NT * k];
    const uint32_t kC = 0x9e3779b1u;
    const uint32_t x0 = (uint32_t) (SPT * tid) * kC;

    for (int64_t b = blockIdx.x; b < P.nb; b += gridDim.x) {
        if (!P.active[b]) continue;
        int fc = -1;
        if (P.cin) {
            fc = P.cin[b];
            if (fc < 0) continue;
        }
        if (tid < 128) {
            const int p = tid >> 2, j = tid & 3;
            const int wi = widx_d(p, j);
            const float dd = h2f(P.dcur[b]);
            Tt[p][j] = P.w[b * 128 + wi] / dd;
            Hh[p][j] = P.h ? P.h[b * 128 + wi] : 1.0f;
        }
        __syncthreads();

        float D[SPT];
        // tables of step q into parity q & 1
        auto build = [&](int q) {
            if constexpr (COMP) return;
            const int p = (P.rot + q) & 31;
            const float t0 = Tt[p][0], t1 = Tt[p][1], t2 = Tt[p][2], t3 = Tt[p][3];
            const float h0 = Hh[p][0], h1 = Hh[p][1], h2 = Hh[p][2], h3 = Hh[p][3];
#pragma unroll
            for (int k = 0; k < NP; ++k) {
                const float a0 = t0 - L[k].x, a1 = t1 - L[k].y, a2 = t2 - L[k].x, a3 = t3 - L[k].y;
                tab[q & 1][0][tid + NT * k] = h0 * a0 * a0 + h1 * a1 * a1;
                tab[q & 1][1][tid + NT * k] = h2 * a2 * a2 + h3 * a3 * a3;
            }
        };
        build(0);
        __syncthreads();
        {
            float tc[4], hc[4];
            if constexpr (COMP) {
                const int p0 = P.rot & 31;
                for (int i = 0; i < 4; ++i) { tc[i] = Tt[p0][i]; hc[i] = Hh[p0][i]; }
            }
#pragma unroll
            for (int j = 0; j < SPT; ++j) {
                const uint32_t x = x0 + (uint32_t) j * kC;
                float v;
                if constexpr (COMP) v = comp_cost((uint32_t) (SPT * tid + j), tc, hc);
                else v = tab[0][0][x >> 21] + tab[0][1][(x >> 10) & 2047u];
                if (fc >= 0 && (((SPT * tid + j) & (G - 1)) != fc)) v = kInf;
                D[j] = v;
            }
        }
        for (int q = 1; q < 32; ++q) {
            const int par = q & 1;
            build(q);
            // group minima of the previous column
            uint16_t* bpq = bp + (size_t) q * G;
            if constexpr (SPT >= SPAN) {
#pragma unroll
                for (int g = 0; g < GPT; ++g) {
                    float mv = D[g * SPAN];
                    int ma = 0;
#pragma unroll
                    for (int e = 1; e < SPAN; ++e) {
                        const float v = D[g * SPAN + e];
                        if (v < mv) { mv = v; ma = e; }
                    }
                    const int a = ((SPT * tid) >> K) + g;
                    mbuf[par][a] = mv;
                    bpq[a] = (uint16_t) (SPT * tid + g * SPAN + ma);
                }
            } else {
                // a group spans PC consecutive threads (the same wave): min/argmin of the thread, then across the PC
                float mv = D[0];
                int ma = 0;
#pragma unroll
                for (int e = 1; e < SPT; ++e) {
                    const float v = D[e];
                    if (v < mv) { mv = v; ma = e; }
                }
                int st = SPT * tid + ma;
#pragma unroll
                for (int off = 1; off < PC; off <<= 1) {
                    const float ov = __shfl_xor(mv, off);
                    const int os = __shfl_xor(st, off);
                    if (ov < mv || (ov == mv && os < st)) { mv = ov; st = os; }
                }
                if ((tid & (PC - 1)) == 0) {
                    const int a = (SPT * tid) >> K;
                    mbuf[par][a] = mv;
                    bpq[a] = (uint16_t) st;
                }
            }
            __syncthreads();
            const float4* m4 = reinterpret_cast<const float4*>(&mbuf[par][(SPT * tid) & (G - 1)]);
            // keep the compiler from caching the 2 * SPT hashed table addresses across the steps (they are loop invariant,
            // and 128 live registers spill): an empty asm makes x0 "new" every step
            uint32_t x0s = x0;
            asm volatile("" : "+v"(x0s));
            float tc[4], hc[4];
            if constexpr (COMP) {
                const int pq = (P.rot + q) & 31;
                for (int i = 0; i < 4; ++i) { tc[i] = Tt[pq][i]; hc[i] = Hh[pq][i]; }
            }
#pragma unroll
            for (int j4 = 0; j4 < SPT / 4; ++j4) {
                const float4 mm = m4[j4];
                const uint32_t xa = x0s + (uint32_t) (4 * j4) * kC;
                const uint32_t xb = xa + kC, xc = xb + kC, xd = xc + kC;
                if constexpr (COMP) {
                    const uint32_t sa = (uint32_t) (SPT * tid + 4 * j4);
                    D[4 * j4 + 0] = mm.x + comp_cost(sa, tc, hc);
                    D[4 * j4 + 1] = mm.y + comp_cost(sa + 1, tc, hc);
                    D[4 * j4 + 2] = mm.z + comp_cost(sa + 2, tc, hc);
                    D[4 * j4 + 3] = mm.w + comp_cost(sa + 3, tc, hc);
                } else {
                    D[4 * j4 + 0] = mm.x + tab[par][0][xa >> 21] + tab[par][1][(xa >> 10) & 2047u];
                    D[4 * j4 + 1] = mm.y + tab[par][0][xb >> 21] + tab[par][1][(xb >> 10) & 2047u];
                    D[4 * j4 + 2] = mm.z + tab[par][0][xc >> 21] + tab[par][1][(xc >> 10) & 2047u];
                    D[4 * j4 + 3] = mm.w + tab[par][0][xd >> 21] + tab[par][1][(xd >> 10) & 2047u];
                }
                if ((j4 & 3) == 3) __builtin_amdgcn_sched_barrier(0);   // 16 states at a time: do not batch all the gathers
            }
        }
        // final state
        const int lo = fc < 0 ? 0 : (fc << K), hi = fc < 0 ? 32768 : lo + SPAN;
        float bv = 3e38f;
        int bi = -1;
#pragma unroll
        for (int j = 0; j < SPT; ++j) {
            const int s = SPT * tid + j;
            if (s >= lo && s < hi && D[j] < bv) { bv = D[j]; bi = s; }
        }
#pragma unroll
        for (int off = 16; off >= 1; off >>= 1) {
            const float ov = __shfl_xor(bv, off);
            const int oi = __shfl_xor(bi, off);
            if (ov < bv || (ov == bv && oi >= 0 && (bi < 0 || oi < bi))) { bv = ov; bi = oi; }
        }
        if ((tid & 31) == 0) { redv[tid >> 5] = bv; redi[tid >> 5] = bi; }
        __syncthreads();
        if (tid == 0) {
            float v = redv[0];
            int i = redi[0];
            for (int w2 = 1; w2 < NT / 32; ++w2)
                if (redv[w2] < v || (redv[w2] == v && redi[w2] >= 0 && (i < 0 || redi[w2] < i))) { v = redv[w2]; i = redi[w2]; }
            // traceback
            if (i < 0 || v >= kInf * 0.5f) {
                if (P.cin) P.cost_out[b] = kInf; else P.c_out[b] = 0;
            } else if (!P.cin) {
                const int qj = (31 - P.rot) & 31;   // the step that processes position 31
                int s = i;
                for (int q = 31; q > qj; --q) s = bp[(size_t) q * G + (s & (G - 1))];
                P.c_out[b] = s >> K;
            } else {
                int s = i;
                P.st_out[b * 32 + ((P.rot + 31) & 31)] = (uint16_t) s;
                for (int q = 31; q >= 1; --q) {
                    s = bp[(size_t) q * G + (s & (G - 1))];
                    P.st_out[b * 32 + ((P.rot + q - 1) & 31)] = (uint16_t) s;
                }
                P.cost_out[b] = v;
            }
        }
        __syncthreads();
    }
}

// ------------------------------------------------------------------------------------------------ per-block stages

__device__ __forceinline__ void state_cb(const float2* lutq, int comp, uint32_t s, float* c) {
    if (comp) { comp_vals(s, c); return; }
    const uint32_t x = s * 0x9e3779b1u;
    const float2 a = lutq[x >> 21], b = lutq[(x >> 10) & 2047u];
    c[0] = a.x; c[1] = a.y; c[2] = b.x; c[3] = b.y;
}

__global__ void init_kernel(const float* w, const float* h, int64_t nb, uint16_t* dcur, uint8_t* active, uint8_t* status,
                            double* sumhw2, double mult, int first, uint16_t fixed_scale, double cb_rms) {
    const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
    if (b >= nb) return;
    double s2 = 0, sh2 = 0, amax = 0;
    bool fin = true;
    for (int i = 0; i < 128; ++i) {
        const float x = w[b * 128 + i];
        const float hi = h ? h[b * 128 + i] : 1.0f;
        if (!isfinite(x) || !isfinite(hi)) fin = false;
        s2 += (double) x * x;
        sh2 += (double) hi * x * x;
        amax = fmax(amax, fabs((double) x));
    }
    if (first) {
        status[b] = !fin ? 2 : (amax == 0.0 ? 1 : 0);
        sumhw2[b] = sh2;
    }
    if (status[b] != 0) { active[b] = 0; return; }
    const double rms = sqrt(s2 / 128.0);
    double dd = rms / cb_rms * mult;
    uint16_t dh = f2h((float) fmin(dd, 65504.0));
    if (dh == 0) dh = 1;
    if (fixed_scale) dh = fixed_scale;
    dcur[b] = dh;
    active[b] = 1;
}

// error of (states by position st, fp16 scale dh)
__device__ double fit_d(const float* w, const float* h, int64_t b, const uint16_t* st, uint16_t dh, const float2* lutq, int comp) {
    const float d = h2f(dh);
    double e = 0;
    for (int p = 0; p < 32; ++p) {
        float c[4];
        state_cb(lutq, comp, st[p], c);
        for (int j = 0; j < 4; ++j) {
            const int wi = widx_d(p, j);
            const float rec = d * c[j];
            const double df = (double) w[b * 128 + wi] - rec;
            e += (h ? (double) h[b * 128 + wi] : 1.0) * df * df;
        }
    }
    return e;
}

// pick the best candidate, then the scale refinement step of encode_block
__global__ void refine_kernel(const float* w, const float* h, int64_t nb, int nc, const uint16_t* st_c, const float* cost_c,
                              uint16_t* dcur, uint8_t* active, uint16_t* best_d, double* best_err, uint16_t* best_st,
                              const float2* lutq, int fixed_scale, int comp) {
    const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
    if (b >= nb || !active[b]) return;
    int bc = -1;
    float bcost = kInf * 0.5f;
    for (int r = 0; r < nc; ++r) {
        const float c = cost_c[(int64_t) r * nb + b];
        if (c < bcost) { bcost = c; bc = r; }
    }
    if (bc < 0) { active[b] = 0; return; }
    uint16_t st[32];
    for (int p = 0; p < 32; ++p) st[p] = st_c[((int64_t) bc * nb + b) * 32 + p];
    const uint16_t dh = dcur[b];
    const double e = fit_d(w, h, b, st, dh, lutq, comp);
    if (e < best_err[b]) {
        best_err[b] = e;
        best_d[b] = dh;
        for (int p = 0; p < 32; ++p) best_st[b * 32 + p] = st[p];
    }
    if (e == 0.0 || fixed_scale) { active[b] = 0; return; }
    double num = 0, den = 0;
    for (int p = 0; p < 32; ++p) {
        float c[4];
        state_cb(lutq, comp, st[p], c);
        for (int j = 0; j < 4; ++j) {
            const int wi = widx_d(p, j);
            const double hi = h ? (double) h[b * 128 + wi] : 1.0, cc = c[j];
            num += hi * w[b * 128 + wi] * cc;
            den += hi * cc * cc;
        }
    }
    if (den <= 0 || num <= 0) { active[b] = 0; return; }
    const uint16_t nh = f2h((float) fmin(num / den, 65504.0));
    if (nh == 0 || nh == dh) { active[b] = 0; return; }
    const double en = fit_d(w, h, b, st, nh, lutq, comp);
    if (en < best_err[b]) {
        best_err[b] = en;
        best_d[b] = nh;
        for (int p = 0; p < 32; ++p) best_st[b * 32 + p] = st[p];
    }
    if (en >= e) { active[b] = 0; return; }
    dcur[b] = nh;
}

__global__ void dedupe_kernel(int64_t nb, int nc, int* c_all) {
    const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
    if (b >= nb) return;
    for (int r = 1; r < nc; ++r) {
        const int c = c_all[(int64_t) r * nb + b];
        for (int q = 0; q < r; ++q)
            if (c_all[(int64_t) q * nb + b] == c) { c_all[(int64_t) r * nb + b] = -1; break; }
    }
}

__global__ void pack_kernel(int K, int64_t nb, const uint8_t* status, const double* sumhw2, const uint16_t* best_d,
                            const double* best_err, const uint16_t* best_st, uint8_t* dst, double* err_out) {
    const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
    if (b >= nb) return;
    const int bb = 2 + 4 * K;
    uint8_t* o = dst + b * bb;
    for (int i = 0; i < bb; ++i) o[i] = 0;
    if (status[b] == 2) { err_out[b] = NAN; return; }
    if (status[b] == 1 || best_err[b] >= sumhw2[b]) { err_out[b] = status[b] == 1 ? 0.0 : sumhw2[b]; return; }
    o[0] = (uint8_t) (best_d[b] & 255);
    o[1] = (uint8_t) (best_d[b] >> 8);
    uint8_t* qs = o + 2;
    for (int u = 0; u < 32; ++u) {
        const uint32_t x = best_st[b * 32 + u] & ((1u << K) - 1u);
        for (int bit = 0; bit < K; ++bit)
            if ((x >> bit) & 1u) qs[(u * K + bit) >> 3] |= (uint8_t) (1u << ((u * K + bit) & 7));
    }
    err_out[b] = best_err[b];
}

__global__ void init_best_kernel(int64_t nb, double* best_err, uint8_t* have) {
    const int64_t b = blockIdx.x * (int64_t) blockDim.x + threadIdx.x;
    if (b >= nb) return;
    best_err[b] = 1e300;
}

template <int K, bool COMP>
void launch_pass(const Pass& P, int grid) {
    hipLaunchKernelGGL((pass_kernel<K, kSL, COMP>), dim3(grid), dim3(NT), 0, 0, P);
}

void launch_pass_k(int K, bool comp, const Pass& P, int grid) {
    if (comp) {   // TQK5X / TQK4X
        if (K == 4) launch_pass<4, true>(P, grid);
        else launch_pass<5, true>(P, grid);
        return;
    }
    switch (K) {
        case 4: launch_pass<4, false>(P, grid); break;
        case 5: launch_pass<5, false>(P, grid); break;
        case 6: launch_pass<6, false>(P, grid); break;
        default: launch_pass<7, false>(P, grid); break;
    }
}

struct Dev {
    int device = 0;
    int nsm = 0;
    float2* lutq = nullptr;
    bool ready = false;
} g_dev;

void ensure_dev(int device) {
    if (g_dev.ready && g_dev.device == device) return;
    HCHECK(hipSetDevice(device));
    hipDeviceProp_t pr;
    HCHECK(hipGetDeviceProperties(&pr, device));
    g_dev.device = device;
    g_dev.nsm = pr.multiProcessorCount;
    // the 2048 codebook points as the decoder sees them: round(fp16 * 8192) / 8192 (tq_cpu.cpp fill_table)
    std::vector<float2> pts(2048);
    const float* pt = tqenc::point_table();
    for (int i = 0; i < 2048; ++i) pts[i] = make_float2(pt[2 * i], pt[2 * i + 1]);
    if (g_dev.lutq) HCHECK(hipFree(g_dev.lutq));
    HCHECK(hipMalloc(&g_dev.lutq, 2048 * sizeof(float2)));
    HCHECK(hipMemcpy(g_dev.lutq, pts.data(), 2048 * sizeof(float2), hipMemcpyHostToDevice));
    g_dev.ready = true;
}

template <typename T>
T* dalloc(size_t n) {
    T* p = nullptr;
    HCHECK(hipMalloc(&p, n * sizeof(T)));
    return p;
}

}  // namespace

extern "C" {
#if defined(_WIN32)
#define TQH_EXPORT __declspec(dllexport)
#else
#define TQH_EXPORT __attribute__((visibility("default")))
#endif

// GPU encode of nblocks blocks of TQK4X / 5X / 6 / 7.  Same arguments as tqe_encode (iters, ncand, scale_tries; exact
// search is CPU only).  stats[0] = nonfinite blocks, stats[1] = zero blocks; timing[0] = kernel seconds, [1] = passes.
TQH_EXPORT double tqh_encode(int type, const float* w, const float* h, int64_t nblocks, uint8_t* dst, int iters, int ncand,
                             int scale_tries, int device, double* err_out, int64_t* stats, double* timing,
                             int fixed_scale_bits) {
    const int K = tqenc::type_kbits(type);
    if (K < 4 || K > 7) return -1.0;
    const bool comp = tqenc::type_computed(type);
    const double cb_rms = tqenc::codebook_rms(type);
    ensure_dev(device);
    const int bb = 2 + 4 * K, G = 1 << (15 - K);
    const int nc = std::max(1, std::min(ncand, 8));
    static const int rots[8] = {16, 8, 24, 12, 20, 4, 28, 0};
    static const double mult[7] = {1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3};
    const int tries = fixed_scale_bits ? 1 : std::max(1, std::min(scale_tries, 7));
    const int64_t CH = 1 << 16;
    const int grid_cap = g_dev.nsm * 4;
    uint16_t* bp = dalloc<uint16_t>((size_t) grid_cap * 32 * G);
    double tot = 0;
    int64_t nf = 0, zb = 0;
    double ksec = 0;
    int64_t npass = 0;
    for (int64_t c0 = 0; c0 < nblocks; c0 += CH) {
        const int64_t nb = std::min(CH, nblocks - c0);
        float* dw = dalloc<float>((size_t) nb * 128);
        float* dh = h ? dalloc<float>((size_t) nb * 128) : nullptr;
        HCHECK(hipMemcpy(dw, w + c0 * 128, (size_t) nb * 128 * 4, hipMemcpyHostToDevice));
        if (h) HCHECK(hipMemcpy(dh, h + c0 * 128, (size_t) nb * 128 * 4, hipMemcpyHostToDevice));
        uint16_t* dcur = dalloc<uint16_t>(nb);
        uint8_t* active = dalloc<uint8_t>(nb);
        uint8_t* status = dalloc<uint8_t>(nb);
        double* sumhw2 = dalloc<double>(nb);
        uint16_t* best_d = dalloc<uint16_t>(nb);
        double* best_err = dalloc<double>(nb);
        uint16_t* best_st = dalloc<uint16_t>((size_t) nb * 32);
        uint16_t* st_c = dalloc<uint16_t>((size_t) nc * nb * 32);
        float* cost_c = dalloc<float>((size_t) nc * nb);
        int* c_all = dalloc<int>((size_t) nc * nb);
        uint8_t* d_dst = dalloc<uint8_t>((size_t) nb * bb);
        double* d_err = dalloc<double>(nb);
        uint8_t* have = nullptr;
        const int tb = 128, gb = (int) ((nb + tb - 1) / tb);
        hipLaunchKernelGGL(init_best_kernel, dim3(gb), dim3(tb), 0, 0, nb, best_err, have);
        const auto t0 = std::chrono::steady_clock::now();
        for (int tr = 0; tr < tries; ++tr) {
            hipLaunchKernelGGL(init_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, dcur, active, status, sumhw2, mult[tr],
                               tr == 0 ? 1 : 0, (uint16_t) fixed_scale_bits, cb_rms);
            for (int it = 0; it <= (fixed_scale_bits ? 0 : iters); ++it) {
                Pass P{};
                P.w = dw; P.h = dh; P.dcur = dcur; P.active = active; P.nb = nb; P.lutq = g_dev.lutq; P.bp_scratch = bp;
                const int grid = (int) std::min<int64_t>(nb, grid_cap);
                for (int r = 0; r < nc; ++r) {
                    P.rot = rots[r]; P.cin = nullptr; P.c_out = c_all + (int64_t) r * nb;
                    launch_pass_k(K, comp, P, grid);
                }
                hipLaunchKernelGGL(dedupe_kernel, dim3(gb), dim3(tb), 0, 0, nb, nc, c_all);
                HCHECK(hipMemset(cost_c, 0x7f, (size_t) nc * nb * sizeof(float)));   // skipped candidates cost "infinity"
                for (int r = 0; r < nc; ++r) {
                    P.rot = 0; P.cin = c_all + (int64_t) r * nb; P.c_out = nullptr;
                    P.st_out = st_c + (int64_t) r * nb * 32; P.cost_out = cost_c + (int64_t) r * nb;
                    launch_pass_k(K, comp, P, grid);
                }
                hipLaunchKernelGGL(refine_kernel, dim3(gb), dim3(tb), 0, 0, dw, dh, nb, nc, st_c, cost_c, dcur, active,
                                   best_d, best_err, best_st, g_dev.lutq, fixed_scale_bits, comp ? 1 : 0);
                npass += 2 * nc;
                if (std::getenv("TQH_DEBUG")) {
                    HCHECK(hipDeviceSynchronize());
                    int hc[4] = {}; float hcost[4] = {}; double he[4] = {}; uint16_t hd[4] = {}; uint8_t ha[4] = {};
                    HCHECK(hipMemcpy(hc, c_all, 16, hipMemcpyDeviceToHost));
                    HCHECK(hipMemcpy(hcost, cost_c, 16, hipMemcpyDeviceToHost));
                    HCHECK(hipMemcpy(he, best_err, 32, hipMemcpyDeviceToHost));
                    HCHECK(hipMemcpy(hd, dcur, 8, hipMemcpyDeviceToHost));
                    HCHECK(hipMemcpy(ha, active, 4, hipMemcpyDeviceToHost));
                    for (int i = 0; i < 4; ++i)
                        std::fprintf(stderr, "[dbg try %d it %d] block %d: c %d cost %g best_err %g dcur %g active %d\n", tr, it, i, hc[i], hcost[i], he[i],
                                     tqenc::half_to_float(hd[i]), ha[i]);
                }
            }
        }
        hipLaunchKernelGGL(pack_kernel, dim3(gb), dim3(tb), 0, 0, K, nb, status, sumhw2, best_d, best_err, best_st, d_dst, d_err);
        HCHECK(hipDeviceSynchronize());
        ksec += std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
        HCHECK(hipMemcpy(dst + (size_t) c0 * bb, d_dst, (size_t) nb * bb, hipMemcpyDeviceToHost));
        std::vector<double> herr(nb);
        std::vector<uint8_t> hst(nb);
        HCHECK(hipMemcpy(herr.data(), d_err, nb * 8, hipMemcpyDeviceToHost));
        HCHECK(hipMemcpy(hst.data(), status, nb, hipMemcpyDeviceToHost));
        for (int64_t i = 0; i < nb; ++i) {
            if (hst[i] == 2) ++nf;
            else tot += herr[i];
            if (hst[i] == 1) ++zb;
            if (err_out) err_out[c0 + i] = herr[i];
        }
        for (void* p : {(void*) dw, (void*) dh, (void*) dcur, (void*) active, (void*) status, (void*) sumhw2, (void*) best_d,
                        (void*) best_err, (void*) best_st, (void*) st_c, (void*) cost_c, (void*) c_all, (void*) d_dst,
                        (void*) d_err})
            if (p) HCHECK(hipFree(p));
    }
    HCHECK(hipFree(bp));
    if (stats) { stats[0] = nf; stats[1] = zb; }
    if (timing) { timing[0] = ksec; timing[1] = (double) npass; }
    return tot;
}
}