Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
25.0 KB
Raw Blame
// Strata. The trellis routed experts (TQK6 / TQK7 / TQK5X / TQK4X / TQK5T / TQK4T, ds4_iq.cuh's is_tq) on the device for the
// DeepSeek-V4.1 engine path.  The weights stay as the file has them (a row of k values = k / 128 blocks of {fp16 d; 4 K stream
// bytes}); the blob in VRAM is the file's bytes, so the expert cache, the tensor-parallel cuts and the CPU lane read the same
// layout.  Arithmetic as the CPU lane's (ds4_tq_cpu.cpp): the activations are int8 with an F32 scale per 32 values (the
// engine's q8_quant), the weights exact (fp16 codebook values times the block's fp16 scale); the sums are F32.
//
// gfx12 (RDNA4, R9700): v_wmma_f32_16x16x16_f16.  The weights are the B operand (16 rows of a tile = the 16 columns, a lane
// holds one row's 8 consecutive k), the activations the A operand (16 entries = the 16 rows, split x = hi + lo in two fp16
// so the product keeps ~2^-22), D[entry][weight row] in F32: a lane holds the entries 8 kg .. 8 kg + 7 of its row, so the
// block scale d is the lane's own.  Lane l = r + 16 kg owns row r and the 16 steps 16 kg .. 16 kg + 15 of every block, whose
// 15-bit states are one 128-bit window of the stream (the tile code of tools/dsv41_lab/wmma_emu.hpp, checked there against a
// double reference and on the card by tests/dsv41/tqw_hip_test.cpp).  Elsewhere the same lane mapping in plain FMAs.
#include "ds4_launch.h"   // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_iq.cuh"

#include <cuda_fp16.h>

#include "ggml-common-tq.h"   // tq_lut_f16 (the 2048-point codebook as fp16 bit patterns, __device__)

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>

#if defined(STRATA_USE_HIP) && !defined(DS4_TQ_PORTABLE) && (defined(__GFX12__) || defined(__gfx1200__) || defined(__gfx1201__))
#define DS4_TQ_WMMA 1
#endif

#if !defined(STRATA_USE_HIP) && defined(__CUDACC__)   // (HIP's non-sync shuffle name, for the portable lane on CUDA)
#define __shfl_xor(v, m) __shfl_xor_sync(0xffffffffu, (v), (m))
#endif
namespace strata::ds4::gguf::iq::tq {
namespace {

constexpr int kWaves = 4, kThreads = kWaves * 32;   // a block: 4 waves, a wave = one 16-row tile of each matrix
constexpr int kTileRows = 16;
constexpr int kMaxEnt = 16;   // entries (columns) a pass

#if defined(DS4_TQ_WMMA)
typedef _Float16 half8_t __attribute__((ext_vector_type(8)));
typedef float float8_t __attribute__((ext_vector_type(8)));
#endif

constexpr bool computed_code(int type) { return type == kTQK5X || type == kTQK4X; }

// ---------------------------------------------------------------- the codebook and the stream
// a step's 15-bit state -> the four weights as two words of two fp16 (lo, hi): w0 w1 | w2 w3
template <bool COMPUTED>
__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1) {
    if (COMPUTED) {
        w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
        w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
    } else {
        const uint32_t x = s * 0x9e3779b1u;
        w0 = lut[x >> 21];
        w1 = lut[(x >> 10) & 2047u];
    }
}
__device__ __forceinline__ float half_bits_f(uint32_t b) { return __half2float(__ushort_as_half((unsigned short) b)); }

// the 128-bit window of a lane's 16 steps in one block record: record = {u16 d; stream}, lane half kg takes stream bits
// [(16 - 16 kg) K, (32 - 16 kg) K) plus the 15 - K bits after them (circular), so state(u), u = 0..15, is the 15 bits at
// (15 - u) K of the window.
template <int K>
__device__ __forceinline__ void stream_window(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {
    const uint16_t* st = (const uint16_t*) (rec + 2);
    constexpr int NW = (15 * K + 15 + 15) / 16;   // u16 words that hold 15 K + 15 bits
    uint64_t w[2] = {0ull, 0ull};
    if (kg == 1) {
#pragma unroll
        for (int i = 0; i < NW; ++i) w[i >> 2] |= (uint64_t) st[i] << (16 * (i & 3));
    } else {   // the stream's upper half [16 K, 32 K) = K words; the bits past its end wrap to the stream's start
        const uint16_t* up = st + K;
#pragma unroll
        for (int i = 0; i < K; ++i) w[i >> 2] |= (uint64_t) up[i] << (16 * (i & 3));
        const uint64_t wrap = (uint64_t) st[0] & ((1ull << (15 - K)) - 1ull);   // 15 - K <= 11 bits: inside word 0
        const int p = 16 * K;
        w[p >> 6] |= wrap << (p & 63);   // (p & 63) + 15 - K <= 56: never past the word
    }
    lo = w[0];
    hi = w[1];
}
__device__ __forceinline__ uint32_t state_at(uint64_t lo, uint64_t hi, int pos) {   // 15 bits at bit pos of the 128-bit window
    uint64_t v;
    if (pos + 15 <= 64) v = lo >> pos;
    else if (pos >= 64) v = hi >> (pos - 64);
    else v = (lo >> pos) | (hi << (64 - pos));
    return (uint32_t) v & 0x7fffu;
}

// ---------------------------------------------------------------- activations: where a lane's 8 values of an entry come from
// Each loader: valid(m) says entry m (this pass's column) exists; load8(m, k0, v) the 8 values k0..k0+7 as F32.
struct ActQ8 {   // int8 rows with an F32 scale a 32: entry m = row rows[m] of q (n values a row), scales d (n / 32 a row)
    const int8_t* q;
    const float* d;
    const int* rows;
    int n;
    __device__ __forceinline__ void load8(int m, int k0, float* v) const {
        const size_t r = (size_t) rows[m];
        const int2 w = *(const int2*) (q + r * n + k0);
        const float sc = d[r * (n / 32) + (k0 >> 5)];
        const int8_t* b = (const int8_t*) &w;
#pragma unroll
        for (int j = 0; j < 8; ++j) v[j] = (float) b[j] * sc;
    }
};
struct ActF32 {   // F32 rows: entry m = row rows[m] (ld floats a row)
    const float* x;
    const int* rows;
    size_t ld;
    __device__ __forceinline__ void load8(int m, int k0, float* v) const {
        const float4* p = (const float4*) (x + (size_t) rows[m] * ld + k0);
        const float4 a = p[0], b = p[1];
        v[0] = a.x; v[1] = a.y; v[2] = a.z; v[3] = a.w; v[4] = b.x; v[5] = b.y; v[6] = b.z; v[7] = b.w;
    }
};
struct ActF16 {   // F16 rows: entry m = row rows[m] (ld halves a row)
    const uint16_t* x;
    const int* rows;
    size_t ld;
    __device__ __forceinline__ void load8(int m, int k0, float* v) const {
        const uint4 w = *(const uint4*) (x + (size_t) rows[m] * ld + k0);
        const uint32_t u[4] = {w.x, w.y, w.z, w.w};
#pragma unroll
        for (int j = 0; j < 4; ++j) {
            v[2 * j] = half_bits_f(u[j] & 0xffffu);
            v[2 * j + 1] = half_bits_f(u[j] >> 16);
        }
    }
};

// ---------------------------------------------------------------- the stream window by 32-bit loads
// The same 128-bit window as stream_window, from aligned 32-bit loads (a record is 2 + 4 K bytes: its scale, then the stream, at
// any 2-byte alignment): the array starts at the 4-byte floor of the record, the stream at bit 16 or 32 of it.  Loads past the
// end of the record are skipped (the window bits above what a lane needs are junk anyway).
template <int K>
__device__ __forceinline__ void stream_window_u32(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {
    constexpr int bsz = 2 + 4 * K;
    const uintptr_t ad = (uintptr_t) rec;
    const uint32_t* pa = (const uint32_t*) (ad & ~(uintptr_t) 3);
    const int off = 16 + 8 * (int) (ad & 3);                    // the stream's first bit in the array: 16 or 32
    const int start = off + (kg ? 0 : 16 * K);
    const int w0 = start >> 5, s = start & 31;
    const int nwords = (int) (((ad & 3) + bsz + 3) >> 2);       // words that hold the record
    uint32_t v[5];
#pragma unroll
    for (int i = 0; i < 5; ++i) v[i] = (w0 + i < nwords) ? pa[w0 + i] : 0u;
    uint32_t r[4];
#pragma unroll
    for (int i = 0; i < 4; ++i) r[i] = (uint32_t) ((((uint64_t) v[i + 1] << 32) | v[i]) >> s);
    lo = ((uint64_t) r[1] << 32) | r[0];
    hi = ((uint64_t) r[3] << 32) | r[2];
    if (!kg) {   // the upper half [16 K, 32 K) of the stream (16 K >= 64 bits); the bits past its end wrap to the stream's start
        hi &= (1ull << (16 * K - 64)) - 1ull;
        const uint32_t first = off == 16 ? (pa[0] >> 16) : pa[1];
        const uint64_t wrap = (uint64_t) first & ((1ull << (15 - K)) - 1ull);
        const int p = 16 * K - 64;
        hi |= wrap << p;
    }
}

// ---------------------------------------------------------------- NM 16-row tiles (matrices sharing the activations) over up to 16 entries
// acc_out[mi][i] (i < 8) = D[entry 8 kg + i][row r] of matrix mi's tile: sum over the row's k values of x_entry * w, k % 128 == 0.
// wrows[mi]: the tile's row r (this lane's) of matrix mi in the blob; nblk blocks.  The activation fragments are made once for all NM.
template <int K, bool COMPUTED, int NM, bool WORDS, class Act>
__device__ __forceinline__ void tile_dot(const uint8_t* const* wrows, int nblk, const uint32_t* lut, const Act& act, int ne,
                                         float (*acc_out)[8]) {
    constexpr int bsz = 2 + 4 * K;
    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;
    float tot[NM][8];
#pragma unroll
    for (int mi = 0; mi < NM; ++mi)
#pragma unroll
        for (int i = 0; i < 8; ++i) tot[mi][i] = 0.0f;
    const bool have = r < ne;   // this lane's A rows (the entry m = r)
    for (int b = 0; b < nblk; ++b) {
        float d[NM];
        uint64_t lo[NM], hi[NM];
#pragma unroll
        for (int mi = 0; mi < NM; ++mi) {
            const uint8_t* rec = wrows[mi] + (size_t) b * bsz;
            d[mi] = half_bits_f(*(const uint16_t*) rec);
            if (WORDS) stream_window_u32<K>(rec, kg, lo[mi], hi[mi]);
            else stream_window<K>(rec, kg, lo[mi], hi[mi]);
        }
#if defined(DS4_TQ_WMMA)
        float8_t acc[NM];
#pragma unroll
        for (int mi = 0; mi < NM; ++mi) acc[mi] = float8_t{0, 0, 0, 0, 0, 0, 0, 0};
#pragma unroll
        for (int m = 0; m < 8; ++m) {
            half8_t ah, al;
            {
                float v[8];
                if (have) act.load8(r, b * 128 + 64 * kg + 8 * m, v);
                else {
#pragma unroll
                    for (int j = 0; j < 8; ++j) v[j] = 0.0f;
                }
#pragma unroll
                for (int j = 0; j < 8; ++j) {
                    const _Float16 h = (_Float16) v[j];
                    ah[j] = h;
                    al[j] = (_Float16) (v[j] - (float) h);
                }
            }
#pragma unroll
            for (int mi = 0; mi < NM; ++mi) {
                half8_t wf;
#pragma unroll
                for (int s2 = 0; s2 < 2; ++s2) {
                    const int u = 2 * m + s2;
                    uint32_t w0, w1;
                    step_words<COMPUTED>(lut, state_at(lo[mi], hi[mi], (15 - u) * K), w0, w1);
                    wf[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));
                    wf[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));
                    wf[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));
                    wf[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));
                }
                acc[mi] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(ah, wf, acc[mi]);   // D[m = entry][n = weight row]
                acc[mi] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(al, wf, acc[mi]);
            }
        }
#pragma unroll
        for (int mi = 0; mi < NM; ++mi)
#pragma unroll
            for (int i = 0; i < 8; ++i) tot[mi][i] += d[mi] * acc[mi][i];
#else
        // plain FMAs: this lane's row r, its 64 weights of the block against every entry (lane r + 16 kg' sums the halves)
        float part[NM][kMaxEnt];
#pragma unroll
        for (int mi = 0; mi < NM; ++mi)
#pragma unroll
            for (int e = 0; e < kMaxEnt; ++e) part[mi][e] = 0.0f;
#pragma unroll
        for (int m = 0; m < 8; ++m) {
            float w[NM][8];
#pragma unroll
            for (int mi = 0; mi < NM; ++mi)
#pragma unroll
                for (int s2 = 0; s2 < 2; ++s2) {
                    uint32_t w0, w1;
                    step_words<COMPUTED>(lut, state_at(lo[mi], hi[mi], (15 - (2 * m + s2)) * K), w0, w1);
                    w[mi][4 * s2 + 0] = half_bits_f(w0 & 0xffffu);
                    w[mi][4 * s2 + 1] = half_bits_f(w0 >> 16);
                    w[mi][4 * s2 + 2] = half_bits_f(w1 & 0xffffu);
                    w[mi][4 * s2 + 3] = half_bits_f(w1 >> 16);
                }
#pragma unroll
            for (int e = 0; e < kMaxEnt; ++e)
                if (e < ne) {
                    float v[8];
                    act.load8(e, b * 128 + 64 * kg + 8 * m, v);
#pragma unroll
                    for (int mi = 0; mi < NM; ++mi)
#pragma unroll
                        for (int j = 0; j < 8; ++j) part[mi][e] = fmaf(w[mi][j], v[j], part[mi][e]);
                }
        }
        // entries 0..7 into tot[i] of the kg = 0 lane, 8..15 of the kg = 1 lane, so the output indexing matches the WMMA's
#pragma unroll
        for (int mi = 0; mi < NM; ++mi)
#pragma unroll
            for (int i = 0; i < 8; ++i) {
                const float p0 = part[mi][i], p1 = part[mi][8 + i];
                const float q0 = __shfl_xor_sync(0xffffffffu, p0, 16), q1 = __shfl_xor_sync(0xffffffffu, p1, 16);
                tot[mi][i] += d[mi] * (kg == 0 ? p0 + q0 : p1 + q1);
            }
#endif
    }
#pragma unroll
    for (int mi = 0; mi < NM; ++mi)
#pragma unroll
        for (int i = 0; i < 8; ++i) acc_out[mi][i] = tot[mi][i];
}

__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {   // ds4_iq.cu's
    if (limit > 0.0f) {
        gv = fminf(gv, limit);
        uv = fminf(fmaxf(uv, -limit), limit);
    }
    return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
}

template <int K, bool COMPUTED>
__device__ __forceinline__ void load_lut(uint32_t* lut) {
    if (!COMPUTED) {
        // the table as the CPU decoder has it (tq_cpu.cpp's int16 state table): fp16 * 2^13 rounded half away from zero, over
        // 2^13 - exact for values >= 2^-3 and within 2^-14 below; the result is still an exact fp16 (fewer significant bits)
        for (int i = threadIdx.x; i < 2048; i += blockDim.x) {
            uint32_t pair = 0;
#pragma unroll
            for (int j = 0; j < 2; ++j) {
                const float v = half_bits_f(tq_lut_f16[2 * i + j]) * 8192.0f;
                const int n = (int) (v < 0.0f ? v - 0.5f : v + 0.5f);
                pair |= (uint32_t) __half_as_ushort(__float2half_rn((float) n / 8192.0f)) << (16 * j);
            }
            lut[i] = pair;
        }
        __syncthreads();
    }
}

// ---------------------------------------------------------------- token / window experts
__device__ __forceinline__ bool group_of(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
    if (a.grp_ptr == nullptr) {
        const unsigned long long p = a.plan_ptr[grp];
        if (p == 0ull) return false;
        blob = (const uint8_t*) p;
        m0 = grp;
        m1 = grp + 1;
        return true;
    }
    if (grp >= *a.n_groups) return false;
    blob = (const uint8_t*) a.grp_ptr[grp];
    m0 = a.grp_start[grp];
    m1 = a.grp_start[grp + 1];
    return true;
}

// grid (n_ff / 64, groups): a wave = a 16-row tile of the gate and of the up of the group's expert, 16 entries a pass
template <int K, bool COMPUTED, bool WORDS>
__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
    __shared__ int ent[kMaxEnt], xr[kMaxEnt];
    const uint8_t* blob;
    int m0, m1;
    if (!group_of(a, blockIdx.y, blob, m0, m1)) return;   // (uniform)
    load_lut<K, COMPUTED>(lut);
    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
    const int tile = blockIdx.x * kWaves + wave;
    const int row = tile * kTileRows + r;
    const size_t rb = (size_t) (a.n_embd / 128) * (2 + 4 * K);
    const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
    const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
    for (int mm = m0; mm < m1; mm += kMaxEnt) {
        const int ne = min(kMaxEnt, m1 - mm);
        __syncthreads();
        if ((int) threadIdx.x < ne) {
            const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
            ent[threadIdx.x] = e;
            xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
        }
        __syncthreads();
        if (tile * kTileRows >= a.n_ff) continue;
        const ActQ8 act{a.xq, a.xd, xr, a.n_embd};
        const uint8_t* wr[2] = {g, u};
        float gu[2][8];
        tile_dot<K, COMPUTED, 2, WORDS>(wr, a.n_embd / 128, lut, act, ne, gu);
#pragma unroll
        for (int i = 0; i < 8; ++i) {
            const int m = 8 * kg + i;
            if (m < ne) a.h[(size_t) ent[m] * a.n_ff + row] = swiglu(gu[0][i], gu[1][i], a.limit);
        }
    }
}

// grid (n_embd / 64, groups): the down's tile of a wave against the entries' int8 h
template <int K, bool COMPUTED, bool WORDS>
__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
    __shared__ int ent[kMaxEnt];
    const uint8_t* blob;
    int m0, m1;
    if (!group_of(a, blockIdx.y, blob, m0, m1)) return;
    load_lut<K, COMPUTED>(lut);
    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
    const int tile = blockIdx.x * kWaves + wave;
    const int row = tile * kTileRows + r;
    const size_t rb = (size_t) (a.n_ff / 128) * (2 + 4 * K);
    const uint8_t* d = blob + a.tr[2] + (size_t) row * rb;
    for (int mm = m0; mm < m1; mm += kMaxEnt) {
        const int ne = min(kMaxEnt, m1 - mm);
        __syncthreads();
        if ((int) threadIdx.x < ne) ent[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
        __syncthreads();
        if (tile * kTileRows >= a.n_embd) continue;
        const ActQ8 act{a.hq, a.hd, ent, a.n_ff};
        const uint8_t* wr[1] = {d};
        float dd[1][8];
        tile_dot<K, COMPUTED, 1, WORDS>(wr, a.n_ff / 128, lut, act, ne, dd);
#pragma unroll
        for (int i = 0; i < 8; ++i) {
            const int m = 8 * kg + i;
            if (m < ne) a.part[(size_t) ent[m] * a.n_embd + row] = dd[0][i];
        }
    }
}

// ---------------------------------------------------------------- the prompt path's sets
// grid (n_ff / 64, chunks of 16 rows, n_exp): expert e = blockIdx.z owns the sorted rows [bounds[e], bounds[e + 1]); local
// row = sorted row - row0.  gate / up: F32 x[idx[local]] (ldx) -> F16 swiglu h[local * ldo + c]; down: F16 xh[local] (ldx) ->
// F32 y[local * ldo + c].
template <int K, bool COMPUTED, bool GATE_UP, bool WORDS>
__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {
    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
    __shared__ int rowsel[kMaxEnt];
    const int e = blockIdx.z;
    const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
    const int r0 = lo + blockIdx.y * kMaxEnt;
    if (r0 >= hi) return;   // (uniform)
    load_lut<K, COMPUTED>(lut);
    const int ne = min(kMaxEnt, hi - r0);
    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
    const int N = GATE_UP ? a.n_ff : a.n_embd, Kin = GATE_UP ? a.n_embd : a.n_ff;
    const int tile = blockIdx.x * kWaves + wave;
    const int row = tile * kTileRows + r;
    const size_t rb = (size_t) (Kin / 128) * (2 + 4 * K);
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    if ((int) threadIdx.x < ne) rowsel[threadIdx.x] = GATE_UP ? a.idx[r0 + threadIdx.x] : r0 + (int) threadIdx.x;
    __syncthreads();
    if (tile * kTileRows >= N) return;
    float res[2][8];
    if (GATE_UP) {
        const ActF32 act{a.xf, rowsel, (size_t) a.ldx};
        const uint8_t* wr[2] = {blob + a.tr[0] + (size_t) row * rb, blob + a.tr[1] + (size_t) row * rb};
        tile_dot<K, COMPUTED, 2, WORDS>(wr, Kin / 128, lut, act, ne, res);
    } else {
        const ActF16 act{a.xh, rowsel, (size_t) a.ldx};
        const uint8_t* wr[1] = {blob + a.tr[2] + (size_t) row * rb};
        tile_dot<K, COMPUTED, 1, WORDS>(wr, Kin / 128, lut, act, ne, res);
    }
    const int kgl = threadIdx.x >> 4 & 1;
#pragma unroll
    for (int i = 0; i < 8; ++i) {
        const int m = 8 * kgl + i;
        if (m >= ne) continue;
        const size_t lr = (size_t) (r0 + m);
        if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(res[0][i], res[1][i], a.limit)));
        else a.y[lr * a.ldo + row] = res[0][i];
    }
}

// ---------------------------------------------------------------- GEMV jobs
// grid (n / 64): y[c * ycs + r] = alpha (row r . x_c) + bias[r] for ncol (<= 16) int8 activation columns
template <int K, bool COMPUTED, bool WORDS>
__global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {
    __shared__ uint32_t lut[COMPUTED ? 1 : 2048];
    __shared__ int cols[kMaxEnt];
    load_lut<K, COMPUTED>(lut);
    if ((int) threadIdx.x < kMaxEnt) cols[threadIdx.x] = (int) threadIdx.x;
    __syncthreads();
    const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4, wave = threadIdx.x >> 5;
    const int tile = blockIdx.x * kWaves + wave;
    if (tile * kTileRows >= j.n) return;
    const int row = tile * kTileRows + r;
    const ActQ8 act{j.xq, j.xd, cols, j.k};
    const uint8_t* wr[1] = {j.w + (size_t) row * j.ld};
    float accm[1][8];
    tile_dot<K, COMPUTED, 1, WORDS>(wr, j.k / 128, lut, act, j.ncol, accm);
#pragma unroll
    for (int i = 0; i < 8; ++i) {
        const int m = 8 * kg + i;
        if (m < j.ncol && row < j.n) j.y[(size_t) m * j.ycs + row] = j.alpha * accm[0][i] + (j.bias != nullptr ? j.bias[row] : 0.0f);
    }
}

// ---------------------------------------------------------------- dispatch
#define DS4_TQ_SWITCH(t, F)                         \
    switch (t) {                                    \
        case kTQK6: F(6, false); break;             \
        case kTQK7: F(7, false); break;             \
        case kTQK5X: F(5, true); break;             \
        case kTQK4X: F(4, true); break;             \
        case kTQK5T: F(5, false); break;            \
        case kTQK4T: F(4, false); break;            \
        default: return false;                      \
    }

// 32-bit stream loads (STRATA_DS4_TQ_WORDS=1) or the 16-bit ones (default)
bool use_words() {
    static const bool on = [] {
        const char* e = std::getenv("STRATA_DS4_TQ_WORDS");
        return e != nullptr && std::atoi(e) != 0;   // (no gain measured on the R9700: 13.8 vs 14.2 ms a token)
    }();
    return on;
}

bool bad(const char* what) {
    std::fprintf(stderr, "ds4 tq: %s\n", what);
    return false;
}
}  // namespace

bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
    const bool w = use_words();
    (void) ne_max;
    if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("gate_up: geometry not covered");
    const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) groups);
#define GU_(K, C) { if (w) strata_ds4_launch(gate_up_kernel<K, C, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(gate_up_kernel<K, C, false>, grid, kThreads, 0, s, a); }
    DS4_TQ_SWITCH(a.tg, GU_)
#undef GU_
    return true;
}

bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
    const bool w = use_words();
    (void) ne_max;
    if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("down: geometry not covered");
    const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) groups);
#define DN_(K, C) { if (w) strata_ds4_launch(down_kernel<K, C, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(down_kernel<K, C, false>, grid, kThreads, 0, s, a); }
    DS4_TQ_SWITCH(a.td, DN_)
#undef DN_
    return true;
}

bool set_gate_up(const SetArgs& a, cudaStream_t s) {
    const bool w = use_words();
    if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("set_gate_up: geometry not covered");
    const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);
#define SG_(K, C) { if (w) strata_ds4_launch(set_kernel<K, C, true, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(set_kernel<K, C, true, false>, grid, kThreads, 0, s, a); }
    DS4_TQ_SWITCH(a.tg, SG_)
#undef SG_
    return true;
}

bool set_down(const SetArgs& a, cudaStream_t s) {
    const bool w = use_words();
    if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("set_down: geometry not covered");
    const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);
#define SD_(K, C) { if (w) strata_ds4_launch(set_kernel<K, C, false, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(set_kernel<K, C, false, false>, grid, kThreads, 0, s, a); }
    DS4_TQ_SWITCH(a.td, SD_)
#undef SD_
    return true;
}

bool mv(const MvJob* jobs, int n, cudaStream_t s) {
    for (int i = 0; i < n; ++i) {
        const MvJob& j = jobs[i];
        const bool w = use_words();
        if (j.k % 128 != 0 || j.n % kTileRows != 0 || j.ncol < 1 || j.ncol > kMaxEnt) return bad("mv: geometry not covered");
        const dim3 grid((unsigned) ((j.n + kWaves * kTileRows - 1) / (kWaves * kTileRows)));
#define MV_(K, C) { if (w) strata_ds4_launch(mv_kernel<K, C, true>, grid, kThreads, 0, s, j); else strata_ds4_launch(mv_kernel<K, C, false>, grid, kThreads, 0, s, j); }
        DS4_TQ_SWITCH(j.type, MV_)
#undef MV_
    }
    return true;
}

}  // namespace strata::ds4::gguf::iq::tq