Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
83.2 KB
Raw Blame
// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh).  The block layouts and grids are
// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.
#include "ds4_launch.h"   // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_iq.cuh"
#if defined(STRATA_METAL)
#include <ds4_metal_host.h>
#endif
#include "ds4_numerics.cuh"
#include "ds4_experts_rule.hpp"

#include <cuda_fp16.h>
#if !defined(STRATA_USE_HIP)
#include <mma.h>
#endif

#define GGML_COMMON_DECL_CUDA
#define GGML_COMMON_IMPL_CUDA
#include "ggml-common.h"

#include <algorithm>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <type_traits>
#include <map>
#include <mutex>
#include <unordered_map>

namespace strata::ds4::gguf::iq {
namespace {

constexpr int kWarps = 8, kThreads = kWarps * 32;

// ---------------------------------------------------------------- the types
template <int T>
struct Ty;
template <> struct Ty<kIQ2_XXS> { static constexpr int B = 66, TAB = 512, MIN = 0; };
template <> struct Ty<kIQ2_XS> { static constexpr int B = 74, TAB = 1024, MIN = 0; };
template <> struct Ty<kIQ2_S> { static constexpr int B = 82, TAB = 2048, MIN = 0; };
template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN = 0; };
template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };
template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0; };
template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };
// The 1.5- and 1.75-bit types (IQ1_S, IQ1_M: an 11-bit index into iq1s_grid_gpu, whose 8 values are 4-bit codes) and
// IQ4_NL (32 values a block: its codebook's 16 int8 values are the first 4 words of its table)
template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };
template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };
template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };
template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };
template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };
// the dense matrices' 5- and 6-bit K-quants (GEMV only: ds4_iq's mv; not an expert type)
template <> struct Ty<kQ5_K> { static constexpr int B = 176, TAB = 0, MIN = 1; };
template <> struct Ty<kQ6_K> { static constexpr int B = 210, TAB = 0, MIN = 0; };
// MXFP4 (ds4_gguf.cu's own kernels everywhere else): a block of 32 values, an E8M0 scale then 16 code bytes; the light
// sets take it too
constexpr int kMXFP4 = 39;
template <> struct Ty<kMXFP4> { static constexpr int B = 17, TAB = 0, MIN = 0; };
template <int T> constexpr int kQK = T == kMXFP4 || T == kIQ4_NL ? 32 : 256;   // values a block
static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_xs) == 74 && sizeof(block_iq2_s) == 82 &&
                  sizeof(block_iq3_xxs) == 98 &&
                  sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 && sizeof(block_q4_K) == 144 &&
                  sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110 && sizeof(block_iq1_s) == 50 &&
                  sizeof(block_iq1_m) == 56 && sizeof(block_iq4_nl) == 18,
              "ggml's block sizes");

template <int T>
__host__ __device__ __forceinline__ size_t row_bytes_t(int k) { return (size_t) (k / kQK<T>) * Ty<T>::B; }

// the type's codebook into shared memory (u32 words; IQ4_XS: its 16 values, from bytes - an int8 table's alignment)
template <int T>
__device__ __forceinline__ void load_tab(uint32_t* tab) {
    if constexpr (T == kIQ4_XS || T == kIQ4_NL) {
        if (threadIdx.x < 4) {
            const uint8_t* kv = (const uint8_t*) kvalues_iq4nl + 4 * threadIdx.x;
            tab[threadIdx.x] = (uint32_t) kv[0] | (uint32_t) kv[1] << 8 | (uint32_t) kv[2] << 16 | (uint32_t) kv[3] << 24;
        }
    } else if constexpr (Ty<T>::TAB > 0) {
        const uint32_t* src = T == kIQ2_XXS  ? (const uint32_t*) iq2xxs_grid
                              : T == kIQ2_XS ? (const uint32_t*) iq2xs_grid
                              : T == kIQ2_S  ? (const uint32_t*) iq2s_grid
                              : T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid
                              : T == kIQ1_S || T == kIQ1_M ? (const uint32_t*) iq1s_grid_gpu
                                              : (const uint32_t*) iq3s_grid;
        for (int i = threadIdx.x; i < Ty<T>::TAB; i += blockDim.x) tab[i] = src[i];
    }
}

__device__ __forceinline__ uint32_t ld2(const uint8_t* p) {   // 4 bytes from a 2-byte aligned address
    const uint16_t* q = (const uint16_t*) p;
    return (uint32_t) q[0] | ((uint32_t) q[1] << 16);
}
__device__ __forceinline__ uint32_t ld4(const uint8_t* p) { return *(const uint32_t*) p; }
__device__ __forceinline__ float h2f(const uint8_t* p) { return __half2float(__ushort_as_half(*(const uint16_t*) p)); }
// 7 sign bits (bits 0..6 of v) -> the 8 signs, bit 7 their parity (ggml's ksigns_iq2xs)
__device__ __forceinline__ uint32_t ksigns8(uint32_t v) {
    v &= 127u;
    return v | ((__popc(v) & 1u) << 7);
}
// 4 grid bytes with the signs of bits 0..3 of b applied (bit i: byte i negative) - llama.cpp's HIP form, cheap where the
// SIMD byte instructions (__vcmpne4 / __vsub4) are emulated (Volta, RDNA): every grid byte is nonzero and <= 62, so
// (g ^ 0xff) + 1 = -g in each byte with no carry into the next
__device__ __forceinline__ int sgn4(uint32_t g, uint32_t b) {
    const uint32_t lo = (((b & 15u) * 0x10204080u) & 0x80808080u) >> 7;
    return (int) ((g ^ (lo * 255u)) + lo);
}
__device__ __forceinline__ int2 tab16(uint32_t q4, const uint32_t* t) {   // vecdotq's get_int_from_table_16
    uint32_t tmp[2];
    const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
#pragma unroll
    for (int i = 0; i < 2; ++i) {
        const uint32_t sh = 16 * i;
        const uint32_t lo = __byte_perm(t[0], t[1], q4 >> sh), hi = __byte_perm(t[2], t[3], q4 >> sh);
        tmp[i] = __byte_perm(lo, hi, sel >> sh);
    }
    return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));
}
// iq1s_grid_gpu's 8 values in 4 bytes, 4 bits each (low nibbles: values 0..3, high: 4..7; 0 is -1, 2 is +1) -> the 4
// values as int8 codes 8 (n - 1) + 1, or - 1 when the group's delta is negative: ggml's weight dl ((n - 1) +- 0.125), whose
// dl (2 sc + 1) / 8 is this kernel's per-sub-block scale, is then d * (s0 lo + s1 hi) exactly
__device__ __forceinline__ uint32_t iq1_codes(uint32_t n, bool neg) {
    const uint32_t c = (uint32_t) __vsub4((int) (n * 8u), 0x07070707);   // (n * 8 - 7) in each byte: no carry (n <= 15)
    return neg ? (uint32_t) __vsub4((int) c, 0x02020202) : c;
}
// IQ1_M has no d field: its F16 scale is the top nibbles of its four u16 scales (ggml's dequantize_row_iq1_m)
__device__ __forceinline__ uint16_t iq1m_d(const uint8_t* b) {
    const uint32_t s0 = (uint32_t) b[48] | (uint32_t) b[49] << 8, s1 = (uint32_t) b[50] | (uint32_t) b[51] << 8;
    const uint32_t s2 = (uint32_t) b[52] | (uint32_t) b[53] << 8, s3 = (uint32_t) b[54] | (uint32_t) b[55] << 8;
    return (uint16_t) ((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) | (s3 & 0xf000u));
}
__device__ __forceinline__ int q3k_scale(const uint8_t* sc, int is) {
    const int lo = is < 8 ? sc[is] & 15 : sc[is - 8] >> 4;
    const int hi = (sc[8 + (is & 3)] >> (2 * (is >> 2))) & 3;
    return (lo | (hi << 4)) - 32;
}
__device__ __forceinline__ void q4k_scale(const uint8_t* q, int j, int& d, int& m) {   // ggml's get_scale_min_k4
    if (j < 4) {
        d = q[j] & 63;
        m = q[j + 4] & 63;
    } else {
        d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
        m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4);
    }
}

// a 32-value sub-block (kb: its index in the row) as 32 int8 codes and its scales: weight i = d s q_i - m, s and m of
// its half (i < 16: s0, m0; else s1, m1)
struct Sub {
    int q[8];
    float d, m0, m1;
    int s0, s1;
};
// (b: the block's address, ib: the sub-block in it (0 for the 32-value types); the token kernels keep ib fixed a lane)
template <int T>
__device__ __forceinline__ void decode_b(const uint8_t* __restrict__ b, int ib, const uint32_t* tab, Sub& s) {
    if constexpr (T == kMXFP4) {   // codes: value j in byte j's low nibble, j + 16 in its high one (any alignment)
        const uint32_t t[4] = {0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu};   // ggml's kvalues_mxfp4
#pragma unroll
        for (int i = 0; i < 4; ++i) {
            const uint8_t* q = b + 1 + 4 * i;
            const int2 v = tab16((uint32_t) q[0] | (uint32_t) q[1] << 8 | (uint32_t) q[2] << 16 | (uint32_t) q[3] << 24, t);
            s.q[i] = v.x;
            s.q[4 + i] = v.y;
        }
        const uint32_t e = b[0];   // 2^(e - 128): ggml's GGML_E8M0_TO_FP32_HALF (the codes are the E2M1 values doubled)
        s.d = __uint_as_float(e < 2 ? 0x00200000u << e : (e - 1) << 23);
        s.s0 = s.s1 = 1;
        (void) ib;
        (void) tab;
    } else if constexpr (T == kIQ2_XXS) {
        const uint8_t* q = b + 2 + 8 * ib;
        const uint32_t a0 = ld2(q), a1 = ld2(q + 4);
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint2 g = ((const uint2*) tab)[(a0 >> (8 * l)) & 255u];
            const uint32_t sg = ksigns8(a1 >> (7 * l));
            s.q[2 * l] = sgn4(g.x, sg);
            s.q[2 * l + 1] = sgn4(g.y, sg >> 4);
        }
        s.s0 = s.s1 = 2 * (int) (a1 >> 28) + 1;
        s.d = h2f(b) * 0.125f;
    } else if constexpr (T == kIQ2_XS) {   // d, qs[32] u16 (9-bit grid index, 7-bit sign index), scales[8] (two 4-bit)
        const uint8_t* q = b + 2 + 8 * ib;
        const uint32_t a0 = ld2(q), a1 = ld2(q + 4);
        const uint32_t v[4] = {a0 & 0xffffu, a0 >> 16, a1 & 0xffffu, a1 >> 16};
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint2 g = ((const uint2*) tab)[v[l] & 511u];
            const uint32_t sg = ksigns8(v[l] >> 9);
            s.q[2 * l] = sgn4(g.x, sg);
            s.q[2 * l + 1] = sgn4(g.y, sg >> 4);
        }
        const uint8_t sc = b[66 + ib];
        s.s0 = 2 * (int) (sc & 15) + 1;
        s.s1 = 2 * (int) (sc >> 4) + 1;
        s.d = h2f(b) * 0.125f;
    } else if constexpr (T == kIQ2_S) {
        const uint8_t* qs = b + 2;
        const uint32_t i4 = ld2(qs + 4 * ib), sg4 = ld2(qs + 32 + 4 * ib), h = b[66 + ib], sc = b[74 + ib];
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint2 g = ((const uint2*) tab)[((i4 >> (8 * l)) & 255u) | ((h << (8 - 2 * l)) & 0x300u)];
            const uint32_t sg = sg4 >> (8 * l);
            s.q[2 * l] = sgn4(g.x, sg);
            s.q[2 * l + 1] = sgn4(g.y, sg >> 4);
        }
        s.s0 = 2 * (int) (sc & 15) + 1;
        s.s1 = 2 * (int) (sc >> 4) + 1;
        s.d = h2f(b) * 0.125f;
    } else if constexpr (T == kIQ3_XXS) {
        const uint8_t* qs = b + 2;
        const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), aux = ld2(qs + 64 + 4 * ib);
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint32_t w = l < 2 ? w0 : w1, sh = 16 * (l & 1);
            const uint32_t sg = ksigns8(aux >> (7 * l));
            s.q[2 * l] = sgn4(tab[(w >> sh) & 255u], sg);
            s.q[2 * l + 1] = sgn4(tab[(w >> (sh + 8)) & 255u], sg >> 4);
        }
        s.s0 = s.s1 = 2 * (int) (aux >> 28) + 1;
        s.d = h2f(b) * 0.25f;
    } else if constexpr (T == kIQ3_S) {
        const uint8_t* qs = b + 2;
        const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), h = b[66 + ib], s4 = ld2(b + 74 + 4 * ib);
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint32_t w = l < 2 ? w0 : w1, sh = 16 * (l & 1);
            const uint32_t i1 = ((w >> sh) & 255u) | ((h << (8 - 2 * l)) & 256u);
            const uint32_t i2 = ((w >> (sh + 8)) & 255u) | ((h << (7 - 2 * l)) & 256u);
            const uint32_t sg = s4 >> (8 * l);
            s.q[2 * l] = sgn4(tab[i1], sg);
            s.q[2 * l + 1] = sgn4(tab[i2], sg >> 4);
        }
        s.s0 = s.s1 = 1 + 2 * (int) ((b[106 + (ib >> 1)] >> (4 * (ib & 1))) & 15);
        s.d = h2f(b);
    } else if constexpr (T == kIQ4_XS) {
        const uint32_t sh = *(const uint16_t*) (b + 2);
        const int ls = ((b[4 + (ib >> 1)] >> (4 * (ib & 1))) & 15) | (((sh >> (2 * ib)) & 3) << 4);
        const uint8_t* qs = b + 8 + 16 * ib;
#pragma unroll
        for (int t = 0; t < 4; ++t) {
            const int2 v = tab16(ld4(qs + 4 * t), tab);
            s.q[t] = v.x;
            s.q[4 + t] = v.y;
        }
        s.s0 = s.s1 = ls - 32;
        s.d = h2f(b);
    } else if constexpr (T == kIQ1_S) {   // d, qs[32] (4 a 32-value sub-block), qh[8] u16: 11-bit grid, 3-bit scale, delta
        const uint8_t* qs = b + 2;
        const uint16_t qh = ((const uint16_t*) (b + 34))[ib];
        const bool neg = (qh & 0x8000u) != 0;
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint32_t g = tab[qs[4 * ib + l] | (((qh >> (3 * l)) & 7u) << 8)];
            s.q[2 * l] = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);
            s.q[2 * l + 1] = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);
        }
        s.s0 = s.s1 = 2 * (int) ((qh >> 12) & 7u) + 1;
        s.d = h2f(b) * 0.125f;
    } else if constexpr (T == kIQ1_M) {   // qs[32], qh[16], scales[8]: no d field; 11-bit grid, two 3-bit scales a 32
        const uint8_t* qs = b + 4 * ib;
        const uint8_t qh0 = b[32 + 2 * ib], qh1 = b[33 + 2 * ib];
        const uint16_t sc = ((const uint16_t*) (b + 48))[ib >> 1];
        const int sh = 6 * (ib & 1);
        const uint32_t ii[4] = {qs[0] | ((uint32_t) (qh0 << 8) & 0x700u), qs[1] | ((uint32_t) (qh0 << 4) & 0x700u),
                                qs[2] | ((uint32_t) (qh1 << 8) & 0x700u), qs[3] | ((uint32_t) (qh1 << 4) & 0x700u)};
        const bool neg[4] = {(qh0 & 0x08u) != 0, (qh0 & 0x80u) != 0, (qh1 & 0x08u) != 0, (qh1 & 0x80u) != 0};
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint32_t g = tab[ii[l]];
            s.q[2 * l] = (int) iq1_codes(g & 0x0F0F0F0Fu, neg[l]);
            s.q[2 * l + 1] = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg[l]);
        }
        s.s0 = 2 * (int) ((sc >> sh) & 7u) + 1;
        s.s1 = 2 * (int) ((sc >> (sh + 3)) & 7u) + 1;
        s.d = __half2float(__ushort_as_half(iq1m_d(b))) * 0.125f;
    } else if constexpr (T == kIQ4_NL) {   // d, qs[16]: a block of 32 values, its codebook's int8 a nibble
        const uint8_t* qs = b + 2;   // (18-byte blocks: even, so ld2's 2-byte alignment - not ld4's 4)
#pragma unroll
        for (int t = 0; t < 4; ++t) {
            const int2 v = tab16(ld2(qs + 4 * t), tab);
            s.q[t] = v.x;
            s.q[4 + t] = v.y;
        }
        s.s0 = s.s1 = 1;
        s.d = h2f(b);
    } else if constexpr (T == kQ4_K) {
        int sc, m;
        q4k_scale(b + 4, ib, sc, m);
        const uint8_t* qs = b + 16 + 32 * (ib >> 1);
        const int sh = 4 * (ib & 1);
#pragma unroll
        for (int t = 0; t < 8; ++t) s.q[t] = (int) ((ld4(qs + 4 * t) >> sh) & 0x0f0f0f0fu);
        s.s0 = s.s1 = sc;
        s.d = h2f(b);
        s.m0 = s.m1 = h2f(b + 2) * (float) m;
    } else if constexpr (T == kQ5_K) {   // d, dmin, scales[12], qh[32], qs[128]: weight = d sc (nibble + 16 hbit) - dmin m
        int sc, m;
        q4k_scale(b + 4, ib, sc, m);
        const uint8_t* qs = b + 48 + 32 * (ib >> 1);
        const uint8_t* qh = b + 16;
        const int sh = 4 * (ib & 1);
#pragma unroll
        for (int t = 0; t < 8; ++t)
            s.q[t] = (int) (((ld4(qs + 4 * t) >> sh) & 0x0f0f0f0fu) | (((ld4(qh + 4 * t) >> ib) & 0x01010101u) << 4));
        s.s0 = s.s1 = sc;
        s.d = h2f(b);
        s.m0 = s.m1 = h2f(b + 2) * (float) m;
    } else if constexpr (T == kQ6_K) {   // ql[128], qh[64], scales int8[16], d: weight = d sc (q - 32), q 6 bits
        const int h = ib >> 2, quarter = ib & 3;
        const uint8_t* ql = b + 64 * h + (quarter & 1 ? 32 : 0);
        const uint8_t* qh = b + 128 + 32 * h;
        const int sh = 4 * (quarter >> 1);
#pragma unroll
        for (int t = 0; t < 8; ++t) {
            const uint32_t x = ((ld2(ql + 4 * t) >> sh) & 0x0f0f0f0fu) | (((ld2(qh + 4 * t) >> (2 * quarter)) & 0x03030303u) << 4);
            s.q[t] = (int) __vsub4(x, 0x20202020u);
        }
        const int8_t* scl = (const int8_t*) (b + 192) + 8 * h + 2 * quarter;
        s.s0 = scl[0];
        s.s1 = scl[1];
        s.d = h2f(b + 208);
    } else if constexpr (T == kQ2_K) {
        const int n = ib >> 2, j = ib & 3;
        const uint8_t* qs = b + 16 + 32 * n;
#pragma unroll
        for (int t = 0; t < 8; ++t) s.q[t] = (int) ((ld4(qs + 4 * t) >> (2 * j)) & 0x03030303u);
        const int c0 = b[8 * n + 2 * j], c1 = b[8 * n + 2 * j + 1];
        const float dmin = h2f(b + 82);
        s.s0 = c0 & 15;
        s.s1 = c1 & 15;
        s.d = h2f(b + 80);
        s.m0 = dmin * (float) (c0 >> 4);
        s.m1 = dmin * (float) (c1 >> 4);
    } else {   // Q3_K
        const int n = ib >> 2, j = ib & 3;
        const uint8_t* qs = b + 32 + 32 * n;
#pragma unroll
        for (int t = 0; t < 8; ++t) {
            const uint32_t ql = (ld2(qs + 4 * t) >> (2 * j)) & 0x03030303u;
            const uint32_t neg = ((~ld2(b + 4 * t)) >> (4 * n + j)) & 0x01010101u;
            s.q[t] = (int) (ql | (neg * 0xFCu));   // (q - 4 in a byte: q + 0xFC, no carry for q <= 3)
        }
        s.s0 = q3k_scale(b + 96, 8 * n + 2 * j);
        s.s1 = q3k_scale(b + 96, 8 * n + 2 * j + 1);
        s.d = h2f(b + 108);
    }
}
template <int T>
__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {
    decode_b<T>(row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B, kb & 7, tab, s);
}
// the sub-block against 32 int8 activations (xa: values 0..15, xb: 16..31), before their scale: an exact integer dot,
// its scales in F32 (explicit roundings: every instantiation the same bits)
template <int T>
__device__ __forceinline__ float sub_dot(const Sub& s, const int4& xa, const int4& xb) {
    int lo = __dp4a(s.q[0], xa.x, 0);
    lo = __dp4a(s.q[1], xa.y, lo);
    lo = __dp4a(s.q[2], xa.z, lo);
    lo = __dp4a(s.q[3], xa.w, lo);
    int hi = __dp4a(s.q[4], xb.x, 0);
    hi = __dp4a(s.q[5], xb.y, hi);
    hi = __dp4a(s.q[6], xb.z, hi);
    hi = __dp4a(s.q[7], xb.w, hi);
    float v = __fmul_rn(s.d, (float) (s.s0 * lo + s.s1 * hi));
    if constexpr (Ty<T>::MIN) {
        int sl = __dp4a(0x01010101, xa.x, 0);
        sl = __dp4a(0x01010101, xa.y, sl);
        sl = __dp4a(0x01010101, xa.z, sl);
        sl = __dp4a(0x01010101, xa.w, sl);
        int sh = __dp4a(0x01010101, xb.x, 0);
        sh = __dp4a(0x01010101, xb.y, sh);
        sh = __dp4a(0x01010101, xb.z, sh);
        sh = __dp4a(0x01010101, xb.w, sh);
        v = __fsub_rn(v, __fadd_rn(__fmul_rn(s.m0, (float) sl), __fmul_rn(s.m1, (float) sh)));
    }
    return v;
}

// values [8 p, 8 p + 8) of sub-block kb, as F32 (the prompt sets' tiles): ggml's dequantize_row_* values
template <int T>
__device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, int p, const uint32_t* tab, float* w) {
    const int ib = kb & 7;
    const uint8_t* b = row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B;
    if constexpr (T == kIQ2_XXS) {
        const uint8_t* q = b + 2 + 8 * ib;
        const uint32_t a1 = ld2(q + 4);
        const uint2 g = ((const uint2*) tab)[q[p]];
        const uint32_t v = (a1 >> (7 * p)) & 127u, sg = v | ((__popc(v) & 1u) << 7);
        const float dl = h2f(b) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;
#pragma unroll
        for (int j = 0; j < 8; ++j) {
            const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
            w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
        }
    } else if constexpr (T == kIQ2_XS) {
        const uint16_t v = ((const uint16_t*) (b + 2 + 8 * ib))[p];
        const uint2 g = ((const uint2*) tab)[v & 511u];
        const uint32_t sg = ksigns8(v >> 9);
        const uint8_t sc = b[66 + ib];
        const float dl = h2f(b) * (float) (2 * (int) (p < 2 ? sc & 15 : sc >> 4) + 1) * 0.125f;
#pragma unroll
        for (int j = 0; j < 8; ++j) {
            const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
            w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
        }
    } else if constexpr (T == kIQ2_S) {
        const uint8_t* qs = b + 2;
        const uint32_t idx = qs[4 * ib + p] | ((((uint32_t) b[66 + ib]) << (8 - 2 * p)) & 0x300u);
        const uint32_t sg = qs[32 + 4 * ib + p], sc = b[74 + ib];
        const uint2 g = ((const uint2*) tab)[idx];
        const float dl = h2f(b) * (float) (2 * (int) (p < 2 ? sc & 15 : sc >> 4) + 1) * 0.125f;
#pragma unroll
        for (int j = 0; j < 8; ++j) {
            const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
            w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
        }
    } else if constexpr (T == kIQ3_XXS || T == kIQ3_S) {
        const uint8_t* qs = b + 2;
        uint32_t g1, g2, sg;
        float dl;
        if constexpr (T == kIQ3_XXS) {
            const uint32_t aux = ld2(qs + 64 + 4 * ib), v = (aux >> (7 * p)) & 127u;
            g1 = tab[qs[8 * ib + 2 * p]];
            g2 = tab[qs[8 * ib + 2 * p + 1]];
            sg = v | ((__popc(v) & 1u) << 7);
            dl = h2f(b) * (float) (2 * (int) (aux >> 28) + 1) * 0.25f;
        } else {
            const uint32_t h = b[66 + ib];
            g1 = tab[qs[8 * ib + 2 * p] | ((h << (8 - 2 * p)) & 256u)];
            g2 = tab[qs[8 * ib + 2 * p + 1] | ((h << (7 - 2 * p)) & 256u)];
            sg = b[74 + 4 * ib + p];
            dl = h2f(b) * (float) (1 + 2 * (int) ((b[106 + (ib >> 1)] >> (4 * (ib & 1))) & 15));
        }
#pragma unroll
        for (int j = 0; j < 8; ++j) {
            const float gv = (float) ((j < 4 ? g1 >> (8 * j) : g2 >> (8 * (j - 4))) & 255u);
            w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
        }
    } else if constexpr (T == kIQ4_XS) {
        const uint32_t sh = *(const uint16_t*) (b + 2);
        const int ls = ((b[4 + (ib >> 1)] >> (4 * (ib & 1))) & 15) | (((sh >> (2 * ib)) & 3) << 4);
        const float dl = h2f(b) * (float) (ls - 32);
        const uint8_t* qs = b + 8 + 16 * ib + 8 * (p & 1);
        const int8_t* kv = (const int8_t*) tab;
#pragma unroll
        for (int j = 0; j < 8; ++j) w[j] = dl * (float) kv[p < 2 ? qs[j] & 15 : qs[j] >> 4];
    } else if constexpr (T == kIQ1_S) {
        const uint16_t qh = ((const uint16_t*) (b + 34))[ib];
        const uint32_t g = tab[b[2 + 4 * ib + p] | (((qh >> (3 * p)) & 7u) << 8)];
        const uint32_t c0 = iq1_codes(g & 0x0F0F0F0Fu, (qh & 0x8000u) != 0);
        const uint32_t c1 = iq1_codes((g >> 4) & 0x0F0F0F0Fu, (qh & 0x8000u) != 0);
        const float dl = h2f(b) * (float) (2 * (int) ((qh >> 12) & 7u) + 1) * 0.125f;
#pragma unroll
        for (int j = 0; j < 4; ++j) w[j] = dl * (float) (int8_t) (c0 >> (8 * j));
#pragma unroll
        for (int j = 0; j < 4; ++j) w[4 + j] = dl * (float) (int8_t) (c1 >> (8 * j));
    } else if constexpr (T == kIQ1_M) {
        const uint8_t* qs = b + 4 * ib;
        const uint8_t qh0 = b[32 + 2 * ib], qh1 = b[33 + 2 * ib], qh = p < 2 ? qh0 : qh1;
        const uint16_t sc = ((const uint16_t*) (b + 48))[ib >> 1];
        const uint32_t g = tab[qs[p] | ((uint32_t) (qh << (p & 1 ? 4 : 8)) & 0x700u)];
        const bool neg = (p & 1 ? qh & 0x80u : qh & 0x08u) != 0;
        const uint32_t c0 = iq1_codes(g & 0x0F0F0F0Fu, neg), c1 = iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);
        const int sh = (int) ((sc >> (6 * (ib & 1) + (p < 2 ? 0 : 3))) & 7u);
        const float dl = __half2float(__ushort_as_half(iq1m_d(b))) * (float) (2 * sh + 1) * 0.125f;
#pragma unroll
        for (int j = 0; j < 4; ++j) w[j] = dl * (float) (int8_t) (c0 >> (8 * j));
#pragma unroll
        for (int j = 0; j < 4; ++j) w[4 + j] = dl * (float) (int8_t) (c1 >> (8 * j));
    } else if constexpr (T == kIQ4_NL) {
        const float dl = h2f(b);
        const uint8_t* qs = b + 2 + 8 * (p & 1);
        const int8_t* kv = (const int8_t*) tab;
#pragma unroll
        for (int j = 0; j < 8; ++j) w[j] = dl * (float) kv[p < 2 ? qs[j] & 15 : qs[j] >> 4];
    } else if constexpr (T == kQ4_K) {
        int sc, m;
        q4k_scale(b + 4, ib, sc, m);
        const float d1 = h2f(b) * (float) sc, m1 = h2f(b + 2) * (float) m;
        const uint8_t* qs = b + 16 + 32 * (ib >> 1) + 8 * p;
#pragma unroll
        for (int j = 0; j < 8; ++j) w[j] = d1 * (float) ((ib & 1) ? qs[j] >> 4 : qs[j] & 15) - m1;
    } else if constexpr (T == kQ2_K) {
        const int n = ib >> 2, jj = ib & 3;
        const int c = b[8 * n + 2 * jj + (p >> 1)];
        const float dl = h2f(b + 80) * (float) (c & 15), ml = h2f(b + 82) * (float) (c >> 4);
        const uint8_t* qs = b + 16 + 32 * n + 8 * p;
#pragma unroll
        for (int j = 0; j < 8; ++j) w[j] = dl * (float) ((qs[j] >> (2 * jj)) & 3) - ml;
    } else {   // Q3_K
        const int n = ib >> 2, jj = ib & 3;
        const float dl = h2f(b + 108) * (float) q3k_scale(b + 96, 8 * n + 2 * jj + (p >> 1));
        const uint8_t* qs = b + 32 + 32 * n + 8 * p;
        const uint8_t* hm = b + 8 * p;
#pragma unroll
        for (int j = 0; j < 8; ++j)
            w[j] = dl * (float) ((int) ((qs[j] >> (2 * jj)) & 3) - (((hm[j] >> (4 * n + jj)) & 1) ? 0 : 4));
    }
}


// ---------------------------------------------------------------- the token kernels' faster decode (CUDA only)
// The token kernels (gate_up_kernel / down_kernel) are bound by instructions per weight, not by memory, so on CUDA a type may
// decode through a larger shared-memory table that folds more of the work into one lookup.  The arithmetic is unchanged
// (the same integer codes into the same dp4a sums, the same F32 scales in the same order), so the results are the same bits.
#if !defined(STRATA_USE_HIP)
template <int T> struct Fast { static constexpr int WORDS = 0; };   // 0: this type decodes as decode<T>
// IQ1_M: the 12-bit (neg, 11-bit grid index) -> its 8 int8 codes (iq1_codes with the sub-block delta's sign), 2 words
template <> struct Fast<kIQ1_M> { static constexpr int WORDS = 8192; };
__device__ __align__(16) const uint2 kIq1mCodes[4096] = {
#include "ds4_iq1m_codes.inc"
};
#else
template <int T> struct Fast { static constexpr int WORDS = 0; };
#endif
// F: whether the fast form is wanted (the kernels of many rows a pass are bound by the dot products and their registers, and the
// larger codebook's shared memory costs them occupancy: they keep decode<T>)
template <int T, bool F = true> constexpr int kTabWords = F && Fast<T>::WORDS > 0 ? Fast<T>::WORDS : (Ty<T>::TAB > 0 ? Ty<T>::TAB : 1);

// Persistent grids (blocks looping over work items) only for the types with a codebook worth keeping in the block (8 KiB and
// more); the others run a block (tile, group) as before - their blocks leave at once when the group is empty.
#if defined(STRATA_USE_HIP)
template <int T>
constexpr bool kPersist = false;   // (HIP keeps the release grids)
#else
template <int T>
constexpr bool kPersist = kTabWords<T> * 4 >= 8192;
#endif

template <int T, bool F = true>
__device__ __forceinline__ void load_tab_fast(uint32_t* tab) {
    if constexpr (!F || Fast<T>::WORDS == 0) {
        load_tab<T>(tab);
#if !defined(STRATA_USE_HIP)
    } else if constexpr (T == kIQ1_M) {   // a copy of the generated table (ds4_iq1m_codes.inc): 2048 x 16 bytes
        const uint4* src = (const uint4*) kIq1mCodes;
        uint4* dst = (uint4*) tab;
        for (int j = threadIdx.x; j < 2048; j += blockDim.x) dst[j] = src[j];
#endif
    }
}
template <int T, bool F = true>
__device__ __forceinline__ void decode_fast_b(const uint8_t* __restrict__ b, int ib, const uint32_t* tab, Sub& s) {
    if constexpr (!F || Fast<T>::WORDS == 0) {
        decode_b<T>(b, ib, tab, s);
    } else if constexpr (T == kIQ1_M) {   // (56-byte blocks in 1120-byte rows at 256-aligned offsets: 8-byte aligned)
        const uint32_t qsw = *(const uint32_t*) (b + 4 * ib);
        const uint32_t qhw = *(const uint16_t*) (b + 32 + 2 * ib);
        const uint2 sw = *(const uint2*) (b + 48);
#pragma unroll
        for (int l = 0; l < 4; ++l) {
            const uint32_t i = ((qsw >> (8 * l)) & 255u) | ((l < 3 ? qhw << (8 - 4 * l) : qhw >> 4) & 0xF00u);
            const uint2 c = ((const uint2*) tab)[i];
            s.q[2 * l] = (int) c.x;
            s.q[2 * l + 1] = (int) c.y;
        }
        const uint32_t word = ib < 4 ? sw.x : sw.y;
        const uint32_t sc16 = (ib & 2 ? word >> 16 : word) & 0xffffu;
        const int sh = 6 * (ib & 1);
        s.s0 = 2 * (int) ((sc16 >> sh) & 7u) + 1;
        s.s1 = 2 * (int) ((sc16 >> (sh + 3)) & 7u) + 1;
        const uint32_t d16 = (sw.x >> 12 & 0xfu) | ((sw.x >> 24) & 0xf0u) | ((sw.y >> 4) & 0xf00u) | ((sw.y >> 16) & 0xf000u);
        s.d = __half2float(__ushort_as_half((uint16_t) d16)) * 0.125f;
    }
}

__device__ __forceinline__ float warp_sum(float v) {
#pragma unroll
    for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
    return v;
}
__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {   // (ds4_gguf.cu's)
    if (limit > 0.0f) {
        gv = fminf(gv, limit);
        uv = fminf(fmaxf(uv, -limit), limit);
    }
    return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
}
__device__ __forceinline__ bool group(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
    if (a.grp_ptr == nullptr) {
        const unsigned long long p = a.plan_ptr[grp];
        if (p == 0ull) return false;
        blob = (const uint8_t*) p;
        m0 = grp;
        m1 = grp + 1;
        return true;
    }
    if (grp >= *a.n_groups) return false;
    blob = (const uint8_t*) a.grp_ptr[grp];
    m0 = a.grp_start[grp];
    m1 = a.grp_start[grp + 1];
    return true;
}

// ---------------------------------------------------------------- token / window experts
// RW rows a warp, 32 / RW lanes a row (a lane a sub-block a step), their sums over those lanes
template <int RW>
__device__ __forceinline__ float lanes_sum(float v) {
#pragma unroll
    for (int o = 16 / RW; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
    return v;
}
constexpr int kGuRW = 1, kDnRW = 2;   // rows a warp: gate / up (n_embd 5120: 10 sub-blocks a lane), down (2304: 9)

// grid (n_ff / (8 RW), groups): RW rows of the gate and of the up a warp, NE of the group's entries a pass (each
// sub-block decoded once for them) -> h = the clamped swiglu
template <int T, int NE, int RW = kGuRW>
__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a, int groups) {
    constexpr int LPR = 32 / RW;
    constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = LPR / kBPS;   // sub-blocks a block; blocks a lane steps
    static_assert(LPR % kBPS == 0, "a lane keeps one sub-block position");
    __shared__ __align__(16) uint32_t tab[kTabWords<T>];
    __shared__ int es[NE], xr[NE];
    bool loaded = false;   // (the codebook once a block, when its first live item comes: a block of empty groups leaves at once)
    const int lane32 = threadIdx.x & 31;
    const size_t rb = row_bytes_t<T>(a.n_embd);
    const int nsb = a.n_embd / 32, nblk = nsb / kBPS;
    const int ntile = a.n_ff / (kWarps * RW);
    for (int item = kPersist<T> ? blockIdx.x : 0; item < (kPersist<T> ? groups * ntile : 1); item += kPersist<T> ? gridDim.x : 1) {
    const int grp = kPersist<T> ? item / ntile : blockIdx.y, tile = kPersist<T> ? item % ntile : blockIdx.x;
    const uint8_t* blob;
    int m0, m1;
    if (!group(a, grp, blob, m0, m1)) continue;   // (uniform)
    if (!loaded) { load_tab_fast<T>(tab); loaded = true; }
    for (int mm = m0; mm < m1; mm += NE) {
        const int ne = min(NE, m1 - mm);
        __syncthreads();
        if ((int) threadIdx.x < ne) {
            const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
            es[threadIdx.x] = e;
            xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
        }
        __syncthreads();
        {
            const int row = (tile * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;
            const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;
            const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
            const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
            float ag[NE], au[NE];
#pragma unroll
            for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;
            // a lane's sub-blocks: kb = lane + LPR j = 8 blk + ib with ib fixed (the 32-value types: blk = kb)
            for (int blk = blk0; blk < nblk; blk += kStep) {
                const int kb = blk * kBPS + ib0;
                Sub sg, su;
                decode_fast_b<T>(g + (size_t) blk * Ty<T>::B, ib0, tab, sg);
                decode_fast_b<T>(u + (size_t) blk * Ty<T>::B, ib0, tab, su);
#pragma unroll
                for (int i = 0; i < NE; ++i)
                    if (i < ne) {
                        const int4* xp = (const int4*) (a.xq + (size_t) xr[i] * a.n_embd + (size_t) kb * 32);
                        const int4 xa = xp[0], xb = xp[1];
                        const float dx = a.xd[(size_t) xr[i] * nsb + kb];
                        ag[i] = fmaf(dx, sub_dot<T>(sg, xa, xb), ag[i]);
                        au[i] = fmaf(dx, sub_dot<T>(su, xa, xb), au[i]);
                    }
            }
#pragma unroll
            for (int i = 0; i < NE; ++i)
                if (i < ne) {
                    const float gv = lanes_sum<RW>(ag[i]), uv = lanes_sum<RW>(au[i]);
                    if (lane == 0) a.h[(size_t) es[i] * a.n_ff + row] = swiglu(gv, uv, a.limit);
                }
        }
    }
    }
}

// grid (n_embd / (8 RW), groups): RW rows of the down a warp for NE entries a pass, against their int8 h -> part
template <int T, int NE, int RW = kDnRW>
__global__ void __launch_bounds__(kThreads) down_kernel(const Args a, int groups) {
    constexpr int LPR = 32 / RW;
    constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = LPR / kBPS;   // sub-blocks a block; blocks a lane steps
    static_assert(LPR % kBPS == 0, "a lane keeps one sub-block position");
    __shared__ __align__(16) uint32_t tab[kTabWords<T>];
    __shared__ int es[NE];
    bool loaded = false;
    const int lane32 = threadIdx.x & 31;
    const int nsb = a.n_ff / 32, nblk = nsb / kBPS;
    const int ntile = a.n_embd / (kWarps * RW);
    for (int item = kPersist<T> ? blockIdx.x : 0; item < (kPersist<T> ? groups * ntile : 1); item += kPersist<T> ? gridDim.x : 1) {
    const int grp = kPersist<T> ? item / ntile : blockIdx.y, tile = kPersist<T> ? item % ntile : blockIdx.x;
    const uint8_t* blob;
    int m0, m1;
    if (!group(a, grp, blob, m0, m1)) continue;
    if (!loaded) { load_tab_fast<T>(tab); loaded = true; }
    for (int mm = m0; mm < m1; mm += NE) {
        const int ne = min(NE, m1 - mm);
        __syncthreads();
        if ((int) threadIdx.x < ne) es[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
        __syncthreads();
        {
            const int row = (tile * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;
            const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;
            const uint8_t* d = blob + a.tr[2] + (size_t) row * row_bytes_t<T>(a.n_ff);
            float ad[NE];
#pragma unroll
            for (int i = 0; i < NE; ++i) ad[i] = 0.0f;
            for (int blk = blk0; blk < nblk; blk += kStep) {
                const int kb = blk * kBPS + ib0;
                Sub sd;
                decode_fast_b<T>(d + (size_t) blk * Ty<T>::B, ib0, tab, sd);
#pragma unroll
                for (int i = 0; i < NE; ++i)
                    if (i < ne) {
                        const int4* hp = (const int4*) (a.hq + (size_t) es[i] * a.n_ff + (size_t) kb * 32);
                        const int4 ha = hp[0], hb = hp[1];
                        ad[i] = fmaf(a.hd[(size_t) es[i] * nsb + kb], sub_dot<T>(sd, ha, hb), ad[i]);
                    }
            }
#pragma unroll
            for (int i = 0; i < NE; ++i)
                if (i < ne) {
                    const float v = lanes_sum<RW>(ad[i]);
                    if (lane == 0) a.part[(size_t) es[i] * a.n_embd + row] = v;
                }
        }
    }
    }
}

// ---------------------------------------------------------------- the prompt path's sets
constexpr int BM = 64, BN = 64, TM = 4, TN = 4;
static_assert((BM / TM) * (BN / TN) == kThreads, "one output tile a thread");

// F32 FMAs (any device): a block RM rows (64, or 16 for experts of a few rows: a short prompt's) x BN outputs, a 32-value
// sub-block of K a step, the weights dequantised into shared memory (each thread 8 values of a row), each thread RT x TN
// outputs.  (deq8 is the tile loader a tensor-core form takes too.)
template <int T, bool GATE_UP, int RM = BM>
__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {
    constexpr int RT = RM / (kThreads / (BN / TN));
    static_assert(RT >= 1 && RT * (kThreads / (BN / TN)) == RM, "the block's rows");
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    __shared__ float wg[32][BN + 1];
    __shared__ float wu[GATE_UP ? 32 : 1][BN + 1];
    __shared__ float xs[RM][33];
    __shared__ int src[RM];
    const int e = blockIdx.z;
    const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
    const int r0 = lo + blockIdx.x * RM;
    if (r0 >= hi || hi - lo <= a.skip_le) return;   // (uniform; skip_le: the light kernels' experts)
    if (a.skip_ge > 0 && hi - lo >= a.skip_ge) return;   // (the caller's FP16 GEMMs')
    load_tab<T>(tab);
    const int nr = min(RM, hi - r0);
    const int c0 = blockIdx.y * BN;
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    const int K = GATE_UP ? a.n_embd : a.n_ff;
    const size_t rb = row_bytes_t<T>(K);
    const int tid = (int) threadIdx.x;
    if (tid < RM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
    __syncthreads();
    const int ty = tid / (BN / TN), tx = tid % (BN / TN);
    const int tr = tid >> 2, tq = tid & 3;
    float ag[RT][TN], au[RT][TN];
#pragma unroll
    for (int i = 0; i < RT; ++i)
#pragma unroll
        for (int j = 0; j < TN; ++j) ag[i][j] = au[i][j] = 0.0f;
    for (int kb = 0; kb < K / 32; ++kb) {
        float w8[8];
        deq8<T>(blob + a.tr[GATE_UP ? 0 : 2] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);
#pragma unroll
        for (int j = 0; j < 8; ++j) wg[tq * 8 + j][tr] = w8[j];
        if (GATE_UP) {
            deq8<T>(blob + a.tr[1] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);
#pragma unroll
            for (int j = 0; j < 8; ++j) wu[tq * 8 + j][tr] = w8[j];
        }
        for (int i = tid; i < RM * 32; i += kThreads) {
            const int r = i / 32, c = i % 32, sr = src[r];
            float v = 0.0f;
            if (sr >= 0) {
                const size_t at = (size_t) sr * a.ldx + (size_t) kb * 32 + c;
                v = GATE_UP ? a.xf[at] : __half2float(__ushort_as_half(a.xh[at]));
            }
            xs[r][c] = v;
        }
        __syncthreads();
#pragma unroll 8
        for (int kk = 0; kk < 32; ++kk) {
            float xv[RT], gw[TN], uw[TN];
#pragma unroll
            for (int i = 0; i < RT; ++i) xv[i] = xs[ty * RT + i][kk];
#pragma unroll
            for (int j = 0; j < TN; ++j) {
                gw[j] = wg[kk][tx * TN + j];
                if (GATE_UP) uw[j] = wu[kk][tx * TN + j];
            }
#pragma unroll
            for (int i = 0; i < RT; ++i)
#pragma unroll
                for (int j = 0; j < TN; ++j) {
                    ag[i][j] = fmaf(xv[i], gw[j], ag[i][j]);
                    if (GATE_UP) au[i][j] = fmaf(xv[i], uw[j], au[i][j]);
                }
        }
        __syncthreads();
    }
#pragma unroll
    for (int i = 0; i < RT; ++i) {
        const int r = ty * RT + i;
        if (r >= nr) continue;
        const size_t lr = (size_t) (r0 + r);
#pragma unroll
        for (int j = 0; j < TN; ++j) {
            const int c = c0 + tx * TN + j;
            if (GATE_UP) a.h[lr * a.ldo + c] = __half_as_ushort(__float2half_rn(swiglu(ag[i][j], au[i][j], a.limit)));
            else a.y[lr * a.ldo + c] = ag[i][j];
        }
    }
}

#if !defined(STRATA_USE_HIP)
// the same on the tensor cores (sm_70+): 8 warps each 16 rows x 32 outputs, the tiles F16 in shared memory
namespace wm = nvcuda::wmma;
constexpr int kLdT = 40;
template <int T, bool GATE_UP>
__global__ void __launch_bounds__(kThreads) set_tc_kernel(const SetArgs a) {
    struct Tiles {
        __half x[BM][kLdT], wg[BN][kLdT], wu[GATE_UP ? BN : 1][kLdT];
    };
    struct Out {
        float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];
    };
    __shared__ __align__(32) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    __shared__ int src[BM];
    Tiles& Tl = *reinterpret_cast<Tiles*>(smem);
    Out& O = *reinterpret_cast<Out*>(smem);
    const int e = blockIdx.z;
    const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
    const int r0 = lo + blockIdx.x * BM;
    if (r0 >= hi || hi - lo <= a.skip_le) return;   // (uniform; skip_le: the light kernels' experts)
    if (a.skip_ge > 0 && hi - lo >= a.skip_ge) return;   // (the caller's FP16 GEMMs')
    load_tab<T>(tab);
    const int nr = min(BM, hi - r0);
    const int c0 = blockIdx.y * BN;
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    const int K = GATE_UP ? a.n_embd : a.n_ff;
    const size_t rb = row_bytes_t<T>(K);
    const int tid = (int) threadIdx.x, warp = tid >> 5;
    if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
    __syncthreads();
    const int wm_ = warp >> 1, wn = (warp & 1) * 2;
    wm::fragment<wm::accumulator, 16, 16, 16, float> fg[2], fu[2];
    for (int j = 0; j < 2; ++j) {
        wm::fill_fragment(fg[j], 0.0f);
        wm::fill_fragment(fu[j], 0.0f);
    }
    const int tr = tid >> 2, tq = tid & 3;
    for (int kb = 0; kb < K / 32; ++kb) {
        for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {
            float w8[8];
            deq8<T>(blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);
            __half* dst = m == 0 ? &Tl.wg[tr][tq * 8] : &Tl.wu[tr][tq * 8];
#pragma unroll
            for (int j = 0; j < 8; ++j) dst[j] = __float2half_rn(w8[j]);
        }
        const int sr = src[tr];
#pragma unroll
        for (int j = 0; j < 8; ++j) {
            __half v = __float2half_rn(0.0f);
            if (sr >= 0) {
                const size_t at = (size_t) sr * a.ldx + (size_t) kb * 32 + tq * 8 + j;
                v = GATE_UP ? __float2half_rn(a.xf[at]) : __ushort_as_half(a.xh[at]);
            }
            Tl.x[tr][tq * 8 + j] = v;
        }
        __syncthreads();
#pragma unroll
        for (int kk = 0; kk < 32; kk += 16) {
            wm::fragment<wm::matrix_a, 16, 16, 16, __half, wm::row_major> fa;
            wm::load_matrix_sync(fa, &Tl.x[wm_ * 16][kk], kLdT);
#pragma unroll
            for (int j = 0; j < 2; ++j) {
                wm::fragment<wm::matrix_b, 16, 16, 16, __half, wm::col_major> fb;
                wm::load_matrix_sync(fb, &Tl.wg[(wn + j) * 16][kk], kLdT);
                wm::mma_sync(fg[j], fa, fb, fg[j]);
                if (GATE_UP) {
                    wm::load_matrix_sync(fb, &Tl.wu[(wn + j) * 16][kk], kLdT);
                    wm::mma_sync(fu[j], fa, fb, fu[j]);
                }
            }
        }
        __syncthreads();
    }
    for (int j = 0; j < 2; ++j) {
        wm::store_matrix_sync(&O.g[wm_ * 16][(wn + j) * 16], fg[j], BN + 4, wm::mem_row_major);
        if (GATE_UP) wm::store_matrix_sync(&O.u[wm_ * 16][(wn + j) * 16], fu[j], BN + 4, wm::mem_row_major);
    }
    __syncthreads();
    for (int i = tid; i < BM * BN; i += kThreads) {
        const int r = i / BN, c = i % BN;
        if (r >= nr) continue;
        const size_t lr = (size_t) (r0 + r);
        if (GATE_UP) a.h[lr * a.ldo + c0 + c] = __half_as_ushort(__float2half_rn(swiglu(O.g[r][c], O.u[r][c], a.limit)));
        else a.y[lr * a.ldo + c0 + c] = O.g[r][c];
    }
}
#endif

// ---------------------------------------------------------------- GEMVs
constexpr int kMaxJobs = 8;
struct Jobs {
    MvJob j[kMaxJobs];
};
template <int T, int NC>
__global__ void __launch_bounds__(kThreads) mv_kernel(const Jobs J) {
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    load_tab<T>(tab);
    __syncthreads();
    const MvJob& b = J.j[blockIdx.y];
    const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
    if (row >= b.n) return;
    const uint8_t* w = b.w + (size_t) row * b.ld;
    const int nsb = b.k / 32;
    float acc[NC];
#pragma unroll
    for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
    for (int kb = lane; kb < nsb; kb += 32) {
        Sub s;
        decode<T>(w, kb, tab, s);
#pragma unroll
        for (int c = 0; c < NC; ++c)
            if (c < b.ncol) {
                const int4* xp = (const int4*) (b.xq + (size_t) c * b.k + (size_t) kb * 32);
                acc[c] = fmaf(b.xd[(size_t) c * nsb + kb], sub_dot<T>(s, xp[0], xp[1]), acc[c]);
            }
    }
#pragma unroll
    for (int c = 0; c < NC; ++c)
        if (c < b.ncol) {
            const float v = warp_sum(acc[c]);
            if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
        }
}

// ---------------------------------------------------------------- the prompt sets' light experts
// A set's experts of a few rows (<= kLightRows: a prompt of a few hundred tokens spreads ~10 over most of 384 experts)
// in the token kernels' form: each weight sub-block decoded once and dotted with every row's int8 input (rounded per 32
// values as the token path rounds them) - the weights read once at the memory's speed, where a 64-row GEMM tile did 64
// rows' work for them.  Each expert by its own rows: the light kernels take the experts of <= kLightRows rows, the tiles
// the others (SetArgs::skip_le: they skip the light ones) - every expert's sums in one fixed order whatever the set.
// Expert e's row i is input row e kLightRows + i of the scratch, output (local) row bounds[e] - row0 + i.
constexpr int kLightRows = 16;
constexpr int kLightMaxExp = 512;   // a set's experts at most (the scratch: kLightMaxExp kLightRows rows)

// grid (k / 32 / 8, n_exp kLightRows): the light experts' rows of k values (gate / up: F32 x[idx[lr]]; down: F16 h[lr],
// ldx apart) -> int8 q[e kLightRows + i][k], scales d[..][k / 32]
__global__ void __launch_bounds__(kThreads) light_q8_kernel(const float* __restrict__ xf, const uint16_t* __restrict__ xh,
                                                           const int* __restrict__ idx, int ldx, int k,
                                                           const int* __restrict__ bounds, int row0,
                                                           int8_t* __restrict__ q, float* __restrict__ d) {
    const int e = blockIdx.y / kLightRows, i = blockIdx.y % kLightRows;
    const int rows = bounds[e + 1] - bounds[e];
    const int b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
    if (i >= rows || rows > kLightRows || b * 32 >= k) return;
    const int lr = bounds[e] - row0 + i;
    const size_t at = (size_t) b * 32 + lane, qr = (size_t) blockIdx.y;
    const float v = xf != nullptr ? xf[(size_t) idx[lr] * ldx + at] : __half2float(__ushort_as_half(xh[(size_t) lr * ldx + at]));
    float am = fabsf(v);
#pragma unroll
    for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
    const float sc = am / 127.0f;   // (ds4_gguf.cu's q8_quant: the same operations)
    q[qr * k + at] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
    if (lane == 0) d[qr * (k / 32) + b] = sc;
}

// grid (n_ff / 8 or n_embd / 8, n_exp): a light expert's rows, a warp a row of the weights (gate and up, or down), every
// row of the expert a pass -> h16 (the F16 swiglu) or y
// one work item: the rows [tile kWarps, ...) of expert e (see light_kernel)
template <int T, int NE, bool GATE_UP, bool F>
__device__ __forceinline__ void light_item(const SetArgs& a, const int8_t* __restrict__ xq, const float* __restrict__ xd,
                                           const uint32_t* tab, int item, int ntile) {
    constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = 32 / kBPS;   // (as the token kernels: a lane keeps its sub-block position)
    const int K = GATE_UP ? a.n_embd : a.n_ff, N = GATE_UP ? a.n_ff : a.n_embd, nsb = K / 32, nblk = nsb / kBPS;
    const int lane = threadIdx.x & 31;
    const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;
    const int e = item / ntile, row = (item % ntile) * kWarps + (threadIdx.x >> 5);
    const int lo = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];
    if (ne <= 0 || ne > NE || row >= N) return;   // (ne: uniform, the tiles take an expert of more rows)
    const size_t rb = row_bytes_t<T>(K);
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    const size_t q0 = (size_t) e * kLightRows;
    const uint8_t* w0 = blob + a.tr[GATE_UP ? 0 : 2] + (size_t) row * rb;
    const uint8_t* w1 = blob + a.tr[1] + (size_t) row * rb;
    float a0[NE], a1[NE];
#pragma unroll
    for (int i = 0; i < NE; ++i) a0[i] = a1[i] = 0.0f;
    for (int blk = blk0; blk < nblk; blk += kStep) {
        const int kb = blk * kBPS + ib0;
        Sub s0, s1;
        decode_fast_b<T, F>(w0 + (size_t) blk * Ty<T>::B, ib0, tab, s0);
        if (GATE_UP) decode_fast_b<T, F>(w1 + (size_t) blk * Ty<T>::B, ib0, tab, s1);
#pragma unroll
        for (int i = 0; i < NE; ++i)
            if (i < ne) {
                const int4* xp = (const int4*) (xq + (q0 + i) * K + (size_t) kb * 32);
                const int4 xa = xp[0], xb = xp[1];
                const float dx = xd[(q0 + i) * nsb + kb];
                a0[i] = fmaf(dx, sub_dot<T>(s0, xa, xb), a0[i]);
                if (GATE_UP) a1[i] = fmaf(dx, sub_dot<T>(s1, xa, xb), a1[i]);
            }
    }
#pragma unroll
    for (int i = 0; i < NE; ++i)
        if (i < ne) {
            const float v0 = warp_sum(a0[i]);
            const float v1 = GATE_UP ? warp_sum(a1[i]) : 0.0f;
            if (lane == 0) {
                const size_t lr = (size_t) (lo + i);
                if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(v0, v1, a.limit)));
                else a.y[lr * a.ldo + row] = v0;
            }
        }
}

template <int T, int NE, bool GATE_UP>
__global__ void __launch_bounds__(kThreads) light_kernel(const SetArgs a, const int8_t* __restrict__ xq,
                                                                          const float* __restrict__ xd) {
    constexpr bool kF = NE <= 4;   // (the fast codebook only for the few-row kernels, which loop over work items round the grid)
    __shared__ __align__(16) uint32_t tab[kTabWords<T, kF>];
    const int N = GATE_UP ? a.n_ff : a.n_embd, ntile = (N + kWarps - 1) / kWarps;
    bool loaded = false;   // (the codebook when the first live item comes)
    for (int item = kPersist<T> ? blockIdx.x : 0; item < (kPersist<T> ? a.n_exp * ntile : 1); item += kPersist<T> ? gridDim.x : 1) {
        const int e = kPersist<T> ? item / ntile : blockIdx.y, ne = a.bounds[e + 1] - a.bounds[e];
        if (ne <= 0 || ne > NE) continue;   // (uniform)
        if (!loaded) { load_tab_fast<T, kF>(tab); __syncthreads(); loaded = true; }
        light_item<T, NE, GATE_UP, kF>(a, xq, xd, tab, kPersist<T> ? item : (int) (blockIdx.y * ntile + blockIdx.x), ntile);
    }
}

// The release's form, for the experts of more rows (5 .. 16: compute bound, registers tuned): grid (n_ff / 8 or n_embd / 8, n_exp), a light expert's rows, a warp a row of the weights (gate and up, or down), every
// row of the expert a pass -> h16 (the F16 swiglu) or y
template <int T, int NE, bool GATE_UP>
__global__ void __launch_bounds__(kThreads) light_kernel_rows(const SetArgs a, const int8_t* __restrict__ xq,
                                                          const float* __restrict__ xd) {
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    const int e = blockIdx.y;
    const int lo = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];
    if (ne <= 0 || ne > NE) return;   // (uniform: the tiles take an expert of more rows)
    load_tab<T>(tab);
    __syncthreads();
    const int K = GATE_UP ? a.n_embd : a.n_ff, N = GATE_UP ? a.n_ff : a.n_embd, nsb = K / 32;
    const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
    if (row >= N) return;
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    const size_t rb = row_bytes_t<T>(K);
    const uint8_t* w0 = blob + a.tr[GATE_UP ? 0 : 2] + (size_t) row * rb;
    const uint8_t* w1 = blob + a.tr[1] + (size_t) row * rb;
    const size_t q0 = (size_t) e * kLightRows;
    float a0[NE], a1[NE];
#pragma unroll
    for (int i = 0; i < NE; ++i) a0[i] = a1[i] = 0.0f;
    for (int kb = lane; kb < nsb; kb += 32) {
        Sub s0, s1;
        decode<T>(w0, kb, tab, s0);
        if (GATE_UP) decode<T>(w1, kb, tab, s1);
#pragma unroll
        for (int i = 0; i < NE; ++i)
            if (i < ne) {
                const int4* xp = (const int4*) (xq + (q0 + i) * K + (size_t) kb * 32);
                const int4 xa = xp[0], xb = xp[1];
                const float dx = xd[(q0 + i) * nsb + kb];
                a0[i] = fmaf(dx, sub_dot<T>(s0, xa, xb), a0[i]);
                if (GATE_UP) a1[i] = fmaf(dx, sub_dot<T>(s1, xa, xb), a1[i]);
            }
    }
#pragma unroll
    for (int i = 0; i < NE; ++i)
        if (i < ne) {
            const float v0 = warp_sum(a0[i]);
            const float v1 = GATE_UP ? warp_sum(a1[i]) : 0.0f;
            if (lane == 0) {
                const size_t lr = (size_t) (lo + i);
                if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(v0, v1, a.limit)));
                else a.y[lr * a.ldo + row] = v0;
            }
        }
}

bool tc_ok() {   // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps the F32 kernel
#if defined(STRATA_USE_HIP)
    return false;
#else
    // (per device: the cards of a multi-GPU run need not be the same generation)
    static std::atomic<int> state[16];   // 0: unknown, 1: tensor cores, 2: the F32 kernel
    int dev = 0;
    cudaGetDevice(&dev);
    if (dev < 0 || dev >= 16) return false;
    int v = state[dev].load(std::memory_order_relaxed);
    if (v == 0) {
        int major = 0;
        cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
        const char* e = std::getenv("STRATA_DS4_NO_TC");
        v = major >= 7 && !(e != nullptr && e[0] == '1') ? 1 : 2;
        state[dev].store(v, std::memory_order_relaxed);
    }
    return v == 1;
#endif
}

// ---------------------------------------------------------------- dispatch
// F(T) for the type t at run time; false when it is not one of these
#define DS4_IQ_SWITCH(t, F)          \
    switch (t) {                     \
        case kIQ2_XXS: F(kIQ2_XXS); break; \
        case kIQ2_XS: F(kIQ2_XS); break;   \
        case kIQ2_S: F(kIQ2_S); break;     \
        case kIQ3_XXS: F(kIQ3_XXS); break; \
        case kIQ3_S: F(kIQ3_S); break;     \
        case kIQ4_XS: F(kIQ4_XS); break;   \
        case kIQ1_S: F(kIQ1_S); break;     \
        case kIQ1_M: F(kIQ1_M); break;     \
        case kIQ4_NL: F(kIQ4_NL); break;   \
        case kQ4_K: F(kQ4_K); break;       \
        case kQ2_K: F(kQ2_K); break;       \
        case kQ3_K: F(kQ3_K); break;       \
        default: return false;       \
    }

// CUDA: a persistent grid (the resident blocks of the device, split over the groups), each block looping over row tiles with
// its codebook loaded once; HIP keeps the full grid (one tile a block)
template <auto Kernel>
int resident_ctas() {   // (a static a kernel: the occupancy differs by the codebook's shared memory)
#if defined(STRATA_USE_HIP) || defined(STRATA_METAL)
    return 1 << 30;
#else
    static std::atomic<int> cache[16];
    int dev = 0;
    cudaGetDevice(&dev);
    if (dev < 0 || dev >= 16) return 1 << 30;
    int v = cache[dev].load(std::memory_order_relaxed);
    if (v == 0) {
        int sms = 0, per = 0;
        cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
        cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per, Kernel, kThreads, 0);
        if (const char* e = std::getenv("DS4_IQ_MAX_SMS")) sms = std::min(sms, std::max(1, std::atoi(e)));   // (benchmarking: a smaller card)
        if (const char* e = std::getenv("DS4_IQ_CTA_PER_SM")) per = std::min(per, std::max(1, std::atoi(e)));
        v = std::max(1, sms * std::max(per, 1));
        cache[dev].store(v, std::memory_order_relaxed);
    }
    return v;
#endif
}
// The grid: CUDA takes the device's resident blocks (at most one a work item) and every block loops over the items (group, row
// tile) round the grid (group by group, so that the blocks work on one expert's rows together): whichever groups are active,
// they share all the blocks, and a block loads its codebook once.  HIP: a block an item (the full grid).
template <int T>
inline int items_grid(int items, int resident) { return std::max(1, std::min(items, resident)); }
template <int T, int NE>
void launch_gu(const Args& a, int groups, cudaStream_t s) {
    const int tiles = a.n_ff / (kWarps * kGuRW);
    if constexpr (kPersist<T>)
        strata_ds4_launch(gate_up_kernel<T, NE>, items_grid<T>(groups * tiles, resident_ctas<gate_up_kernel<T, NE, kGuRW>>()), kThreads, 0, s, a, groups);
    else
        strata_ds4_launch(gate_up_kernel<T, NE>, dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s, a, groups);
}
template <int T, int NE>
void launch_dn(const Args& a, int groups, cudaStream_t s) {
    const int tiles = a.n_embd / (kWarps * kDnRW);
    if constexpr (kPersist<T>)
        strata_ds4_launch(down_kernel<T, NE>, items_grid<T>(groups * tiles, resident_ctas<down_kernel<T, NE, kDnRW>>()), kThreads, 0, s, a, groups);
    else
        strata_ds4_launch(down_kernel<T, NE>, dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s, a, groups);
}
template <int T>
void gu_ne(const Args& a, int groups, int ne, cudaStream_t s) {
    if (ne <= 1) launch_gu<T, 1>(a, groups, s);
    else if (ne <= 4) launch_gu<T, 4>(a, groups, s);
    else if (ne <= 8) launch_gu<T, 8>(a, groups, s);
    else launch_gu<T, 16>(a, groups, s);
}
template <int T>
void dn_ne(const Args& a, int groups, int ne, cudaStream_t s) {
    if (ne <= 1) launch_dn<T, 1>(a, groups, s);
    else if (ne <= 4) launch_dn<T, 4>(a, groups, s);
    else if (ne <= 8) launch_dn<T, 8>(a, groups, s);
    else launch_dn<T, 16>(a, groups, s);
}
#include "ds4_iq_hip.inc"
template <int T, bool GU>
void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {
#if defined(STRATA_USE_HIP)
    if (hipk::launch<T, GU>(a, s)) return;   // (gfx12, STRATA_DS4_HIPK=1: the matrix-core tiles)
#endif
#if !defined(STRATA_USE_HIP)
    if (tc_ok()) {
        strata_ds4_launch(set_tc_kernel<T, GU>, grid, kThreads, 0, s, a);
        return;
    }
#endif
    if (a.max_rows <= 16) {   // (a block of 64 rows would do 64 rows' FMAs for the few there are)
        grid.x = (unsigned) ((a.max_rows + 15) / 16);
        strata_ds4_launch(set_kernel<T, GU, 16>, grid, kThreads, 0, s, a);
        return;
    }
    strata_ds4_launch(set_kernel<T, GU>, grid, kThreads, 0, s, a);
}
// the light sets' int8 rows, per stream (kLightMaxExp kLightRows rows of k)
struct LightBuf {
    int8_t* q = nullptr;
    float* d = nullptr;
    size_t cap = 0;
};
std::mutex g_lmu;
std::map<std::pair<int, cudaStream_t>, LightBuf> g_light;   // (device, stream): the default stream 0 is one handle on every device
LightBuf* light_buf(cudaStream_t s, int k) {
    std::lock_guard<std::mutex> lk(g_lmu);
    int dev = 0;
    cudaGetDevice(&dev);
    LightBuf& b = g_light[{dev, s}];
    const size_t need = (size_t) kLightMaxExp * kLightRows * (size_t) std::max(k, 8192);
    if (b.cap < need) {
        if (b.q != nullptr) cudaFree(b.q);
        if (b.d != nullptr) cudaFree(b.d);
        b = LightBuf{};
        if (cudaMalloc(&b.q, need) != cudaSuccess || cudaMalloc(&b.d, need / 32 * sizeof(float)) != cudaSuccess) {
            std::fprintf(stderr, "ds4 iq: the light sets' scratch (%zu MB) failed\n", need >> 20);
            return nullptr;
        }
        b.cap = need;
    }
    return &b;
}
bool light_on() {   // STRATA_DS4_NO_LIGHT=1: every set through the GEMM tiles
    static const bool on = [] {
        const char* v = std::getenv("STRATA_DS4_NO_LIGHT");
        return !(v != nullptr && v[0] == '1');
    }();
    return on;
}
template <int T, bool GU>
void launch_light(const SetArgs& a, const LightBuf& b, cudaStream_t s) {
    const int tiles = ((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps;
    if (a.max_rows <= 4) {   // few rows: work items round a persistent grid (the codebook per block is then the cost)
        if constexpr (kPersist<T>)
            strata_ds4_launch(light_kernel<T, 4, GU>, items_grid<T>(a.n_exp * tiles, resident_ctas<light_kernel<T, 4, GU>>()), kThreads, 0, s, a, b.q, b.d);
        else   // (a small codebook: the release's kernel, which is the faster there)
            strata_ds4_launch(light_kernel_rows<T, 4, GU>, dim3((unsigned) tiles, (unsigned) a.n_exp), kThreads, 0, s, a, b.q, b.d);
    } else {   // more rows: compute bound, the release's kernel and grid
        const dim3 grid((unsigned) tiles, (unsigned) a.n_exp);
        if (a.max_rows <= 8) strata_ds4_launch(light_kernel_rows<T, 8, GU>, grid, kThreads, 0, s, a, b.q, b.d);
        else strata_ds4_launch(light_kernel_rows<T, kLightRows, GU>, grid, kThreads, 0, s, a, b.q, b.d);
    }
}
bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {
    const int K = gu ? a.n_embd : a.n_ff;
    LightBuf* b = light_buf(s, K);
    if (b == nullptr) return false;
    const dim3 gq((unsigned) ((K / 32 + kWarps - 1) / kWarps), (unsigned) (a.n_exp * kLightRows));
    strata_ds4_launch(light_q8_kernel, gq, kThreads, 0, s, gu ? a.xf : nullptr, gu ? nullptr : a.xh, a.idx, a.ldx, K, a.bounds,
                                            a.row0, b->q, b->d);
    if (gu) {
        if (a.tg == kMXFP4) {
            launch_light<kMXFP4, true>(a, *b, s);
            return true;
        }
#define LG_(T) launch_light<T, true>(a, *b, s)
        DS4_IQ_SWITCH(a.tg, LG_)
#undef LG_
    } else {
        if (a.td == kMXFP4) {
            launch_light<kMXFP4, false>(a, *b, s);
            return true;
        }
#define LD_(T) launch_light<T, false>(a, *b, s)
        DS4_IQ_SWITCH(a.td, LD_)
#undef LD_
    }
    return true;
}

template <int T>
void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
    const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);
    if (max_c <= 1) strata_ds4_launch(mv_kernel<T, 1>, grid, kThreads, 0, s, J);
    else if (max_c <= 4) strata_ds4_launch(mv_kernel<T, 4>, grid, kThreads, 0, s, J);
    else strata_ds4_launch(mv_kernel<T, 16>, grid, kThreads, 0, s, J);
}


// ---------------------------------------------------------------- whole matrices as 16-bit rows (the chunked prompt's GEMMs)
// n rows of k values from `raw` (ld bytes a row): one thread a 32-value sub-block, the sub-block's codes and scales from decode<>
// (value = d s q - m of its half, ggml's dequantize_row_*), rounded to BF16 (bf16) or F16.
// The 32 values go through shared memory and are written coalesced (a warp's 32 threads cover 1024 contiguous values): a
// thread writing its own 32 values straight to dst wrote 2 bytes every 64 (a warp's store touches 32 cache lines), which
// measured 0.5 G weights/s on the R9700 - the 8K prompt's q_b stage 4 s.  DS4_DQ_OLD=1 keeps the old kernel for A/B.
template <int T>
__global__ void __launch_bounds__(kThreads) dq_rows_kernel2(const uint8_t* __restrict__ raw, size_t ld, int k, int n,
                                                            uint16_t* __restrict__ dst, bool bf16) {
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    __shared__ uint16_t sm[kThreads * 32];
    load_tab<T>(tab);
    __syncthreads();
    const int nsb = k / 32;
    const size_t total = (size_t) n * (size_t) nsb;   // 32-value sub-blocks
    const size_t vals = (size_t) n * (size_t) k;
    for (size_t base = (size_t) blockIdx.x * kThreads; base < total; base += (size_t) gridDim.x * kThreads) {
        const size_t i = base + threadIdx.x;
        uint16_t* o = sm + threadIdx.x * 32;
        if (i < total) {
            const size_t r = i / (size_t) nsb;
            const int kb = (int) (i - r * (size_t) nsb);
            Sub sb;
            sb.m0 = sb.m1 = 0.0f;
            decode<T>(raw + r * ld, kb, tab, sb);
            const float d0 = __fmul_rn(sb.d, (float) sb.s0), d1 = __fmul_rn(sb.d, (float) sb.s1);
#pragma unroll
            for (int j = 0; j < 32; ++j) {
                const int q = (int) (int8_t) ((uint32_t) sb.q[j >> 2] >> (8 * (j & 3)));
                const float v = j < 16 ? __fsub_rn(__fmul_rn(d0, (float) q), sb.m0) : __fsub_rn(__fmul_rn(d1, (float) q), sb.m1);
                o[j] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));
            }
        } else {
#pragma unroll
            for (int j = 0; j < 32; ++j) o[j] = 0;
        }
        __syncthreads();
        // sm[m] is the block's value m (the sub-blocks are contiguous in the row-major output, rows included): write it
        // a thread's 32 values apart, so a warp's stores are 512 contiguous bytes each
        const size_t go = base * 32;
        uint16_t* d = dst + go;
#pragma unroll
        for (int j = 0; j < 32; ++j) {
            const size_t m = (size_t) j * kThreads + threadIdx.x;
            if (go + m < vals) d[m] = sm[m];
        }
        __syncthreads();
    }
}

template <int T>
__global__ void __launch_bounds__(kThreads) dq_rows_kernel(const uint8_t* __restrict__ raw, size_t ld, int k, int n,
                                                           uint16_t* __restrict__ dst, bool bf16) {
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    load_tab<T>(tab);
    __syncthreads();
    const int nsb = k / 32;
    const size_t total = (size_t) n * (size_t) nsb;
    // A warp stages its 32 sub-blocks in shared through contiguous uint4 loads and decodes from there: a lane reading its own
    // 176-byte block at that stride touched 32 scattered sectors per instruction - the dequant's remaining cost once the stores
    // were vectorised (15.4 s of a 4K prompt, 23 GB of quantised input at ~1.5 GB/s).  Each lane decodes with the index relative
    // to the staged buffer (kb - kb0), which is exactly equivalent because kb0 is a multiple of the 8 sub-blocks in a block.
    constexpr int kQKv = kQK<T>;
    constexpr int kSpan = (kQKv == 32 ? 32 : 4) * Ty<T>::B;      // bytes of a warp's 32 sub-blocks
    constexpr bool kCanStage = Ty<T>::TAB == 0;                  // (a codebook table already fills shared for the i-quants)
    const bool staged = (nsb % 32 == 0) && kCanStage;
    __shared__ uint4 sin4[kCanStage ? (kThreads / 32) * 44 : 1];
    const int lane32 = threadIdx.x & 31, wq = threadIdx.x >> 5;
    for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {
        const size_t r = i / (size_t) nsb;
        const int kb = (int) (i - r * (size_t) nsb);
        Sub sb;
        sb.m0 = sb.m1 = 0.0f;
        if (staged) {
            const size_t wbase = i - (size_t) lane32;   // (uniform in the warp: blockDim and the stride are multiples of 32)
            const size_t r0 = wbase / (size_t) nsb;
            const int kb0 = (int) (wbase - r0 * (size_t) nsb);
            const uint8_t* srcp = raw + r0 * ld + (size_t) (kb0 * Ty<T>::B) / (kQKv / 32);
            uint4* st = sin4 + wq * 44;
            for (int c = lane32; c < kSpan / 16; c += 32) st[c] = ((const uint4*) srcp)[c];
            __threadfence_block();
            __syncwarp();
            decode<T>((const uint8_t*) st, kb - kb0, tab, sb);
        } else
            decode<T>(raw + r * ld, kb, tab, sb);
        const float d0 = __fmul_rn(sb.d, (float) sb.s0), d1 = __fmul_rn(sb.d, (float) sb.s1);
        uint16_t* o = dst + r * (size_t) k + (size_t) kb * 32;
        // (the 32 halves packed into four 16-byte stores: 32 two-byte stores a lane, each lane 64 bytes from the next, write
        // ~16x the bytes they carry in DRAM sectors - measured as the dense dequant's whole cost: 6.6 GB to FP16 in 20.0 s)
        uint32_t hw[16];
#pragma unroll
        for (int j = 0; j < 16; ++j) {
            const int q0 = (int) (int8_t) ((uint32_t) sb.q[j >> 1] >> (8 * ((2 * j) & 3)));
            const int q1 = (int) (int8_t) ((uint32_t) sb.q[j >> 1] >> (8 * ((2 * j + 1) & 3)));
            const bool lo = (2 * j) < 16;
            const float v0 = lo ? __fsub_rn(__fmul_rn(d0, (float) q0), sb.m0) : __fsub_rn(__fmul_rn(d1, (float) q0), sb.m1);
            const bool lo1 = (2 * j + 1) < 16;
            const float v1 = lo1 ? __fsub_rn(__fmul_rn(d0, (float) q1), sb.m0) : __fsub_rn(__fmul_rn(d1, (float) q1), sb.m1);
            const uint32_t h0 = bf16 ? strata::ds4::bf16_rn(v0) : __half_as_ushort(__float2half_rn(v0));
            const uint32_t h1 = bf16 ? strata::ds4::bf16_rn(v1) : __half_as_ushort(__float2half_rn(v1));
            hw[j] = h0 | (h1 << 16);
        }
        uint4* o4 = (uint4*) o;
        o4[0] = make_uint4(hw[0], hw[1], hw[2], hw[3]);
        o4[1] = make_uint4(hw[4], hw[5], hw[6], hw[7]);
        o4[2] = make_uint4(hw[8], hw[9], hw[10], hw[11]);
        o4[3] = make_uint4(hw[12], hw[13], hw[14], hw[15]);
    }
}

}  // namespace

// Which dense dequantizer: dq_rows_kernel2 (the coalesced transpose, 0.2.7 DS Support: 566 -> 592 ppts on the R9700, ids identical) or the
// row-per-lane dq_rows_kernel, which main's fd33cb6ee gave 16-byte stores and a staged input: the Strix Halo (gfx11) 4K
// prompt 79.5 -> 88.5 tok/s with it, so a gfx11 device defaults to it.  DS4_DQ_OLD=1 forces it anywhere, DS4_DQ_OLD=0 forces kernel2
// anywhere.  The detection is the device's gcnArchName (HIP only; a CUDA build has no gfx11).  Decided once per device.
static bool dq_use_old_kernel() {
    const char* dq_env = std::getenv("DS4_DQ_OLD");
    const int env = dq_env != nullptr && *dq_env != '\0' ? (std::atoi(dq_env) != 0 ? 1 : 0) : -1;
    const bool v026 = strata::ds4::defaults_026();   // DS4_DEFAULTS=026: 0.2.6's row-per-lane dequantizer (the same values, bit for bit)
    if (env >= 0 || v026) return strata::ds4::dq_old_default(env, false, v026);
#if defined(STRATA_USE_HIP)
    static std::atomic<int> cache[64];   // 0 unknown, 1 no, 2 yes
    int dev = 0;
    cudaGetDevice(&dev);
    if (dev < 0 || dev >= 64) return false;
    int c = cache[dev].load();
    if (c == 0) {
        cudaDeviceProp prop{};
        const bool yes = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0;
        c = yes ? 2 : 1;
        cache[dev].store(c);
    }
    return c == 2;
#else
    return false;
#endif
}

bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {
    if (raw == nullptr || k % 256 != 0 || n <= 0) return false;
    const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);
    const bool old = dq_use_old_kernel();
#define DS4_DQ(T) do {                                                                                    \
        if (old)                                                                                          \
            strata_ds4_launch(dq_rows_kernel<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16);      \
        else                                                                                              \
            strata_ds4_launch(dq_rows_kernel2<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16);     \
    } while (0)
    switch (type) {
        case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;
        case kIQ2_XS: DS4_DQ(kIQ2_XS); break;
        case kIQ2_S: DS4_DQ(kIQ2_S); break;
        case kIQ3_XXS: DS4_DQ(kIQ3_XXS); break;
        case kIQ3_S: DS4_DQ(kIQ3_S); break;
        case kIQ4_XS: DS4_DQ(kIQ4_XS); break;
        case kIQ1_S: DS4_DQ(kIQ1_S); break;
        case kIQ1_M: DS4_DQ(kIQ1_M); break;
        case kQ4_K: DS4_DQ(kQ4_K); break;
        case kQ5_K: DS4_DQ(kQ5_K); break;
        case kQ6_K: DS4_DQ(kQ6_K); break;
        case kQ2_K: DS4_DQ(kQ2_K); break;
        case kQ3_K: DS4_DQ(kQ3_K); break;
        default: return false;
    }
#undef DS4_DQ
    return cudaGetLastError() == cudaSuccess;
}

bool supported(int t) {
#if defined(STRATA_METAL)
    // The MSL kernels (metal/src/engine/msl/ds4_iq.metal) implement the 2.0-bit types plus IQ1_M; IQ1_S and IQ4_NL
    // have CUDA kernels here but no Metal ones.  Saying so makes those layers "cpu only" (ds4_model.cu, expert_layout),
    // so their experts stay on the CPU lane instead of stopping in "kernel launch with no Metal dispatch".
    if (t == kIQ1_S || t == kIQ4_NL) return false;
#endif
    // the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)
    switch (t) {
        case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:
        case kIQ1_S: case kIQ1_M: case kIQ4_NL:
        case kQ4_K: case kQ5_K: case kQ6_K: case kQ2_K: case kQ3_K:
            return true;
        default:
            return is_tq(t);
    }
}

bool known(int t) { return block_bytes(t) > 0; }

int block_values(int t) {
    if (t == kIQ4_NL) return 32;   // (ggml's QK4_NL: the only type here whose block is not 256 values)
    return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0;
}

int block_bytes(int t) {
    if (is_tq(t)) return 2 + 4 * tq_kbits(t);
    switch (t) {
        case kIQ2_XXS: return Ty<kIQ2_XXS>::B;
        case kIQ2_XS: return Ty<kIQ2_XS>::B;
        case kIQ2_S: return Ty<kIQ2_S>::B;
        case kIQ3_XXS: return Ty<kIQ3_XXS>::B;
        case kIQ3_S: return Ty<kIQ3_S>::B;
        case kIQ4_XS: return Ty<kIQ4_XS>::B;
        case kQ4_K: return Ty<kQ4_K>::B;
        case kQ5_K: return Ty<kQ5_K>::B;
        case kQ6_K: return Ty<kQ6_K>::B;
        case kQ2_K: return Ty<kQ2_K>::B;
        case kQ3_K: return Ty<kQ3_K>::B;
        case kIQ1_S: return Ty<kIQ1_S>::B;
        case kIQ1_M: return Ty<kIQ1_M>::B;
        case kIQ4_NL: return Ty<kIQ4_NL>::B;
        default: return 0;
    }
}

const char* name(int t) {
    switch (t) {
        case kTQK6: return "TQK6";
        case kTQK7: return "TQK7";
        case kTQK5X: return "TQK5X";
        case kTQK4X: return "TQK4X";
        case kTQK5T: return "TQK5T";
        case kTQK4T: return "TQK4T";
        case kIQ2_XXS: return "IQ2_XXS";
        case kIQ2_XS: return "IQ2_XS";
        case kIQ2_S: return "IQ2_S";
        case kIQ3_XXS: return "IQ3_XXS";
        case kIQ3_S: return "IQ3_S";
        case kIQ4_XS: return "IQ4_XS";
        case kQ4_K: return "Q4_K";
        case kQ5_K: return "Q5_K";
        case kQ6_K: return "Q6_K";
        case kQ2_K: return "Q2_K";
        case kQ3_K: return "Q3_K";
        case kIQ1_S: return "IQ1_S";
        case kIQ1_M: return "IQ1_M";
        case kIQ4_NL: return "IQ4_NL";
        default: return "?";
    }
}

bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
#if defined(STRATA_METAL)
    // gate_up_kernel on the Metal dispatch (the prompt / window path's expert GEMM); the launch below is the fallback
    if (::strata::dsv41::metal_host::iq_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,
                                           a.n_ff, a.limit, a.xq, a.xd, a.h, a.tg, groups))
        return true;
#endif
    if (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);
    if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;
#define GU_(T) gu_ne<T>(a, groups, ne_max, s)
    DS4_IQ_SWITCH(a.tg, GU_)
#undef GU_
    return true;
}

bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
#if defined(STRATA_METAL)
    // down_kernel on the Metal dispatch; the launch below is the fallback
    if (::strata::dsv41::metal_host::iq_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,
                                           a.n_ff, a.hq, a.hd, a.part, a.td, groups))
        return true;
#endif
    if (is_tq(a.td)) return tq::down(a, groups, ne_max, s);
    if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;
#define DN_(T) dn_ne<T>(a, groups, ne_max, s)
    DS4_IQ_SWITCH(a.td, DN_)
#undef DN_
    return true;
}

bool light_gate_up(const SetArgs& a, cudaStream_t s) {
    if (is_tq(a.tg)) return false;   // (the trellis sets take the tiles for every expert)
    if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;
    return light_set(a, true, s);
}
bool light_down(const SetArgs& a, cudaStream_t s) {
    if (is_tq(a.td)) return false;
    if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;
    return light_set(a, false, s);
}

bool hipk_force(int mode) {
#if defined(STRATA_USE_HIP)
    hipk::force_mode().store(mode, std::memory_order_relaxed);
    return mode < 0 || hipk::ok_device() || mode == 0;
#else
    (void) mode;
    return false;
#endif
}
long long hipk_launches() {
#if defined(STRATA_USE_HIP)
    return hipk::launches().load(std::memory_order_relaxed);
#else
    return -1;
#endif
}

bool set_gate_up(const SetArgs& a, cudaStream_t s) {
#if defined(STRATA_METAL)
    if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_gu(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xf, a.ldx, a.idx, a.h, a.ldo, a.tg, a.td))
        return true;
#endif
    if (is_tq(a.tg)) return tq::set_gate_up(a, s);
    if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;
    SetArgs t = a;
    if (light_gate_up(a, s)) {   // the experts of a few rows; the tiles the others
        if (a.max_rows <= kLightRows) return true;
        t.skip_le = kLightRows;
    }
    const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (a.n_ff / BN), (unsigned) a.n_exp);
#define SG_(T) launch_set<T, true>(t, grid, s)
    DS4_IQ_SWITCH(a.tg, SG_)
#undef SG_
    return true;
}

bool set_down(const SetArgs& a, cudaStream_t s) {
#if defined(STRATA_METAL)
    if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_dn(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xh, a.ldx, a.y, a.ldo, a.tg, a.td))
        return true;
#endif
    if (is_tq(a.td)) return tq::set_down(a, s);
    if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;
    SetArgs t = a;
    if (light_down(a, s)) {
        if (a.max_rows <= kLightRows) return true;
        t.skip_le = kLightRows;
    }
    const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (a.n_embd / BN), (unsigned) a.n_exp);
#define SD_(T) launch_set<T, false>(t, grid, s)
    DS4_IQ_SWITCH(a.td, SD_)
#undef SD_
    return true;
}

// ---------------------------------------------------------------- the busy experts' FP16 weights (f16_rows)
// 8 values a thread: row r's sub-block kb, piece p - deq8's values rounded as the tensor-core tiles round them
template <int T>
__global__ void __launch_bounds__(kThreads) f16_rows_kernel(const uint8_t* __restrict__ w, int k, int n,
                                                             uint16_t* __restrict__ dst, int ldd) {
    __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
    load_tab<T>(tab);
    __syncthreads();
    const int nq = k / 8;
    const size_t rb = row_bytes_t<T>(k);
    for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < (long long) n * nq;
         i += (long long) gridDim.x * blockDim.x) {
        const int r = (int) (i / nq), q = (int) (i % nq);
        float w8[8];
        deq8<T>(w + (size_t) r * rb, q >> 2, q & 3, tab, w8);
        union {
            uint4 v;
            uint16_t h[8];
        } u;
#pragma unroll
        for (int j = 0; j < 8; ++j) u.h[j] = __half_as_ushort(__float2half_rn(w8[j]));
        *reinterpret_cast<uint4*>(dst + (size_t) r * ldd + (size_t) q * 8) = u.v;
    }
}
__global__ void __launch_bounds__(kThreads) gather_f16_kernel(const uint16_t* __restrict__ x, int ldx,
                                                               const int* __restrict__ idx, int n, int k,
                                                               uint16_t* __restrict__ dst) {
    const int nq = k / 8;
    for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < (long long) n * nq;
         i += (long long) gridDim.x * blockDim.x) {
        const int r = (int) (i / nq), q = (int) (i % nq);
        *reinterpret_cast<uint4*>(dst + (size_t) r * k + (size_t) q * 8) =
            *reinterpret_cast<const uint4*>(x + (size_t) idx[r] * ldx + (size_t) q * 8);
    }
}
__global__ void __launch_bounds__(kThreads) swiglu_f16_kernel(const float* __restrict__ gu, int ld, int n, int ff,
                                                               float limit, uint16_t* __restrict__ h, int ldh) {
    for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < (long long) n * ff;
         i += (long long) gridDim.x * blockDim.x) {
        const int r = (int) (i / ff), c = (int) (i % ff);
        const float* g = gu + (size_t) r * ld;
        h[(size_t) r * ldh + c] = __half_as_ushort(__float2half_rn(swiglu(g[c], g[ff + c], limit)));
    }
}
static unsigned grid_for(long long items) {
    return (unsigned) std::max<long long>(1, std::min<long long>(4096, (items + kThreads - 1) / kThreads));
}
bool f16_rows_ok(int type) {   // (the DS4_IQ_SWITCH list)
    switch (type) {
        case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS: case kIQ1_S: case kIQ1_M:
        case kIQ4_NL: case kQ4_K: case kQ2_K: case kQ3_K:
            return true;
        default:
            return false;
    }
}
bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s) {
    if (k % 256 != 0 || n < 1 || ldd % 8 != 0 || ((uintptr_t) dst & 15) != 0) return false;
    const unsigned blocks = grid_for((long long) n * (k / 8));
#define FR_(T) strata_ds4_launch(f16_rows_kernel<T>, blocks, kThreads, 0, s, w, k, n, dst, ldd)
    DS4_IQ_SWITCH(type, FR_)
#undef FR_
    return cudaGetLastError() == cudaSuccess;
}
void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s) {
    strata_ds4_launch(gather_f16_kernel, grid_for((long long) n * (k / 8)), kThreads, 0, s, x, ldx, idx, n, k, dst);
}
void swiglu_f16(const float* gu, int ld, int n, int ff, float limit, uint16_t* h, int ldh, cudaStream_t s) {
    strata_ds4_launch(swiglu_f16_kernel, grid_for((long long) n * ff), kThreads, 0, s, gu, ld, n, ff, limit, h, ldh);
}

bool mv(const MvJob* jobs, int n, cudaStream_t s) {
#if defined(STRATA_METAL)
    if (n > 0 && n <= 64 && ::strata::dsv41::metal_host::iq_mv((const void*) jobs, n)) return true;
#endif
    {   // the trellis jobs go to their kernels, the others on
        MvJob tqj[64], oth[64];
        int nt = 0, no = 0;
        if (n > 64) return false;
        for (int i = 0; i < n; ++i) (is_tq(jobs[i].type) ? tqj[nt++] : oth[no++]) = jobs[i];
        if (nt > 0) {
            if (!tq::mv(tqj, nt, s)) return false;
            if (no == 0) return true;
            return mv(oth, no, s);
        }
    }
    // the jobs by type, up to kMaxJobs a launch
    bool done[64] = {};
    if (n > 64) return false;
    for (int i = 0; i < n; ++i)
        while (!done[i]) {
            const int t = jobs[i].type;
            Jobs J{};
            int nj = 0, max_n = 0, max_c = 1;
            for (int k = i; k < n && nj < kMaxJobs; ++k) {
                if (done[k] || jobs[k].type != t) continue;
                if (jobs[k].k % 256 != 0 || jobs[k].ncol < 1 || jobs[k].ncol > 16) return false;
                J.j[nj++] = jobs[k];
                max_n = std::max(max_n, jobs[k].n);
                max_c = std::max(max_c, jobs[k].ncol);
                done[k] = true;
            }
#define MV_(T) mv_nc<T>(J, nj, max_n, max_c, s)
            switch (t) {   // the experts' types and the dense matrices' 5- and 6-bit ones
                case kQ5_K: MV_(kQ5_K); break;
                case kQ6_K: MV_(kQ6_K); break;
                default: DS4_IQ_SWITCH(t, MV_)
            }
#undef MV_
        }
    return true;
}

}  // namespace strata::ds4::gguf::iq