Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
75.7 KB
Raw Blame
// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/kernels/cuda/exl3.cu - the EXL3 device kernels (the format: include/strata/kernels/exl3.hpp).
//
// The codebooks and the trellis window arithmetic follow exllamav3 (MIT, turboderp - exllamav3_ext/quant/
// codebook.cuh, exl3_dq.cuh, hadamard_inner.cuh), and so does decoding a tile straight into tensor-core fragments
// (its small-m GEMV and GEMM); the kernels around them are shaped for this engine's decode:
//
//   * a block owns 128 outputs (8 tiles - exactly one output rotation block) over a slice of at most 4096 inputs;
//     warp w takes tile column w through every 16-row slice, decoding 8 weights per lane (one fragment: 2 columns x
//     4 rows) - on sm_80 and later straight into one mma.m16n8k16 a tile with up to 8 activations as its columns,
//     F32 accumulation; before sm_80 (and on HIP) F32 FMAs on the CUDA cores;
//   * the input rotation runs per block, 128 inputs at a time (F32 or q8_1 in, FP16 in shared memory), so the
//     engine's activation buffers are used as they are; the next 128 inputs' tile words load while these decode;
//   * a matrix with more than 4096 inputs, or too few output blocks to fill the device, splits its inputs over
//     blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in
//     split order (deterministic) and runs the output rotation.
//
// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy
// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s).  The
// prompt path below is this file's on every GPU.
#include "../ds4_launch.h"   // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#if defined(STRATA_METAL)
#include <ds4_metal_host.h>
#endif
#include "strata/dsv41/exl3.hpp"

#include <cuda_fp16.h>
#include <cuda_runtime.h>

#include <algorithm>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <mutex>
#include <utility>

#if !defined(STRATA_USE_HIP)
namespace strata::dsv41k::exl3::legacy {   // exl3_legacy.cu: v1.1.8's decode kernels
int launch_errors();
bool prepare(cudaStream_t s);
bool several_ok();
bool mv(const MvJob* jobs, int n, cudaStream_t s);
void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s);
void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);
bool win_experts(const WinArgs& a, cudaStream_t s);
}  // namespace strata::dsv41k::exl3::legacy
#define EXL3_LEGACY(call) \
    if (use_legacy()) return legacy::call
#else
#define EXL3_LEGACY(call) (void) 0
#endif

namespace strata::dsv41k::exl3 {
namespace {

constexpr int kThreads = 256, kWarps = 8;
constexpr int kCols = 128;                // outputs per block: 8 tiles, one rotation block
constexpr int kMaxKR = 4096;              // inputs per block at most (mv's split)
// a routed expert's widths at most (moe_* / win_*: a block streams its inputs through block_gemv's two 128-value chunk
// buffers, so the shared memory does not grow with them; DeepSeek-V4.1's n_embd is 5120)
constexpr int kMaxMoeK = 8192;
constexpr float kHad = 0.088388347648f;   // 1 / sqrt(128)

struct Q81 {   // the engine's q8_1 block (ggml block_q8_1): d, sum, 32 int8
    __half2 ds;
    int8_t qs[32];
};

std::atomic<int> g_launch_errors{0};
thread_local const int* g_skip = nullptr;   // set_skip: the calling thread's flag (nullptr: none)
void launch_check(const char* what) {
    const cudaError_t e = cudaGetLastError();
    if (e != cudaSuccess) {
        std::fprintf(stderr, "exl3 %s: %s\n", what, cudaGetErrorString(e));
        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
    }
}

// ---------------------------------------------------------------- the codebooks
__device__ __forceinline__ uint32_t bytesum_plus(uint32_t x, uint32_t acc) {
#if defined(STRATA_USE_HIP)
    return acc + (x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24);
#else
    return __dp4a(x, 0x01010101u, acc);
#endif
}

// two states -> two values, FP16 arithmetic as exllamav3's decode_3inst_2 (the tensor-core kernels take the half2)
template <int cb>
__device__ __forceinline__ __half2 decode2h(uint32_t s0, uint32_t s1) {
    if constexpr (cb == 2) {
        const uint32_t a = bytesum_plus(s0 * 0x83DCD12Du, 0x6400u);
        const uint32_t b = bytesum_plus(s1 * 0x83DCD12Du, 0x6400u);
        const __half2 h = __halves2half2(__ushort_as_half((unsigned short) a), __ushort_as_half((unsigned short) b));
        return __hfma2(h, __half2half2(__ushort_as_half(0x1eee)), __half2half2(__ushort_as_half(0xc931)));
    } else {
        uint32_t x0 = cb == 1 ? s0 * 0xCBAC1FEDu : s0 * 89226354u + 64248484u;
        uint32_t x1 = cb == 1 ? s1 * 0xCBAC1FEDu : s1 * 89226354u + 64248484u;
        x0 = (x0 & 0x8fff8fffu) ^ 0x3b603b60u;
        x1 = (x1 & 0x8fff8fffu) ^ 0x3b603b60u;
        const __half2 lo = __halves2half2(__ushort_as_half((unsigned short) (x0 & 0xffffu)),
                                          __ushort_as_half((unsigned short) (x1 & 0xffffu)));
        const __half2 hi = __halves2half2(__ushort_as_half((unsigned short) (x0 >> 16)),
                                          __ushort_as_half((unsigned short) (x1 >> 16)));
        return __hadd2(lo, hi);
    }
}
template <int cb>
__device__ __forceinline__ float2 decode2(uint32_t s0, uint32_t s1) {
    return __half22float2(decode2h<cb>(s0, s1));
}

__device__ __forceinline__ uint32_t h2u(__half2 h) { return *reinterpret_cast<uint32_t*>(&h); }
__device__ __forceinline__ __half2 u2h(uint32_t u) { return *reinterpret_cast<__half2*>(&u); }

// (hi : lo) >> s, the low 32 bits
__device__ __forceinline__ uint32_t fsh(uint32_t lo, uint32_t hi, int s) {
    return (uint32_t) ((((uint64_t) hi << 32) | (uint64_t) lo) >> s);
}

// Step p's state is the 16-bit window ending at bit (p + 1) K of the tile's MSB-first bit stream (w: its 8 K words),
// wrapping around the tile.  A lane decodes its 8 consecutive steps lane * 8 .. + 7 together, the windows of a group
// of steps cut from one or two words with funnel shifts - exllamav3's grouping per K (exl3_dq.cuh: dq8_aligned_*bits,
// dq8, dq4, dq2x2), which the steps' fixed offsets keep within two words.  f[i]: steps 2i, 2i + 1 as FP16 values.
// Step lane * 8 + j sits at tile row (lane & 3) * 2 + (j & 1) + 8 (j >> 1 & 1), column (lane >> 2) + 8 (j >> 2).
template <int cb>
__device__ __forceinline__ void decode_states(const uint32_t (&s)[8], uint32_t (&f)[4]) {
#pragma unroll
    for (int i = 0; i < 4; ++i) f[i] = h2u(decode2h<cb>(s[2 * i] & 0xffffu, s[2 * i + 1] & 0xffffu));
}
// K = 4: the lane's steps end in its own word b (word lane), the first three reach into a (word lane - 1)
__device__ __forceinline__ void states4(uint32_t a, uint32_t b, uint32_t (&s)[8]) {
    const uint32_t x = fsh(b, a, 20);
    s[0] = x >> 8;
    s[1] = x >> 4;
    s[2] = x;
    s[3] = b >> 16;
    s[4] = b >> 12;
    s[5] = b >> 8;
    s[6] = b >> 4;
    s[7] = b;
}
// step p .. p + G - 1's windows (G = 4 or 2) from the two words that hold them
template <int K, int G>
__device__ __forceinline__ void states_group(const uint32_t* w, int p, uint32_t* s) {
    constexpr int NW = 8 * K;
    const int b0 = (p + 257) * K - 16, b2 = b0 + (G - 1) * K + 16;
    const int i0 = b0 >> 5, i2 = (b2 - 1) >> 5, s2 = ((i2 + 1) << 5) - b2;
    const uint32_t a = w[i0 % NW], b = w[i2 % NW];
#pragma unroll
    for (int j = 0; j < G; ++j) s[G - 1 - j] = fsh(b, a, s2 + j * K);
}
template <int K, int cb>
__device__ __forceinline__ void frag8(const uint32_t* w, int lane, uint32_t (&f)[4]) {
    const int p = lane * 8;
    uint32_t s[8];
    if constexpr (K == 4) {
        states4(w[(lane + 31) & 31], w[lane], s);
    } else if constexpr (K == 1 || K == 2) {
        // the 8 windows overlap in one 32-bit window: steps p + j end 8 - 1 - j K bits apart
        constexpr int NW = 8 * K, per = 32 / (8 * K);   // lanes per word
        const int i1 = lane / per, i0 = (i1 + NW - 1) % NW;
        const uint32_t b = fsh(w[i1], w[i0], (per - 1 - lane % per) * 8 * K);
#pragma unroll
        for (int j = 0; j < 8; ++j) s[j] = b >> ((7 - j) * K);
    } else if constexpr (K == 3) {
        // all 8 windows within two words: two funnel shifts, the rest by shifting
        constexpr int NW = 24;
        const int b1 = (p + 257) * 3, b0 = b1 - 16, b2 = b1 + 21;
        const int i0 = b0 >> 5, i2 = (b2 - 1) >> 5, s2 = ((i2 + 1) << 5) - b2;
        const uint32_t a = w[i0 % NW], b = w[i2 % NW];
        const uint32_t hi = fsh(b, a, s2), lo = fsh(b, a, s2 + 12);
#pragma unroll
        for (int j = 0; j < 4; ++j) {
            s[7 - j] = hi >> (3 * j);
            s[3 - j] = lo >> (3 * j);
        }
    } else if constexpr (K == 7) {
#pragma unroll
        for (int g = 0; g < 4; ++g) states_group<K, 2>(w, p + 2 * g, s + 2 * g);
    } else {   // 5, 6, 8
        states_group<K, 4>(w, p, s);
        states_group<K, 4>(w, p + 4, s + 4);
    }
    decode_states<cb>(s, f);
}

// the lane's 8 steps as F32: v[j] = step lane * 8 + j
template <int K, int cb>
__device__ __forceinline__ void decode8(const uint32_t* w, int lane, float* v) {
    uint32_t f[4];
    frag8<K, cb>(w, lane, f);
#pragma unroll
    for (int i = 0; i < 4; ++i) {
        const float2 d = __half22float2(u2h(f[i]));
        v[2 * i] = d.x;
        v[2 * i + 1] = d.y;
    }
}

// ---------------------------------------------------------------- the rotation
// the 128-point transform of the 128 values a warp holds 4 per lane (value lane * 4 + i), over sqrt(128)
__device__ __forceinline__ void had128(float* v, int lane) {
    const float s0 = v[0] + v[1], d0 = v[0] - v[1], s1 = v[2] + v[3], d1 = v[2] - v[3];
    v[0] = s0 + s1;
    v[1] = d0 + d1;
    v[2] = s0 - s1;
    v[3] = d0 - d1;
#pragma unroll
    for (int m = 1; m < 32; m <<= 1) {
#pragma unroll
        for (int i = 0; i < 4; ++i) {
            const float o = __shfl_xor_sync(0xffffffffu, v[i], m);
            v[i] = (lane & m) ? o - v[i] : v[i] + o;
        }
    }
#pragma unroll
    for (int i = 0; i < 4; ++i) v[i] *= kHad;
}

__device__ __forceinline__ float h2f(uint16_t b) { return __half2float(__ushort_as_half(b)); }

// the activation a warp rotates into its block's FP16 chunks: F32 (f) or the engine's q8_1 blocks (q)
struct Xin {
    const float* f = nullptr;
    const Q81* q = nullptr;
};

// the 128-input chunk of x from k0 on: the lane's 4 values and their suh ...
__device__ __forceinline__ void chunk_load(const Xin& x, const uint16_t* __restrict__ suh, int k0, int lane,
                                           float (&v)[4], float (&s)[4]) {
    const int base = k0 + lane * 4;
    if (x.f != nullptr) {
#pragma unroll
        for (int i = 0; i < 4; ++i) v[i] = x.f[base + i];
    } else {
        const Q81* b = x.q + base / 32;
        const float d = __low2float(b->ds);
#pragma unroll
        for (int i = 0; i < 4; ++i) v[i] = d * (float) b->qs[base % 32 + i];
    }
#pragma unroll
    for (int i = 0; i < 4; ++i) s[i] = h2f(suh[base + i]);
}
// ... and H(x * suh) of the chunk, FP16, into dst[0..127]
__device__ __forceinline__ void chunk_store(float (&v)[4], const float (&s)[4], int lane, __half* dst) {
#pragma unroll
    for (int i = 0; i < 4; ++i) v[i] *= s[i];
    had128(v, lane);
    __half2* d = reinterpret_cast<__half2*>(dst + lane * 4);
    d[0] = __floats2half2_rn(v[0], v[1]);
    d[1] = __floats2half2_rn(v[2], v[3]);
}

// ---------------------------------------------------------------- the block GEMV
// A block owns 128 outputs - 8 tiles, warp w the tile column tc0 + w through all of the block's inputs [k0, k1) - for
// nc <= kTok activations at once.  Per 128-input chunk, warp c (< nc) scales activation c's chunk by suh and rotates
// it into FP16 (xb, two buffers), then each warp decodes its tile of each of the chunk's 8 slices and multiplies it
// with every activation: on sm_80 and later as one mma.m16n8k16 a tile (the decoded tile is the A operand - a lane's 8
// steps are its A fragment - and the activations are the B operand's columns, as exllamav3's small-m GEMV and its
// tensor-core GEMM decode tiles straight into fragments), F32 accumulation; elsewhere on the CUDA cores, in F32.  An
// activation's sums do not depend on the other columns, so a window's token gets the token path's bits.  The next
// chunk's tile words are loaded while this one is decoded.  out[c * 128 + i]: activation c's sum for output i.  Ends
// with a __syncthreads (out complete).
constexpr int kTok = 8;   // activations a block takes at once (the MMA's n)

#if !defined(STRATA_USE_HIP) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
#define EXL3_TC 1
#endif

#if !defined(EXL3_TC)
// a lane's 4 weights of one column times the activation's 4 inputs, added to a (the CUDA-core product)
__device__ __forceinline__ float dot4_acc(float a, const float* v, float x0, float x1, float x2, float x3) {
    return __fadd_rn(a, __fmaf_rn(v[3], x3, __fmaf_rn(v[2], x2, __fmaf_rn(v[1], x1, __fmul_rn(v[0], x0)))));
}
#else
__device__ __forceinline__ void mma_f32(float (&c)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
                                        uint32_t b0, uint32_t b1) {
    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
                 "{%0,%1,%2,%3};\n"
                 : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
                 : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));
}
#endif

// a tile's words, and a lane's share of them (word lane + 32 r)
template <int K>
struct Words {
    static constexpr int W = 8 * K, R = (W + 31) / 32;
};

// the warp's tile of each of a chunk's 8 slices (slice s at tile0 + s sstride)
template <int K>
__device__ __forceinline__ void load_tiles(const uint32_t* __restrict__ tile0, size_t sstride, int lane,
                                           uint32_t (&w)[8][Words<K>::R]) {
    constexpr int W = Words<K>::W, R = Words<K>::R;
#pragma unroll
    for (int s = 0; s < 8; ++s)
#pragma unroll
        for (int r = 0; r < R; ++r) {
            const int i = lane + 32 * r;
            w[s][r] = (W % 32 == 0 || i < W) ? __ldg(tile0 + (size_t) s * sstride + i) : 0u;
        }
}

// one slice's fragment from the lane's words: K = 4 straight from the registers (the lane's steps end in its own word,
// the first three reach into the previous lane's), other K through the warp's stage (8 K words)
template <int K, int cb>
__device__ __forceinline__ void slice_frags(const uint32_t (&w)[Words<K>::R], uint32_t* st, int lane, uint32_t (&f)[4]) {
    if constexpr (K == 4) {
        uint32_t s[8];
        states4(__shfl_sync(0xffffffffu, w[0], (lane + 31) & 31), w[0], s);
        decode_states<cb>(s, f);
    } else {
        constexpr int W = Words<K>::W, R = Words<K>::R;
        __syncwarp();
#pragma unroll
        for (int r = 0; r < R; ++r)
            if (W % 32 == 0 || lane + 32 * r < W) st[lane + 32 * r] = w[r];
        __syncwarp();
        frag8<K, cb>(st, lane, f);
    }
}

// the blocks an SM keeps of the block_gemv kernels (their registers), as measured on an RTX 3090 and an RTX 5060 Ti:
// the routed experts four at K <= 4 (a token's 8 experts, 256 blocks, are one wave on 82 SMs), two at wider codes;
// mv two, three at K > 4 on sm_80 .. sm_90 (sm_120 spills at the 80 registers that takes).  Before sm_80 (the CUDA
// cores' product, more registers) two; HIP reads the second bound differently and keeps its default.
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
template <int K>
constexpr int kMinBlocksX = K <= 4 ? 4 : 2;
#else
template <int K>
constexpr int kMinBlocksX = 2;
#endif
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 1000
template <int K>
constexpr int kMinBlocksMv = K <= 4 ? 2 : 3;
#else
template <int K>
constexpr int kMinBlocksMv = 2;
#endif
#if defined(STRATA_USE_HIP)
#define EXL3_BOUNDS(n) __launch_bounds__(kThreads)
#else
#define EXL3_BOUNDS(n) __launch_bounds__(kThreads, n)
#endif

// the shared memory of a block_gemv block
template <int K>
struct BlockSmem {
    __half xb[2 * kTok * kCols];       // two chunk buffers, kTok activations each
    uint32_t stage[kWarps * 8 * K];    // a tile per warp (K != 4)
    float out[kTok * kCols];
};

template <int K, int cb>
__device__ void block_gemv(const uint32_t* __restrict__ tr, int ld, int tc0, int k0, int k1, const Xin& x,
                           const uint16_t* __restrict__ suh, int nc, BlockSmem<K>& sm) {
    constexpr int W = Words<K>::W, R = Words<K>::R;
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;
    const size_t sstride = (size_t) ld * W;
    const uint32_t* tile = tr + ((size_t) (k0 / 16) * ld + tc0 + warp) * W;
    uint32_t* st = sm.stage + warp * W;
    const bool rot = warp < nc;
    const int nq = (k1 - k0) / kCols;
    uint32_t wc[8][R], wn[8][R];
    float xv[4], xs4[4];
    load_tiles<K>(tile, sstride, lane, wc);
    if (rot) chunk_load(x, suh, k0, lane, xv, xs4);
#if defined(EXL3_TC)
    float c[4] = {0.0f, 0.0f, 0.0f, 0.0f};
#else
    float acc[kTok][2];
#pragma unroll
    for (int cc = 0; cc < kTok; ++cc) acc[cc][0] = acc[cc][1] = 0.0f;
#endif
    for (int q = 0; q < nq; ++q) {
        __half* xq = sm.xb + (q & 1) * (kTok * kCols);
        if (rot) chunk_store(xv, xs4, lane, xq + warp * kCols);
        __syncthreads();
        if (q + 1 < nq) {
            load_tiles<K>(tile + (size_t) (q + 1) * 8 * sstride, sstride, lane, wn);
            if (rot) chunk_load(x, suh, k0 + (q + 1) * kCols, lane, xv, xs4);
        }
#pragma unroll
        for (int s = 0; s < 8; ++s) {
            uint32_t f[4];
            slice_frags<K, cb>(wc[s], st, lane, f);
#if defined(EXL3_TC)
            uint32_t b0 = 0u, b1 = 0u;
            if (g < nc) {
                const __half* xr = xq + g * kCols + s * 16 + 2 * t;
                b0 = *reinterpret_cast<const uint32_t*>(xr);
                b1 = *reinterpret_cast<const uint32_t*>(xr + 8);
            }
            // A: rows (outputs) g / g + 8, columns (inputs) 2t.. / 2t + 8..: steps 0,1 / 4,5 / 2,3 / 6,7
            mma_f32(c, f[0], f[2], f[1], f[3], b0, b1);
#else
            float v[8];
#pragma unroll
            for (int i = 0; i < 4; ++i) {
                const float2 d = __half22float2(u2h(f[i]));
                v[2 * i] = d.x;
                v[2 * i + 1] = d.y;
            }
#pragma unroll
            for (int cc = 0; cc < kTok; ++cc)
                if (cc < nc) {
                    const __half* xr = xq + cc * kCols + s * 16 + 2 * t;
                    const float x0 = __half2float(xr[0]), x1 = __half2float(xr[1]);
                    const float x2 = __half2float(xr[8]), x3 = __half2float(xr[9]);
                    acc[cc][0] = dot4_acc(acc[cc][0], v, x0, x1, x2, x3);
                    acc[cc][1] = dot4_acc(acc[cc][1], v + 4, x0, x1, x2, x3);
                }
#endif
        }
#pragma unroll
        for (int s = 0; s < 8; ++s)
#pragma unroll
            for (int r = 0; r < R; ++r) wc[s][r] = wn[s][r];
    }
    const int o = warp * 16 + g;
#if defined(EXL3_TC)
    // c: the tile's outputs g (c[0], c[1]) and g + 8 (c[2], c[3]) for activations 2t (c[0], c[2]) and 2t + 1
    if (2 * t < nc) {
        sm.out[(2 * t) * kCols + o] = c[0];
        sm.out[(2 * t) * kCols + o + 8] = c[2];
    }
    if (2 * t + 1 < nc) {
        sm.out[(2 * t + 1) * kCols + o] = c[1];
        sm.out[(2 * t + 1) * kCols + o + 8] = c[3];
    }
#else
    // the 4 lanes of one g hold parts of the same two outputs
#pragma unroll
    for (int cc = 0; cc < kTok; ++cc)
        if (cc < nc) {
#pragma unroll
            for (int h = 0; h < 2; ++h) {
                acc[cc][h] += __shfl_xor_sync(0xffffffffu, acc[cc][h], 1);
                acc[cc][h] += __shfl_xor_sync(0xffffffffu, acc[cc][h], 2);
            }
            if (t == 0) {
                sm.out[cc * kCols + o] = acc[cc][0];
                sm.out[cc * kCols + o + 8] = acc[cc][1];
            }
        }
#endif
    __syncthreads();
}

// one warp: y[c] = alpha * (H(red) * svh)[c] + bias[c] for the block's 128 outputs
__device__ __forceinline__ void epilogue(const float* red, const uint16_t* svh, float alpha, const float* bias,
                                         float* y) {
    const int lane = threadIdx.x & 31;
    float v[4];
#pragma unroll
    for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
    had128(v, lane);
#pragma unroll
    for (int i = 0; i < 4; ++i) {
        const int c = lane * 4 + i;
        y[c] = __fmaf_rn(__fmul_rn(alpha, v[i]), h2f(svh[c]), bias != nullptr ? bias[c] : 0.0f);
    }
}

// ---------------------------------------------------------------- mv
struct KJob {
    const uint32_t* tr;
    const uint16_t* suh;
    const uint16_t* svh;
    const float* xf;
    const Q81* xq;
    float* y;
    const float* bias;
    float* part;       // nc x S x n partial sums (S > 1)
    unsigned* cnt;     // n / 128 counters, zero between launches
    float alpha;
    int k, n, ld, S, kr;
    int nc;            // the activations (1..kTok), xf / xq / y advanced by these per activation
    size_t xf_cs, xq_cs, y_cs;   // (floats, bytes, floats)
};
struct KBatch {
    KJob j[kMaxMvJobs];
    int blk_end[kMaxMvJobs];
    int n;
    const int* skip;   // set_skip's flag: non-zero = a pipelined window's token past a wrong draft (nullptr: none)
};

// a launch's blocks: its jobs' (output block, input split) pairs, each for all of the job's nc activations at once
// (block_gemv).  S > 1: the partial sums go to the scratch and the last block of an output block adds them up in split
// order (deterministic) and runs the epilogue
template <int K, int cb>
__global__ void EXL3_BOUNDS(kMinBlocksMv<K>) mv_kernel(const KBatch b) {
    __shared__ __align__(16) BlockSmem<K> sm;
    __shared__ int last;
    int ji = 0;
    while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;
    const KJob& J = b.j[ji];
    const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);
    const int g = local / J.S, sp = local % J.S;
    const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);
    const int warp = threadIdx.x >> 5;
    // a skipped token (set_skip): no work.  The flag changes only between launches (skip_gate, on the same stream), so
    // every block of the launch returns and no split counter moves
    if (b.skip != nullptr && *b.skip) return;
    Xin x;
    if (warp < J.nc) {
        if (J.xf != nullptr) x.f = J.xf + warp * J.xf_cs;
        else x.q = (const Q81*) ((const uint8_t*) J.xq + warp * J.xq_cs);
    }
    block_gemv<K, cb>(J.tr, J.ld, g * 8, k0, k1, x, J.suh, J.nc, sm);
    const float* bias = J.bias != nullptr ? J.bias + g * kCols : nullptr;
    if (J.S == 1) {
        if (warp < J.nc)
            epilogue(sm.out + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
        return;
    }
    if (threadIdx.x < kCols)
        for (int c = 0; c < J.nc; ++c)
            J.part[((size_t) c * J.S + sp) * J.n + g * kCols + threadIdx.x] = sm.out[c * kCols + threadIdx.x];
    __threadfence();
    __syncthreads();
    if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);
    __syncthreads();
    if (!last) return;
    __threadfence();
    if (threadIdx.x < kCols)
        for (int c = 0; c < J.nc; ++c) {
            float s = 0.0f;
            for (int i = 0; i < J.S; ++i)
                s += *(volatile const float*) &J.part[((size_t) c * J.S + i) * J.n + g * kCols + threadIdx.x];
            sm.out[c * kCols + threadIdx.x] = s;
        }
    __syncthreads();
    if (warp < J.nc)
        epilogue(sm.out + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
    if (threadIdx.x == 0) J.cnt[g] = 0u;
}

// ---------------------------------------------------------------- the routed experts
struct KMoe {
    const unsigned long long* plan_ptr;
    const float* plan_w;
    const float* cpu_part;
    const int* cpu_flag;
    const float* x;
    float* h;            // [k][n_ff]: the gate's values, then the swiglu
    float* u;            // [k][n_ff]: the up's values
    const float* sh_out;
    float* out;
    float* part;         // [k][n_embd]
    unsigned* cnt;       // n_embd / 128
    unsigned* cnt_gu;    // k x n_ff / 128: the gate / up pairs
    size_t suh[3], svh[3], tr[3];
    int k, n_embd, n_ff;
    float limit;
    const int* skip;     // set_skip's flag (nullptr: none)
};

// the pieces the token's experts (moe_*_kernel) and a window's grouped experts (win_*_kernel) share, so each window
// token gets the token path's bits: one warp's output block of 128 (H, then svh) ...
__device__ __forceinline__ void block_out(const float* red, const uint16_t* svh, float* dst) {
    const int lane = threadIdx.x & 31;
    float v[4];
#pragma unroll
    for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
    had128(v, lane);
#pragma unroll
    for (int i = 0; i < 4; ++i) dst[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[lane * 4 + i]));
}
// ... the clamped swiglu ...
__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {
    if (limit > 0.0f) {
        gv = fminf(gv, limit);
        uv = fminf(fmaxf(uv, -limit), limit);
    }
    return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
}
// ... the second block of a (gate, up) pair to finish (its counter back at zero for the next launch) ...
__device__ __forceinline__ bool pair_last(unsigned* cnt, int& last) {
    __threadfence();
    __syncthreads();
    if (threadIdx.x == 0) last = atomicAdd(cnt, 1u) == 1u;
    __syncthreads();
    if (!last) return false;
    __threadfence();
    if (threadIdx.x == 0) *cnt = 0u;
    return true;
}
// ... and one output's sum: the experts' rows in plan order (p[i]: expert i's, 0 for one not on the device), the CPU
// lane's part when flagged (cpu_part: null when not), the shared expert's row (sh: 0 without one)
__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {
    float m = 0.0f;
    for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);
    if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);
    return __fadd_rn(m, sh);
}

// grid (n_ff / 128, k, 2): expert blockIdx.y's gate (z 0, into h) or up (z 1, into u) for 128 h values; the pair's
// second block to finish turns them into the clamped swiglu, in h
template <int K, int cb>
__global__ void EXL3_BOUNDS(kMinBlocksX<K>) moe_gate_up_kernel(const KMoe a) {
    __shared__ __align__(16) BlockSmem<K> sm;
    __shared__ int last;
    const int ei = blockIdx.y, g = blockIdx.x, r = blockIdx.z;
    const unsigned long long ptr = a.plan_ptr[ei];
    if (ptr == 0ull) return;
    if (a.skip != nullptr && *a.skip) return;   // a skipped token (set_skip, as mv_kernel)
    const uint8_t* blob = (const uint8_t*) ptr;
    const int warp = threadIdx.x >> 5;
    Xin x;
    x.f = a.x;
    block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd, x,
                      (const uint16_t*) (blob + a.suh[r]), 1, sm);
    const size_t row = (size_t) ei * a.n_ff + g * kCols;
    if (warp == 0) block_out(sm.out, (const uint16_t*) (blob + a.svh[r]) + g * kCols, (r == 0 ? a.h : a.u) + row);
    if (!pair_last(a.cnt_gu + ei * (a.n_ff / kCols) + g, last)) return;
    if (threadIdx.x < kCols)
        a.h[row + threadIdx.x] = swiglu(*(volatile const float*) &a.h[row + threadIdx.x],
                                        *(volatile const float*) &a.u[row + threadIdx.x], a.limit);
}

// grid (n_embd / 128, k): expert blockIdx.y's down rows for 128 outputs; the last block of each output block adds
// the experts up in plan order (as the engine's moe_down), then the CPU lane's part and the shared expert
template <int K, int cb>
__global__ void EXL3_BOUNDS(kMinBlocksX<K>) moe_down_kernel(const KMoe a) {
    __shared__ __align__(16) BlockSmem<K> sm;
    __shared__ int last;
    const int ei = blockIdx.y, g = blockIdx.x;
    const unsigned long long ptr = a.plan_ptr[ei];
    const int warp = threadIdx.x >> 5;
    float* part = a.part + (size_t) ei * a.n_embd + g * kCols;
    if (a.skip != nullptr && *a.skip) return;   // a skipped token (set_skip, as mv_kernel)
    if (ptr != 0ull) {
        const uint8_t* blob = (const uint8_t*) ptr;
        Xin x;
        x.f = a.h + (size_t) ei * a.n_ff;
        block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff, x,
                          (const uint16_t*) (blob + a.suh[2]), 1, sm);
        if (warp == 0) block_out(sm.out, (const uint16_t*) (blob + a.svh[2]) + g * kCols, part);
    } else if (threadIdx.x < kCols) {
        part[threadIdx.x] = 0.0f;
    }
    __threadfence();
    __syncthreads();
    if (threadIdx.x == 0) last = atomicAdd(&a.cnt[g], 1u) == (unsigned) (a.k - 1);
    __syncthreads();
    if (!last) return;
    __threadfence();
    if (threadIdx.x < kCols) {
        const int r = g * kCols + threadIdx.x;
        float p[8];
        for (int i = 0; i < a.k; ++i) p[i] = *(volatile const float*) &a.part[(size_t) i * a.n_embd + r];
        a.out[r] = combine(a.plan_w, p, a.k, a.cpu_flag != nullptr && *a.cpu_flag ? a.cpu_part + r : nullptr,
                           a.sh_out != nullptr ? a.sh_out[r] : 0.0f);
    }
    if (threadIdx.x == 0) a.cnt[g] = 0u;
}

// ---------------------------------------------------------------- a verify window's routed experts
// gf::moe_win_experts' grouping (group g: one expert, its blob at grp_ptr[g]; its entries e = t * 8 + i at
// grp_ent[grp_start[g] ..)), each group's weights read and decoded once for up to kTok of its tokens at a time (as
// exllamav3's fused MoE runs a run of slots that picked one expert as the rows of one MMA tile), with the token
// path's arithmetic per entry: every token's output is moe_gate_up + moe_down's for it alone, bit for bit
struct KWin {
    const unsigned long long* grp_ptr;
    const int *grp_start, *grp_ent, *n_groups;
    const unsigned long long* plan_ptr;
    const float* plan_w;
    const float* cpu_part;
    size_t cpu_stride;
    const int* cpu_flag;
    const float* x;
    size_t x_stride;
    float* h;            // [T * 8][n_ff]: the gate's values, then the swiglu
    float* u;            // [T * 8][n_ff]: the up's values
    float* part;         // [T * 8][n_embd]
    const float* sh_out; // [T][n_embd]
    float* out;
    size_t out_stride;
    unsigned* cnt_gu;    // groups x n_ff / 128: the gate / up pairs
    size_t suh[3], svh[3], tr[3];
    int k, n_embd, n_ff;
    float limit;
};

// grid (n_ff / 128, groups at most, 2): group blockIdx.y's gate (z 0, into h) or up (z 1, into u) for 128 h values of
// each of its entries, kTok entries a pass; the pair's second block to finish turns them into the clamped swiglu
template <int K, int cb>
__global__ void EXL3_BOUNDS(kMinBlocksX<K>) win_gu_kernel(const KWin a) {
    __shared__ __align__(16) BlockSmem<K> sm;
    __shared__ int last;
    const int grp = blockIdx.y, g = blockIdx.x, r = blockIdx.z;
    if (grp >= *a.n_groups) return;
    const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
    const int warp = threadIdx.x >> 5;
    const int m0 = a.grp_start[grp], m1 = a.grp_start[grp + 1];
    float* dst = r == 0 ? a.h : a.u;
    for (int mm = m0; mm < m1; mm += kTok) {
        const int nc = min(kTok, m1 - mm);
        Xin x;
        if (warp < nc) x.f = a.x + (size_t) (a.grp_ent[mm + warp] >> 3) * a.x_stride;
        block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd, x,
                          (const uint16_t*) (blob + a.suh[r]), nc, sm);
        if (warp < nc)
            block_out(sm.out + warp * kCols, (const uint16_t*) (blob + a.svh[r]) + g * kCols,
                      dst + (size_t) a.grp_ent[mm + warp] * a.n_ff + g * kCols);
        __syncthreads();
    }
    if (!pair_last(a.cnt_gu + grp * (a.n_ff / kCols) + g, last)) return;
    if (threadIdx.x < kCols)
        for (int m = m0; m < m1; ++m) {
            const size_t row = (size_t) a.grp_ent[m] * a.n_ff + g * kCols + threadIdx.x;
            a.h[row] = swiglu(*(volatile const float*) &a.h[row], *(volatile const float*) &a.u[row], a.limit);
        }
}

// grid (n_embd / 128, groups at most): group blockIdx.y's down rows for 128 outputs of each entry into its part rows
template <int K, int cb>
__global__ void EXL3_BOUNDS(kMinBlocksX<K>) win_down_kernel(const KWin a) {
    __shared__ __align__(16) BlockSmem<K> sm;
    const int grp = blockIdx.y, g = blockIdx.x;
    if (grp >= *a.n_groups) return;
    const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
    const int warp = threadIdx.x >> 5;
    const int m0 = a.grp_start[grp], m1 = a.grp_start[grp + 1];
    for (int mm = m0; mm < m1; mm += kTok) {
        const int nc = min(kTok, m1 - mm);
        Xin x;
        if (warp < nc) x.f = a.h + (size_t) a.grp_ent[mm + warp] * a.n_ff;
        block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff, x,
                          (const uint16_t*) (blob + a.suh[2]), nc, sm);
        if (warp < nc)
            block_out(sm.out + warp * kCols, (const uint16_t*) (blob + a.svh[2]) + g * kCols,
                      a.part + (size_t) a.grp_ent[mm + warp] * a.n_embd + g * kCols);
        __syncthreads();
    }
}

// grid (n_embd / 128, T): each token's outputs, moe_down_kernel's combine (an entry off the device adds 0)
__global__ void __launch_bounds__(kCols) win_combine_kernel(const KWin a) {
    const int t = blockIdx.y, r = blockIdx.x * kCols + threadIdx.x;
    float p[8];
    for (int i = 0; i < a.k; ++i) {
        const int e = t * 8 + i;
        p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;
    }
    a.out[(size_t) t * a.out_stride + r] =
        combine(a.plan_w + t * 8, p, a.k,
                a.cpu_flag != nullptr && a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.cpu_stride + r : nullptr,
                a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);
}

__global__ void win_wait_kernel(const unsigned* p, unsigned v) {
    if (threadIdx.x != 0) return;
    while (*(volatile const unsigned*) p != v) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
        __nanosleep(200);
#endif
    }
    __threadfence_system();
}

// ---------------------------------------------------------------- reconstruction
// grid (n / 128, k / 128): the 128 x 128 block of W = diag(suh) H W' H diag(svh), written transposed (dst[c][r])
template <int K, int cb>
__global__ void __launch_bounds__(kThreads) recon_rows_kernel(const uint32_t* __restrict__ tr, int ld,
                                                              const uint16_t* __restrict__ suh,
                                                              const uint16_t* __restrict__ svh, int k, uint16_t* dst,
                                                              bool bf16) {
    constexpr int W = 8 * K;
    constexpr int P = kCols + 2;   // padded row (halves)
    __shared__ __half tile[kCols * P];
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
    const int nb = blockIdx.x, kb = blockIdx.y;
    // decode: warp w takes the 16-row slices w (8 tiles each)
    for (int ks = warp; ks < 8; ks += kWarps) {
        const uint32_t* src = tr + ((size_t) (kb * 8 + ks) * ld + nb * 8) * W;
        for (int t = 0; t < 8; ++t) {
            float v[8];
            decode8<K, cb>(src + t * W, lane, v);
#pragma unroll
            for (int j = 0; j < 8; ++j) {
                const int r = ks * 16 + (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);
                const int c = t * 16 + (lane >> 2) + 8 * (j >> 2);
                tile[r * P + c] = __float2half_rn(v[j]);
            }
        }
    }
    __syncthreads();
    // H along the outputs (each row of 128 is one block)
    for (int r = warp; r < kCols; r += kWarps) {
        float v[4];
#pragma unroll
        for (int i = 0; i < 4; ++i) v[i] = __half2float(tile[r * P + lane * 4 + i]);
        had128(v, lane);
#pragma unroll
        for (int i = 0; i < 4; ++i) tile[r * P + lane * 4 + i] = __float2half_rn(v[i]);
    }
    __syncthreads();
    // H along the inputs, the scales, the transposed store
    for (int c = warp; c < kCols; c += kWarps) {
        float v[4];
#pragma unroll
        for (int i = 0; i < 4; ++i) v[i] = __half2float(tile[(lane * 4 + i) * P + c]);
        had128(v, lane);
        const float sv = h2f(svh[nb * kCols + c]);
        uint16_t* d = dst + (size_t) (nb * kCols + c) * k + kb * kCols + lane * 4;
#pragma unroll
        for (int i = 0; i < 4; ++i) {
            const float w = v[i] * sv * h2f(suh[kb * kCols + lane * 4 + i]);
            if (bf16) {
                const uint32_t u = __float_as_uint(w);
                d[i] = (uint16_t) ((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);   // round to nearest even
            } else {
                d[i] = __half_as_ushort(__float2half_rn(w));
            }
        }
    }
}

// grid (n / 128, k / 16): W' row-major, one tile per warp
template <int K, int cb>
__global__ void __launch_bounds__(kThreads) recon_inner_kernel(const uint32_t* __restrict__ tr, int ld, int n,
                                                               uint16_t* dst) {
    constexpr int W = 8 * K;
    const int lane = threadIdx.x & 31, t = threadIdx.x >> 5;
    const int tc = blockIdx.x * 8 + t, ks = blockIdx.y;
    float v[8];
    decode8<K, cb>(tr + ((size_t) ks * ld + tc) * W, lane, v);
#pragma unroll
    for (int j = 0; j < 8; ++j) {
        const int r = ks * 16 + (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);
        const int c = tc * 16 + (lane >> 2) + 8 * (j >> 2);
        dst[(size_t) r * n + c] = __half_as_ushort(__float2half_rn(v[j]));
    }
}

// grid (n / 128, k / 128): W' of the 128 x 128 block, transposed (dst[c][r]) - the decoded values alone.  Each warp
// decodes its slice's 8 tiles into the block's tile (half2 pairs of rows, conflict-free), then writes 256-byte rows.
template <int K, int cb>
__global__ void __launch_bounds__(kThreads) inner_rows_kernel(const uint32_t* __restrict__ tr, int ld, int k,
                                                              uint16_t* __restrict__ dst) {
    constexpr int W = 8 * K;
    constexpr int P = kCols + 8;   // a column's 128 rows, padded (halves; 16-byte rows)
    __shared__ __align__(16) __half tile[kCols * P];   // tile[c * P + r]
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;
    const int nb = blockIdx.x, kb = blockIdx.y;
    for (int ks = warp; ks < 8; ks += kWarps) {
        const uint32_t* src = tr + ((size_t) (kb * 8 + ks) * ld + nb * 8) * W;
        const int r = ks * 16 + 2 * t;
#pragma unroll
        for (int tc = 0; tc < 8; ++tc) {
            uint32_t f[4];
            frag8<K, cb>(src + tc * W, lane, f);
            const int c = tc * 16 + g;
            *reinterpret_cast<uint32_t*>(&tile[c * P + r]) = f[0];
            *reinterpret_cast<uint32_t*>(&tile[c * P + r + 8]) = f[1];
            *reinterpret_cast<uint32_t*>(&tile[(c + 8) * P + r]) = f[2];
            *reinterpret_cast<uint32_t*>(&tile[(c + 8) * P + r + 8]) = f[3];
        }
    }
    __syncthreads();
    for (int c = warp; c < kCols; c += kWarps)
        *reinterpret_cast<uint2*>(dst + (size_t) (nb * kCols + c) * k + kb * kCols + lane * 4) =
            *reinterpret_cast<const uint2*>(&tile[c * P + lane * 4]);
}

// one warp per 128 values of a row: the rotated FP16 input of an EXL3 matrix (rows_rot_f16) ...
__global__ void __launch_bounds__(kThreads) rows_rot_f16_kernel(const float* __restrict__ x, int ldx,
                                                                const int* __restrict__ idx, int nrows, int ncols,
                                                                const uint16_t* __restrict__ suh, uint16_t* out) {
    const int lane = threadIdx.x & 31, nb = ncols / kCols;
    const int w = blockIdx.x * kWarps + (threadIdx.x >> 5);
    if (w >= nrows * nb) return;
    const int r = w / nb, b = w % nb;
    Xin xin;
    xin.f = x + (size_t) (idx != nullptr ? idx[r] : r) * ldx;
    float v[4], sc[4];
    chunk_load(xin, suh, b * kCols, lane, v, sc);
    chunk_store(v, sc, lane, reinterpret_cast<__half*>(out) + (size_t) r * ncols + b * kCols);
}
// ... and its output rotation in place (block_out's arithmetic)
__global__ void __launch_bounds__(kThreads) rows_rot_out_kernel(float* y, int ldy, int nrows, int ncols,
                                                                const uint16_t* __restrict__ svh) {
    const int lane = threadIdx.x & 31, nb = ncols / kCols;
    const int w = blockIdx.x * kWarps + (threadIdx.x >> 5);
    if (w >= nrows * nb) return;
    const int r = w / nb, b = w % nb;
    float* p = y + (size_t) r * ldy + b * kCols;
    float v[4];
#pragma unroll
    for (int i = 0; i < 4; ++i) v[i] = p[lane * 4 + i];
    had128(v, lane);
#pragma unroll
    for (int i = 0; i < 4; ++i) p[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[b * kCols + lane * 4 + i]));
}

__global__ void rows_f16_kernel(const float* __restrict__ x, int ldx, const int* __restrict__ idx, int ncols,
                                uint16_t* __restrict__ out) {
    const int r = blockIdx.y;
    const float* src = x + (size_t) (idx != nullptr ? idx[r] : r) * ldx;
    uint16_t* dst = out + (size_t) r * ncols;
    for (int c = blockIdx.x * blockDim.x + threadIdx.x; c < ncols; c += gridDim.x * blockDim.x)
        dst[c] = __half_as_ushort(__float2half_rn(src[c]));
}

// ---------------------------------------------------------------- the prompt path's tensor-core GEMMs (sm_80+)
// A block: BM rows x 128 outputs (one output rotation block).  Per 128 inputs the block's rows are scaled by suh,
// rotated and stored FP16 in shared memory (the A operands); per 16-row slice each warp decodes its 16 x 16 tile
// straight into mma.m16n8k16's B fragments - a lane's 8 steps ARE the fragment order (steps 0,1 / 2,3 rows 2t+{0,1} /
// 2t+8+{0,1} of column g, steps 4..7 the same of column g + 8) - and multiplies every m-tile of the block with them.
// The output rotation and svh run on the F32 tile in shared memory.
#if !defined(STRATA_USE_HIP)
namespace tc {

constexpr int BM = 32, MT = BM / 16, AP = 136;   // rows a block, its m-tiles, the A rows' pitch (halves)

__device__ __forceinline__ void mma16816(float* c, const uint32_t* a, uint32_t b0, uint32_t b1) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
                 "{%0,%1,%2,%3};\n"
                 : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
                 : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1));
#endif
}
// the lane's 4 B registers of tile t (words staged in st): b[0..1] columns 0..7, b[2..3] columns 8..15
template <int K, int cb>
__device__ __forceinline__ void tile_frags(const uint32_t* st, int lane, uint32_t* b) {
    uint32_t f[4];
    frag8<K, cb>(st, lane, f);
#pragma unroll
    for (int i = 0; i < 4; ++i) b[i] = f[i];
}

// A[r][j] = FP16(H(x_r * suh)[j]), j < 128, for the block's rows (zero past nrows).  x_r: F32 row idx[r] of xf (ldx)
// when xf is given, else FP16 row r of xh (ldx).  Every warp, BM / 8 rows each.
__device__ __forceinline__ void load_a(const float* __restrict__ xf, const int* __restrict__ idx,
                                       const __half* __restrict__ xh, int ldx, int r0, int nrows, int k0,
                                       const uint16_t* __restrict__ suh, __half* A) {
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
    for (int rr = warp; rr < BM; rr += kWarps) {
        float v[4] = {0.0f, 0.0f, 0.0f, 0.0f};
        const int r = r0 + rr;
        if (rr < nrows) {
            const int c = k0 + lane * 4;
            if (xf != nullptr) {
                const float* src = xf + (size_t) idx[r] * ldx + c;
#pragma unroll
                for (int i = 0; i < 4; ++i) v[i] = src[i];
            } else {
                const __half* src = xh + (size_t) r * ldx + c;
#pragma unroll
                for (int i = 0; i < 4; ++i) v[i] = __half2float(src[i]);
            }
#pragma unroll
            for (int i = 0; i < 4; ++i) v[i] *= h2f(suh[c + i]);
            had128(v, lane);
        }
#pragma unroll
        for (int i = 0; i < 4; ++i) A[rr * AP + lane * 4 + i] = __float2half_rn(v[i]);
    }
}

// the m-tile's A registers for the 16-column slice s of A
__device__ __forceinline__ void a_frags(const __half* A, int mt, int s, int lane, uint32_t* a) {
    const int g = lane >> 2, t = lane & 3;
    const __half* p = A + (mt * 16 + g) * AP + s * 16 + 2 * t;
    a[0] = *reinterpret_cast<const uint32_t*>(p);
    a[1] = *reinterpret_cast<const uint32_t*>(p + 8 * AP);
    a[2] = *reinterpret_cast<const uint32_t*>(p + 8);
    a[3] = *reinterpret_cast<const uint32_t*>(p + 8 * AP + 8);
}

// acc (this warp's 16 columns of the block's BM rows) -> T[BM][128] F32
__device__ __forceinline__ void store_acc(const float (&acc)[MT][2][4], float* T) {
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;
#pragma unroll
    for (int mt = 0; mt < MT; ++mt)
#pragma unroll
        for (int ns = 0; ns < 2; ++ns) {
            const int c = warp * 16 + ns * 8 + 2 * t, r = mt * 16 + g;
            T[r * 128 + c] = acc[mt][ns][0];
            T[r * 128 + c + 1] = acc[mt][ns][1];
            T[(r + 8) * 128 + c] = acc[mt][ns][2];
            T[(r + 8) * 128 + c + 1] = acc[mt][ns][3];
        }
}

struct GArgs {
    const uint8_t* wbase;     // expert e's blob at wbase + e * stride (dense: 0, the offsets are pointers)
    size_t stride;
    size_t tr[3], suh[3], svh[3];
    int ld[3];                // tiles per 16-row slice of each matrix
    const int* bounds;        // device: expert e's rows [bounds[e], bounds[e + 1]) (null: one matrix, rows [0, m))
    int row0, m;              // local row of a sorted row: bounds[e] - row0 + r
    int k, n;                 // of the matrix (gate/up: n_embd, n_ff)
    const float* xf;          // F32 input rows by index (idx[local row]) ...
    const int* idx;
    const __half* xh;         // ... or FP16 rows by local row
    int ldx;
    float* y;                 // F32 output rows (ldy), y = beta y + out ...
    int ldy;
    float beta;
    __half* h;                // ... or (gate/up) the swiglu as FP16 rows (ldh)
    int ldh;
    float limit;
};

// one matrix: y = beta y + H(H(x * suh) W') * svh.  grid (row blocks, n / 128, experts)
template <int K, int cb>
__global__ void __launch_bounds__(kThreads) gemm_kernel(const GArgs a) {
    constexpr int W = 8 * K;
    __shared__ __align__(16) unsigned char sm[BM * 128 * 4];   // A (BM x AP halves), then the F32 tile
    __shared__ uint32_t stage[kWarps][64];
    __half* A = reinterpret_cast<__half*>(sm);
    float* T = reinterpret_cast<float*>(sm);
    const int e = blockIdx.z, rb = blockIdx.x, g = blockIdx.y;
    const int e0 = a.bounds != nullptr ? a.bounds[e] - a.row0 : 0;
    const int ne = a.bounds != nullptr ? a.bounds[e + 1] - a.bounds[e] : a.m;
    const int r0 = e0 + rb * BM, nrows = min(BM, ne - rb * BM);
    if (nrows <= 0) return;
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    const uint32_t* tr = (const uint32_t*) (blob + a.tr[0]);
    const uint16_t* suh = (const uint16_t*) (blob + a.suh[0]);
    const uint16_t* svh = (const uint16_t*) (blob + a.svh[0]);
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
    float acc[MT][2][4] = {};
    for (int k0 = 0; k0 < a.k; k0 += 128) {
        __syncthreads();
        load_a(a.xf, a.idx, a.xh, a.ldx, r0, nrows, k0, suh, A);
        __syncthreads();
#pragma unroll 1
        for (int s = 0; s < 8; ++s) {
            const int ks = k0 / 16 + s;
            const uint32_t* src = tr + ((size_t) ks * a.ld[0] + g * 8 + warp) * W;
            __syncwarp();
            for (int i = lane; i < W; i += 32) stage[warp][i] = src[i];
            __syncwarp();
            uint32_t b[4];
            tile_frags<K, cb>(stage[warp], lane, b);
#pragma unroll
            for (int mt = 0; mt < MT; ++mt) {
                uint32_t af[4];
                a_frags(A, mt, s, lane, af);
                mma16816(acc[mt][0], af, b[0], b[1]);
                mma16816(acc[mt][1], af, b[2], b[3]);
            }
        }
    }
    __syncthreads();
    store_acc(acc, T);
    __syncthreads();
    for (int rr = warp; rr < nrows; rr += kWarps) {
        float v[4];
#pragma unroll
        for (int i = 0; i < 4; ++i) v[i] = T[rr * 128 + lane * 4 + i];
        had128(v, lane);
        float* yr = a.y + (size_t) (r0 + rr) * a.ldy + g * 128 + lane * 4;
#pragma unroll
        for (int i = 0; i < 4; ++i) {
            const float o = v[i] * h2f(svh[g * 128 + lane * 4 + i]);
            yr[i] = a.beta != 0.0f ? a.beta * yr[i] + o : o;
        }
    }
}

// gate and up (matrices 0 and 1) over the same rows, the clamped swiglu, FP16 h.  grid (row blocks, n / 128, experts)
template <int K, int cb>
__global__ void __launch_bounds__(kThreads) gate_up_kernel(const GArgs a) {
    constexpr int W = 8 * K;
    __shared__ __align__(16) unsigned char sm[2 * BM * 128 * 4];   // A_g, A_u, then the two F32 tiles
    __shared__ uint32_t stage[kWarps][64];
    __half* A[2] = {reinterpret_cast<__half*>(sm), reinterpret_cast<__half*>(sm) + BM * AP};
    float* T[2] = {reinterpret_cast<float*>(sm), reinterpret_cast<float*>(sm) + BM * 128};
    const int e = blockIdx.z, rb = blockIdx.x, g = blockIdx.y;
    const int e0 = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];
    const int r0 = e0 + rb * BM, nrows = min(BM, ne - rb * BM);
    if (nrows <= 0) return;
    const uint8_t* blob = a.wbase + (size_t) e * a.stride;
    const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
    float acc[2][MT][2][4] = {};
    for (int k0 = 0; k0 < a.k; k0 += 128) {
        __syncthreads();
        for (int m = 0; m < 2; ++m)
            load_a(a.xf, a.idx, a.xh, a.ldx, r0, nrows, k0, (const uint16_t*) (blob + a.suh[m]), A[m]);
        __syncthreads();
#pragma unroll 1
        for (int s = 0; s < 8; ++s) {
            const int ks = k0 / 16 + s;
#pragma unroll
            for (int m = 0; m < 2; ++m) {
                const uint32_t* src = (const uint32_t*) (blob + a.tr[m]) + ((size_t) ks * a.ld[m] + g * 8 + warp) * W;
                __syncwarp();
                for (int i = lane; i < W; i += 32) stage[warp][i] = src[i];
                __syncwarp();
                uint32_t b[4];
                tile_frags<K, cb>(stage[warp], lane, b);
#pragma unroll
                for (int mt = 0; mt < MT; ++mt) {
                    uint32_t af[4];
                    a_frags(A[m], mt, s, lane, af);
                    mma16816(acc[m][mt][0], af, b[0], b[1]);
                    mma16816(acc[m][mt][1], af, b[2], b[3]);
                }
            }
        }
    }
    __syncthreads();
    store_acc(acc[0], T[0]);
    store_acc(acc[1], T[1]);
    __syncthreads();
    const uint16_t* svg = (const uint16_t*) (a.wbase + (size_t) e * a.stride + a.svh[0]) + g * 128;
    const uint16_t* svu = (const uint16_t*) (a.wbase + (size_t) e * a.stride + a.svh[1]) + g * 128;
    for (int rr = warp; rr < nrows; rr += kWarps) {
        float vg[4], vu[4];
#pragma unroll
        for (int i = 0; i < 4; ++i) {
            vg[i] = T[0][rr * 128 + lane * 4 + i];
            vu[i] = T[1][rr * 128 + lane * 4 + i];
        }
        had128(vg, lane);
        had128(vu, lane);
        __half* hr = a.h + (size_t) (r0 + rr) * a.ldh + g * 128 + lane * 4;
#pragma unroll
        for (int i = 0; i < 4; ++i) {
            float gv = vg[i] * h2f(svg[lane * 4 + i]), uv = vu[i] * h2f(svu[lane * 4 + i]);
            if (a.limit > 0.0f) {
                gv = fminf(gv, a.limit);
                uv = fminf(fmaxf(uv, -a.limit), a.limit);
            }
            hr[i] = __float2half_rn((gv / (1.0f + expf(-gv))) * uv);
        }
    }
}

}  // namespace tc
#endif

// ---------------------------------------------------------------- dispatch
// F(K, cb) for K 1..8 and the three codebooks
template <template <int, int> class F, typename... A>
bool dispatch(int K, int cb, A&&... args) {
#define EXL3_CASE(KK)                                                    \
    case KK:                                                             \
        if (cb == 2) F<KK, 2>::run(std::forward<A>(args)...);            \
        else if (cb == 1) F<KK, 1>::run(std::forward<A>(args)...);       \
        else F<KK, 0>::run(std::forward<A>(args)...);                    \
        return true;
    switch (K) {
        EXL3_CASE(1) EXL3_CASE(2) EXL3_CASE(3) EXL3_CASE(4) EXL3_CASE(5) EXL3_CASE(6) EXL3_CASE(7) EXL3_CASE(8)
        default: return false;
    }
#undef EXL3_CASE
}

template <int K, int cb> struct RunMv {
    static void run(const KBatch& b, int blocks, cudaStream_t s) { strata_ds4_launch(mv_kernel<K, cb>, blocks, kThreads, 0, s, b); }
};
template <int K, int cb> struct RunWin {
    static void run(const KWin& a, int groups, cudaStream_t s) {
        strata_ds4_launch(win_gu_kernel<K, cb>, dim3((unsigned) (a.n_ff / kCols), (unsigned) groups, 2u), kThreads, 0, s, a);
    }
};
template <int K, int cb> struct RunWinDown {
    static void run(const KWin& a, int groups, cudaStream_t s) {
        strata_ds4_launch(win_down_kernel<K, cb>, dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, 0, s, a);
    }
};
template <int K, int cb> struct RunGU {
    static void run(const KMoe& a, cudaStream_t s) {
        strata_ds4_launch(moe_gate_up_kernel<K, cb>, dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k, 2u), kThreads, 0, s, a);
    }
};
template <int K, int cb> struct RunDown {
    static void run(const KMoe& a, cudaStream_t s) {
        strata_ds4_launch(moe_down_kernel<K, cb>, dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s, a);
    }
};
template <int K, int cb> struct RunRows {
    static void run(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
        strata_ds4_launch(recon_rows_kernel<K, cb>, dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s, 
            m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.suh, m.svh, m.k, dst, bf16);
    }
};
#if !defined(STRATA_USE_HIP)
template <int K, int cb> struct RunGemm {
    static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(tc::gemm_kernel<K, cb>, grid, kThreads, 0, s, a); }
};
template <int K, int cb> struct RunGateUp {
    static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) {
        strata_ds4_launch(tc::gate_up_kernel<K, cb>, grid, kThreads, 0, s, a);
    }
};
#endif
template <int K, int cb> struct RunInnerRows {
    static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {
        strata_ds4_launch(inner_rows_kernel<K, cb>, dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s, 
            m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.k, dst);
    }
};
template <int K, int cb> struct RunInner {
    static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {
        strata_ds4_launch(recon_inner_kernel<K, cb>, dim3((unsigned) (m.n / kCols), (unsigned) (m.k / 16)), kThreads, 0, s, 
            m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.n, dst);
    }
};

// ---------------------------------------------------------------- per-stream scratch
constexpr size_t kPartFloats = (size_t) 2 << 20;   // split partials: 8 MB
constexpr size_t kCounters = 16384;
constexpr size_t kCntGU = 8192;                    // the counters from here on: the experts' gate / up pairs
constexpr size_t kHFloats = (size_t) 8 * 8192;     // the experts' h
constexpr size_t kUFloats = (size_t) 128 * 4096;   // the experts' up values: a window's 16 tokens x 8, n_ff <= 4096
struct Scratch {
    float* part = nullptr;
    unsigned* cnt = nullptr;
    float* h = nullptr;
    float* u = nullptr;
};
std::mutex g_mu;
std::map<std::pair<int, cudaStream_t>, Scratch> g_scratch;

Scratch* scratch(cudaStream_t s) {
    int dev = 0;
    cudaGetDevice(&dev);
    // (the last stream this thread used: an engine thread per device launches without the lock)
    thread_local int t_dev = -1;
    thread_local cudaStream_t t_s = nullptr;
    thread_local Scratch* t_sc = nullptr;
    if (t_sc != nullptr && t_dev == dev && t_s == s) return t_sc;
    std::lock_guard<std::mutex> lk(g_mu);
    auto it = g_scratch.find({dev, s});
    if (it != g_scratch.end()) {
        t_dev = dev, t_s = s, t_sc = &it->second;
        return &it->second;
    }
    cudaStreamCaptureStatus cs = cudaStreamCaptureStatusNone;
    if (cudaStreamIsCapturing(s, &cs) == cudaSuccess && cs != cudaStreamCaptureStatusNone) {
        std::fprintf(stderr, "exl3: scratch for a stream first seen inside a graph capture (call exl3::prepare)\n");
        return nullptr;
    }
    Scratch sc;
    if (cudaMalloc(&sc.part, kPartFloats * sizeof(float)) != cudaSuccess ||
        cudaMalloc(&sc.cnt, kCounters * sizeof(unsigned)) != cudaSuccess ||
        cudaMalloc(&sc.h, kHFloats * sizeof(float)) != cudaSuccess ||
        cudaMalloc(&sc.u, kUFloats * sizeof(float)) != cudaSuccess ||
        cudaMemsetAsync(sc.cnt, 0, kCounters * sizeof(unsigned), s) != cudaSuccess) {
        std::fprintf(stderr, "exl3: scratch did not allocate\n");
        return nullptr;
    }
    cudaStreamSynchronize(s);
    return &(g_scratch[{dev, s}] = sc);
}

int sm_count() {
    int dev = 0, sms = 0;
    cudaGetDevice(&dev);
    cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
    return std::max(1, sms);
}

// Whether the current device decodes with v1.1.8's kernels (exl3_legacy.cu): NVIDIA before sm_80, where this file's
// tiles run on the CUDA cores and were slower; STRATA_GLM_EXL3_DECODE=legacy | new picks one on any GPU (A/B).  The
// token path and the verify windows take the same family on a device, so a window stays bit for bit its token path.
// Asked once per device.
bool use_legacy() {
#if defined(STRATA_USE_HIP) || defined(STRATA_METAL)
    // HIP has no v1.1.8 kernels.  Metal has neither them nor a compute capability to ask for: the shim answers
    // cudaDeviceGetAttribute with an error, so `major` stayed 0 and every Metal run without STRATA_GLM_EXL3_DECODE=new
    // took this branch and aborted in exl3::legacy::prepare's stub.  The 2.25 bpw pack never uses EXL3.
    return false;
#else
    static std::atomic<int> cache[64];   // 0 not asked, 1 v1.1.8's kernels, 2 this file's
    int dev = 0;
    cudaGetDevice(&dev);
    if (dev < 0 || dev >= 64) return false;
    int c = cache[dev].load(std::memory_order_acquire);
    if (c == 0) {
        int major = 0;
        cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
        const char* v = std::getenv("STRATA_GLM_EXL3_DECODE");
        c = v != nullptr && std::strcmp(v, "legacy") == 0 ? 1 : v != nullptr && std::strcmp(v, "new") == 0 ? 2
            : major < 8                                    ? 1
                                                           : 2;
        if (c == 1) std::fprintf(stderr, "exl3: CUDA%d decodes with v1.1.8's kernels\n", dev);
        cache[dev].store(c, std::memory_order_release);
    }
    return c == 1;
#endif
}

// the blocks of mv_kernel<K> an SM of the current device keeps at once (its registers and shared memory), asked once
// per device and K
template <int K, int cb> struct OccMv {
    static void run(int& n) {
        if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&n, mv_kernel<K, cb>, kThreads, 0) != cudaSuccess) {
            cudaGetLastError();
            n = 2;
        }
    }
};
int mv_occ(int K) {
    static std::atomic<int> cache[64][9];   // 0: not asked
    int dev = 0;
    cudaGetDevice(&dev);
    if (dev < 0 || dev >= 64 || K < 1 || K > 8) return 2;
    int n = cache[dev][K].load(std::memory_order_acquire);
    if (n == 0) {
        n = 2;
        dispatch<OccMv>(K, 2, n);
        n = std::max(1, n);
        cache[dev][K].store(n, std::memory_order_release);
    }
    return n;
}

// mv's input split for m: S blocks of kr inputs per output block, at most kMaxKR inputs a block (the same for every
// activation count, so a job of several sums each activation as a job of one).  About two blocks per SM; where an SM
// keeps three or more (`occ`: the RTX 3090's at K > 4) four a SM, down to 128 inputs a block - its decode sets the pace
// there, and the more, shorter blocks balance it over the SMs (RTX 3090, K6 / K5, us: a KDA layer's 8192 x 4096
// output 47.6 -> 45.5, the dense layers' 4096 x 12288 gate 58.9 -> 52.5 and 12288 x 4096 down 59.3 -> 52.2, the
// shared expert's 4096 x 2048 17.4 -> 16.5); with two a SM, a launch that would spill a short second wave (more blocks
// than the SMs keep, less than one and a half times) takes twice the splits (RTX 5060 Ti, K5: that gate 99.4 -> 84.8
// us, that down 100.6 -> 84.9)
// call_groups: the output blocks the call's jobs of this code count together against the target, not the job's own -
// where an SM keeps three or more, all of them (RTX 3090, K6, a KDA layer's q, k and v, three 4096 x 8192 jobs: 1536
// blocks 119.6 us, 384 blocks 105.9); at two a SM only jobs of the same shape (RTX 5060 Ti: those q, k and v 191 ->
// 185 us, a shared expert's gate and up 35.9 -> 35.0; the MLA layer's q_a and kv_a together would lose 8%).  Never the
// activations: a window's token gets the token path's split
void mv_split(const Mat& m, int sms, int occ, int& S, int& kr, int call_groups = 0) {
    const bool many = occ >= 3;
    const int groups = std::max(m.n / kCols, call_groups);
    const int kr_min = many ? 128 : 256, target = (many ? 4 : 2) * sms, P = occ * sms;
    S = (m.k + kMaxKR - 1) / kMaxKR;
    while (groups * S < target && m.k / (S * 2) >= kr_min) S *= 2;
    if (!many && groups * S > P && 2 * groups * S < 3 * P && m.k / (S * 2) >= kr_min) S *= 2;
    kr = ((m.k + S - 1) / S + 127) / 128 * 128;
    S = (m.k + kr - 1) / kr;
}

}  // namespace

Mat view(const Mat& m, int c0, int nc) {
    Mat v = m;
    v.ld = m.ld > 0 ? m.ld : m.n / 16;
    v.trellis = m.trellis + (size_t) (c0 / 16) * 8 * m.K;
    v.svh = m.svh + c0;
    v.n = nc;
    return v;
}

bool supported(const Mat& m) {
    return m.trellis != nullptr && m.suh != nullptr && m.svh != nullptr && m.k > 0 && m.n > 0 && m.k % 128 == 0 &&
           m.n % 128 == 0 && m.K >= 1 && m.K <= 8 && m.cb >= 0 && m.cb <= 2;
}

void set_skip(const int* flag) { g_skip = flag; }

int mv_splits(const Mat& m) {
    if (!supported(m)) return 0;
    int S = 1, kr = 0;
    mv_split(m, sm_count(), mv_occ(m.K), S, kr);
    return S;
}

int mv_cols(const Mat& m) {
    if (!supported(m)) return 0;
#if !defined(STRATA_USE_HIP)
    if (use_legacy()) return 4;   // (v1.1.8's kernels: the runs of four their callers have always used)
#endif
    const int S = mv_splits(m);
    if (S <= 0) return 0;
    if (S == 1) return kTok;
    return (int) std::max<size_t>(1, std::min<size_t>(kTok, kPartFloats / ((size_t) S * m.n)));
}

int launch_errors() {
#if !defined(STRATA_USE_HIP)
    return g_launch_errors.load(std::memory_order_relaxed) + legacy::launch_errors();
#else
    return g_launch_errors.load(std::memory_order_relaxed);
#endif
}

bool prepare(cudaStream_t s) {
    EXL3_LEGACY(prepare(s));
    return scratch(s) != nullptr;
}

bool several_ok() {
    EXL3_LEGACY(several_ok());
    return true;
}

bool mv(const MvJob* jobs, int n, cudaStream_t s) {
    EXL3_LEGACY(mv(jobs, n, s));
    if (n <= 0 || n > kMaxMvJobs) return false;
    Scratch* sc = scratch(s);
    if (sc == nullptr) return false;
    const int sms = sm_count();
    // the launch units: a job's activations in runs of up to kTok (one pass over its weights each)
    struct Unit {
        int j, c0, nc;
    };
    Unit units[kMaxMvJobs * kMaxMvCols];
    int nu = 0;
    const int run = kTok;
    for (int j = 0; j < n; ++j) {
        const Mat& m = *jobs[j].m;
        if (!supported(m) || jobs[j].y == nullptr || (jobs[j].xf == nullptr && jobs[j].xq == nullptr) ||
            jobs[j].ncol < 1 || jobs[j].ncol > kMaxMvCols) {
            std::fprintf(stderr, "exl3 mv: job %d unsupported (k %d n %d K %d cb %d, %d activations)\n", j, m.k, m.n,
                         m.K, m.cb, jobs[j].ncol);
            return false;
        }
        for (int c0 = 0; c0 < jobs[j].ncol; c0 += run) units[nu++] = {j, c0, std::min(run, jobs[j].ncol - c0)};
    }
    // each job's call_groups (mv_split): the output blocks of the call's jobs of its code - of its shape too where an SM
    // keeps fewer than three
    int cg[kMaxMvJobs] = {};
    for (int j = 0; j < n; ++j) {
        const Mat& a = *jobs[j].m;
        const bool many = mv_occ(a.K) >= 3;
        for (int q = 0; q < n; ++q) {
            const Mat& b = *jobs[q].m;
            if (b.K == a.K && b.cb == a.cb && (many || (b.k == a.k && b.n == a.n))) cg[j] += b.n / kCols;
        }
    }
    bool done[sizeof(units) / sizeof(units[0])] = {};
    for (int i = 0; i < nu; ++i) {
        if (done[i]) continue;
        const Mat& m0 = *jobs[units[i].j].m;
        // a launch: the units of one (K, codebook), as many as KBatch and the split scratch take (the next launch on
        // the stream may use the scratch again: this one's last blocks are done with it by then)
        KBatch b{};
        int blocks = 0, nb = 0;
        size_t part_at = 0, cnt_at = 0;
        for (int u = i; u < nu && nb < kMaxMvJobs; ++u) {
            const MvJob& job = jobs[units[u].j];
            const Mat& m = *job.m;
            if (done[u] || m.K != m0.K || m.cb != m0.cb) continue;
            const int c0 = units[u].c0, nc = units[u].nc, groups = m.n / kCols;
            int S = 1, kr = 0;
            mv_split(m, sms, mv_occ(m.K), S, kr, cg[units[u].j]);
            const size_t need = S > 1 ? (size_t) S * m.n * nc : 0;
            if (S > 1 && (part_at + need > kPartFloats || cnt_at + (size_t) groups > kCounters)) {
                if (nb > 0) continue;   // the next launch
                std::fprintf(stderr, "exl3 mv: scratch too small for k %d n %d (%d activations)\n", m.k, m.n, nc);
                return false;
            }
            done[u] = true;
            KJob& J = b.j[nb];
            J.tr = m.trellis;
            J.suh = m.suh;
            J.svh = m.svh;
            J.xf = job.xf != nullptr ? job.xf + c0 * job.xf_cs : nullptr;
            J.xq = job.xq != nullptr ? (const Q81*) ((const uint8_t*) job.xq + c0 * job.xq_cs) : nullptr;
            J.y = job.y + c0 * job.y_cs;
            J.bias = job.bias;
            J.alpha = job.alpha;
            J.k = m.k;
            J.n = m.n;
            J.ld = m.ld > 0 ? m.ld : m.n / 16;
            J.S = S;
            J.kr = kr;
            J.part = sc->part + part_at;
            J.cnt = sc->cnt + cnt_at;
            J.nc = nc;
            J.xf_cs = job.xf_cs;
            J.xq_cs = job.xq_cs;
            J.y_cs = job.y_cs;
            part_at += need;
            if (S > 1) cnt_at += (size_t) groups;
            blocks += groups * S;
            b.blk_end[nb++] = blocks;
        }
        b.n = nb;
        b.skip = g_skip;
        if (!dispatch<RunMv>(m0.K, m0.cb, b, blocks, s)) return false;
    }
    launch_check("mv");
    return true;
}

static bool moe_args(const MoeArgs& a, KMoe& m, cudaStream_t s) {
    Scratch* sc = scratch(s);
    if (sc == nullptr) return false;
    if (a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxMoeK ||
        a.n_ff > kMaxMoeK || (size_t) a.k * a.n_ff > kHFloats || (size_t) a.k * a.n_embd > kPartFloats ||
        (size_t) (a.n_embd / kCols) > kCntGU || kCntGU + (size_t) a.k * (a.n_ff / kCols) > kCounters) {
        std::fprintf(stderr, "exl3 moe: geometry not covered (k %d n_embd %d n_ff %d)\n", a.k, a.n_embd, a.n_ff);
        return false;
    }
    m = KMoe{};
    m.plan_ptr = a.plan_ptr;
    m.plan_w = a.plan_w;
    m.cpu_part = a.cpu_part;
    m.cpu_flag = a.cpu_flag;
    m.h = sc->h;
    m.u = sc->u;
    m.part = sc->part;
    m.cnt = sc->cnt;
    m.cnt_gu = sc->cnt + kCntGU;
    for (int r = 0; r < 3; ++r) {
        m.suh[r] = a.lay.suh[r];
        m.svh[r] = a.lay.svh[r];
        m.tr[r] = a.lay.trellis[r];
    }
    m.k = a.k;
    m.n_embd = a.n_embd;
    m.n_ff = a.n_ff;
    m.limit = a.limit;
    m.skip = g_skip;
    return true;
}

void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s) {
    EXL3_LEGACY(moe_gate_up(a, x, h, s));
    KMoe m;
    if (!moe_args(a, m, s)) return;
    if (a.lay.K[0] != a.lay.K[1]) {
        std::fprintf(stderr, "exl3 moe_gate_up: gate K %d != up K %d\n", a.lay.K[0], a.lay.K[1]);
        return;
    }
    m.x = x;
    if (h != nullptr) m.h = h;
    if (!dispatch<RunGU>(a.lay.K[0], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_gate_up: K %d\n", a.lay.K[0]);
    launch_check("moe_gate_up");
}

void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {
    EXL3_LEGACY(moe_down(a, h, sh_out, out, s));
    KMoe m;
    if (!moe_args(a, m, s)) return;
    if (h != nullptr) m.h = const_cast<float*>(h);
    m.sh_out = sh_out;
    m.out = out;
    if (!dispatch<RunDown>(a.lay.K[2], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_down: K %d\n", a.lay.K[2]);
    launch_check("moe_down");
}

bool win_experts(const WinArgs& a, cudaStream_t s) {
    EXL3_LEGACY(win_experts(a, s));
    if (a.T < 1 || a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxMoeK ||
        a.n_ff > kMaxMoeK || a.lay.K[0] != a.lay.K[1] || a.h == nullptr || a.part == nullptr || a.out == nullptr) {
        std::fprintf(stderr, "exl3 win_experts: not covered (T %d k %d n_embd %d n_ff %d, gate K %d up K %d)\n", a.T,
                     a.k, a.n_embd, a.n_ff, a.lay.K[0], a.lay.K[1]);
        return false;
    }
    Scratch* sc = scratch(s);
    if (sc == nullptr) return false;
    const int groups = a.T * a.k;   // at most (the blocks past the real count return at once)
    if ((size_t) a.T * 8 * a.n_ff > kUFloats || kCntGU + (size_t) groups * (a.n_ff / kCols) > kCounters) {
        std::fprintf(stderr, "exl3 win_experts: %d tokens are more than the scratch takes\n", a.T);
        return false;
    }
    KWin w{};
    w.grp_ptr = a.grp_ptr;
    w.grp_start = a.grp_start;
    w.grp_ent = a.grp_ent;
    w.n_groups = a.n_groups;
    w.plan_ptr = a.plan_ptr;
    w.plan_w = a.plan_w;
    w.cpu_part = a.cpu_part;
    w.cpu_stride = a.cpu_stride != 0 ? a.cpu_stride : (size_t) a.n_embd;
    w.cpu_flag = a.cpu_flag;
    w.x = a.x;
    w.x_stride = a.x_stride;
    w.h = a.h;
    w.u = sc->u;
    w.cnt_gu = sc->cnt + kCntGU;
    w.part = a.part;
    w.sh_out = a.sh_out;
    w.out = a.out;
    w.out_stride = a.out_stride;
    for (int r = 0; r < 3; ++r) {
        w.suh[r] = a.lay.suh[r];
        w.svh[r] = a.lay.svh[r];
        w.tr[r] = a.lay.trellis[r];
    }
    w.k = a.k;
    w.n_embd = a.n_embd;
    w.n_ff = a.n_ff;
    w.limit = a.limit;
    if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||
        !dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))
        return false;
    if (a.wait_ptr != nullptr) strata_ds4_launch(win_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);
    strata_ds4_launch(win_combine_kernel, dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s, w);
    launch_check("win_experts");
    return true;
}

bool mma_supported() {
#if defined(STRATA_USE_HIP)
    return false;
#else
    int dev = 0, major = 0;
    cudaGetDevice(&dev);
    cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
    return major >= 8;
#endif
}

void set_gate_up(const SetArgs& sa, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,
                 cudaStream_t s) {
#if !defined(STRATA_USE_HIP)
    if (sa.lay.K[0] != sa.lay.K[1]) {
        std::fprintf(stderr, "exl3 set_gate_up: gate K %d != up K %d\n", sa.lay.K[0], sa.lay.K[1]);
        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
        return;
    }
    if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
    tc::GArgs a{};
    a.wbase = sa.wbase;
    a.stride = sa.stride;
    for (int r = 0; r < 2; ++r) {
        a.tr[r] = sa.lay.trellis[r];
        a.suh[r] = sa.lay.suh[r];
        a.svh[r] = sa.lay.svh[r];
        a.ld[r] = sa.n_ff / 16;
    }
    a.bounds = sa.bounds;
    a.row0 = sa.row0;
    a.k = sa.n_embd;
    a.n = sa.n_ff;
    a.xf = x;
    a.idx = row_idx;
    a.ldx = ldx;
    a.h = (__half*) h16;
    a.ldh = ldh;
    a.limit = sa.limit;
    const dim3 grid((unsigned) ((sa.max_rows + tc::BM - 1) / tc::BM), (unsigned) (sa.n_ff / kCols), (unsigned) sa.n_exp);
    if (!dispatch<RunGateUp>(sa.lay.K[0], sa.lay.cb, a, grid, s)) std::fprintf(stderr, "exl3 set_gate_up: K\n");
    launch_check("set_gate_up");
#else
    (void) sa; (void) x; (void) ldx; (void) row_idx; (void) h16; (void) ldh; (void) s;
#endif
}

void set_down(const SetArgs& sa, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s) {
#if !defined(STRATA_USE_HIP)
    if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
    tc::GArgs a{};
    a.wbase = sa.wbase;
    a.stride = sa.stride;
    a.tr[0] = sa.lay.trellis[2];
    a.suh[0] = sa.lay.suh[2];
    a.svh[0] = sa.lay.svh[2];
    a.ld[0] = sa.n_embd / 16;
    a.bounds = sa.bounds;
    a.row0 = sa.row0;
    a.k = sa.n_ff;
    a.n = sa.n_embd;
    a.xh = (const __half*) h16;
    a.ldx = ldh;
    a.y = y;
    a.ldy = ldy;
    const dim3 grid((unsigned) ((sa.max_rows + tc::BM - 1) / tc::BM), (unsigned) (sa.n_embd / kCols), (unsigned) sa.n_exp);
    if (!dispatch<RunGemm>(sa.lay.K[2], sa.lay.cb, a, grid, s)) std::fprintf(stderr, "exl3 set_down: K\n");
    launch_check("set_down");
#else
    (void) sa; (void) h16; (void) ldh; (void) y; (void) ldy; (void) s;
#endif
}

void gemm_rows(const Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta, cudaStream_t s) {
#if !defined(STRATA_USE_HIP)
    if (rows <= 0) return;
    tc::GArgs a{};
    a.wbase = nullptr;
    a.tr[0] = (size_t) m.trellis;
    a.suh[0] = (size_t) m.suh;
    a.svh[0] = (size_t) m.svh;
    a.ld[0] = m.ld > 0 ? m.ld : m.n / 16;
    a.m = rows;
    a.k = m.k;
    a.n = m.n;
    a.xh = (const __half*) x16;
    a.ldx = ldx;
    a.y = y;
    a.ldy = ldy;
    a.beta = beta;
    const dim3 grid((unsigned) ((rows + tc::BM - 1) / tc::BM), (unsigned) (m.n / kCols), 1);
    if (!supported(m) || !dispatch<RunGemm>(m.K, m.cb, a, grid, s)) {
        std::fprintf(stderr, "exl3 gemm_rows: unsupported matrix\n");
        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
        return;
    }
    launch_check("gemm_rows");
#else
    (void) m; (void) x16; (void) ldx; (void) rows; (void) y; (void) ldy; (void) beta; (void) s;
#endif
}

void reconstruct_rows(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
    if (!supported(m) || !dispatch<RunRows>(m.K, m.cb, m, dst, bf16, s)) {
        std::fprintf(stderr, "exl3 reconstruct_rows: unsupported matrix\n");
        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
        return;
    }
    launch_check("reconstruct_rows");
}

void reconstruct_inner_rows(const Mat& m, uint16_t* dst, cudaStream_t s) {
    if (!supported(m) || !dispatch<RunInnerRows>(m.K, m.cb, m, dst, s)) {
        std::fprintf(stderr, "exl3 reconstruct_inner_rows: unsupported matrix\n");
        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
        return;
    }
    launch_check("reconstruct_inner_rows");
}

void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, const uint16_t* suh, uint16_t* out,
                  cudaStream_t s) {
    if (nrows <= 0) return;
    const long long warps = (long long) nrows * (ncols / kCols);
    strata_ds4_launch(rows_rot_f16_kernel, (unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s, x, ldx, idx, nrows, ncols, suh,
                                                                                         out);
    launch_check("rows_rot_f16");
}

void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s) {
    if (nrows <= 0) return;
    const long long warps = (long long) nrows * (ncols / kCols);
    strata_ds4_launch(rows_rot_out_kernel, (unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s, y, ldy, nrows, ncols, svh);
    launch_check("rows_rot_out");
}

void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s) {
    if (nrows <= 0) return;
#if defined(STRATA_METAL)
    if (::strata::dsv41::metal_host::rows_f16(x, ldx, idx, nrows, ncols, out)) return;
#endif
    strata_ds4_launch(rows_f16_kernel, dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s, x, ldx, idx,
                                                                                                         ncols, out);
    launch_check("rows_f16");
}

void reconstruct_inner(const Mat& m, uint16_t* dst, cudaStream_t s) {
    if (!supported(m) || !dispatch<RunInner>(m.K, m.cb, m, dst, s)) {
        std::fprintf(stderr, "exl3 reconstruct_inner: unsupported matrix\n");
        g_launch_errors.fetch_add(1, std::memory_order_relaxed);
        return;
    }
    launch_check("reconstruct_inner");
}

}  // namespace strata::dsv41k::exl3