Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).
#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_gguf.cuh"
#if defined(STRATA_METAL)
#include <ds4_metal_host.h>
#endif
#include "ds4_iq.cuh"
#if defined(STRATA_METAL)
#include "ds4_gguf_cpu.hpp" // gg_row_to_bf16 (the DS4_DBG_IQ check)
#endif
#include "ds4_numerics.cuh"
#include <cuda_fp16.h>
#if !defined(STRATA_USE_HIP)
#include <mma.h>
#endif
#include <algorithm>
#include <cstdlib>
#include <atomic>
#include <cstdio>
#include <cstring>
#include <mutex>
#include <unordered_map>
namespace strata::ds4::gguf {
namespace {
std::atomic<int> g_errors{0};
void check(const char* what) {
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) {
std::fprintf(stderr, "ds4 gguf %s: %s\n", what, cudaGetErrorString(e));
g_errors.fetch_add(1, std::memory_order_relaxed);
}
}
void fail(const char* what) {
std::fprintf(stderr, "ds4 gguf %s\n", what);
g_errors.fetch_add(1, std::memory_order_relaxed);
}
constexpr int kWarps = 8, kThreads = kWarps * 32;
constexpr int kMaxEnt = 16; // a group's entries in one pass (a window's tokens at most)
__device__ __forceinline__ float warp_sum(float v) {
#pragma unroll
for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
return v;
}
// ---------------------------------------------------------------- BF16 matrices
struct BfJob {
const void* w;
size_t ld; // values a row (BF16), bytes a row (MXFP4)
const float* x;
size_t xcs;
float* y;
size_t ycs;
const float* bias;
float alpha;
int k, n, ncol;
const int8_t* xq; // (Q8_0: the activations rounded to int8, q8_quant's)
const float* xd;
};
struct BfJobs {
BfJob j[x3::kMaxMvJobs];
};
// grid (max rows / 8, jobs): a warp a row, each of its activations' dot in F32 - a lane 8 BF16 weights (16 bytes) and
// 8 activations a step (VEC), or one of each (rows or activations not 16-byte aligned)
template <int NC, bool VEC>
__global__ void __launch_bounds__(kThreads) bf16_mv_kernel(const BfJobs J) {
const BfJob& b = J.j[blockIdx.y];
const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
if (row >= b.n) return;
const uint16_t* w = (const uint16_t*) b.w + (size_t) row * b.ld;
float acc[NC];
#pragma unroll
for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
if (VEC) {
for (int i = lane * 8; i < b.k; i += 256) {
const uint4 q = *(const uint4*) (w + i);
const float wf[8] = {__uint_as_float(q.x << 16), __uint_as_float(q.x & 0xffff0000u),
__uint_as_float(q.y << 16), __uint_as_float(q.y & 0xffff0000u),
__uint_as_float(q.z << 16), __uint_as_float(q.z & 0xffff0000u),
__uint_as_float(q.w << 16), __uint_as_float(q.w & 0xffff0000u)};
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) {
const float4* xp = (const float4*) (b.x + c * b.xcs + i);
const float4 x0 = __ldg(xp), x1 = __ldg(xp + 1);
float a = acc[c];
a = fmaf(wf[0], x0.x, a);
a = fmaf(wf[1], x0.y, a);
a = fmaf(wf[2], x0.z, a);
a = fmaf(wf[3], x0.w, a);
a = fmaf(wf[4], x1.x, a);
a = fmaf(wf[5], x1.y, a);
a = fmaf(wf[6], x1.z, a);
a = fmaf(wf[7], x1.w, a);
acc[c] = a;
}
}
} else {
for (int i = lane; i < b.k; i += 32) {
const float wf = __uint_as_float((uint32_t) w[i] << 16);
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) acc[c] = fmaf(wf, b.x[c * b.xcs + i], acc[c]);
}
}
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) {
const float v = warp_sum(acc[c]);
if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
}
__global__ void q8_rows_kernel(const uint8_t* __restrict__ w, size_t ld, int k, int n, uint16_t* __restrict__ dst,
bool bf16) {
const size_t total = (size_t) n * (size_t) k;
for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {
const size_t r = i / (size_t) k, c = i - r * (size_t) k;
const uint8_t* p = w + r * ld + (c / 32) * kQ8Bytes;
const float v = __half2float(__ushort_as_half(*(const uint16_t*) p)) * (float) (int8_t) p[2 + c % 32];
dst[i] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));
}
}
__global__ void bf16_rows_kernel(const uint16_t* __restrict__ w, size_t ld, int k, int n, uint16_t* __restrict__ dst,
bool bf16) {
const size_t total = (size_t) n * (size_t) k;
for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {
const size_t r = i / (size_t) k, c = i - r * (size_t) k;
const uint16_t b = w[r * ld + c];
dst[i] = bf16 ? b : __half_as_ushort(__float2half_rn(__uint_as_float((uint32_t) b << 16)));
}
}
// ---------------------------------------------------------------- MXFP4 experts
__device__ __forceinline__ float e8m0_half(uint32_t e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF
return __uint_as_float(e < 2 ? 0x00200000u << e : (e - 1) << 23);
}
__device__ __forceinline__ void lut_init(float* lut) {
if (threadIdx.x < 16) {
const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};
lut[threadIdx.x] = threadIdx.x < 8 ? v[threadIdx.x] : -v[threadIdx.x - 8];
}
}
// block p's 32 values, exactly (a code times a power of two)
__device__ __forceinline__ void mx_block(const uint8_t* __restrict__ p, const float* lut, float* w) {
const float d = e8m0_half(p[0]);
#pragma unroll
for (int j = 0; j < 16; ++j) {
const uint32_t q = p[1 + j];
w[j] = lut[q & 15u] * d;
w[j + 16] = lut[q >> 4] * d;
}
}
__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {
if (limit > 0.0f) {
gv = fminf(gv, limit);
uv = fminf(fmaxf(uv, -limit), limit);
}
return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
}
__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {
float m = 0.0f;
for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);
if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);
return __fadd_rn(m, sh);
}
// a Q8_0 block's 32 values, exactly (an FP16 scale times an int8; the block is only 2-byte aligned)
__device__ __forceinline__ void q8_block(const uint8_t* __restrict__ p, float* w) {
const float d = __half2float(__ushort_as_half(*(const uint16_t*) p));
const uint16_t* q = (const uint16_t*) (p + 2);
#pragma unroll
for (int j = 0; j < 16; ++j) {
const uint16_t v = q[j];
w[2 * j] = d * (float) (int8_t) (v & 0xff);
w[2 * j + 1] = d * (float) (int8_t) (v >> 8);
}
}
// ---- int8 activations (llama.cpp's MMVQ: its q8_1 blocks' rounding)
// rows of k F32 values (row r at x + r ldx; k % 32 == 0) -> q[r][k] int8 and d[r][k / 32] = max|x| / 127 per 32
__global__ void __launch_bounds__(kThreads) q8_quant_kernel(const float* __restrict__ x, size_t ldx, int k,
int8_t* __restrict__ q, float* __restrict__ d) {
const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
if (b * 32 >= k) return;
const float v = x[(size_t) r * ldx + (size_t) b * 32 + lane];
float am = fabsf(v);
#pragma unroll
for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
const float sc = am / 127.0f;
q[(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
if (lane == 0) d[(size_t) r * (k / 32) + b] = sc;
}
void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {
const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);
#if defined(STRATA_METAL)
if (::strata::dsv41::metal_host::q8_quant_rows(x, ldx, rows, k, (int8_t*) q, d)) return;
#endif
strata_ds4_launch(q8_quant_kernel, grid, kThreads, 0, s, x, ldx, k, q, d);
}
// Several activations' rounding in one launch (a GEMV call's jobs: wo_a's eight groups, the shared expert's two, ...): item z is
// q8_quant's (x, ldx, rows, k, q, d), the same per-element arithmetic, so the bytes are the single launches'. Identical items
// (the same x, stride, rows and k: two matrices reading one input) are rounded once and share the result.
constexpr int kQuantItems = x3::kMaxMvJobs;
struct QuantItems {
const float* x[kQuantItems];
size_t ldx[kQuantItems];
int k[kQuantItems], rows[kQuantItems];
int8_t* q[kQuantItems];
float* d[kQuantItems];
int n;
};
__global__ void __launch_bounds__(kThreads) q8_quant_multi_kernel(QuantItems it) {
const int z = blockIdx.z;
if (z >= it.n) return;
const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
const int k = it.k[z];
if (r >= it.rows[z] || b * 32 >= k) return;
const float v = it.x[z][(size_t) r * it.ldx[z] + (size_t) b * 32 + lane];
float am = fabsf(v);
#pragma unroll
for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
const float sc = am / 127.0f;
it.q[z][(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
if (lane == 0) it.d[z][(size_t) r * (k / 32) + b] = sc;
}
// adds an item unless an identical one is queued; returns the item whose result to read (its q and d)
int quant_add(QuantItems& it, const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {
static const bool nodedup = std::getenv("DS4_QUANT_NODEDUP") != nullptr;
for (int i = 0; i < it.n && !nodedup; ++i)
if (it.x[i] == x && it.ldx[i] == ldx && it.rows[i] == rows && it.k[i] == k) return i;
const int i = it.n++;
it.x[i] = x;
it.ldx[i] = ldx;
it.rows[i] = rows;
it.k[i] = k;
it.q[i] = q;
it.d[i] = d;
return i;
}
void quant_flush(const QuantItems& it, cudaStream_t s) {
if (it.n <= 0) return;
static const bool single = std::getenv("DS4_QUANT_SINGLE") != nullptr;
#if defined(STRATA_METAL)
if (!single && std::getenv("DS4_NO_Q8_BATCH") == nullptr) { // the call's roundings in one dispatch (a dispatch a job was 45% of the decode token's)
struct Spec { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } sp[kQuantItems];
for (int i = 0; i < it.n; ++i) sp[i] = Spec{it.x[i], (uint64_t) it.ldx[i], (uint32_t) it.rows[i], (uint32_t) it.k[i], it.q[i], it.d[i]};
if (::strata::dsv41::metal_host::q8_quant_multi(sp, it.n)) return;
}
for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);
return;
#endif
if (single) {
for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);
return;
}
int kmax = 0, rmax = 0;
for (int i = 0; i < it.n; ++i) {
kmax = std::max(kmax, it.k[i]);
rmax = std::max(rmax, it.rows[i]);
}
const dim3 grid((unsigned) ((kmax / 32 + kWarps - 1) / kWarps), (unsigned) rmax, (unsigned) it.n);
strata_ds4_launch(q8_quant_multi_kernel, grid, kThreads, 0, s, it);
}
// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),
// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4
__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {
const uint32_t t0 = 0x03020100u, t1 = 0x0C080604u, t2 = 0xFDFEFF00u, t3 = 0xF4F8FAFCu;
const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
uint32_t tmp[2];
#pragma unroll
for (int i = 0; i < 2; ++i) {
const uint32_t idx = (q4 >> (16 * i)) & 0x7777u;
const uint32_t lo = __byte_perm(t0, t1, idx), hi = __byte_perm(t2, t3, idx);
tmp[i] = __byte_perm(lo, hi, sel >> (16 * i));
}
return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));
}
// an MXFP4 block's codes (4 words) against 32 int8 activations (xa: values 0..15, xb: 16..31)
__device__ __forceinline__ int mx_dot(const uint4& q, const int4& xa, const int4& xb) {
int sum = 0;
int2 v = e2m1_x8(q.x);
sum = __dp4a(v.x, xa.x, sum);
sum = __dp4a(v.y, xb.x, sum);
v = e2m1_x8(q.y);
sum = __dp4a(v.x, xa.y, sum);
sum = __dp4a(v.y, xb.y, sum);
v = e2m1_x8(q.z);
sum = __dp4a(v.x, xa.z, sum);
sum = __dp4a(v.y, xb.z, sum);
v = e2m1_x8(q.w);
sum = __dp4a(v.x, xa.w, sum);
sum = __dp4a(v.y, xb.w, sum);
return sum;
}
// the warp copies the blocks [b0, b0 + nbk) of a row (BB bytes each; the row 4-byte aligned, nbk BB % 4 == 0) into its
// stage in 4-byte words; every lane calls it
template <int BB>
__device__ __forceinline__ void stage_chunk(const uint8_t* row, int b0, int nbk, uint32_t* st) {
const uint32_t* src = (const uint32_t*) (row + (size_t) b0 * BB);
__syncwarp(); // (the lanes are done with the chunk before)
for (int i = threadIdx.x & 31; i < nbk * BB / 4; i += 32) st[i] = src[i];
__syncwarp();
}
// a row's next chunk of 32 blocks (BB bytes each) in the lanes' registers, then into the warp's stage
template <int BB>
struct Fetch {
static constexpr int W = (32 * BB / 4 + 31) / 32; // words a lane
uint32_t r[W];
int n = 0;
__device__ __forceinline__ void load(const uint8_t* row, int b0, int nbk) {
const uint32_t* src = (const uint32_t*) (row + (size_t) b0 * BB);
n = nbk * BB / 4;
#pragma unroll
for (int j = 0; j < W; ++j) {
const int i = (int) (threadIdx.x & 31) + 32 * j;
r[j] = i < n ? src[i] : 0u;
}
}
__device__ __forceinline__ void store(uint32_t* st) const {
__syncwarp(); // (the lanes are done with the chunk before)
#pragma unroll
for (int j = 0; j < W; ++j) {
const int i = (int) (threadIdx.x & 31) + 32 * j;
if (i < n) st[i] = r[j];
}
__syncwarp();
}
};
// MXFP4 block i of a stage (17-byte blocks from a 4-byte aligned start): its scale byte, its 16 code bytes as 4 words
__device__ __forceinline__ uint32_t mx_words_s(const uint32_t* st, int i, uint4& q) {
const int o = 17 * i, w = o >> 2, sh = (o & 3) * 8;
const uint32_t a0 = st[w], a1 = st[w + 1], a2 = st[w + 2], a3 = st[w + 3], a4 = st[w + 4];
const uint32_t s0 = __funnelshift_r(a0, a1, sh), s1 = __funnelshift_r(a1, a2, sh), s2 = __funnelshift_r(a2, a3, sh),
s3 = __funnelshift_r(a3, a4, sh), s4 = a4 >> sh;
q.x = __funnelshift_r(s0, s1, 8);
q.y = __funnelshift_r(s1, s2, 8);
q.z = __funnelshift_r(s2, s3, 8);
q.w = __funnelshift_r(s3, s4, 8);
return s0 & 0xffu;
}
// ... the same from global memory, byte by byte (rows not 4-byte aligned)
__device__ __forceinline__ uint32_t mx_words_g(const uint8_t* __restrict__ p, uint4& q) {
uint32_t w[4];
#pragma unroll
for (int j = 0; j < 4; ++j)
w[j] = (uint32_t) p[1 + 4 * j] | (uint32_t) p[2 + 4 * j] << 8 | (uint32_t) p[3 + 4 * j] << 16 |
(uint32_t) p[4 + 4 * j] << 24;
q = make_uint4(w[0], w[1], w[2], w[3]);
return p[0];
}
// Q8_0 block i of a stage (34-byte blocks): its scale, its 32 int8 as 8 words
__device__ __forceinline__ float q8_words_s(const uint32_t* st, int i, int* v) {
const int o = 34 * i, w = o >> 2, sh = (o & 3) * 8;
uint32_t a[9];
#pragma unroll
for (int j = 0; j < 9; ++j) a[j] = st[w + j];
uint32_t sw[9];
#pragma unroll
for (int j = 0; j < 8; ++j) sw[j] = __funnelshift_r(a[j], a[j + 1], sh);
sw[8] = a[8] >> sh;
#pragma unroll
for (int j = 0; j < 8; ++j) v[j] = (int) __funnelshift_r(sw[j], sw[j + 1], 16);
return __half2float(__ushort_as_half((uint16_t) (sw[0] & 0xffffu)));
}
__device__ __forceinline__ float q8_words_g(const uint8_t* __restrict__ p, int* v) {
#pragma unroll
for (int j = 0; j < 8; ++j)
v[j] = (int) ((uint32_t) p[2 + 4 * j] | (uint32_t) p[3 + 4 * j] << 8 | (uint32_t) p[4 + 4 * j] << 16 |
(uint32_t) p[5 + 4 * j] << 24);
return __half2float(__ushort_as_half((uint16_t) (p[0] | p[1] << 8)));
}
// grid (max rows / 8, jobs): a Q8_0 matrix's rows against the jobs' int8 activations (q8_quant's: activation c of job
// j at xq + c k, its scales at xd + c k / 32) - a warp a row, a lane a block a step, the blocks staged (ST) or read
template <int NC, bool ST>
__global__ void __launch_bounds__(kThreads) q8_mv_kernel(const BfJobs J) {
__shared__ __align__(16) uint32_t stage[kWarps][32 * kQ8Bytes / 4];
const BfJob& b = J.j[blockIdx.y];
const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
if (row >= b.n) return;
const uint8_t* w = (const uint8_t*) b.w + (size_t) row * b.ld;
const int nb = b.k / 32;
float acc[NC];
#pragma unroll
for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
Fetch<kQ8Bytes> F;
if (ST) F.load(w, 0, min(32, nb));
for (int b0 = 0; b0 < nb; b0 += 32) {
const int nbk = min(32, nb - b0), kb = b0 + lane;
if (ST) {
F.store(stage[threadIdx.x >> 5]);
if (b0 + 32 < nb) F.load(w, b0 + 32, min(32, nb - b0 - 32));
}
if (lane >= nbk) continue;
int v[8];
const float dw = ST ? q8_words_s(stage[threadIdx.x >> 5], lane, v) : q8_words_g(w + (size_t) kb * kQ8Bytes, v);
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) {
const int4* xp = (const int4*) (b.xq + (size_t) c * b.k + (size_t) kb * 32);
const int4 xa = xp[0], xb = xp[1];
int sum = __dp4a(v[0], xa.x, 0);
sum = __dp4a(v[1], xa.y, sum);
sum = __dp4a(v[2], xa.z, sum);
sum = __dp4a(v[3], xa.w, sum);
sum = __dp4a(v[4], xb.x, sum);
sum = __dp4a(v[5], xb.y, sum);
sum = __dp4a(v[6], xb.z, sum);
sum = __dp4a(v[7], xb.w, sum);
acc[c] = fmaf(dw * b.xd[(size_t) c * nb + kb], (float) sum, acc[c]);
}
}
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) {
const float v = warp_sum(acc[c]);
if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
}
// grid (max rows / 8, jobs): bf16_mv_kernel for MXFP4 (Q8 false) or Q8_0 (Q8 true) matrices - a lane a 32-value block
// a step, dequantised once for the job's activations
template <int NC, bool VEC, bool Q8>
__global__ void __launch_bounds__(kThreads) mx_mv_kernel(const BfJobs J) {
__shared__ float lut[16];
lut_init(lut);
__syncthreads();
const BfJob& b = J.j[blockIdx.y];
const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
if (row >= b.n) return;
const uint8_t* w = (const uint8_t*) b.w + (size_t) row * b.ld;
float acc[NC];
#pragma unroll
for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
for (int kb = lane; kb < b.k / kMxBlock; kb += 32) {
float wv[32];
if (Q8) q8_block(w + (size_t) kb * kQ8Bytes, wv);
else mx_block(w + (size_t) kb * kMxBytes, lut, wv);
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) {
const float* x = b.x + c * b.xcs + kb * kMxBlock;
float a = acc[c];
if (VEC) {
#pragma unroll
for (int q = 0; q < 8; ++q) {
const float4 xv = __ldg((const float4*) x + q);
a = fmaf(wv[4 * q], xv.x, a);
a = fmaf(wv[4 * q + 1], xv.y, a);
a = fmaf(wv[4 * q + 2], xv.z, a);
a = fmaf(wv[4 * q + 3], xv.w, a);
}
} else {
#pragma unroll
for (int j = 0; j < 32; ++j) a = fmaf(wv[j], x[j], a);
}
acc[c] = a;
}
}
#pragma unroll
for (int c = 0; c < NC; ++c)
if (c < b.ncol) {
const float v = warp_sum(acc[c]);
if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
}
// a token's experts (grp_ptr null: group i = expert i of the plan, plan_ptr[i] its blob or 0 off the device, its one
// entry i, every entry's input x) or a window's groups (group g: one expert, blob grp_ptr[g], entries e = t * 8 + i at
// grp_ent[grp_start[g] ..), entry e's input x + (e >> 3) x_stride)
struct MxArgs {
const unsigned long long* plan_ptr;
const unsigned long long* grp_ptr;
const int *grp_start, *grp_ent, *n_groups;
const float* x;
size_t x_stride;
float* h; // [entries][n_ff]
float* part; // [entries][n_embd]
size_t tr[3];
int n_embd, n_ff;
float limit;
const int8_t* xq; // x rounded to int8 (a token's: row 0; a window's: row t), q8_quant's rows of n_embd
const float* xd;
const int8_t* hq; // h rounded to int8, row e = the entry, rows of n_ff
const float* hd;
};
__device__ __forceinline__ bool mx_group(const MxArgs& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
if (a.grp_ptr == nullptr) {
const unsigned long long p = a.plan_ptr[grp];
if (p == 0ull) return false;
blob = (const uint8_t*) p;
m0 = grp;
m1 = grp + 1;
return true;
}
if (grp >= *a.n_groups) return false;
blob = (const uint8_t*) a.grp_ptr[grp];
m0 = a.grp_start[grp];
m1 = a.grp_start[grp + 1];
return true;
}
// grid (n_ff / 8, groups): a warp a row of the gate and of the up, for NE of the group's entries a pass (each weight
// block decoded once for them) against the entries' int8 x -> h = the clamped swiglu
template <int NE, bool ST>
__global__ void __launch_bounds__(kThreads) mx_gate_up_kernel(const MxArgs a) {
__shared__ __align__(16) uint32_t stage[kWarps][2][32 * kMxBytes / 4];
__shared__ int es[NE], xr[NE];
const uint8_t* blob;
int m0, m1;
if (!mx_group(a, blockIdx.y, blob, m0, m1)) return; // (uniform)
const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
const size_t rb = mx_row_bytes(a.n_embd);
const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
const int nb = a.n_embd / kMxBlock;
for (int mm = m0; mm < m1; mm += NE) {
const int ne = min(NE, m1 - mm);
__syncthreads();
if ((int) threadIdx.x < ne) {
const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
es[threadIdx.x] = e;
xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
}
__syncthreads();
if (row >= a.n_ff) continue;
float ag[NE], au[NE];
#pragma unroll
for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;
Fetch<kMxBytes> FG, FU;
if (ST) {
FG.load(g, 0, min(32, nb));
FU.load(u, 0, min(32, nb));
}
for (int b0 = 0; b0 < nb; b0 += 32) {
const int nbk = min(32, nb - b0), b = b0 + lane;
if (ST) {
FG.store(stage[wp][0]);
FU.store(stage[wp][1]);
if (b0 + 32 < nb) {
FG.load(g, b0 + 32, min(32, nb - b0 - 32));
FU.load(u, b0 + 32, min(32, nb - b0 - 32));
}
}
if (lane >= nbk) continue;
uint4 qg, qu;
const uint32_t eg = ST ? mx_words_s(stage[wp][0], lane, qg) : mx_words_g(g + (size_t) b * kMxBytes, qg);
const uint32_t eu = ST ? mx_words_s(stage[wp][1], lane, qu) : mx_words_g(u + (size_t) b * kMxBytes, qu);
const float dg = e8m0_half(eg), du = e8m0_half(eu);
#pragma unroll
for (int i = 0; i < NE; ++i)
if (i < ne) {
const int4* xp = (const int4*) (a.xq + (size_t) xr[i] * a.n_embd + (size_t) b * kMxBlock);
const int4 xa = xp[0], xb = xp[1];
const float dx = a.xd[(size_t) xr[i] * nb + b];
ag[i] = fmaf(dg * dx, (float) mx_dot(qg, xa, xb), ag[i]);
au[i] = fmaf(du * dx, (float) mx_dot(qu, xa, xb), au[i]);
}
}
#pragma unroll
for (int i = 0; i < NE; ++i)
if (i < ne) {
const float gv = warp_sum(ag[i]), uv = warp_sum(au[i]);
if (lane == 0) a.h[(size_t) es[i] * a.n_ff + row] = swiglu(gv, uv, a.limit);
}
}
}
// grid (n_embd / 8, groups): a warp a row of the down for NE entries a pass, against their int8 h -> part
template <int NE, bool ST>
__global__ void __launch_bounds__(kThreads) mx_down_kernel(const MxArgs a) {
__shared__ __align__(16) uint32_t stage[kWarps][32 * kMxBytes / 4];
__shared__ int es[NE];
const uint8_t* blob;
int m0, m1;
if (!mx_group(a, blockIdx.y, blob, m0, m1)) return;
const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
const uint8_t* d = blob + a.tr[2] + (size_t) row * mx_row_bytes(a.n_ff);
const int nb = a.n_ff / kMxBlock;
for (int mm = m0; mm < m1; mm += NE) {
const int ne = min(NE, m1 - mm);
__syncthreads();
if ((int) threadIdx.x < ne) es[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
__syncthreads();
if (row >= a.n_embd) continue;
float ad[NE];
#pragma unroll
for (int i = 0; i < NE; ++i) ad[i] = 0.0f;
Fetch<kMxBytes> FD;
if (ST) FD.load(d, 0, min(32, nb));
for (int b0 = 0; b0 < nb; b0 += 32) {
const int nbk = min(32, nb - b0), b = b0 + lane;
if (ST) {
FD.store(stage[wp]);
if (b0 + 32 < nb) FD.load(d, b0 + 32, min(32, nb - b0 - 32));
}
if (lane >= nbk) continue;
uint4 q;
const float dd = e8m0_half(ST ? mx_words_s(stage[wp], lane, q) : mx_words_g(d + (size_t) b * kMxBytes, q));
#pragma unroll
for (int i = 0; i < NE; ++i)
if (i < ne) {
const int4* hp = (const int4*) (a.hq + (size_t) es[i] * a.n_ff + (size_t) b * kMxBlock);
const int4 ha = hp[0], hb = hp[1];
ad[i] = fmaf(dd * a.hd[(size_t) es[i] * nb + b], (float) mx_dot(q, ha, hb), ad[i]);
}
}
#pragma unroll
for (int i = 0; i < NE; ++i)
if (i < ne) {
const float v = warp_sum(ad[i]);
if (lane == 0) a.part[(size_t) es[i] * a.n_embd + row] = v;
}
}
}
struct CombArgs {
const unsigned long long* plan_ptr; // [T][8]
const float* plan_w; // [T][8]
const float* part; // [T * 8][n_embd]
const float* cpu_part;
size_t cpu_stride;
const int* cpu_flag;
const float* sh_out; // [T][n_embd]
float* out;
size_t out_stride;
int k, n_embd;
};
// grid (n_embd / 128, T): each token's outputs - the experts in plan order (an expert off the device adds 0), then
// the CPU lane's part and the shared expert's (x3's combine)
__global__ void __launch_bounds__(128) mx_combine_kernel(const CombArgs a) {
const int t = blockIdx.y, r = blockIdx.x * 128 + threadIdx.x;
if (r >= a.n_embd) return;
float p[8];
for (int i = 0; i < a.k; ++i) {
const int e = t * 8 + i;
p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;
}
a.out[(size_t) t * a.out_stride + r] =
combine(a.plan_w + t * 8, p, a.k,
a.cpu_flag != nullptr && a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.cpu_stride + r : nullptr,
a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);
}
__global__ void mx_wait_kernel(const unsigned* p, unsigned v) {
if (threadIdx.x != 0) return;
while (*(volatile const unsigned*) p != v) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
__nanosleep(200);
#endif
}
__threadfence_system();
}
// ---------------------------------------------------------------- the prompt path's expert sets
// a set's expert i (blob wbase + i stride) owns the sorted rows [bounds[i], bounds[i + 1]); local row = sorted row -
// row0. A block: BM of its rows x BN outputs, the weights dequantised a 32-value block at a time into shared memory,
// F32 FMAs (each thread TM x TN outputs)
constexpr int BM = 64, BN = 64, TM = 4, TN = 4;
static_assert((BM / TM) * (BN / TN) == kThreads, "one output tile a thread");
struct SetK {
const uint8_t* wbase;
size_t stride;
size_t tr[3];
const int* bounds;
int row0;
int n_embd, n_ff;
float limit;
const float* xf; // gate / up: F32 input rows x[idx[local]] (ldx)
const int* idx;
const __half* xh; // down: F16 rows (ldx)
int ldx;
__half* h; // gate / up: F16 out
float* y; // down: F32 out
int ldo;
int skip_le; // the experts of at most this many rows left out (the light kernels took them; ds4_iq.cuh)
};
// one 32-value block of BN weight rows (rows c0 .., row bytes rb, block kb) -> ws[32][BN + 1]
__device__ __forceinline__ void mx_tile(const uint8_t* w, size_t rb, int c0, int kb, const float* lut,
float (*ws)[BN + 1]) {
const int r = threadIdx.x >> 2, q = threadIdx.x & 3; // row r of the tile, values [8 q, 8 q + 8)
const uint8_t* p = w + (size_t) (c0 + r) * rb + (size_t) kb * kMxBytes;
const float d = e8m0_half(p[0]);
const int b0 = (q & 1) * 8, hi = q >> 1;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const uint32_t c = p[1 + b0 + j];
ws[q * 8 + j][r] = lut[hi ? (c >> 4) : (c & 15u)] * d;
}
}
template <bool GATE_UP>
__global__ void __launch_bounds__(kThreads) mx_set_kernel(const SetK a) {
__shared__ float lut[16];
__shared__ float wg[32][BN + 1];
__shared__ float wu[GATE_UP ? 32 : 1][BN + 1];
__shared__ float xs[BM][33];
__shared__ int src[BM];
lut_init(lut);
const int e = blockIdx.z;
const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
const int r0 = lo + blockIdx.x * BM;
if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform: before any barrier but lut_init's writes, which nobody
// reads then; skip_le: the light kernels' experts)
const int nr = min(BM, hi - r0);
const int c0 = blockIdx.y * BN;
const uint8_t* blob = a.wbase + (size_t) e * a.stride;
const int K = GATE_UP ? a.n_embd : a.n_ff;
const size_t rb = mx_row_bytes(K);
const int tid = (int) threadIdx.x;
if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
__syncthreads();
const int ty = threadIdx.x / (BN / TN), tx = threadIdx.x % (BN / TN);
float ag[TM][TN], au[TM][TN];
#pragma unroll
for (int i = 0; i < TM; ++i)
#pragma unroll
for (int j = 0; j < TN; ++j) ag[i][j] = au[i][j] = 0.0f;
for (int kb = 0; kb < K / kMxBlock; ++kb) {
mx_tile(blob + a.tr[GATE_UP ? 0 : 2], rb, c0, kb, lut, wg);
if (GATE_UP) mx_tile(blob + a.tr[1], rb, c0, kb, lut, wu);
for (int i = threadIdx.x; i < BM * kMxBlock; i += kThreads) { // the rows' 32 inputs
const int r = i / kMxBlock, c = i % kMxBlock, sr = src[r];
float v = 0.0f;
if (sr >= 0) {
const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + c;
v = GATE_UP ? a.xf[at] : __half2float(a.xh[at]);
}
xs[r][c] = v;
}
__syncthreads();
#pragma unroll 8
for (int kk = 0; kk < kMxBlock; ++kk) {
float xv[TM], gw[TN], uw[TN];
#pragma unroll
for (int i = 0; i < TM; ++i) xv[i] = xs[ty * TM + i][kk];
#pragma unroll
for (int j = 0; j < TN; ++j) {
gw[j] = wg[kk][tx * TN + j];
if (GATE_UP) uw[j] = wu[kk][tx * TN + j];
}
#pragma unroll
for (int i = 0; i < TM; ++i)
#pragma unroll
for (int j = 0; j < TN; ++j) {
ag[i][j] = fmaf(xv[i], gw[j], ag[i][j]);
if (GATE_UP) au[i][j] = fmaf(xv[i], uw[j], au[i][j]);
}
}
__syncthreads();
}
#pragma unroll
for (int i = 0; i < TM; ++i) {
const int r = ty * TM + i;
if (r >= nr) continue;
const size_t lr = (size_t) (r0 + r);
#pragma unroll
for (int j = 0; j < TN; ++j) {
const int c = c0 + tx * TN + j;
if (GATE_UP) a.h[lr * a.ldo + c] = __float2half_rn(swiglu(ag[i][j], au[i][j], a.limit));
else a.y[lr * a.ldo + c] = ag[i][j];
}
}
}
// the same on the tensor cores: a block BM rows x BN outputs, 8 warps each 16 rows x 32 outputs (2 fragments, gate and
// up: 4), a 32-value MXFP4 block of K a step (2 WMMA k-steps), the tiles in shared memory as F16
#if !defined(STRATA_USE_HIP)
namespace wm = nvcuda::wmma;
constexpr int kLdT = 40; // halves a tile row (32 + 8: WMMA wants a multiple of 8; the pad spreads the banks)
template <bool GATE_UP>
__global__ void __launch_bounds__(kThreads) mx_set_tc_kernel(const SetK a) {
struct Tiles {
__half x[BM][kLdT], wg[BN][kLdT], wu[GATE_UP ? BN : 1][kLdT];
};
struct Out {
float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];
};
__shared__ __align__(32) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];
__shared__ float lut[16];
__shared__ int src[BM];
Tiles& T = *reinterpret_cast<Tiles*>(smem);
Out& O = *reinterpret_cast<Out*>(smem);
lut_init(lut);
const int e = blockIdx.z;
const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
const int r0 = lo + blockIdx.x * BM;
if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)
const int nr = min(BM, hi - r0);
const int c0 = blockIdx.y * BN;
const uint8_t* blob = a.wbase + (size_t) e * a.stride;
const int K = GATE_UP ? a.n_embd : a.n_ff;
const size_t rb = mx_row_bytes(K);
const int tid = (int) threadIdx.x, warp = tid >> 5;
if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
__syncthreads();
const int wm_ = warp >> 1, wn = (warp & 1) * 2; // the warp's 16 rows, its 2 x 16 outputs
wm::fragment<wm::accumulator, 16, 16, 16, float> fg[2], fu[2];
for (int j = 0; j < 2; ++j) {
wm::fill_fragment(fg[j], 0.0f);
wm::fill_fragment(fu[j], 0.0f);
}
const int tr = tid >> 2, tq = tid & 3; // the thread's tile row, its 8 values [8 tq, 8 tq + 8)
for (int kb = 0; kb < K / kMxBlock; ++kb) {
{ // the weights: row tr of the outputs, values 8 tq ..
for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {
const uint8_t* q = blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb + (size_t) kb * kMxBytes;
const float d = e8m0_half(q[0]);
const int b0 = (tq & 1) * 8, hn = tq >> 1;
__half* dst = m == 0 ? &T.wg[tr][tq * 8] : &T.wu[tr][tq * 8];
#pragma unroll
for (int j = 0; j < 8; ++j) {
const uint32_t c = q[1 + b0 + j];
dst[j] = __float2half_rn(lut[hn ? (c >> 4) : (c & 15u)] * d);
}
}
// the inputs: row tr, values 8 tq ..
const int sr = src[tr];
#pragma unroll
for (int j = 0; j < 8; ++j) {
__half v = __float2half_rn(0.0f);
if (sr >= 0) {
const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + tq * 8 + j;
v = GATE_UP ? __float2half_rn(a.xf[at]) : a.xh[at];
}
T.x[tr][tq * 8 + j] = v;
}
}
__syncthreads();
#pragma unroll
for (int kk = 0; kk < 32; kk += 16) {
wm::fragment<wm::matrix_a, 16, 16, 16, __half, wm::row_major> fa;
wm::load_matrix_sync(fa, &T.x[wm_ * 16][kk], kLdT);
#pragma unroll
for (int j = 0; j < 2; ++j) {
wm::fragment<wm::matrix_b, 16, 16, 16, __half, wm::col_major> fb;
wm::load_matrix_sync(fb, &T.wg[(wn + j) * 16][kk], kLdT);
wm::mma_sync(fg[j], fa, fb, fg[j]);
if (GATE_UP) {
wm::load_matrix_sync(fb, &T.wu[(wn + j) * 16][kk], kLdT);
wm::mma_sync(fu[j], fa, fb, fu[j]);
}
}
}
__syncthreads();
}
for (int j = 0; j < 2; ++j) {
wm::store_matrix_sync(&O.g[wm_ * 16][(wn + j) * 16], fg[j], BN + 4, wm::mem_row_major);
if (GATE_UP) wm::store_matrix_sync(&O.u[wm_ * 16][(wn + j) * 16], fu[j], BN + 4, wm::mem_row_major);
}
__syncthreads();
for (int i = tid; i < BM * BN; i += kThreads) {
const int r = i / BN, c = i % BN;
if (r >= nr) continue;
const size_t lr = (size_t) (r0 + r);
if (GATE_UP) a.h[lr * a.ldo + c0 + c] = __float2half_rn(swiglu(O.g[r][c], O.u[r][c], a.limit));
else a.y[lr * a.ldo + c0 + c] = O.g[r][c];
}
}
#endif
#if defined(STRATA_USE_HIP)
// the same on an AMD wave32 GPU's WMMA (RDNA3 / RDNA3.5 gfx11, RDNA4 gfx12): 8 waves each 16 rows x 32 outputs, the
// tiles in shared memory as F16, F32 accumulation - each lane gives its row / column's 16 values of a k-step (gfx12:
// its half of them), and holds 8 of the 16 x 16 results (rows 2 i + hi on gfx11, 8 hi + i on gfx12; column lane % 16)
using h16v = _Float16 __attribute__((ext_vector_type(16)));
using f8v = float __attribute__((ext_vector_type(8)));
__device__ __forceinline__ f8v set_wmma(h16v a, h16v b, f8v c) {
#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \
defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__)
return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
#elif defined(__gfx1200__) || defined(__gfx1201__)
using h8 = _Float16 __attribute__((ext_vector_type(8)));
const int k0 = (threadIdx.x & 31) / 16 * 8;
h8 aa, bb;
#pragma unroll
for (int i = 0; i < 8; ++i) {
aa[i] = a[k0 + i];
bb[i] = b[k0 + i];
}
return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(aa, bb, c);
#else
__builtin_trap();
return c;
#endif
}
__device__ __forceinline__ int set_wmma_row(int i, int hi) {
#if defined(__gfx1200__) || defined(__gfx1201__)
return 8 * hi + i;
#else
return 2 * i + hi;
#endif
}
constexpr int kLdH = 40; // halves a tile row (32 + 8: the pad spreads the banks)
template <bool GATE_UP>
__global__ void __launch_bounds__(kThreads) mx_set_wmma_kernel(const SetK a) {
struct Tiles {
_Float16 x[BM][kLdH], wg[BN][kLdH], wu[GATE_UP ? BN : 1][kLdH];
};
struct Out {
float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];
};
__shared__ __align__(16) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];
__shared__ float lut[16];
__shared__ int src[BM];
Tiles& T = *reinterpret_cast<Tiles*>(smem);
Out& O = *reinterpret_cast<Out*>(smem);
lut_init(lut);
const int e = blockIdx.z;
const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;
const int r0 = lo + blockIdx.x * BM;
if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)
const int nr = min(BM, hi_r - r0);
const int c0 = blockIdx.y * BN;
const uint8_t* blob = a.wbase + (size_t) e * a.stride;
const int K = GATE_UP ? a.n_embd : a.n_ff;
const size_t rb = mx_row_bytes(K);
const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
__syncthreads();
const int wm_ = warp >> 1, wn = (warp & 1) * 2; // the wave's 16 rows, its 2 x 16 outputs
f8v fg[2], fu[2];
#pragma unroll
for (int j = 0; j < 2; ++j)
#pragma unroll
for (int i = 0; i < 8; ++i) fg[j][i] = fu[j][i] = 0.0f;
const int tr = tid >> 2, tq = tid & 3; // the thread's tile row, its 8 values [8 tq, 8 tq + 8)
for (int kb = 0; kb < K / kMxBlock; ++kb) {
for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {
const uint8_t* q = blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb + (size_t) kb * kMxBytes;
const float d = e8m0_half(q[0]);
const int b0 = (tq & 1) * 8, hn = tq >> 1;
_Float16* dst = m == 0 ? &T.wg[tr][tq * 8] : &T.wu[tr][tq * 8];
#pragma unroll
for (int j = 0; j < 8; ++j) {
const uint32_t cc = q[1 + b0 + j];
dst[j] = (_Float16) (lut[hn ? (cc >> 4) : (cc & 15u)] * d);
}
}
const int sr = src[tr];
#pragma unroll
for (int j = 0; j < 8; ++j) {
float v = 0.0f;
if (sr >= 0) {
const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + tq * 8 + j;
v = GATE_UP ? a.xf[at] : __half2float(a.xh[at]);
}
T.x[tr][tq * 8 + j] = (_Float16) v;
}
__syncthreads();
#pragma unroll
for (int kk = 0; kk < 32; kk += 16) {
h16v av;
#pragma unroll
for (int i = 0; i < 16; ++i) av[i] = T.x[wm_ * 16 + l16][kk + i];
#pragma unroll
for (int j = 0; j < 2; ++j) {
h16v bv;
#pragma unroll
for (int i = 0; i < 16; ++i) bv[i] = T.wg[(wn + j) * 16 + l16][kk + i];
fg[j] = set_wmma(av, bv, fg[j]);
if (GATE_UP) {
#pragma unroll
for (int i = 0; i < 16; ++i) bv[i] = T.wu[(wn + j) * 16 + l16][kk + i];
fu[j] = set_wmma(av, bv, fu[j]);
}
}
}
__syncthreads();
}
#pragma unroll
for (int j = 0; j < 2; ++j)
#pragma unroll
for (int i = 0; i < 8; ++i) {
const int r = wm_ * 16 + set_wmma_row(i, hi), cc = (wn + j) * 16 + l16;
O.g[r][cc] = fg[j][i];
if (GATE_UP) O.u[r][cc] = fu[j][i];
}
__syncthreads();
for (int i = tid; i < BM * BN; i += kThreads) {
const int r = i / BN, cc = i % BN;
if (r >= nr) continue;
const size_t lr = (size_t) (r0 + r);
if (GATE_UP) a.h[lr * a.ldo + c0 + cc] = __float2half_rn(swiglu(O.g[r][cc], O.u[r][cc], a.limit));
else a.y[lr * a.ldo + c0 + cc] = O.g[r][cc];
}
}
#endif
bool tc_ok() { // sm_70 and later (WMMA F16) / AMD gfx11, gfx12 (wave32 WMMA); STRATA_DS4_NO_TC=1 keeps the F32 kernel
#if defined(STRATA_USE_HIP)
static const bool ok = [] {
int dev = 0;
cudaGetDevice(&dev);
cudaDeviceProp prop{};
if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
const char* v = std::getenv("STRATA_DS4_NO_TC");
if (v != nullptr && v[0] == '1') return false;
const char* arch = prop.gcnArchName;
for (const char* p : {"gfx1100", "gfx1101", "gfx1102", "gfx1103", "gfx1150", "gfx1151", "gfx1152", "gfx1200", "gfx1201"})
if (std::strncmp(arch, p, 7) == 0) return true;
return false;
}();
return ok;
#else
static const bool ok = [] {
int dev = 0, major = 0;
cudaGetDevice(&dev);
cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
const char* v = std::getenv("STRATA_DS4_NO_TC");
return major >= 7 && !(v != nullptr && v[0] == '1');
}();
return ok;
#endif
}
// ---------------------------------------------------------------- scratch
constexpr int kMaxEmbd = 8192;
constexpr int kMaxQK = 32768; // a Q8_0 GEMV's inputs at most
struct Scratch {
float* part = nullptr; // [8][kMaxEmbd]: a token's experts' down outputs
int8_t *xq = nullptr, *hq = nullptr, *mq = nullptr; // int8 rows: x [kMaxEnt], h [kMaxEnt * 8], the GEMVs'
float *xd = nullptr, *hd = nullptr, *md = nullptr; // [kMaxMvJobs * kMaxMvCols] inputs; their scales
};
std::mutex g_mu;
std::unordered_map<cudaStream_t, Scratch> g_scratch;
Scratch* scratch(cudaStream_t s) {
std::lock_guard<std::mutex> lk(g_mu);
auto it = g_scratch.find(s);
if (it != g_scratch.end()) return &it->second;
Scratch sc;
const size_t nx = (size_t) kMaxEnt * kMaxEmbd, nh = (size_t) kMaxEnt * 8 * kMaxEmbd,
nm = (size_t) x3::kMaxMvJobs * x3::kMaxMvCols * kMaxQK;
if (cudaMalloc(&sc.part, (size_t) 8 * kMaxEmbd * sizeof(float)) != cudaSuccess ||
cudaMalloc(&sc.xq, nx) != cudaSuccess || cudaMalloc(&sc.xd, nx / 32 * 4) != cudaSuccess ||
cudaMalloc(&sc.hq, nh) != cudaSuccess || cudaMalloc(&sc.hd, nh / 32 * 4) != cudaSuccess ||
cudaMalloc(&sc.mq, nm) != cudaSuccess || cudaMalloc(&sc.md, nm / 32 * 4) != cudaSuccess) {
fail("scratch: cudaMalloc failed");
return nullptr;
}
return &g_scratch.emplace(s, sc).first->second;
}
bool mx_geometry(const char* who, int k, int n_embd, int n_ff) {
if (k < 1 || k > 8 || n_embd % BN != 0 || n_ff % BN != 0 || n_embd > kMaxEmbd || n_ff % kMxBlock != 0) {
std::fprintf(stderr, "ds4 gguf %s: geometry not covered (k %d n_embd %d n_ff %d)\n", who, k, n_embd, n_ff);
g_errors.fetch_add(1, std::memory_order_relaxed);
return false;
}
return true;
}
} // namespace
// ---------------------------------------------------------------- entry points
bool prepare(cudaStream_t s) { return scratch(s) != nullptr; }
int launch_errors() { return g_errors.load(std::memory_order_relaxed); }
size_t type_row_bytes(int type, int64_t k) {
if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;
// known(), not supported(): a type with no device kernel still has a layout, and its rows are read by the CPU lane
return iq::known(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;
}
int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::known(type) ? iq::block_values(type) : 0; }
const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }
bool type_on_device(int type) { return type == kTypeMXFP4 || iq::supported(type); }
bool type_on_chunk(int type) {
#if defined(STRATA_METAL)
// ds4_mm.metal instantiates the tile GEMM for IQ2_XXS / IQ2_XS / IQ2_S only; IQ1_M (the 2.00 bpw pack's type) has
// the matvec kernel but no tile one, so a chunked prompt would stop in "kernel launch with no Metal dispatch".
if (type == iq::kIQ1_S || type == iq::kIQ4_NL) return false; // (IQ1_M: the tile kernels since DS4_MM_INST(29) and iq_bs)
return type_on_device(type);
#else
(void) type;
return true; // CUDA and HIP: the chunk path takes whatever the layer holds (types with no GPU kernel stay on the CPU lane)
#endif
}
namespace {
size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a row
if (m.ld > 0) return (size_t) m.ld;
return m.fmt == kFmtMXFP4 ? mx_row_bytes(m.k)
: m.fmt == kFmtQ8_0 ? q8_row_bytes(m.k)
: m.fmt == kFmtGG ? type_row_bytes(m.K, m.k)
: (size_t) m.k;
}
// a layer's low-bit experts as ds4_iq's arguments (false: its types not covered - printed)
bool iq_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit, iq::Args& ia, const char* who) {
ia = iq::Args{};
for (int r = 0; r < 3; ++r) ia.tr[r] = lay.trellis[r];
ia.n_embd = n_embd;
ia.n_ff = n_ff;
ia.limit = limit;
ia.tg = expert_type(lay, 0);
ia.td = expert_type(lay, 2);
if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % iq::block_values(ia.tg) != 0 ||
n_ff % iq::block_values(ia.td) != 0) {
std::fprintf(stderr, "ds4 gguf %s: expert types %s / %s / %s (n_embd %d, n_ff %d) not covered\n", who,
type_name(expert_type(lay, 0)), type_name(expert_type(lay, 1)), type_name(expert_type(lay, 2)),
n_embd, n_ff);
g_errors.fetch_add(1, std::memory_order_relaxed);
return false;
}
return true;
}
template <int NC>
void mv_launch(int f, const BfJobs& J, int nj, int max_n, bool vec, cudaStream_t s) {
const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);
if (f == 1) {
#if defined(STRATA_METAL)
// the MXFP4 matvec: the engine's jobs go across as they are (same layout on both sides)
if (max_n > 0 && nj > 0 &&
::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 1)) return;
#endif
if (vec) strata_ds4_launch(mx_mv_kernel<NC, true, false>, grid, kThreads, 0, s, J);
else strata_ds4_launch(mx_mv_kernel<NC, false, false>, grid, kThreads, 0, s, J);
} else if (f == 2) { // (vec here: every row 4-byte aligned, so staged)
#if defined(STRATA_METAL)
if (max_n > 0 && nj > 0 &&
::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 2)) return;
#endif
if (vec) strata_ds4_launch(q8_mv_kernel<NC, true>, grid, kThreads, 0, s, J);
else strata_ds4_launch(q8_mv_kernel<NC, false>, grid, kThreads, 0, s, J);
} else {
#if defined(STRATA_METAL)
// the BF16 matvec (vec: the rows and activations 16-byte aligned, so the 8-value step is safe)
if (vec && max_n > 0 && nj > 0 &&
::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 0)) return;
#endif
if (vec) strata_ds4_launch(bf16_mv_kernel<NC, true>, grid, kThreads, 0, s, J);
else strata_ds4_launch(bf16_mv_kernel<NC, false>, grid, kThreads, 0, s, J);
}
}
void mv_set(int f, const BfJobs& J, int nj, int max_n, int max_c, bool vec, cudaStream_t s) {
if (nj == 0) return;
if (max_c <= 1) mv_launch<1>(f, J, nj, max_n, vec, s);
else if (max_c <= 2) mv_launch<2>(f, J, nj, max_n, vec, s);
else if (max_c <= 4) mv_launch<4>(f, J, nj, max_n, vec, s);
else if (max_c <= 8) mv_launch<8>(f, J, nj, max_n, vec, s);
else mv_launch<16>(f, J, nj, max_n, vec, s);
}
} // namespace
x3::Mat view(const x3::Mat& m, int c0, int nc) {
x3::Mat v = m;
const size_t ld = row_stride(m);
v.ld = (int) ld;
v.raw = m.fmt != kFmtBF16 ? (const void*) ((const uint8_t*) m.raw + (size_t) c0 * ld)
: (const void*) ((const uint16_t*) m.raw + (size_t) c0 * ld);
v.n = nc;
return v;
}
bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
if (n <= 0 || n > x3::kMaxMvJobs) return false;
#if defined(STRATA_METAL)
if (ds4_dbg("DS4_MV_LOG")) { // how the engine batches its matvec jobs (one call = one quantiser dispatch)
static std::atomic<unsigned long long> calls{0}, jobsum{0};
static std::atomic<unsigned long long> hist[9];
static std::atomic<int> seen{0};
calls.fetch_add(1); jobsum.fetch_add((unsigned long long) n);
hist[n < 8 ? n : 8].fetch_add(1);
if (seen.fetch_add(1) % 256 == 255)
std::fprintf(stderr, "mvlog calls=%llu jobs=%llu avg=%.2f hist1..4=%llu,%llu,%llu,%llu 5+=%llu,%llu,%llu,%llu\n",
(unsigned long long) calls.load(), (unsigned long long) jobsum.load(),
(double) jobsum.load() / (double) calls.load(),
(unsigned long long) hist[1].load(), (unsigned long long) hist[2].load(),
(unsigned long long) hist[3].load(), (unsigned long long) hist[4].load(),
(unsigned long long) hist[5].load(), (unsigned long long) hist[6].load(),
(unsigned long long) hist[7].load(), (unsigned long long) hist[8].load());
}
#endif
BfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch each
int nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};
bool vec[3] = {true, true, true};
iq::MvJob IJ[x3::kMaxMvJobs]; // the low-bit ones (ds4_iq: a launch a type)
int nij = 0;
QuantItems QI; // the int8 roundings of this call's jobs, one launch (identical inputs once)
QI.n = 0;
for (int i = 0; i < n; ++i) {
const x3::MvJob& q = jobs[i];
const x3::Mat& m = *q.m;
if (m.fmt == kFmtGG) { // the activations rounded to int8 (job i's rows of the scratch), then ds4_iq's GEMV
Scratch* sc = scratch(s);
if (sc == nullptr || m.raw == nullptr || q.xf == nullptr || q.y == nullptr || q.ncol < 1 ||
#if !defined(STRATA_METAL)
q.ncol > x3::kMaxMvCols ||
#endif
m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {
std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (type %s k %d n %d)\n", i, type_name(m.K), m.k, m.n);
g_errors.fetch_add(1, std::memory_order_relaxed);
return false;
}
const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
const int qi = quant_add(QI, q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32);
iq::MvJob& b = IJ[nij++];
b.w = (const uint8_t*) m.raw;
b.ld = row_stride(m);
b.type = m.K;
b.k = m.k;
b.n = m.n;
b.ncol = q.ncol;
b.xq = QI.q[qi];
b.xd = QI.d[qi];
b.y = q.y;
b.ycs = q.y_cs;
b.bias = q.bias;
b.alpha = q.alpha;
continue;
}
const int f = m.fmt == kFmtMXFP4 ? 1 : m.fmt == kFmtQ8_0 ? 2 : 0;
if ((m.fmt != kFmtBF16 && m.fmt != kFmtMXFP4 && m.fmt != kFmtQ8_0) || m.raw == nullptr || q.xf == nullptr ||
q.y == nullptr || q.ncol < 1 || q.ncol > x3::kMaxMvCols || m.k <= 0 || m.n <= 0 ||
(f != 0 && m.k % kMxBlock != 0)) {
std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (fmt %d k %d n %d, %d activations)\n", i, m.fmt, m.k,
m.n, q.ncol);
g_errors.fetch_add(1, std::memory_order_relaxed);
return false;
}
BfJob& b = J[f].j[nj[f]++];
b.w = m.raw;
b.ld = row_stride(m);
b.x = q.xf;
b.xcs = q.xf_cs;
b.y = q.y;
b.ycs = q.y_cs;
b.bias = q.bias;
b.alpha = q.alpha;
b.k = m.k;
b.n = m.n;
b.ncol = q.ncol;
max_n[f] = std::max(max_n[f], m.n);
max_c[f] = std::max(max_c[f], q.ncol);
const bool xa = ((uintptr_t) b.x & 15u) == 0 && (q.ncol == 1 || b.xcs % 4 == 0);
if (f == 2) { // the activations rounded to int8 first (each job its own rows of the scratch)
Scratch* sc = scratch(s);
if (sc == nullptr || m.k > kMaxQK) {
fail("mv: Q8_0 inputs past the scratch");
return false;
}
const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
const int qi = quant_add(QI, q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32);
b.xq = QI.q[qi];
b.xd = QI.d[qi];
vec[f] = vec[f] && b.ld % 4 == 0 && ((uintptr_t) b.w & 3u) == 0 && (m.k / 32) % 2 == 0;
} else {
vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));
}
}
quant_flush(QI, s);
for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);
#if defined(STRATA_METAL)
if (ds4_dbg("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recompute
ds4_metal_flush();
const BfJob& b = J[2].j[0];
const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);
const int8_t* xq = (const int8_t*) strata::gpu::host_view(b.xq);
const float* xd = (const float*) strata::gpu::host_view(b.xd);
const float* y = (const float*) strata::gpu::host_view(b.y);
const float* xf = (const float*) strata::gpu::host_view(b.x);
for (int c = 0; c < std::min(b.ncol, 2); ++c)
for (int r = 0; r < 2; ++r) {
double acc = 0.0, accf = 0.0;
for (int kb = 0; kb < b.k / 32; ++kb) {
const uint8_t* p = w + (size_t) r * b.ld + (size_t) kb * 34;
uint16_t dh;
std::memcpy(&dh, p, 2);
float dw = __half2float(*(const __half*) &dh);
int sum = 0;
for (int j = 0; j < 32; ++j) sum += (int) (int8_t) p[2 + j] * (int) xq[(size_t) c * b.k + kb * 32 + j];
acc += (double) dw * xd[(size_t) c * (b.k / 32) + kb] * sum;
for (int j = 0; j < 32; ++j) accf += (double) dw * (int) (int8_t) p[2 + j] * xf[(size_t) c * b.xcs + kb * 32 + j];
}
std::fprintf(stderr, "DBG q8 job0 k=%d n=%d ncol=%d ld=%zu xcs=%zu c=%d r=%d recompute=%.5g exactf=%.5g kernel_y=%.5g xd0=%g xq0=%d\n", b.k, b.n, b.ncol,
b.ld, b.xcs, c, r, acc, accf, y[(size_t) c * b.ycs + r], xd[(size_t) c * (b.k / 32)], (int) xq[(size_t) c * b.k]);
}
}
#endif
if (nij > 0 && !iq::mv(IJ, nij, s)) {
fail("mv: a low-bit job not covered");
return false;
}
#if defined(STRATA_METAL)
if (ds4_dbg("DS4_DBG_IQ") && nij > 0) { // DBG: the low-bit matvec's first job against a host decode of its rows
ds4_metal_flush();
for (int ji = 0; ji < std::min(nij, 2); ++ji) {
const iq::MvJob& b = IJ[ji];
const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);
const int8_t* xq = (const int8_t*) strata::gpu::host_view(b.xq);
const float* xd = (const float*) strata::gpu::host_view(b.xd);
const float* y = (const float*) strata::gpu::host_view(b.y);
std::vector<uint16_t> row((size_t) b.k);
for (int r = 0; r < 3; ++r) {
if (!gg_row_to_bf16(b.type, w + (size_t) r * b.ld, b.k, row.data())) { std::fprintf(stderr, "DBG iq: no host decode for type %d\n", b.type); break; }
double acc = 0.0;
for (int i = 0; i < b.k; ++i) {
uint32_t u = (uint32_t) row[(size_t) i] << 16; float wf; std::memcpy(&wf, &u, 4);
acc += (double) wf * xq[i] * xd[i / 32];
}
std::fprintf(stderr, "DBG iq job %d type %d k=%d n=%d ncol=%d ld=%zu r=%d host=%.5g kernel=%.5g w=%p\n", ji, b.type, b.k, b.n, b.ncol, b.ld, r, acc, y[r], (const void*) b.w);
}
}
}
#endif
check("mv");
return true;
}
void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
#if defined(STRATA_METAL)
if (!bf16 && m.raw != nullptr) { // the prompt chunk's dense matrices: F16 rows for the MPP GEMM
const int kind = m.fmt == kFmtGG ? m.K : m.fmt == kFmtQ8_0 ? 8 : m.fmt == kFmtBF16 ? 0 : -1;
if (kind >= 0 && ::strata::dsv41::metal_host::dequant_rows(kind, m.raw, row_stride(m), m.k, m.n, dst)) return;
}
#endif
if (m.fmt == kFmtGG && m.raw != nullptr) { // a K-quant / i-quant dense matrix (the IST-DAS sets' Q5_K): its own dequantizer
if (!iq::dequant_rows(m.K, m.raw, row_stride(m), m.k, m.n, dst, bf16, s)) fail("rows: a low-bit type not covered");
return;
}
if ((m.fmt != kFmtBF16 && m.fmt != kFmtQ8_0) || m.raw == nullptr) {
fail("rows: not a BF16 or Q8_0 matrix");
return;
}
const size_t ld = row_stride(m);
const size_t total = (size_t) m.n * (size_t) m.k;
const unsigned blocks = (unsigned) std::min<size_t>((total + 255) / 256, 65535);
if (m.fmt == kFmtQ8_0) strata_ds4_launch(q8_rows_kernel, blocks, 256, 0, s, (const uint8_t*) m.raw, ld, m.k, m.n, dst, bf16);
else strata_ds4_launch(bf16_rows_kernel, blocks, 256, 0, s, (const uint16_t*) m.raw, ld, m.k, m.n, dst, bf16);
check("rows");
}
namespace {
// (staged when the matrix's rows are 4-byte aligned: 17 nb % 4 == 0; the blobs' offsets are)
template <int NE>
struct GateUpK {
static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {
if (mx_row_bytes(m.n_embd) % 4 == 0) strata_ds4_launch(mx_gate_up_kernel<NE, true>, grid, kThreads, 0, s, m);
else strata_ds4_launch(mx_gate_up_kernel<NE, false>, grid, kThreads, 0, s, m);
}
};
template <int NE>
struct DownK {
static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {
if (mx_row_bytes(m.n_ff) % 4 == 0) strata_ds4_launch(mx_down_kernel<NE, true>, grid, kThreads, 0, s, m);
else strata_ds4_launch(mx_down_kernel<NE, false>, grid, kThreads, 0, s, m);
}
};
template <template <int> class Kern>
void launch_groups(const MxArgs& m, int rows, int groups, int ne_max, cudaStream_t s) {
const dim3 grid((unsigned) ((rows + kWarps - 1) / kWarps), (unsigned) groups);
if (ne_max <= 1) Kern<1>::run(m, grid, s);
else if (ne_max <= 4) Kern<4>::run(m, grid, s);
else if (ne_max <= 8) Kern<8>::run(m, grid, s);
else Kern<16>::run(m, grid, s);
}
MxArgs mx_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit) {
MxArgs m{};
for (int r = 0; r < 3; ++r) m.tr[r] = lay.trellis[r];
m.n_embd = n_embd;
m.n_ff = n_ff;
m.limit = limit;
return m;
}
} // namespace
void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s) {
if (!mx_geometry("moe_gate_up", a.k, a.n_embd, a.n_ff) || h == nullptr) return;
Scratch* sc = scratch(s);
if (sc == nullptr) return;
if (!mx_experts(a.lay)) { // low-bit experts: ds4_iq's kernels
iq::Args ia;
if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "moe_gate_up")) return;
ia.plan_ptr = a.plan_ptr;
ia.h = h;
ia.xq = sc->xq;
ia.xd = sc->xd;
q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);
iq::gate_up(ia, a.k, 1, s);
check("moe_gate_up");
return;
}
MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);
m.plan_ptr = a.plan_ptr;
m.x = x;
m.h = h;
m.xq = sc->xq;
m.xd = sc->xd;
q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);
#if defined(STRATA_METAL)
// mx_gate_up_kernel on the Metal dispatch (the MXFP4 expert GEMM); the launch below is the fallback
if (::strata::dsv41::metal_host::mx_gu(a.plan_ptr, nullptr, nullptr, nullptr, nullptr, a.lay.trellis, a.n_embd,
a.n_ff, a.limit, sc->xq, sc->xd, h, a.k)) {
check("moe_gate_up");
return;
}
#endif
launch_groups<GateUpK>(m, a.n_ff, a.k, 1, s);
check("moe_gate_up");
}
void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {
if (!mx_geometry("moe_down", a.k, a.n_embd, a.n_ff) || h == nullptr) return;
Scratch* sc = scratch(s);
if (sc == nullptr) return;
MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);
m.plan_ptr = a.plan_ptr;
m.h = const_cast<float*>(h);
m.part = sc->part;
m.hq = sc->hq;
m.hd = sc->hd;
q8_quant(h, (size_t) a.n_ff, a.k, a.n_ff, sc->hq, sc->hd, s);
if (!mx_experts(a.lay)) { // low-bit experts: ds4_iq's down, then the same combine
iq::Args ia;
if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "moe_down")) return;
ia.plan_ptr = a.plan_ptr;
ia.part = sc->part;
ia.hq = sc->hq;
ia.hd = sc->hd;
iq::down(ia, a.k, 1, s);
} else {
#if defined(STRATA_METAL)
// mx_down_kernel on the Metal dispatch; the launch below is the fallback
if (!::strata::dsv41::metal_host::mx_dn(a.plan_ptr, nullptr, nullptr, nullptr, nullptr, a.lay.trellis, a.n_embd,
a.n_ff, sc->hq, sc->hd, sc->part, a.k))
#endif
launch_groups<DownK>(m, a.n_embd, a.k, 1, s);
}
CombArgs c{};
c.plan_ptr = a.plan_ptr;
c.plan_w = a.plan_w;
c.part = sc->part;
c.cpu_part = a.cpu_part;
c.cpu_stride = (size_t) a.n_embd;
c.cpu_flag = a.cpu_flag;
c.sh_out = sh_out;
c.out = out;
c.out_stride = (size_t) a.n_embd;
c.k = a.k;
c.n_embd = a.n_embd;
#if defined(STRATA_METAL)
if (!::strata::dsv41::metal_host::mx_combine(c.plan_ptr, c.plan_w, c.part, c.cpu_part, c.cpu_stride,
c.cpu_flag, c.sh_out, c.out, c.out_stride, 1, c.k, c.n_embd))
#endif
strata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s, c);
check("moe_down");
}
bool win_experts(const x3::WinArgs& a, cudaStream_t s) {
if (a.T < 1 || a.T > kMaxEnt || !mx_geometry("win_experts", a.k, a.n_embd, a.n_ff) || a.h == nullptr ||
a.part == nullptr || a.out == nullptr) {
std::fprintf(stderr, "ds4 gguf win_experts: not covered (T %d)\n", a.T);
return false;
}
MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);
m.grp_ptr = a.grp_ptr;
m.grp_start = a.grp_start;
m.grp_ent = a.grp_ent;
m.n_groups = a.n_groups;
Scratch* sc = scratch(s);
if (sc == nullptr) return false;
m.x = a.x;
m.x_stride = a.x_stride;
m.h = a.h;
m.part = a.part;
m.xq = sc->xq;
m.xd = sc->xd;
m.hq = sc->hq;
m.hd = sc->hd;
const int groups = a.T * a.k; // at most (the blocks past the real count return at once)
const bool mx = mx_experts(a.lay);
iq::Args ia;
if (!mx) { // low-bit experts: ds4_iq's kernels on the same groups and scratch
if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "win_experts")) return false;
ia.grp_ptr = a.grp_ptr;
ia.grp_start = a.grp_start;
ia.grp_ent = a.grp_ent;
ia.n_groups = a.n_groups;
ia.h = a.h;
ia.part = a.part;
ia.xq = sc->xq;
ia.xd = sc->xd;
ia.hq = sc->hq;
ia.hd = sc->hd;
}
q8_quant(a.x, a.x_stride, a.T, a.n_embd, sc->xq, sc->xd, s);
if (mx) {
#if defined(STRATA_METAL)
// mx_gate_up_kernel on the Metal dispatch; the launch below is the fallback
if (!::strata::dsv41::metal_host::mx_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups,
a.lay.trellis, a.n_embd, a.n_ff, a.limit, sc->xq, sc->xd, a.h, groups))
#endif
launch_groups<GateUpK>(m, a.n_ff, groups, a.T, s);
} else {
iq::gate_up(ia, groups, a.T, s);
}
q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)
if (mx) {
#if defined(STRATA_METAL)
// mx_down_kernel on the Metal dispatch; the launch below is the fallback
if (!::strata::dsv41::metal_host::mx_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups,
a.lay.trellis, a.n_embd, a.n_ff, sc->hq, sc->hd, a.part, groups))
#endif
launch_groups<DownK>(m, a.n_embd, groups, a.T, s);
} else {
iq::down(ia, groups, a.T, s);
}
#if !defined(STRATA_METAL)
// (this port's dispatches are synchronous and nothing is published to wait for: ds4_kernels.cu's wait_seq)
if (a.wait_ptr != nullptr) strata_ds4_launch(mx_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);
#endif
CombArgs c{};
c.plan_ptr = a.plan_ptr;
c.plan_w = a.plan_w;
c.part = a.part;
c.cpu_part = a.cpu_part;
c.cpu_stride = a.cpu_stride != 0 ? a.cpu_stride : (size_t) a.n_embd;
c.cpu_flag = a.cpu_flag;
c.sh_out = a.sh_out;
c.out = a.out;
c.out_stride = a.out_stride;
c.k = a.k;
c.n_embd = a.n_embd;
#if defined(STRATA_METAL)
// mx_combine_kernel on the Metal dispatch; the launch below is the fallback
if (!::strata::dsv41::metal_host::mx_combine(c.plan_ptr, c.plan_w, c.part, c.cpu_part, c.cpu_stride,
c.cpu_flag, c.sh_out, c.out, c.out_stride, a.T, c.k, c.n_embd))
#endif
strata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s, c);
check("win_experts");
return true;
}
namespace {
iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (the inputs / outputs and types to fill in)
iq::SetArgs q;
q.wbase = sa.wbase;
q.stride = sa.stride;
for (int r = 0; r < 3; ++r) q.tr[r] = sa.lay.trellis[r];
q.bounds = sa.bounds;
q.row0 = sa.row0;
q.max_rows = sa.max_rows;
q.n_exp = sa.n_exp;
q.n_embd = sa.n_embd;
q.n_ff = sa.n_ff;
q.limit = sa.limit;
q.xf16 = sa.x16;
q.skip_ge = sa.skip_ge;
#if defined(STRATA_METAL)
q.hbounds = sa.hbounds;
q.total_rows = sa.total_rows;
#endif
return q;
}
} // namespace
void set_gate_up(const x3::SetArgs& sa, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,
cudaStream_t s) {
if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
if (!mx_geometry("set_gate_up", 1, sa.n_embd, sa.n_ff)) return;
if (!mx_experts(sa.lay)) { // low-bit experts: ds4_iq's sets
iq::Args ia;
if (!iq_args(sa.lay, sa.n_embd, sa.n_ff, sa.limit, ia, "set_gate_up")) return;
iq::SetArgs q = iq_set(sa);
q.xf = x;
q.idx = row_idx;
q.ldx = ldx;
q.h = h16;
q.ldo = ldh;
q.tg = ia.tg;
q.td = ia.td;
if (!iq::set_gate_up(q, s)) fail("set_gate_up: not covered");
check("set_gate_up");
return;
}
int skip = 0;
{ // the experts of at most 16 rows: the light kernels (each weight read once; ds4_iq.cuh), the others the tiles
iq::SetArgs q = iq_set(sa);
q.xf = x;
q.idx = row_idx;
q.ldx = ldx;
q.h = h16;
q.ldo = ldh;
q.tg = q.td = kTypeMXFP4;
if (iq::light_gate_up(q, s)) {
if (sa.max_rows <= 16) {
check("set_gate_up");
return;
}
skip = 16;
}
}
SetK k{};
k.skip_le = skip;
k.wbase = sa.wbase;
k.stride = sa.stride;
for (int r = 0; r < 3; ++r) k.tr[r] = sa.lay.trellis[r];
k.bounds = sa.bounds;
k.row0 = sa.row0;
k.n_embd = sa.n_embd;
k.n_ff = sa.n_ff;
k.limit = sa.limit;
k.xf = x;
k.idx = row_idx;
k.ldx = ldx;
k.h = (__half*) h16;
k.ldo = ldh;
const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_ff / BN), (unsigned) sa.n_exp);
#if !defined(STRATA_USE_HIP)
if (tc_ok()) strata_ds4_launch(mx_set_tc_kernel<true>, grid, kThreads, 0, s, k);
#else
if (tc_ok()) strata_ds4_launch(mx_set_wmma_kernel<true>, grid, kThreads, 0, s, k);
#endif
else
strata_ds4_launch(mx_set_kernel<true>, grid, kThreads, 0, s, k);
check("set_gate_up");
}
void set_down(const x3::SetArgs& sa, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s) {
if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
if (!mx_geometry("set_down", 1, sa.n_embd, sa.n_ff)) return;
if (!mx_experts(sa.lay)) { // low-bit experts: ds4_iq's sets
iq::Args ia;
if (!iq_args(sa.lay, sa.n_embd, sa.n_ff, sa.limit, ia, "set_down")) return;
iq::SetArgs q = iq_set(sa);
q.xh = h16;
q.ldx = ldh;
q.y = y;
q.ldo = ldy;
q.tg = ia.tg;
q.td = ia.td;
if (!iq::set_down(q, s)) fail("set_down: not covered");
check("set_down");
return;
}
int skip = 0;
{ // (as set_gate_up's)
iq::SetArgs q = iq_set(sa);
q.xh = h16;
q.ldx = ldh;
q.y = y;
q.ldo = ldy;
q.tg = q.td = kTypeMXFP4;
if (iq::light_down(q, s)) {
if (sa.max_rows <= 16) {
check("set_down");
return;
}
skip = 16;
}
}
SetK k{};
k.skip_le = skip;
k.wbase = sa.wbase;
k.stride = sa.stride;
for (int r = 0; r < 3; ++r) k.tr[r] = sa.lay.trellis[r];
k.bounds = sa.bounds;
k.row0 = sa.row0;
k.n_embd = sa.n_embd;
k.n_ff = sa.n_ff;
k.xh = (const __half*) h16;
k.ldx = ldh;
k.y = y;
k.ldo = ldy;
const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_embd / BN), (unsigned) sa.n_exp);
#if !defined(STRATA_USE_HIP)
if (tc_ok()) strata_ds4_launch(mx_set_tc_kernel<false>, grid, kThreads, 0, s, k);
#else
if (tc_ok()) strata_ds4_launch(mx_set_wmma_kernel<false>, grid, kThreads, 0, s, k);
#endif
else
strata_ds4_launch(mx_set_kernel<false>, grid, kThreads, 0, s, k);
check("set_down");
}
} // namespace strata::ds4::gguf