Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// metal/src/engine/msl/ds4_init.metal - the first kernels the engine asks for, ported from ds4_kernels.cu.
//
// ds4_kernels.cu:1477 __global__ void init_pre_kernel(float* pre, int T) {
// const int i = blockIdx.x * blockDim.x + threadIdx.x;
// if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;
// }
#include <metal_stdlib>
using namespace metal;
// the grid is threadgroups of 128; the flat id is what CUDA's blockIdx*blockDim+threadIdx was
kernel void ds4_init_pre(device float* pre [[buffer(0)]], constant uint& T [[buffer(1)]],
uint gid [[thread_position_in_grid]]) {
if (gid < T * 8u) pre[gid] = (gid % 8u) == 0u ? 1.0f : 0.0f;
}
// ds4_kernels.cu:130 embed_kernel: v = bf16(rows[t*dim + e]); h[(t*hc + s)*dim + e] = v for each of hc copies
kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [[buffer(1)]],
constant uint& dim [[buffer(2)]], constant uint& hc [[buffer(3)]],
uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {
const uint e = tgp.x * 256u + uint(lane);
if (e >= dim) return;
const uint t = tgp.y;
const uint bits = uint(rows[(size_t) t * dim + e]) << 16; // BF16 -> F32 is the top half
const float v = as_type<float>(bits);
for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;
}
// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, n_mix, T, hcd, ...) - the mHC mixes: mixes[t][m] = h[t] . fn[m]
// one thread an output element; both operands F32, alpha = 1, beta = 0 (every call site seen)
kernel void ds4_sgemm_tn(device const float* A [[buffer(0)]], device const float* B [[buffer(1)]],
device float* C [[buffer(2)]], constant uint& m [[buffer(3)]], constant uint& nt [[buffer(4)]],
constant uint& k [[buffer(5)]], constant uint& lda [[buffer(6)]], constant uint& ldb [[buffer(7)]],
constant uint& ldc [[buffer(8)]], uint gid [[thread_position_in_grid]]) {
const uint total = m * nt;
if (gid >= total) return;
const uint t = gid / m, mm = gid % m;
const device float* a = A + (size_t) mm * lda; // row mm of A
const device float* b = B + (size_t) t * ldb; // row t of B
float acc = 0.0f;
for (uint i = 0; i < k; ++i) acc += a[i] * b[i];
C[(size_t) t * ldc + mm] = acc;
}
// ds4_kernels.cu:1894 finish_kernel<false> - RMS norm (f.w), mrope (f.rd), rounding (f.quant), then store to x or dst.
// The mrope and rounding branches are not written yet; the host wrapper will decline until they are.
struct Ds4Finish {
device float* w;
float eps;
int rd, pos, inverse, vpp, pos_step;
device float* freqs;
int quant;
device ushort* dst;
long dst_stride;
device uchar* pdst;
long pdst_stride;
};
static inline float ds4_warp_sum(float v) {
v += simd_shuffle_xor(v, 16);
v += simd_shuffle_xor(v, 8);
v += simd_shuffle_xor(v, 4);
v += simd_shuffle_xor(v, 2);
v += simd_shuffle_xor(v, 1);
return v;
}
static inline float ds4_block_sum(float v, threadgroup float* red, ushort lane, ushort w) {
v = ds4_warp_sum(v);
if (lane == 0) red[w] = v;
threadgroup_barrier(mem_flags::mem_threadgroup);
if (w == 0) {
float t = lane < 8 ? red[lane] : 0.0f;
t = ds4_warp_sum(t);
if (lane == 0) red[0] = t;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
return red[0];
}
// ds4_kernels.cu:65 pow2_ceil / :88 fp8_rt / :90 e2m1_rt - the KV quantiser's scale and roundings, shared by finish_pack
// (the packed rows) and ds4_finish's rounding branch
static inline float ds4_pow2_ceil(float x) {
const uint b = as_type<uint>(x);
const int e = int((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);
return as_type<float>(uint(e + 127) << 23);
}
static inline float ds4_e4m3_round(float v) { // E4M3, nearest even (the input is clamped)
const float a = fabs(v);
if (!(a == a)) return v;
float r;
if (a >= 448.0f) r = 448.0f;
else {
const int e = int((as_type<uint>(a) >> 23) & 0xffu) - 127;
const float step = e < -6 ? 0.001953125f : as_type<float>(uint(e - 3 + 127) << 23);
r = a == 0.0f ? 0.0f : rint(a / step) * step;
r = fmin(r, 448.0f);
}
return copysign(r, v);
}
static inline float ds4_e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even
const float a = fabs(v);
float r;
if (a <= 0.25f) r = 0.0f;
else if (a < 0.75f) r = 0.5f;
else if (a <= 1.25f) r = 1.0f;
else if (a < 1.75f) r = 1.5f;
else if (a <= 2.5f) r = 2.0f;
else if (a < 3.5f) r = 3.0f;
else if (a <= 5.0f) r = 4.0f;
else r = 6.0f;
return copysign(r, v);
}
kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],
constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {
threadgroup float red[8];
device float* v = x + (long) b * stride;
const uint nj = (len + 255u) / 256u;
float r[32];
float ss = 0.0f;
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
r[j] = e < len ? v[e] : 0.0f;
ss += r[j] * r[j];
}
if (f.w != nullptr) {
const float tot = ds4_block_sum(ss, red, lane, w);
const float inv = rsqrt(tot / float(len) + f.eps);
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
if (e < len) r[j] = r[j] * inv * f.w[e];
}
}
if (f.rd > 0) {
// mrope: the last rd values move in (re, im) pairs a lane apart (ds4_kernels.cu:161): the partner comes from a
// lane xor 1, the angle from the token's position and the frequency of the pair
const uint r0 = len - uint(f.rd);
const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
const float other = simd_shuffle_xor(r[j], 1);
if (e >= r0 && e < len) {
const uint i = (e - r0) >> 1;
const float ang = float(pos) * f.freqs[i];
const float c = cos(ang);
const float sn = f.inverse ? -sin(ang) : sin(ang);
r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;
}
}
}
if (f.quant != 0) { // the KV rounding without the pack: the group's scale, then the value rounded to the format
const uint g = f.quant == 2 ? 16u : 32u;
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
float a = e < len ? fabs(r[j]) : 0.0f;
for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));
if (e >= len) continue;
if (f.quant == 1) {
const float s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));
r[j] = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f)) * s;
} else if (f.quant == 2) {
const float s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);
r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;
} else {
const float s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;
}
}
}
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
if (e >= len) continue;
if (f.dst != nullptr) {
const uint bits = as_type<uint>(r[j]);
f.dst[(long) b * f.dst_stride + (long) e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even, the vector's own row
} else {
v[e] = r[j];
}
}
}
// ds4_kernels.cu:1677 hc_post_inplace_kernel: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]
kernel void ds4_hc_post(device const float* y [[buffer(0)]], device float* h [[buffer(1)]],
device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],
constant uint& dim [[buffer(4)]], constant uint& hc [[buffer(5)]],
uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {
const uint e = tgp.x * 256u + uint(lane);
if (e >= dim) return;
const uint t = tgp.y;
device float* r = h + (size_t) t * hc * dim;
float rv[8];
for (uint j = 0; j < 8u; ++j) if (j < hc) rv[j] = r[(size_t) j * dim + e];
const float yv = y[(size_t) t * dim + e];
for (uint k = 0; k < 8u; ++k) {
if (k >= hc) break;
float a = post[(size_t) t * 8u + k] * yv;
for (uint j = 0; j < 8u; ++j) if (j < hc) a += comb[(size_t) t * 64u + j * hc + k] * rv[j];
r[(size_t) k * dim + e] = a;
}
}
// ds4_kernels.cu:1692 hc_mix_scale_kernel: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps), one threadgroup a token
kernel void ds4_hc_mix_scale(device const float* h [[buffer(0)]], device float* mixes [[buffer(1)]],
constant uint& hcdim [[buffer(2)]], constant uint& n_mix [[buffer(3)]],
constant float& eps [[buffer(4)]], uint b [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort w [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[8];
const device float* x = h + (size_t) b * hcdim;
float ss = 0.0f;
for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
const float4 hv = *(const device float4*) (x + k);
ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
}
ss = ds4_block_sum(ss, red, lane, w);
const float inv = rsqrt(ss / float(hcdim) + eps);
if (tid < n_mix) mixes[(size_t) b * 32u + tid] *= inv;
}
// ds4_kernels.cu:1017 hc_mix_kernel: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps), one threadgroup a (m, t)
kernel void ds4_hc_mix(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],
device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],
constant float& eps [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort w [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[8];
const uint m = tgp.x, t = tgp.y;
const device float* x = h + (size_t) t * hcdim;
const device float* f = fn + (size_t) m * hcdim;
float acc = 0.0f, ss = 0.0f;
for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
const float4 hv = *(const device float4*) (x + k);
const float4 fv = *(const device float4*) (f + k);
acc += hv.x * fv.x + hv.y * fv.y + hv.z * fv.z + hv.w * fv.w;
ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
}
acc = ds4_block_sum(acc, red, lane, w);
ss = ds4_block_sum(ss, red, lane, w);
if (tid == 0) mixes[(size_t) t * 32u + m] = acc * rsqrt(ss / float(hcdim) + eps);
}
// hc_mix for a prompt chunk: one threadgroup a token for four mixes a pass (the token's row is read once per four mixes instead of once per
// mix). Every mix's partial sums keep the single-mix kernel's order, so the result is bit-identical.
kernel void ds4_hc_mix_all(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],
device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],
constant float& eps [[buffer(4)]], constant uint& n_mix [[buffer(5)]],
uint t [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort w [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[8];
const device float* x = h + (size_t) t * hcdim;
float ss = 0.0f;
for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
const float4 hv = *(const device float4*) (x + k);
ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
}
ss = ds4_block_sum(ss, red, lane, w);
const float inv = rsqrt(ss / float(hcdim) + eps);
for (uint m0 = 0; m0 < n_mix; m0 += 4u) {
const uint nm = min(4u, n_mix - m0);
float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;
const device float* f0 = fn + (size_t) m0 * hcdim;
const device float* f1 = f0 + (nm > 1u ? hcdim : 0u);
const device float* f2 = f0 + (nm > 2u ? 2u * hcdim : 0u);
const device float* f3 = f0 + (nm > 3u ? 3u * hcdim : 0u);
for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
const float4 hv = *(const device float4*) (x + k);
const float4 v0 = *(const device float4*) (f0 + k);
const float4 v1 = *(const device float4*) (f1 + k);
const float4 v2 = *(const device float4*) (f2 + k);
const float4 v3 = *(const device float4*) (f3 + k);
a0 += hv.x * v0.x + hv.y * v0.y + hv.z * v0.z + hv.w * v0.w;
a1 += hv.x * v1.x + hv.y * v1.y + hv.z * v1.z + hv.w * v1.w;
a2 += hv.x * v2.x + hv.y * v2.y + hv.z * v2.z + hv.w * v2.w;
a3 += hv.x * v3.x + hv.y * v3.y + hv.z * v3.z + hv.w * v3.w;
}
a0 = ds4_block_sum(a0, red, lane, w);
a1 = ds4_block_sum(a1, red, lane, w);
a2 = ds4_block_sum(a2, red, lane, w);
a3 = ds4_block_sum(a3, red, lane, w);
if (tid == 0) {
mixes[(size_t) t * 32u + m0] = a0 * inv;
if (nm > 1u) mixes[(size_t) t * 32u + m0 + 1u] = a1 * inv;
if (nm > 2u) mixes[(size_t) t * 32u + m0 + 2u] = a2 * inv;
if (nm > 3u) mixes[(size_t) t * 32u + m0 + 3u] = a3 * inv;
}
}
}
// hc mix + split in one dispatch. The mix is ds4_hc_mix_all's (four mixes a pass, every one keeping the single-mix
// kernel's summation order, so the values are bit-identical), then the same threadgroup runs the sinkhorn split of the
// 24 mixes into pre / post / comb. Two dispatches become one, and nothing else changes.
kernel void ds4_hc_mix_split(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],
device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],
constant float& eps [[buffer(4)]], constant uint& n_mix [[buffer(5)]],
device const float* scale [[buffer(6)]], device const float* base [[buffer(7)]],
device float* pre [[buffer(8)]], device float* post [[buffer(9)]],
device float* comb [[buffer(10)]], constant uint& hc [[buffer(11)]],
constant uint& iters [[buffer(12)]], constant float& hc_eps [[buffer(13)]],
uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort wid [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[8];
const device float* x = h + (size_t) t * hcdim;
float ss = 0.0f;
for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
const float4 hv = *(const device float4*) (x + k);
ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
}
ss = ds4_block_sum(ss, red, lane, wid);
const float inv = rsqrt(ss / float(hcdim) + eps);
for (uint m0 = 0; m0 < n_mix; m0 += 4u) {
const uint nm = min(4u, n_mix - m0);
float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;
const device float* f0 = fn + (size_t) m0 * hcdim;
const device float* f1 = f0 + (nm > 1u ? hcdim : 0u);
const device float* f2 = f0 + (nm > 2u ? 2u * hcdim : 0u);
const device float* f3 = f0 + (nm > 3u ? 3u * hcdim : 0u);
for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
const float4 hv = *(const device float4*) (x + k);
const float4 v0 = *(const device float4*) (f0 + k);
const float4 v1 = *(const device float4*) (f1 + k);
const float4 v2 = *(const device float4*) (f2 + k);
const float4 v3 = *(const device float4*) (f3 + k);
a0 += hv.x * v0.x + hv.y * v0.y + hv.z * v0.z + hv.w * v0.w;
a1 += hv.x * v1.x + hv.y * v1.y + hv.z * v1.z + hv.w * v1.w;
a2 += hv.x * v2.x + hv.y * v2.y + hv.z * v2.z + hv.w * v2.w;
a3 += hv.x * v3.x + hv.y * v3.y + hv.z * v3.z + hv.w * v3.w;
}
a0 = ds4_block_sum(a0, red, lane, wid);
a1 = ds4_block_sum(a1, red, lane, wid);
a2 = ds4_block_sum(a2, red, lane, wid);
a3 = ds4_block_sum(a3, red, lane, wid);
if (tid == 0) {
mixes[(size_t) t * 32u + m0] = a0 * inv;
if (nm > 1u) mixes[(size_t) t * 32u + m0 + 1u] = a1 * inv;
if (nm > 2u) mixes[(size_t) t * 32u + m0 + 2u] = a2 * inv;
if (nm > 3u) mixes[(size_t) t * 32u + m0 + 3u] = a3 * inv;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// the split: ds4_hc_split4's math, in the first warp (every lane of it takes the same path)
const device float* mx = mixes + (size_t) t * 32u;
if (tid < 4u) {
pre[(size_t) t * 8u + tid] = 1.0f / (1.0f + exp(-(mx[tid] * scale[0] + base[tid]))) + hc_eps;
post[(size_t) t * 8u + tid] = 2.0f / (1.0f + exp(-(mx[4u + tid] * scale[1] + base[4u + tid])));
}
if (tid < 32u) {
const bool on = lane < 16u;
float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;
float m = fmax(c, simd_shuffle_xor(c, ushort(1)));
m = fmax(m, simd_shuffle_xor(m, ushort(2)));
c = on ? exp(c - m) : 0.0f;
float rs = c + simd_shuffle_xor(c, ushort(1));
rs += simd_shuffle_xor(rs, ushort(2));
c = c / rs + hc_eps;
c = on ? c : 0.0f;
float cs = c + simd_shuffle_xor(c, ushort(4));
cs += simd_shuffle_xor(cs, ushort(8));
c = c / (cs + hc_eps);
for (uint it = 1; it < iters; ++it) {
float r = c + simd_shuffle_xor(c, ushort(1));
r += simd_shuffle_xor(r, ushort(2));
c = c / (r + hc_eps);
float q = c + simd_shuffle_xor(c, ushort(4));
q += simd_shuffle_xor(q, ushort(8));
c = c / (q + hc_eps);
}
if (on) comb[(size_t) t * 64u + lane] = c;
}
}
// ds4_kernels.cu:268 hc_split_kernel - the sinkhorn normalisation of the hyper-connection mix.
// This is the SCALAR path of the CUDA kernel (lane 0 of the threadgroup does the token, a local 8x8 matrix), applied for every
// hc including 4: it computes the same mathematics as the warp-lane version, only without the shuffles. Correctness first -
// the lane-parallel hc == 4 version can be added behind a profile.
kernel void ds4_hc_split(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],
device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],
device float* post [[buffer(4)]], device float* comb [[buffer(5)]],
constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],
constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],
uint lane [[thread_index_in_threadgroup]]) {
if (lane != 0u) return;
const device float* mx = mixes + (size_t) b * 32u;
device float* pr = pre + (size_t) b * 8u;
device float* po = post + (size_t) b * 8u;
device float* cb = comb + (size_t) b * 64u;
for (uint i = 0; i < 8u; ++i) {
if (i >= hc) break;
pr[i] = 1.0f / (1.0f + exp(-(mx[i] * scale[0] + base[i]))) + eps;
po[i] = 2.0f / (1.0f + exp(-(mx[hc + i] * scale[1] + base[hc + i])));
}
float c[8][8];
for (uint j = 0; j < 8u; ++j) {
if (j >= hc) break;
float m = -3.402823466e+38f;
for (uint k = 0; k < 8u; ++k) {
if (k >= hc) break;
c[j][k] = mx[2u * hc + j * hc + k] * scale[2] + base[2u * hc + j * hc + k];
m = fmax(m, c[j][k]);
}
float s = 0.0f;
for (uint k = 0; k < 8u; ++k) {
if (k >= hc) break;
c[j][k] = exp(c[j][k] - m);
s += c[j][k];
}
for (uint k = 0; k < 8u; ++k) {
if (k >= hc) break;
c[j][k] = c[j][k] / s + eps;
}
}
for (uint k = 0; k < 8u; ++k) {
if (k >= hc) break;
float s = 0.0f;
for (uint j = 0; j < 8u; ++j) if (j < hc) s += c[j][k];
for (uint j = 0; j < 8u; ++j) if (j < hc) c[j][k] /= s + eps;
}
for (uint it = 1; it < iters; ++it) {
for (uint j = 0; j < 8u; ++j) {
if (j >= hc) break;
float s = 0.0f;
for (uint k = 0; k < 8u; ++k) if (k < hc) s += c[j][k];
for (uint k = 0; k < 8u; ++k) if (k < hc) c[j][k] /= s + eps;
}
for (uint k = 0; k < 8u; ++k) {
if (k >= hc) break;
float s = 0.0f;
for (uint j = 0; j < 8u; ++j) if (j < hc) s += c[j][k];
for (uint j = 0; j < 8u; ++j) if (j < hc) c[j][k] /= s + eps;
}
}
for (uint j = 0; j < 8u; ++j) {
if (j >= hc) break;
for (uint k = 0; k < 8u; ++k) { if (k >= hc) break; cb[j * hc + k] = c[j][k]; }
}
}
// ds4_kernels.cu:1938 moe_sort - count each virtual slot's picks, prefix-sum the counts, then fill in token order.
// kPicks = 8 (ds4_kernels.cuh:19): idx is [T][8], so q -> (t = q/8, i = q%8).
kernel void ds4_bounds_scan(device int* bounds [[buffer(0)]], constant uint& n_vs [[buffer(1)]],
uint tid [[thread_index_in_threadgroup]]) {
if (tid != 0u) return;
bounds[0] = 0;
for (uint s = 0; s < n_vs; ++s) bounds[s + 1] += bounds[s];
}
// simd_ballot returns a simd_vote, which does not cast to an integer in this toolchain; the same counts come from the
// prefix-sum and sum primitives, which give a lane its offset inside its simdgroup and the group its total.
static inline int ds4_warp_prefix(bool hit) { return simd_prefix_exclusive_sum(hit ? 1 : 0); }
static inline int ds4_warp_total(bool hit) { return simd_sum(hit ? 1 : 0); }
kernel void ds4_moe_sort_count(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],
constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],
device int* bounds [[buffer(4)]], uint slot [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
threadgroup int warp_n[8];
threadgroup int base_s;
const uint wp = tid / 32u, lane = tid % 32u;
if (tid == 0u) base_s = 0;
threadgroup_barrier(mem_flags::mem_threadgroup);
const uint n = T * kk;
for (uint q = tid; q < n; q += 256u) {
const uint t = q / kk, i = q - t * kk;
const bool hit = vs[idx[t * 8u + i]] == int(slot);
// one vote a warp: the running offset needs each warp's count, accumulated in slot order
const int wtot = ds4_warp_total(hit);
if ((tid % 32u) == 0u) warp_n[wp] = wtot;
threadgroup_barrier(mem_flags::mem_threadgroup);
int total = 0;
for (uint w = 0; w < 8u; ++w) total += warp_n[w];
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid == 0u) base_s += total;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
if (tid == 0u) bounds[slot + 1] = base_s;
}
kernel void ds4_moe_sort_fill(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],
constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],
device int* bounds [[buffer(4)]], device int* row_tok [[buffer(5)]],
device int* pos [[buffer(6)]], uint slot [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
threadgroup int warp_n[8];
threadgroup int base_s;
const uint wp = tid / 32u, lane = tid % 32u;
if (tid == 0u) base_s = bounds[slot];
threadgroup_barrier(mem_flags::mem_threadgroup);
const uint n = T * kk;
for (uint q = tid; q < n; q += 256u) {
const uint t = q / kk, i = q - t * kk;
const bool hit = vs[idx[t * 8u + i]] == int(slot);
const int wpre = ds4_warp_prefix(hit), wtot = ds4_warp_total(hit);
if ((tid % 32u) == 0u) warp_n[wp] = wtot;
threadgroup_barrier(mem_flags::mem_threadgroup);
int before = 0, total = 0;
for (uint w = 0; w < 8u; ++w) { if (w < wp) before += warp_n[w]; total += warp_n[w]; }
if (hit) {
const int r = base_s + before + wpre;
row_tok[r] = int(t);
pos[t * 8u + i] = r;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid == 0u) base_s += total;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
// ds4_kernels.cu:1690 engram_dequant_tok_kernel - one hash-table row a threadgroup: IQ4_NL blocks to F32.
// fmt 2 (this model's Engram): an 18-byte block is fp16 d then 16 packed nibbles; the low nibble of q[j] is value j and the
// high nibble value j + 16, each an index into ggml-common.h's kvalues_iq4nl (ds4_kernels.cu:1280).
constant int ds4_kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
kernel void ds4_engram_dequant(device const uchar* src [[buffer(0)]], constant ulong& per [[buffer(1)]],
constant uint& cols [[buffer(2)]], constant uint& hd [[buffer(3)]],
device float* out [[buffer(4)]], constant int& fmt [[buffer(5)]],
uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint t = r / cols, c = r - t * cols;
const device uchar* rows = src + (size_t) t * per + (size_t) c * hd;
if (fmt == 2) {
for (uint e = tid; e < hd; e += 256u) {
const device uchar* b = rows + (size_t) (e / 32u) * 18u;
const uint q = b[2u + (e & 15u)];
const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);
const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);
out[(size_t) r * hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);
}
return;
}
// fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: write NaN, so a wrong path shows as NaN in the output rather than
// as a silent zero.
for (uint e = tid; e < hd; e += 256u) out[(size_t) r * hd + e] = NAN;
}
// ds4_kernels.cu:341 hc_pre_norm_kernel - per token: a = sum_s mix[s]*h[token][s][e], one RMS over dim, then x = a*inv
// (times w when given). One threadgroup a token, 1024 threads; a thread holds at most 8 values (dim <= 8*1024 = 8192).
kernel void ds4_hc_pre_norm(device const float* h [[buffer(0)]], device const float* mix [[buffer(1)]],
device const float* w [[buffer(2)]], constant uint& dim [[buffer(3)]],
constant uint& hc [[buffer(4)]], constant float& eps [[buffer(5)]],
device float* x [[buffer(6)]], constant uint& has_w [[buffer(7)]],
uint b [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[32];
const device float* hh = h + (size_t) b * hc * dim;
const device float* mm = mix + (size_t) b * 8u;
device float* xx = x + (size_t) b * dim;
float r[8];
float ss = 0.0f;
uint n = 0;
for (uint e = tid; e < dim; e += 1024u, ++n) {
float a = 0.0f;
for (uint s = 0; s < 8u; ++s) if (s < hc) a += mm[s] * hh[(size_t) s * dim + e];
if (n < 8u) r[n] = a;
ss += a * a;
}
// a 1024-thread reduction: 32 simdgroups, each into red[], then the first simdgroup sums those
float v = ds4_warp_sum(ss);
if (lane == 0u) red[wg] = v;
threadgroup_barrier(mem_flags::mem_threadgroup);
if (wg == 0u) {
float t = lane < 32u ? red[lane] : 0.0f;
t = ds4_warp_sum(t);
if (lane == 0u) red[0] = t;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
const float inv = rsqrt(red[0] / float(dim) + eps);
n = 0;
for (uint e = tid; e < dim; e += 1024u, ++n) {
if (n >= 8u) break;
xx[e] = has_w != 0u ? r[n] * inv * w[e] : r[n];
}
}
// ds4_gguf.cu:186 q8_quant_kernel: a warp a 32-group - amax across the warp, sc = amax/127, q = rint(v/sc), d = sc.
// grid = (ceil((k/32) / 8 warps), rows), threadgroup = 256 (8 warps of 32).
kernel void ds4_q8_quant(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],
constant uint& k [[buffer(2)]], device char* q [[buffer(3)]],
device float* d [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
const uint warp = tid / 32u, lane = tid % 32u;
const uint b = tgp.x * 8u + warp; // the 32-group index
const uint r = tgp.y;
if ((size_t) b * 32u >= (size_t) k) return;
const device float* xr = x + (size_t) r * ldx;
const float v = xr[(size_t) b * 32u + lane];
const float am = simd_max(fabs(v));
const float sc = am / 127.0f;
q[(size_t) r * k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);
if (lane == 0u) d[(size_t) r * (k / 32u) + b] = sc;
}
// ds4_gguf.cu:371 mx_mv_kernel - the MXFP4 expert matrix-vector.
// grid = (ceil(n / 8 warps), jobs), threadgroup = 256. A warp a row; a lane strides the 32-value blocks; NC columns of
// activations share each decoded block (the T=8 window), then a warp sum and y[c][row] = alpha*v + bias[row].
// Block layout (kMxBytes = 17 a 32 values): the E8M0 scale byte, then 16 bytes whose low nibble is value j and high nibble
// value j+16, each an index into the 16-entry codebook {0,1,2,3,4,6,8,12} and its negatives.
struct Ds4MxJob {
device const void* w;
ulong ld; // bytes a row
device const float* x;
ulong xcs;
device float* y;
ulong ycs;
device const float* bias;
float alpha;
int k, n, ncol;
device const char* xq;
device const float* xd;
};
static inline float ds4_e8m0(uint e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF
return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));
}
static inline void ds4_mx_block(device const uchar* p, threadgroup const float* lut, thread float* w) {
const float d = ds4_e8m0(uint(p[0]));
for (uint j = 0; j < 16u; ++j) {
const uint q = uint(p[1u + j]);
w[j] = lut[q & 15u] * d;
w[j + 16u] = lut[q >> 4] * d;
}
}
kernel void ds4_mx_mv(device const uchar* jobs [[buffer(0)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]]) {
threadgroup float lut[16];
if (tid < 16u) {
const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};
lut[tid] = tid < 8u ? v[tid] : -v[tid - 8u];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
const uint row = tgp.x * 8u + (tid >> 5);
if (int(row) >= b.n) return;
const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;
thread float acc[8];
for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
const uint nblk = uint(b.k) / 32u;
for (uint kb = lane; kb < nblk; kb += 32u) {
float wv[32];
ds4_mx_block(w + (size_t) kb * 17u, lut, wv);
for (uint c = 0; c < 8u; ++c) {
if (int(c) >= b.ncol) break;
const device float* x = b.x + c * b.xcs + (size_t) kb * 32u;
float a = acc[c];
for (uint j = 0; j < 32u; ++j) a = fma(wv[j], x[j], a);
acc[c] = a;
}
}
for (uint c = 0; c < 8u; ++c) {
if (int(c) >= b.ncol) break;
float v = acc[c];
v += simd_shuffle_xor(v, 16);
v += simd_shuffle_xor(v, 8);
v += simd_shuffle_xor(v, 4);
v += simd_shuffle_xor(v, 2);
v += simd_shuffle_xor(v, 1);
if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
}
// ds4_gguf.cu:320 q8_mv_kernel - the Q8_0 matvec against quantised activations.
// grid = (ceil(n / 8), jobs), threadgroup 256: a warp a row, a lane strides the 34-byte blocks (fp16 d + 32 int8), the dot is
// integer (CUDA's __dp4a in four-value groups; here the same sum written out), then acc += (dw * xd[c]) * sum.
// The staged-prefetch (ST) and vector variants of the CUDA kernel are prefetch strategies only - this reads the blocks directly.
kernel void ds4_q8_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
const uint row = tgp.x * 8u + (tid >> 5);
if (int(row) >= b.n) return;
const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;
const uint nb = uint(b.k) / 32u;
float acc[8];
for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
for (uint kb = lane; kb < nb; kb += 32u) {
const device uchar* p = w + (size_t) kb * 34u;
const ushort dh = ushort(p[0]) | (ushort(p[1]) << 8);
const float dw = float(as_type<half>(dh));
int wv[32];
for (uint j = 0; j < 32u; ++j) wv[j] = int((char) p[2u + j]);
for (uint c = 0; c < 8u; ++c) {
if (int(c) >= b.ncol) break;
const device char* xq = b.xq + (size_t) c * b.k + (size_t) kb * 32u;
int sum = 0;
for (uint j = 0; j < 32u; ++j) sum += wv[j] * int(xq[j]);
acc[c] = fma(dw * b.xd[(size_t) c * nb + kb], float(sum), acc[c]);
}
}
for (uint c = 0; c < 8u; ++c) {
if (int(c) >= b.ncol) break;
float v = acc[c];
v += simd_shuffle_xor(v, 16);
v += simd_shuffle_xor(v, 8);
v += simd_shuffle_xor(v, 4);
v += simd_shuffle_xor(v, 2);
v += simd_shuffle_xor(v, 1);
if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
}
// ds4_kernels.cu:1092 ring_to_ext_kernel - one ring row of the KV cache to BF16: each value an E4M3 code times its group's
// E8M0 power of two (row[e] and row[d + e/32]); rows before the window start write zero.
static inline float ds4_e4m3(uint b) { // E4M3 (bias 7, no infinities, max 448): sign, 4-bit exp, 3-bit mantissa
const float s = (b & 0x80u) ? -1.0f : 1.0f;
const uint ex = (b >> 3) & 0xfu, m = b & 7u;
if (ex == 0u) return s * float(m) * (1.0f / 8.0f) * (1.0f / 64.0f);
if (ex == 15u && m == 7u) return NAN;
return s * exp2(float(ex) - 7.0f) * (1.0f + float(m) * 0.125f);
}
static inline float ds4_ue8m0(uint b) { return as_type<float>(b << 23); } // 2^(b - 127)
static inline ushort ds4_f2bf16(float f) {
const uint u = as_type<uint>(f);
return ushort((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);
}
// ds4_kernels.cu:1104 ext_to_ring_kernel - the window's BF16 rows packed back into the E4M3 ring, each 32-value group at the
// smallest power-of-two scale it fits under (k), so v / 2^k is an E4M3 value times a power of two.
static inline uint ds4_e4m3_code(float v, int k) {
const uint sign = (v < 0.0f) ? 0x80u : 0u;
float a = fabs(v) * exp2(-float(k));
if (a == 0.0f) return sign;
int e = int(floor(log2(a)));
int mant = int(round((a / exp2(float(e)) - 1.0f) * 8.0f));
int ex = e + 7;
if (mant >= 8) { mant = 0; ++ex; }
if (ex <= 0) { mant = 0; ex = 0; }
if (ex > 15) { mant = 7; ex = 15; }
return sign | (uint(ex) << 3) | uint(mant & 7);
}
kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],
device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],
constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],
constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]], constant uint& has_backup [[buffer(8)]],
uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
if (T - t > n) return; // overwritten by a later row of the window anyway
device uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;
if (has_backup != 0u) { // (a bound buffer is never a null pointer here: the host says whether there is a backup)
for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const device ushort* src = ext + (size_t) (n + t) * d;
for (uint e0 = 0; e0 < d; e0 += 256u) {
const uint e = e0 + tid;
const float v = e < d ? as_type<float>(uint(src[e]) << 16) : 0.0f; // (BF16 bits, not half)
uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bits
for (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));
const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);
if (e >= d) continue;
row[e] = uchar(ds4_e4m3_code(v, k));
if ((e & 31u) == 0u) row[d + e / 32u] = uchar(k + 127);
}
}
// ds4_kernels.cu:916 unpack32 - 16 threads expand one MXFP4 KV row (288 B at d=512) into 32 BF16 values a thread:
// 16 code bytes at part*16, two E4M3 scale bytes at 256 + part*2, and the 4-bit code is itself an FP4 bit pattern
// (e2m1_val14: sign and exponent/mantissa bits spliced into a half).
static inline float ds4_e2m1_val14(uint c) {
const ushort h = (ushort) (((c & 8u) << 12) | ((c & 7u) << 9));
return float(as_type<half>(h));
}
// ds4_kernels.cu:937 unpack_sel_kernel - the selected rows of the compressed cache into the attention scratch, one row a
// group of 16 threads. n_sel comes either from the generated schedule (gen: min(kcap, (pos+1)/R)) or from nsel[t].
kernel void ds4_unpack_sel(device const uchar* ckv [[buffer(0)]], device const int* sel [[buffer(1)]],
constant uint& sel_stride [[buffer(2)]], constant int& gen [[buffer(3)]],
constant long& p0 [[buffer(4)]], constant int& t0 [[buffer(5)]],
constant int& R [[buffer(6)]], constant int& kcap [[buffer(7)]],
device const int* nsel [[buffer(8)]], constant uint& cap [[buffer(9)]],
device ushort* out [[buffer(10)]], constant ulong& rb [[buffer(11)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint t = tgp.y;
int n_sel;
if (gen != 0) {
const long pa = p0 + long(t0) + long(t);
n_sel = R > 0 ? int(min(long(kcap), (pa + 1) / long(R))) : 0;
} else {
n_sel = nsel[t];
}
const uint i = tgp.x * 16u + (tid >> 4u), part = tid & 15u;
if (int(i) >= n_sel) return;
const size_t r = (size_t) t * cap + i;
const device uchar* rr = ckv + (size_t) sel[(size_t) t * sel_stride + i] * rb;
const uint4 cw = *(const device uint4*) (rr + part * 16u);
const float k0 = ds4_e4m3(uint(rr[256u + part * 2u])) * 16384.0f;
const float k1 = ds4_e4m3(uint(rr[256u + part * 2u + 1u])) * 16384.0f;
const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};
uint o[16];
for (uint k = 0; k < 16u; ++k) {
const uint b = wv[k >> 2] >> (8u * (k & 3u));
const float ks = k < 8u ? k0 : k1;
const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
}
device uint* dst = (device uint*) (out + r * 512u + part * 32u);
for (uint q = 0; q < 4u; ++q) {
dst[q * 4u + 0u] = o[4u * q + 0u];
dst[q * 4u + 1u] = o[4u * q + 1u];
dst[q * 4u + 2u] = o[4u * q + 2u];
dst[q * 4u + 3u] = o[4u * q + 3u];
}
}
// ds4_kernels.cu:1772 moe_combine_kernel - a token's expert outputs back into its row: for each pick whose routed row is in
// [r0, r1), acc += w[t][i] * y[r - r0][e]; untouched when no pick falls in range.
kernel void ds4_moe_combine(device float* acc [[buffer(0)]], device const float* y [[buffer(1)]],
device const int* pos [[buffer(2)]], device const float* w [[buffer(3)]],
constant int& kk [[buffer(4)]], constant int& r0 [[buffer(5)]],
constant int& r1 [[buffer(6)]], constant uint& dim [[buffer(7)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint e = tgp.x * 256u + tid;
if (e >= dim) return;
const uint t = tgp.y;
float a = acc[(size_t) t * dim + e];
bool any = false;
for (int i = 0; i < 8; ++i) {
if (i >= kk) break;
const int r = pos[(size_t) t * 8u + i];
if (r < r0 || r >= r1) continue;
a = fma(w[(size_t) t * 8u + i], y[(size_t) (r - r0) * dim + e], a);
any = true;
}
if (any) acc[(size_t) t * dim + e] = a;
}
// ds4_kernels.cu:362 hc_post_kernel - like hc_post_inplace, but reads a separate residual and writes a separate output:
// out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]
kernel void ds4_hc_post_v(device const float* y [[buffer(0)]], device const float* res [[buffer(1)]],
device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],
device float* out [[buffer(4)]], constant uint& dim [[buffer(5)]],
constant uint& hc [[buffer(6)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
const uint i = tgp.x * 256u + tid;
if (i >= dim * hc) return;
const uint t = tgp.y, k = i / dim, e = i - k * dim;
const device float* r = res + (size_t) t * hc * dim;
float a = post[(size_t) t * 8u + k] * y[(size_t) t * dim + e];
for (uint j = 0; j < 8u; ++j) {
if (j >= hc) break;
a += comb[(size_t) t * 64u + j * hc + k] * r[(size_t) j * dim + e];
}
out[(size_t) t * hc * dim + i] = a;
}
// ds4_kernels.cu:441 gemv_f32_kernel - a warp a row of F32 weights: float4 loads of W[row] and x[t], four FMAs a step,
// lane-strided over k/4, then the warp sum into y[t][row].
kernel void ds4_gemv_f32(device const float* W [[buffer(0)]], device const float* x [[buffer(1)]],
constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],
device float* y [[buffer(4)]], constant uint& ldy [[buffer(5)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]]) {
const uint row = tgp.x * 8u + (tid >> 5);
if (row >= n) return;
const uint t = tgp.y;
const device float4* w = (const device float4*) (W + (size_t) row * kk);
const device float4* xv = (const device float4*) (x + (size_t) t * kk);
float acc = 0.0f;
for (uint c = lane; c < kk / 4u; c += 32u) {
const float4 a = w[c], b = xv[c];
acc = fma(a.x, b.x, acc);
acc = fma(a.y, b.y, acc);
acc = fma(a.z, b.z, acc);
acc = fma(a.w, b.w, acc);
}
acc += simd_shuffle_xor(acc, 16);
acc += simd_shuffle_xor(acc, 8);
acc += simd_shuffle_xor(acc, 4);
acc += simd_shuffle_xor(acc, 2);
acc += simd_shuffle_xor(acc, 1);
if (lane == 0u) y[(size_t) t * ldy + row] = acc;
}
// ds4_kernels.cu:1353 route_publish_kernel - the MoE router: sc = sqrt(softplus(logit)), bs = sc + bias, then k rounds of a
// block-wide argmax (a strided max, a warp reduction, a 16-warp pass), the picks' sc summed into the weights, and each pick's
// device address published to plan_ptr. The prefetch requests (req) are NOT published here yet: that is the schedule, not the
// answer, and it can be turned on separately.
kernel void ds4_route_publish(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],
constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],
constant float& route_scale [[buffer(4)]], device const ulong* vram [[buffer(5)]],
device int* idx [[buffer(6)]], device float* w [[buffer(7)]],
device ulong* plan_ptr [[buffer(8)]], device const float* bias_vl [[buffer(9)]],
constant uint& vl [[buffer(10)]], constant uint& has_vl [[buffer(11)]],
uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
threadgroup float sc[1024];
threadgroup float bs[1024];
threadgroup float bv[16];
threadgroup int bi[16];
threadgroup int pick[8];
const device float* lg = logits + (size_t) t * 1024u;
const device float* b = (has_vl != 0u && ((vl >> t) & 1u) != 0u) ? bias_vl : bias;
for (uint i = tid; i < n; i += 512u) {
const float l = lg[i];
const float sp = l > 20.0f ? l : log(1.0f + exp(l));
sc[i] = sqrt(sp);
bs[i] = sc[i] + b[i];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
const uint lane = tid & 31u, wp = tid >> 5u;
for (uint r = 0; r < 8u; ++r) {
if (r >= kk) break;
float best = -3.402823466e+38f;
int bix = 0x7fffffff;
for (uint i = tid; i < n; i += 512u) {
if (bs[i] > best || (bs[i] == best && int(i) < bix)) { best = bs[i]; bix = int(i); }
}
for (uint o = 16u; o > 0u; o >>= 1u) {
const float ov = simd_shuffle_xor(best, (ushort) o);
const int oi = simd_shuffle_xor(bix, (ushort) o);
if (ov > best || (ov == best && oi < bix)) { best = ov; bix = oi; }
}
if (lane == 0u && wp < 16u) { bv[wp] = best; bi[wp] = bix; }
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid == 0u) {
float bb = -3.402823466e+38f;
int ix = 0x7fffffff;
for (uint q = 0; q < 16u; ++q) {
if (bv[q] > bb || (bv[q] == bb && bi[q] < ix)) { bb = bv[q]; ix = bi[q]; }
}
pick[r] = ix;
bs[ix] = -3.402823466e+38f;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
if (tid == 0u) {
float sum = 0.0f;
for (uint r = 0; r < 8u; ++r) { if (r >= kk) break; sum += sc[pick[r]]; }
for (uint r = 0; r < 8u; ++r) {
if (r >= kk) break;
const uint p_ = uint(pick[r]);
const float wv = sc[p_] / (sum + 1e-20f) * route_scale;
idx[(size_t) t * 8u + r] = int(p_);
w[(size_t) t * 8u + r] = wv;
plan_ptr[(size_t) t * 8u + r] = vram[p_];
}
}
}
// ds4_kernels.cu swiglu_kernel - the expert MLP's activation: h = silu(g) * u, with the clamp when lim > 0
kernel void ds4_swiglu(device const float* g [[buffer(0)]], device const float* u [[buffer(1)]],
constant uint& n [[buffer(2)]], constant float& lim [[buffer(3)]],
device float* h [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
if (gid >= n) return;
float gv = g[gid], uv = u[gid];
if (lim > 0.0f) { // both inputs clamped, then the gate itself
gv = fmin(gv, lim);
uv = fmin(fmax(uv, -lim), lim);
}
h[gid] = (gv / (1.0f + exp(-gv))) * uv; // silu(g) * u
}
kernel void ds4_ring_to_ext(device const uchar* ring [[buffer(0)]], device ushort* ext [[buffer(1)]],
constant long& p0 [[buffer(2)]], constant uint& n [[buffer(3)]],
constant uint& d [[buffer(4)]], constant ulong& rb [[buffer(5)]],
uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const long p = p0 - long(n) + long(r);
const device uchar* row = ring + (size_t) (p >= 0 ? (p % long(n)) : 0) * rb;
for (uint e = tid; e < d; e += 256u) {
ext[(size_t) r * d + e] = p >= 0 ? ds4_f2bf16(ds4_e4m3(uint(row[e])) * ds4_ue8m0(uint(row[d + e / 32u]))) : ushort(0);
}
}
// ---------------------------------------------------------------- ds4_gguf.cu:577 mx_combine_kernel
//
// A token's expert outputs summed in plan order (an expert off the device adds 0), the CPU lane's part added when its
// flag is set, then the shared expert's - ds4_gguf.cu:169's combine(). Grid (n_embd / 128, T), 128 threads.
struct Ds4Comb {
device const ulong* plan_ptr; // [T][8]: nonzero when the device holds that expert
device const float* plan_w; // [T][8]
device const float* part; // [entries][n_embd]
device const float* cpu_part; // [T][cpu_stride], or null
device const int* cpu_flag; // [T], or null
device const float* sh_out; // [T][n_embd], or null
device float* out;
long cpu_stride, out_stride;
int k, n_embd;
};
kernel void ds4_mx_combine(constant Ds4Comb& a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
const uint r = tgp.x * 128u + tid;
if (int(r) >= a.n_embd) return;
const int t = int(tgp.y);
float m = 0.0f;
for (int i = 0; i < a.k; ++i) { // kPicks <= 8
const int e = t * 8 + i;
const float pi = a.plan_ptr[e] != 0ul ? a.part[(size_t) e * (size_t) a.n_embd + r] : 0.0f;
m = fma(a.plan_w[t * 8 + i], pi, m);
}
if (a.cpu_flag != nullptr && a.cpu_flag[t] != 0) m += a.cpu_part[(size_t) t * (size_t) a.cpu_stride + r];
m += a.sh_out != nullptr ? a.sh_out[(size_t) t * (size_t) a.n_embd + r] : 0.0f;
a.out[(size_t) t * (size_t) a.out_stride + r] = m;
}
// ---------------------------------------------------------------- ds4_kernels.cu:1286 engram_dequant_kernel / :1311 engram_gate_kernel
//
// The Engram tables' row decoder and gate (this model's tables are fmt 2: IQ4_NL, an 18-byte block of 32 values whose
// nibbles index ds4_kIq4nl above).
kernel void ds4_engram_dequant_rows(device const uchar* rows [[buffer(0)]], constant uint& hd [[buffer(1)]],
device float* out [[buffer(2)]], constant int& fmt [[buffer(3)]],
uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
if (fmt == 2) {
const device uchar* row = rows + (size_t) r * (size_t) hd;
for (uint e = tid; e < hd; e += 256u) {
const device uchar* b = row + (size_t) (e / 32u) * 18u;
const uint q = b[2u + (e & 15u)];
const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);
const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);
out[(size_t) r * (size_t) hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);
}
return;
}
// fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: NaN rather than a silent zero (as ds4_engram_dequant does)
for (uint e = tid; e < hd; e += 256u) out[(size_t) r * (size_t) hd + e] = NAN;
}
// ds4_kernels.cu's block_sum for a 1024-thread threadgroup: a warp sum a warp, then every warp sums the 32 partials (the
// same result in all of them, so no warp diverges at a barrier)
static inline float ds4_block_sum32(float v, threadgroup float* red, ushort lane, ushort w) {
v = ds4_warp_sum(v);
if (lane == 0) red[w] = v;
threadgroup_barrier(mem_flags::mem_threadgroup);
float t = lane < 32u ? red[lane] : 0.0f;
t = ds4_warp_sum(t);
threadgroup_barrier(mem_flags::mem_threadgroup);
return t;
}
kernel void ds4_engram_gate(device float* h [[buffer(0)]], device const float* kv [[buffer(1)]],
device const float* w [[buffer(2)]], constant uint& hc [[buffer(3)]],
constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[32];
const uint k = tgp.x, t = tgp.y;
device float* hk = h + ((size_t) t * hc + (size_t) k) * (size_t) dim;
const device float* kvt = kv + (size_t) t * (size_t) (hc + 1u) * (size_t) dim;
const device float* key = kvt + (size_t) k * (size_t) dim;
const device float* val = kvt + (size_t) hc * (size_t) dim;
const device float* wk = w + (size_t) k * (size_t) dim;
float sh = 0.0f, sk = 0.0f, sd = 0.0f;
for (uint e = tid; e < dim; e += 1024u) {
const float hv = hk[e], kve = key[e];
sh += hv * hv;
sk += kve * kve;
sd += hv * wk[e] * kve;
}
sh = ds4_block_sum32(sh, red, lane, wg);
sk = ds4_block_sum32(sk, red, lane, wg);
sd = ds4_block_sum32(sd, red, lane, wg);
const float rstd = rsqrt(sh / float(dim) + eps) * rsqrt(sk / float(dim) + eps);
const float dot = sd * rstd * rsqrt(float(dim));
const float sq = copysign(sqrt(fmax(fabs(dot), 1e-6f)), dot);
const float gate = 1.0f / (1.0f + exp(-sq));
for (uint e = tid; e < dim; e += 1024u) hk[e] = hk[e] + gate * val[e];
}
// ds4_gguf.cu:72 bf16_mv_kernel<NC, true> - a BF16 matrix's matvec against F32 activations (the engram WKV here, and the
// BF16 dense matrices the jobs' f == 0 names). Same frame as ds4_mx_mv above: a warp a row, a lane strides the row; here a
// lane 8 BF16 weights (16 bytes = one uint4) and 8 activations, so the row is walked in 256-value steps.
// A pair's F32 is bit-exact - the low half zeroed - which is bf16_mv_kernel's __uint_as_float(w << 16).
kernel void ds4_bf16_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
const uint row = tgp.x * 8u + (tid >> 5);
if (int(row) >= b.n) return;
const device ushort* w = (const device ushort*) b.w + (size_t) row * b.ld;
thread float acc[16]; // (the CUDA template's largest NC; ncol cuts the rest)
for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
for (uint i = uint(lane) * 8u; i < uint(b.k); i += 256u) {
const uint4 q = *(const device uint4*) (w + i);
const float wf[8] = {as_type<float>(q.x << 16), as_type<float>(q.x & 0xffff0000u),
as_type<float>(q.y << 16), as_type<float>(q.y & 0xffff0000u),
as_type<float>(q.z << 16), as_type<float>(q.z & 0xffff0000u),
as_type<float>(q.w << 16), as_type<float>(q.w & 0xffff0000u)};
for (uint c = 0; c < 16u; ++c) {
if (int(c) >= b.ncol) break;
const device float4* xp = (const device float4*) (b.x + c * b.xcs + (size_t) i);
const float4 x0 = xp[0], x1 = xp[1];
float a = acc[c];
a = fma(wf[0], x0.x, a);
a = fma(wf[1], x0.y, a);
a = fma(wf[2], x0.z, a);
a = fma(wf[3], x0.w, a);
a = fma(wf[4], x1.x, a);
a = fma(wf[5], x1.y, a);
a = fma(wf[6], x1.z, a);
a = fma(wf[7], x1.w, a);
acc[c] = a;
}
}
for (uint c = 0; c < 16u; ++c) {
if (int(c) >= b.ncol) break;
float v = acc[c];
v += simd_shuffle_xor(v, 16);
v += simd_shuffle_xor(v, 8);
v += simd_shuffle_xor(v, 4);
v += simd_shuffle_xor(v, 2);
v += simd_shuffle_xor(v, 1);
if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
}
// ---------------------------------------------------------------- ds4_kernels.cu:524 compress_kernel / :582 index_score_kernel
//
// The compressor (ratio > 1 layers): each token's kv / score row into its slot, and every ratio'th token pooled by a
// per-value softmax over the slots, scaled by a row RMS and the norm weights. Grid (1), 512 threads (d <= 1024 values).
static inline float ds4_block_sum_n(float v, threadgroup float* red, ushort lane, ushort w, uint nw) {
v = ds4_warp_sum(v);
if (lane == 0) red[w] = v;
threadgroup_barrier(mem_flags::mem_threadgroup);
float t = lane < nw ? red[lane] : 0.0f;
t = ds4_warp_sum(t); // (every warp sums the same partials: no divergence at a barrier)
threadgroup_barrier(mem_flags::mem_threadgroup);
return t;
}
kernel void ds4_compress(device const float* kv [[buffer(0)]], device const float* score [[buffer(1)]],
device float* kv_state [[buffer(2)]], device float* score_state [[buffer(3)]],
constant int& ratio [[buffer(4)]], constant int& p0 [[buffer(5)]], constant int& T [[buffer(6)]],
constant uint& d [[buffer(7)]], device const float* wn [[buffer(8)]],
constant float& eps [[buffer(9)]], device float* latent [[buffer(10)]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort wg [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[32];
const float kfmax = -3.4028234663852886e+38f; // -FLT_MAX, as the CUDA writes it
int g = 0;
for (int t = 0; t < T; ++t) {
const int slot = (p0 + t) % ratio;
const bool pool = (p0 + t + 1) % ratio == 0;
for (uint e = tid; e < d; e += 512u) {
kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];
score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (!pool) continue;
float val[2] = {0.0f, 0.0f}; // (d <= 2 * 512)
float ss = 0.0f;
uint n = 0;
for (uint e = tid; e < d; e += 512u, ++n) {
float m = kfmax;
for (int s = 0; s < ratio; ++s) m = fmax(m, score_state[(size_t) s * d + e]);
float den = 0.0f;
for (int s = 0; s < ratio; ++s) den += exp(score_state[(size_t) s * d + e] - m);
float acc = 0.0f;
for (int s = 0; s < ratio; ++s)
acc += kv_state[(size_t) s * d + e] * (exp(score_state[(size_t) s * d + e] - m) / den);
val[n] = acc;
ss += acc * acc;
}
const float inv = rsqrt(ds4_block_sum_n(ss, red, lane, wg, 16u) / float(d) + eps);
n = 0;
for (uint e = tid; e < d; e += 512u, ++n) latent[(size_t) g * d + e] = val[n] * inv * wn[e];
++g;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
// ---------------------------------------------------------------- ds4_kernels.cu:582 index_score_kernel<128, KF>
//
// A thread a position: its key row (kfmt 3: 64 bytes of E2M1 codes (value 2i in the low nibble) then 4 UE8M0 scales; kfmt
// 0: BF16) against every head's query (a threadgroup tile), relu, then the heads' weighted sum. Grid (ceil(n / 256), T).
struct Ds4Rows { // ds4_kernels.cuh:115 Rows (kWin = 16)
int n[16];
int k[16];
int gen, R, kcap, ccap, blk;
long p0;
};
static inline void ds4_rows_at(constant Ds4Rows& r, int t, thread int& n, thread int& k) {
if (r.gen == 0) {
n = r.n[t];
k = r.k[t];
return;
}
const int nn = int((r.p0 + t + 1) / r.R);
const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);
switch (r.gen) {
case 1: n = nn; k = min(r.kcap, nn); break;
case 2: n = nn; k = cb; break;
case 3: n = cb * r.blk; k = min(r.kcap, nn); break;
default: n = (nn + r.blk - 1) / r.blk; k = r.kcap; break;
}
}
kernel void ds4_index_score(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],
device const uchar* kc [[buffer(2)]], device const int* cand [[buffer(3)]],
device float* out [[buffer(4)]], constant Ds4Rows& lim [[buffer(5)]],
constant int& heads [[buffer(6)]], constant int& n [[buffer(7)]],
constant int& cand_stride [[buffer(8)]], constant int& blk [[buffer(9)]],
constant long& out_stride [[buffer(10)]], constant int& kfmt [[buffer(11)]],
constant int& has_cand [[buffer(12)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
constexpr int HD = 128;
threadgroup float qs[32 * HD + 32];
const int t = int(tgp.y);
for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qs[i] = q[(size_t) t * (size_t) heads * HD + i];
for (uint i = tid; i < uint(heads); i += 256u) qs[uint(heads) * HD + i] = wq[(size_t) t * 64 + i];
threadgroup_barrier(mem_flags::mem_threadgroup);
const int i = int(tgp.x) * 256 + int(tid);
if (i >= n) return;
device float* o = out + (long) t * out_stride;
int ln, lk;
ds4_rows_at(lim, t, ln, lk);
int p = i;
if (has_cand != 0) {
if (i >= lk * blk) {
o[i] = -INFINITY;
return;
}
p = cand[(size_t) t * (size_t) cand_stride + (size_t) (i / blk)] * blk + i % blk;
}
if (p >= ln) {
o[i] = -INFINITY;
return;
}
const size_t kb = kfmt == 3 ? (size_t) ((HD / 2 + HD / 32 + 15) / 16 * 16) : (size_t) HD * 2;
const device uchar* kr = kc + (size_t) p * kb;
float acc[32];
for (uint h = 0; h < 32u; ++h) acc[h] = 0.0f;
for (int c = 0; c < HD; c += 8) {
float kvv[8];
if (kfmt == 3) { // 8 codes in 4 bytes, the group's power-of-two scale (2^(b - 127))
const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |
(uint(kr[c / 2 + 3]) << 24);
const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);
for (uint x = 0; x < 8u; ++x) kvv[x] = ds4_e2m1_val14(cw >> (4 * x)) * 16384.0f * ks;
} else {
const device ushort* b = (const device ushort*) (kr + (size_t) c * 2);
for (uint x = 0; x < 8u; ++x) kvv[x] = as_type<float>(uint(b[x]) << 16);
}
for (uint h = 0; h < 32u; ++h) {
if (int(h) >= heads) break;
threadgroup const float* qh = qs + h * HD + c;
for (uint x = 0; x < 8u; ++x) acc[h] += qh[x] * kvv[x];
}
}
float s = 0.0f;
for (uint h = 0; h < 32u; ++h) {
if (int(h) >= heads) break;
s += fmax(acc[h], 0.0f) * qs[uint(heads) * HD + h];
}
o[i] = s;
}
// The sparse-attention indexer with the GPU's matrix units. The scalar ds4_index_score spends 16.3 s of a 32K prompt
// (30 % of it) doing 32 head-dots a (query, key) pair one FMA at a time; this stages 16 fp4 keys and one query's q as
// halves and runs the 32 x 128 x 16 product through simdgroup_matrix, which measured 3.17-6.10x faster in the standalone
// harness (tools/ds4/idx_harness/idx_mma) with every output matching the scalar path to half precision (<= 7e-4
// relative). Same arithmetic, same order per (query, key) up to that rounding.
//
// The fp4 weights are staged as e2m1 * ks (<= 12, half-safe) with the 16384 applied once in the epilogue - the full
// 16384 * ks reaches 2e5 and overflows half.
kernel void ds4_index_score_mma(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],
device const uchar* kc [[buffer(2)]], device float* out [[buffer(4)]],
constant Ds4Rows& lim [[buffer(5)]], constant int& heads [[buffer(6)]],
constant int& n [[buffer(7)]], constant int& T [[buffer(8)]],
constant long& out_stride [[buffer(9)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
uint sg [[simdgroup_index_in_threadgroup]]) {
constexpr int HD = 128;
constexpr int KB = 80; // the fp4 row: 64 code bytes + 4 scale bytes, padded to 80
constexpr int NK = 16; // keys a threadgroup scores (two key-blocks of 8)
threadgroup half qst[32 * HD]; // [head][dim]
threadgroup half kst[HD * NK]; // [dim][key], e2m1 * ks
threadgroup float part[4][NK]; // the four head-blocks' partial sums
threadgroup float accs[8][64]; // a simdgroup's 8x8 tile
const int k0 = int(tgp.x) * NK;
const int t = int(tgp.y);
if (t >= T) return;
for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qst[i] = half(q[(size_t) t * (size_t) heads * HD + i]);
for (int key = int(tid) / 8; key < NK; key += 32) {
const int idx = k0 + key;
if (idx >= n) continue;
const device uchar* kr = kc + (size_t) idx * (size_t) KB;
for (int c = 0; c < HD; c += 8) {
const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |
(uint(kr[c / 2 + 3]) << 24);
const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);
for (uint x = 0; x < 8u; ++x) kst[(c + int(x)) * NK + key] = half(ds4_e2m1_val14(cw >> (4 * x)) * ks);
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
const int hb = int(sg) / 2, kb = int(sg) % 2; // eight heads x eight keys a simdgroup
simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (int c = 0; c < HD; c += 8) {
simdgroup_half8x8 a, b;
simdgroup_load(a, qst, HD, ulong2(ulong(c), ulong(hb) * 8ul));
simdgroup_load(b, kst, NK, ulong2(ulong(kb) * 8ul, ulong(c)));
simdgroup_multiply_accumulate(acc, a, b, acc);
}
simdgroup_store(acc, accs[sg], 8);
simdgroup_barrier(mem_flags::mem_threadgroup);
const uint lane = tid % 32u;
if (lane < 8u) {
const int key = kb * 8 + int(lane);
float ssum = 0.0f;
for (uint h = 0; h < 8u; ++h) {
const int hh = hb * 8 + int(h);
if (hh >= heads) break;
ssum += fmax(accs[sg][h * 8 + lane] * 16384.0f, 0.0f) * wq[(size_t) t * 64 + hh];
}
part[hb][key] = ssum;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < uint(NK)) {
const int key = int(tid);
const int idx = k0 + key;
if (idx >= n) return;
int ln, lk;
ds4_rows_at(lim, t, ln, lk);
device float* o = out + (long) t * out_stride;
o[idx] = idx >= ln ? -INFINITY : (part[0][key] + part[1][key] + part[2][key] + part[3][key]);
}
}
// ---------------------------------------------------------------- ds4_kernels.cu:142 finish_kernel<true> (the KV pack)
//
// The same load, RMS and mrope as ds4_finish above, then the row written as the cache's packed codes (kv_row_bytes): the
// quantiser's scale byte a group (32 values, or 16 for the 4-bit MXFP4-style rows), the codes themselves from bit 0 up -
// for the 4-bit formats two values a byte, the second from the next lane. Written from the CUDA of the same name.
static inline uint ds4_e2m1_code(float q) { // (bit 3 the sign, -0 kept; the magnitudes as 0 .. 7)
const float a = fabs(q);
const uint m = a <= 1.5f ? uint(a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;
return m | ((as_type<uint>(q) >> 31) << 3);
}
kernel void ds4_finish_pack(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],
constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {
threadgroup float red[8];
device float* v = x + (long) b * stride;
const uint nj = (len + 255u) / 256u;
float r[32];
float ss = 0.0f;
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
r[j] = e < len ? v[e] : 0.0f;
ss += r[j] * r[j];
}
if (f.w != nullptr) {
const float tot = ds4_block_sum(ss, red, lane, w);
const float inv = rsqrt(tot / float(len) + f.eps);
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
if (e < len) r[j] = r[j] * inv * f.w[e];
}
}
if (f.rd > 0) {
const uint r0 = len - uint(f.rd);
const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
const float other = simd_shuffle_xor(r[j], 1);
if (e >= r0 && e < len) {
const uint i = (e - r0) >> 1;
const float ang = float(pos) * f.freqs[i];
const float c = cos(ang);
const float sn = f.inverse ? -sin(ang) : sin(ang);
r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;
}
}
}
// the packed row: a group's scale and the group's codes as the format's (quant 1: d E4M3 codes at pr[e] and d/32
// scale bytes at pr[len + e/32]; 2: E2M1 pairs at pr[e/2] with E4M3 scales at pr[len/2 + e/16]; 3: E2M1 pairs with
// UE8M0 scales at pr[len/2 + e/32])
const uint g = f.quant == 2 ? 16u : 32u;
device uchar* pr = f.pdst + (long) b * f.pdst_stride;
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
float a = e < len ? fabs(r[j]) : 0.0f;
for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));
float q = 0.0f, s = 1.0f;
if (e < len) {
if (f.quant == 1) {
s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));
q = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f));
} else if (f.quant == 2) {
s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);
q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));
} else {
s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));
}
r[j] = q * s;
}
if (f.quant == 1) {
if (e < len) pr[e] = uchar(ds4_e4m3_code(q, 0));
} else {
const uint c = ds4_e2m1_code(q);
const uint hi = simd_shuffle_down(c, 1); // (value e + 1: the next lane's)
if (e < len && (e & 1u) == 0u) pr[e >> 1] = uchar(c | (hi << 4));
}
if (e < len && e % g == 0u)
pr[(f.quant == 1 ? len : len / 2u) + e / g] =
uchar(f.quant == 2 ? ds4_e4m3_code(s, 0) : (as_type<uint>(s) >> 23));
}
device ushort* d = f.dst != nullptr ? f.dst + (long) b * f.dst_stride : nullptr;
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
if (e >= len) continue;
if (d != nullptr) {
const uint bits = as_type<uint>(r[j]);
d[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even
}
}
}
// ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations (the indexer's wproj here): a warp a row
// of the matrix, a lane 8 halves (16 bytes, the uint4 above) and 8 activations a step, kRowsT = 2 tokens a block. Grid
// (ceil(n / 8), ceil(T / 2)), threadgroup 256.
kernel void ds4_gemv_f16_rows(device const ushort* W [[buffer(0)]], device const float* x [[buffer(1)]],
constant ulong& ldx [[buffer(2)]], constant int& n [[buffer(3)]],
constant int& k [[buffer(4)]], constant int& T [[buffer(5)]],
constant float& alpha [[buffer(6)]], device float* y [[buffer(7)]],
constant ulong& ldy [[buffer(8)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
constexpr int kRowsT = 2;
const int row = int(tgp.x) * 8 + int(tid >> 5);
const int t0 = int(tgp.y) * kRowsT, nt = min(kRowsT, T - t0);
if (row >= n) return;
const device ushort* w = W + (size_t) row * (size_t) k;
float acc[kRowsT];
for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;
for (int c = int(lane) * 8; c < k; c += 32 * 8) {
const uint4 q = *(const device uint4*) (w + c);
const float2 f0 = float2(as_type<half2>(q.x)), f1 = float2(as_type<half2>(q.y));
const float2 f2 = float2(as_type<half2>(q.z)), f3 = float2(as_type<half2>(q.w));
const float2 fs[4] = {f0, f1, f2, f3};
float xv[kRowsT][8];
for (int t = 0; t < kRowsT; ++t)
if (t < nt) {
const device float4* xp = (const device float4*) (x + (size_t) (t0 + t) * ldx + (size_t) c);
const float4 a4 = xp[0], b4 = xp[1];
xv[t][0] = a4.x;
xv[t][1] = a4.y;
xv[t][2] = a4.z;
xv[t][3] = a4.w;
xv[t][4] = b4.x;
xv[t][5] = b4.y;
xv[t][6] = b4.z;
xv[t][7] = b4.w;
}
for (int i = 0; i < 4; ++i) {
const float2 f = fs[i];
for (int t = 0; t < kRowsT; ++t)
if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];
}
}
for (int t = 0; t < kRowsT; ++t)
if (t < nt) {
float a = acc[t];
a += simd_shuffle_xor(a, 16);
a += simd_shuffle_xor(a, 8);
a += simd_shuffle_xor(a, 4);
a += simd_shuffle_xor(a, 2);
a += simd_shuffle_xor(a, 1);
if (lane == 0u) y[(size_t) (t0 + t) * ldy + row] = alpha * a;
}
}
// ---------------------------------------------------------------- ds4_kernels.cu:650 topk_kernel
//
// A token's k largest scores: four 8-bit radix passes over the order keys of the row (a threadgroup of 1024, shared
// histograms), then the picks written in index order by a scan a pass, so the attention sums them in the same order every
// run and the ties kept are the lowest positions.
static inline uint ds4_ord_key(float f) {
const uint b = as_type<uint>(f);
return (b & 0x80000000u) ? ~b : (b | 0x80000000u);
}
kernel void ds4_topk(device const float* vals [[buffer(0)]], constant long& stride [[buffer(1)]],
constant Ds4Rows& r [[buffer(2)]], device const int* cand [[buffer(3)]],
constant int& cand_stride [[buffer(4)]], constant int& blk [[buffer(5)]],
device int* out [[buffer(6)]], constant int& out_stride [[buffer(7)]],
constant int& has_cand [[buffer(8)]], uint tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort wid [[simdgroup_index_in_threadgroup]]) {
threadgroup uint hist[256];
threadgroup uint prefix_s, mask_s;
threadgroup int remaining_s, cnt_gt, cnt_eq;
threadgroup int wgt[32], weq[32];
const int t = int(tgp);
int n, k;
ds4_rows_at(r, t, n, k);
device const float* v = vals + (long) t * stride;
device int* o = out + (size_t) t * (size_t) out_stride;
device const int* cd = has_cand != 0 ? cand + (size_t) t * (size_t) cand_stride : nullptr;
if (n <= 0) return;
if (k >= n) {
for (uint i = tid; i < uint(n); i += 1024u)
o[i] = cd != nullptr ? cd[i / uint(blk)] * blk + int(i % uint(blk)) : int(i);
return;
}
if (tid == 0u) {
prefix_s = 0u;
mask_s = 0u;
remaining_s = k;
cnt_gt = 0;
cnt_eq = 0;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int shift = 24; shift >= 0; shift -= 8) {
for (uint i = tid; i < 256u; i += 1024u) hist[i] = 0u;
threadgroup_barrier(mem_flags::mem_threadgroup);
const uint prefix = prefix_s, mask = mask_s;
for (int i = int(tid); i < n; i += 1024) {
const uint u = ds4_ord_key(v[i]);
if ((u & mask) == prefix) {
threadgroup atomic_uint* hp = (threadgroup atomic_uint*) &hist[(u >> shift) & 0xffu];
atomic_fetch_add_explicit(hp, 1u, memory_order_relaxed);
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid == 0u) {
const int rem = remaining_s;
int cum = 0;
for (int d = 255; d >= 0; --d) {
const int h = int(hist[d]);
if (cum + h >= rem) {
prefix_s = prefix | (uint(d) << shift);
mask_s = mask | (0xffu << shift);
remaining_s = rem - cum;
break;
}
cum += h;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const uint thr = prefix_s;
const int remaining = remaining_s;
const int n_gt = k - remaining;
for (int i0 = 0; i0 < n; i0 += 1024) {
const int i = i0 + int(tid);
const uint u = i < n ? ds4_ord_key(v[i]) : 0u;
const bool gt = i < n && u > thr, eq = i < n && u == thr;
const auto bgv = simd_ballot(gt);
const auto bev = simd_ballot(eq);
const uint bg = uint((simd_vote::vote_t)bgv), be = uint((simd_vote::vote_t)bev);
if (lane == 0u) {
wgt[wid] = int(popcount(bg));
weq[wid] = int(popcount(be));
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (wid == 0u) {
const int g0 = wgt[lane], e0 = weq[lane];
int gs = g0, es = e0;
for (uint o2 = 1u; o2 < 32u; o2 <<= 1) {
const int a = simd_shuffle_up(gs, ushort(o2)), b = simd_shuffle_up(es, ushort(o2));
if (uint(lane) >= o2) {
gs += a;
es += b;
}
}
wgt[lane] = cnt_gt + gs - g0;
weq[lane] = cnt_eq + es - e0;
simdgroup_barrier(mem_flags::mem_none);
if (lane == 31u) {
cnt_gt += gs;
cnt_eq += es;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
int slot = -1;
const uint below = (1u << lane) - 1u;
if (gt) slot = wgt[wid] + int(popcount(bg & below));
else if (eq) {
const int e = weq[wid] + int(popcount(be & below));
if (e < remaining) slot = n_gt + e;
}
if (slot >= 0) o[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
// ds4_kernels.cu:751 block_max_kernel - a block of `blk` positions' largest score (the indexer's candidate blocks), the
// newest position's block pinned in at +inf. Grid (ceil(nb_max / 256), T), 256 threads.
kernel void ds4_block_max(device const float* score [[buffer(0)]], constant long& stride [[buffer(1)]],
constant Ds4Rows& r [[buffer(2)]], constant int& blk [[buffer(3)]],
device float* bm [[buffer(4)]], constant long& bm_stride [[buffer(5)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const int b = int(tgp.x) * 256 + int(tid), t = int(tgp.y);
int n, k_;
ds4_rows_at(r, t, n, k_);
const int nb = (n + blk - 1) / blk;
if (b >= nb) return;
float m = -INFINITY;
for (int j = 0; j < blk; ++j) {
const int p = b * blk + j;
if (p < n) m = fmax(m, score[(long) t * stride + p]);
}
if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in
bm[(long) t * bm_stride + b] = m;
}
// ---------------------------------------------------------------- ds4_kernels.cu:1141 hc_mean_kernel / :1150 iota_rows_kernel
//
// The two small shapes the decode loop asks for: a token's mean over its hc streams, and a row's indices.
kernel void ds4_hc_mean(device const float* h [[buffer(0)]], constant int& hc [[buffer(1)]],
constant int& dim [[buffer(2)]], device float* out [[buffer(3)]],
constant long& out_stride [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
const int e = int(tgp.x) * 256 + int(tid), t = int(tgp.y);
if (e >= dim) return;
float a = 0.0f;
for (int s = 0; s < hc; ++s) a += h[((size_t) t * (size_t) hc + (size_t) s) * (size_t) dim + (size_t) e];
out[(long) t * out_stride + e] = a / float(hc);
}
kernel void ds4_iota_rows(device int* out [[buffer(0)]], constant int& n [[buffer(1)]], constant int& stride [[buffer(2)]],
uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint t = tg;
for (uint i = tid; i < uint(n); i += 64u) out[(size_t) t * (size_t) stride + i] = int(i);
}
// ---------------------------------------------------------------- vendor/sampler.cu:93 sampler_greedy_kernel
//
// The plain argmax over the vocabulary, ONE BLOCK (1024) a token: a thread's strided candidates with the repetition
// penalties applied (a threadgroup bitmap of the history window says which tokens can be penalised at all), a warp
// reduction, then warp 0's - a tie keeps the larger value and the SMALLER index, so this is the serial scan's answer.
kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constant int& n_vocab [[buffer(1)]],
device const int* history [[buffer(2)]], constant int& history_len [[buffer(3)]],
constant int& plen [[buffer(4)]], constant float& pen_repeat [[buffer(5)]],
constant float& pen_freq [[buffer(6)]], constant float& pen_present [[buffer(7)]],
device const float* logit_bias [[buffer(8)]], constant int& has_bias [[buffer(9)]],
device int* out [[buffer(10)]], uint tg [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort wid [[simdgroup_index_in_threadgroup]]) {
threadgroup uint penal_bits[4096]; // (n_vocab + 31) / 32 words (the host checks it fits)
threadgroup float sv[32];
threadgroup int si[32];
const int t = int(tg);
device const float* l = logits + (size_t) t * (size_t) n_vocab;
device const int* hrow = history != nullptr ? history + (size_t) t * (size_t) history_len : nullptr;
int hlen = 0;
if (hrow != nullptr) {
hlen = min(plen, history_len);
if (hlen < 0) hlen = 0;
hrow += history_len - hlen; // the window is the tail
}
const int bits_words = (n_vocab + 31) / 32;
const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
if (use_bits) {
for (int w = int(tid); w < bits_words; w += 1024) penal_bits[w] = 0u;
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int i = int(tid); i < hlen; i += 1024)
if (hrow[i] >= 0 && hrow[i] < n_vocab) {
threadgroup atomic_uint* pb = (threadgroup atomic_uint*) &penal_bits[hrow[i] >> 5];
atomic_fetch_or_explicit(pb, 1u << (hrow[i] & 31), memory_order_relaxed);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
float bv = -INFINITY;
int best = n_vocab; // (the "no candidate" index: it loses every comparison)
for (int v = int(tid); v < n_vocab; v += 1024) {
float s = has_bias != 0 ? l[v] + logit_bias[v] : l[v];
int cnt = 0;
if (use_bits && (penal_bits[v >> 5] & (1u << (v & 31))) != 0u)
for (int i = 0; i < hlen; ++i)
if (hrow[i] == v) ++cnt;
if (cnt > 0) {
if (s <= 0.0f) s *= pen_repeat;
else s /= pen_repeat;
s -= float(cnt) * pen_freq + 1.0f * pen_present;
}
if (s > bv) {
bv = s;
best = v;
}
}
for (uint o = 16u; o > 0u; o >>= 1) {
const float ov = simd_shuffle_down(bv, ushort(o));
const int oi = simd_shuffle_down(best, ushort(o));
if (ov > bv || (ov == bv && oi < best)) {
bv = ov;
best = oi;
}
}
if (lane == 0u) {
sv[wid] = bv;
si[wid] = best;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (wid == 0u) {
float wv = sv[lane];
int wi = si[lane];
for (uint o = 16u; o > 0u; o >>= 1) {
const float ov = simd_shuffle_down(wv, ushort(o));
const int oi = simd_shuffle_down(wi, ushort(o));
if (ov > wv || (ov == wv && oi < wi)) {
wv = ov;
wi = oi;
}
}
if (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)
}
}
// ---------------------------------------------------------------- sparse attention (the prompt path)
//
// ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false>. The packed kernel stages sixteen
// rows of a tile in threadgroup memory (the window rows as they are, the compressed rows decoded to BF16) once for all
// G heads of a threadgroup, then does each head's dots, softmax and weighted sum. The BF16 kernel is the same arithmetic
// for row sets already stored as BF16 (a DSpark block); one head and query a threadgroup.
struct Ds4AttnRows { // ds4_kernels.cuh:149 AttnRows (kWin = 16)
int win0[16], nwin[16], nsel[16];
int gen, t0, win, R, kcap;
long p0;
};
static inline void ds4_attn_counts(constant Ds4AttnRows& ar, int t, thread int& n_win, thread int& n_sel, thread int& w0) {
if (ar.gen != 0) {
const long ta = (long) ar.t0 + (long) t;
const long pa = ar.p0 + ta;
const long a = (long) ar.win - ar.p0, b = ta + 1;
const long first = a > b ? a : b; // max(win - p0, ta + 1)
w0 = int(first);
n_win = int((long) ar.win + ta - first + 1);
n_sel = ar.R > 0 ? int(((pa + 1) / (long) ar.R) < (long) ar.kcap ? (pa + 1) / (long) ar.R : (long) ar.kcap) : 0;
} else {
w0 = ar.win0[t];
n_win = ar.nwin[t];
n_sel = ar.nsel[t];
}
if (n_win < 0) n_win = 0;
if (n_sel < 0) n_sel = 0;
}
static inline float ds4_bf16_u32(uint v) { return as_type<float>(v << 16); }
// 32 values of a packed compressed row (kKvFp4, 512 values) as BF16 in four uint4s
static inline void ds4_attn_unpack32(const device uchar* rr, int part, threadgroup uint4* dst) {
const uint4 cw = *(const device uint4*) (rr + (size_t) part * 16u);
const float k0 = ds4_e4m3(uint(rr[256u + uint(part) * 2u])) * 16384.0f;
const float k1 = ds4_e4m3(uint(rr[256u + uint(part) * 2u + 1u])) * 16384.0f;
const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};
uint o[16];
for (uint k = 0; k < 16u; ++k) {
const uint b = wv[k >> 2] >> (8u * (k & 3u));
const float ks = k < 8u ? k0 : k1;
const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
}
dst[0] = uint4(o[0], o[1], o[2], o[3]);
dst[1] = uint4(o[4], o[5], o[6], o[7]);
dst[2] = uint4(o[8], o[9], o[10], o[11]);
dst[3] = uint4(o[12], o[13], o[14], o[15]);
}
// rows [s0, s0 + n) of the tile at t0 into tile4; a thread stages one row's 32 values
static inline void ds4_attn_stage(device const ushort* wbase, device const uchar* ckv, device const int* sl,
int n_win, int t0, int s0, int n, int sr, int part, threadgroup uint4* tile4) {
if (sr >= n) return;
const int j = s0 + sr;
threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) part * 4u;
const int jj = t0 + j;
if (jj < n_win) {
const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) part * 32u);
dst[0] = src[0];
dst[1] = src[1];
dst[2] = src[2];
dst[3] = src[3];
return;
}
const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;
ds4_attn_unpack32(rr, part, dst);
}
static inline float ds4_attn_warp_max(float v) {
v = fmax(v, simd_shuffle_xor(v, 16));
v = fmax(v, simd_shuffle_xor(v, 8));
v = fmax(v, simd_shuffle_xor(v, 4));
v = fmax(v, simd_shuffle_xor(v, 2));
v = fmax(v, simd_shuffle_xor(v, 1));
return v;
}
template <int G>
static inline void ds4_sparse_packed_run(device const float* q, device const ushort* win, device const uchar* ckv,
device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,
device const float* sink, uint heads, float scale, device float* out,
threadgroup uint4* tile4, threadgroup float* sc, threadgroup float* m_run_s,
threadgroup float* tscale_s, threadgroup float* den_s, uint2 tgp, uint tid,
ushort lane_in, ushort wp_in) {
constexpr int W = 8 / G; // warps a head
const int hb = int(tgp.x) * G, t = int(tgp.y);
int n_win, n_sel, w0;
ds4_attn_counts(ar, t, n_win, n_sel, w0);
const device ushort* wbase = win + (size_t) w0 * 512u;
const device int* sl = sel + (size_t) t * (size_t) sel_stride;
const uint lane = uint(lane_in), wp = uint(wp_in);
const uint gw = wp / uint(W), wr = wp % uint(W);
float qr[16];
for (int x = 0; x < 16; ++x)
qr[x] = q[((size_t) t * heads + uint(hb) + gw) * 512u + lane * 16u + uint(x)];
if (tid < uint(G)) {
m_run_s[tid] = -1e30f;
den_s[tid] = 0.0f;
}
float o0[G], o1[G];
for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;
const int c = int(tid) * 2;
const int sr = int(tid) >> 4, part = int(tid) & 15;
threadgroup_barrier(mem_flags::mem_threadgroup);
const int total = n_win + n_sel;
for (int t0 = 0; t0 < total; t0 += 64) {
const int nt = min(64, total - t0);
for (int s0 = 0; s0 < nt; s0 += 16) { // the dots
const int n = min(16, nt - s0);
ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int r = int(wr); r < n; r += W) {
const threadgroup uint4* rp = tile4 + (size_t) r * 64u + lane * 2u;
const uint4 a = rp[0], b = rp[1];
const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};
float s = 0.0f;
for (int x = 0; x < 4; ++x) {
s += qr[2 * x] * ds4_bf16_u32(av[x] & 0xffffu);
s += qr[2 * x + 1] * ds4_bf16_u32(av[x] >> 16);
}
for (int x = 0; x < 4; ++x) {
s += qr[8 + 2 * x] * ds4_bf16_u32(bv[x] & 0xffffu);
s += qr[8 + 2 * x + 1] * ds4_bf16_u32(bv[x] >> 16);
}
s = ds4_warp_sum(s);
if (lane == 0u) sc[uint(gw) * 64u + uint(s0 + r)] = s * scale;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
if (wp < uint(G)) { // head wp's softmax over the tile
threadgroup float* scg = sc + wp * 64u;
float m = -INFINITY;
for (int j = int(lane); j < nt; j += 32) m = fmax(m, scg[j]);
m = ds4_attn_warp_max(m);
const float m_old = m_run_s[wp];
const float m_new = fmax(m_old, m);
float s = 0.0f;
for (int j = int(lane); j < nt; j += 32) {
const float p = exp(scg[j] - m_new);
scg[j] = p;
s += p;
}
s = ds4_warp_sum(s);
if (lane == 0u) {
tscale_s[wp] = exp(m_old - m_new);
den_s[wp] = den_s[wp] * tscale_s[wp] + s;
m_run_s[wp] = m_new;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int g = 0; g < G; ++g) {
const float cs = tscale_s[g];
o0[g] *= cs;
o1[g] *= cs;
}
const threadgroup uint* tu = (const threadgroup uint*) tile4;
for (int s0 = 0; s0 < nt; s0 += 16) { // the weighted sums
const int n = min(16, nt - s0);
ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int r = 0; r < n; ++r) {
const uint pr = tu[(size_t) r * 256u + uint(tid)];
const float v0 = ds4_bf16_u32(pr & 0xffffu), v1 = ds4_bf16_u32(pr >> 16);
for (int g = 0; g < G; ++g) {
const float p = sc[uint(g) * 64u + uint(s0 + r)];
o0[g] = fma(p, v0, o0[g]);
o1[g] = fma(p, v1, o1[g]);
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
for (int g = 0; g < G; ++g) {
const float den = den_s[g] + exp(sink[hb + g] - m_run_s[g]);
device float* ov = out + ((size_t) t * heads + uint(hb + g)) * 512u;
ov[c] = o0[g] / den;
ov[c + 1] = o1[g] / den;
}
}
#define DS4_SPARSE_PACKED_SIG(NAME) \
kernel void NAME(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]], \
device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]], \
constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]], \
device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]], \
constant float& scale [[buffer(8)]], device float* out [[buffer(9)]], \
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], \
ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]])
#define DS4_SPARSE_PACKED_BODY(G) \
{ \
threadgroup uint4 tile4[16u * 64u]; \
threadgroup float sc[8u * 64u]; \
threadgroup float m_run_s[8u], tscale_s[8u], den_s[8u]; \
ds4_sparse_packed_run<G>(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, tile4, sc, m_run_s, \
tscale_s, den_s, tgp, tid, lane, wp); \
}
DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g8) DS4_SPARSE_PACKED_BODY(8)
DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g4) DS4_SPARSE_PACKED_BODY(4)
DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g2) DS4_SPARSE_PACKED_BODY(2)
DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g1) DS4_SPARSE_PACKED_BODY(1)
#undef DS4_SPARSE_PACKED_SIG
#undef DS4_SPARSE_PACKED_BODY
#ifndef DS4_MX_PLO
#define DS4_MX_PLO 0 // 1: the softmax weights as BF16 high + low (rms error vs the scalar kernel 2e-6 instead of 3e-4)
#endif
struct Ds4AttnRaw { uint4 v[4]; int kind; }; // a thread's 32 values of one row, as read: kind 0 window (v[0..3] BF16), 1 compressed (v[0] codes, v[1].x the two scale bytes), 2 none
static inline void ds4_attn_ld(device const ushort* wbase, device const uchar* ckv, device const int* sl, int n_win, int t0, int n,
int sr, int pt, thread Ds4AttnRaw& r) {
r.kind = 2;
if (sr >= n) return;
const int jj = t0 + sr;
if (jj < n_win) {
const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) pt * 32u);
r.v[0] = src[0]; r.v[1] = src[1]; r.v[2] = src[2]; r.v[3] = src[3];
r.kind = 0;
} else {
const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;
r.v[0] = *(const device uint4*) (rr + (size_t) pt * 16u);
r.v[1] = uint4(uint(rr[256u + uint(pt) * 2u]), uint(rr[256u + uint(pt) * 2u + 1u]), 0u, 0u);
r.kind = 1;
}
}
static inline void ds4_attn_st(thread const Ds4AttnRaw& r, int sr, int pt, threadgroup uint4* tile4) {
threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) pt * 4u;
if (r.kind == 0) {
dst[0] = r.v[0]; dst[1] = r.v[1]; dst[2] = r.v[2]; dst[3] = r.v[3];
} else if (r.kind == 1) {
const float k0 = ds4_e4m3(r.v[1].x) * 16384.0f;
const float k1 = ds4_e4m3(r.v[1].y) * 16384.0f;
const uint wv[4] = {r.v[0].x, r.v[0].y, r.v[0].z, r.v[0].w};
uint o[16];
for (uint k = 0; k < 16u; ++k) {
const uint b = wv[k >> 2] >> (8u * (k & 3u));
const float ks = k < 8u ? k0 : k1;
const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
}
dst[0] = uint4(o[0], o[1], o[2], o[3]);
dst[1] = uint4(o[4], o[5], o[6], o[7]);
dst[2] = uint4(o[8], o[9], o[10], o[11]);
dst[3] = uint4(o[12], o[13], o[14], o[15]);
} else {
dst[0] = uint4(0); dst[1] = uint4(0); dst[2] = uint4(0); dst[3] = uint4(0);
}
}
// ---------------------------------------------------------------- sparse attention on the matrix units (G = 8 heads a threadgroup)
//
// The same attention as ds4_sparse_attn_g8 (a token, eight heads, the window rows plus the selected compressed rows, softmax with the head's
// sink) but the dots and the weighted sum run on simdgroup 8x8 matrices: eight rows a pass are staged once as BF16 (exact), q and the
// softmax weights are split into a BF16 high and low part (so the products keep about 16 bits), accumulation is fp32, and the softmax is the
// online one (the running maximum rescales the accumulators through a diagonal matrix).
#ifndef DS4_MX_QK_LO
#define DS4_MX_QK_LO 0 // 0: the QK dot uses the BF16 high part of q only (faster, ~8 fewer bits)
#endif
// ATTN_PROBE (function constant 25): a timing probe. 1 skips the P.V accumulate, 2 skips the QK matmul - the numbers
// it produces are wrong on purpose; it exists to say which half of the pass the 20 s of a 32K prompt is in.
constant int ATTN_PROBE [[function_constant(25)]];
constant int attn_probe = is_function_constant_defined(ATTN_PROBE) ? ATTN_PROBE : 0;
kernel void ds4_sparse_attn_mx(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {
threadgroup uint4 tile4[8u * 64u];
threadgroup float qbuf[4096]; // q hi / lo as BF16 (8 KB each), later the output rows as float
threadgroup float part[8u * 64u];
threadgroup float ssm[64];
threadgroup bfloat phi[64], plo[64];
threadgroup float dg[64];
threadgroup float m_s[8], den_s[8], sc_s[8];
threadgroup bfloat* qhi = (threadgroup bfloat*) qbuf;
threadgroup bfloat* qlo = qhi + 4096;
threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;
const int hb = int(tgp.x) * 8, t = int(tgp.y);
int n_win, n_sel, w0;
ds4_attn_counts(ar, t, n_win, n_sel, w0);
const device ushort* wbase = win + (size_t) w0 * 512u;
const device int* sl = sel + (size_t) t * (size_t) sel_stride;
for (uint x = 0; x < 16u; ++x) {
const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
const float v = q[((size_t) t * heads + uint(hb) + h) * 512u + d];
const bfloat hi = bfloat(v);
qhi[idx] = hi;
if (DS4_MX_QK_LO) qlo[idx] = bfloat(v - float(hi)); // (unused when the QK dot takes the high part only)
}
if (tid < 8u) {
m_s[tid] = -1e30f;
den_s[tid] = 0.0f;
}
simdgroup_float8x8 O[8];
for (int c = 0; c < 8; ++c) O[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
const int sr = int(tid) >> 4, pt = int(tid) & 15;
const int total = n_win + n_sel;
threadgroup_barrier(mem_flags::mem_threadgroup);
Ds4AttnRaw raw;
raw.kind = 2;
if (sr < 8) ds4_attn_ld(wbase, ckv, sl, n_win, 0, min(8, total), sr, pt, raw);
for (int t0 = 0; t0 < total; t0 += 8) {
const int n = min(8, total - t0);
if (sr < 8 && attn_probe != 5) ds4_attn_st(raw, sr, pt, tile4);
threadgroup_barrier(mem_flags::mem_threadgroup);
if (sr < 8) ds4_attn_ld(wbase, ckv, sl, n_win, t0 + 8, min(8, total - t0 - 8), sr, pt, raw); // the next pass's rows, in flight during this one
if (attn_probe != 2 && attn_probe != 6) { // S = q . K^T over this simdgroup's eighth of the 512 values
simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {
simdgroup_bfloat8x8 a_hi, a_lo, b;
simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);
simdgroup_load(a_hi, qhi, 512, ulong2(ulong(kb) * 8ul, 0ul));
simdgroup_load(a_lo, qlo, 512, ulong2(ulong(kb) * 8ul, 0ul));
simdgroup_multiply_accumulate(acc, a_hi, b, acc);
if (DS4_MX_QK_LO) simdgroup_multiply_accumulate(acc, a_lo, b, acc); // (the low part of q: exactness vs speed)
}
simdgroup_store(acc, part + uint(sg) * 64u, 8);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < 64u && attn_probe != 4 && attn_probe != 6) {
float s = 0.0f;
for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];
ssm[tid] = s * scale;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (attn_probe != 3) { // head sg's online softmax over the pass's rows
const bool valid = int(lane) < n;
const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;
const float m = ds4_attn_warp_max(s);
const float m_old = m_s[sg];
const float m_new = fmax(m_old, m);
const float p = valid ? exp(s - m_new) : 0.0f;
const float ps = ds4_warp_sum(p);
if (lane < 8u) {
const bfloat hi = bfloat(p);
phi[uint(sg) * 8u + uint(lane)] = hi;
plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));
}
if (lane == 0u) {
const float cs = exp(m_old - m_new);
sc_s[sg] = cs;
den_s[sg] = den_s[sg] * cs + ps;
m_s[sg] = m_new;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[tid >> 3] : 0.0f;
threadgroup_barrier(mem_flags::mem_threadgroup);
if (attn_probe != 1 && attn_probe != 6) { // O = diag(rescale) O + P V on this simdgroup's 64 columns
simdgroup_float8x8 D;
simdgroup_bfloat8x8 ph, pl;
simdgroup_load(D, dg, 8);
simdgroup_load(ph, phi, 8);
simdgroup_load(pl, plo, 8);
for (int c = 0; c < 8; ++c) {
simdgroup_float8x8 tmp;
simdgroup_multiply(tmp, D, O[c]);
simdgroup_bfloat8x8 v;
simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
simdgroup_multiply_accumulate(tmp, ph, v, tmp);
#if DS4_MX_PLO
simdgroup_multiply_accumulate(tmp, pl, v, tmp);
#endif
O[c] = tmp;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
for (int c = 0; c < 8; ++c) simdgroup_store(O[c], qbuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint x = 0; x < 16u; ++x) {
const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
const float den = den_s[h] + exp(sink[uint(hb) + h] - m_s[h]);
out[((size_t) t * heads + uint(hb) + h) * 512u + d] = qbuf[idx] / den;
}
}
// TWO QUERIES A THREADGROUP, written out in full. The staged keys are shared; each query keeps its own accumulators,
// its own online-softmax state and its own key count (the causal shape makes the second query's set a superset, so the
// union staged is the second query's own set and the first masks its tail).
kernel void ds4_sparse_attn_mx2(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {
threadgroup uint4 tile4[8u * 64u];
threadgroup float qbuf[4096]; // two queries' q as BF16 (8 KB each), later a query's output rows
threadgroup float part[8u * 64u];
threadgroup float ssm[64];
threadgroup bfloat phi[64], plo[64];
threadgroup float dg[64];
threadgroup float m_s[16], den_s[16], sc_s[16];
threadgroup bfloat* qhiA = (threadgroup bfloat*) qbuf;
threadgroup bfloat* qhiB = qhiA + 4096; // a query's q is 4096 bf16
threadgroup float* obuf = qbuf; // 8 heads x 512 floats: qbuf again, once the key loop is done
threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;
const int hb = int(tgp.x) * 8;
const int tA = int(tgp.y) * 2;
const int rows_total = (ar.gen == 0 && ar.t0 > 0) ? ar.t0 : 0x7fffffff;
const int nq = (tA + 1 < rows_total) ? 2 : 1;
int nwA, nsA, w0A, nwB, nsB, w0B;
ds4_attn_counts(ar, tA, nwA, nsA, w0A);
nwB = nwA; nsB = nsA; w0B = w0A;
if (nq > 1) ds4_attn_counts(ar, tA + 1, nwB, nsB, w0B);
const int totalA = nwA + nsA;
const int totalB = (nq > 1) ? nwB + nsB : totalA;
const int total = totalA > totalB ? totalA : totalB;
const device ushort* wbase = win + (size_t) w0A * 512u;
const device int* sl = sel + (size_t) tA * (size_t) sel_stride;
for (uint x = 0; x < 16u; ++x) {
const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
qhiA[idx] = bfloat(q[((size_t) tA * heads + uint(hb) + h) * 512u + d]);
if (nq > 1) qhiB[idx] = bfloat(q[((size_t) (tA + 1) * heads + uint(hb) + h) * 512u + d]);
}
if (tid < 16u) {
m_s[tid] = -1e30f;
den_s[tid] = 0.0f;
}
simdgroup_float8x8 OA[8], OB[8];
for (int c = 0; c < 8; ++c) {
OA[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
OB[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
const int sr = int(tid) >> 4, pt = int(tid) & 15;
threadgroup_barrier(mem_flags::mem_threadgroup);
Ds4AttnRaw raw;
raw.kind = 2;
if (sr < 8) ds4_attn_ld(wbase, ckv, sl, nwB, 0, min(8, total), sr, pt, raw);
for (int t0 = 0; t0 < total; t0 += 8) {
const int np = min(8, total - t0);
(void) np;
if (sr < 8) ds4_attn_st(raw, sr, pt, tile4);
threadgroup_barrier(mem_flags::mem_threadgroup);
if (sr < 8) ds4_attn_ld(wbase, ckv, sl, nwB, t0 + 8, min(8, total - t0 - 8), sr, pt, raw);
{ // query A: S = q . K^T
simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {
simdgroup_bfloat8x8 a_hi, b;
simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);
simdgroup_load(a_hi, qhiA, 512, ulong2(ulong(kb) * 8ul, 0ul));
simdgroup_multiply_accumulate(acc, a_hi, b, acc);
}
simdgroup_store(acc, part + uint(sg) * 64u, 8);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < 64u) {
float s = 0.0f;
for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];
ssm[tid] = s * scale;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
{ // query A: online softmax
const int nq_here = min(8, totalA - t0);
const bool valid = int(lane) < nq_here;
const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;
const float m = ds4_attn_warp_max(s);
const float m_old = m_s[0 + sg];
const float m_new = fmax(m_old, m);
const float p = valid ? exp(s - m_new) : 0.0f;
const float ps = ds4_warp_sum(p);
if (lane < 8u) {
const bfloat hi = bfloat(p);
phi[uint(sg) * 8u + uint(lane)] = hi;
plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));
}
if (lane == 0u) {
const float cs = exp(m_old - m_new);
sc_s[0 + sg] = cs;
den_s[0 + sg] = den_s[0 + sg] * cs + ps;
m_s[0 + sg] = m_new;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[0 + (tid >> 3)] : 0.0f;
threadgroup_barrier(mem_flags::mem_threadgroup);
{ // query A: O = diag(rescale) O + P V
simdgroup_float8x8 D;
simdgroup_bfloat8x8 ph, pl;
simdgroup_load(D, dg, 8);
simdgroup_load(ph, phi, 8);
simdgroup_load(pl, plo, 8);
for (int c = 0; c < 8; ++c) {
simdgroup_float8x8 tmp;
simdgroup_multiply(tmp, D, OA[c]);
simdgroup_bfloat8x8 v;
simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
simdgroup_multiply_accumulate(tmp, ph, v, tmp);
#if DS4_MX_PLO
simdgroup_multiply_accumulate(tmp, pl, v, tmp);
#endif
OA[c] = tmp;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (nq > 1) {
{ // query B: S = q . K^T
simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {
simdgroup_bfloat8x8 a_hi, b;
simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);
simdgroup_load(a_hi, qhiB, 512, ulong2(ulong(kb) * 8ul, 0ul));
simdgroup_multiply_accumulate(acc, a_hi, b, acc);
}
simdgroup_store(acc, part + uint(sg) * 64u, 8);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < 64u) {
float s = 0.0f;
for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];
ssm[tid] = s * scale;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
{ // query B: online softmax
const int nq_here = min(8, totalB - t0);
const bool valid = int(lane) < nq_here;
const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;
const float m = ds4_attn_warp_max(s);
const float m_old = m_s[8 + sg];
const float m_new = fmax(m_old, m);
const float p = valid ? exp(s - m_new) : 0.0f;
const float ps = ds4_warp_sum(p);
if (lane < 8u) {
const bfloat hi = bfloat(p);
phi[uint(sg) * 8u + uint(lane)] = hi;
plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));
}
if (lane == 0u) {
const float cs = exp(m_old - m_new);
sc_s[8 + sg] = cs;
den_s[8 + sg] = den_s[8 + sg] * cs + ps;
m_s[8 + sg] = m_new;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[8 + (tid >> 3)] : 0.0f;
threadgroup_barrier(mem_flags::mem_threadgroup);
{ // query B: O = diag(rescale) O + P V
simdgroup_float8x8 D;
simdgroup_bfloat8x8 ph, pl;
simdgroup_load(D, dg, 8);
simdgroup_load(ph, phi, 8);
simdgroup_load(pl, plo, 8);
for (int c = 0; c < 8; ++c) {
simdgroup_float8x8 tmp;
simdgroup_multiply(tmp, D, OB[c]);
simdgroup_bfloat8x8 v;
simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
simdgroup_multiply_accumulate(tmp, ph, v, tmp);
#if DS4_MX_PLO
simdgroup_multiply_accumulate(tmp, pl, v, tmp);
#endif
OB[c] = tmp;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
for (int c = 0; c < 8; ++c) simdgroup_store(OA[c], obuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint x = 0; x < 16u; ++x) {
const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
out[((size_t) tA * heads + uint(hb) + h) * 512u + d] = obuf[idx] / (den_s[h] + exp(sink[uint(hb) + h] - m_s[h]));
}
if (nq > 1) {
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int c = 0; c < 8; ++c) simdgroup_store(OB[c], obuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint x = 0; x < 16u; ++x) {
const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
out[((size_t) (tA + 1) * heads + uint(hb) + h) * 512u + d] =
obuf[idx] / (den_s[8u + h] + exp(sink[uint(hb) + h] - m_s[8u + h]));
}
}
}
// kContig == false (the row set is BF16 already): one head and query a threadgroup, a warp a row's dot and warp 0 the
// softmax, every thread two output values.
static inline void ds4_sparse_bf16_run(device const float* q, device const ushort* win, device const ushort* ckv,
device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,
device const float* sink, uint heads, float scale, device float* out,
threadgroup float* qs, threadgroup float* sc, threadgroup int* rowix,
threadgroup float* m_run_s, threadgroup float* tscale_s,
threadgroup float* den_s, uint2 tgp, uint tid, ushort lane_in,
ushort wp_in) {
const int h = int(tgp.x), t = int(tgp.y);
int n_win, n_sel, w0;
ds4_attn_counts(ar, t, n_win, n_sel, w0);
const device ushort* wbase = win + (size_t) w0 * 512u;
const device int* sl = sel + (size_t) t * (size_t) sel_stride;
const uint lane = uint(lane_in), wp = uint(wp_in);
for (uint i = tid; i < 512u; i += 256u) qs[i] = q[((size_t) t * heads + uint(h)) * 512u + i];
if (tid == 0u) {
m_run_s[0] = -1e30f;
den_s[0] = 0.0f;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
float o0 = 0.0f, o1 = 0.0f;
const int c = int(tid) * 2;
const int total = n_win + n_sel;
for (int t0 = 0; t0 < total; t0 += 64) {
const int nt = min(64, total - t0);
if (tid < uint(nt)) {
const int j = t0 + int(tid);
rowix[tid] = j < n_win ? -1 - j : sl[j - n_win];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
for (int jj = int(wp); jj < nt; jj += 8) {
const int code = rowix[jj];
const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u
: ckv + (size_t) code * 512u;
const device uint4* rp = (const device uint4*) (row + lane * 16u);
const uint4 a = rp[0], b = rp[1];
const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};
float s = 0.0f;
for (int x = 0; x < 4; ++x) {
s += qs[lane * 16u + uint(2 * x)] * ds4_bf16_u32(av[x] & 0xffffu);
s += qs[lane * 16u + uint(2 * x + 1)] * ds4_bf16_u32(av[x] >> 16);
}
for (int x = 0; x < 4; ++x) {
s += qs[lane * 16u + uint(8 + 2 * x)] * ds4_bf16_u32(bv[x] & 0xffffu);
s += qs[lane * 16u + uint(9 + 2 * x)] * ds4_bf16_u32(bv[x] >> 16);
}
s = ds4_warp_sum(s);
if (lane == 0u) sc[jj] = s * scale;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (wp == 0u) {
float m = -INFINITY;
for (int j = int(lane); j < nt; j += 32) m = fmax(m, sc[j]);
m = ds4_attn_warp_max(m);
const float m_old = m_run_s[0];
const float m_new = fmax(m_old, m);
float s = 0.0f;
for (int j = int(lane); j < nt; j += 32) {
const float p = exp(sc[j] - m_new);
sc[j] = p;
s += p;
}
s = ds4_warp_sum(s);
if (lane == 0u) {
tscale_s[0] = exp(m_old - m_new);
den_s[0] = den_s[0] * tscale_s[0] + s;
m_run_s[0] = m_new;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
const float cs = tscale_s[0];
o0 *= cs;
o1 *= cs;
for (int j = 0; j < nt; ++j) {
const int code = rowix[j];
const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u
: ckv + (size_t) code * 512u;
const uint pr = *(const device uint*) (row + c);
o0 = fma(sc[j], ds4_bf16_u32(pr & 0xffffu), o0);
o1 = fma(sc[j], ds4_bf16_u32(pr >> 16), o1);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
const float den = den_s[0] + exp(sink[h] - m_run_s[0]);
device float* ov = out + ((size_t) t * heads + uint(h)) * 512u;
ov[c] = o0 / den;
ov[c + 1] = o1 / den;
}
kernel void ds4_sparse_attn_bf16(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
device const ushort* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {
threadgroup float qs[512];
threadgroup float sc[64];
threadgroup int rowix[64];
threadgroup float m_run_s, tscale_s, den_s;
ds4_sparse_bf16_run(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, qs, sc, rowix, &m_run_s,
&tscale_s, &den_s, tgp, tid, lane, wp);
}
// ds4_kernels.cu:461 route_kernel - softplus scores, their top k with bias, weights from the unbiased scores. One
// threadgroup (512) a token, as the CUDA kernel; rows 1024 values apart, the picks kPicks = 8 apart.
kernel void ds4_route(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],
constant uint& n [[buffer(2)]], constant uint& k [[buffer(3)]], constant float& route_scale [[buffer(4)]],
device int* idx [[buffer(5)]], device float* w [[buffer(6)]], device const float* bias_vl [[buffer(7)]],
constant uint& has_vl [[buffer(8)]], device const uint* vlw [[buffer(9)]],
uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {
threadgroup float sc[1024], bs[1024];
threadgroup float bv[32];
threadgroup int bi[32];
threadgroup int pick[16];
const uint t = tg;
device const float* l = logits + (size_t) t * 1024u;
device const float* bp = bias;
if (has_vl != 0u && ((vlw[t >> 5u] >> (t & 31u)) & 1u) != 0u) bp = bias_vl;
for (uint i = tid; i < n; i += 512u) {
const float v = l[i];
const float sp = v > 20.0f ? v : log(1.0f + exp(v));
sc[i] = sqrt(sp);
bs[i] = sc[i] + bp[i];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint r = 0; r < k; ++r) {
float best = -INFINITY;
int bix = 0x7fffffff;
for (uint i = tid; i < n; i += 512u)
if (bs[i] > best || (bs[i] == best && int(i) < bix)) {
best = bs[i];
bix = int(i);
}
for (uint o = 16u; o > 0u; o >>= 1u) {
const float ov = simd_shuffle_xor(best, o);
const int oi = simd_shuffle_xor(bix, o);
if (ov > best || (ov == best && oi < bix)) {
best = ov;
bix = oi;
}
}
if (lane == 0u) {
bv[wp] = best;
bi[wp] = bix;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (tid == 0u) {
float b = -INFINITY;
int ix = 0x7fffffff;
for (uint q = 0; q < 16u; ++q)
if (bv[q] > b || (bv[q] == b && bi[q] < ix)) {
b = bv[q];
ix = bi[q];
}
pick[r] = ix;
bs[ix] = -INFINITY;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
if (tid == 0u) {
float s = 0.0f;
for (uint r = 0; r < k; ++r) s += sc[pick[r]];
for (uint r = 0; r < k; ++r) {
idx[(size_t) t * 8u + r] = pick[r];
w[(size_t) t * 8u + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;
}
}
}
// ds4_kernels.cu:1336 gather_kernel and :1342 combine_kernel - a token's gathered expert-output rows and the MoE's sum.
struct Ds4Gather {
int src[16 * 8];
};
struct Ds4Combine {
int n[16];
int row[16][8];
float w[16][8];
int cpu[16];
};
kernel void ds4_gather(device const float* x [[buffer(0)]], constant Ds4Gather& g [[buffer(1)]],
constant uint& cols [[buffer(2)]], device float* out [[buffer(3)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint r = tgp.y;
const uint c = tgp.x * 256u + tid;
if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];
}
kernel void ds4_combine(device const float* ed [[buffer(0)]], device const float* cpu [[buffer(1)]],
device const float* sh [[buffer(2)]], constant Ds4Combine& cm [[buffer(3)]],
constant uint& dim [[buffer(4)]], constant uint& has_cpu [[buffer(5)]],
constant uint& has_sh [[buffer(6)]], device float* out [[buffer(7)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint e = tgp.x * 256u + tid, t = tgp.y;
if (e >= dim) return;
float a = 0.0f;
for (int j = 0; j < cm.n[t]; ++j) a = fma(cm.w[t][j], ed[(size_t) cm.row[t][j] * dim + e], a);
if (has_cpu != 0u && cm.cpu[t] != 0) a += cpu[(size_t) t * dim + e];
if (has_sh != 0u) a += sh[(size_t) t * dim + e];
out[(size_t) t * dim + e] = a;
}
// ---------------------------------------------------------------- ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel
//
// The drafter's head (DSpark's Markov chain and its confidence): markov_add is a warp a vocabulary row - the previous
// token's E row (BF16) staged as floats in the threadgroup, then E . H[v] added to logits[v] (H is F16, the pack's
// markov_head.head.weight); confidence is a threadgroup a token over x and the same E row against the head's weights,
// its block sum through the logistic.
struct Ds4MarkovJob {
device float* logits; // [V]: += the bias
device const ushort* E; // [vocab][R] BF16
device const ushort* H; // [vocab][R] F16
device const int* tok; // the previous token (one int)
int V, R;
};
struct Ds4ConfJob {
device const float* x; // [T][dim]
device const ushort* E; // [vocab][R] BF16
device const int* tok; // [T]
device const float* w; // [dim + R]
device float* out; // [T]
int dim, R;
};
// a BF16 bit pattern as its float (ds4_kernels.cu:28 bf16f: the top half, no rounding)
static inline float ds4_bf16f(ushort b) { return as_type<float>(uint(b) << 16u); }
// a warp a vocabulary row: logits[v] += E[*tok] . H[v]
kernel void ds4_markov_add(constant Ds4MarkovJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
threadgroup float er[1024]; // (R <= 1024: the host wrapper refuses more)
const device ushort* erow = j.E + (size_t) (*j.tok) * (size_t) j.R;
for (int i = int(tid); i < j.R; i += 256) er[i] = ds4_bf16f(erow[i]);
threadgroup_barrier(mem_flags::mem_threadgroup);
const int v = int(tg) * 8 + int(tid >> 5);
if (v >= j.V) return;
const device ushort* hr = j.H + (size_t) v * (size_t) j.R;
float a = 0.0f;
for (int c = int(lane) * 8; c < j.R; c += 256) {
for (int i = 0; i < 8; ++i) a += float(as_type<half>(hr[c + i])) * er[c + i];
}
a = ds4_warp_sum(a);
if (lane == 0u) j.logits[v] += a;
}
// a threadgroup a token: the logistic of the head's sum over x and the token's E row
kernel void ds4_confidence(constant Ds4ConfJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
ushort w [[simdgroup_index_in_threadgroup]]) {
threadgroup float red[32];
const int t = int(tg);
const device float* xt = j.x + (size_t) t * (size_t) j.dim;
const device ushort* er = j.E + (size_t) j.tok[t] * (size_t) j.R;
float a = 0.0f;
for (int i = int(tid); i < j.dim + j.R; i += 1024) a += j.w[i] * (i < j.dim ? xt[i] : ds4_bf16f(er[i - j.dim]));
a = ds4_block_sum_n(a, red, lane, w, 32u);
if (tid == 0u) j.out[t] = 1.0f / (1.0f + exp(-a));
}
// embed by raw device addresses (the runtime's MTLResidencySet makes gpuAddress pointers valid kernel arguments)
struct Ds4EmbedArgs {
device ushort* rows;
device float* h;
uint dim;
uint hc;
};
kernel void ds4_embed_j(device const Ds4EmbedArgs* a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
ushort lane [[thread_index_in_threadgroup]]) {
const uint e = tgp.x * 256u + uint(lane);
if (e >= a->dim) return;
const uint t = tgp.y;
const uint bits = uint(a->rows[(size_t) t * a->dim + e]) << 16;
const float v = as_type<float>(bits);
for (uint s = 0; s < a->hc; ++s) a->h[((size_t) t * a->hc + s) * a->dim + e] = v;
}
// (diagnostic) write a constant through a device pointer passed as a value, and one passed in a buffer
kernel void ds4_test_write(device float* out [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {
if (gid < n) out[gid] = 3.0f;
}
kernel void ds4_test_write_v(device const uint64_t* p [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {
device float* out = (device float*) (uintptr_t) p[0];
if (gid < n) out[gid] = 7.0f;
}
// (diagnostic) report the arguments the kernel received
kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [[buffer(1)]],
constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {
if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }
}
// hc == 4 (every layer of this model): the warp-lane form of the CUDA kernel - lane j*4+k holds comb[j][k], the row and column
// sums are xor-shuffles (1, 2 within a row; 4, 8 within a column). One simdgroup a token instead of one lane.
kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],
device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],
device float* post [[buffer(4)]], device float* comb [[buffer(5)]],
constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],
constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],
uint lane [[thread_index_in_threadgroup]]) {
const device float* mx = mixes + (size_t) b * 32u;
if (lane < 4u) {
pre[(size_t) b * 8u + lane] = 1.0f / (1.0f + exp(-(mx[lane] * scale[0] + base[lane]))) + eps;
post[(size_t) b * 8u + lane] = 2.0f / (1.0f + exp(-(mx[4u + lane] * scale[1] + base[4u + lane])));
}
const bool on = lane < 16u;
float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;
float m = fmax(c, simd_shuffle_xor(c, ushort(1)));
m = fmax(m, simd_shuffle_xor(m, ushort(2)));
c = on ? exp(c - m) : 0.0f;
float rs = c + simd_shuffle_xor(c, ushort(1));
rs += simd_shuffle_xor(rs, ushort(2));
c = c / rs + eps;
c = on ? c : 0.0f;
float cs = c + simd_shuffle_xor(c, ushort(4));
cs += simd_shuffle_xor(cs, ushort(8));
c = c / (cs + eps);
for (uint it = 1; it < iters; ++it) {
float r = c + simd_shuffle_xor(c, ushort(1));
r += simd_shuffle_xor(r, ushort(2));
c = c / (r + eps);
float s = c + simd_shuffle_xor(c, ushort(4));
s += simd_shuffle_xor(s, ushort(8));
c = c / (s + eps);
}
if (on) comb[(size_t) b * 64u + lane] = c;
}
// Plain byte copies and fills on the context's queue: cudaMemcpyAsync / cudaMemsetAsync between runtime allocations (unified
// memory: a "device" and a "host" buffer are both just memory) ride in the same command buffer as the kernels around them,
// so their order is the program order and a layer needs no queue-to-queue handshake.
kernel void ds4_copy4(device const uint* src [[buffer(0)]], device uint* dst [[buffer(1)]], constant ulong& n4 [[buffer(2)]],
uint gid [[thread_position_in_grid]]) {
if (ulong(gid) < n4) dst[gid] = src[gid];
}
kernel void ds4_copy1(device const uchar* src [[buffer(0)]], device uchar* dst [[buffer(1)]], constant ulong& n [[buffer(2)]],
uint gid [[thread_position_in_grid]]) {
if (ulong(gid) < n) dst[gid] = src[gid];
}
kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffer(1)]], constant uint& v [[buffer(2)]],
uint gid [[thread_position_in_grid]]) {
if (ulong(gid) < n) dst[gid] = uchar(v);
}
// ---------------------------------------------------------------------------------------------------------------------
// The prompt chunk's dense GEMM operands (see ds4_gemm_f16_nt on the host): float rows -> half, a quantized matrix -> half rows,
// strided half rows packed, and the float result scattered into the engine's strided output.
kernel void ds4_rows_f16(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],
device const int* idx [[buffer(2)]], constant uint& has_idx [[buffer(3)]],
constant uint& ncols [[buffer(4)]], device half* out [[buffer(5)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint r = tgp.y, c = tgp.x * 256u + tid;
if (c >= ncols) return;
const ulong row = has_idx != 0u ? ulong(idx[r]) : ulong(r);
out[(ulong) r * ncols + c] = half(x[row * ldx + c]);
}
static inline void ds4_kscale2(const device uchar* q, int j, thread int& d, thread int& m) {
if (j < 4) {
d = int(q[j]) & 63;
m = int(q[j + 4]) & 63;
} else {
d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);
m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);
}
}
// a thread a 32-value sub-block of a Q5_K matrix (n rows of k values, ld bytes a row) -> out[n][k] half
kernel void ds4_dq_q5k(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
const uint nb = k / 256u;
const ulong total = (ulong) n * nb * 8ul;
if (ulong(gid) >= total) return;
const uint ib = gid & 7u;
const uint blk = (gid >> 3) % nb;
const uint row = uint(ulong(gid) / (8ul * nb));
const device uchar* p = W + (ulong) row * ld + (ulong) blk * 176ul;
const float d = float(as_type<half>(*(const device ushort*) p));
const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
int sc, mm;
ds4_kscale2(p + 4, int(ib), sc, mm);
const device uchar* qh = p + 16;
const device uchar* qs = p + 48 + 32u * (ib >> 1);
const uint sh = 4u * (ib & 1u);
device packed_half4* o = (device packed_half4*) (out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u);
const float ds = d * float(sc), dm = dmin * float(mm);
for (uint l = 0; l < 32u; l += 4u) { // four values a step: 4-byte loads, 8-byte stores (the same arithmetic a value)
const uint4 a = uint4(*(const device packed_uchar4*) (qs + l));
const uint4 b = uint4(*(const device packed_uchar4*) (qh + l));
const uint4 q = ((a >> sh) & 15u) | (((b >> ib) & 1u) << 4);
o[l >> 2] = packed_half4(half4(float4(ds) * float4(q) - float4(dm)));
}
}
// Q8_0: 34-byte blocks (fp16 d, 32 int8)
kernel void ds4_dq_q8(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
const uint ng = k / 32u;
const ulong total = (ulong) n * ng;
if (ulong(gid) >= total) return;
const uint g = gid % ng, row = gid / ng;
const device uchar* p = W + (ulong) row * ld + (ulong) g * 34ul;
const float d = float(as_type<half>(*(const device ushort*) p));
device half* o = out + (ulong) row * k + (ulong) g * 32ul;
for (uint l = 0; l < 32u; ++l) o[l] = half(d * float(char(p[2 + l])));
}
// BF16 values (ld values a row) -> half
kernel void ds4_dq_bf16(device const ushort* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
const ulong total = (ulong) n * k;
if (ulong(gid) >= total) return;
const ulong r = ulong(gid) / k, c = ulong(gid) - r * k;
out[gid] = half(as_type<float>(uint(W[r * ld + c]) << 16));
}
// half rows with a row stride -> contiguous rows
kernel void ds4_pack_h16(device const ushort* x [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
device ushort* out [[buffer(3)]], uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint c = tgp.x * 256u + tid;
if (c < k) out[(ulong) tgp.y * k + c] = x[(ulong) tgp.y * ld + c];
}
// y[r][c] = alpha * t[r][c] + beta * y[r][c]
kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* y [[buffer(1)]], constant ulong& ldy [[buffer(2)]],
constant uint& n [[buffer(3)]], constant float& alpha [[buffer(4)]], constant float& beta [[buffer(5)]],
uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
const uint c = tgp.x * 256u + tid;
if (c >= n) return;
const ulong yi = (ulong) tgp.y * ldy + c;
y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);
}
// ds4_q8_quant for several jobs in one dispatch (a matvec call's activations): grid (groups, max rows, jobs)
struct Ds4Q8Spec {
device const float* x;
ulong ldx;
uint rows;
uint k;
device char* q;
device float* d;
};
kernel void ds4_q8_quant_multi(device const Ds4Q8Spec* specs [[buffer(0)]], uint3 tgp [[threadgroup_position_in_grid]],
uint tid [[thread_index_in_threadgroup]]) {
const device Ds4Q8Spec& s = specs[tgp.z];
const uint warp = tid / 32u, lane = tid % 32u;
const uint b = tgp.x * 8u + warp;
const uint r = tgp.y;
if (r >= s.rows || (size_t) b * 32u >= (size_t) s.k) return;
const device float* xr = s.x + (size_t) r * s.ldx;
const float v = xr[(size_t) b * 32u + lane];
const float am = simd_max(fabs(v));
const float sc = am / 127.0f;
s.q[(size_t) r * s.k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);
if (lane == 0u) s.d[(size_t) r * (s.k / 32u) + b] = sc;
}