Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#version 450
// chain_attn.comp's attention for a prompt chunk: the same results (to rounding), the
// K and V rows read once per block of query rows instead of once per (head, row).
// chain_attn runs one workgroup per (head, row), and each reads every position the row
// sees: a chunk of S rows reads its KV cache S * H times (68 GB a layer for 2048 rows of
// Qwen3.6, 1.2 s on a Radeon 780M). Here one workgroup takes the G = H / KVH query
// heads of one KV head for BR consecutive rows, P = G * BR (head, row) pairs, at most 32,
// and walks the positions of its rows in tiles of BC (a specialization constant, 4 to 16
// as the device's shared memory allows): each tile's K and V rows go to shared memory
// once and serve every pair. 8 lanes per pair; lane l owns the query dims l, l + 8, ...
// (at most 32) and the output dims likewise, in registers:
// - each lane's partial dot products with the tile's keys go to shared memory, summed
// per pair (score = sum of the 8 partials, times the scale);
// - a position the pair's row does not see (past it, before its window, past its
// list) scores nothing; the online softmax (running max and sum, the accumulator
// rescaled when the max grows) as chain_attn's;
// - the output, divided by the sum, times sigmoid(gate) when has_gate.
// Everything chain_attn takes: a sliding window (win), a ring (ring), V's own head dim
// (vd), position-major rows (kv_pm), a sink logit (sink, sink_off), and a selection list
// per row (sel_row > 0): then BR = 1, the row's list is the workgroup's.
// The order of the sums differs from chain_attn's and the CPU's; all agree to rounding.
// Push constants: chain_attn's, then br (rows per workgroup, br * H / KVH <= 32).
// part != 0 (vk_kvsplit.h's host part on the device, from its shadow of the host's
// rows): a row's positions are its causal ones before its share of a split cache, the
// blocks pos/B - anchor and on, so [first, (pos/B - anchor)*B); the output is the part
// (the value sum unnormalized at o_off + s*o_row + h*vd, (m, l) at st_off + (s*H + h)*2)
// for vkc_kvs_merge. Its tiles start at multiples of BC: a row's sums then depend on its
// own positions only, whatever rows share its workgroup (a tile it does not see changes
// nothing), so a row gets the same bits however a forward is cut into steps. No lists,
// no sink, no ring there. With g_off > 0 the positions are cut into chunks of g_off (a
// multiple of BC), workgroup z taking [z*g_off, (z+1)*g_off): its part at o_off +
// z*g_row + ... and its (m, l) at st_off + z*g_seg + ..., for chain_kvs.comp's join (a
// decode row's thousands of positions over many workgroups; the chunks are fixed by
// position, so the bits still are the row's own).
// Limits: hd, vd <= 256.
layout(local_size_x = 256) in;
layout(constant_id = 0) const int BC = 8;
layout(std430, binding = 0) readonly buffer QB { float q[]; };
layout(std430, binding = 1) readonly buffer KB { float kc[]; };
layout(std430, binding = 2) readonly buffer VB { float vc[]; };
layout(std430, binding = 3) buffer OB { float o[]; };
layout(std430, binding = 4) readonly buffer GB { float gt[]; };
layout(std430, binding = 5) readonly buffer LB { int sel[]; };
layout(std430, binding = 6) readonly buffer SB { float snk[]; };
layout(push_constant) uniform P {
int S, H, KVH, hd, pos_base, cap;
int q_off, q_row, q_seg;
int g_off, g_row, g_seg, has_gate;
int o_off, o_row;
int sel_off, sel_row;
float scale;
int k_off, v_off;
int win, ring, vd, kv_pm, sink, sink_off;
int br;
int B, anchor, part, st_off;
} p;
shared float ks[BC * 256]; // the tile's keys [j][d]
shared float vs[BC * 256]; // the tile's values [j][d]
shared float part[32 * 8 * BC]; // [pair][lane][position]
shared float scr[32 * BC]; // [pair][position]
shared int tpos[BC]; // the tile's positions (-1: none)
void main() {
int tid = int(gl_LocalInvocationID.x);
int pair = tid >> 3, lane = tid & 7;
int kvh = int(gl_WorkGroupID.x);
int G = p.H / p.KVH;
int s0 = int(gl_WorkGroupID.y) * p.br;
int r = pair / G, g = pair - r * G;
int s = s0 + r;
bool live = r < p.br && s < p.S;
int h = kvh * G + g;
float qv[32], av[32];
int qb = p.q_off + s * p.q_row + h * p.q_seg;
for (int i = 0; i < 32; i++) {
int d = lane + 8 * i;
qv[i] = live && d < p.hd ? q[qb + d] : 0.0;
av[i] = 0.0;
}
// the block's positions: the union of its rows' ranges, or the one row's list
int slast = min(s0 + p.br, p.S) - 1;
int pos = p.pos_base + s;
int first = p.win > 0 ? max(0, pos - p.win + 1) : 0; // this pair's row
int bfirst = p.win > 0 ? max(0, p.pos_base + s0 - p.win + 1) : 0;
int n = p.pos_base + slast + 1 - bfirst;
int end = pos + 1; // this pair's row: positions before end
if (p.part != 0) {
end = (pos / p.B - p.anchor) * p.B;
int bend = ((p.pos_base + slast) / p.B - p.anchor) * p.B; // the block's last row's
bfirst -= bfirst % BC;
if (p.g_off > 0) {
int c0 = int(gl_WorkGroupID.z) * p.g_off, c1 = c0 + p.g_off;
bfirst = max(bfirst, c0); bend = min(bend, c1);
first = max(first, c0); end = min(end, c1);
}
n = max(0, bend - bfirst);
}
bool list = false;
int lb = 0;
if (p.sel_row > 0) {
lb = p.sel_off + s0 * p.sel_row;
int c = sel[lb];
if (c >= 0) { n = c; list = true; }
}
float m = p.sink != 0 && live ? snk[p.sink_off + h] : -3.0e38, l = p.sink != 0 ? 1.0 : 0.0;
int kbase = kvh * p.cap;
for (int t0 = 0; t0 < n; t0 += BC) {
if (tid < BC) {
int i = t0 + tid;
tpos[tid] = i < n ? (list ? sel[lb + 1 + i] : bfirst + i) : -1;
}
barrier();
// the tile's keys and values into shared memory, once for every pair
for (int x = tid; x < BC * p.hd; x += 256) {
int j = x / p.hd, d = x - j * p.hd, t = tpos[j];
float kv = 0.0;
if (t >= 0) {
int rr = p.ring > 0 ? t % p.ring : t;
kv = kc[p.k_off + (p.kv_pm != 0 ? rr * p.KVH + kvh : kbase + rr) * p.hd + d];
}
ks[j * 256 + d] = kv;
}
for (int x = tid; x < BC * p.vd; x += 256) {
int j = x / p.vd, d = x - j * p.vd, t = tpos[j];
float vv = 0.0;
if (t >= 0) {
int rr = p.ring > 0 ? t % p.ring : t;
vv = vc[p.v_off + (p.kv_pm != 0 ? rr * p.KVH + kvh : kbase + rr) * p.vd + d];
}
vs[j * 256 + d] = vv;
}
barrier();
// the partial dot products of this lane's query dims with the tile's keys
for (int j = 0; j < BC; j++) {
float a = 0.0;
for (int i = 0; i < 32; i++) {
int d = lane + 8 * i;
if (d < p.hd) a += qv[i] * ks[j * 256 + d];
}
part[(pair * 8 + lane) * BC + j] = a;
}
barrier();
// the scores: lane j sums position j's (and j + 8's) partials
for (int j = lane; j < BC; j += 8) {
int t = tpos[j];
float sc = -3.0e38;
if (live && t >= 0 && (list || (t >= first && t < end))) {
float a = 0.0;
for (int k = 0; k < 8; k++) a += part[(pair * 8 + k) * BC + j];
sc = a * p.scale;
}
scr[pair * BC + j] = sc;
}
barrier();
// the online softmax over the tile, each lane of the pair alike
float mx = m;
for (int j = 0; j < BC; j++) mx = max(mx, scr[pair * BC + j]);
float corr = exp(m - mx), sum = 0.0;
for (int i = 0; i < 32; i++) av[i] *= corr;
for (int j = 0; j < BC; j++) {
float sc = scr[pair * BC + j];
if (sc <= -3.0e38) continue;
float w = exp(sc - mx);
sum += w;
for (int i = 0; i < 32; i++) {
int d = lane + 8 * i;
if (d < p.vd) av[i] += w * vs[j * 256 + d];
}
}
l = l * corr + sum;
m = mx;
barrier();
}
if (!live) return;
int ob = p.o_off + s * p.o_row + h * p.vd;
if (p.part != 0) { // the part (a chunk's, with g_off), for the merge (or the join)
int z = p.g_off > 0 ? int(gl_WorkGroupID.z) : 0;
int obz = ob + z * p.g_row, sb = p.st_off + z * p.g_seg + (s * p.H + h) * 2;
for (int i = 0; i < 32; i++) {
int d = lane + 8 * i;
if (d >= p.vd) break;
o[obz + d] = av[i];
}
if (lane == 0) { o[sb] = m; o[sb + 1] = l; }
return;
}
float inv = l > 0.0 ? 1.0 / l : 0.0;
int gb = p.g_off + s * p.g_row + h * p.g_seg;
for (int i = 0; i < 32; i++) {
int d = lane + 8 * i;
if (d >= p.vd) break;
float v = av[i] * inv;
if (p.has_gate != 0) v *= 1.0 / (1.0 + exp(-gt[gb + d]));
o[ob + d] = v;
}
}