Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#version 450
// The dense chain's attention core (vk_chain.c, VKC_P_ATTN): grouped-query attention
// of S query rows over a KV cache that lives on the device, with Qwen's output gate.
// One workgroup per (head h = WorkGroupID.x, row s = WorkGroupID.y):
// score_t = (q . K[kvh][t]) * scale over the row's positions,
// ctx = sum_t softmax(score)_t V[kvh][t], times sigmoid(gate) when has_gate.
// The positions are 0..pos_base+s (causal) or, with a selection list (Qwen3.8's QSA
// indexer, chain_qsa.comp), the row's list: sel[sel_off + s*sel_row] = count, then the
// positions; a negative count means every position 0..pos_base+s.
// The softmax runs online over tiles of 128 positions (running max and sum, the
// accumulator rescaled when the max grows), so no score row is stored and a context
// of any length costs one pass over K and V. The CPU's order of the sums differs (it
// normalizes first); the two agree to rounding.
// Cache layout as the engines keep it on the host: K[(kvh*cap + t)*hd + d].
// Limits: hd <= 256 (two dims per thread).
//
// The trailing push constants (all zero for the callers that predate them, who get
// exactly the arithmetic above):
// win a sliding window: row s sees the positions max(0, pos - win + 1)..pos,
// pos = pos_base + s (MiMo's SWA layers); 0 = every position from 0
// ring the cache is a ring of `ring` rows: position t sits in row t % ring;
// 0 = position t in row t
// vd V's head dim when it differs from K's (the output's per-head stride too);
// vk_chain.c passes hd for 0. vd <= 256
// kv_pm 1: position-major rows, K[(row*KVH + kvh)*hd + d], V[(row*KVH + kvh)*vd + d]
// (MiMo's host layout); 0: the head-major layout above
// sink, sink_off 1: one more logit per head, snk[sink_off + h], joins the softmax
// denominator and nothing else (MiMo's attention sink): the running max and
// sum start from it instead of from nothing
layout(local_size_x = 128) in;
layout(std430, binding = 0) readonly buffer QB { float q[]; };
layout(std430, binding = 1) readonly buffer KB { float kc[]; };
layout(std430, binding = 2) readonly buffer VB { float vc[]; };
layout(std430, binding = 3) buffer OB { float o[]; };
layout(std430, binding = 4) readonly buffer GB { float gt[]; };
layout(std430, binding = 5) readonly buffer LB { int sel[]; };
layout(std430, binding = 6) readonly buffer SB { float snk[]; };
layout(push_constant) uniform P {
int S, H, KVH, hd, pos_base, cap;
int q_off, q_row, q_seg;
int g_off, g_row, g_seg, has_gate;
int o_off, o_row;
int sel_off, sel_row;
float scale;
int k_off, v_off; // where this layer's cache starts in kc / vc
int win, ring, vd, kv_pm, sink, sink_off;
} p;
shared float qs[256];
shared float pe[128];
shared int pt[128];
shared float red[128];
void main() {
int h = int(gl_WorkGroupID.x), s = int(gl_WorkGroupID.y);
int tid = int(gl_LocalInvocationID.x);
int kvh = h / (p.H / p.KVH);
int qb = p.q_off + s * p.q_row + h * p.q_seg;
for (int d = tid; d < p.hd; d += 128) qs[d] = q[qb + d];
int first = p.win > 0 ? max(0, p.pos_base + s - p.win + 1) : 0;
int n = p.pos_base + s + 1 - first;
bool list = false;
int lb = 0;
if (p.sel_row > 0) {
lb = p.sel_off + s * p.sel_row;
int c = sel[lb];
if (c >= 0) { n = c; list = true; }
}
barrier();
float m = p.sink != 0 ? snk[p.sink_off + h] : -3.0e38, l = p.sink != 0 ? 1.0 : 0.0, acc0 = 0.0, acc1 = 0.0;
int kbase = kvh * p.cap;
for (int t0 = 0; t0 < n; t0 += 128) {
int i = t0 + tid;
float sv = -3.0e38;
if (i < n) {
int t = list ? sel[lb + 1 + i] : first + i;
int r = p.ring > 0 ? t % p.ring : t;
pt[tid] = r;
int kb = p.k_off + (p.kv_pm != 0 ? r * p.KVH + kvh : kbase + r) * p.hd;
float a = 0.0;
for (int d = 0; d < p.hd; d++) a += qs[d] * kc[kb + d];
sv = a * p.scale;
}
red[tid] = sv;
barrier();
for (int off = 64; off > 0; off >>= 1) {
if (tid < off) red[tid] = max(red[tid], red[tid + off]);
barrier();
}
float mn = max(m, red[0]);
barrier();
float e = i < n ? exp(sv - mn) : 0.0;
pe[tid] = e;
red[tid] = e;
barrier();
for (int off = 64; off > 0; off >>= 1) {
if (tid < off) red[tid] += red[tid + off];
barrier();
}
float corr = exp(m - mn);
l = l * corr + red[0];
acc0 *= corr; acc1 *= corr;
int cnt = min(128, n - t0);
int d0 = tid, d1 = tid + 128;
for (int j = 0; j < cnt; j++) {
int vb = p.v_off + (p.kv_pm != 0 ? pt[j] * p.KVH + kvh : kbase + pt[j]) * p.vd;
float w = pe[j];
if (d0 < p.vd) acc0 += w * vc[vb + d0];
if (d1 < p.vd) acc1 += w * vc[vb + d1];
}
m = mn;
barrier();
}
float inv = l > 0.0 ? 1.0 / l : 0.0;
int ob = p.o_off + s * p.o_row + h * p.vd;
int gb = p.g_off + s * p.g_row + h * p.g_seg;
for (int k = 0; k < 2; k++) {
int d = tid + 128 * k;
if (d >= p.vd) break;
float v = (k == 0 ? acc0 : acc1) * inv;
if (p.has_gate != 0) v *= 1.0 / (1.0 + exp(-gt[gb + d]));
o[ob + d] = v;
}
}