Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: KDA, hyper-connections and the k-pooled indexer on the chain

JustVugg6052a4c2341c
diff --git a/c/Makefile b/c/Makefile
index 73b8305c..a60020f5 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -719,6 +719,8 @@ VK_SPV += shaders/chain_norm.spv shaders/chain_rope.spv shaders/chain_attn.spv
# the MLA ops (vk_chain.h): attention core, RoPE and LayerNorm rows, per-head weight
# blocks, the DSA indexer's selection
VK_SPV += shaders/chain_mla.spv shaders/chain_hgemv.spv shaders/chain_dsa.spv
+# Kimi Delta Attention and manifold-constrained hyper-connections (vk_chain.h)
+VK_SPV += shaders/chain_kda.spv shaders/chain_mhc.spv
endif
# Optional AMD XDNA2 (Ryzen AI NPU) lane, Windows. Same opt-in shape as VK=1:
diff --git a/c/shaders/chain_dsa.comp b/c/shaders/chain_dsa.comp
index fa55bfdf..90e11d6c 100644
--- a/c/shaders/chain_dsa.comp
+++ b/c/shaders/chain_dsa.comp
@@ -15,6 +15,23 @@
// integer keys (32 counting passes); -0 counts as +0, as the float compare does.
// sel[s*sel_row] = keep, then the positions. Scores live in sc[s*sc_row + t].
// Limits: IH*ID <= 4096.
+//
+// With k-pooling (GLM-5.3, sparse_index.h's coli_sparse_index_select_range):
+// mode 1, the pooled keys: one workgroup per pool p = p0 + WorkGroupID.x, complete (its
+// `pool` positions all written). Per channel d, the pool's key is the softmax mixture of
+// its members' keys under their gate logits plus the position bias ape[j][d]:
+// pk[p][d] = sum_j softmax_j(gate[p*pool + j][d] + ape[j][d]) * key[p*pool + j][d]
+// in the CPU's order. A pool's key depends on its own rows only: the engine computes
+// each once, when a step completes it, and keeps it (the CPU recomputes it every step).
+// mode 2, the selection: one workgroup per query row s, position q = pos_base + s. The
+// complete pools visible to it (p < (q+1)/pool) score
+// sum_h [dot_h > 0] (w_h / wdiv) * dot_h * scale, dot_h = q_h . pk[p]
+// and the top topk/pool of them, in rank order (score, then the lower pool first), give
+// their positions: rank r's pool fills slots [r*pool, r*pool + pool); the incomplete tail
+// (the positions after the last complete pool, up to pool - 1 of them) fills the slots
+// from topk on when `tail`; every other slot is -1 (skipped by the attention core).
+// sel[s*sel_row] = topk (+ pool - 1 with the tail), then the slots. Limits: IH*ID <=
+// 4096, topk/pool <= 1024.
#extension GL_KHR_shader_subgroup_arithmetic : require
layout(local_size_x = 256) in;
@@ -23,16 +40,49 @@ layout(std430, binding = 1) readonly buffer HW { float hw[]; };
layout(std430, binding = 2) readonly buffer KC { float kc[]; };
layout(std430, binding = 3) buffer SC { float sc[]; };
layout(std430, binding = 4) buffer SL { int sel[]; };
+layout(std430, binding = 5) buffer PK { float pk[]; }; // 1: written; 2: read
+layout(std430, binding = 6) readonly buffer GT { float gt[]; }; // 1: gate logits [cap][ID]
+layout(std430, binding = 7) readonly buffer AP { float prm[]; }; // 1: ape [pool][ID]
-layout(push_constant) uniform P {
- int mode, S, pos_base, IH, ID, topk, force;
- int q_off, q_row, w_off, w_row, k_off, k_row, sc_row, sel_row;
- float qscale, wscale;
-} p;
+// one block of 32 words, read per mode (vk_chain.h: VkcDsa, VkcDsaPool, VkcDsaPick)
+layout(push_constant) uniform P { int v[32]; } p;
+#define S_ p.v[1]
+#define POS_BASE p.v[2]
+#define IH_ p.v[3]
+#define ID_ p.v[4]
+#define TOPK p.v[5]
+#define FORCE p.v[6]
+#define Q_OFF p.v[7]
+#define Q_ROW p.v[8]
+#define W_OFF p.v[9]
+#define W_ROW p.v[10]
+#define K_OFF p.v[11]
+#define K_ROW p.v[12]
+#define SC_ROW p.v[13]
+#define SEL_ROW p.v[14]
+#define QSCALE intBitsToFloat(p.v[15])
+#define WSCALE intBitsToFloat(p.v[16])
+// mode 1: ID_ at 4, then
+#define POOL1 p.v[2]
+#define P0 p.v[3]
+#define G_OFF p.v[5]
+#define G_ROW p.v[6]
+#define APE_OFF p.v[7]
+#define PK_OFF1 p.v[8]
+#define K_OFF1 p.v[9]
+#define K_ROW1 p.v[10]
+// mode 2: S_, POS_BASE, IH_, ID_, TOPK, Q_OFF.., W_.., SC_ROW, SEL_ROW as mode 0, then
+#define POOL2 p.v[6]
+#define PK_OFF2 p.v[11]
+#define TAIL p.v[12]
+#define WDIV intBitsToFloat(p.v[15])
+#define PSCALE intBitsToFloat(p.v[16])
shared float qs[4096];
shared float ws[64];
shared uint cnt[256];
+shared int plist[1024];
+shared uint pkey[1024];
uint okey(float f) {
uint u = floatBitsToUint(f);
@@ -67,27 +117,27 @@ uint wscan(uint v, int tid) {
return incl - v;
}
-void main() {
+void token_select() {
int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);
- int nk = p.pos_base + s + 1;
- int lb = s * p.sel_row;
- if (nk <= p.topk && p.force == 0) { if (tid == 0) sel[lb] = -1; return; }
- int keep = min(nk, p.topk);
- int nq = p.IH * p.ID, qb = p.q_off + s * p.q_row;
+ int nk = POS_BASE + s + 1;
+ int lb = s * SEL_ROW;
+ if (nk <= TOPK && FORCE == 0) { if (tid == 0) sel[lb] = -1; return; }
+ int keep = min(nk, TOPK);
+ int nq = IH_ * ID_, qb = Q_OFF + s * Q_ROW;
for (int i = tid; i < nq; i += 256) qs[i] = iq[qb + i];
- for (int h = tid; h < p.IH; h += 256) ws[h] = hw[p.w_off + s * p.w_row + h];
+ for (int h = tid; h < IH_; h += 256) ws[h] = hw[W_OFF + s * W_ROW + h];
barrier();
- int sb = s * p.sc_row;
+ int sb = s * SC_ROW;
for (int t = tid; t < nk; t += 256) {
- int kb = p.k_off + t * p.k_row;
+ int kb = K_OFF + t * K_ROW;
float a = 0.0;
- for (int h = 0; h < p.IH; h++) {
+ for (int h = 0; h < IH_; h++) {
float d = 0.0;
- for (int i = 0; i < p.ID; i++) d += qs[h * p.ID + i] * kc[kb + i];
- d *= p.qscale;
+ for (int i = 0; i < ID_; i++) d += qs[h * ID_ + i] * kc[kb + i];
+ d *= QSCALE;
if (d > 0.0) a += ws[h] * d;
}
- sc[sb + t] = a * p.wscale;
+ sc[sb + t] = a * WSCALE;
}
memoryBarrierBuffer();
barrier();
@@ -110,3 +160,90 @@ void main() {
for (int t = c0; t < c1 && o < uint(keep); t++) if (okey(sc[sb + t]) == K) sel[lb + 1 + int(o++)] = t;
if (tid == 0) sel[lb] = keep;
}
+
+// the keep-th largest key among n scores at sc[sb..], bit by bit from the top
+uint kth_key(int sb, int n, int keep, int tid) {
+ uint K = 0u;
+ for (int b = 31; b >= 0; b--) {
+ uint cand = K | (1u << uint(b));
+ uint c = 0u;
+ for (int t = tid; t < n; t += 256) if (okey(sc[sb + t]) >= cand) c++;
+ if (wsum(c, tid) >= uint(keep)) K = cand;
+ }
+ return K;
+}
+
+void pool_keys() {
+ int pp = P0 + int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);
+ int start = pp * POOL1;
+ for (int d = tid; d < ID_; d += 256) {
+ float mxv = -3.402823466e38;
+ for (int j = 0; j < POOL1; j++) {
+ float lg = gt[G_OFF + (start + j) * G_ROW + d] + prm[APE_OFF + j * ID_ + d];
+ if (lg > mxv) mxv = lg;
+ }
+ float total = 0.0;
+ for (int j = 0; j < POOL1; j++)
+ total += exp(gt[G_OFF + (start + j) * G_ROW + d] + prm[APE_OFF + j * ID_ + d] - mxv);
+ float mixed = 0.0;
+ for (int j = 0; j < POOL1; j++) {
+ float wgt = exp(gt[G_OFF + (start + j) * G_ROW + d] + prm[APE_OFF + j * ID_ + d] - mxv) / total;
+ mixed += wgt * kc[K_OFF1 + (start + j) * K_ROW1 + d];
+ }
+ pk[PK_OFF1 + pp * ID_ + d] = mixed;
+ }
+}
+
+void pool_select() {
+ int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);
+ int q = POS_BASE + s, pool = POOL2;
+ int nvis = (q + 1) / pool, wanted = TOPK / pool, keep = min(wanted, nvis);
+ int width = TAIL != 0 ? TOPK + pool - 1 : TOPK;
+ int lb = s * SEL_ROW;
+ for (int i = tid; i < width; i += 256) sel[lb + 1 + i] = -1;
+ int nq = IH_ * ID_, qb = Q_OFF + s * Q_ROW;
+ for (int i = tid; i < nq; i += 256) qs[i] = iq[qb + i];
+ for (int h = tid; h < IH_; h += 256) ws[h] = hw[W_OFF + s * W_ROW + h] / WDIV;
+ barrier();
+ int sb = s * SC_ROW;
+ for (int pp = tid; pp < nvis; pp += 256) {
+ float score = 0.0;
+ for (int h = 0; h < IH_; h++) {
+ float dot = 0.0;
+ for (int d = 0; d < ID_; d++) dot += qs[h * ID_ + d] * pk[PK_OFF2 + pp * ID_ + d];
+ if (dot > 0.0) score += ws[h] * dot * PSCALE;
+ }
+ sc[sb + pp] = score;
+ }
+ memoryBarrierBuffer();
+ barrier();
+ if (keep > 0) {
+ uint K = kth_key(sb, nvis, keep, tid);
+ int chunk = (nvis + 255) / 256, c0 = tid * chunk, c1 = min(nvis, c0 + chunk);
+ uint above = 0u, at = 0u;
+ for (int t = c0; t < c1; t++) { uint k = okey(sc[sb + t]); above += k > K ? 1u : 0u; at += k == K ? 1u : 0u; }
+ uint n1 = wsum(above, tid);
+ uint o = wscan(above, tid);
+ for (int t = c0; t < c1; t++) { uint k = okey(sc[sb + t]); if (k > K) { plist[o] = t; pkey[o] = k; o++; } }
+ o = n1 + wscan(at, tid);
+ for (int t = c0; t < c1 && o < uint(keep); t++) { uint k = okey(sc[sb + t]); if (k == K) { plist[o] = t; pkey[o] = k; o++; } }
+ barrier();
+ // rank order: the higher score first, then the lower pool
+ for (int i = tid; i < keep; i += 256) {
+ uint ki = pkey[i]; int pi = plist[i], r = 0;
+ for (int j = 0; j < keep; j++) { uint kj = pkey[j]; if (kj > ki || (kj == ki && plist[j] < pi)) r++; }
+ for (int j = 0; j < pool; j++) sel[lb + 1 + r * pool + j] = pi * pool + j;
+ }
+ }
+ if (TAIL != 0 && tid == 0) {
+ int visible = q + 1, tail = visible % pool, ts = visible - tail;
+ for (int j = 0; j < tail && j < pool - 1; j++) sel[lb + 1 + TOPK + j] = ts + j;
+ }
+ if (tid == 0) sel[lb] = width;
+}
+
+void main() {
+ if (p.v[0] == 0) token_select();
+ else if (p.v[0] == 1) pool_keys();
+ else pool_select();
+}
diff --git a/c/shaders/chain_dsa.spv b/c/shaders/chain_dsa.spv
index 53c70bf2..73bd037f 100644
Binary files a/c/shaders/chain_dsa.spv and b/c/shaders/chain_dsa.spv differ
diff --git a/c/shaders/chain_kda.comp b/c/shaders/chain_kda.comp
new file mode 100644
index 00000000..a30505ce
--- /dev/null
+++ b/c/shaders/chain_kda.comp
@@ -0,0 +1,150 @@
+#version 450
+#extension GL_EXT_control_flow_attributes : require
+// Kimi Delta Attention on the dense chain (vk_chain.c, VKC_P_KDA): the linear-attention
+// layers of GLM-5.3 (and Kimi K3), delta_attention.h's coli_kda_step for S rows in
+// order, the state and the convolution window on the device across steps.
+//
+// mode 0, the short convolution: one thread per channel c of the C = 3P channels (q, k,
+// v, P each), the rows in order. The channel's window holds its last K inputs, oldest
+// first, as the CPU keeps it ([C][K]); per row it drops the oldest, takes the input
+// (part c / P at in_off + part*in_part + s*in_row + c % P) and the output is
+// silu(sum_tap w[c][tap] * window[tap]), the taps in order, at out_off + s*out_row + c.
+//
+// mode 1, the recurrence and the output norm: one workgroup per head h, thread v owns
+// the state's column S[h][:, v] in registers (KD, the key dim, is a specialization
+// constant; the value dim VD <= 128). Per row, from the convolution's output m (q at
+// h*KD, k at P + h*KD, v at 2P + h*VD) and the raw projections:
+// q, k l2-normalized with eps inside the root, q also times 1/sqrt(KD)
+// alpha = exp(lb * sigmoid(exp(A_log[h]) * (f[s][h*KD+k] + dt[h*KD+k]))), per key row
+// beta = sigmoid(b[s][h])
+// S[k] *= alpha[k]; mem = sum_k k[k] S[k]; S[k] += k[k] (v - mem) beta; o = sum_k q[k] S[k]
+// y[s][h*VD+v] = o * rsqrt(mean(o^2) + eps) * norm[v] * sigmoid(g[s][h*VD+v])
+// Each column runs the CPU's products and sums in its order; the two l2 sums and the
+// output norm's sum are made by one thread in order, as on the CPU.
+// Parameters in prm at prm_off: A_log[H], dt[P], norm[VD].
+layout(local_size_x = 128) in;
+layout(constant_id = 0) const int KD = 128;
+
+layout(std430, binding = 0) readonly buffer IN { float xin[]; }; // 0: q|k|v projections; 1: conv output
+layout(std430, binding = 1) readonly buffer WB { float w[]; }; // 0: taps; 1: prm
+layout(std430, binding = 2) buffer RB { float st[]; }; // 0: window; 1: state [H][KD][VD]
+layout(std430, binding = 3) writeonly buffer OB { float o[]; }; // 0: conv output; 1: y
+layout(std430, binding = 4) readonly buffer FB { float fr[]; }; // 1: decay projection f
+layout(std430, binding = 5) readonly buffer BB { float br[]; }; // 1: beta projection b
+layout(std430, binding = 6) readonly buffer GB { float gr[]; }; // 1: output gate projection g
+
+layout(push_constant) uniform P { int v[32]; } p;
+// mode 0
+#define S_ p.v[1]
+#define C_ p.v[2]
+#define KW p.v[3]
+#define PW p.v[4]
+#define IN_OFF p.v[5]
+#define IN_ROW p.v[6]
+#define IN_PART p.v[7]
+#define OUT_OFF p.v[8]
+#define OUT_ROW p.v[9]
+#define W_OFF p.v[10]
+#define WIN_OFF p.v[11]
+// mode 1
+#define H_ p.v[2]
+#define VD p.v[3]
+#define M_OFF p.v[5]
+#define M_ROW p.v[6]
+#define F_OFF p.v[7]
+#define F_ROW p.v[8]
+#define B_OFF p.v[9]
+#define B_ROW p.v[10]
+#define G_OFF p.v[11]
+#define G_ROW p.v[12]
+#define Y_OFF p.v[13]
+#define Y_ROW p.v[14]
+#define ST_OFF p.v[15]
+#define PRM_OFF p.v[16]
+#define LB intBitsToFloat(p.v[17])
+#define NEPS intBitsToFloat(p.v[18])
+#define EPS intBitsToFloat(p.v[19])
+
+shared float sq[KD];
+shared float sk[KD];
+shared float al[KD];
+shared float ov[128];
+shared float scal[2];
+
+float sig(float x) { // hyper_connections.h's and glm53.c's two-sided form
+ if (x >= 0.0) return 1.0 / (1.0 + exp(-x));
+ float e = exp(x);
+ return e / (1.0 + e);
+}
+
+void conv() {
+ int c = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 128u);
+ if (c >= C_) return;
+ int part = c / PW, j = c - part * PW;
+ float hist[8];
+ int wb = WIN_OFF + c * KW, tb = W_OFF + c * KW;
+ for (int k = 0; k < 8; k++) hist[k] = k < KW ? st[wb + k] : 0.0;
+ for (int s = 0; s < S_; s++) {
+ for (int k = 0; k < 7; k++) if (k + 1 < KW) hist[k] = hist[k + 1];
+ float cur = xin[IN_OFF + part * IN_PART + s * IN_ROW + j];
+ for (int k = 0; k < 8; k++) if (k == KW - 1) hist[k] = cur;
+ float sum = 0.0;
+ for (int k = 0; k < 8; k++) if (k < KW) sum += w[tb + k] * hist[k];
+ o[OUT_OFF + s * OUT_ROW + c] = sum / (1.0 + exp(-sum));
+ }
+ for (int k = 0; k < 8; k++) if (k < KW) st[wb + k] = hist[k];
+}
+
+void rec() {
+ int h = int(gl_WorkGroupID.x), v = int(gl_LocalInvocationID.x);
+ int Pw = PW;
+ bool own = v < VD;
+ float col[KD];
+ int sb = ST_OFF + h * KD * VD;
+ [[unroll]] for (int k = 0; k < KD; k++) col[k] = own ? st[sb + k * VD + v] : 0.0;
+ float ea = exp(w[PRM_OFF + h]);
+ float nw = own ? w[PRM_OFF + H_ + Pw + v] : 0.0;
+ for (int s = 0; s < S_; s++) {
+ int mb = M_OFF + s * M_ROW;
+ for (int k = v; k < KD; k += 128) {
+ sq[k] = xin[mb + h * KD + k];
+ sk[k] = xin[mb + Pw + h * KD + k];
+ float f = fr[F_OFF + s * F_ROW + h * KD + k] + w[PRM_OFF + H_ + h * KD + k];
+ al[k] = exp(LB * sig(ea * f));
+ }
+ barrier();
+ if (v == 0) { // the l2 sums, in order, eps first
+ float qs = NEPS, ks = NEPS;
+ for (int k = 0; k < KD; k++) { qs += sq[k] * sq[k]; ks += sk[k] * sk[k]; }
+ scal[0] = (1.0 / sqrt(float(KD))) / sqrt(qs);
+ scal[1] = 1.0 / sqrt(ks);
+ }
+ barrier();
+ float qn = scal[0], kn = scal[1];
+ float beta = sig(br[B_OFF + s * B_ROW + h]);
+ float val = own ? xin[mb + 2 * Pw + h * VD + v] : 0.0;
+ float mem = 0.0;
+ [[unroll]] for (int k = 0; k < KD; k++) { col[k] *= al[k]; mem += (sk[k] * kn) * col[k]; }
+ float res = 0.0;
+ [[unroll]] for (int k = 0; k < KD; k++) {
+ col[k] += (sk[k] * kn) * (val - mem) * beta;
+ res += (sq[k] * qn) * col[k];
+ }
+ ov[v] = own ? res : 0.0;
+ barrier();
+ if (v == 0) {
+ float sum = 0.0;
+ for (int d = 0; d < VD; d++) sum += ov[d] * ov[d];
+ scal[0] = 1.0 / sqrt(sum / float(VD) + EPS);
+ }
+ barrier();
+ if (own) o[Y_OFF + s * Y_ROW + h * VD + v] = res * scal[0] * nw * sig(gr[G_OFF + s * G_ROW + h * VD + v]);
+ barrier();
+ }
+ if (own) [[unroll]] for (int k = 0; k < KD; k++) st[sb + k * VD + v] = col[k];
+}
+
+void main() {
+ if (p.v[0] == 0) conv();
+ else rec();
+}
diff --git a/c/shaders/chain_kda.spv b/c/shaders/chain_kda.spv
new file mode 100644
index 00000000..110ddcd8
Binary files /dev/null and b/c/shaders/chain_kda.spv differ
diff --git a/c/shaders/chain_mhc.comp b/c/shaders/chain_mhc.comp
new file mode 100644
index 00000000..7121f353
--- /dev/null
+++ b/c/shaders/chain_mhc.comp
@@ -0,0 +1,118 @@
+#version 450
+// Manifold-constrained hyper-connections (mHC) on the dense chain (vk_chain.c,
+// VKC_P_MHC), hyper_connections.h's arithmetic for S rows of H residual streams of D
+// floats ([S][H*D]), and a clamped SwiGLU. Five modes:
+//
+// mode 0, the split: one workgroup per row s. r = 1 / sqrt(mean(x^2) + eps) over the
+// row's H*D stream values; the mix logits are mix[s][i] * r (mix = the raw products
+// hc_fn . x, from a matmul); then, by one thread in the CPU's order:
+// pre[i] = sigmoid(m[i] * scale[0] + base[i]) + hc_eps
+// post[i] = 2 * sigmoid(m[H+i] * scale[1] + base[H+i])
+// comb = softmax per row of m[2H + row*H + col] * scale[2] + base[...], + hc_eps,
+// then iters - 1 rounds of Sinkhorn (rows, then columns) after a first
+// column normalization
+// into hp[s][0..H) pre, [H..2H) post, [2H..2H+H*H) comb. scale (3) and base (2H+H*H)
+// at prm_off in prm.
+// mode 1, collapse: y[s][d] = sum_i pre[i] * x[s][i*D + d], the streams in order.
+// mode 2, write back: y[s][j*D + d] = sum_i comb[i*H + j] * x[s][i*D + d] + post[j] * b[s][d]
+// mode 3, mean: y[s][d] = (sum_i x[s][i*D + d]) / H (the model's last collapse).
+// mode 4, clamped SwiGLU: y[i] = silu(min(a[i], lim)) * clamp(b[i], -lim, lim) over n
+// values (GLM-5.3's dense MLPs and shared experts; lim from the config).
+// Offsets in floats; a row of x / y is x_row / y_row floats.
+layout(local_size_x = 256) in;
+
+layout(std430, binding = 0) readonly buffer XB { float x[]; }; // streams; 4: a
+layout(std430, binding = 1) readonly buffer MB { float mx[]; }; // 0: mix; 2: branch; 4: b
+layout(std430, binding = 2) buffer HB { float hp[]; }; // pre | post | comb per row
+layout(std430, binding = 3) readonly buffer PB { float prm[]; }; // 0: scale, base
+layout(std430, binding = 4) writeonly buffer YB { float y[]; };
+
+layout(push_constant) uniform P {
+ int mode, S, H, D, iters;
+ int x_off, x_row, m_off, m_row, hp_off, hp_row, y_off, y_row, prm_off, n;
+ float eps, hc_eps, lim;
+} p;
+
+shared float red[256];
+shared float ms[96];
+
+float sig(float v) {
+ if (v >= 0.0) { float d = exp(-v); return 1.0 / (1.0 + d); }
+ float g = exp(v);
+ return g / (1.0 + g);
+}
+
+void split() {
+ int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);
+ int H = p.H, HD = H * p.D, nm = (2 + H) * H;
+ int xb = p.x_off + s * p.x_row;
+ float a = 0.0;
+ for (int i = tid; i < HD; i += 256) { float v = x[xb + i]; a += v * v; }
+ red[tid] = a;
+ barrier();
+ for (int off = 128; off > 0; off >>= 1) {
+ if (tid < off) red[tid] += red[tid + off];
+ barrier();
+ }
+ float r = 1.0 / sqrt(red[0] / float(HD) + p.eps);
+ for (int i = tid; i < nm; i += 256) ms[i] = mx[p.m_off + s * p.m_row + i] * r;
+ barrier();
+ if (tid != 0) return;
+ int hb = p.hp_off + s * p.hp_row, pb = p.prm_off;
+ for (int i = 0; i < H; i++) {
+ hp[hb + i] = sig(ms[i] * prm[pb] + prm[pb + 3 + i]) + p.hc_eps;
+ hp[hb + H + i] = 2.0 * sig(ms[H + i] * prm[pb + 1] + prm[pb + 3 + H + i]);
+ }
+ int cb = hb + 2 * H, mo = 2 * H;
+ for (int row = 0; row < H; row++) {
+ float mxv = -3.0e38;
+ for (int col = 0; col < H; col++) {
+ float v = ms[mo + row * H + col] * prm[pb + 2] + prm[pb + 3 + mo + row * H + col];
+ hp[cb + row * H + col] = v;
+ if (v > mxv) mxv = v;
+ }
+ float sum = 0.0;
+ for (int col = 0; col < H; col++) { float v = exp(hp[cb + row * H + col] - mxv); hp[cb + row * H + col] = v; sum += v; }
+ for (int col = 0; col < H; col++) hp[cb + row * H + col] = hp[cb + row * H + col] / sum + p.hc_eps;
+ }
+ float sums[8];
+ for (int col = 0; col < H; col++) { float sm = 0.0; for (int row = 0; row < H; row++) sm += hp[cb + row * H + col]; sums[col] = sm; }
+ for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[col] + p.hc_eps;
+ for (int it = 1; it < p.iters; it++) {
+ for (int row = 0; row < H; row++) { float sm = 0.0; for (int col = 0; col < H; col++) sm += hp[cb + row * H + col]; sums[row] = sm; }
+ for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[row] + p.hc_eps;
+ for (int col = 0; col < H; col++) { float sm = 0.0; for (int row = 0; row < H; row++) sm += hp[cb + row * H + col]; sums[col] = sm; }
+ for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[col] + p.hc_eps;
+ }
+}
+
+void main() {
+ if (p.mode == 0) { split(); return; }
+ int i = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);
+ if (p.mode == 4) {
+ if (i >= p.n) return;
+ float g = x[p.x_off + i], u = mx[p.m_off + i];
+ g = g > p.lim ? p.lim : g;
+ u = u < -p.lim ? -p.lim : (u > p.lim ? p.lim : u);
+ y[p.y_off + i] = (g / (1.0 + exp(-g))) * u;
+ return;
+ }
+ int H = p.H, D = p.D;
+ if (p.mode == 2) {
+ if (i >= p.S * H * D) return;
+ int s = i / (H * D), r = i - s * H * D, j = r / D, d = r - j * D;
+ int xb = p.x_off + s * p.x_row, hb = p.hp_off + s * p.hp_row;
+ float v = 0.0;
+ for (int k = 0; k < H; k++) v += hp[hb + 2 * H + k * H + j] * x[xb + k * D + d];
+ v += hp[hb + H + j] * mx[p.m_off + s * p.m_row + d];
+ y[p.y_off + s * p.y_row + j * D + d] = v;
+ return;
+ }
+ if (i >= p.S * D) return;
+ int s = i / D, d = i - s * D;
+ int xb = p.x_off + s * p.x_row, hb = p.hp_off + s * p.hp_row;
+ float v = 0.0;
+ if (p.mode == 1) { for (int k = 0; k < H; k++) v += hp[hb + k] * x[xb + k * D + d]; }
+ else { for (int k = 0; k < H; k++) v += x[xb + k * D + d]; v = v / float(H); }
+ y[p.y_off + s * p.y_row + d] = v;
+}
diff --git a/c/shaders/chain_mhc.spv b/c/shaders/chain_mhc.spv
new file mode 100644
index 00000000..c13871ca
Binary files /dev/null and b/c/shaders/chain_mhc.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index b016afe7..a17d1626 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -25,14 +25,25 @@
* lists with skipped entries, a gate, prefill rows, a long context from
* kv_start, latents up to 1024)
* dsa the indexer's top-k selection bit for bit, ties included
+ * kpool the k-pooled indexer (GLM-5.3): pooled keys, the pools' rank order and
+ * the tail, slot for slot against sparse_index.h
+ * kda GLM-5.3's KDA layer around delta_attention.h's step: the convolution
+ * and its window, the recurrence and its state over two submissions, the
+ * decay, beta, output norm and gate
+ * mhc hyper_connections.h's split, collapse and write back, the mean, and the
+ * clamped SwiGLU
*
* make vk-chain-check VK=1 (VK_ICD_FILENAMES=.../lvp_icd.json for Lavapipe) */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
+#include <float.h>
#include "../backend_vulkan.h"
#include "../vk_chain.h"
+#include "../delta_attention.h" /* the KDA step the KDA ops follow */
+#include "../hyper_connections.h" /* the mHC arithmetic */
+#include "../sparse_index.h" /* GLM-5.3's k-pooled indexer */
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
@@ -790,6 +801,181 @@ static void test_dsa(int S, int pos_base, int IH, int ID, int topk, int force, i
free(q); free(k); free(w); free(ref);
}
+/* ---- KDA, mHC, k-pooling ----------------------------------------------------------------- */
+/* glm53.c's KDA layer around delta_attention.h's step, S rows over two calls (the state
+ * and the window carried on the device between them) */
+static void test_kda(int H, int KD, int CK, float xs) { /* xs: the inputs' scale (small: the l2 eps counts) */
+ int VD = KD, P = H * KD, C = 3 * P, S = 5;
+ float lb = -5.f, neps = 1e-6f, eps = 1e-5f;
+ float *qkv = fvec((size_t)S * C, xs), *f = fvec((size_t)S * P, 1.f), *b = fvec((size_t)S * H, 2.f), *g = fvec((size_t)S * P, 2.f);
+ float *taps = fvec((size_t)C * CK, 0.7f), *win = fvec((size_t)C * CK, 1.f), *state = fvec((size_t)H * KD * VD, 0.3f);
+ float *prm = malloc((size_t)(H + P + VD) * 4);
+ for (int i = 0; i < H; i++) prm[i] = frnd() * 0.5f;
+ for (int i = 0; i < P; i++) prm[H + i] = frnd() * 0.5f;
+ for (int i = 0; i < VD; i++) prm[H + P + i] = 0.5f + (rnd() % 100) / 100.f;
+ /* reference: glm53.c's kda_layer row by row */
+ float *rs = malloc((size_t)H * KD * VD * 4), *rw = malloc((size_t)C * CK * 4), *ref = malloc((size_t)S * P * 4);
+ memcpy(rs, state, (size_t)H * KD * VD * 4); memcpy(rw, win, (size_t)C * CK * 4);
+ float *scratch = malloc((size_t)coli_kda_scratch_floats(H, KD, VD) * 4), *core = malloc((size_t)P * 4);
+ float *dk = malloc((size_t)P * 4), *bt = malloc((size_t)H * 4);
+ for (int s = 0; s < S; s++) {
+ for (int h = 0; h < H; h++) for (int d = 0; d < KD; d++) {
+ int i = h * KD + d; float sv = expf(prm[h]) * (f[s * P + i] + prm[H + i]);
+ float sg = sv >= 0 ? 1.f / (1.f + expf(-sv)) : expf(sv) / (1.f + expf(sv));
+ dk[i] = lb * sg;
+ }
+ for (int h = 0; h < H; h++) { float v = b[s * H + h]; bt[h] = v >= 0 ? 1.f / (1.f + expf(-v)) : expf(v) / (1.f + expf(v)); }
+ coli_kda_step(core, rs, rw, qkv + (size_t)s * C, taps, dk, bt, H, KD, VD, CK, neps, scratch);
+ for (int h = 0; h < H; h++) {
+ float sq = 0; for (int d = 0; d < VD; d++) sq += core[h * VD + d] * core[h * VD + d];
+ float inv = 1.f / sqrtf(sq / VD + eps);
+ for (int d = 0; d < VD; d++) {
+ float gv = g[s * P + h * VD + d], sg = gv >= 0 ? 1.f / (1.f + expf(-gv)) : expf(gv) / (1.f + expf(gv));
+ ref[s * P + h * VD + d] = core[h * VD + d] * inv * prm[H + P + d] * sg;
+ }
+ }
+ }
+ /* the device: q, k, v in three blocks [3][S][P] (in_part = S*P) */
+ float *blk = malloc((size_t)3 * S * P * 4);
+ for (int s = 0; s < S; s++) for (int part = 0; part < 3; part++)
+ memcpy(blk + (size_t)part * S * P + (size_t)s * P, qkv + (size_t)s * C + (size_t)part * P, (size_t)P * 4);
+ VkcBuf *ib = up(blk, (size_t)3 * S * P), *tb = up(taps, (size_t)C * CK), *wb = up(win, (size_t)C * CK), *sb = up(state, (size_t)H * KD * VD);
+ VkcBuf *fb = up(f, (size_t)S * P), *bb = up(b, (size_t)S * H), *gb = up(g, (size_t)S * P), *pb = up(prm, (size_t)H + P + VD);
+ VkcBuf *mb = vkc_buf((size_t)S * C * 4, VKC_DEV), *yb = vkc_buf((size_t)S * P * 4, VKC_DEV);
+ int ok = 1, s0 = 0;
+ for (int part = 0; part < 2 && ok; part++) {
+ int n = part ? S - 3 : 3;
+ VkcKdaConv cp = {n, C, CK, P, s0 * P, P, S * P, 0, C, 0, 0};
+ VkcKdaRec rp = {n, H, VD, P, 0, C, s0 * P, P, s0 * H, H, s0 * P, P, s0 * P, P, 0, 0, lb, neps, eps};
+ ok = vkc_begin() && vkc_kda_conv(ib, tb, wb, mb, &cp) && vkc_kda_rec(KD, mb, fb, bb, gb, pb, sb, yb, &rp) && vkc_submit(part);
+ s0 += n;
+ }
+ float *y = down(yb, 0, (size_t)S * P), *st2 = down(sb, 0, (size_t)H * KD * VD), *w2 = down(wb, 0, (size_t)C * CK);
+ double e = relerr(y, ref, (size_t)S * P, 1e-3), es = relerr(st2, rs, (size_t)H * KD * VD, 1e-3), ew = relerr(w2, rw, (size_t)C * CK, 1e-3);
+ CHECK(ok && e < 1e-5 && es < 1e-5 && ew == 0 && !bad(y, (size_t)S * P), "kda H %d KD %d K %d: ok %d out %.2e state %.2e window %.2e",
+ H, KD, CK, ok, e, es, ew);
+ if (getenv("VKC_TEST_VERBOSE")) printf("kda H %d KD %d: out %.2e state %.2e\n", H, KD, e, es);
+ vkc_free(ib); vkc_free(tb); vkc_free(wb); vkc_free(sb); vkc_free(fb); vkc_free(bb); vkc_free(gb); vkc_free(pb); vkc_free(mb); vkc_free(yb);
+ free(qkv); free(f); free(b); free(g); free(taps); free(win); free(state); free(prm); free(rs); free(rw); free(ref);
+ free(scratch); free(core); free(dk); free(bt); free(blk); free(y); free(st2); free(w2);
+}
+
+/* hyper_connections.h's pre and post against the split, collapse and write back, the
+ * mean, and the clamped SwiGLU */
+static void test_mhc(int H, int D, int iters) {
+ int S = 3, HD = H * D, nm = (2 + H) * H;
+ float eps = 1e-5f, hce = 1e-6f, lim = 1.5f;
+ float *x = fvec((size_t)S * HD, 1.f), *fn = fvec((size_t)nm * HD, 0.05f), *br = fvec((size_t)S * D, 1.f);
+ float *prm = malloc((size_t)(3 + nm) * 4);
+ for (int i = 0; i < 3; i++) prm[i] = 0.5f + (rnd() % 100) / 100.f;
+ for (int i = 0; i < nm; i++) prm[3 + i] = frnd() * 0.5f;
+ float *rpre = malloc((size_t)S * D * 4), *rpost = malloc((size_t)S * H * 4), *rcomb = malloc((size_t)S * H * H * 4);
+ float *rout = malloc((size_t)S * HD * 4), *rmean = malloc((size_t)S * D * 4);
+ for (int s = 0; s < S; s++) {
+ coli_hc_pre(rpre + s * D, rpost + s * H, rcomb + s * H * H, x + s * HD, fn, prm, prm + 3, H, D, iters, eps, hce);
+ coli_hc_post(rout + s * HD, br + s * D, x + s * HD, rpost + s * H, rcomb + s * H * H, H, D);
+ for (int d = 0; d < D; d++) { float sm = 0; for (int i = 0; i < H; i++) sm += x[s * HD + i * D + d]; rmean[s * D + d] = sm / H; }
+ }
+ ColiVkTensor *t = NULL;
+ if (!coli_vk_tensor_ensure(&t, fn, NULL, 10, HD, nm, 0)) { CHECK(0, "mhc: upload"); return; }
+ VkcBuf *xb = up(x, (size_t)S * HD), *bb = up(br, (size_t)S * D), *pb = up(prm, 3 + (size_t)nm);
+ VkcBuf *mb = vkc_buf((size_t)S * nm * 4, VKC_DEV), *hb = vkc_buf((size_t)S * (2 * H + H * H) * 4, VKC_DEV);
+ VkcBuf *cb = vkc_buf((size_t)S * D * 4, VKC_DEV), *ob = vkc_buf((size_t)S * HD * 4, VKC_DEV), *eb = vkc_buf((size_t)S * D * 4, VKC_DEV);
+ int hrow = 2 * H + H * H;
+ VkcMhc sp = {S, H, D, iters, 0, HD, 0, nm, 0, hrow, 0, D, 0, 0, eps, hce, 0.f};
+ VkcMhc po = {S, H, D, iters, 0, HD, 0, D, 0, hrow, 0, HD, 0, 0, eps, hce, 0.f};
+ int ok = vkc_begin() && vkc_matmul(t, xb, 0, mb, 0, S) && vkc_mhc(VKC_MHC_SPLIT, xb, mb, hb, pb, NULL, &sp) &&
+ vkc_mhc(VKC_MHC_COLLAPSE, xb, NULL, hb, NULL, cb, &sp) && vkc_mhc(VKC_MHC_POST, xb, bb, hb, NULL, ob, &po) &&
+ vkc_mhc(VKC_MHC_MEAN, xb, NULL, NULL, NULL, eb, &sp) && vkc_submit(1);
+ float *hp = down(hb, 0, (size_t)S * hrow), *col = down(cb, 0, (size_t)S * D), *out = down(ob, 0, (size_t)S * HD), *mean = down(eb, 0, (size_t)S * D);
+ /* pre, post and comb from coli_hc_split_sinkhorn on the CPU's own mixes */
+ float *rp = malloc((size_t)S * hrow * 4), *mixs = malloc((size_t)nm * 4);
+ for (int s = 0; s < S; s++) {
+ float ms = 0; for (int i = 0; i < HD; i++) ms += x[s * HD + i] * x[s * HD + i];
+ float ir = 1.f / sqrtf(ms / HD + eps);
+ for (int r = 0; r < nm; r++) { float a = 0; for (int i = 0; i < HD; i++) a += fn[(size_t)r * HD + i] * x[s * HD + i]; mixs[r] = a * ir; }
+ coli_hc_split_sinkhorn(rp + s * hrow, rp + s * hrow + H, rp + s * hrow + 2 * H, mixs, prm, prm + 3, H, iters, hce);
+ }
+ double epre = 0;
+ for (int s = 0; s < S; s++) for (int i = 0; i < hrow; i++) { double d = fabs(hp[s * hrow + i] - rp[s * hrow + i]); if (d > epre) epre = d; }
+ free(rp); free(mixs);
+ double ep = 0, ec = relerr(col, rpre, (size_t)S * D, 1e-3), eo = relerr(out, rout, (size_t)S * HD, 1e-3), em = relerr(mean, rmean, (size_t)S * D, 1e-3);
+ for (int s = 0; s < S; s++) {
+ for (int i = 0; i < H; i++) { double d = fabs(hp[s * hrow + H + i] - rpost[s * H + i]); if (d > ep) ep = d; }
+ for (int i = 0; i < H * H; i++) { double d = fabs(hp[s * hrow + 2 * H + i] - rcomb[s * H * H + i]); if (d > ep) ep = d; }
+ }
+ CHECK(ok && ep < 2e-5 && epre < 5e-7 && ec < 2e-5 && eo < 2e-5 && em < 1e-6,
+ "mhc H %d D %d iters %d: ok %d split %.2e post/comb %.2e collapse %.2e write back %.2e mean %.2e",
+ H, D, iters, ok, epre, ep, ec, eo, em);
+ if (getenv("VKC_TEST_VERBOSE")) printf("mhc H %d D %d: split %.2e post/comb %.2e collapse %.2e write back %.2e mean %.2e\n", H, D, epre, ep, ec, eo, em);
+ /* the clamped SwiGLU, on values both sides of the limit */
+ int n = 333;
+ float *ga = fvec(n, 3.f), *ua = fvec(n, 3.f), *rr = malloc((size_t)n * 4);
+ for (int i = 0; i < n; i++) { float gg = ga[i] > lim ? lim : ga[i], uu = ua[i] < -lim ? -lim : (ua[i] > lim ? lim : ua[i]); rr[i] = gg / (1.f + expf(-gg)) * uu; }
+ VkcBuf *gab = up(ga, n), *uab = up(ua, n), *yb = vkc_buf((size_t)n * 4, VKC_DEV);
+ VkcMhc sw = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, n, 0.f, 0.f, lim};
+ ok = vkc_begin() && vkc_mhc(VKC_SWIGLU_CLAMP, gab, uab, NULL, NULL, yb, &sw) && vkc_submit(1);
+ float *yy = down(yb, 0, n);
+ double es = relerr(yy, rr, n, 1e-3);
+ CHECK(ok && es < 1e-6, "swiglu clamp: ok %d err %.2e", ok, es);
+ vkc_free(xb); vkc_free(bb); vkc_free(pb); vkc_free(mb); vkc_free(hb); vkc_free(cb); vkc_free(ob); vkc_free(eb);
+ vkc_free(gab); vkc_free(uab); vkc_free(yb); coli_vk_tensor_free(t);
+ free(x); free(fn); free(br); free(prm); free(rpre); free(rpost); free(rcomb); free(rout); free(rmean);
+ free(hp); free(col); free(out); free(mean); free(ga); free(ua); free(rr); free(yy);
+}
+
+/* k-pooling against sparse_index.h. Equal gates and no bias make each pooled key the
+ * exact mean of its members, so with quarter-step values and power-of-two scales the
+ * scores are exact on both sides and the selection must be the CPU's slot for slot;
+ * a second pass with random gates and bias checks the pooled keys to rounding. */
+static void test_kpool(int S, int pos_base, int IH, int ID, int pool, int topk, int tail) {
+ int T = pos_base + S, width = coli_sparse_index_width(topk, pool, tail), selrow = width + 1;
+ int npool = T / pool, scrow = npool + 1;
+ float *keys = malloc((size_t)T * ID * 4), *gates = calloc((size_t)T * ID, 4), *ape = calloc((size_t)pool * ID, 4);
+ float *q = malloc((size_t)S * IH * ID * 4), *hw = malloc((size_t)S * IH * 4);
+ for (int i = 0; i < T * ID; i++) keys[i] = ((int)(rnd() % 17) - 8) * 0.25f;
+ for (int i = 0; i < S * IH * ID; i++) q[i] = ((int)(rnd() % 17) - 8) * 0.25f;
+ for (int i = 0; i < S * IH; i++) hw[i] = ((int)(rnd() % 9) - 4) * 0.5f * sqrtf((float)IH);
+ for (int t = 0; t < T; t += 7) for (int i = 0; i < ID; i++) keys[t * ID + i] = keys[(t % 3) * ID + i]; /* equal pools: ties */
+ unsigned char *valid = malloc((size_t)T); memset(valid, 1, (size_t)T);
+ int *ref = malloc((size_t)S * width * sizeof(int));
+ float *hwd = malloc((size_t)S * IH * 4);
+ for (int i = 0; i < S * IH; i++) hwd[i] = hw[i] / sqrtf((float)IH);
+ coli_sparse_index_select_range(ref, q, keys, gates, hwd, ape, valid, T, IH, ID, pool, topk, tail, pos_base, T);
+ VkcBuf *kb = up(keys, (size_t)T * ID), *gb = up(gates, (size_t)T * ID), *ab = up(ape, (size_t)pool * ID);
+ VkcBuf *qb = up(q, (size_t)S * IH * ID), *wb = up(hw, (size_t)S * IH), *pk = vkc_buf((size_t)(npool + 1) * ID * 4, VKC_DEV);
+ VkcBuf *scb = vkc_buf((size_t)S * scrow * 4, VKC_DEV), *sb = vkc_buf((size_t)S * selrow * 4, VKC_DOWN);
+ VkcDsaPool kp = {npool, pool, 0, ID, 0, ID, 0, 0, 0, ID};
+ VkcDsaPick sp = {S, pos_base, IH, ID, topk, pool, 0, IH * ID, 0, IH, 0, tail, scrow, selrow, sqrtf((float)IH), 1.f / sqrtf((float)ID)};
+ int ok = vkc_begin() && vkc_dsa_pool_keys(kb, gb, ab, pk, &kp) && vkc_dsa_pool_select(qb, wb, pk, scb, sb, &sp) && vkc_submit(1);
+ const int *got = (const int *)vkc_ptr(sb);
+ int same = ok;
+ for (int s = 0; same && s < S; s++) {
+ same = got[s * selrow] == width;
+ for (int i = 0; same && i < width; i++) same = got[s * selrow + 1 + i] == ref[s * width + i];
+ if (!same && getenv("DSA_DEBUG")) { for (int i = 0; i < width; i++) printf(" %d/%d", got[s * selrow + 1 + i], ref[s * width + i]); printf("\n"); }
+ }
+ CHECK(same, "kpool S %d pos %d IH %d ID %d pool %d topk %d tail %d: ok %d, the selection differs", S, pos_base, IH, ID, pool, topk, tail, ok);
+ /* the pooled keys with real gates and bias, against sparse_index.h's arithmetic */
+ for (int i = 0; i < T * ID; i++) gates[i] = frnd() * 2.f;
+ for (int i = 0; i < pool * ID; i++) ape[i] = frnd() * 0.5f;
+ float *rk = malloc((size_t)npool * ID * 4);
+ for (int p = 0; p < npool; p++) for (int d = 0; d < ID; d++) {
+ float mx = -FLT_MAX, tot = 0, mix = 0;
+ for (int j = 0; j < pool; j++) { float l = gates[(p * pool + j) * ID + d] + ape[j * ID + d]; if (l > mx) mx = l; }
+ for (int j = 0; j < pool; j++) tot += expf(gates[(p * pool + j) * ID + d] + ape[j * ID + d] - mx);
+ for (int j = 0; j < pool; j++) mix += expf(gates[(p * pool + j) * ID + d] + ape[j * ID + d] - mx) / tot * keys[(p * pool + j) * ID + d];
+ rk[p * ID + d] = mix;
+ }
+ ok = vkc_begin() && vkc_write(gb, 0, gates, (size_t)T * ID * 4) && vkc_write(ab, 0, ape, (size_t)pool * ID * 4) &&
+ vkc_dsa_pool_keys(kb, gb, ab, pk, &kp) && vkc_submit(1);
+ float *gk = down(pk, 0, (size_t)npool * ID);
+ double e = relerr(gk, rk, (size_t)npool * ID, 1e-3);
+ CHECK(ok && e < 2e-6, "kpool keys pool %d ID %d: ok %d err %.2e", pool, ID, ok, e);
+ vkc_free(kb); vkc_free(gb); vkc_free(ab); vkc_free(qb); vkc_free(wb); vkc_free(pk); vkc_free(scb); vkc_free(sb);
+ free(keys); free(gates); free(ape); free(q); free(hw); free(valid); free(ref); free(hwd); free(rk); free(gk);
+}
+
/* ---- frames: many ops, frames in flight, ordering ---------------------------------------- */
static void test_frames(void) {
int n = 1000;
@@ -895,6 +1081,17 @@ int main(int argc, char **argv) {
test_dsa(3, 5, 2, 16, 64, 1, 0); test_dsa(2, 700, 3, 64, 100, 0, 0); test_dsa(1, 9, 64, 64, 4, 0, 0);
test_dsa(2, 900, 2, 16, 50, 0, 1);
printf("dsa done\n");
+ test_kpool(3, 20, 4, 16, 2, 8, 1); test_kpool(2, 9, 1, 16, 4, 8, 1); test_kpool(4, 300, 4, 64, 4, 64, 1);
+ test_kpool(1, 2, 4, 16, 2, 4, 0); test_kpool(2, 30, 16, 16, 2, 6, 1);
+ printf("kpool done\n");
+ }
+ if (!vkc_kda_ready() || !vkc_mhc_ready()) { fails++; printf("FAIL: the KDA or mHC shaders did not load\n"); }
+ else {
+ test_kda(3, 16, 4, 1.f); test_kda(2, 32, 4, 1.f); test_kda(4, 128, 3, 1.f); test_kda(1, 64, 2, 1.f);
+ test_kda(2, 16, 4, 0.002f);
+ printf("kda done\n");
+ test_mhc(2, 64, 3); test_mhc(4, 96, 20); test_mhc(3, 40, 1);
+ printf("mhc done\n");
}
VkcStats st; vkc_stats(&st);
printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);
diff --git a/c/vk_chain.c b/c/vk_chain.c
index 03adb438..ee112a1d 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -36,13 +36,16 @@ static const char *const pipe_file[P_NPIPE] = {
/* COLI_VK_CHAIN_PROF=1: device time per kind of op (timestamps after every op; ops
* that run side by side between two barriers share the time unevenly) */
enum { PK_GEMV8, PK_GEMV, PK_GEMM, PK_NORM, PK_ROPE, PK_ATTN, PK_DNCONV, PK_DNREC, PK_EW, PK_QSA, PK_PLE, PK_COPY,
- PK_MLA, PK_MLAW, PK_DSA, PK_N };
+ PK_MLA, PK_MLAW, PK_DSA, PK_KDA, PK_MHC, PK_N };
static const char *const pk_name[PK_N] = {"GEMV int8", "GEMV other", "tiled GEMM", "norm", "rope", "attention",
"dn conv", "dn recurrence", "element-wise", "qsa", "ple", "copy",
- "mla", "mla weights", "dsa"};
-/* the MLA pipelines: optional (an engine without MLA layers needs none of them) */
-enum { PM_MLA, PM_HGEMV, PM_DSA, PM_N };
-static const char *const mla_file[PM_N] = {"chain_mla.spv", "chain_hgemv.spv", "chain_dsa.spv"};
+ "mla", "mla weights", "dsa", "kda", "mhc"};
+/* the MLA, KDA and mHC pipelines: optional (an engine without such layers needs none
+ * of them, and a missing one turns off only its own ops) */
+enum { PM_MLA, PM_HGEMV, PM_DSA, PM_KDA, PM_MHC, PM_N };
+static const char *const mla_file[PM_N] = {"chain_mla.spv", "chain_hgemv.spv", "chain_dsa.spv", "chain_kda.spv",
+ "chain_mhc.spv"};
+#define VKC_KDA_MAX 8
/* ---- memory: blocks per kind, buffers bound at offsets inside them -------------- */
typedef struct { VkDeviceMemory mem; uint8_t *map; } VkcBlock;
@@ -80,6 +83,7 @@ static struct {
VkPipeline dnrec[VKC_DNREC_MAX]; int dnrec_kd[VKC_DNREC_MAX], ndnrec;
VkShaderModule mod_gemv4; VkPipeline gemv4; int gemv4_xs; /* chain_gemv.comp: the vectorized decode GEMV */
VkShaderModule mmod[PM_N]; VkPipeline mpipe[PM_N]; int mla_ok; /* chain_mla, chain_hgemv, chain_dsa */
+ VkPipeline kdarec[VKC_KDA_MAX]; int kdarec_kd[VKC_KDA_MAX], nkdarec; /* chain_kda's recurrence per key dim */
VkCommandPool cpool;
VkcFrame fr[VKC_FRAMES];
int cur; /* the open frame, -1 none */
@@ -281,8 +285,7 @@ int vkc_init(void) {
if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;
}
}
- /* the MLA shaders, optional: without them only the MLA ops decline */
- K.mla_ok = 1;
+ /* the MLA, KDA and mHC shaders, optional: without one only its ops decline */
for (int i = 0; i < PM_N; i++) {
char path[1200];
const char *sl = strrchr(K.core.spv_path, '/');
@@ -293,9 +296,9 @@ int vkc_init(void) {
f = fopen(path, "rb");
}
if (f) fclose(f);
- if (!f || !(K.mmod[i] = load_module(K.core.spv_path, mla_file[i])) || !(K.mpipe[i] = make_pipe(K.mmod[i], NULL)))
- K.mla_ok = 0;
+ if (f && (K.mmod[i] = load_module(K.core.spv_path, mla_file[i]))) K.mpipe[i] = make_pipe(K.mmod[i], NULL);
}
+ K.mla_ok = K.mpipe[PM_MLA] && K.mpipe[PM_HGEMV] && K.mpipe[PM_DSA];
/* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */
K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;
for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {
@@ -835,6 +838,70 @@ int vkc_dsa_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *sc, VkcBuf *sel
return record(K.mpipe[PM_DSA], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
}
+/* k-pooling (chain_dsa.comp modes 1 and 2) */
+int vkc_dsa_pool_keys(VkcBuf *keys, VkcBuf *gates, VkcBuf *prm, VkcBuf *pk, const VkcDsaPool *p) {
+ K.kind = PK_DSA;
+ if (!K.mla_ok || p->np < 0 || p->pool < 1 || p->p0 < 0 || p->ID < 1) return 0;
+ if (p->np == 0) return open_frame() && !K.lost;
+ struct { int mode; VkcDsaPool b; } pc = {1, *p};
+ VkcBind bd[8] = {B(NULL, 0), B(NULL, 0), B(keys, 0), B(NULL, 0), B(NULL, 0), B(pk, 1), B(gates, 0), B(prm, 0)};
+ return record(K.mpipe[PM_DSA], bd, 8, &pc, sizeof pc, (uint32_t)p->np, 1, 1);
+}
+int vkc_dsa_pool_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *pk, VkcBuf *sc, VkcBuf *sel, const VkcDsaPick *p) {
+ K.kind = PK_DSA;
+ if (!K.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||
+ p->pool < 1 || p->topk < p->pool || p->topk % p->pool || p->topk / p->pool > 1024 || p->pos_base < 0 ||
+ p->sc_row < (p->pos_base + p->S) / p->pool || p->sel_row < 1 + p->topk + (p->tail ? p->pool - 1 : 0)) return 0;
+ struct { int mode; VkcDsaPick b; } pc = {2, *p};
+ VkcBind bd[6] = {B(iq, 0), B(hw, 0), B(NULL, 0), B(sc, 1), B(sel, 1), B(pk, 0)};
+ return record(K.mpipe[PM_DSA], bd, 6, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
+}
+
+/* ---- Kimi Delta Attention (chain_kda.comp) ------------------------------------------ */
+int vkc_kda_ready(void) { return vkc_ready() && K.mpipe[PM_KDA]; }
+int vkc_kda_conv(VkcBuf *in, VkcBuf *w, VkcBuf *win, VkcBuf *out, const VkcKdaConv *p) {
+ K.kind = PK_KDA;
+ if (!K.mpipe[PM_KDA] || p->K < 1 || p->K > 8 || p->P < 1 || p->C < 1 || p->S < 1) return 0;
+ struct { int mode; VkcKdaConv b; } pc = {0, *p};
+ VkcBind bd[4] = {B(in, 0), B(w, 0), B(win, 1), B(out, 1)};
+ uint32_t gx, gy; grid(((uint64_t)p->C + 127) / 128, &gx, &gy);
+ return record(K.mpipe[PM_KDA], bd, 4, &pc, sizeof pc, gx, gy, 1);
+}
+static VkPipeline kda_pipe(int KD) {
+ for (int i = 0; i < K.nkdarec; i++) if (K.kdarec_kd[i] == KD) return K.kdarec[i];
+ if (K.nkdarec == VKC_KDA_MAX || !K.mmod[PM_KDA]) return VK_NULL_HANDLE;
+ int32_t kd = KD;
+ VkSpecializationMapEntry me = {0, 0, 4};
+ VkSpecializationInfo si = {1, &me, 4, &kd};
+ VkPipeline p = make_pipe(K.mmod[PM_KDA], &si);
+ if (!p) return VK_NULL_HANDLE;
+ K.kdarec[K.nkdarec] = p; K.kdarec_kd[K.nkdarec++] = KD;
+ return p;
+}
+int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y, const VkcKdaRec *p) {
+ K.kind = PK_KDA;
+ if (!K.mpipe[PM_KDA] || KD < 1 || KD > 256 || p->VD < 1 || p->VD > 128 || p->H < 1 || p->S < 1) return 0;
+ VkPipeline pipe = kda_pipe(KD);
+ if (!pipe) return 0;
+ struct { int mode; VkcKdaRec b; } pc = {1, *p};
+ VkcBind bd[7] = {B(m, 0), B(prm, 0), B(st, 1), B(y, 1), B(f, 0), B(b, 0), B(g, 0)};
+ return record(pipe, bd, 7, &pc, sizeof pc, (uint32_t)p->H, 1, 1);
+}
+
+/* ---- manifold-constrained hyper-connections (chain_mhc.comp) ------------------------ */
+int vkc_mhc_ready(void) { return vkc_ready() && K.mpipe[PM_MHC]; }
+int vkc_mhc(int mode, VkcBuf *x, VkcBuf *m, VkcBuf *hp, VkcBuf *prm, VkcBuf *y, const VkcMhc *p) {
+ K.kind = PK_MHC;
+ if (!K.mpipe[PM_MHC] || mode < 0 || mode > 4 || p->S < 0 || (mode != 4 && (p->H < 1 || p->H > 8 || p->D < 1))) return 0;
+ if (mode == 0 && p->iters < 1) return 0;
+ struct { int mode; VkcMhc b; } pc = {mode, *p};
+ VkcBind bd[5] = {B(x, 0), B(m, 0), B(hp, mode == 0), B(prm, 0), B(y, mode != 0)};
+ uint64_t n = mode == 4 ? (uint64_t)p->n : mode == 2 ? (uint64_t)p->S * p->H * p->D : (uint64_t)p->S * p->D;
+ if (mode == 0) return p->S == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
+ uint32_t gx, gy; grid((n + 255) / 256, &gx, &gy);
+ return n == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, gx, gy, 1);
+}
+
int vkc_mla_scratch(VkcMlaScratch *s, const VkcMla *m, int rows) {
if (s->rows >= rows && s->q) return 1;
size_t r = (size_t)rows, H = (size_t)m->H;
@@ -951,6 +1018,7 @@ void vkc_shutdown(void) {
if (K.pipe[i]) vkDestroyPipeline(K.dev, K.pipe[i], NULL);
if (K.mod[i]) vkDestroyShaderModule(K.dev, K.mod[i], NULL);
}
+ for (int i = 0; i < K.nkdarec; i++) vkDestroyPipeline(K.dev, K.kdarec[i], NULL);
for (int i = 0; i < PM_N; i++) {
if (K.mpipe[i]) vkDestroyPipeline(K.dev, K.mpipe[i], NULL);
if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);
diff --git a/c/vk_chain.h b/c/vk_chain.h
index 62a04692..784b3d2b 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -26,7 +26,9 @@
* at its top: chain_norm, chain_rope, chain_attn, chain_dnconv, chain_dnrec,
* chain_ew, chain_qsa, chain_ple. Offsets and strides are in floats.
* - multi-head latent attention: the MLA ops and the layer op below (chain_mla,
- * chain_hgemv, chain_dsa), loaded beside the others but optional.
+ * chain_hgemv, chain_dsa, its k-pooled modes included), Kimi Delta Attention
+ * (chain_kda) and manifold-constrained hyper-connections (chain_mhc): loaded beside
+ * the others but optional.
*
* Threading: the engine thread only (the main queue is the backend's, used from the
* same thread by coli_vk_matmul; the expert tier submits on its own queue).
@@ -193,6 +195,19 @@ typedef struct { int S, pos_base, IH, ID, topk, force, q_off, q_row, w_off, w_ro
float qscale, wscale; } VkcDsa;
int vkc_dsa_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *sc, VkcBuf *sel, const VkcDsa *p);
+/* chain_dsa.comp with k-pooling (GLM-5.3): mode 1, the pooled keys of the np complete
+ * pools from p0 on, each the per-channel softmax mixture (gate logits plus ape, at
+ * ape_off in prm, [pool][ID]) of its members' keys, into pk[pk_off + p*ID]; mode 2, row
+ * s's selection: the complete pools visible to position pos_base + s scored
+ * sum_h [dot > 0] (w_h / wdiv) * dot * scale (dot = q_h . pk[p]), the top topk/pool in
+ * rank order filling `pool` slots each, the incomplete tail from slot topk (tail), -1
+ * elsewhere; sel[s*sel_row] = topk (+ pool - 1 with the tail). sc: [S][sc_row] scores. */
+typedef struct { int np, pool, p0, ID, g_off, g_row, ape_off, pk_off, k_off, k_row; } VkcDsaPool;
+int vkc_dsa_pool_keys(VkcBuf *keys, VkcBuf *gates, VkcBuf *prm, VkcBuf *pk, const VkcDsaPool *p);
+typedef struct { int S, pos_base, IH, ID, topk, pool, q_off, q_row, w_off, w_row, pk_off, tail, sc_row, sel_row;
+ float wdiv, scale; } VkcDsaPick;
+int vkc_dsa_pool_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *pk, VkcBuf *sc, VkcBuf *sel, const VkcDsaPick *p);
+
/* The layer op. Tensors are the resident copies the per-matrix path uploads
* (coli_vk_tensor_ensure), in any format the backend holds. */
typedef struct {
@@ -231,6 +246,46 @@ int vkc_mla_attn(const VkcMla *m, VkcMlaScratch *s, int S, int pos_base, int kv_
int vkc_mla(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int S, int pos_base, int kv_start,
VkcBuf *cs, VkcMlaCache *c, VkcBuf *out, size_t out_off);
+/* ---- Kimi Delta Attention (chain_kda.comp) --------------------------------------------
+ * delta_attention.h's coli_kda_step for S rows in order, the state and the short
+ * convolution's window on the device (GLM-5.3's linear layers; Kimi K3's KDA).
+ * vkc_kda_conv: C = 3P channels (q, k, v), the input of part c / P at in_off + part*in_part
+ * + s*in_row + c % P, the window [C][K] at win_off (the last K inputs, oldest first, as
+ * the CPU keeps it), taps [C][K] at w_off; out[out_off + s*out_row + c] = silu(sum).
+ * vkc_kda_rec: one workgroup per head, the key dim KD a specialization (<= 256), VD <= 128;
+ * m = the convolution's output rows (q at h*KD, k at P + h*KD, v at 2P + h*VD); f, b, g
+ * the raw decay, beta and output-gate projections; prm at prm_off: A_log[H], dt[P],
+ * norm[VD]; alpha = exp(lb * sigmoid(exp(A_log) * (f + dt))), beta = sigmoid(b), q and k
+ * l2-normalized with neps inside the root; y = RMSNorm(o, eps) * norm * sigmoid(g).
+ * The state [H][KD][VD] at st_off. */
+int vkc_kda_ready(void);
+typedef struct { int S, C, K, P, in_off, in_row, in_part, out_off, out_row, w_off, win_off; } VkcKdaConv;
+int vkc_kda_conv(VkcBuf *in, VkcBuf *w, VkcBuf *win, VkcBuf *out, const VkcKdaConv *p);
+typedef struct { int S, H, VD, P, m_off, m_row, f_off, f_row, b_off, b_row, g_off, g_row, y_off, y_row, st_off, prm_off;
+ float lb, neps, eps; } VkcKdaRec;
+int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y, const VkcKdaRec *p);
+
+/* ---- manifold-constrained hyper-connections (chain_mhc.comp) ------------------------
+ * hyper_connections.h for S rows of H <= 8 streams of D floats ([S][H*D] at x_off,
+ * x_row apart), GLM-5.3's (and DeepSeek V4's) residual:
+ * VKC_MHC_SPLIT m: the raw mix products hc_fn . x ([S][(2+H)*H], from a matmul),
+ * scaled by the rows' 1/rms (eps); pre, post and the Sinkhorn-projected
+ * comb (iters, hc_eps; scale[3] and base at prm_off in prm) into
+ * hp[hp_off + s*hp_row]: H pre, H post, H*H comb
+ * VKC_MHC_COLLAPSE y[s][d] = sum_i pre[i] x[s][i*D + d]
+ * VKC_MHC_POST y[s][j*D + d] = sum_i comb[i*H + j] x[s][i*D + d] + post[j] m[s][d]
+ * VKC_MHC_MEAN y[s][d] = (sum_i x[s][i*D + d]) / H
+ * VKC_SWIGLU_CLAMP y[i] = silu(min(x[i], lim)) * clamp(m[i], -lim, lim), i < n */
+#define VKC_MHC_SPLIT 0
+#define VKC_MHC_COLLAPSE 1
+#define VKC_MHC_POST 2
+#define VKC_MHC_MEAN 3
+#define VKC_SWIGLU_CLAMP 4
+int vkc_mhc_ready(void);
+typedef struct { int S, H, D, iters, x_off, x_row, m_off, m_row, hp_off, hp_row, y_off, y_row, prm_off, n;
+ float eps, hc_eps, lim; } VkcMhc;
+int vkc_mhc(int mode, VkcBuf *x, VkcBuf *m, VkcBuf *hp, VkcBuf *prm, VkcBuf *y, const VkcMhc *p);
+
/* counters, for the engines' [VK] lines */
typedef struct {
unsigned long long frames, waits, ops, matmuls, gemms, barriers, bytes_up, bytes_down;
diff --git a/docs/vulkan.md b/docs/vulkan.md
index a651aeae..30bd2e67 100644
--- a/docs/vulkan.md
+++ b/docs/vulkan.md
@@ -885,6 +885,15 @@ records what reads the projections (a DSA indexer reads the normalized q latent
leaves in its scratch). Limits: K up to 1024, R up to 128, Q up to 1024 where kv_b's key
rows are read transposed, an indexer of up to 64 heads and 4096 query floats.
+GLM-5.3 Flash adds three more things, as ops of their own in `vk_chain.h`, which the
+DeepSeek V4 and Kimi K3 chains can take as they are:
+
+| Op | Shader | What it does |
+|---|---|---|
+| `vkc_dsa_pool_keys`, `vkc_dsa_pool_select` | `chain_dsa` (modes 1, 2) | the k-pooled indexer (`sparse_index.h`): a pool's key, the per-channel softmax mixture of its members' keys under their gate logits plus a position bias, computed once when a step completes the pool; each row's top pools in rank order (score, then the lower pool), their positions, the incomplete tail, -1 in the unused slots |
+| `vkc_kda_conv`, `vkc_kda_rec` | `chain_kda` | Kimi Delta Attention (`delta_attention.h`): the short convolution with its window, the gated delta rule with a decay per key row and the state on the device, GLM-5.3's output RMSNorm and sigmoid gate; the l2 sums and the norm's sum in the CPU's order |
+| `vkc_mhc` | `chain_mhc` | manifold-constrained hyper-connections (`hyper_connections.h`): the mix logits' split with Sinkhorn, the collapse, the write back, the plain mean of the last collapse; and a clamped SwiGLU |
+
`make vk-chain-check VK=1` runs them against double-precision references of colibri.c's
absorbed attention: every weight format both ways through the per-head blocks, RoPE in
both styles in place and into a cache row, LayerNorm with and without a bias, and the
@@ -894,7 +903,10 @@ context from a nonzero start, GLM-5.2's head shape, a latent of 1024). Measured
Lavapipe and on an Intel Iris Xe (Mesa's Dozen): the layer's output within 5e-7 of its
largest value and the new cache rows within 7e-7 (the test's bound is 2e-5). The
indexer's selection is checked bit for bit, ties included, on scores that are exact in
-float on both sides.
+float on both sides. The k-pooled selection is checked slot for slot against `sparse_index.h` the same
+way, the KDA layer against `delta_attention.h` over two submissions (the state and the
+window carried between them, inputs small enough that the l2 eps counts), the mHC ops
+against `hyper_connections.h`; each within 3e-7 on Lavapipe and on the Iris Xe.
## Adding an engine to the tier