Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: KDA, hyper-connections and the k-pooled indexer on the chain
JustVugg6052a4c2341c
diff --git a/c/Makefile b/c/Makefileindex 73b8305c..a60020f5 100644--- a/c/Makefile+++ b/c/Makefile@@ -719,6 +719,8 @@ VK_SPV += shaders/chain_norm.spv shaders/chain_rope.spv shaders/chain_attn.spv# the MLA ops (vk_chain.h): attention core, RoPE and LayerNorm rows, per-head weight# blocks, the DSA indexer's selectionVK_SPV += shaders/chain_mla.spv shaders/chain_hgemv.spv shaders/chain_dsa.spv+# Kimi Delta Attention and manifold-constrained hyper-connections (vk_chain.h)+VK_SPV += shaders/chain_kda.spv shaders/chain_mhc.spvendif# Optional AMD XDNA2 (Ryzen AI NPU) lane, Windows. Same opt-in shape as VK=1:diff --git a/c/shaders/chain_dsa.comp b/c/shaders/chain_dsa.compindex fa55bfdf..90e11d6c 100644--- a/c/shaders/chain_dsa.comp+++ b/c/shaders/chain_dsa.comp@@ -15,6 +15,23 @@// integer keys (32 counting passes); -0 counts as +0, as the float compare does.// sel[s*sel_row] = keep, then the positions. Scores live in sc[s*sc_row + t].// Limits: IH*ID <= 4096.+//+// With k-pooling (GLM-5.3, sparse_index.h's coli_sparse_index_select_range):+// mode 1, the pooled keys: one workgroup per pool p = p0 + WorkGroupID.x, complete (its+// `pool` positions all written). Per channel d, the pool's key is the softmax mixture of+// its members' keys under their gate logits plus the position bias ape[j][d]:+// pk[p][d] = sum_j softmax_j(gate[p*pool + j][d] + ape[j][d]) * key[p*pool + j][d]+// in the CPU's order. A pool's key depends on its own rows only: the engine computes+// each once, when a step completes it, and keeps it (the CPU recomputes it every step).+// mode 2, the selection: one workgroup per query row s, position q = pos_base + s. The+// complete pools visible to it (p < (q+1)/pool) score+// sum_h [dot_h > 0] (w_h / wdiv) * dot_h * scale, dot_h = q_h . pk[p]+// and the top topk/pool of them, in rank order (score, then the lower pool first), give+// their positions: rank r's pool fills slots [r*pool, r*pool + pool); the incomplete tail+// (the positions after the last complete pool, up to pool - 1 of them) fills the slots+// from topk on when `tail`; every other slot is -1 (skipped by the attention core).+// sel[s*sel_row] = topk (+ pool - 1 with the tail), then the slots. Limits: IH*ID <=+// 4096, topk/pool <= 1024.#extension GL_KHR_shader_subgroup_arithmetic : requirelayout(local_size_x = 256) in;@@ -23,16 +40,49 @@ layout(std430, binding = 1) readonly buffer HW { float hw[]; };layout(std430, binding = 2) readonly buffer KC { float kc[]; };layout(std430, binding = 3) buffer SC { float sc[]; };layout(std430, binding = 4) buffer SL { int sel[]; };+layout(std430, binding = 5) buffer PK { float pk[]; }; // 1: written; 2: read+layout(std430, binding = 6) readonly buffer GT { float gt[]; }; // 1: gate logits [cap][ID]+layout(std430, binding = 7) readonly buffer AP { float prm[]; }; // 1: ape [pool][ID]-layout(push_constant) uniform P {- int mode, S, pos_base, IH, ID, topk, force;- int q_off, q_row, w_off, w_row, k_off, k_row, sc_row, sel_row;- float qscale, wscale;-} p;+// one block of 32 words, read per mode (vk_chain.h: VkcDsa, VkcDsaPool, VkcDsaPick)+layout(push_constant) uniform P { int v[32]; } p;+#define S_ p.v[1]+#define POS_BASE p.v[2]+#define IH_ p.v[3]+#define ID_ p.v[4]+#define TOPK p.v[5]+#define FORCE p.v[6]+#define Q_OFF p.v[7]+#define Q_ROW p.v[8]+#define W_OFF p.v[9]+#define W_ROW p.v[10]+#define K_OFF p.v[11]+#define K_ROW p.v[12]+#define SC_ROW p.v[13]+#define SEL_ROW p.v[14]+#define QSCALE intBitsToFloat(p.v[15])+#define WSCALE intBitsToFloat(p.v[16])+// mode 1: ID_ at 4, then+#define POOL1 p.v[2]+#define P0 p.v[3]+#define G_OFF p.v[5]+#define G_ROW p.v[6]+#define APE_OFF p.v[7]+#define PK_OFF1 p.v[8]+#define K_OFF1 p.v[9]+#define K_ROW1 p.v[10]+// mode 2: S_, POS_BASE, IH_, ID_, TOPK, Q_OFF.., W_.., SC_ROW, SEL_ROW as mode 0, then+#define POOL2 p.v[6]+#define PK_OFF2 p.v[11]+#define TAIL p.v[12]+#define WDIV intBitsToFloat(p.v[15])+#define PSCALE intBitsToFloat(p.v[16])shared float qs[4096];shared float ws[64];shared uint cnt[256];+shared int plist[1024];+shared uint pkey[1024];uint okey(float f) {uint u = floatBitsToUint(f);@@ -67,27 +117,27 @@ uint wscan(uint v, int tid) {return incl - v;}-void main() {+void token_select() {int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);- int nk = p.pos_base + s + 1;- int lb = s * p.sel_row;- if (nk <= p.topk && p.force == 0) { if (tid == 0) sel[lb] = -1; return; }- int keep = min(nk, p.topk);- int nq = p.IH * p.ID, qb = p.q_off + s * p.q_row;+ int nk = POS_BASE + s + 1;+ int lb = s * SEL_ROW;+ if (nk <= TOPK && FORCE == 0) { if (tid == 0) sel[lb] = -1; return; }+ int keep = min(nk, TOPK);+ int nq = IH_ * ID_, qb = Q_OFF + s * Q_ROW;for (int i = tid; i < nq; i += 256) qs[i] = iq[qb + i];- for (int h = tid; h < p.IH; h += 256) ws[h] = hw[p.w_off + s * p.w_row + h];+ for (int h = tid; h < IH_; h += 256) ws[h] = hw[W_OFF + s * W_ROW + h];barrier();- int sb = s * p.sc_row;+ int sb = s * SC_ROW;for (int t = tid; t < nk; t += 256) {- int kb = p.k_off + t * p.k_row;+ int kb = K_OFF + t * K_ROW;float a = 0.0;- for (int h = 0; h < p.IH; h++) {+ for (int h = 0; h < IH_; h++) {float d = 0.0;- for (int i = 0; i < p.ID; i++) d += qs[h * p.ID + i] * kc[kb + i];- d *= p.qscale;+ for (int i = 0; i < ID_; i++) d += qs[h * ID_ + i] * kc[kb + i];+ d *= QSCALE;if (d > 0.0) a += ws[h] * d;}- sc[sb + t] = a * p.wscale;+ sc[sb + t] = a * WSCALE;}memoryBarrierBuffer();barrier();@@ -110,3 +160,90 @@ void main() {for (int t = c0; t < c1 && o < uint(keep); t++) if (okey(sc[sb + t]) == K) sel[lb + 1 + int(o++)] = t;if (tid == 0) sel[lb] = keep;}++// the keep-th largest key among n scores at sc[sb..], bit by bit from the top+uint kth_key(int sb, int n, int keep, int tid) {+ uint K = 0u;+ for (int b = 31; b >= 0; b--) {+ uint cand = K | (1u << uint(b));+ uint c = 0u;+ for (int t = tid; t < n; t += 256) if (okey(sc[sb + t]) >= cand) c++;+ if (wsum(c, tid) >= uint(keep)) K = cand;+ }+ return K;+}++void pool_keys() {+ int pp = P0 + int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);+ int start = pp * POOL1;+ for (int d = tid; d < ID_; d += 256) {+ float mxv = -3.402823466e38;+ for (int j = 0; j < POOL1; j++) {+ float lg = gt[G_OFF + (start + j) * G_ROW + d] + prm[APE_OFF + j * ID_ + d];+ if (lg > mxv) mxv = lg;+ }+ float total = 0.0;+ for (int j = 0; j < POOL1; j++)+ total += exp(gt[G_OFF + (start + j) * G_ROW + d] + prm[APE_OFF + j * ID_ + d] - mxv);+ float mixed = 0.0;+ for (int j = 0; j < POOL1; j++) {+ float wgt = exp(gt[G_OFF + (start + j) * G_ROW + d] + prm[APE_OFF + j * ID_ + d] - mxv) / total;+ mixed += wgt * kc[K_OFF1 + (start + j) * K_ROW1 + d];+ }+ pk[PK_OFF1 + pp * ID_ + d] = mixed;+ }+}++void pool_select() {+ int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);+ int q = POS_BASE + s, pool = POOL2;+ int nvis = (q + 1) / pool, wanted = TOPK / pool, keep = min(wanted, nvis);+ int width = TAIL != 0 ? TOPK + pool - 1 : TOPK;+ int lb = s * SEL_ROW;+ for (int i = tid; i < width; i += 256) sel[lb + 1 + i] = -1;+ int nq = IH_ * ID_, qb = Q_OFF + s * Q_ROW;+ for (int i = tid; i < nq; i += 256) qs[i] = iq[qb + i];+ for (int h = tid; h < IH_; h += 256) ws[h] = hw[W_OFF + s * W_ROW + h] / WDIV;+ barrier();+ int sb = s * SC_ROW;+ for (int pp = tid; pp < nvis; pp += 256) {+ float score = 0.0;+ for (int h = 0; h < IH_; h++) {+ float dot = 0.0;+ for (int d = 0; d < ID_; d++) dot += qs[h * ID_ + d] * pk[PK_OFF2 + pp * ID_ + d];+ if (dot > 0.0) score += ws[h] * dot * PSCALE;+ }+ sc[sb + pp] = score;+ }+ memoryBarrierBuffer();+ barrier();+ if (keep > 0) {+ uint K = kth_key(sb, nvis, keep, tid);+ int chunk = (nvis + 255) / 256, c0 = tid * chunk, c1 = min(nvis, c0 + chunk);+ uint above = 0u, at = 0u;+ for (int t = c0; t < c1; t++) { uint k = okey(sc[sb + t]); above += k > K ? 1u : 0u; at += k == K ? 1u : 0u; }+ uint n1 = wsum(above, tid);+ uint o = wscan(above, tid);+ for (int t = c0; t < c1; t++) { uint k = okey(sc[sb + t]); if (k > K) { plist[o] = t; pkey[o] = k; o++; } }+ o = n1 + wscan(at, tid);+ for (int t = c0; t < c1 && o < uint(keep); t++) { uint k = okey(sc[sb + t]); if (k == K) { plist[o] = t; pkey[o] = k; o++; } }+ barrier();+ // rank order: the higher score first, then the lower pool+ for (int i = tid; i < keep; i += 256) {+ uint ki = pkey[i]; int pi = plist[i], r = 0;+ for (int j = 0; j < keep; j++) { uint kj = pkey[j]; if (kj > ki || (kj == ki && plist[j] < pi)) r++; }+ for (int j = 0; j < pool; j++) sel[lb + 1 + r * pool + j] = pi * pool + j;+ }+ }+ if (TAIL != 0 && tid == 0) {+ int visible = q + 1, tail = visible % pool, ts = visible - tail;+ for (int j = 0; j < tail && j < pool - 1; j++) sel[lb + 1 + TOPK + j] = ts + j;+ }+ if (tid == 0) sel[lb] = width;+}++void main() {+ if (p.v[0] == 0) token_select();+ else if (p.v[0] == 1) pool_keys();+ else pool_select();+}diff --git a/c/shaders/chain_dsa.spv b/c/shaders/chain_dsa.spvindex 53c70bf2..73bd037f 100644Binary files a/c/shaders/chain_dsa.spv and b/c/shaders/chain_dsa.spv differdiff --git a/c/shaders/chain_kda.comp b/c/shaders/chain_kda.compnew file mode 100644index 00000000..a30505ce--- /dev/null+++ b/c/shaders/chain_kda.comp@@ -0,0 +1,150 @@+#version 450+#extension GL_EXT_control_flow_attributes : require+// Kimi Delta Attention on the dense chain (vk_chain.c, VKC_P_KDA): the linear-attention+// layers of GLM-5.3 (and Kimi K3), delta_attention.h's coli_kda_step for S rows in+// order, the state and the convolution window on the device across steps.+//+// mode 0, the short convolution: one thread per channel c of the C = 3P channels (q, k,+// v, P each), the rows in order. The channel's window holds its last K inputs, oldest+// first, as the CPU keeps it ([C][K]); per row it drops the oldest, takes the input+// (part c / P at in_off + part*in_part + s*in_row + c % P) and the output is+// silu(sum_tap w[c][tap] * window[tap]), the taps in order, at out_off + s*out_row + c.+//+// mode 1, the recurrence and the output norm: one workgroup per head h, thread v owns+// the state's column S[h][:, v] in registers (KD, the key dim, is a specialization+// constant; the value dim VD <= 128). Per row, from the convolution's output m (q at+// h*KD, k at P + h*KD, v at 2P + h*VD) and the raw projections:+// q, k l2-normalized with eps inside the root, q also times 1/sqrt(KD)+// alpha = exp(lb * sigmoid(exp(A_log[h]) * (f[s][h*KD+k] + dt[h*KD+k]))), per key row+// beta = sigmoid(b[s][h])+// S[k] *= alpha[k]; mem = sum_k k[k] S[k]; S[k] += k[k] (v - mem) beta; o = sum_k q[k] S[k]+// y[s][h*VD+v] = o * rsqrt(mean(o^2) + eps) * norm[v] * sigmoid(g[s][h*VD+v])+// Each column runs the CPU's products and sums in its order; the two l2 sums and the+// output norm's sum are made by one thread in order, as on the CPU.+// Parameters in prm at prm_off: A_log[H], dt[P], norm[VD].+layout(local_size_x = 128) in;+layout(constant_id = 0) const int KD = 128;++layout(std430, binding = 0) readonly buffer IN { float xin[]; }; // 0: q|k|v projections; 1: conv output+layout(std430, binding = 1) readonly buffer WB { float w[]; }; // 0: taps; 1: prm+layout(std430, binding = 2) buffer RB { float st[]; }; // 0: window; 1: state [H][KD][VD]+layout(std430, binding = 3) writeonly buffer OB { float o[]; }; // 0: conv output; 1: y+layout(std430, binding = 4) readonly buffer FB { float fr[]; }; // 1: decay projection f+layout(std430, binding = 5) readonly buffer BB { float br[]; }; // 1: beta projection b+layout(std430, binding = 6) readonly buffer GB { float gr[]; }; // 1: output gate projection g++layout(push_constant) uniform P { int v[32]; } p;+// mode 0+#define S_ p.v[1]+#define C_ p.v[2]+#define KW p.v[3]+#define PW p.v[4]+#define IN_OFF p.v[5]+#define IN_ROW p.v[6]+#define IN_PART p.v[7]+#define OUT_OFF p.v[8]+#define OUT_ROW p.v[9]+#define W_OFF p.v[10]+#define WIN_OFF p.v[11]+// mode 1+#define H_ p.v[2]+#define VD p.v[3]+#define M_OFF p.v[5]+#define M_ROW p.v[6]+#define F_OFF p.v[7]+#define F_ROW p.v[8]+#define B_OFF p.v[9]+#define B_ROW p.v[10]+#define G_OFF p.v[11]+#define G_ROW p.v[12]+#define Y_OFF p.v[13]+#define Y_ROW p.v[14]+#define ST_OFF p.v[15]+#define PRM_OFF p.v[16]+#define LB intBitsToFloat(p.v[17])+#define NEPS intBitsToFloat(p.v[18])+#define EPS intBitsToFloat(p.v[19])++shared float sq[KD];+shared float sk[KD];+shared float al[KD];+shared float ov[128];+shared float scal[2];++float sig(float x) { // hyper_connections.h's and glm53.c's two-sided form+ if (x >= 0.0) return 1.0 / (1.0 + exp(-x));+ float e = exp(x);+ return e / (1.0 + e);+}++void conv() {+ int c = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 128u);+ if (c >= C_) return;+ int part = c / PW, j = c - part * PW;+ float hist[8];+ int wb = WIN_OFF + c * KW, tb = W_OFF + c * KW;+ for (int k = 0; k < 8; k++) hist[k] = k < KW ? st[wb + k] : 0.0;+ for (int s = 0; s < S_; s++) {+ for (int k = 0; k < 7; k++) if (k + 1 < KW) hist[k] = hist[k + 1];+ float cur = xin[IN_OFF + part * IN_PART + s * IN_ROW + j];+ for (int k = 0; k < 8; k++) if (k == KW - 1) hist[k] = cur;+ float sum = 0.0;+ for (int k = 0; k < 8; k++) if (k < KW) sum += w[tb + k] * hist[k];+ o[OUT_OFF + s * OUT_ROW + c] = sum / (1.0 + exp(-sum));+ }+ for (int k = 0; k < 8; k++) if (k < KW) st[wb + k] = hist[k];+}++void rec() {+ int h = int(gl_WorkGroupID.x), v = int(gl_LocalInvocationID.x);+ int Pw = PW;+ bool own = v < VD;+ float col[KD];+ int sb = ST_OFF + h * KD * VD;+ [[unroll]] for (int k = 0; k < KD; k++) col[k] = own ? st[sb + k * VD + v] : 0.0;+ float ea = exp(w[PRM_OFF + h]);+ float nw = own ? w[PRM_OFF + H_ + Pw + v] : 0.0;+ for (int s = 0; s < S_; s++) {+ int mb = M_OFF + s * M_ROW;+ for (int k = v; k < KD; k += 128) {+ sq[k] = xin[mb + h * KD + k];+ sk[k] = xin[mb + Pw + h * KD + k];+ float f = fr[F_OFF + s * F_ROW + h * KD + k] + w[PRM_OFF + H_ + h * KD + k];+ al[k] = exp(LB * sig(ea * f));+ }+ barrier();+ if (v == 0) { // the l2 sums, in order, eps first+ float qs = NEPS, ks = NEPS;+ for (int k = 0; k < KD; k++) { qs += sq[k] * sq[k]; ks += sk[k] * sk[k]; }+ scal[0] = (1.0 / sqrt(float(KD))) / sqrt(qs);+ scal[1] = 1.0 / sqrt(ks);+ }+ barrier();+ float qn = scal[0], kn = scal[1];+ float beta = sig(br[B_OFF + s * B_ROW + h]);+ float val = own ? xin[mb + 2 * Pw + h * VD + v] : 0.0;+ float mem = 0.0;+ [[unroll]] for (int k = 0; k < KD; k++) { col[k] *= al[k]; mem += (sk[k] * kn) * col[k]; }+ float res = 0.0;+ [[unroll]] for (int k = 0; k < KD; k++) {+ col[k] += (sk[k] * kn) * (val - mem) * beta;+ res += (sq[k] * qn) * col[k];+ }+ ov[v] = own ? res : 0.0;+ barrier();+ if (v == 0) {+ float sum = 0.0;+ for (int d = 0; d < VD; d++) sum += ov[d] * ov[d];+ scal[0] = 1.0 / sqrt(sum / float(VD) + EPS);+ }+ barrier();+ if (own) o[Y_OFF + s * Y_ROW + h * VD + v] = res * scal[0] * nw * sig(gr[G_OFF + s * G_ROW + h * VD + v]);+ barrier();+ }+ if (own) [[unroll]] for (int k = 0; k < KD; k++) st[sb + k * VD + v] = col[k];+}++void main() {+ if (p.v[0] == 0) conv();+ else rec();+}diff --git a/c/shaders/chain_kda.spv b/c/shaders/chain_kda.spvnew file mode 100644index 00000000..110ddcd8Binary files /dev/null and b/c/shaders/chain_kda.spv differdiff --git a/c/shaders/chain_mhc.comp b/c/shaders/chain_mhc.compnew file mode 100644index 00000000..7121f353--- /dev/null+++ b/c/shaders/chain_mhc.comp@@ -0,0 +1,118 @@+#version 450+// Manifold-constrained hyper-connections (mHC) on the dense chain (vk_chain.c,+// VKC_P_MHC), hyper_connections.h's arithmetic for S rows of H residual streams of D+// floats ([S][H*D]), and a clamped SwiGLU. Five modes:+//+// mode 0, the split: one workgroup per row s. r = 1 / sqrt(mean(x^2) + eps) over the+// row's H*D stream values; the mix logits are mix[s][i] * r (mix = the raw products+// hc_fn . x, from a matmul); then, by one thread in the CPU's order:+// pre[i] = sigmoid(m[i] * scale[0] + base[i]) + hc_eps+// post[i] = 2 * sigmoid(m[H+i] * scale[1] + base[H+i])+// comb = softmax per row of m[2H + row*H + col] * scale[2] + base[...], + hc_eps,+// then iters - 1 rounds of Sinkhorn (rows, then columns) after a first+// column normalization+// into hp[s][0..H) pre, [H..2H) post, [2H..2H+H*H) comb. scale (3) and base (2H+H*H)+// at prm_off in prm.+// mode 1, collapse: y[s][d] = sum_i pre[i] * x[s][i*D + d], the streams in order.+// mode 2, write back: y[s][j*D + d] = sum_i comb[i*H + j] * x[s][i*D + d] + post[j] * b[s][d]+// mode 3, mean: y[s][d] = (sum_i x[s][i*D + d]) / H (the model's last collapse).+// mode 4, clamped SwiGLU: y[i] = silu(min(a[i], lim)) * clamp(b[i], -lim, lim) over n+// values (GLM-5.3's dense MLPs and shared experts; lim from the config).+// Offsets in floats; a row of x / y is x_row / y_row floats.+layout(local_size_x = 256) in;++layout(std430, binding = 0) readonly buffer XB { float x[]; }; // streams; 4: a+layout(std430, binding = 1) readonly buffer MB { float mx[]; }; // 0: mix; 2: branch; 4: b+layout(std430, binding = 2) buffer HB { float hp[]; }; // pre | post | comb per row+layout(std430, binding = 3) readonly buffer PB { float prm[]; }; // 0: scale, base+layout(std430, binding = 4) writeonly buffer YB { float y[]; };++layout(push_constant) uniform P {+ int mode, S, H, D, iters;+ int x_off, x_row, m_off, m_row, hp_off, hp_row, y_off, y_row, prm_off, n;+ float eps, hc_eps, lim;+} p;++shared float red[256];+shared float ms[96];++float sig(float v) {+ if (v >= 0.0) { float d = exp(-v); return 1.0 / (1.0 + d); }+ float g = exp(v);+ return g / (1.0 + g);+}++void split() {+ int s = int(gl_WorkGroupID.x), tid = int(gl_LocalInvocationID.x);+ int H = p.H, HD = H * p.D, nm = (2 + H) * H;+ int xb = p.x_off + s * p.x_row;+ float a = 0.0;+ for (int i = tid; i < HD; i += 256) { float v = x[xb + i]; a += v * v; }+ red[tid] = a;+ barrier();+ for (int off = 128; off > 0; off >>= 1) {+ if (tid < off) red[tid] += red[tid + off];+ barrier();+ }+ float r = 1.0 / sqrt(red[0] / float(HD) + p.eps);+ for (int i = tid; i < nm; i += 256) ms[i] = mx[p.m_off + s * p.m_row + i] * r;+ barrier();+ if (tid != 0) return;+ int hb = p.hp_off + s * p.hp_row, pb = p.prm_off;+ for (int i = 0; i < H; i++) {+ hp[hb + i] = sig(ms[i] * prm[pb] + prm[pb + 3 + i]) + p.hc_eps;+ hp[hb + H + i] = 2.0 * sig(ms[H + i] * prm[pb + 1] + prm[pb + 3 + H + i]);+ }+ int cb = hb + 2 * H, mo = 2 * H;+ for (int row = 0; row < H; row++) {+ float mxv = -3.0e38;+ for (int col = 0; col < H; col++) {+ float v = ms[mo + row * H + col] * prm[pb + 2] + prm[pb + 3 + mo + row * H + col];+ hp[cb + row * H + col] = v;+ if (v > mxv) mxv = v;+ }+ float sum = 0.0;+ for (int col = 0; col < H; col++) { float v = exp(hp[cb + row * H + col] - mxv); hp[cb + row * H + col] = v; sum += v; }+ for (int col = 0; col < H; col++) hp[cb + row * H + col] = hp[cb + row * H + col] / sum + p.hc_eps;+ }+ float sums[8];+ for (int col = 0; col < H; col++) { float sm = 0.0; for (int row = 0; row < H; row++) sm += hp[cb + row * H + col]; sums[col] = sm; }+ for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[col] + p.hc_eps;+ for (int it = 1; it < p.iters; it++) {+ for (int row = 0; row < H; row++) { float sm = 0.0; for (int col = 0; col < H; col++) sm += hp[cb + row * H + col]; sums[row] = sm; }+ for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[row] + p.hc_eps;+ for (int col = 0; col < H; col++) { float sm = 0.0; for (int row = 0; row < H; row++) sm += hp[cb + row * H + col]; sums[col] = sm; }+ for (int row = 0; row < H; row++) for (int col = 0; col < H; col++) hp[cb + row * H + col] /= sums[col] + p.hc_eps;+ }+}++void main() {+ if (p.mode == 0) { split(); return; }+ int i = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);+ if (p.mode == 4) {+ if (i >= p.n) return;+ float g = x[p.x_off + i], u = mx[p.m_off + i];+ g = g > p.lim ? p.lim : g;+ u = u < -p.lim ? -p.lim : (u > p.lim ? p.lim : u);+ y[p.y_off + i] = (g / (1.0 + exp(-g))) * u;+ return;+ }+ int H = p.H, D = p.D;+ if (p.mode == 2) {+ if (i >= p.S * H * D) return;+ int s = i / (H * D), r = i - s * H * D, j = r / D, d = r - j * D;+ int xb = p.x_off + s * p.x_row, hb = p.hp_off + s * p.hp_row;+ float v = 0.0;+ for (int k = 0; k < H; k++) v += hp[hb + 2 * H + k * H + j] * x[xb + k * D + d];+ v += hp[hb + H + j] * mx[p.m_off + s * p.m_row + d];+ y[p.y_off + s * p.y_row + j * D + d] = v;+ return;+ }+ if (i >= p.S * D) return;+ int s = i / D, d = i - s * D;+ int xb = p.x_off + s * p.x_row, hb = p.hp_off + s * p.hp_row;+ float v = 0.0;+ if (p.mode == 1) { for (int k = 0; k < H; k++) v += hp[hb + k] * x[xb + k * D + d]; }+ else { for (int k = 0; k < H; k++) v += x[xb + k * D + d]; v = v / float(H); }+ y[p.y_off + s * p.y_row + d] = v;+}diff --git a/c/shaders/chain_mhc.spv b/c/shaders/chain_mhc.spvnew file mode 100644index 00000000..c13871caBinary files /dev/null and b/c/shaders/chain_mhc.spv differdiff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.cindex b016afe7..a17d1626 100644--- a/c/tests/test_vk_chain.c+++ b/c/tests/test_vk_chain.c@@ -25,14 +25,25 @@* lists with skipped entries, a gate, prefill rows, a long context from* kv_start, latents up to 1024)* dsa the indexer's top-k selection bit for bit, ties included+ * kpool the k-pooled indexer (GLM-5.3): pooled keys, the pools' rank order and+ * the tail, slot for slot against sparse_index.h+ * kda GLM-5.3's KDA layer around delta_attention.h's step: the convolution+ * and its window, the recurrence and its state over two submissions, the+ * decay, beta, output norm and gate+ * mhc hyper_connections.h's split, collapse and write back, the mean, and the+ * clamped SwiGLU** make vk-chain-check VK=1 (VK_ICD_FILENAMES=.../lvp_icd.json for Lavapipe) */#include <stdio.h>#include <stdlib.h>#include <string.h>#include <math.h>+#include <float.h>#include "../backend_vulkan.h"#include "../vk_chain.h"+#include "../delta_attention.h" /* the KDA step the KDA ops follow */+#include "../hyper_connections.h" /* the mHC arithmetic */+#include "../sparse_index.h" /* GLM-5.3's k-pooled indexer */static int fails;#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)@@ -790,6 +801,181 @@ static void test_dsa(int S, int pos_base, int IH, int ID, int topk, int force, ifree(q); free(k); free(w); free(ref);}+/* ---- KDA, mHC, k-pooling ----------------------------------------------------------------- */+/* glm53.c's KDA layer around delta_attention.h's step, S rows over two calls (the state+ * and the window carried on the device between them) */+static void test_kda(int H, int KD, int CK, float xs) { /* xs: the inputs' scale (small: the l2 eps counts) */+ int VD = KD, P = H * KD, C = 3 * P, S = 5;+ float lb = -5.f, neps = 1e-6f, eps = 1e-5f;+ float *qkv = fvec((size_t)S * C, xs), *f = fvec((size_t)S * P, 1.f), *b = fvec((size_t)S * H, 2.f), *g = fvec((size_t)S * P, 2.f);+ float *taps = fvec((size_t)C * CK, 0.7f), *win = fvec((size_t)C * CK, 1.f), *state = fvec((size_t)H * KD * VD, 0.3f);+ float *prm = malloc((size_t)(H + P + VD) * 4);+ for (int i = 0; i < H; i++) prm[i] = frnd() * 0.5f;+ for (int i = 0; i < P; i++) prm[H + i] = frnd() * 0.5f;+ for (int i = 0; i < VD; i++) prm[H + P + i] = 0.5f + (rnd() % 100) / 100.f;+ /* reference: glm53.c's kda_layer row by row */+ float *rs = malloc((size_t)H * KD * VD * 4), *rw = malloc((size_t)C * CK * 4), *ref = malloc((size_t)S * P * 4);+ memcpy(rs, state, (size_t)H * KD * VD * 4); memcpy(rw, win, (size_t)C * CK * 4);+ float *scratch = malloc((size_t)coli_kda_scratch_floats(H, KD, VD) * 4), *core = malloc((size_t)P * 4);+ float *dk = malloc((size_t)P * 4), *bt = malloc((size_t)H * 4);+ for (int s = 0; s < S; s++) {+ for (int h = 0; h < H; h++) for (int d = 0; d < KD; d++) {+ int i = h * KD + d; float sv = expf(prm[h]) * (f[s * P + i] + prm[H + i]);+ float sg = sv >= 0 ? 1.f / (1.f + expf(-sv)) : expf(sv) / (1.f + expf(sv));+ dk[i] = lb * sg;+ }+ for (int h = 0; h < H; h++) { float v = b[s * H + h]; bt[h] = v >= 0 ? 1.f / (1.f + expf(-v)) : expf(v) / (1.f + expf(v)); }+ coli_kda_step(core, rs, rw, qkv + (size_t)s * C, taps, dk, bt, H, KD, VD, CK, neps, scratch);+ for (int h = 0; h < H; h++) {+ float sq = 0; for (int d = 0; d < VD; d++) sq += core[h * VD + d] * core[h * VD + d];+ float inv = 1.f / sqrtf(sq / VD + eps);+ for (int d = 0; d < VD; d++) {+ float gv = g[s * P + h * VD + d], sg = gv >= 0 ? 1.f / (1.f + expf(-gv)) : expf(gv) / (1.f + expf(gv));+ ref[s * P + h * VD + d] = core[h * VD + d] * inv * prm[H + P + d] * sg;+ }+ }+ }+ /* the device: q, k, v in three blocks [3][S][P] (in_part = S*P) */+ float *blk = malloc((size_t)3 * S * P * 4);+ for (int s = 0; s < S; s++) for (int part = 0; part < 3; part++)+ memcpy(blk + (size_t)part * S * P + (size_t)s * P, qkv + (size_t)s * C + (size_t)part * P, (size_t)P * 4);+ VkcBuf *ib = up(blk, (size_t)3 * S * P), *tb = up(taps, (size_t)C * CK), *wb = up(win, (size_t)C * CK), *sb = up(state, (size_t)H * KD * VD);+ VkcBuf *fb = up(f, (size_t)S * P), *bb = up(b, (size_t)S * H), *gb = up(g, (size_t)S * P), *pb = up(prm, (size_t)H + P + VD);+ VkcBuf *mb = vkc_buf((size_t)S * C * 4, VKC_DEV), *yb = vkc_buf((size_t)S * P * 4, VKC_DEV);+ int ok = 1, s0 = 0;+ for (int part = 0; part < 2 && ok; part++) {+ int n = part ? S - 3 : 3;+ VkcKdaConv cp = {n, C, CK, P, s0 * P, P, S * P, 0, C, 0, 0};+ VkcKdaRec rp = {n, H, VD, P, 0, C, s0 * P, P, s0 * H, H, s0 * P, P, s0 * P, P, 0, 0, lb, neps, eps};+ ok = vkc_begin() && vkc_kda_conv(ib, tb, wb, mb, &cp) && vkc_kda_rec(KD, mb, fb, bb, gb, pb, sb, yb, &rp) && vkc_submit(part);+ s0 += n;+ }+ float *y = down(yb, 0, (size_t)S * P), *st2 = down(sb, 0, (size_t)H * KD * VD), *w2 = down(wb, 0, (size_t)C * CK);+ double e = relerr(y, ref, (size_t)S * P, 1e-3), es = relerr(st2, rs, (size_t)H * KD * VD, 1e-3), ew = relerr(w2, rw, (size_t)C * CK, 1e-3);+ CHECK(ok && e < 1e-5 && es < 1e-5 && ew == 0 && !bad(y, (size_t)S * P), "kda H %d KD %d K %d: ok %d out %.2e state %.2e window %.2e",+ H, KD, CK, ok, e, es, ew);+ if (getenv("VKC_TEST_VERBOSE")) printf("kda H %d KD %d: out %.2e state %.2e\n", H, KD, e, es);+ vkc_free(ib); vkc_free(tb); vkc_free(wb); vkc_free(sb); vkc_free(fb); vkc_free(bb); vkc_free(gb); vkc_free(pb); vkc_free(mb); vkc_free(yb);+ free(qkv); free(f); free(b); free(g); free(taps); free(win); free(state); free(prm); free(rs); free(rw); free(ref);+ free(scratch); free(core); free(dk); free(bt); free(blk); free(y); free(st2); free(w2);+}++/* hyper_connections.h's pre and post against the split, collapse and write back, the+ * mean, and the clamped SwiGLU */+static void test_mhc(int H, int D, int iters) {+ int S = 3, HD = H * D, nm = (2 + H) * H;+ float eps = 1e-5f, hce = 1e-6f, lim = 1.5f;+ float *x = fvec((size_t)S * HD, 1.f), *fn = fvec((size_t)nm * HD, 0.05f), *br = fvec((size_t)S * D, 1.f);+ float *prm = malloc((size_t)(3 + nm) * 4);+ for (int i = 0; i < 3; i++) prm[i] = 0.5f + (rnd() % 100) / 100.f;+ for (int i = 0; i < nm; i++) prm[3 + i] = frnd() * 0.5f;+ float *rpre = malloc((size_t)S * D * 4), *rpost = malloc((size_t)S * H * 4), *rcomb = malloc((size_t)S * H * H * 4);+ float *rout = malloc((size_t)S * HD * 4), *rmean = malloc((size_t)S * D * 4);+ for (int s = 0; s < S; s++) {+ coli_hc_pre(rpre + s * D, rpost + s * H, rcomb + s * H * H, x + s * HD, fn, prm, prm + 3, H, D, iters, eps, hce);+ coli_hc_post(rout + s * HD, br + s * D, x + s * HD, rpost + s * H, rcomb + s * H * H, H, D);+ for (int d = 0; d < D; d++) { float sm = 0; for (int i = 0; i < H; i++) sm += x[s * HD + i * D + d]; rmean[s * D + d] = sm / H; }+ }+ ColiVkTensor *t = NULL;+ if (!coli_vk_tensor_ensure(&t, fn, NULL, 10, HD, nm, 0)) { CHECK(0, "mhc: upload"); return; }+ VkcBuf *xb = up(x, (size_t)S * HD), *bb = up(br, (size_t)S * D), *pb = up(prm, 3 + (size_t)nm);+ VkcBuf *mb = vkc_buf((size_t)S * nm * 4, VKC_DEV), *hb = vkc_buf((size_t)S * (2 * H + H * H) * 4, VKC_DEV);+ VkcBuf *cb = vkc_buf((size_t)S * D * 4, VKC_DEV), *ob = vkc_buf((size_t)S * HD * 4, VKC_DEV), *eb = vkc_buf((size_t)S * D * 4, VKC_DEV);+ int hrow = 2 * H + H * H;+ VkcMhc sp = {S, H, D, iters, 0, HD, 0, nm, 0, hrow, 0, D, 0, 0, eps, hce, 0.f};+ VkcMhc po = {S, H, D, iters, 0, HD, 0, D, 0, hrow, 0, HD, 0, 0, eps, hce, 0.f};+ int ok = vkc_begin() && vkc_matmul(t, xb, 0, mb, 0, S) && vkc_mhc(VKC_MHC_SPLIT, xb, mb, hb, pb, NULL, &sp) &&+ vkc_mhc(VKC_MHC_COLLAPSE, xb, NULL, hb, NULL, cb, &sp) && vkc_mhc(VKC_MHC_POST, xb, bb, hb, NULL, ob, &po) &&+ vkc_mhc(VKC_MHC_MEAN, xb, NULL, NULL, NULL, eb, &sp) && vkc_submit(1);+ float *hp = down(hb, 0, (size_t)S * hrow), *col = down(cb, 0, (size_t)S * D), *out = down(ob, 0, (size_t)S * HD), *mean = down(eb, 0, (size_t)S * D);+ /* pre, post and comb from coli_hc_split_sinkhorn on the CPU's own mixes */+ float *rp = malloc((size_t)S * hrow * 4), *mixs = malloc((size_t)nm * 4);+ for (int s = 0; s < S; s++) {+ float ms = 0; for (int i = 0; i < HD; i++) ms += x[s * HD + i] * x[s * HD + i];+ float ir = 1.f / sqrtf(ms / HD + eps);+ for (int r = 0; r < nm; r++) { float a = 0; for (int i = 0; i < HD; i++) a += fn[(size_t)r * HD + i] * x[s * HD + i]; mixs[r] = a * ir; }+ coli_hc_split_sinkhorn(rp + s * hrow, rp + s * hrow + H, rp + s * hrow + 2 * H, mixs, prm, prm + 3, H, iters, hce);+ }+ double epre = 0;+ for (int s = 0; s < S; s++) for (int i = 0; i < hrow; i++) { double d = fabs(hp[s * hrow + i] - rp[s * hrow + i]); if (d > epre) epre = d; }+ free(rp); free(mixs);+ double ep = 0, ec = relerr(col, rpre, (size_t)S * D, 1e-3), eo = relerr(out, rout, (size_t)S * HD, 1e-3), em = relerr(mean, rmean, (size_t)S * D, 1e-3);+ for (int s = 0; s < S; s++) {+ for (int i = 0; i < H; i++) { double d = fabs(hp[s * hrow + H + i] - rpost[s * H + i]); if (d > ep) ep = d; }+ for (int i = 0; i < H * H; i++) { double d = fabs(hp[s * hrow + 2 * H + i] - rcomb[s * H * H + i]); if (d > ep) ep = d; }+ }+ CHECK(ok && ep < 2e-5 && epre < 5e-7 && ec < 2e-5 && eo < 2e-5 && em < 1e-6,+ "mhc H %d D %d iters %d: ok %d split %.2e post/comb %.2e collapse %.2e write back %.2e mean %.2e",+ H, D, iters, ok, epre, ep, ec, eo, em);+ if (getenv("VKC_TEST_VERBOSE")) printf("mhc H %d D %d: split %.2e post/comb %.2e collapse %.2e write back %.2e mean %.2e\n", H, D, epre, ep, ec, eo, em);+ /* the clamped SwiGLU, on values both sides of the limit */+ int n = 333;+ float *ga = fvec(n, 3.f), *ua = fvec(n, 3.f), *rr = malloc((size_t)n * 4);+ for (int i = 0; i < n; i++) { float gg = ga[i] > lim ? lim : ga[i], uu = ua[i] < -lim ? -lim : (ua[i] > lim ? lim : ua[i]); rr[i] = gg / (1.f + expf(-gg)) * uu; }+ VkcBuf *gab = up(ga, n), *uab = up(ua, n), *yb = vkc_buf((size_t)n * 4, VKC_DEV);+ VkcMhc sw = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, n, 0.f, 0.f, lim};+ ok = vkc_begin() && vkc_mhc(VKC_SWIGLU_CLAMP, gab, uab, NULL, NULL, yb, &sw) && vkc_submit(1);+ float *yy = down(yb, 0, n);+ double es = relerr(yy, rr, n, 1e-3);+ CHECK(ok && es < 1e-6, "swiglu clamp: ok %d err %.2e", ok, es);+ vkc_free(xb); vkc_free(bb); vkc_free(pb); vkc_free(mb); vkc_free(hb); vkc_free(cb); vkc_free(ob); vkc_free(eb);+ vkc_free(gab); vkc_free(uab); vkc_free(yb); coli_vk_tensor_free(t);+ free(x); free(fn); free(br); free(prm); free(rpre); free(rpost); free(rcomb); free(rout); free(rmean);+ free(hp); free(col); free(out); free(mean); free(ga); free(ua); free(rr); free(yy);+}++/* k-pooling against sparse_index.h. Equal gates and no bias make each pooled key the+ * exact mean of its members, so with quarter-step values and power-of-two scales the+ * scores are exact on both sides and the selection must be the CPU's slot for slot;+ * a second pass with random gates and bias checks the pooled keys to rounding. */+static void test_kpool(int S, int pos_base, int IH, int ID, int pool, int topk, int tail) {+ int T = pos_base + S, width = coli_sparse_index_width(topk, pool, tail), selrow = width + 1;+ int npool = T / pool, scrow = npool + 1;+ float *keys = malloc((size_t)T * ID * 4), *gates = calloc((size_t)T * ID, 4), *ape = calloc((size_t)pool * ID, 4);+ float *q = malloc((size_t)S * IH * ID * 4), *hw = malloc((size_t)S * IH * 4);+ for (int i = 0; i < T * ID; i++) keys[i] = ((int)(rnd() % 17) - 8) * 0.25f;+ for (int i = 0; i < S * IH * ID; i++) q[i] = ((int)(rnd() % 17) - 8) * 0.25f;+ for (int i = 0; i < S * IH; i++) hw[i] = ((int)(rnd() % 9) - 4) * 0.5f * sqrtf((float)IH);+ for (int t = 0; t < T; t += 7) for (int i = 0; i < ID; i++) keys[t * ID + i] = keys[(t % 3) * ID + i]; /* equal pools: ties */+ unsigned char *valid = malloc((size_t)T); memset(valid, 1, (size_t)T);+ int *ref = malloc((size_t)S * width * sizeof(int));+ float *hwd = malloc((size_t)S * IH * 4);+ for (int i = 0; i < S * IH; i++) hwd[i] = hw[i] / sqrtf((float)IH);+ coli_sparse_index_select_range(ref, q, keys, gates, hwd, ape, valid, T, IH, ID, pool, topk, tail, pos_base, T);+ VkcBuf *kb = up(keys, (size_t)T * ID), *gb = up(gates, (size_t)T * ID), *ab = up(ape, (size_t)pool * ID);+ VkcBuf *qb = up(q, (size_t)S * IH * ID), *wb = up(hw, (size_t)S * IH), *pk = vkc_buf((size_t)(npool + 1) * ID * 4, VKC_DEV);+ VkcBuf *scb = vkc_buf((size_t)S * scrow * 4, VKC_DEV), *sb = vkc_buf((size_t)S * selrow * 4, VKC_DOWN);+ VkcDsaPool kp = {npool, pool, 0, ID, 0, ID, 0, 0, 0, ID};+ VkcDsaPick sp = {S, pos_base, IH, ID, topk, pool, 0, IH * ID, 0, IH, 0, tail, scrow, selrow, sqrtf((float)IH), 1.f / sqrtf((float)ID)};+ int ok = vkc_begin() && vkc_dsa_pool_keys(kb, gb, ab, pk, &kp) && vkc_dsa_pool_select(qb, wb, pk, scb, sb, &sp) && vkc_submit(1);+ const int *got = (const int *)vkc_ptr(sb);+ int same = ok;+ for (int s = 0; same && s < S; s++) {+ same = got[s * selrow] == width;+ for (int i = 0; same && i < width; i++) same = got[s * selrow + 1 + i] == ref[s * width + i];+ if (!same && getenv("DSA_DEBUG")) { for (int i = 0; i < width; i++) printf(" %d/%d", got[s * selrow + 1 + i], ref[s * width + i]); printf("\n"); }+ }+ CHECK(same, "kpool S %d pos %d IH %d ID %d pool %d topk %d tail %d: ok %d, the selection differs", S, pos_base, IH, ID, pool, topk, tail, ok);+ /* the pooled keys with real gates and bias, against sparse_index.h's arithmetic */+ for (int i = 0; i < T * ID; i++) gates[i] = frnd() * 2.f;+ for (int i = 0; i < pool * ID; i++) ape[i] = frnd() * 0.5f;+ float *rk = malloc((size_t)npool * ID * 4);+ for (int p = 0; p < npool; p++) for (int d = 0; d < ID; d++) {+ float mx = -FLT_MAX, tot = 0, mix = 0;+ for (int j = 0; j < pool; j++) { float l = gates[(p * pool + j) * ID + d] + ape[j * ID + d]; if (l > mx) mx = l; }+ for (int j = 0; j < pool; j++) tot += expf(gates[(p * pool + j) * ID + d] + ape[j * ID + d] - mx);+ for (int j = 0; j < pool; j++) mix += expf(gates[(p * pool + j) * ID + d] + ape[j * ID + d] - mx) / tot * keys[(p * pool + j) * ID + d];+ rk[p * ID + d] = mix;+ }+ ok = vkc_begin() && vkc_write(gb, 0, gates, (size_t)T * ID * 4) && vkc_write(ab, 0, ape, (size_t)pool * ID * 4) &&+ vkc_dsa_pool_keys(kb, gb, ab, pk, &kp) && vkc_submit(1);+ float *gk = down(pk, 0, (size_t)npool * ID);+ double e = relerr(gk, rk, (size_t)npool * ID, 1e-3);+ CHECK(ok && e < 2e-6, "kpool keys pool %d ID %d: ok %d err %.2e", pool, ID, ok, e);+ vkc_free(kb); vkc_free(gb); vkc_free(ab); vkc_free(qb); vkc_free(wb); vkc_free(pk); vkc_free(scb); vkc_free(sb);+ free(keys); free(gates); free(ape); free(q); free(hw); free(valid); free(ref); free(hwd); free(rk); free(gk);+}+/* ---- frames: many ops, frames in flight, ordering ---------------------------------------- */static void test_frames(void) {int n = 1000;@@ -895,6 +1081,17 @@ int main(int argc, char **argv) {test_dsa(3, 5, 2, 16, 64, 1, 0); test_dsa(2, 700, 3, 64, 100, 0, 0); test_dsa(1, 9, 64, 64, 4, 0, 0);test_dsa(2, 900, 2, 16, 50, 0, 1);printf("dsa done\n");+ test_kpool(3, 20, 4, 16, 2, 8, 1); test_kpool(2, 9, 1, 16, 4, 8, 1); test_kpool(4, 300, 4, 64, 4, 64, 1);+ test_kpool(1, 2, 4, 16, 2, 4, 0); test_kpool(2, 30, 16, 16, 2, 6, 1);+ printf("kpool done\n");+ }+ if (!vkc_kda_ready() || !vkc_mhc_ready()) { fails++; printf("FAIL: the KDA or mHC shaders did not load\n"); }+ else {+ test_kda(3, 16, 4, 1.f); test_kda(2, 32, 4, 1.f); test_kda(4, 128, 3, 1.f); test_kda(1, 64, 2, 1.f);+ test_kda(2, 16, 4, 0.002f);+ printf("kda done\n");+ test_mhc(2, 64, 3); test_mhc(4, 96, 20); test_mhc(3, 40, 1);+ printf("mhc done\n");}VkcStats st; vkc_stats(&st);printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);diff --git a/c/vk_chain.c b/c/vk_chain.cindex 03adb438..ee112a1d 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -36,13 +36,16 @@ static const char *const pipe_file[P_NPIPE] = {/* COLI_VK_CHAIN_PROF=1: device time per kind of op (timestamps after every op; ops* that run side by side between two barriers share the time unevenly) */enum { PK_GEMV8, PK_GEMV, PK_GEMM, PK_NORM, PK_ROPE, PK_ATTN, PK_DNCONV, PK_DNREC, PK_EW, PK_QSA, PK_PLE, PK_COPY,- PK_MLA, PK_MLAW, PK_DSA, PK_N };+ PK_MLA, PK_MLAW, PK_DSA, PK_KDA, PK_MHC, PK_N };static const char *const pk_name[PK_N] = {"GEMV int8", "GEMV other", "tiled GEMM", "norm", "rope", "attention","dn conv", "dn recurrence", "element-wise", "qsa", "ple", "copy",- "mla", "mla weights", "dsa"};-/* the MLA pipelines: optional (an engine without MLA layers needs none of them) */-enum { PM_MLA, PM_HGEMV, PM_DSA, PM_N };-static const char *const mla_file[PM_N] = {"chain_mla.spv", "chain_hgemv.spv", "chain_dsa.spv"};+ "mla", "mla weights", "dsa", "kda", "mhc"};+/* the MLA, KDA and mHC pipelines: optional (an engine without such layers needs none+ * of them, and a missing one turns off only its own ops) */+enum { PM_MLA, PM_HGEMV, PM_DSA, PM_KDA, PM_MHC, PM_N };+static const char *const mla_file[PM_N] = {"chain_mla.spv", "chain_hgemv.spv", "chain_dsa.spv", "chain_kda.spv",+ "chain_mhc.spv"};+#define VKC_KDA_MAX 8/* ---- memory: blocks per kind, buffers bound at offsets inside them -------------- */typedef struct { VkDeviceMemory mem; uint8_t *map; } VkcBlock;@@ -80,6 +83,7 @@ static struct {VkPipeline dnrec[VKC_DNREC_MAX]; int dnrec_kd[VKC_DNREC_MAX], ndnrec;VkShaderModule mod_gemv4; VkPipeline gemv4; int gemv4_xs; /* chain_gemv.comp: the vectorized decode GEMV */VkShaderModule mmod[PM_N]; VkPipeline mpipe[PM_N]; int mla_ok; /* chain_mla, chain_hgemv, chain_dsa */+ VkPipeline kdarec[VKC_KDA_MAX]; int kdarec_kd[VKC_KDA_MAX], nkdarec; /* chain_kda's recurrence per key dim */VkCommandPool cpool;VkcFrame fr[VKC_FRAMES];int cur; /* the open frame, -1 none */@@ -281,8 +285,7 @@ int vkc_init(void) {if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;}}- /* the MLA shaders, optional: without them only the MLA ops decline */- K.mla_ok = 1;+ /* the MLA, KDA and mHC shaders, optional: without one only its ops decline */for (int i = 0; i < PM_N; i++) {char path[1200];const char *sl = strrchr(K.core.spv_path, '/');@@ -293,9 +296,9 @@ int vkc_init(void) {f = fopen(path, "rb");}if (f) fclose(f);- if (!f || !(K.mmod[i] = load_module(K.core.spv_path, mla_file[i])) || !(K.mpipe[i] = make_pipe(K.mmod[i], NULL)))- K.mla_ok = 0;+ if (f && (K.mmod[i] = load_module(K.core.spv_path, mla_file[i]))) K.mpipe[i] = make_pipe(K.mmod[i], NULL);}+ K.mla_ok = K.mpipe[PM_MLA] && K.mpipe[PM_HGEMV] && K.mpipe[PM_DSA];/* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {@@ -835,6 +838,70 @@ int vkc_dsa_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *sc, VkcBuf *selreturn record(K.mpipe[PM_DSA], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);}+/* k-pooling (chain_dsa.comp modes 1 and 2) */+int vkc_dsa_pool_keys(VkcBuf *keys, VkcBuf *gates, VkcBuf *prm, VkcBuf *pk, const VkcDsaPool *p) {+ K.kind = PK_DSA;+ if (!K.mla_ok || p->np < 0 || p->pool < 1 || p->p0 < 0 || p->ID < 1) return 0;+ if (p->np == 0) return open_frame() && !K.lost;+ struct { int mode; VkcDsaPool b; } pc = {1, *p};+ VkcBind bd[8] = {B(NULL, 0), B(NULL, 0), B(keys, 0), B(NULL, 0), B(NULL, 0), B(pk, 1), B(gates, 0), B(prm, 0)};+ return record(K.mpipe[PM_DSA], bd, 8, &pc, sizeof pc, (uint32_t)p->np, 1, 1);+}+int vkc_dsa_pool_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *pk, VkcBuf *sc, VkcBuf *sel, const VkcDsaPick *p) {+ K.kind = PK_DSA;+ if (!K.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||+ p->pool < 1 || p->topk < p->pool || p->topk % p->pool || p->topk / p->pool > 1024 || p->pos_base < 0 ||+ p->sc_row < (p->pos_base + p->S) / p->pool || p->sel_row < 1 + p->topk + (p->tail ? p->pool - 1 : 0)) return 0;+ struct { int mode; VkcDsaPick b; } pc = {2, *p};+ VkcBind bd[6] = {B(iq, 0), B(hw, 0), B(NULL, 0), B(sc, 1), B(sel, 1), B(pk, 0)};+ return record(K.mpipe[PM_DSA], bd, 6, &pc, sizeof pc, (uint32_t)p->S, 1, 1);+}++/* ---- Kimi Delta Attention (chain_kda.comp) ------------------------------------------ */+int vkc_kda_ready(void) { return vkc_ready() && K.mpipe[PM_KDA]; }+int vkc_kda_conv(VkcBuf *in, VkcBuf *w, VkcBuf *win, VkcBuf *out, const VkcKdaConv *p) {+ K.kind = PK_KDA;+ if (!K.mpipe[PM_KDA] || p->K < 1 || p->K > 8 || p->P < 1 || p->C < 1 || p->S < 1) return 0;+ struct { int mode; VkcKdaConv b; } pc = {0, *p};+ VkcBind bd[4] = {B(in, 0), B(w, 0), B(win, 1), B(out, 1)};+ uint32_t gx, gy; grid(((uint64_t)p->C + 127) / 128, &gx, &gy);+ return record(K.mpipe[PM_KDA], bd, 4, &pc, sizeof pc, gx, gy, 1);+}+static VkPipeline kda_pipe(int KD) {+ for (int i = 0; i < K.nkdarec; i++) if (K.kdarec_kd[i] == KD) return K.kdarec[i];+ if (K.nkdarec == VKC_KDA_MAX || !K.mmod[PM_KDA]) return VK_NULL_HANDLE;+ int32_t kd = KD;+ VkSpecializationMapEntry me = {0, 0, 4};+ VkSpecializationInfo si = {1, &me, 4, &kd};+ VkPipeline p = make_pipe(K.mmod[PM_KDA], &si);+ if (!p) return VK_NULL_HANDLE;+ K.kdarec[K.nkdarec] = p; K.kdarec_kd[K.nkdarec++] = KD;+ return p;+}+int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y, const VkcKdaRec *p) {+ K.kind = PK_KDA;+ if (!K.mpipe[PM_KDA] || KD < 1 || KD > 256 || p->VD < 1 || p->VD > 128 || p->H < 1 || p->S < 1) return 0;+ VkPipeline pipe = kda_pipe(KD);+ if (!pipe) return 0;+ struct { int mode; VkcKdaRec b; } pc = {1, *p};+ VkcBind bd[7] = {B(m, 0), B(prm, 0), B(st, 1), B(y, 1), B(f, 0), B(b, 0), B(g, 0)};+ return record(pipe, bd, 7, &pc, sizeof pc, (uint32_t)p->H, 1, 1);+}++/* ---- manifold-constrained hyper-connections (chain_mhc.comp) ------------------------ */+int vkc_mhc_ready(void) { return vkc_ready() && K.mpipe[PM_MHC]; }+int vkc_mhc(int mode, VkcBuf *x, VkcBuf *m, VkcBuf *hp, VkcBuf *prm, VkcBuf *y, const VkcMhc *p) {+ K.kind = PK_MHC;+ if (!K.mpipe[PM_MHC] || mode < 0 || mode > 4 || p->S < 0 || (mode != 4 && (p->H < 1 || p->H > 8 || p->D < 1))) return 0;+ if (mode == 0 && p->iters < 1) return 0;+ struct { int mode; VkcMhc b; } pc = {mode, *p};+ VkcBind bd[5] = {B(x, 0), B(m, 0), B(hp, mode == 0), B(prm, 0), B(y, mode != 0)};+ uint64_t n = mode == 4 ? (uint64_t)p->n : mode == 2 ? (uint64_t)p->S * p->H * p->D : (uint64_t)p->S * p->D;+ if (mode == 0) return p->S == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);+ uint32_t gx, gy; grid((n + 255) / 256, &gx, &gy);+ return n == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, gx, gy, 1);+}+int vkc_mla_scratch(VkcMlaScratch *s, const VkcMla *m, int rows) {if (s->rows >= rows && s->q) return 1;size_t r = (size_t)rows, H = (size_t)m->H;@@ -951,6 +1018,7 @@ void vkc_shutdown(void) {if (K.pipe[i]) vkDestroyPipeline(K.dev, K.pipe[i], NULL);if (K.mod[i]) vkDestroyShaderModule(K.dev, K.mod[i], NULL);}+ for (int i = 0; i < K.nkdarec; i++) vkDestroyPipeline(K.dev, K.kdarec[i], NULL);for (int i = 0; i < PM_N; i++) {if (K.mpipe[i]) vkDestroyPipeline(K.dev, K.mpipe[i], NULL);if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);diff --git a/c/vk_chain.h b/c/vk_chain.hindex 62a04692..784b3d2b 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -26,7 +26,9 @@* at its top: chain_norm, chain_rope, chain_attn, chain_dnconv, chain_dnrec,* chain_ew, chain_qsa, chain_ple. Offsets and strides are in floats.* - multi-head latent attention: the MLA ops and the layer op below (chain_mla,- * chain_hgemv, chain_dsa), loaded beside the others but optional.+ * chain_hgemv, chain_dsa, its k-pooled modes included), Kimi Delta Attention+ * (chain_kda) and manifold-constrained hyper-connections (chain_mhc): loaded beside+ * the others but optional.** Threading: the engine thread only (the main queue is the backend's, used from the* same thread by coli_vk_matmul; the expert tier submits on its own queue).@@ -193,6 +195,19 @@ typedef struct { int S, pos_base, IH, ID, topk, force, q_off, q_row, w_off, w_rofloat qscale, wscale; } VkcDsa;int vkc_dsa_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *sc, VkcBuf *sel, const VkcDsa *p);+/* chain_dsa.comp with k-pooling (GLM-5.3): mode 1, the pooled keys of the np complete+ * pools from p0 on, each the per-channel softmax mixture (gate logits plus ape, at+ * ape_off in prm, [pool][ID]) of its members' keys, into pk[pk_off + p*ID]; mode 2, row+ * s's selection: the complete pools visible to position pos_base + s scored+ * sum_h [dot > 0] (w_h / wdiv) * dot * scale (dot = q_h . pk[p]), the top topk/pool in+ * rank order filling `pool` slots each, the incomplete tail from slot topk (tail), -1+ * elsewhere; sel[s*sel_row] = topk (+ pool - 1 with the tail). sc: [S][sc_row] scores. */+typedef struct { int np, pool, p0, ID, g_off, g_row, ape_off, pk_off, k_off, k_row; } VkcDsaPool;+int vkc_dsa_pool_keys(VkcBuf *keys, VkcBuf *gates, VkcBuf *prm, VkcBuf *pk, const VkcDsaPool *p);+typedef struct { int S, pos_base, IH, ID, topk, pool, q_off, q_row, w_off, w_row, pk_off, tail, sc_row, sel_row;+ float wdiv, scale; } VkcDsaPick;+int vkc_dsa_pool_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *pk, VkcBuf *sc, VkcBuf *sel, const VkcDsaPick *p);+/* The layer op. Tensors are the resident copies the per-matrix path uploads* (coli_vk_tensor_ensure), in any format the backend holds. */typedef struct {@@ -231,6 +246,46 @@ int vkc_mla_attn(const VkcMla *m, VkcMlaScratch *s, int S, int pos_base, int kv_int vkc_mla(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int S, int pos_base, int kv_start,VkcBuf *cs, VkcMlaCache *c, VkcBuf *out, size_t out_off);+/* ---- Kimi Delta Attention (chain_kda.comp) --------------------------------------------+ * delta_attention.h's coli_kda_step for S rows in order, the state and the short+ * convolution's window on the device (GLM-5.3's linear layers; Kimi K3's KDA).+ * vkc_kda_conv: C = 3P channels (q, k, v), the input of part c / P at in_off + part*in_part+ * + s*in_row + c % P, the window [C][K] at win_off (the last K inputs, oldest first, as+ * the CPU keeps it), taps [C][K] at w_off; out[out_off + s*out_row + c] = silu(sum).+ * vkc_kda_rec: one workgroup per head, the key dim KD a specialization (<= 256), VD <= 128;+ * m = the convolution's output rows (q at h*KD, k at P + h*KD, v at 2P + h*VD); f, b, g+ * the raw decay, beta and output-gate projections; prm at prm_off: A_log[H], dt[P],+ * norm[VD]; alpha = exp(lb * sigmoid(exp(A_log) * (f + dt))), beta = sigmoid(b), q and k+ * l2-normalized with neps inside the root; y = RMSNorm(o, eps) * norm * sigmoid(g).+ * The state [H][KD][VD] at st_off. */+int vkc_kda_ready(void);+typedef struct { int S, C, K, P, in_off, in_row, in_part, out_off, out_row, w_off, win_off; } VkcKdaConv;+int vkc_kda_conv(VkcBuf *in, VkcBuf *w, VkcBuf *win, VkcBuf *out, const VkcKdaConv *p);+typedef struct { int S, H, VD, P, m_off, m_row, f_off, f_row, b_off, b_row, g_off, g_row, y_off, y_row, st_off, prm_off;+ float lb, neps, eps; } VkcKdaRec;+int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y, const VkcKdaRec *p);++/* ---- manifold-constrained hyper-connections (chain_mhc.comp) ------------------------+ * hyper_connections.h for S rows of H <= 8 streams of D floats ([S][H*D] at x_off,+ * x_row apart), GLM-5.3's (and DeepSeek V4's) residual:+ * VKC_MHC_SPLIT m: the raw mix products hc_fn . x ([S][(2+H)*H], from a matmul),+ * scaled by the rows' 1/rms (eps); pre, post and the Sinkhorn-projected+ * comb (iters, hc_eps; scale[3] and base at prm_off in prm) into+ * hp[hp_off + s*hp_row]: H pre, H post, H*H comb+ * VKC_MHC_COLLAPSE y[s][d] = sum_i pre[i] x[s][i*D + d]+ * VKC_MHC_POST y[s][j*D + d] = sum_i comb[i*H + j] x[s][i*D + d] + post[j] m[s][d]+ * VKC_MHC_MEAN y[s][d] = (sum_i x[s][i*D + d]) / H+ * VKC_SWIGLU_CLAMP y[i] = silu(min(x[i], lim)) * clamp(m[i], -lim, lim), i < n */+#define VKC_MHC_SPLIT 0+#define VKC_MHC_COLLAPSE 1+#define VKC_MHC_POST 2+#define VKC_MHC_MEAN 3+#define VKC_SWIGLU_CLAMP 4+int vkc_mhc_ready(void);+typedef struct { int S, H, D, iters, x_off, x_row, m_off, m_row, hp_off, hp_row, y_off, y_row, prm_off, n;+ float eps, hc_eps, lim; } VkcMhc;+int vkc_mhc(int mode, VkcBuf *x, VkcBuf *m, VkcBuf *hp, VkcBuf *prm, VkcBuf *y, const VkcMhc *p);+/* counters, for the engines' [VK] lines */typedef struct {unsigned long long frames, waits, ops, matmuls, gemms, barriers, bytes_up, bytes_down;diff --git a/docs/vulkan.md b/docs/vulkan.mdindex a651aeae..30bd2e67 100644--- a/docs/vulkan.md+++ b/docs/vulkan.md@@ -885,6 +885,15 @@ records what reads the projections (a DSA indexer reads the normalized q latentleaves in its scratch). Limits: K up to 1024, R up to 128, Q up to 1024 where kv_b's keyrows are read transposed, an indexer of up to 64 heads and 4096 query floats.+GLM-5.3 Flash adds three more things, as ops of their own in `vk_chain.h`, which the+DeepSeek V4 and Kimi K3 chains can take as they are:++| Op | Shader | What it does |+|---|---|---|+| `vkc_dsa_pool_keys`, `vkc_dsa_pool_select` | `chain_dsa` (modes 1, 2) | the k-pooled indexer (`sparse_index.h`): a pool's key, the per-channel softmax mixture of its members' keys under their gate logits plus a position bias, computed once when a step completes the pool; each row's top pools in rank order (score, then the lower pool), their positions, the incomplete tail, -1 in the unused slots |+| `vkc_kda_conv`, `vkc_kda_rec` | `chain_kda` | Kimi Delta Attention (`delta_attention.h`): the short convolution with its window, the gated delta rule with a decay per key row and the state on the device, GLM-5.3's output RMSNorm and sigmoid gate; the l2 sums and the norm's sum in the CPU's order |+| `vkc_mhc` | `chain_mhc` | manifold-constrained hyper-connections (`hyper_connections.h`): the mix logits' split with Sinkhorn, the collapse, the write back, the plain mean of the last collapse; and a clamped SwiGLU |+`make vk-chain-check VK=1` runs them against double-precision references of colibri.c'sabsorbed attention: every weight format both ways through the per-head blocks, RoPE inboth styles in place and into a cache row, LayerNorm with and without a bias, and the@@ -894,7 +903,10 @@ context from a nonzero start, GLM-5.2's head shape, a latent of 1024). MeasuredLavapipe and on an Intel Iris Xe (Mesa's Dozen): the layer's output within 5e-7 of itslargest value and the new cache rows within 7e-7 (the test's bound is 2e-5). Theindexer's selection is checked bit for bit, ties included, on scores that are exact in-float on both sides.+float on both sides. The k-pooled selection is checked slot for slot against `sparse_index.h` the same+way, the KDA layer against `delta_attention.h` over two submissions (the state and the+window carried between them, inputs small enough that the l2 eps counts), the mHC ops+against `hyper_connections.h`; each within 3e-7 on Lavapipe and on the Iris Xe.## Adding an engine to the tier