Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: the dense chain, a layer recorded into one submission

JustVugga4ddd6c95b0c
diff --git a/c/Makefile b/c/Makefile
index 2a91104c..58ff051a 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -529,6 +529,7 @@ TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_frami
test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \
test_e8x4g64_loader test_qwenimage_vae test_vk_tier \
test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contract
+TEST_EXCLUDE += test_vk_chain
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
@@ -708,6 +709,12 @@ endif
VK_OBJ = backend_vulkan.o vk_tier.o
VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \
shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spv
+# vk_chain.o: a layer's dense chain recorded into one submission (vk_chain.h), and
+# its shaders; an engine that does not call it links nothing from it.
+VK_OBJ += vk_chain.o
+VK_SPV += shaders/chain_norm.spv shaders/chain_rope.spv shaders/chain_attn.spv shaders/chain_dnconv.spv \
+ shaders/chain_dnrec.spv shaders/chain_ew.spv shaders/chain_qsa.spv shaders/chain_ple.spv \
+ shaders/chain_gemv.spv
endif
# Optional AMD XDNA2 (Ryzen AI NPU) lane, Windows. Same opt-in shape as VK=1:
@@ -843,6 +850,8 @@ backend_vulkan.o: backend_vulkan.c .build-config
$(CC) $(CFLAGS) -c backend_vulkan.c -o $@
vk_tier.o: vk_tier.c .build-config
$(CC) $(CFLAGS) -c vk_tier.c -o $@
+vk_chain.o: vk_chain.c .build-config
+ $(CC) $(CFLAGS) -c vk_chain.c -o $@
shaders/%.spv: shaders/%.comp
@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
$(GLSLC) --target-env=vulkan1.2 $< -o $@
@@ -1957,6 +1966,16 @@ tests/test_vk_tier$(EXE): tests/test_vk_tier.c $(VK_OBJ) $(VK_SPV)
vk-tier-check: tests/test_vk_tier$(EXE)
./tests/test_vk_tier$(EXE) shaders/qmatmul.spv
+# The dense chain's ops (vk_chain.c, shaders/chain_*.comp) against CPU references, on
+# a Vulkan device (CI: Lavapipe); like test_vk_tier it needs VK=1 and stays out of
+# `check`.
+tests/test_vk_chain$(EXE): tests/test_vk_chain.c $(VK_OBJ) $(VK_SPV)
+ @[ "$(VK)" = 1 ] || { echo "tests/test_vk_chain needs VK=1" >&2; exit 1; }
+ $(CC) $(CFLAGS) $< $(VK_OBJ) -o $@ $(LDFLAGS)
+.PHONY: vk-chain-check
+vk-chain-check: tests/test_vk_chain$(EXE)
+ ./tests/test_vk_chain$(EXE) shaders/qmatmul.spv
+
tests/test_rans$(EXE): tests/test_rans.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index b4cd83e0..7cd85bfd 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -1219,6 +1219,46 @@ int coli_vk_dense_decide(const char *engine, int tier_on, int def) {
}
int coli_vk_dense(void) { return g_dense_on; }
+/* Where a layer's dense chain runs (vk_chain.c): the whole layer recorded into one
+ * submission with the residual stream and the recurrent/KV state on the device, the
+ * alternative to one synchronous coli_vk_matmul per matrix. Returns COLI_VK_CHAIN_OFF,
+ * _ON (every forward) or _PREFILL (forwards of more than two rows: prompts, not decode
+ * steps or an MTP verify).
+ * COLI_VK_CHAIN set and non-empty: 0 off, 2 prefill only, any other number on.
+ * Unset, a discrete GPU: on.
+ * Unset, an integrated GPU: `igpu` when the engine runs the expert tier (tier_on),
+ * else off. `igpu` is what the engine measured on one (a Radeon 780M, docs/vulkan.md,
+ * "The dense chain"): qwen36 passes ON (its chain won decode, 9.9 against 8.0 tok/s,
+ * and prefill, 9.5 against 12.2 s); qwen38 OFF (its chain won prefill, 30.1 against
+ * 38.7 s, and lost decode, 3.2 against 3.8 tok/s: the device's GEMV at the GPU's
+ * 800 MHz floor is slower than the CPU's int8 one on a 4.3 GB trunk; prompts only
+ * still lost 5% of decode, the trunk's device copy taking from the tier's budget).
+ * Unset, a CPU device (Lavapipe): off.
+ * Printed as a [VK] line with an engine name (NULL: silent). */
+int coli_vk_chain_decide(const char *engine, int tier_on, int igpu) {
+ const char *e = getenv("COLI_VK_CHAIN");
+ char why[192];
+ int on;
+ if (e && *e) {
+ int v = atoi(e);
+ on = v == 0 ? COLI_VK_CHAIN_OFF : v == 2 ? COLI_VK_CHAIN_PREFILL : COLI_VK_CHAIN_ON;
+ snprintf(why, sizeof why, "COLI_VK_CHAIN=%s", e);
+ } else if (coli_vk_device_integrated()) {
+ on = tier_on ? igpu : COLI_VK_CHAIN_OFF;
+ snprintf(why, sizeof why, "an integrated GPU%s: %s; COLI_VK_CHAIN=0 off, 1 on, 2 prompts only",
+ tier_on ? " with the expert tier" : " without the expert tier",
+ on == COLI_VK_CHAIN_ON ? "measured faster on decode and prefill"
+ : on == COLI_VK_CHAIN_PREFILL ? "measured faster on prefill, slower on decode"
+ : tier_on ? "measured slower on decode" : "not measured");
+ } else if (coli_vk_device_shares_ram()) {
+ on = COLI_VK_CHAIN_OFF;
+ snprintf(why, sizeof why, "a CPU device; COLI_VK_CHAIN=1 turns it on");
+ } else { on = COLI_VK_CHAIN_ON; snprintf(why, sizeof why, "a discrete GPU%s", tier_on ? ", beside the expert tier" : ""); }
+ if (engine) fprintf(stderr, "[VK] %s: dense chain %s (%s)\n", engine,
+ on == COLI_VK_CHAIN_ON ? "on" : on == COLI_VK_CHAIN_PREFILL ? "on for prompts" : "off", why);
+ return on;
+}
+
int coli_vk_init_env_tier(const char *engine, int tier_on) {
const char *on = getenv("COLI_VULKAN");
if (!on || !atoi(on)) return 0;
@@ -2675,6 +2715,38 @@ void coli_vk_shutdown(void) {
memset(&G, 0, sizeof(G));
}
+/* ---- the dense chain's view of the device (vk_chain.c) -------------------------
+ * The chain records its own command buffers on the main queue, from the engine
+ * thread, with pipelines of its own built from the same shader directory; it reads
+ * the resident tensors the engines already uploaded (their VkBuffers below). */
+int coli_vk_core(ColiVkCore *o) {
+ if (!G.ready || !o) return 0;
+ memset(o, 0, sizeof *o);
+ o->instance = (void *)G.inst; o->phys = (void *)G.phys; o->device = (void *)G.dev;
+ o->queue = (void *)G.queue; o->qfam = G.qfam;
+ o->memtype_host = G.memtype; o->memtype_cached = G.memtype_cached; o->memtype_dev = G.memtype_dev;
+ o->ssbo_align = G.ssbo_align; o->ssbo_range = G.ssbo_range; o->spv_path = G.spv_path;
+ o->has_prio = G.has_prio;
+ for (int k = 0; k < VK_GEMM_SLOTS && k < 4; k++) {
+ if (!G.pipe_gemm[k]) break;
+ VkGemmTile t = G.gemm_t[k];
+ o->gemm_tile[k][0] = t.bm; o->gemm_tile[k][1] = t.bn; o->gemm_tile[k][2] = t.bk;
+ o->gemm_tile[k][3] = t.tm; o->gemm_tile[k][4] = t.tn; o->gemm_tile[k][5] = t.pf;
+ o->gemm_tiles = k + 1;
+ }
+ o->gemm_min_s = G.gemm_min_s; o->gemm_min_so = G.gemm_min_so;
+ o->integrated = coli_vk_device_integrated(); o->shares_ram = coli_vk_device_shares_ram();
+ return 1;
+}
+int coli_vk_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *o) {
+ if (!t || !o || t->dev != 0) return 0;
+ o->wbuf = (void *)t->wbuf; o->sbuf = (void *)t->sbuf;
+ o->fmt = t->fmt; o->I = t->I; o->O = t->O; o->rowWords = t->rowWords; o->gs = t->gs;
+ return 1;
+}
+/* A fence the chain waited on failed: the device is gone, everyone falls back. */
+void coli_vk_mark_lost(void) { G.ready = 0; }
+
#ifdef VK_TEST
// ---- standalone GPU-vs-CPU validation + microbench --------------------------
#include <math.h>
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index d7f0f315..1c3ac2ed 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -230,6 +230,34 @@ const char *coli_vk_shader_path(char *buf, size_t n);
/* How many coli_vk_matmul calls ran on the device: a check that a path is really used. */
unsigned long long coli_vk_matmul_calls(void);
+/* ---- the dense chain (vk_chain.h) ----------------------------------------------
+ * Whether an engine runs its layers' dense chain on the device (COLI_VK_CHAIN): set,
+ * 0 off, 2 prompts only, else on; unset, on for a discrete GPU, `igpu` (the engine's
+ * measured choice) on an integrated GPU with the expert tier on, off otherwise. With an
+ * engine name the decision is printed as a [VK] line. */
+#define COLI_VK_CHAIN_OFF 0
+#define COLI_VK_CHAIN_ON 1
+#define COLI_VK_CHAIN_PREFILL 2 /* forwards of more than two rows only */
+int coli_vk_chain_decide(const char *engine, int tier_on, int igpu);
+/* The device as the chain sees it: Vulkan handles as void * (VkInstance,
+ * VkPhysicalDevice, VkDevice, VkQueue), the memory types the backend picked, the
+ * shader directory's qmatmul.spv and the fp32 GEMM's tiles. 0 before coli_vk_init. */
+typedef struct {
+ void *instance, *phys, *device, *queue;
+ uint32_t qfam, memtype_host, memtype_cached, memtype_dev;
+ size_t ssbo_align, ssbo_range;
+ const char *spv_path;
+ int gemm_tiles, gemm_tile[4][6]; /* bm, bn, bk, tm, tn, pf */
+ int gemm_min_s, gemm_min_so;
+ int has_prio, integrated, shares_ram;
+} ColiVkCore;
+int coli_vk_core(ColiVkCore *out);
+/* A resident tensor's buffers (VkBuffer as void *) and layout; 0 for a COLI_VK_DEV2 one. */
+typedef struct { void *wbuf, *sbuf; int fmt, I, O, rowWords, gs; } ColiVkTensorInfo;
+int coli_vk_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *out);
+/* The chain's fence wait failed: the device is lost, the backend stops. */
+void coli_vk_mark_lost(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/c/shaders/chain_attn.comp b/c/shaders/chain_attn.comp
new file mode 100644
index 00000000..655f083b
--- /dev/null
+++ b/c/shaders/chain_attn.comp
@@ -0,0 +1,108 @@
+#version 450
+// The dense chain's attention core (vk_chain.c, VKC_P_ATTN): grouped-query attention
+// of S query rows over a KV cache that lives on the device, with Qwen's output gate.
+// One workgroup per (head h = WorkGroupID.x, row s = WorkGroupID.y):
+// score_t = (q . K[kvh][t]) * scale over the row's positions,
+// ctx = sum_t softmax(score)_t V[kvh][t], times sigmoid(gate) when has_gate.
+// The positions are 0..pos_base+s (causal) or, with a selection list (Qwen3.8's QSA
+// indexer, chain_qsa.comp), the row's list: sel[sel_off + s*sel_row] = count, then the
+// positions; a negative count means every position 0..pos_base+s.
+// The softmax runs online over tiles of 128 positions (running max and sum, the
+// accumulator rescaled when the max grows), so no score row is stored and a context
+// of any length costs one pass over K and V. The CPU's order of the sums differs (it
+// normalizes first); the two agree to rounding.
+// Cache layout as the engines keep it on the host: K[(kvh*cap + t)*hd + d].
+// Limits: hd <= 256 (two dims per thread).
+layout(local_size_x = 128) in;
+
+layout(std430, binding = 0) readonly buffer QB { float q[]; };
+layout(std430, binding = 1) readonly buffer KB { float kc[]; };
+layout(std430, binding = 2) readonly buffer VB { float vc[]; };
+layout(std430, binding = 3) buffer OB { float o[]; };
+layout(std430, binding = 4) readonly buffer GB { float gt[]; };
+layout(std430, binding = 5) readonly buffer LB { int sel[]; };
+
+layout(push_constant) uniform P {
+ int S, H, KVH, hd, pos_base, cap;
+ int q_off, q_row, q_seg;
+ int g_off, g_row, g_seg, has_gate;
+ int o_off, o_row;
+ int sel_off, sel_row;
+ float scale;
+ int k_off, v_off; // where this layer's cache starts in kc / vc
+} p;
+
+shared float qs[256];
+shared float pe[128];
+shared int pt[128];
+shared float red[128];
+
+void main() {
+ int h = int(gl_WorkGroupID.x), s = int(gl_WorkGroupID.y);
+ int tid = int(gl_LocalInvocationID.x);
+ int kvh = h / (p.H / p.KVH);
+ int qb = p.q_off + s * p.q_row + h * p.q_seg;
+ for (int d = tid; d < p.hd; d += 128) qs[d] = q[qb + d];
+ int n = p.pos_base + s + 1;
+ bool list = false;
+ int lb = 0;
+ if (p.sel_row > 0) {
+ lb = p.sel_off + s * p.sel_row;
+ int c = sel[lb];
+ if (c >= 0) { n = c; list = true; }
+ }
+ barrier();
+ float m = -3.0e38, l = 0.0, acc0 = 0.0, acc1 = 0.0;
+ int kbase = kvh * p.cap;
+ for (int t0 = 0; t0 < n; t0 += 128) {
+ int i = t0 + tid;
+ float sv = -3.0e38;
+ if (i < n) {
+ int t = list ? sel[lb + 1 + i] : i;
+ pt[tid] = t;
+ int kb = p.k_off + (kbase + t) * p.hd;
+ float a = 0.0;
+ for (int d = 0; d < p.hd; d++) a += qs[d] * kc[kb + d];
+ sv = a * p.scale;
+ }
+ red[tid] = sv;
+ barrier();
+ for (int off = 64; off > 0; off >>= 1) {
+ if (tid < off) red[tid] = max(red[tid], red[tid + off]);
+ barrier();
+ }
+ float mn = max(m, red[0]);
+ barrier();
+ float e = i < n ? exp(sv - mn) : 0.0;
+ pe[tid] = e;
+ red[tid] = e;
+ barrier();
+ for (int off = 64; off > 0; off >>= 1) {
+ if (tid < off) red[tid] += red[tid + off];
+ barrier();
+ }
+ float corr = exp(m - mn);
+ l = l * corr + red[0];
+ acc0 *= corr; acc1 *= corr;
+ int cnt = min(128, n - t0);
+ int d0 = tid, d1 = tid + 128;
+ for (int j = 0; j < cnt; j++) {
+ int vb = p.v_off + (kbase + pt[j]) * p.hd;
+ float w = pe[j];
+ if (d0 < p.hd) acc0 += w * vc[vb + d0];
+ if (d1 < p.hd) acc1 += w * vc[vb + d1];
+ }
+ m = mn;
+ barrier();
+ }
+ float inv = l > 0.0 ? 1.0 / l : 0.0;
+ int ob = p.o_off + s * p.o_row + h * p.hd;
+ int gb = p.g_off + s * p.g_row + h * p.g_seg;
+ for (int k = 0; k < 2; k++) {
+ int d = tid + 128 * k;
+ if (d >= p.hd) break;
+ float v = (k == 0 ? acc0 : acc1) * inv;
+ if (p.has_gate != 0) v *= 1.0 / (1.0 + exp(-gt[gb + d]));
+ o[ob + d] = v;
+ }
+}
diff --git a/c/shaders/chain_attn.spv b/c/shaders/chain_attn.spv
new file mode 100644
index 00000000..43fffc79
Binary files /dev/null and b/c/shaders/chain_attn.spv differ
diff --git a/c/shaders/chain_dnconv.comp b/c/shaders/chain_dnconv.comp
new file mode 100644
index 00000000..830c2d97
--- /dev/null
+++ b/c/shaders/chain_dnconv.comp
@@ -0,0 +1,53 @@
+#version 450
+// Gated DeltaNet's causal depthwise convolution (vk_chain.c, VKC_P_DNCONV): one thread
+// per channel c walks the S rows in order with the channel's ring of the last ck-1
+// inputs (which lives on the device across steps):
+// acc = sum_k w[c][k] * ring[k] + w[c][ck-1] * in[s][c] (order 0: the taps first)
+// acc = w[c][ck-1] * in[s][c] + sum_k w[c][k] * ring[k] (order 1: the input first)
+// out[s][c] = silu(acc); the ring drops its oldest input and takes in[s][c].
+// The two orders are the two engines' (qwen36.c, qwen38_core.h), so the sum rounds as
+// on the CPU. snap_row >= 0: the ring as it stands after that row is copied to snap
+// (the state an MTP verify's first row leaves, for the rollback of its draft).
+layout(local_size_x = 64) in;
+
+layout(std430, binding = 0) readonly buffer IN { float xin[]; };
+layout(std430, binding = 1) readonly buffer WB { float w[]; };
+layout(std430, binding = 2) buffer RB { float ring[]; };
+layout(std430, binding = 3) writeonly buffer OB { float o[]; };
+layout(std430, binding = 4) writeonly buffer SB { float snap[]; };
+
+layout(push_constant) uniform P {
+ int S, CD, CK;
+ int in_off, in_row, out_off, out_row;
+ int snap_row, order;
+ int w_off, ring_off, snap_off;
+} p;
+
+void main() {
+ int c = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 64u);
+ if (c >= p.CD) return;
+ int nh = p.CK - 1;
+ float hist[8];
+ for (int k = 0; k < 8; k++) hist[k] = k < nh ? ring[p.ring_off + c * nh + k] : 0.0;
+ int wb = p.w_off + c * p.CK;
+ float wl = w[wb + p.CK - 1];
+ for (int s = 0; s < p.S; s++) {
+ float cur = xin[p.in_off + s * p.in_row + c];
+ float acc;
+ /* constant trip counts: the ring stays in registers */
+ if (p.order == 0) {
+ acc = 0.0;
+ for (int k = 0; k < 8; k++) if (k < nh) acc += w[wb + k] * hist[k];
+ acc += wl * cur;
+ } else {
+ acc = wl * cur;
+ for (int k = 0; k < 8; k++) if (k < nh) acc += w[wb + k] * hist[k];
+ }
+ o[p.out_off + s * p.out_row + c] = acc / (1.0 + exp(-acc));
+ for (int k = 0; k < 7; k++) if (k + 1 < nh) hist[k] = hist[k + 1];
+ for (int k = 0; k < 8; k++) if (k == nh - 1) hist[k] = cur;
+ if (s == p.snap_row)
+ for (int k = 0; k < 8; k++) if (k < nh) snap[p.snap_off + c * nh + k] = hist[k];
+ }
+ for (int k = 0; k < 8; k++) if (k < nh) ring[p.ring_off + c * nh + k] = hist[k];
+}
diff --git a/c/shaders/chain_dnconv.spv b/c/shaders/chain_dnconv.spv
new file mode 100644
index 00000000..7e2ed88b
Binary files /dev/null and b/c/shaders/chain_dnconv.spv differ
diff --git a/c/shaders/chain_dnrec.comp b/c/shaders/chain_dnrec.comp
new file mode 100644
index 00000000..cfa0568f
--- /dev/null
+++ b/c/shaders/chain_dnrec.comp
@@ -0,0 +1,108 @@
+#version 450
+#extension GL_EXT_control_flow_attributes : require
+// Gated DeltaNet's recurrence with its gated RMSNorm (vk_chain.c, VKC_P_DNREC), one
+// workgroup per value head h, rows in order. The state S[h] = [KD][VD] lives on the
+// device across steps; thread v owns its column S[h][:, v] in registers for the whole
+// call (KD is a specialization constant, so the column is a register array).
+// Per row, from the convolution's output cv (q, k at key head h / (VH/KH), v at h):
+// q, k <- l2-normalized (eps 1e-6 inside the root), q also times qscale (1/sqrt(KD))
+// beta = sigmoid(b[s][h]); decay = exp(-exp(A_log[h]) * softplus(a[s][h] + dt_bias[h]))
+// S *= decay; kv = k . S; delta = (v - kv) * beta; S += k (x) delta; o = q . S
+// y[s][h*VD + v] = o * rsqrt(mean(o^2) + eps) * norm[v] * gate(z[s][h*VD + v])
+// with gate = silu (flags 0, Qwen3.6) or sigmoid (flags 1, Qwen3.8's gated norm).
+// Every column runs the CPU's own sequence of products and sums (the k loop in order);
+// the norms reduce in float where the CPU uses double.
+// snap_row >= 0: the state after that row is also written to snap (MTP verify).
+// Limits: VD <= 128 (one column per thread), KD the specialization constant; the k
+// loops are unrolled so the column stays in registers.
+layout(local_size_x = 128) in;
+layout(constant_id = 0) const int KD = 128;
+
+layout(std430, binding = 0) readonly buffer CV { float cv[]; }; // conv output rows
+layout(std430, binding = 1) readonly buffer AB { float ab[]; }; // b and a projections
+layout(std430, binding = 2) readonly buffer ZB { float z[]; }; // z projection rows
+layout(std430, binding = 3) buffer ST { float st[]; }; // state [VH][KD][VD]
+layout(std430, binding = 4) readonly buffer PB { float prm[]; }; // A_log[VH], dt_bias[VH], norm[VD]
+layout(std430, binding = 5) writeonly buffer OB { float y[]; };
+layout(std430, binding = 6) writeonly buffer SB { float snap[]; };
+
+layout(push_constant) uniform P {
+ int S, VH, KH, VD, Ktot;
+ int cv_off, cv_row;
+ int b_off, b_row, a_off, a_row;
+ int z_off, z_row, y_off, y_row;
+ int snap_row, flags;
+ float eps, qscale;
+ int st_off, snap_off, prm_off;
+} p;
+
+shared float qn[KD];
+shared float kn[KD];
+shared float red[128];
+shared float red2[128];
+
+float softplus(float x) {
+ if (x > 20.0) return x;
+ float e = exp(x);
+ return e < 1e-4 ? e - 0.5 * e * e : log(1.0 + e); // log1p
+}
+
+void main() {
+ int h = int(gl_WorkGroupID.x);
+ int v = int(gl_LocalInvocationID.x);
+ bool own = v < p.VD;
+ int kh = h / (p.VH / p.KH);
+ float col[KD];
+ int sb = p.st_off + h * KD * p.VD, nb = p.snap_off + h * KD * p.VD;
+ [[unroll]] for (int k = 0; k < KD; k++) col[k] = own ? st[sb + k * p.VD + v] : 0.0;
+ float alog = prm[p.prm_off + h], dtb = prm[p.prm_off + p.VH + h];
+ float nw = own ? prm[p.prm_off + 2 * p.VH + v] : 0.0;
+ for (int s = 0; s < p.S; s++) {
+ int cb = p.cv_off + s * p.cv_row;
+ float qa = 0.0, ka = 0.0;
+ for (int k = v; k < KD; k += 128) {
+ float qv = cv[cb + kh * KD + k], kv = cv[cb + p.Ktot + kh * KD + k];
+ qn[k] = qv; kn[k] = kv;
+ qa += qv * qv; ka += kv * kv;
+ }
+ red[v] = qa; red2[v] = ka;
+ barrier();
+ for (int off = 64; off > 0; off >>= 1) {
+ if (v < off) { red[v] += red[v + off]; red2[v] += red2[v + off]; }
+ barrier();
+ }
+ float qsc = 1.0 / sqrt(red[0] + 1e-6) * p.qscale;
+ float ksc = 1.0 / sqrt(red2[0] + 1e-6);
+ barrier();
+ for (int k = v; k < KD; k += 128) { qn[k] *= qsc; kn[k] *= ksc; }
+ barrier();
+ float bv = ab[p.b_off + s * p.b_row + h], av = ab[p.a_off + s * p.a_row + h];
+ float beta = 1.0 / (1.0 + exp(-bv));
+ float decay = exp(-exp(alog) * softplus(av + dtb));
+ float o = 0.0;
+ if (own) {
+ float vin = cv[cb + 2 * p.Ktot + h * p.VD + v];
+ float kvs = 0.0;
+ [[unroll]] for (int k = 0; k < KD; k++) { col[k] *= decay; kvs += kn[k] * col[k]; }
+ float delta = (vin - kvs) * beta;
+ [[unroll]] for (int k = 0; k < KD; k++) col[k] += kn[k] * delta;
+ [[unroll]] for (int k = 0; k < KD; k++) o += qn[k] * col[k];
+ }
+ red[v] = o * o;
+ barrier();
+ for (int off = 64; off > 0; off >>= 1) {
+ if (v < off) red[v] += red[v + off];
+ barrier();
+ }
+ float r = 1.0 / sqrt(red[0] / float(p.VD) + p.eps);
+ if (own) {
+ float zv = z[p.z_off + s * p.z_row + h * p.VD + v];
+ float g = (p.flags & 1) != 0 ? 1.0 / (1.0 + exp(-zv)) : zv / (1.0 + exp(-zv));
+ y[p.y_off + s * p.y_row + h * p.VD + v] = o * r * nw * g;
+ if (s == p.snap_row)
+ [[unroll]] for (int k = 0; k < KD; k++) snap[nb + k * p.VD + v] = col[k];
+ }
+ barrier();
+ }
+ if (own) [[unroll]] for (int k = 0; k < KD; k++) st[sb + k * p.VD + v] = col[k];
+}
diff --git a/c/shaders/chain_dnrec.spv b/c/shaders/chain_dnrec.spv
new file mode 100644
index 00000000..50e1c1c1
Binary files /dev/null and b/c/shaders/chain_dnrec.spv differ
diff --git a/c/shaders/chain_ew.comp b/c/shaders/chain_ew.comp
new file mode 100644
index 00000000..decabdf5
--- /dev/null
+++ b/c/shaders/chain_ew.comp
@@ -0,0 +1,68 @@
+#version 450
+// The dense chain's element-wise steps (vk_chain.c, VKC_P_EW), one op per dispatch,
+// one thread per output element (i < n). Buffers: y (written), a, b, c, e (read; any
+// may alias another or y where an op reads and writes the same element).
+// 0 ADD y[i] = a[i] + b[i] (the residual add)
+// 1 COMBINE the MoE output joining the residual, row r = i / D, d = i % D:
+// t = (flags&1 ? b[r*D+d] : 0) (routed experts, summed on the host)
+// t = t + g * c[r*D+d], g = flags&4 ? sigmoid(e[r*e_row]) : 1 (flags&2: the shared expert)
+// y = flags&8 ? t : a[r*D+d] + t
+// which is the CPU's order: routed in rank order, then the gated shared expert,
+// then the residual.
+// 2 SWIGLU y[i] = silu(a[i]) * b[i] (the shared expert's activation)
+// 3 HC_LOW y[i] = silu(a[i] / fc) (Qwen3.8 hyper-connection read, fc = streams)
+// 4 HC_MIX y[r*D+d] = sum_k sigmoid(a[r*W+k*D+d]) * b[r*W+k*D+d] / fc, k < C, W = C*D
+// 5 HC_INJ y[i] = 2 * sigmoid(a[i] / fc)
+// 6 HC_APPLY y[r*W+k*D+d] += a[r*C+k] * b[r*D+d] (i over S*C*D)
+// Offsets in floats: y_off, a_off, b_off, c_off, e_off.
+layout(local_size_x = 256) in;
+
+layout(std430, binding = 0) buffer YB { float y[]; };
+layout(std430, binding = 1) readonly buffer AB { float a[]; };
+layout(std430, binding = 2) readonly buffer BB { float b[]; };
+layout(std430, binding = 3) readonly buffer CB { float c[]; };
+layout(std430, binding = 4) readonly buffer EB { float e[]; };
+
+layout(push_constant) uniform P {
+ int op, n, D, C, flags, e_row;
+ int y_off, a_off, b_off, c_off, e_off;
+ float fc;
+} p;
+
+float sig(float x) { return 1.0 / (1.0 + exp(-x)); }
+
+void main() {
+ int i = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);
+ if (i >= p.n) return;
+ if (p.op == 0) {
+ y[p.y_off + i] = a[p.a_off + i] + b[p.b_off + i];
+ } else if (p.op == 1) {
+ int r = i / p.D, d = i - r * p.D;
+ float t = (p.flags & 1) != 0 ? b[p.b_off + r * p.D + d] : 0.0;
+ if ((p.flags & 2) != 0) {
+ float g = (p.flags & 4) != 0 ? sig(e[p.e_off + r * p.e_row]) : 1.0;
+ t = t + g * c[p.c_off + r * p.D + d];
+ }
+ y[p.y_off + i] = (p.flags & 8) != 0 ? t : a[p.a_off + i] + t;
+ } else if (p.op == 2) {
+ float g = a[p.a_off + i];
+ y[p.y_off + i] = (g / (1.0 + exp(-g))) * b[p.b_off + i];
+ } else if (p.op == 3) {
+ float v = a[p.a_off + i] / p.fc;
+ y[p.y_off + i] = v * sig(v);
+ } else if (p.op == 4) {
+ int r = i / p.D, d = i - r * p.D, W = p.C * p.D;
+ float v = 0.0;
+ for (int k = 0; k < p.C; k++) {
+ int j = r * W + k * p.D + d;
+ v += sig(a[p.a_off + j]) * b[p.b_off + j];
+ }
+ y[p.y_off + i] = v / p.fc;
+ } else if (p.op == 5) {
+ y[p.y_off + i] = 2.0 * sig(a[p.a_off + i] / p.fc);
+ } else if (p.op == 6) {
+ int W = p.C * p.D;
+ int r = i / W, rem = i - r * W, k = rem / p.D, d = rem - k * p.D;
+ y[p.y_off + i] += a[p.a_off + r * p.C + k] * b[p.b_off + r * p.D + d];
+ }
+}
diff --git a/c/shaders/chain_ew.spv b/c/shaders/chain_ew.spv
new file mode 100644
index 00000000..c261bc08
Binary files /dev/null and b/c/shaders/chain_ew.spv differ
diff --git a/c/shaders/chain_gemv.comp b/c/shaders/chain_gemv.comp
new file mode 100644
index 00000000..4febbda9
--- /dev/null
+++ b/c/shaders/chain_gemv.comp
@@ -0,0 +1,102 @@
+#version 450
+// The dense chain's decode GEMV (vk_chain.c, VKC_P_GEMV4): y[s,o] = sum_i x[s,i] *
+// dequant(W[o,i]), the qmatmul.comp contract (same bindings, push constants and weight
+// layout) for the formats a dense trunk uses, read 16 bytes per lane per step:
+// 1 int8 rows, one scale per row (16 weights per step)
+// 4 int4 in groups of 64, nibble - 8, low nibble = even column (32 per step)
+// 10 f32 (4 per step), 11 bf16, low half = even column (8 per step)
+// qmatmul.comp reads one 32-bit word per lane per step and checks every element
+// against I; on an integrated GPU at its floor clock that leaves the GEMV short of
+// loads in flight and long on instructions (measured on a Radeon 780M: int8 at 22 to
+// 31 GB/s of weights). Here a lane loads a uvec4, decodes with one bitfield extract
+// per value, and four partial sums run side by side; x[s,:] sits in shared memory
+// padded with zeros to the row's padded length, and the weight rows' padding is zero
+// too (the backend zeroes it), so the steps carry no bounds checks.
+// A row is spread over lpr lanes (a power of two up to the subgroup, from the host:
+// about eight steps per lane), so a subgroup works on subgroup/lpr rows at once and
+// a short row does not leave most of its steps to the per-row reduction.
+// The host takes this shader only when a row is a whole number of 16-byte steps
+// (rowWords % 4 == 0), I % 4 == 0, and the padded row fits the staging array (XS
+// vec4s, a specialization constant from the device's shared-memory limit).
+#extension GL_KHR_shader_subgroup_arithmetic : require
+#extension GL_KHR_shader_subgroup_shuffle : require
+#extension GL_EXT_control_flow_attributes : require
+layout(local_size_x = 256) in;
+layout(constant_id = 0) const int XS = 2048;
+
+layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; };
+layout(std430, binding = 1) readonly buffer W4 { uvec4 w4[]; };
+layout(std430, binding = 2) readonly buffer Sc { float scale[]; };
+layout(std430, binding = 3) writeonly buffer Y { float y[]; };
+
+layout(push_constant) uniform P {
+ int fmt, S, I, O, rowWords, gs;
+ int lpr; // lanes per row, a power of two
+} p;
+
+shared vec4 xs[XS];
+
+float sb(uint w, int off) { return float(bitfieldExtract(int(w), off, 8)); } // signed byte
+float nb(uint w, int off) { return float(bitfieldExtract(w, off, 4)) - 8.0; } // nibble - 8
+vec2 bf(uint w) { return vec2(uintBitsToFloat(w << 16), uintBitsToFloat(w & 0xffff0000u)); }
+
+void main() {
+ int s = int(gl_WorkGroupID.y);
+ int tid = int(gl_LocalInvocationID.x);
+ int nq = p.rowWords / 4; // uvec4 steps per row
+ int per = p.fmt == 1 ? 16 : p.fmt == 4 ? 32 : p.fmt == 11 ? 8 : 4; // values per step
+ int nx4 = nq * per / 4, i4 = p.I / 4; // padded x length in vec4s
+ for (int k = tid; k < nx4; k += 256) xs[k] = k < i4 ? x4[s * i4 + k] : vec4(0.0);
+ barrier();
+ int sg = int(gl_SubgroupID), sgs = int(gl_SubgroupSize);
+ int lpr = min(p.lpr, sgs), rps = sgs / lpr;
+ int sub = int(gl_SubgroupInvocationID) / lpr, lane = int(gl_SubgroupInvocationID) % lpr;
+ int nsg = 256 / sgs, stride = int(gl_NumWorkGroups.x) * nsg * rps;
+ int ng = p.gs > 0 ? (p.I + p.gs - 1) / p.gs : 1;
+ for (int o0 = (int(gl_WorkGroupID.x) * nsg + sg) * rps; o0 < p.O; o0 += stride) {
+ int o = o0 + sub;
+ bool live = o < p.O;
+ int rb = o * nq, nql = live ? nq : 0;
+ float a0 = 0.0, a1 = 0.0, a2 = 0.0, a3 = 0.0;
+ if (p.fmt == 1) {
+ for (int q = lane; q < nql; q += lpr) {
+ uvec4 w = w4[rb + q]; int xb = q * 4;
+ vec4 x0 = xs[xb], x1 = xs[xb + 1], x2 = xs[xb + 2], x3 = xs[xb + 3];
+ a0 += x0.x * sb(w.x, 0) + x0.y * sb(w.x, 8) + x0.z * sb(w.x, 16) + x0.w * sb(w.x, 24);
+ a1 += x1.x * sb(w.y, 0) + x1.y * sb(w.y, 8) + x1.z * sb(w.y, 16) + x1.w * sb(w.y, 24);
+ a2 += x2.x * sb(w.z, 0) + x2.y * sb(w.z, 8) + x2.z * sb(w.z, 16) + x2.w * sb(w.z, 24);
+ a3 += x3.x * sb(w.w, 0) + x3.y * sb(w.w, 8) + x3.z * sb(w.w, 16) + x3.w * sb(w.w, 24);
+ }
+ } else if (p.fmt == 4) {
+ for (int q = lane; q < nql; q += lpr) {
+ uvec4 w = w4[rb + q]; int xb = q * 8;
+ float sc = scale[o * ng + (q * 32) / p.gs];
+ float t0 = 0.0, t1 = 0.0, t2 = 0.0, t3 = 0.0;
+ [[unroll]] for (int k = 0; k < 2; k++) {
+ vec4 xa = xs[xb + k], xc = xs[xb + 2 + k], xe = xs[xb + 4 + k], xg = xs[xb + 6 + k];
+ int sh = k * 16;
+ t0 += xa.x * nb(w.x, sh) + xa.y * nb(w.x, sh + 4) + xa.z * nb(w.x, sh + 8) + xa.w * nb(w.x, sh + 12);
+ t1 += xc.x * nb(w.y, sh) + xc.y * nb(w.y, sh + 4) + xc.z * nb(w.y, sh + 8) + xc.w * nb(w.y, sh + 12);
+ t2 += xe.x * nb(w.z, sh) + xe.y * nb(w.z, sh + 4) + xe.z * nb(w.z, sh + 8) + xe.w * nb(w.z, sh + 12);
+ t3 += xg.x * nb(w.w, sh) + xg.y * nb(w.w, sh + 4) + xg.z * nb(w.w, sh + 8) + xg.w * nb(w.w, sh + 12);
+ }
+ a0 += (t0 + t1 + t2 + t3) * sc;
+ }
+ } else if (p.fmt == 11) {
+ for (int q = lane; q < nql; q += lpr) {
+ uvec4 w = w4[rb + q]; int xb = q * 2;
+ vec4 x0 = xs[xb], x1 = xs[xb + 1];
+ vec2 b0 = bf(w.x), b1 = bf(w.y), b2 = bf(w.z), b3 = bf(w.w);
+ a0 += x0.x * b0.x + x0.y * b0.y;
+ a1 += x0.z * b1.x + x0.w * b1.y;
+ a2 += x1.x * b2.x + x1.y * b2.y;
+ a3 += x1.z * b3.x + x1.w * b3.y;
+ }
+ } else {
+ for (int q = lane; q < nql; q += lpr) a0 += dot(xs[q], uintBitsToFloat(w4[rb + q]));
+ }
+ float tot = (a0 + a1) + (a2 + a3);
+ for (int off = lpr / 2; off > 0; off >>= 1) tot += subgroupShuffleXor(tot, uint(off));
+ if (lane == 0 && live) y[s * p.O + o] = p.fmt == 1 ? tot * scale[o] : tot;
+ }
+}
diff --git a/c/shaders/chain_gemv.spv b/c/shaders/chain_gemv.spv
new file mode 100644
index 00000000..8fc2700c
Binary files /dev/null and b/c/shaders/chain_gemv.spv differ
diff --git a/c/shaders/chain_norm.comp b/c/shaders/chain_norm.comp
new file mode 100644
index 00000000..4089e488
--- /dev/null
+++ b/c/shaders/chain_norm.comp
@@ -0,0 +1,54 @@
+#version 450
+// The dense chain's RMSNorm (vk_chain.c, VKC_P_NORM): nseg segments of D floats, each
+// y = x * r * weight * post, r = 1 / sqrt(mean(x^2) + eps)
+// One workgroup per segment. Segment g sits in row g / per_row, at index j = g % per_row
+// of that row: x[x_off + row*x_row + j*x_seg + i] (the same for y). That covers a row of
+// the residual stream (per_row 1), the heads of a projection (q/k norms: per_row heads,
+// x_seg the head's stride, which may exceed D when a gate follows the query) and the
+// hyper-connection streams of Qwen3.8 (per_row 4, a weight slice per stream).
+// weight of segment j: w[w_off + (w_mod > 0 ? (j % w_mod) * D : 0) + i]
+// flags 1: zero-centred weight, scale by (1 + w) (Qwen3.6/3.8 text norms)
+// flags 2: no weight (scale by 1)
+// flags 4: L2 norm: r = 1 / sqrt(sum(x^2) + eps) (no mean)
+// x and y may be the same buffer and the same places (in place): every element is read
+// for the sum before the barrier and written after it by the thread that reads it.
+// The sum is a float tree; the CPU sums in double. The two differ by rounding only.
+layout(local_size_x = 256) in;
+
+layout(std430, binding = 0) buffer X { float x[]; };
+layout(std430, binding = 1) readonly buffer W { float w[]; };
+layout(std430, binding = 2) buffer Y { float y[]; };
+
+layout(push_constant) uniform P {
+ int nseg, D, per_row;
+ int x_off, x_row, x_seg;
+ int y_off, y_row, y_seg;
+ int w_off, w_mod, flags;
+ float eps, post;
+} p;
+
+shared float red[256];
+
+void main() {
+ int g = int(gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x);
+ if (g >= p.nseg) return; // uniform across the workgroup
+ int row = g / p.per_row, j = g - row * p.per_row;
+ int xb = p.x_off + row * p.x_row + j * p.x_seg;
+ int yb = p.y_off + row * p.y_row + j * p.y_seg;
+ int wb = p.w_off + (p.w_mod > 0 ? (j % p.w_mod) * p.D : 0);
+ int tid = int(gl_LocalInvocationID.x);
+ float acc = 0.0;
+ for (int i = tid; i < p.D; i += 256) { float v = x[xb + i]; acc += v * v; }
+ red[tid] = acc;
+ barrier();
+ for (int off = 128; off > 0; off >>= 1) {
+ if (tid < off) red[tid] += red[tid + off];
+ barrier();
+ }
+ float r = (p.flags & 4) != 0 ? 1.0 / sqrt(red[0] + p.eps)
+ : 1.0 / sqrt(red[0] / float(p.D) + p.eps);
+ for (int i = tid; i < p.D; i += 256) {
+ float wv = (p.flags & 2) != 0 ? 1.0 : ((p.flags & 1) != 0 ? 1.0 + w[wb + i] : w[wb + i]);
+ y[yb + i] = x[xb + i] * r * wv * p.post;
+ }
+}
diff --git a/c/shaders/chain_norm.spv b/c/shaders/chain_norm.spv
new file mode 100644
index 00000000..72661396
Binary files /dev/null and b/c/shaders/chain_norm.spv differ
diff --git a/c/shaders/chain_ple.comp b/c/shaders/chain_ple.comp
new file mode 100644
index 00000000..327afb04
--- /dev/null
+++ b/c/shaders/chain_ple.comp
@@ -0,0 +1,104 @@
+#version 450
+// Qwen3.8's per-layer n-gram embedding (PLE) on the device (vk_chain.c, VKC_P_PLE), the
+// part of qwen38_core.h's q38_ple that follows the key and value projections. The
+// n-gram table rows stay on the host; the chain uploads their rows and projects them.
+//
+// mode 0, the gate: one workgroup per (row s, stream k) = WorkGroupID.x = s*C + k, H
+// floats of each:
+// kn = rms0(keys[s][k]), qn = rms0(hyper[s][k]) (zero-centred weights, slices k)
+// dot = kn . qn / sqrt(H); g = sigmoid(sign(dot) * sqrt(max(|dot|, 1e-6)))
+// gated[s][k] = g * value[s]; normv[s][k] = rms0(gated[s][k]) with the conv norm
+// mode 1, the convolution: one thread per channel d < W = C*H, rows in order, with the
+// dilated ring (taps every ngram entries, (ck-1)*ngram of them, on the device):
+// a = conv[d][ck-1] * normv[s][d] + sum_t conv[d][t] * ring[t*ngram]
+// hyper[s][d] += gated[s][d] + silu(a); the ring drops its oldest, takes normv[s][d]
+// snap_row >= 0 also writes the ring after that row to snap (MTP verify rollback).
+// prm: norm_key[W], norm_query[W], norm_conv[W].
+layout(local_size_x = 256) in;
+
+layout(std430, binding = 0) readonly buffer KB { float keys[]; };
+layout(std430, binding = 1) buffer HB { float hyp[]; };
+layout(std430, binding = 2) readonly buffer VB { float val[]; };
+layout(std430, binding = 3) readonly buffer PB { float prm[]; };
+layout(std430, binding = 4) buffer GB { float gated[]; };
+layout(std430, binding = 5) buffer NB { float normv[]; };
+layout(std430, binding = 6) readonly buffer CB { float conv[]; };
+layout(std430, binding = 7) buffer RB { float ring[]; };
+
+layout(push_constant) uniform P {
+ int mode, S, C, H, CK, NG;
+ int keys_off, hyp_off, val_off;
+ int snap_row, snap_off; // the snap ring sits in the ring buffer at snap_off
+ float eps;
+ int prm_off, conv_off, ring_off;
+} p;
+
+shared float r1[256];
+shared float r2[256];
+
+float sig(float x) { return 1.0 / (1.0 + exp(-x)); }
+
+void reduce2(int tid) {
+ barrier();
+ for (int off = 128; off > 0; off >>= 1) {
+ if (tid < off) { r1[tid] += r1[tid + off]; r2[tid] += r2[tid + off]; }
+ barrier();
+ }
+}
+
+void main() {
+ int W = p.C * p.H;
+ int tid = int(gl_LocalInvocationID.x);
+ if (p.mode == 0) {
+ int g = int(gl_WorkGroupID.x), s = g / p.C, k = g - s * p.C;
+ int kb = p.keys_off + s * W + k * p.H, hb = p.hyp_off + s * W + k * p.H, vb = p.val_off + s * p.H;
+ int wk = p.prm_off + k * p.H, wq = p.prm_off + W + k * p.H, wc = p.prm_off + 2 * W + k * p.H;
+ float a = 0.0, b = 0.0;
+ for (int d = tid; d < p.H; d += 256) { float u = keys[kb + d], v = hyp[hb + d]; a += u * u; b += v * v; }
+ r1[tid] = a; r2[tid] = b;
+ reduce2(tid);
+ float rk = 1.0 / sqrt(r1[0] / float(p.H) + p.eps), rq = 1.0 / sqrt(r2[0] / float(p.H) + p.eps);
+ barrier();
+ float dt = 0.0;
+ for (int d = tid; d < p.H; d += 256)
+ dt += (keys[kb + d] * rk * (1.0 + prm[wk + d])) * (hyp[hb + d] * rq * (1.0 + prm[wq + d]));
+ r1[tid] = dt; r2[tid] = 0.0;
+ reduce2(tid);
+ float dot = r1[0] / sqrt(float(p.H));
+ float shaped = sign(dot) * sqrt(max(abs(dot), 1e-6));
+ if (dot == 0.0) shaped = sqrt(1e-6); /* copysign(x, +0) is +x */
+ float gt = sig(shaped);
+ barrier();
+ float c3 = 0.0;
+ for (int d = tid; d < p.H; d += 256) {
+ float u = gt * val[vb + d];
+ gated[s * W + k * p.H + d] = u;
+ c3 += u * u;
+ }
+ r1[tid] = c3; r2[tid] = 0.0;
+ reduce2(tid);
+ float rc = 1.0 / sqrt(r1[0] / float(p.H) + p.eps);
+ for (int d = tid; d < p.H; d += 256)
+ normv[s * W + k * p.H + d] = gated[s * W + k * p.H + d] * rc * (1.0 + prm[wc + d]);
+ return;
+ }
+ int d = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);
+ if (d >= W) return;
+ int SL = (p.CK - 1) * p.NG;
+ float rg[32];
+ for (int t = 0; t < 32; t++) rg[t] = t < SL ? ring[p.ring_off + d * SL + t] : 0.0;
+ int cw = p.conv_off + d * p.CK;
+ float wl = conv[cw + p.CK - 1];
+ for (int s = 0; s < p.S; s++) {
+ float nv = normv[s * W + d];
+ float a = wl * nv;
+ for (int t = 0; t < 31; t++)
+ if (t < p.CK - 1) a += conv[cw + t] * rg[t * p.NG < 32 ? t * p.NG : 0];
+ hyp[p.hyp_off + s * W + d] += gated[s * W + d] + a * sig(a);
+ for (int t = 0; t < 31; t++) if (t + 1 < SL) rg[t] = rg[t + 1];
+ for (int t = 0; t < 32; t++) if (t == SL - 1) rg[t] = nv;
+ if (s == p.snap_row)
+ for (int t = 0; t < 32; t++) if (t < SL) ring[p.snap_off + d * SL + t] = rg[t];
+ }
+ for (int t = 0; t < 32; t++) if (t < SL) ring[p.ring_off + d * SL + t] = rg[t];
+}
diff --git a/c/shaders/chain_ple.spv b/c/shaders/chain_ple.spv
new file mode 100644
index 00000000..70e5c9e6
Binary files /dev/null and b/c/shaders/chain_ple.spv differ
diff --git a/c/shaders/chain_qsa.comp b/c/shaders/chain_qsa.comp
new file mode 100644
index 00000000..d05cf796
--- /dev/null
+++ b/c/shaders/chain_qsa.comp
@@ -0,0 +1,116 @@
+#version 450
+// Qwen3.8's QSA indexer on the device (vk_chain.c, VKC_P_QSA), two modes.
+//
+// mode 0, the block keys: one workgroup per block b = b0 + WorkGroupID.x whose R index
+// keys are all written. Its key is what qwen38_core.h's q38_attention builds for every
+// query again: the mean of the block's R index keys (summed as the CPU does, k/R in
+// order), a zero-centred RMSNorm (1 + w), RoPE at the block's first position (cos/sin
+// row b - b0 of cs). A block's key depends on its own rows only, so the device keeps
+// it (pk[b]) and computes each block once, when a step completes it.
+//
+// mode 1, the selection: one workgroup per query row s, position pos_base + s. With
+// visible = pos + 1 positions, blocks = visible / R complete blocks and
+// take = min(blocks, budget / R): when every block is taken the row attends to all of
+// 0..pos (count -1 in sel, chain_attn.comp's contiguous case). Otherwise each block
+// scores sum_h max(0, q_h . pk[b]) / sqrt(ID) over the IQ index heads (the CPU's sums,
+// in its order), a block is taken when fewer than `take` blocks rank above it (score
+// higher, or equal with a lower index: the CPU's sort order), and sel gets the taken
+// blocks' positions in block order followed by the incomplete tail block's.
+layout(local_size_x = 128) in;
+
+layout(std430, binding = 0) readonly buffer AB { float src[]; }; // mode 0: index keys [cap][ID]; mode 1: index queries
+layout(std430, binding = 1) readonly buffer WB { float w[]; }; // mode 0: k norm weight; mode 1: (unused)
+layout(std430, binding = 2) buffer PB { float pk[]; }; // block keys [nbmax][ID]
+layout(std430, binding = 3) readonly buffer CB { float cs[]; }; // mode 0: cos/sin rows
+layout(std430, binding = 4) buffer SB { float sc[]; }; // mode 1: scratch [S][2*nbmax]
+layout(std430, binding = 5) buffer LB { int sel[]; }; // mode 1: [S][sel_row]
+
+layout(push_constant) uniform P {
+ int mode, ID, R;
+ int b0, half_; // mode 0
+ int S, pos_base, budget, IQ; // mode 1
+ int q_off, q_row, nbmax, sel_row;
+ float eps;
+ int src_off, w_off, pk_off, nb;
+} p;
+
+shared float pool[256];
+shared float red[128];
+
+void main() {
+ int tid = int(gl_LocalInvocationID.x);
+ if (p.mode == 0) {
+ int b = p.b0 + int(gl_WorkGroupID.x);
+ float ss = 0.0;
+ for (int d = tid; d < p.ID; d += 128) {
+ float v = 0.0;
+ for (int r = 0; r < p.R; r++) v += src[p.src_off + (b * p.R + r) * p.ID + d] / float(p.R);
+ pool[d] = v;
+ ss += v * v;
+ }
+ red[tid] = ss;
+ barrier();
+ for (int off = 64; off > 0; off >>= 1) {
+ if (tid < off) red[tid] += red[tid + off];
+ barrier();
+ }
+ float rr = 1.0 / sqrt(red[0] / float(p.ID) + p.eps);
+ barrier();
+ for (int d = tid; d < p.ID; d += 128) pool[d] = pool[d] * rr * (1.0 + w[p.w_off + d]);
+ barrier();
+ int cb = int(gl_WorkGroupID.x) * 2 * p.half_;
+ for (int d = tid; d < p.ID; d += 128) {
+ float v = pool[d];
+ if (d < p.half_) {
+ float c = cs[cb + 2 * d], s = cs[cb + 2 * d + 1];
+ v = pool[d] * c - pool[d + p.half_] * s;
+ } else if (d < 2 * p.half_) {
+ int j = d - p.half_;
+ float c = cs[cb + 2 * j], s = cs[cb + 2 * j + 1];
+ v = pool[d] * c + pool[j] * s;
+ }
+ pk[p.pk_off + b * p.ID + d] = v;
+ }
+ return;
+ }
+ int s = int(gl_WorkGroupID.x);
+ int visible = p.pos_base + s + 1, blocks = visible / p.R;
+ int take = min(blocks, p.budget / p.R);
+ int lb = s * p.sel_row;
+ if (take >= blocks) {
+ if (tid == 0) sel[lb] = -1;
+ return;
+ }
+ int sb = s * 2 * p.nbmax;
+ int qb = p.q_off + s * p.q_row;
+ for (int b = tid; b < blocks; b += 128) {
+ float score = 0.0;
+ for (int h = 0; h < p.IQ; h++) {
+ float a = 0.0;
+ for (int d = 0; d < p.ID; d++) a += src[qb + h * p.ID + d] * pk[p.pk_off + b * p.ID + d];
+ if (a > 0.0) score += a;
+ }
+ sc[sb + b] = score / sqrt(float(p.ID));
+ }
+ memoryBarrierBuffer();
+ barrier();
+ for (int b = tid; b < blocks; b += 128) {
+ float v = sc[sb + b];
+ int rank = 0;
+ for (int b2 = 0; b2 < blocks; b2++) {
+ float v2 = sc[sb + b2];
+ if (v2 > v || (v2 == v && b2 < b)) rank++;
+ }
+ sc[sb + p.nbmax + b] = rank < take ? 1.0 : 0.0;
+ }
+ memoryBarrierBuffer();
+ barrier();
+ if (tid == 0) {
+ int n = 0;
+ for (int b = 0; b < blocks; b++)
+ if (sc[sb + p.nbmax + b] != 0.0)
+ for (int r = 0; r < p.R; r++) sel[lb + 1 + n++] = b * p.R + r;
+ for (int t = blocks * p.R; t < visible; t++) sel[lb + 1 + n++] = t;
+ sel[lb] = n;
+ }
+}
diff --git a/c/shaders/chain_qsa.spv b/c/shaders/chain_qsa.spv
new file mode 100644
index 00000000..73f7365c
Binary files /dev/null and b/c/shaders/chain_qsa.spv differ
diff --git a/c/shaders/chain_rope.comp b/c/shaders/chain_rope.comp
new file mode 100644
index 00000000..87a40512
--- /dev/null
+++ b/c/shaders/chain_rope.comp
@@ -0,0 +1,33 @@
+#version 450
+// The dense chain's RoPE (vk_chain.c, VKC_P_ROPE), in place, rotate-half form: for each
+// segment (a head of a row) and j < half,
+// a = x[j], b = x[j + half]; x[j] = a*c - b*s; x[j + half] = b*c + a*s
+// with (c, s) = cs[cs_off + row*cs_row + 2j], cs[... + 1]. The cosines and sines come
+// from the host, computed with the CPU's own cosf/sinf at the CPU's angles, so a
+// position far from 0 rotates exactly as on the CPU (a device sin of a large angle
+// need not), and M-RoPE's per-axis positions are just another table.
+// Segment g: row g / per_row, head j = g % per_row, at x_off + row*x_row + j*x_seg.
+layout(local_size_x = 64) in;
+
+layout(std430, binding = 0) buffer X { float x[]; };
+layout(std430, binding = 1) readonly buffer CS { float cs[]; };
+
+layout(push_constant) uniform P {
+ int nseg, per_row;
+ int x_off, x_row, x_seg;
+ int half_;
+ int cs_off, cs_row;
+} p;
+
+void main() {
+ int gid = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 64u);
+ if (gid >= p.nseg * p.half_) return;
+ int seg = gid / p.half_, j = gid - seg * p.half_;
+ int row = seg / p.per_row, hh = seg - row * p.per_row;
+ int base = p.x_off + row * p.x_row + hh * p.x_seg;
+ int cb = p.cs_off + row * p.cs_row + 2 * j;
+ float c = cs[cb], s = cs[cb + 1];
+ float a = x[base + j], b = x[base + j + p.half_];
+ x[base + j] = a * c - b * s;
+ x[base + j + p.half_] = b * c + a * s;
+}
diff --git a/c/shaders/chain_rope.spv b/c/shaders/chain_rope.spv
new file mode 100644
index 00000000..24a6bf8d
Binary files /dev/null and b/c/shaders/chain_rope.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
new file mode 100644
index 00000000..cf036906
--- /dev/null
+++ b/c/tests/test_vk_chain.c
@@ -0,0 +1,530 @@
+/* The dense chain's ops (vk_chain.c and shaders/chain_*.comp) against CPU references,
+ * on any Vulkan device (CI: Lavapipe). Every op runs inside recorded frames the way an
+ * engine uses them -- several ops per submission, a submission not waited for, the next
+ * one ordered after it -- and its output is compared with a plain C version of what
+ * qwen36.c / qwen38_core.h compute, within a float tolerance (the device sums in float
+ * and in another order; the references sum in double):
+ * matmul GEMV (chain_gemv.comp's vectorized one and qmatmul.comp's) and tiled
+ * GEMM, int8 rows, int4-g64, f32, bf16, offsets the device can bind and
+ * offsets it cannot (through the frame's temporaries)
+ * norm zero-centred, plain, no weight, L2; heads with a gate between them, a
+ * weight slice per stream, in place
+ * rope rotate-half from a host table, heads at a stride
+ * attn causal GQA with the output gate over a cache at an offset, prefill rows
+ * after earlier ones, and a selection list
+ * dnconv both orders of the sum, the ring carried across calls, the snapshot row
+ * dnrec KD 8 and 128, silu and sigmoid gates, the state carried, the snapshot
+ * ew every element-wise op
+ * qsa block keys, and the selection with ties broken as the CPU sorts
+ * ple the gate and the dilated convolution with its ring and snapshot
+ * frames vkc_write, vkc_read, a frame left in flight and one ordered after it
+ *
+ * make vk-chain-check VK=1 (VK_ICD_FILENAMES=.../lvp_icd.json for Lavapipe) */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "../backend_vulkan.h"
+#include "../vk_chain.h"
+
+static int fails;
+#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
+
+static unsigned rng = 12345;
+static unsigned rnd(void) { rng = rng * 1103515245u + 12345u; return rng >> 8; }
+static float frnd(void) { return (float)((int)(rnd() % 2001) - 1000) / 1000.0f; }
+static float *fvec(size_t n, float scale) { float *v = malloc(n * sizeof *v); for (size_t i = 0; i < n; i++) v[i] = frnd() * scale; return v; }
+static float sigm(float x) { return 1.f / (1.f + expf(-x)); }
+
+/* max |a-b| / (max |b| + floor) */
+static double relerr(const float *a, const float *b, size_t n, double floor) {
+ double m = 0, e = 0;
+ for (size_t i = 0; i < n; i++) { double d = fabs((double)a[i] - b[i]); if (d > e) e = d; if (fabs(b[i]) > m) m = fabs(b[i]); }
+ return e / (m + floor);
+}
+static int bad(const float *a, size_t n) { for (size_t i = 0; i < n; i++) if (!isfinite(a[i])) return 1; return 0; }
+
+static VkcBuf *up(const float *v, size_t n) { /* a device buffer holding v */
+ VkcBuf *b = vkc_buf(n * 4, VKC_DEV);
+ if (!b) return NULL;
+ vkc_begin(); vkc_write(b, 0, v, n * 4); vkc_submit(1);
+ return b;
+}
+static float *down(VkcBuf *b, size_t off, size_t n) {
+ float *v = malloc(n * sizeof *v);
+ if (!vkc_read(b, off, v, n * 4)) { memset(v, 0xff, n * sizeof *v); }
+ return v;
+}
+
+/* ---- matmul ----------------------------------------------------------------------- */
+static uint16_t f2bf(float f) { uint32_t u; memcpy(&u, &f, 4); return (uint16_t)((u + 0x7fff + ((u >> 16) & 1)) >> 16); }
+static float bf2f(uint16_t h) { uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }
+static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {
+ float *x = fvec((size_t)S * I, 1.f), *W = malloc((size_t)O * I * sizeof(float));
+ void *codes = NULL; float *sc = NULL; int gs = 0;
+ if (fmt == 1) { /* int8 rows, one scale per row */
+ int8_t *q = malloc((size_t)O * I); sc = malloc(O * sizeof *sc);
+ for (int o = 0; o < O; o++) { sc[o] = 0.01f + (rnd() % 100) / 5000.f;
+ for (int i = 0; i < I; i++) { q[(size_t)o * I + i] = (int8_t)((int)(rnd() % 255) - 127); W[(size_t)o * I + i] = q[(size_t)o * I + i] * sc[o]; } }
+ codes = q;
+ } else if (fmt == 4) { /* int4 (v+8 nibbles, low = even column), one scale per 64 */
+ gs = 64; int ng = (I + 63) / 64;
+ uint8_t *q = calloc((size_t)O * ((I + 1) / 2), 1); sc = malloc((size_t)O * ng * sizeof *sc);
+ for (int o = 0; o < O; o++) for (int g = 0; g < ng; g++) sc[o * ng + g] = 0.02f + (rnd() % 100) / 3000.f;
+ for (int o = 0; o < O; o++) for (int i = 0; i < I; i++) {
+ int v = (int)(rnd() % 16); q[(size_t)o * ((I + 1) / 2) + i / 2] |= (uint8_t)(v << ((i & 1) * 4));
+ W[(size_t)o * I + i] = (v - 8) * sc[o * ng + i / 64];
+ }
+ codes = q;
+ } else if (fmt == 10) {
+ float *w = fvec((size_t)O * I, 0.05f); memcpy(W, w, (size_t)O * I * sizeof(float)); codes = w;
+ } else { /* 11 bf16 */
+ uint16_t *w = malloc((size_t)O * I * 2);
+ for (size_t i = 0; i < (size_t)O * I; i++) { w[i] = f2bf(frnd() * 0.05f); W[i] = bf2f(w[i]); }
+ codes = w;
+ }
+ ColiVkTensor *t = NULL;
+ if (!coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs)) { CHECK(0, "matmul fmt %d: upload", fmt); return; }
+ float *ref = malloc((size_t)S * O * sizeof *ref);
+ for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {
+ double a = 0; for (int i = 0; i < I; i++) a += (double)x[(size_t)s * I + i] * W[(size_t)o * I + i];
+ ref[(size_t)s * O + o] = (float)a;
+ }
+ VkcBuf *xb = vkc_buf((xo + (size_t)S * I) * 4, VKC_DEV), *yb = vkc_buf((yo + (size_t)S * O) * 4, VKC_DOWN);
+ vkc_begin();
+ vkc_write(xb, xo, x, (size_t)S * I * 4);
+ int ok = vkc_matmul(t, xb, xo, yb, yo, S);
+ vkc_submit(1);
+ float *y = (float *)vkc_ptr(yb) + yo;
+ double e = relerr(y, ref, (size_t)S * O, 1e-3);
+ CHECK(ok && e < 2e-4 && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);
+ vkc_free(xb); vkc_free(yb); coli_vk_tensor_free(t);
+ free(x); free(W); free(codes); free(sc); free(ref);
+}
+
+/* ---- norm ------------------------------------------------------------------------- */
+static void test_norm(int flags, int inplace) {
+ int rows = 3, per = 4, D = 40, seg = 52, rowst = per * seg + 7; /* a gap after each head, like q's gate */
+ size_t n = (size_t)rows * rowst;
+ float *x = fvec(n, 2.f), *w = fvec((size_t)per * D, 0.5f), *ref = malloc(n * sizeof *ref);
+ memcpy(ref, x, n * sizeof *ref);
+ int wmod = flags & VKC_NORM_L2 ? 0 : per;
+ for (int r = 0; r < rows; r++) for (int j = 0; j < per; j++) {
+ const float *xs = x + r * rowst + j * seg; float *ys = ref + r * rowst + j * seg;
+ double ss = 0; for (int i = 0; i < D; i++) ss += (double)xs[i] * xs[i];
+ float rr = flags & VKC_NORM_L2 ? 1.f / sqrtf((float)ss + 1e-6f) : 1.f / sqrtf((float)(ss / D) + 1e-6f);
+ const float *ws = w + (wmod ? (j % wmod) * D : 0);
+ for (int i = 0; i < D; i++) {
+ float wv = flags & VKC_NORM_NOW ? 1.f : flags & VKC_NORM_ADD1 ? 1.f + ws[i] : ws[i];
+ ys[i] = xs[i] * rr * wv * 0.5f;
+ }
+ }
+ VkcBuf *xb = up(x, n), *wb = up(w, (size_t)per * D), *yb = inplace ? xb : vkc_buf(n * 4, VKC_DEV);
+ if (!inplace) { vkc_begin(); vkc_write(yb, 0, x, n * 4); vkc_submit(1); }
+ VkcNorm p = {rows * per, D, per, 0, rowst, seg, 0, rowst, seg, 0, wmod, flags, 1e-6f, 0.5f};
+ vkc_begin(); int ok = vkc_norm(xb, wb, yb, &p); vkc_submit(1);
+ float *y = down(yb, 0, n);
+ double e = relerr(y, ref, n, 1e-3);
+ CHECK(ok && e < 1e-5, "norm flags %d inplace %d: err %.2e", flags, inplace, e);
+ vkc_free(xb); vkc_free(wb); if (!inplace) vkc_free(yb);
+ free(x); free(w); free(ref); free(y);
+}
+
+/* ---- rope ------------------------------------------------------------------------- */
+static void test_rope(void) {
+ int rows = 5, heads = 3, hd = 16, seg = 24, half = 4, rowst = heads * seg;
+ size_t n = (size_t)rows * rowst;
+ float *x = fvec(n, 1.f), *ref = malloc(n * sizeof *ref), *cs = malloc((size_t)rows * half * 2 * sizeof *cs);
+ memcpy(ref, x, n * sizeof *ref);
+ for (int r = 0; r < rows; r++) for (int j = 0; j < half; j++) {
+ float ang = (float)(1000 + 37 * r) * powf(10000.f, -2.f * j / (2 * half));
+ cs[(r * half + j) * 2] = cosf(ang); cs[(r * half + j) * 2 + 1] = sinf(ang);
+ for (int h = 0; h < heads; h++) {
+ float *v = ref + r * rowst + h * seg, a = v[j], b = v[j + half];
+ v[j] = a * cs[(r * half + j) * 2] - b * cs[(r * half + j) * 2 + 1];
+ v[j + half] = b * cs[(r * half + j) * 2] + a * cs[(r * half + j) * 2 + 1];
+ }
+ }
+ (void)hd;
+ VkcBuf *xb = up(x, n), *cb = up(cs, (size_t)rows * half * 2);
+ VkcRope p = {rows * heads, heads, 0, rowst, seg, half, 0, half * 2};
+ vkc_begin(); int ok = vkc_rope(xb, cb, &p); vkc_submit(1);
+ float *y = down(xb, 0, n);
+ double e = relerr(y, ref, n, 1e-3);
+ CHECK(ok && e < 1e-6, "rope: err %.2e", e);
+ vkc_free(xb); vkc_free(cb); free(x); free(ref); free(cs); free(y);
+}
+
+/* ---- attention ---------------------------------------------------------------------- */
+static void test_attn(int S, int pos_base, int hd, int use_list) {
+ int H = 4, KVH = 2, cap = 300, gsz = hd, qseg = hd + gsz, koff = 64;
+ int T = pos_base + S;
+ size_t kn = (size_t)koff + (size_t)KVH * cap * hd;
+ float *q = fvec((size_t)S * H * qseg, 1.f), *kc = fvec(kn, 1.f), *vc = fvec(kn, 1.f);
+ float scale = 1.f / sqrtf((float)hd);
+ int selrow = 1 + T;
+ int *sel = calloc((size_t)S * selrow, sizeof *sel);
+ for (int s = 0; s < S; s++) {
+ int vis = pos_base + s + 1;
+ if (!use_list || s % 2) { sel[s * selrow] = -1; continue; }
+ int n = 0;
+ for (int t = 0; t < vis; t++) if (t % 3 != 1 || t == vis - 1) sel[s * selrow + 1 + n++] = t;
+ sel[s * selrow] = n;
+ }
+ float *ref = malloc((size_t)S * H * hd * sizeof *ref);
+ for (int s = 0; s < S; s++) for (int h = 0; h < H; h++) {
+ int kvh = h / (H / KVH), vis = pos_base + s + 1;
+ const int *list = sel + s * selrow;
+ int n = list[0] >= 0 ? list[0] : vis;
+ double *sc = malloc(n * sizeof *sc), mx = -1e300, sum = 0;
+ for (int j = 0; j < n; j++) {
+ int t = list[0] >= 0 ? list[1 + j] : j;
+ double a = 0; for (int d = 0; d < hd; d++) a += (double)q[(size_t)s * H * qseg + h * qseg + d] * kc[koff + ((size_t)kvh * cap + t) * hd + d];
+ sc[j] = a * scale; if (sc[j] > mx) mx = sc[j];
+ }
+ for (int j = 0; j < n; j++) { sc[j] = exp(sc[j] - mx); sum += sc[j]; }
+ for (int d = 0; d < hd; d++) {
+ double a = 0;
+ for (int j = 0; j < n; j++) { int t = list[0] >= 0 ? list[1 + j] : j; a += sc[j] / sum * vc[koff + ((size_t)kvh * cap + t) * hd + d]; }
+ ref[((size_t)s * H + h) * hd + d] = (float)a * sigm(q[(size_t)s * H * qseg + h * qseg + hd + d]);
+ }
+ free(sc);
+ }
+ VkcBuf *qb = up(q, (size_t)S * H * qseg), *kb = up(kc, kn), *vb = up(vc, kn), *ob = vkc_buf((size_t)S * H * hd * 4, VKC_DOWN);
+ VkcBuf *lb = vkc_buf((size_t)S * selrow * 4, VKC_DEV);
+ vkc_begin(); vkc_write(lb, 0, sel, (size_t)S * selrow * 4); vkc_submit(1);
+ VkcAttn p = {S, H, KVH, hd, pos_base, cap, 0, H * qseg, qseg, hd, H * qseg, qseg, 1, 0, H * hd, 0, use_list ? selrow : 0, scale, koff, koff};
+ vkc_begin(); int ok = vkc_attn(qb, kb, vb, ob, qb, lb, &p); vkc_submit(1);
+ float *o = vkc_ptr(ob);
+ double e = relerr(o, ref, (size_t)S * H * hd, 1e-3);
+ CHECK(ok && e < 2e-5, "attn S %d pos %d hd %d list %d: err %.2e", S, pos_base, hd, use_list, e);
+ vkc_free(qb); vkc_free(kb); vkc_free(vb); vkc_free(ob); vkc_free(lb);
+ free(q); free(kc); free(vc); free(sel); free(ref);
+}
+
+/* ---- DeltaNet: convolution and recurrence ------------------------------------------ */
+static void conv_ref(int S, int CD, int CK, const float *in, const float *w, float *ring, float *out, int order, int snap_row, float *snap) {
+ for (int s = 0; s < S; s++) for (int c = 0; c < CD; c++) {
+ float *rg = ring + c * (CK - 1); const float *wc = w + c * CK; float cur = in[s * CD + c], a;
+ if (order == 0) { a = 0; for (int k = 0; k < CK - 1; k++) a += wc[k] * rg[k]; a += wc[CK - 1] * cur; }
+ else { a = wc[CK - 1] * cur; for (int k = 0; k < CK - 1; k++) a += wc[k] * rg[k]; }
+ out[s * CD + c] = a / (1.f + expf(-a));
+ for (int k = 0; k < CK - 2; k++) rg[k] = rg[k + 1];
+ rg[CK - 2] = cur;
+ if (s == snap_row) memcpy(snap + c * (CK - 1), rg, (CK - 1) * sizeof(float));
+ }
+}
+static void test_dnconv(int order) {
+ int CD = 37, CK = 4, S1 = 5, S2 = 1, nh = CK - 1;
+ float *w = fvec((size_t)CD * CK, 0.5f), *in = fvec((size_t)(S1 + S2) * CD, 1.f);
+ float *ring = calloc((size_t)CD * nh, sizeof *ring), *out = malloc((size_t)(S1 + S2) * CD * sizeof *out);
+ float *snap = calloc((size_t)CD * nh, sizeof *snap);
+ conv_ref(S1, CD, CK, in, w, ring, out, order, 2, snap);
+ float *snap_ref = malloc((size_t)CD * nh * sizeof *snap_ref); memcpy(snap_ref, snap, (size_t)CD * nh * sizeof *snap_ref);
+ conv_ref(S2, CD, CK, in + S1 * CD, w, ring, out + S1 * CD, order, -1, NULL);
+ VkcBuf *wb = up(w, (size_t)CD * CK), *ib = up(in, (size_t)(S1 + S2) * CD);
+ VkcBuf *rb = vkc_buf((size_t)CD * nh * 2 * 4 + 64, VKC_DEV), *ob = vkc_buf((size_t)(S1 + S2) * CD * 4, VKC_DEV);
+ int ro = 16, so = 16 + CD * nh; /* ring and its snapshot in one buffer, at offsets */
+ VkcDnConv p1 = {S1, CD, CK, 0, CD, 0, CD, 2, order, 0, ro, so};
+ VkcDnConv p2 = {S2, CD, CK, S1 * CD, CD, S1 * CD, CD, -1, order, 0, ro, so};
+ vkc_begin(); int ok = vkc_dnconv(ib, wb, rb, ob, rb, &p1); vkc_submit(0); /* left in flight */
+ vkc_begin(); ok &= vkc_dnconv(ib, wb, rb, ob, rb, &p2); vkc_submit(1);
+ float *o = down(ob, 0, (size_t)(S1 + S2) * CD), *r = down(rb, ro, (size_t)CD * nh), *sn = down(rb, so, (size_t)CD * nh);
+ double e1 = relerr(o, out, (size_t)(S1 + S2) * CD, 1e-3), e2 = relerr(r, ring, (size_t)CD * nh, 1e-3), e3 = relerr(sn, snap_ref, (size_t)CD * nh, 1e-3);
+ CHECK(ok && e1 < 1e-6 && e2 == 0 && e3 == 0, "dnconv order %d: out %.2e ring %.2e snap %.2e", order, e1, e2, e3);
+ vkc_free(wb); vkc_free(ib); vkc_free(rb); vkc_free(ob);
+ free(w); free(in); free(ring); free(out); free(snap); free(snap_ref); free(o); free(r); free(sn);
+}
+
+static float softplus(float z) { return z > 20.f ? z : log1pf(expf(z)); }
+/* qwen36.c's deltanet recurrence for S rows, from the conv output */
+static void rec_ref(int S, int VH, int KH, int KD, int VD, const float *cv, const float *b, const float *a, const float *z,
+ const float *alog, const float *dtb, const float *nw, float *st, float *y, int sig_gate, int snap_row, float *snap) {
+ int Kt = KH * KD, rep = VH / KH, CD = 2 * Kt + VH * VD;
+ for (int s = 0; s < S; s++) for (int h = 0; h < VH; h++) {
+ const float *row = cv + (size_t)s * CD;
+ float q[256], k[256], o[128];
+ double sq = 1e-6, sk = 1e-6;
+ for (int d = 0; d < KD; d++) { q[d] = row[(h / rep) * KD + d]; k[d] = row[Kt + (h / rep) * KD + d]; sq += (double)q[d] * q[d]; sk += (double)k[d] * k[d]; }
+ for (int d = 0; d < KD; d++) { q[d] = (float)(q[d] / sqrt(sq) / sqrt((double)KD)); k[d] = (float)(k[d] / sqrt(sk)); }
+ float beta = sigm(b[s * VH + h]), decay = expf(-expf(alog[h]) * softplus(a[s * VH + h] + dtb[h]));
+ float *Sh = st + (size_t)h * KD * VD;
+ for (int t = 0; t < KD * VD; t++) Sh[t] *= decay;
+ for (int v = 0; v < VD; v++) {
+ float kv = 0; for (int d = 0; d < KD; d++) kv += k[d] * Sh[d * VD + v];
+ float delta = (row[2 * Kt + h * VD + v] - kv) * beta;
+ for (int d = 0; d < KD; d++) Sh[d * VD + v] += k[d] * delta;
+ }
+ double ms = 0;
+ for (int v = 0; v < VD; v++) { float acc = 0; for (int d = 0; d < KD; d++) acc += q[d] * Sh[d * VD + v]; o[v] = acc; ms += (double)acc * acc; }
+ float r = 1.f / sqrtf((float)(ms / VD) + 1e-6f);
+ for (int v = 0; v < VD; v++) {
+ float zv = z[(size_t)s * VH * VD + h * VD + v], g = sig_gate ? sigm(zv) : zv / (1.f + expf(-zv));
+ y[(size_t)s * VH * VD + h * VD + v] = o[v] * r * nw[v] * g;
+ }
+ if (s == snap_row) memcpy(snap + (size_t)h * KD * VD, Sh, (size_t)KD * VD * sizeof(float));
+ }
+}
+static void test_dnrec(int KD, int VD, int VH, int KH, int sig_gate) {
+ int S1 = 3, S2 = 2, S = S1 + S2, Kt = KH * KD, CD = 2 * Kt + VH * VD;
+ float *cv = fvec((size_t)S * CD, 1.f), *ab = fvec((size_t)S * VH * 2, 2.f), *z = fvec((size_t)S * VH * VD, 2.f);
+ float *prm = fvec((size_t)2 * VH + VD, 1.f);
+ float *st = fvec((size_t)VH * KD * VD, 0.1f), *st0 = malloc((size_t)VH * KD * VD * sizeof *st0);
+ memcpy(st0, st, (size_t)VH * KD * VD * sizeof *st0);
+ float *b = malloc((size_t)S * VH * sizeof *b), *a = malloc((size_t)S * VH * sizeof *a);
+ for (int s = 0; s < S; s++) for (int h = 0; h < VH; h++) { b[s * VH + h] = ab[s * 2 * VH + h]; a[s * VH + h] = ab[s * 2 * VH + VH + h]; }
+ float *y = malloc((size_t)S * VH * VD * sizeof *y), *snap = malloc((size_t)VH * KD * VD * sizeof *snap);
+ rec_ref(S1, VH, KH, KD, VD, cv, b, a, z, prm, prm + VH, prm + 2 * VH, st, y, sig_gate, 1, snap);
+ rec_ref(S2, VH, KH, KD, VD, cv + (size_t)S1 * CD, b + S1 * VH, a + S1 * VH, z + (size_t)S1 * VH * VD, prm, prm + VH, prm + 2 * VH,
+ st, y + (size_t)S1 * VH * VD, sig_gate, -1, NULL);
+ VkcBuf *cb = up(cv, (size_t)S * CD), *abb = up(ab, (size_t)S * VH * 2), *zb = up(z, (size_t)S * VH * VD), *pb = up(prm, (size_t)2 * VH + VD);
+ size_t SN = (size_t)VH * KD * VD;
+ VkcBuf *sb = vkc_buf(2 * SN * 4, VKC_DEV), *yb = vkc_buf((size_t)S * VH * VD * 4, VKC_DEV);
+ vkc_begin(); vkc_write(sb, 0, st0, SN * 4); vkc_submit(1);
+ VkcDnRec p1 = {S1, VH, KH, VD, Kt, 0, CD, 0, 2 * VH, VH, 2 * VH, 0, VH * VD, 0, VH * VD, 1, sig_gate, 1e-6f,
+ 1.f / sqrtf((float)KD), 0, (int)SN, 0};
+ VkcDnRec p2 = p1;
+ p2.S = S2; p2.cv_off = S1 * CD; p2.b_off = S1 * 2 * VH; p2.a_off = S1 * 2 * VH + VH; p2.z_off = S1 * VH * VD; p2.y_off = S1 * VH * VD; p2.snap_row = -1;
+ vkc_begin(); int ok = vkc_dnrec(KD, cb, abb, zb, sb, pb, yb, sb, &p1); vkc_submit(0);
+ vkc_begin(); ok &= vkc_dnrec(KD, cb, abb, zb, sb, pb, yb, sb, &p2); vkc_submit(1);
+ float *yd = down(yb, 0, (size_t)S * VH * VD), *sd = down(sb, 0, SN), *nd = down(sb, SN, SN);
+ double e1 = relerr(yd, y, (size_t)S * VH * VD, 1e-3), e2 = relerr(sd, st, SN, 1e-3), e3 = relerr(nd, snap, SN, 1e-3);
+ CHECK(ok && e1 < 5e-5 && e2 < 5e-6 && e3 < 5e-6, "dnrec KD %d VD %d gate %d: y %.2e state %.2e snap %.2e", KD, VD, sig_gate, e1, e2, e3);
+ vkc_free(cb); vkc_free(abb); vkc_free(zb); vkc_free(pb); vkc_free(sb); vkc_free(yb);
+ free(cv); free(ab); free(z); free(prm); free(st); free(st0); free(b); free(a); free(y); free(snap); free(yd); free(sd); free(nd);
+}
+
+/* ---- element-wise ------------------------------------------------------------------- */
+static void test_ew(void) {
+ int R = 3, D = 50, C = 4, W = C * D;
+ size_t nw = (size_t)R * W;
+ float *a = fvec(nw, 2.f), *b = fvec(nw, 2.f), *c = fvec(nw, 2.f), *e = fvec(R, 3.f), *ref = malloc(nw * sizeof *ref);
+ VkcBuf *ab = up(a, nw), *bb = up(b, nw), *cb = up(c, nw), *eb = up(e, R), *yb = vkc_buf(nw * 4, VKC_DEV);
+ struct { int op, flags, n; const char *name; } ops[] = {
+ {VKC_EW_ADD, 0, R * D, "add"}, {VKC_EW_COMBINE, 1 | 2 | 4, R * D, "combine"}, {VKC_EW_COMBINE, 2 | 8, R * D, "combine-noresid"},
+ {VKC_EW_COMBINE, 1, R * D, "combine-routed"}, {VKC_EW_SWIGLU, 0, R * D, "swiglu"}, {VKC_EW_HC_LOW, 0, R * D, "hc-low"},
+ {VKC_EW_HC_MIX, 0, R * D, "hc-mix"}, {VKC_EW_HC_INJ, 0, R * C, "hc-inj"}, {VKC_EW_HC_APPLY, 0, R * W, "hc-apply"}};
+ for (size_t k = 0; k < sizeof ops / sizeof *ops; k++) {
+ int n = ops[k].n, op = ops[k].op, f = ops[k].flags;
+ for (int i = 0; i < n; i++) {
+ int r = i / D, d = i % D;
+ switch (op) {
+ case VKC_EW_ADD: ref[i] = a[i] + b[i]; break;
+ case VKC_EW_COMBINE: { float t = f & 1 ? b[r * D + d] : 0.f;
+ if (f & 2) t = t + (f & 4 ? sigm(e[r]) : 1.f) * c[r * D + d];
+ ref[i] = f & 8 ? t : a[i] + t; break; }
+ case VKC_EW_SWIGLU: ref[i] = a[i] / (1.f + expf(-a[i])) * b[i]; break;
+ case VKC_EW_HC_LOW: { float v = a[i] / C; ref[i] = v * sigm(v); break; }
+ case VKC_EW_HC_MIX: { float v = 0; for (int s = 0; s < C; s++) v += sigm(a[r * W + s * D + d]) * b[r * W + s * D + d]; ref[i] = v / C; break; }
+ case VKC_EW_HC_INJ: ref[i] = 2.f * sigm(a[i] / C); break;
+ case VKC_EW_HC_APPLY: { int rr = i / W, rem = i % W, s = rem / D, dd = rem % D;
+ ref[i] = c[i] + a[rr * C + s] * b[rr * D + dd]; break; }
+ }
+ }
+ if (op == VKC_EW_HC_APPLY) { vkc_begin(); vkc_write(yb, 0, c, nw * 4); vkc_submit(1); }
+ VkcEw p = {op, n, D, C, f, 1, 0, 0, 0, 0, 0, (float)C};
+ vkc_begin(); int ok = vkc_ew(yb, ab, bb, cb, eb, &p); vkc_submit(1);
+ float *y = down(yb, 0, n);
+ double err = relerr(y, ref, n, 1e-3);
+ CHECK(ok && err < 1e-6, "ew %s: err %.2e", ops[k].name, err);
+ free(y);
+ }
+ vkc_free(ab); vkc_free(bb); vkc_free(cb); vkc_free(eb); vkc_free(yb);
+ free(a); free(b); free(c); free(e); free(ref);
+}
+
+/* ---- QSA: block keys and selection ----------------------------------------------------- */
+static void test_qsa(void) {
+ int ID = 12, R = 2, half = 3, cap = 40, nbmax = cap / R, IQ = 2, budget = 6; /* take = 3 blocks */
+ float *ik = fvec((size_t)cap * ID, 1.f), *w = fvec(ID, 0.3f), *pk = calloc((size_t)nbmax * ID, sizeof *pk);
+ int nb = 13; /* blocks 0..12 */
+ float *cs = malloc((size_t)nb * half * 2 * sizeof *cs);
+ for (int b = 0; b < nb; b++) for (int j = 0; j < half; j++) {
+ float ang = (float)(b * R) * powf(10000.f, -2.f * j / (2 * half));
+ cs[(b * half + j) * 2] = cosf(ang); cs[(b * half + j) * 2 + 1] = sinf(ang);
+ }
+ for (int b = 0; b < nb; b++) {
+ float pool[64];
+ for (int d = 0; d < ID; d++) { pool[d] = 0; for (int r = 0; r < R; r++) pool[d] += ik[(b * R + r) * ID + d] / R; }
+ double ss = 0; for (int d = 0; d < ID; d++) ss += (double)pool[d] * pool[d];
+ float rr = 1.f / sqrtf((float)(ss / ID) + 1e-6f);
+ for (int d = 0; d < ID; d++) pool[d] = pool[d] * rr * (1.f + w[d]);
+ float *o = pk + b * ID;
+ for (int d = 0; d < ID; d++) o[d] = pool[d];
+ for (int j = 0; j < half; j++) {
+ float c = cs[(b * half + j) * 2], s = cs[(b * half + j) * 2 + 1], x0 = pool[j], x1 = pool[j + half];
+ o[j] = x0 * c - x1 * s; o[j + half] = x1 * c + x0 * s;
+ }
+ }
+ VkcBuf *ib = up(ik, (size_t)cap * ID), *wb = up(w, ID), *pb = vkc_buf((size_t)nbmax * ID * 4, VKC_DEV), *cb = up(cs, (size_t)nb * half * 2);
+ /* blocks in two steps, as two forwards complete them */
+ VkcQsa p0 = {0, ID, R, 0, half, 0, 0, 0, 0, 0, 0, 0, 0, 1e-6f, 0, 0, 0, 7};
+ VkcQsa p1 = p0; p1.b0 = 7; p1.nb = nb - 7;
+ VkcBuf *cb2 = up(cs + 7 * half * 2, (size_t)(nb - 7) * half * 2);
+ vkc_begin(); int ok = vkc_qsa(ib, wb, pb, cb, NULL, NULL, &p0); ok &= vkc_qsa(ib, wb, pb, cb2, NULL, NULL, &p1); vkc_submit(1);
+ float *pd = down(pb, 0, (size_t)nb * ID);
+ double e = relerr(pd, pk, (size_t)nb * ID, 1e-3);
+ CHECK(ok && e < 1e-5, "qsa block keys: err %.2e", e);
+ /* selection: 4 query rows at positions 20..23; give two blocks the same key to test ties */
+ memcpy(pk + 5 * ID, pk + 2 * ID, ID * sizeof(float));
+ vkc_begin(); vkc_write(pb, 0, pk, (size_t)nbmax * ID * 4); vkc_submit(1);
+ int S = 4, pos_base = 20, selrow = 1 + budget + R - 1;
+ float *qi = fvec((size_t)S * IQ * ID, 1.f);
+ VkcBuf *qb = up(qi, (size_t)S * IQ * ID), *sb = vkc_buf((size_t)S * 2 * nbmax * 4, VKC_DEV), *lb = vkc_buf((size_t)S * selrow * 4, VKC_DEV);
+ VkcQsa ps = {1, ID, R, 0, 0, S, pos_base, budget, IQ, 0, IQ * ID, nbmax, selrow, 1e-6f, 0, 0, 0, 0};
+ vkc_begin(); ok = vkc_qsa(qb, NULL, pb, NULL, sb, lb, &ps); vkc_submit(1);
+ int *sel = malloc((size_t)S * selrow * sizeof *sel);
+ ok &= vkc_read(lb, 0, sel, (size_t)S * selrow * 4);
+ for (int s = 0; s < S; s++) {
+ int vis = pos_base + s + 1, blocks = vis / R, take = blocks < budget / R ? blocks : budget / R;
+ float score[64]; int sel_ref[64], n = 0, chosen[64] = {0};
+ for (int b = 0; b < blocks; b++) {
+ float sc = 0; for (int h = 0; h < IQ; h++) { float a = 0; for (int d = 0; d < ID; d++) a += qi[(s * IQ + h) * ID + d] * pk[b * ID + d]; if (a > 0) sc += a; }
+ score[b] = sc / sqrtf((float)ID);
+ }
+ for (int z = 0; z < take; z++) { /* the CPU's sort: score desc, block asc */
+ int best = -1; for (int b = 0; b < blocks; b++) if (!chosen[b] && (best < 0 || score[b] > score[best])) best = b;
+ chosen[best] = 1;
+ }
+ for (int b = 0; b < blocks; b++) if (chosen[b]) for (int r = 0; r < R; r++) sel_ref[n++] = b * R + r;
+ for (int t = blocks * R; t < vis; t++) sel_ref[n++] = t;
+ int same = sel[s * selrow] == n;
+ for (int j = 0; same && j < n; j++) same = sel[s * selrow + 1 + j] == sel_ref[j];
+ CHECK(ok && same, "qsa selection row %d: count %d vs %d", s, sel[s * selrow], n);
+ }
+ vkc_free(ib); vkc_free(wb); vkc_free(pb); vkc_free(cb); vkc_free(cb2); vkc_free(qb); vkc_free(sb); vkc_free(lb);
+ free(ik); free(w); free(pk); free(cs); free(pd); free(qi); free(sel);
+}
+
+/* ---- PLE ---------------------------------------------------------------------------- */
+static void test_ple(void) {
+ int S = 4, C = 3, H = 20, W = C * H, CK = 4, NG = 3, SL = (CK - 1) * NG;
+ float *keys = fvec((size_t)S * W, 1.f), *hyp = fvec((size_t)S * W, 1.f), *val = fvec((size_t)S * H, 1.f);
+ float *prm = fvec((size_t)3 * W, 0.3f), *conv = fvec((size_t)W * CK, 0.5f), *ring = fvec((size_t)W * SL, 0.5f);
+ float *ring0 = malloc((size_t)W * SL * sizeof *ring0); memcpy(ring0, ring, (size_t)W * SL * sizeof *ring0);
+ float *href = malloc((size_t)S * W * sizeof *href); memcpy(href, hyp, (size_t)S * W * sizeof *href);
+ float *snap = malloc((size_t)W * SL * sizeof *snap);
+ float *gated = malloc((size_t)S * W * sizeof *gated), *normv = malloc((size_t)S * W * sizeof *normv);
+ for (int s = 0; s < S; s++) for (int b = 0; b < C; b++) {
+ float kn[64], qn[64]; double sk = 0, sq = 0;
+ for (int d = 0; d < H; d++) { sk += (double)keys[s * W + b * H + d] * keys[s * W + b * H + d]; sq += (double)hyp[s * W + b * H + d] * hyp[s * W + b * H + d]; }
+ float rk = 1.f / sqrtf((float)(sk / H) + 1e-6f), rq = 1.f / sqrtf((float)(sq / H) + 1e-6f);
+ for (int d = 0; d < H; d++) { kn[d] = keys[s * W + b * H + d] * rk * (1.f + prm[b * H + d]); qn[d] = hyp[s * W + b * H + d] * rq * (1.f + prm[W + b * H + d]); }
+ float dot = 0; for (int d = 0; d < H; d++) dot += kn[d] * qn[d]; dot /= sqrtf((float)H);
+ float g = sigm(copysignf(sqrtf(fmaxf(fabsf(dot), 1e-6f)), dot));
+ double sg = 0;
+ for (int d = 0; d < H; d++) { gated[s * W + b * H + d] = g * val[s * H + d]; sg += (double)gated[s * W + b * H + d] * gated[s * W + b * H + d]; }
+ float rg = 1.f / sqrtf((float)(sg / H) + 1e-6f);
+ for (int d = 0; d < H; d++) normv[s * W + b * H + d] = gated[s * W + b * H + d] * rg * (1.f + prm[2 * W + b * H + d]);
+ }
+ for (int s = 0; s < S; s++) for (int d = 0; d < W; d++) {
+ float a = conv[d * CK + CK - 1] * normv[s * W + d];
+ for (int k = 0; k < CK - 1; k++) a += conv[d * CK + k] * ring[d * SL + k * NG];
+ href[s * W + d] += gated[s * W + d] + a * sigm(a);
+ float *rg = ring + d * SL; for (int k = 0; k < SL - 1; k++) rg[k] = rg[k + 1]; rg[SL - 1] = normv[s * W + d];
+ if (s == 1) memcpy(snap + d * SL, rg, SL * sizeof(float));
+ }
+ VkcBuf *kb = up(keys, (size_t)S * W), *hb = up(hyp, (size_t)S * W), *vb = up(val, (size_t)S * H), *pb = up(prm, (size_t)3 * W);
+ VkcBuf *cb = up(conv, (size_t)W * CK), *gb = vkc_buf((size_t)S * W * 4, VKC_DEV), *nb = vkc_buf((size_t)S * W * 4, VKC_DEV);
+ VkcBuf *rb = vkc_buf((size_t)2 * W * SL * 4, VKC_DEV);
+ vkc_begin(); vkc_write(rb, 0, ring0, (size_t)W * SL * 4); vkc_submit(1);
+ VkcPle p0 = {0, S, C, H, CK, NG, 0, 0, 0, -1, 0, 1e-6f, 0, 0, 0};
+ VkcPle p1 = {1, S, C, H, CK, NG, 0, 0, 0, 1, W * SL, 1e-6f, 0, 0, 0};
+ vkc_begin(); int ok = vkc_ple(kb, hb, vb, pb, gb, nb, NULL, NULL, &p0) && vkc_ple(NULL, hb, NULL, NULL, gb, nb, cb, rb, &p1); vkc_submit(1);
+ float *hd = down(hb, 0, (size_t)S * W), *rd = down(rb, 0, (size_t)W * SL), *sd = down(rb, (size_t)W * SL, (size_t)W * SL);
+ double e1 = relerr(hd, href, (size_t)S * W, 1e-3), e2 = relerr(rd, ring, (size_t)W * SL, 1e-3), e3 = relerr(sd, snap, (size_t)W * SL, 1e-3);
+ CHECK(ok && e1 < 2e-5 && e2 < 2e-5 && e3 < 2e-5, "ple: hyper %.2e ring %.2e snap %.2e", e1, e2, e3);
+ vkc_free(kb); vkc_free(hb); vkc_free(vb); vkc_free(pb); vkc_free(cb); vkc_free(gb); vkc_free(nb); vkc_free(rb);
+ free(keys); free(hyp); free(val); free(prm); free(conv); free(ring); free(ring0); free(href); free(snap); free(gated); free(normv);
+ free(hd); free(rd); free(sd);
+}
+
+/* ---- frames: many ops, frames in flight, ordering ---------------------------------------- */
+static void test_frames(void) {
+ int n = 1000;
+ float *a = fvec(n, 1.f), *acc = calloc(n, sizeof *acc);
+ VkcBuf *x = vkc_buf((size_t)n * 4, VKC_DEV), *y = vkc_buf((size_t)n * 4, VKC_DEV);
+ int ok = 1;
+ for (int f = 0; f < 11; f++) { /* more frames than the ring: reuse while others are in flight */
+ ok &= vkc_begin();
+ ok &= vkc_write(y, 0, a, (size_t)n * 4);
+ for (int k = 0; k < 3; k++) {
+ VkcEw p = {VKC_EW_ADD, n, n, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ ok &= vkc_ew(x, x, y, NULL, NULL, &p);
+ for (int i = 0; i < n; i++) acc[i] += a[i];
+ }
+ ok &= vkc_submit(f == 10);
+ for (int i = 0; i < n; i++) a[i] = a[i] * 0.5f + 0.25f;
+ }
+ float *got = down(x, 0, n);
+ double e = relerr(got, acc, n, 1e-3);
+ VkcStats st; vkc_stats(&st);
+ CHECK(ok && e < 1e-5, "frames: err %.2e (%llu frames, %llu barriers)", e, st.frames, st.barriers);
+ vkc_free(x); vkc_free(y); free(a); free(acc); free(got);
+}
+
+/* COLI_VK_CHAIN_BENCH=1: decode-shaped GEMVs back to back in one frame (the chain's
+ * situation: no host gap between matrices), per weight format, in GB/s of weights. */
+#include <time.h>
+static double bnow(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec / 1e9; }
+static void bench_gemv(void) {
+ int shapes[3][2] = {{2560, 10240}, {6144, 2560}, {2560, 640}};
+ int fmts[3] = {1, 4, 11};
+ for (int f = 0; f < 3; f++) for (int k = 0; k < 3; k++) {
+ int I = shapes[k][0], O = shapes[k][1], fmt = fmts[f], gs = fmt == 4 ? 64 : 0, nm = 24;
+ size_t rb = fmt == 1 ? (size_t)I : fmt == 4 ? (size_t)I / 2 : (size_t)I * 2;
+ ColiVkTensor **t = calloc(nm, sizeof *t);
+ uint8_t *w = malloc(rb * O); float *sc = malloc((size_t)O * (I / 64 + 1) * sizeof(float));
+ for (size_t i = 0; i < rb * O; i++) w[i] = (uint8_t)rnd();
+ for (size_t i = 0; i < (size_t)O * (I / 64 + 1); i++) sc[i] = 0.01f;
+ for (int j = 0; j < nm; j++) coli_vk_tensor_ensure(&t[j], w, sc, fmt, I, O, gs);
+ float *x = fvec(I, 1.f);
+ VkcBuf *xb = up(x, I), *yb = vkc_buf((size_t)O * 4 * nm, VKC_DEV);
+ double best = 1e9;
+ for (int r = 0; r < 4; r++) {
+ vkc_begin();
+ for (int j = 0; j < nm; j++) vkc_matmul(t[j], xb, 0, yb, (size_t)j * O, 1);
+ double t0 = bnow(); vkc_submit(1); double dt = bnow() - t0;
+ if (dt < best) best = dt;
+ }
+ printf("bench fmt %2d [%5d x %5d] x %d: %.3f ms per matrix, %.1f GB/s of weights\n", fmt, O, I, nm,
+ best * 1e3 / nm, (double)rb * O * nm / best / 1e9);
+ for (int j = 0; j < nm; j++) coli_vk_tensor_free(t[j]);
+ vkc_free(xb); vkc_free(yb); free(t); free(w); free(sc); free(x);
+ }
+}
+
+int main(int argc, char **argv) {
+ const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
+ if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }
+ if (!vkc_init()) { printf("FAIL: the chain's pipelines did not come up\n"); return 1; }
+ if (getenv("COLI_VK_CHAIN_BENCH")) { bench_gemv(); vkc_shutdown(); coli_vk_shutdown(); return 0; }
+ int fmts[4] = {1, 4, 10, 11};
+ for (int k = 0; k < 4; k++) {
+ test_matmul(fmts[k], 1, 192, 70, 0, 0);
+ test_matmul(fmts[k], 3, 128, 96, 64, 128); /* GEMV rows at bindable offsets */
+ test_matmul(fmts[k], 40, 256, 160, 0, 0); /* the tiled GEMM */
+ test_matmul(fmts[k], 2, 64, 33, 3, 5); /* offsets the device cannot bind */
+ test_matmul(fmts[k], 1, 100, 33, 0, 0); /* rows that are not whole 16-byte steps */
+ test_matmul(fmts[k], 2, 3072, 40, 0, 0); /* a long row (chain_gemv's staging) */
+ }
+ printf("matmul done\n");
+ test_norm(VKC_NORM_ADD1, 0); test_norm(VKC_NORM_ADD1, 1); test_norm(0, 0); test_norm(VKC_NORM_NOW, 1); test_norm(VKC_NORM_L2 | VKC_NORM_NOW, 0);
+ printf("norm done\n");
+ test_rope();
+ test_attn(1, 0, 16, 0); test_attn(1, 140, 32, 0); test_attn(5, 200, 64, 0); test_attn(6, 9, 256, 1); test_attn(130, 3, 24, 0);
+ printf("attn done\n");
+ test_dnconv(0); test_dnconv(1);
+ test_dnrec(8, 8, 8, 4, 0); test_dnrec(4, 4, 4, 2, 1); test_dnrec(128, 128, 4, 2, 0); test_dnrec(32, 100, 6, 3, 1);
+ printf("deltanet done\n");
+ test_ew();
+ test_qsa();
+ test_ple();
+ test_frames();
+ VkcStats st; vkc_stats(&st);
+ printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);
+ vkc_shutdown();
+ coli_vk_shutdown();
+ printf(fails ? "FAIL (%d)\n" : "PASS\n", fails);
+ return fails ? 1 : 0;
+}
diff --git a/c/tools/clean.py b/c/tools/clean.py
index b58a33c3..991c6f5f 100644
--- a/c/tools/clean.py
+++ b/c/tools/clean.py
@@ -29,7 +29,7 @@ FILES = [
# VK=1, XDNA=1 and METAL=1 (qwen36's qpack) objects. Left behind once
# their .d is cleaned, an object would sit in the tree with no record of
# the headers it read.
- "backend_vulkan.o", "vk_tier.o", "backend_xdna.o", "qwen36_qpack.o", "qpack.o",
+ "backend_vulkan.o", "vk_tier.o", "vk_chain.o", "backend_xdna.o", "qwen36_qpack.o", "qpack.o",
"backend_cuda_test", "backend_cuda_test.exe",
"mxfp4_expert_cuda_test", "mxfp4_expert_cuda_test.exe",
"backend_cuda_bench", "backend_cuda_bench.exe",
diff --git a/c/vk_chain.c b/c/vk_chain.c
new file mode 100644
index 00000000..f4b81f04
--- /dev/null
+++ b/c/vk_chain.c
@@ -0,0 +1,810 @@
+/* vk_chain.c -- a layer's dense chain recorded into one Vulkan submission (vk_chain.h).
+ *
+ * The backend (backend_vulkan.c) owns the device and the resident tensors; this file
+ * builds its own pipelines from the same shader directory, with one pipeline layout
+ * for all of them (eight storage buffers, 128 bytes of push constants), and records
+ * them into a ring of frames. A frame is a command buffer, its fence, its descriptor
+ * pools and the staging and temporary buffers its copies read; it is reused once its
+ * fence has signalled.
+ *
+ * Barriers: one global memory barrier (compute and transfer, both ways) whenever an op
+ * touches a buffer the ops since the last barrier wrote, or writes one they read. The
+ * frame opens with that barrier (it orders the frame after everything submitted before
+ * it on the queue) and closes with one to the host. Weights are never written, so they
+ * never cause one. */
+#include "vk_chain.h"
+#include "vk_alloc.h"
+#include <vulkan/vulkan.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+static double vkc_now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }
+
+/* ---- pipelines ------------------------------------------------------------------ */
+enum { P_NORM, P_ROPE, P_ATTN, P_DNCONV, P_EW, P_QSA, P_PLE, P_GEMV, P_NPIPE };
+static const char *const pipe_file[P_NPIPE] = {
+ "chain_norm.spv", "chain_rope.spv", "chain_attn.spv", "chain_dnconv.spv",
+ "chain_ew.spv", "chain_qsa.spv", "chain_ple.spv", "qmatmul.spv"};
+#define VKC_GEMM_MAX 4
+#define VKC_DNREC_MAX 8
+#define VKC_FRAMES 4
+#define VKC_BIND 8
+#define VKC_TRACK 64
+#define VKC_PROF_Q 512
+/* COLI_VK_CHAIN_PROF=1: device time per kind of op (timestamps after every op; ops
+ * that run side by side between two barriers share the time unevenly) */
+enum { PK_GEMV8, PK_GEMV, PK_GEMM, PK_NORM, PK_ROPE, PK_ATTN, PK_DNCONV, PK_DNREC, PK_EW, PK_QSA, PK_PLE, PK_COPY, PK_N };
+static const char *const pk_name[PK_N] = {"GEMV int8", "GEMV other", "tiled GEMM", "norm", "rope", "attention",
+ "dn conv", "dn recurrence", "element-wise", "qsa", "ple", "copy"};
+
+/* ---- memory: blocks per kind, buffers bound at offsets inside them -------------- */
+typedef struct { VkDeviceMemory mem; uint8_t *map; } VkcBlock;
+typedef struct { VkaPool p; uint32_t memtype; int mapped; } VkcPool;
+struct VkcBuf {
+ VkBuffer buf;
+ int kind;
+ size_t bytes;
+ VkaRange r;
+ uint8_t *ptr;
+ unsigned long long last_frame; /* serial of the last frame that bound it */
+};
+
+typedef struct {
+ VkCommandBuffer cmd;
+ VkFence fence;
+ VkDescriptorPool *pools; int npools, cur_pool;
+ VkcBuf **tmp; int ntmp, ctmp; /* staging / temporaries, freed when the frame comes back */
+ VkcBuf *stage; size_t stage_used;
+ int inflight, open;
+ unsigned long long serial;
+ VkQueryPool qp; int nq; unsigned char kind[VKC_PROF_Q]; /* COLI_VK_CHAIN_PROF: a timestamp after each op */
+} VkcFrame;
+
+static struct {
+ int ready, lost;
+ ColiVkCore core;
+ VkDevice dev;
+ VkQueue queue;
+ VkDescriptorSetLayout dsl;
+ VkPipelineLayout pl;
+ VkShaderModule mod[P_NPIPE], mod_gemm, mod_dnrec;
+ VkPipeline pipe[P_NPIPE];
+ VkPipeline gemm[VKC_GEMM_MAX]; int gemm_bm[VKC_GEMM_MAX], gemm_bn[VKC_GEMM_MAX], ngemm;
+ VkPipeline dnrec[VKC_DNREC_MAX]; int dnrec_kd[VKC_DNREC_MAX], ndnrec;
+ VkShaderModule mod_gemv4; VkPipeline gemv4; int gemv4_xs; /* chain_gemv.comp: the vectorized decode GEMV */
+ VkCommandPool cpool;
+ VkcFrame fr[VKC_FRAMES];
+ int cur; /* the open frame, -1 none */
+ unsigned long long serial; /* frames begun so far */
+ VkcPool pool[3];
+ VkcBuf *dummy;
+ size_t align;
+ int gemm_rows; /* -1 the backend's rule, 0 never, else the rows */
+ VkPipeline bound;
+ /* hazard tracking since the last barrier */
+ VkBuffer wr[VKC_TRACK], rd[VKC_TRACK]; int nwr, nrd;
+ VkcStats st;
+ int prof, kind; float ts_period; uint32_t ts_mask;
+ double prof_ms[PK_N]; unsigned long long prof_n[PK_N];
+} K = {.cur = -1, .gemm_rows = -1};
+
+int vkc_ready(void) { return K.ready && !K.lost; }
+int vkc_lost(void) { return K.lost; }
+
+static void lose(const char *what, VkResult r) {
+ if (!K.lost) fprintf(stderr, "[VK] chain: %s failed (%d): the device is lost\n", what, (int)r);
+ K.lost = 1;
+ coli_vk_mark_lost();
+}
+
+/* ---- blocks ---------------------------------------------------------------------- */
+#define VKC_BLOCK ((uint64_t)64 << 20)
+static int pool_grow(VkcPool *P, uint64_t need) {
+ uint64_t cap = vka_block_size_for(&P->p, need);
+ VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+ .allocationSize = cap, .memoryTypeIndex = P->memtype};
+#ifdef VK_EXT_memory_priority
+ VkMemoryPriorityAllocateInfoEXT pri = {.sType = VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT,
+ .priority = 1.0f}; /* rides every chain submit: never evicted before weights */
+ if (K.core.has_prio) ai.pNext = &pri;
+#endif
+ VkcBlock *bk = calloc(1, sizeof *bk);
+ if (!bk) return 0;
+ if (vkAllocateMemory(K.dev, &ai, NULL, &bk->mem) != VK_SUCCESS) { free(bk); return 0; }
+ if (P->mapped && vkMapMemory(K.dev, bk->mem, 0, cap, 0, (void **)&bk->map) != VK_SUCCESS) {
+ vkFreeMemory(K.dev, bk->mem, NULL); free(bk); return 0;
+ }
+ if (vka_pool_add_block(&P->p, cap, bk) < 0) {
+ if (bk->map) vkUnmapMemory(K.dev, bk->mem);
+ vkFreeMemory(K.dev, bk->mem, NULL); free(bk); return 0;
+ }
+ return 1;
+}
+static uint32_t memtype_of(int kind) {
+ return kind == VKC_UP ? K.core.memtype_host : kind == VKC_DOWN ? K.core.memtype_cached : K.core.memtype_dev;
+}
+static int memtype_host_visible(uint32_t t) {
+ VkPhysicalDeviceMemoryProperties mp;
+ vkGetPhysicalDeviceMemoryProperties((VkPhysicalDevice)K.core.phys, &mp);
+ return (mp.memoryTypes[t].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;
+}
+
+static void track_reset(void) { K.nwr = K.nrd = 0; }
+static void zero_now(VkcBuf *b);
+
+VkcBuf *vkc_buf(size_t bytes, int kind) {
+ if (!vkc_ready() || kind < 0 || kind > 2) return NULL;
+ if (!bytes) bytes = 4;
+ bytes = (bytes + 3) & ~(size_t)3;
+ if (K.core.ssbo_range && bytes > K.core.ssbo_range) return NULL; /* one binding could not address it */
+ VkcBuf *b = calloc(1, sizeof *b);
+ if (!b) return NULL;
+ b->kind = kind; b->bytes = bytes;
+ VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bytes,
+ .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
+ if (vkCreateBuffer(K.dev, &bi, NULL, &b->buf) != VK_SUCCESS) { free(b); return NULL; }
+ VkMemoryRequirements req;
+ vkGetBufferMemoryRequirements(K.dev, b->buf, &req);
+ VkcPool *P = &K.pool[kind];
+ uint64_t align = req.alignment > K.align ? req.alignment : K.align;
+ if (!(req.memoryTypeBits & (1u << P->memtype)) ||
+ (!vka_alloc(&P->p, req.size, align, &b->r) &&
+ !(pool_grow(P, req.size + align) && vka_alloc(&P->p, req.size, align, &b->r)))) {
+ vkDestroyBuffer(K.dev, b->buf, NULL); free(b); return NULL;
+ }
+ VkcBlock *bk = P->p.b[b->r.block].user;
+ if (vkBindBufferMemory(K.dev, b->buf, bk->mem, b->r.off) != VK_SUCCESS) {
+ if (vka_free(&P->p, b->r)) { VkcBlock *e = P->p.b[b->r.block].user;
+ if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }
+ vka_pool_drop_block(&P->p, b->r.block); }
+ vkDestroyBuffer(K.dev, b->buf, NULL); free(b); return NULL;
+ }
+ b->ptr = bk->map ? bk->map + b->r.off : NULL;
+ K.st.dev_bytes += bytes;
+ if (b->ptr) memset(b->ptr, 0, bytes);
+ else zero_now(b);
+ return b;
+}
+
+static void buf_release(VkcBuf *b) {
+ VkcPool *P = &K.pool[b->kind];
+ vkDestroyBuffer(K.dev, b->buf, NULL);
+ if (vka_free(&P->p, b->r)) {
+ VkcBlock *e = P->p.b[b->r.block].user;
+ if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }
+ vka_pool_drop_block(&P->p, b->r.block);
+ }
+ K.st.dev_bytes -= b->bytes;
+ free(b);
+}
+
+/* a frame that may still read b must finish first */
+static int frame_wait(VkcFrame *f);
+void vkc_free(VkcBuf *b) {
+ if (!b) return;
+ if (!K.dev) { free(b); return; }
+ if (!K.lost)
+ for (int i = 0; i < VKC_FRAMES; i++) {
+ VkcFrame *f = &K.fr[i];
+ if ((f->inflight || f->open) && f->serial == b->last_frame) {
+ if (f->open) vkc_submit(1); else frame_wait(f);
+ }
+ }
+ for (int i = 0; i < K.nwr; i++) if (K.wr[i] == b->buf) K.wr[i] = VK_NULL_HANDLE;
+ for (int i = 0; i < K.nrd; i++) if (K.rd[i] == b->buf) K.rd[i] = VK_NULL_HANDLE;
+ buf_release(b);
+}
+int vkc_reserve(VkcBuf **b, size_t bytes, int kind) {
+ if (*b && (*b)->bytes >= bytes && (*b)->kind == kind) return 1;
+ size_t want = bytes;
+ if (*b) { want = (*b)->bytes + (*b)->bytes / 2; if (want < bytes) want = bytes; vkc_free(*b); }
+ *b = vkc_buf(want, kind);
+ return *b != NULL;
+}
+void *vkc_ptr(const VkcBuf *b) { return b ? b->ptr : NULL; }
+size_t vkc_bytes(const VkcBuf *b) { return b ? b->bytes : 0; }
+
+/* ---- init ------------------------------------------------------------------------ */
+static VkShaderModule load_module(const char *dir_spv, const char *file) {
+ char path[1200];
+ const char *sl = strrchr(dir_spv, '/');
+ size_t pre = sl ? (size_t)(sl - dir_spv) + 1 : 0;
+ if (pre + strlen(file) + 1 >= sizeof path) return VK_NULL_HANDLE;
+ memcpy(path, dir_spv, pre); strcpy(path + pre, file);
+ FILE *f = fopen(path, "rb");
+ if (!f) return VK_NULL_HANDLE;
+ fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
+ if (n <= 0 || n % 4) { fclose(f); return VK_NULL_HANDLE; }
+ uint32_t *code = malloc((size_t)n);
+ if (!code || fread(code, 1, (size_t)n, f) != (size_t)n) { fclose(f); free(code); return VK_NULL_HANDLE; }
+ fclose(f);
+ VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)n, .pCode = code};
+ VkShaderModule m = VK_NULL_HANDLE;
+ if (vkCreateShaderModule(K.dev, &ci, NULL, &m) != VK_SUCCESS) m = VK_NULL_HANDLE;
+ free(code);
+ if (!m) fprintf(stderr, "[VK] chain: cannot load %s\n", path);
+ return m;
+}
+static VkPipeline make_pipe(VkShaderModule m, const VkSpecializationInfo *si) {
+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT,
+ .module = m, .pName = "main", .pSpecializationInfo = si}, .layout = K.pl};
+ VkPipeline p = VK_NULL_HANDLE;
+ if (vkCreateComputePipelines(K.dev, VK_NULL_HANDLE, 1, &ci, NULL, &p) != VK_SUCCESS) return VK_NULL_HANDLE;
+ return p;
+}
+
+int vkc_init(void) {
+ if (K.ready) return !K.lost;
+ if (!coli_vk_core(&K.core)) return 0;
+ K.dev = (VkDevice)K.core.device; K.queue = (VkQueue)K.core.queue;
+ K.align = K.core.ssbo_align > 16 ? K.core.ssbo_align : 16;
+ VkDescriptorSetLayoutBinding b[VKC_BIND];
+ for (int i = 0; i < VKC_BIND; i++) b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
+ VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
+ .bindingCount = VKC_BIND, .pBindings = b};
+ VkPushConstantRange pr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = 128};
+ if (vkCreateDescriptorSetLayout(K.dev, &li, NULL, &K.dsl) != VK_SUCCESS) return 0;
+ VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
+ .setLayoutCount = 1, .pSetLayouts = &K.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
+ if (vkCreatePipelineLayout(K.dev, &pi, NULL, &K.pl) != VK_SUCCESS) return 0;
+ for (int i = 0; i < P_NPIPE; i++) {
+ K.mod[i] = load_module(K.core.spv_path, pipe_file[i]);
+ if (!K.mod[i] || !(K.pipe[i] = make_pipe(K.mod[i], NULL))) {
+ fprintf(stderr, "[VK] chain: shader %s unavailable, the chain stays off\n", pipe_file[i]);
+ return 0;
+ }
+ }
+ K.mod_dnrec = load_module(K.core.spv_path, "chain_dnrec.spv");
+ if (!K.mod_dnrec) return 0;
+ /* the vectorized decode GEMV, its x staging as large as the device's shared memory
+ * allows (16 KiB floats at most); COLI_VK_CHAIN_GEMV=0 keeps qmatmul.comp's */
+ {
+ const char *e = getenv("COLI_VK_CHAIN_GEMV");
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);
+ int xs = (int)(pp.limits.maxComputeSharedMemorySize / 16);
+ if (xs > 4096) xs = 4096;
+ if (!(e && *e == '0') && xs >= 256 && (K.mod_gemv4 = load_module(K.core.spv_path, "chain_gemv.spv"))) {
+ int32_t v = xs;
+ VkSpecializationMapEntry me = {0, 0, 4};
+ VkSpecializationInfo si = {1, &me, 4, &v};
+ if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;
+ }
+ }
+ /* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */
+ K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;
+ for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {
+ const int *t = K.core.gemm_tile[k];
+ int32_t sv[7] = {t[0], t[1], t[2], t[3], t[4], (t[0] / t[3]) * (t[1] / t[4]), t[5] ? 1 : 0};
+ VkSpecializationMapEntry me[7];
+ for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};
+ VkSpecializationInfo si = {7, me, sizeof sv, sv};
+ if (!(K.gemm[k] = make_pipe(K.mod_gemm, &si))) break;
+ K.gemm_bm[k] = t[0]; K.gemm_bn[k] = t[1]; K.ngemm = k + 1;
+ }
+ VkCommandPoolCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
+ .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = K.core.qfam};
+ if (vkCreateCommandPool(K.dev, &cp, NULL, &K.cpool) != VK_SUCCESS) return 0;
+ for (int i = 0; i < VKC_FRAMES; i++) {
+ VkCommandBufferAllocateInfo ca = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
+ .commandPool = K.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
+ VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
+ if (vkAllocateCommandBuffers(K.dev, &ca, &K.fr[i].cmd) != VK_SUCCESS ||
+ vkCreateFence(K.dev, &fi, NULL, &K.fr[i].fence) != VK_SUCCESS) return 0;
+ }
+ {
+ const char *e = getenv("COLI_VK_CHAIN_PROF");
+ uint32_t nqf = 0; vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)K.core.phys, &nqf, NULL);
+ VkQueueFamilyProperties qf[16]; if (nqf > 16) nqf = 16;
+ vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)K.core.phys, &nqf, qf);
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);
+ uint32_t bits = K.core.qfam < nqf ? qf[K.core.qfam].timestampValidBits : 0;
+ if (e && *e == '1' && bits && pp.limits.timestampPeriod > 0) {
+ K.prof = 1; K.ts_period = pp.limits.timestampPeriod;
+ K.ts_mask = bits >= 32 ? 0xffffffffu : (1u << bits) - 1;
+ for (int i = 0; i < VKC_FRAMES; i++) {
+ VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
+ .queryType = VK_QUERY_TYPE_TIMESTAMP, .queryCount = VKC_PROF_Q};
+ if (vkCreateQueryPool(K.dev, &qi, NULL, &K.fr[i].qp) != VK_SUCCESS) K.prof = 0;
+ }
+ }
+ }
+ for (int k = 0; k < 3; k++) {
+ K.pool[k].memtype = memtype_of(k);
+ K.pool[k].mapped = memtype_host_visible(K.pool[k].memtype);
+ vka_pool_init(&K.pool[k].p, VKC_BLOCK, 0);
+ }
+ if (!K.pool[VKC_UP].mapped || !K.pool[VKC_DOWN].mapped) return 0;
+ K.ready = 1;
+ K.dummy = vkc_buf(256, VKC_DEV);
+ if (!K.dummy) { K.ready = 0; return 0; }
+ return 1;
+}
+
+/* ---- frames ---------------------------------------------------------------------- */
+static void frame_reclaim(VkcFrame *f) {
+ for (int i = 0; i < f->ntmp; i++) buf_release(f->tmp[i]);
+ f->ntmp = 0;
+ f->stage_used = 0;
+ for (int i = 0; i < f->npools; i++) vkResetDescriptorPool(K.dev, f->pools[i], 0);
+ f->cur_pool = 0;
+}
+/* A frame is milliseconds of work and the thread waiting on it has nothing else to do:
+ * poll the fence for up to COLI_VK_CHAIN_SPIN_US (2000 us) before blocking, whose
+ * wake-up costs 50 to 150 us per frame, two frames a layer. */
+static long chain_spin_us(void) {
+ static long v = -1;
+ if (v < 0) { const char *e = getenv("COLI_VK_CHAIN_SPIN_US"); v = e && *e ? atol(e) : 2000; if (v < 0) v = 0; }
+ return v;
+}
+static int frame_wait(VkcFrame *f) {
+ if (!f->inflight) return 1;
+ double t0 = vkc_now_ms();
+ VkResult r = VK_NOT_READY;
+ for (long spin = chain_spin_us(); spin > 0 && r == VK_NOT_READY && (vkc_now_ms() - t0) * 1000.0 < spin; )
+ r = vkGetFenceStatus(K.dev, f->fence);
+ if (r == VK_NOT_READY) r = vkWaitForFences(K.dev, 1, &f->fence, VK_TRUE, 20000000000ULL);
+ K.st.wait_ms += vkc_now_ms() - t0; K.st.waits++;
+ f->inflight = 0;
+ if (r != VK_SUCCESS) { lose("fence wait", r); return 0; }
+ if (K.prof && f->qp && f->nq > 1) {
+ uint64_t ts[VKC_PROF_Q];
+ if (vkGetQueryPoolResults(K.dev, f->qp, 0, (uint32_t)f->nq, sizeof ts, ts, sizeof ts[0],
+ VK_QUERY_RESULT_64_BIT) == VK_SUCCESS)
+ for (int i = 1; i < f->nq; i++) {
+ uint64_t d = (ts[i] - ts[i - 1]) & (K.ts_mask == 0xffffffffu ? ~0ull : (uint64_t)K.ts_mask);
+ K.prof_ms[f->kind[i]] += (double)d * K.ts_period / 1e6; K.prof_n[f->kind[i]]++;
+ }
+ f->nq = 0;
+ }
+ frame_reclaim(f);
+ return 1;
+}
+static void prof_mark(VkcFrame *f, int kind) {
+ if (!K.prof || !f->qp || f->nq >= VKC_PROF_Q) return;
+ vkCmdWriteTimestamp(f->cmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, f->qp, (uint32_t)f->nq);
+ f->kind[f->nq++] = (unsigned char)kind;
+}
+int vkc_finish(void) {
+ if (!K.ready) return 0;
+ if (K.cur >= 0 && !vkc_submit(0)) return 0;
+ for (int i = 0; i < VKC_FRAMES; i++) if (!frame_wait(&K.fr[i])) return 0;
+ return !K.lost;
+}
+
+static void barrier_now(VkCommandBuffer c, VkPipelineStageFlags dst_stage, VkAccessFlags dst) {
+ VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = dst};
+ vkCmdPipelineBarrier(c, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, dst_stage,
+ 0, 1, &mb, 0, NULL, 0, NULL);
+ K.st.barriers++;
+}
+static const VkAccessFlags ALL_RW = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
+ VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
+#define ALL_STAGES (VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT)
+
+int vkc_begin(void) {
+ if (!vkc_ready()) return 0;
+ if (K.cur >= 0) return 1; /* already open */
+ int i = (int)(K.serial % VKC_FRAMES);
+ VkcFrame *f = &K.fr[i];
+ if (!frame_wait(f)) return 0;
+ frame_reclaim(f);
+ if (vkResetCommandBuffer(f->cmd, 0) != VK_SUCCESS) { lose("command buffer reset", VK_ERROR_DEVICE_LOST); return 0; }
+ VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
+ .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
+ vkBeginCommandBuffer(f->cmd, &bi);
+ f->serial = ++K.serial;
+ f->open = 1;
+ K.cur = i;
+ K.bound = VK_NULL_HANDLE;
+ /* after everything submitted earlier on this queue (and the host's writes) */
+ barrier_now(f->cmd, ALL_STAGES, ALL_RW);
+ track_reset();
+ if (K.prof && f->qp) {
+ vkCmdResetQueryPool(f->cmd, f->qp, 0, VKC_PROF_Q);
+ f->nq = 0;
+ vkCmdWriteTimestamp(f->cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, f->qp, 0);
+ f->kind[0] = 0; f->nq = 1;
+ }
+ return 1;
+}
+int vkc_submit(int wait) {
+ if (K.cur < 0) return vkc_ready();
+ VkcFrame *f = &K.fr[K.cur];
+ barrier_now(f->cmd, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_HOST_READ_BIT);
+ vkEndCommandBuffer(f->cmd);
+ f->open = 0; K.cur = -1;
+ if (K.lost) return 0;
+ { /* COLI_VK_CHAIN_FAULT=n (tests): the n-th submission fails as a lost device would */
+ static long fault = -2;
+ if (fault == -2) { const char *e = getenv("COLI_VK_CHAIN_FAULT"); fault = e && *e ? atol(e) : -1; }
+ if (fault > 0 && (long)K.st.frames + 1 >= fault) { lose("queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }
+ }
+ vkResetFences(K.dev, 1, &f->fence);
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &f->cmd};
+ VkResult r = vkQueueSubmit(K.queue, 1, &si, f->fence);
+ if (r != VK_SUCCESS) { lose("queue submit", r); return 0; }
+ f->inflight = 1;
+ K.st.frames++;
+ return wait ? frame_wait(f) : 1;
+}
+
+/* ---- recording helpers ----------------------------------------------------------- */
+static VkcFrame *open_frame(void) { return K.cur >= 0 ? &K.fr[K.cur] : NULL; }
+
+static int in_set(const VkBuffer *s, int n, VkBuffer b) { for (int i = 0; i < n; i++) if (s[i] == b) return 1; return 0; }
+/* a barrier before an op reading rd[] and writing wr[] when it conflicts with the ops
+ * recorded since the last one */
+static void hazard(VkcFrame *f, const VkBuffer *rd, int nr, const VkBuffer *wr, int nw) {
+ int need = 0;
+ for (int i = 0; i < nr && !need; i++) need = in_set(K.wr, K.nwr, rd[i]);
+ for (int i = 0; i < nw && !need; i++) need = in_set(K.wr, K.nwr, wr[i]) || in_set(K.rd, K.nrd, wr[i]);
+ if (need || K.nwr + nw > VKC_TRACK || K.nrd + nr > VKC_TRACK) {
+ barrier_now(f->cmd, ALL_STAGES, ALL_RW);
+ track_reset();
+ }
+ for (int i = 0; i < nr; i++) if (!in_set(K.rd, K.nrd, rd[i])) K.rd[K.nrd++] = rd[i];
+ for (int i = 0; i < nw; i++) if (!in_set(K.wr, K.nwr, wr[i])) K.wr[K.nwr++] = wr[i];
+}
+
+static VkDescriptorSet alloc_set(VkcFrame *f) {
+ for (;;) {
+ if (f->cur_pool < f->npools) {
+ VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
+ .descriptorPool = f->pools[f->cur_pool], .descriptorSetCount = 1, .pSetLayouts = &K.dsl};
+ VkDescriptorSet s;
+ if (vkAllocateDescriptorSets(K.dev, &da, &s) == VK_SUCCESS) return s;
+ f->cur_pool++;
+ continue;
+ }
+ VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 512 * VKC_BIND};
+ VkDescriptorPoolCreateInfo pc = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
+ .maxSets = 512, .poolSizeCount = 1, .pPoolSizes = &ps};
+ VkDescriptorPool *n = realloc(f->pools, (size_t)(f->npools + 1) * sizeof *n);
+ if (!n) return VK_NULL_HANDLE;
+ f->pools = n;
+ if (vkCreateDescriptorPool(K.dev, &pc, NULL, &f->pools[f->npools]) != VK_SUCCESS) return VK_NULL_HANDLE;
+ f->npools++;
+ }
+}
+
+/* One dispatch. bufs[i] NULL: the dummy; offs/ranges in bytes (range 0 = to the end). */
+typedef struct { VkBuffer buf; VkDeviceSize off, range; int written; } VkcBind;
+static int record(VkPipeline pipe, const VkcBind *bd, int n, const void *pc, size_t pcb,
+ uint32_t gx, uint32_t gy, uint32_t gz) {
+ VkcFrame *f = open_frame();
+ if (!f || K.lost || !gx || !gy || !gz) return f && !K.lost;
+ VkBuffer rd[VKC_BIND], wr[VKC_BIND]; int nr = 0, nw = 0;
+ VkDescriptorBufferInfo bi[VKC_BIND];
+ VkWriteDescriptorSet w[VKC_BIND];
+ VkDescriptorSet set = alloc_set(f);
+ if (!set) { fprintf(stderr, "[VK] chain: out of descriptor sets\n"); return 0; }
+ for (int i = 0; i < VKC_BIND; i++) {
+ if (i < n && bd[i].buf) {
+ bi[i] = (VkDescriptorBufferInfo){bd[i].buf, bd[i].off, bd[i].range ? bd[i].range : VK_WHOLE_SIZE};
+ if (bd[i].written) wr[nw++] = bd[i].buf; else rd[nr++] = bd[i].buf;
+ } else bi[i] = (VkDescriptorBufferInfo){K.dummy->buf, 0, VK_WHOLE_SIZE};
+ w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = set,
+ .dstBinding = (uint32_t)i, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
+ .pBufferInfo = &bi[i]};
+ }
+ vkUpdateDescriptorSets(K.dev, VKC_BIND, w, 0, NULL);
+ hazard(f, rd, nr, wr, nw);
+ if (K.bound != pipe) { vkCmdBindPipeline(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); K.bound = pipe; }
+ vkCmdBindDescriptorSets(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, K.pl, 0, 1, &set, 0, NULL);
+ if (pcb) vkCmdPushConstants(f->cmd, K.pl, VK_SHADER_STAGE_COMPUTE_BIT, 0, (uint32_t)pcb, pc);
+ vkCmdDispatch(f->cmd, gx, gy, gz);
+ K.st.ops++;
+ prof_mark(f, K.kind);
+ return 1;
+}
+static VkcBind B(VkcBuf *b, int written) {
+ if (b) b->last_frame = K.serial;
+ return (VkcBind){b ? b->buf : VK_NULL_HANDLE, 0, 0, written};
+}
+/* groups over x, spilling into y past the 65535 limit */
+static void grid(uint64_t groups, uint32_t *gx, uint32_t *gy) {
+ if (!groups) { *gx = *gy = 0; return; }
+ uint64_t x = groups < 65535 ? groups : 65535;
+ *gx = (uint32_t)x; *gy = (uint32_t)((groups + x - 1) / x);
+}
+
+/* ---- transfers ------------------------------------------------------------------- */
+static int copy_bytes(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t bytes) {
+ VkcFrame *f = open_frame();
+ if (!f || K.lost) return 0;
+ if (!bytes) return 1;
+ if (doff + bytes > dst->bytes || soff + bytes > src->bytes) { fprintf(stderr, "[VK] chain: copy out of range\n"); return 0; }
+ VkBuffer r = src->buf, w = dst->buf;
+ hazard(f, &r, 1, &w, 1);
+ VkBufferCopy c = {soff, doff, bytes};
+ vkCmdCopyBuffer(f->cmd, src->buf, dst->buf, 1, &c);
+ prof_mark(f, PK_COPY);
+ src->last_frame = dst->last_frame = K.serial;
+ return 1;
+}
+int vkc_copy(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t n) {
+ return copy_bytes(dst, doff * 4, src, soff * 4, n * 4);
+}
+int vkc_copy_regions(VkcBuf *dst, VkcBuf *src, const VkcRegion *r, int n) {
+ VkcFrame *f = open_frame();
+ if (!f || K.lost) return 0;
+ if (n < 1) return 1;
+ VkBufferCopy *c = malloc((size_t)n * sizeof *c);
+ if (!c) return 0;
+ for (int i = 0; i < n; i++) {
+ if ((r[i].dst + r[i].n) * 4 > dst->bytes || (r[i].src + r[i].n) * 4 > src->bytes) {
+ fprintf(stderr, "[VK] chain: copy region out of range\n"); free(c); return 0;
+ }
+ c[i] = (VkBufferCopy){r[i].src * 4, r[i].dst * 4, r[i].n * 4};
+ }
+ VkBuffer rb = src->buf, wb = dst->buf;
+ hazard(f, &rb, 1, &wb, 1);
+ vkCmdCopyBuffer(f->cmd, src->buf, dst->buf, (uint32_t)n, c);
+ prof_mark(f, PK_COPY);
+ src->last_frame = dst->last_frame = K.serial;
+ free(c);
+ return 1;
+}
+int vkc_zero(VkcBuf *dst, size_t off, size_t n) {
+ VkcFrame *f = open_frame();
+ if (!f || K.lost) return 0;
+ if (!n) return 1;
+ if ((off + n) * 4 > dst->bytes) return 0;
+ VkBuffer w = dst->buf;
+ hazard(f, NULL, 0, &w, 1);
+ vkCmdFillBuffer(f->cmd, dst->buf, off * 4, n * 4, 0);
+ dst->last_frame = K.serial;
+ return 1;
+}
+static void zero_now(VkcBuf *b) {
+ /* a device-only buffer starts zeroed too: its own little frame */
+ int was_open = K.cur >= 0;
+ if (!was_open && !vkc_begin()) return;
+ vkc_zero(b, 0, b->bytes / 4);
+ if (!was_open) vkc_submit(1);
+}
+int vkc_write(VkcBuf *dst, size_t off, const void *src, size_t bytes) {
+ VkcFrame *f = open_frame();
+ if (!f || K.lost) return 0;
+ if (!bytes) return 1;
+ /* always through the staging, even into host-visible memory: the copy runs in
+ * frame order, after the ops recorded before it (a direct memcpy would not) */
+ size_t at = (f->stage_used + 15) & ~(size_t)15;
+ if (!f->stage || at + bytes > f->stage->bytes) {
+ /* a new staging buffer; the old one stays alive until the frame comes back */
+ size_t want = f->stage ? 2 * f->stage->bytes : (size_t)4 << 20;
+ while (want < bytes) want *= 2;
+ VkcBuf *n = vkc_buf(want, VKC_UP);
+ if (!n) return 0;
+ if (f->stage) {
+ if (f->ntmp == f->ctmp) {
+ int c = f->ctmp ? 2 * f->ctmp : 8;
+ VkcBuf **t = realloc(f->tmp, (size_t)c * sizeof *t);
+ if (!t) { buf_release(n); return 0; }
+ f->tmp = t; f->ctmp = c;
+ }
+ f->tmp[f->ntmp++] = f->stage;
+ }
+ f->stage = n; at = 0;
+ }
+ memcpy(f->stage->ptr + at, src, bytes);
+ f->stage_used = at + bytes;
+ K.st.bytes_up += bytes;
+ return copy_bytes(dst, off * 4, f->stage, at, bytes);
+}
+int vkc_read(VkcBuf *src, size_t off, void *dst, size_t bytes) {
+ if (!vkc_ready()) return 0;
+ if (!bytes) return 1;
+ if (!vkc_finish()) return 0;
+ if (src->ptr && src->kind != VKC_UP) { /* the device wrote it into host-readable memory */
+ memcpy(dst, src->ptr + off * 4, bytes);
+ K.st.bytes_down += bytes;
+ return 1;
+ }
+ VkcBuf *rb = vkc_buf(bytes, VKC_DOWN);
+ if (!rb) return 0;
+ int ok = vkc_begin() && copy_bytes(rb, 0, src, off * 4, bytes) && vkc_submit(1);
+ if (ok) { memcpy(dst, rb->ptr, bytes); K.st.bytes_down += bytes; }
+ vkc_free(rb);
+ return ok;
+}
+
+/* ---- matmul ---------------------------------------------------------------------- */
+void vkc_gemm_rows(int rows) { K.gemm_rows = rows; }
+struct VkcPC { int fmt, S, I, O, rowWords, gs; };
+static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcBuf *y, size_t yb, int S) {
+ int path = -1; /* GEMM slot, -1 the GEMV */
+ if (K.ngemm && S >= 2) {
+ int take = K.gemm_rows < 0 ? (K.core.gemm_min_s && S >= K.core.gemm_min_s &&
+ (int64_t)S * ti->O >= K.core.gemm_min_so)
+ : (K.gemm_rows > 0 && S >= K.gemm_rows);
+ if (take) { path = 0; while (path + 1 < K.ngemm && S > K.gemm_bn[path]) path++; }
+ }
+ VkcBind bd[4] = {{x->buf, xb, 0, 0}, {(VkBuffer)ti->wbuf, 0, 0, 0}, {(VkBuffer)ti->sbuf, 0, 0, 0},
+ {y->buf, yb, 0, 1}};
+ x->last_frame = y->last_frame = K.serial;
+ struct VkcPC pc = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs};
+ int ok;
+ /* the vectorized GEMV where the row is whole 16-byte steps and fits the staging */
+ int per = ti->fmt == 1 ? 16 : ti->fmt == 4 ? 32 : ti->fmt == 11 ? 8 : ti->fmt == 10 ? 4 : 0;
+ int v4 = path < 0 && K.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&
+ (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= K.gemv4_xs;
+ K.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;
+ if (v4) {
+ /* each workgroup stages x once: give it enough rows that the staging does not
+ * rival the weights, while keeping enough workgroups to fill the device */
+ /* lanes per row: about sixteen 16-byte steps each (COLI_VK_CHAIN_GEMV_LPR overrides) */
+ const char *e = getenv("COLI_VK_CHAIN_GEMV_LPR");
+ int nq = ti->rowWords / 4, lpr = 4;
+ if (e && *e) lpr = atoi(e); else while (lpr < 64 && lpr * 16 < nq) lpr *= 2;
+ if (lpr < 1) lpr = 1;
+ struct { int fmt, S, I, O, rowWords, gs, lpr; } pc7 = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs, lpr};
+ int rows_wg = 256 / lpr; /* at least, at subgroups of 64 */
+ int wg = (ti->O + rows_wg - 1) / rows_wg; if (wg > 1024) wg = 1024;
+ ok = record(K.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);
+ }
+ else if (path >= 0)
+ ok = record(K.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + K.gemm_bm[path] - 1) / K.gemm_bm[path]),
+ (uint32_t)((S + K.gemm_bn[path] - 1) / K.gemm_bn[path]), 1);
+ else ok = record(K.pipe[P_GEMV], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + 7) / 8), (uint32_t)S, 1);
+ K.st.matmuls += ok; K.st.gemms += ok && path >= 0;
+ return ok;
+}
+int vkc_matmul(ColiVkTensor *t, VkcBuf *x, size_t xo, VkcBuf *y, size_t yo, int S) {
+ ColiVkTensorInfo ti;
+ if (!open_frame() || K.lost || S < 1 || S > 65535 || !coli_vk_tensor_info(t, &ti)) return 0;
+ size_t xb = xo * 4, yb = yo * 4, xn = (size_t)S * ti.I * 4, yn = (size_t)S * ti.O * 4;
+ if (xb + xn > x->bytes || yb + yn > y->bytes) { fprintf(stderr, "[VK] chain: matmul out of range\n"); return 0; }
+ if (xb % K.align == 0 && yb % K.align == 0) return matmul_aligned(&ti, x, xb, y, yb, S);
+ /* an offset the device cannot bind: through temporaries that live with the frame */
+ VkcFrame *f = open_frame();
+ VkcBuf *tx = vkc_buf(xn, VKC_DEV), *ty = vkc_buf(yn, VKC_DEV);
+ if (!tx || !ty) { if (tx) buf_release(tx); if (ty) buf_release(ty); return 0; }
+ if (f->ntmp + 2 > f->ctmp) {
+ int c = f->ctmp ? 2 * f->ctmp : 8; while (c < f->ntmp + 2) c *= 2;
+ VkcBuf **n = realloc(f->tmp, (size_t)c * sizeof *n);
+ if (!n) { buf_release(tx); buf_release(ty); return 0; }
+ f->tmp = n; f->ctmp = c;
+ }
+ f->tmp[f->ntmp++] = tx; f->tmp[f->ntmp++] = ty;
+ return copy_bytes(tx, 0, x, xb, xn) && matmul_aligned(&ti, tx, 0, ty, 0, S) && copy_bytes(y, yb, ty, 0, yn);
+}
+
+/* ---- the chain's shaders ----------------------------------------------------------- */
+int vkc_norm(VkcBuf *x, VkcBuf *w, VkcBuf *y, const VkcNorm *p) {
+ K.kind = PK_NORM;
+ VkcBind bd[3] = {B(x, x == y), B(w, 0), B(y, 1)};
+ uint32_t gx, gy; grid((uint64_t)p->nseg, &gx, &gy);
+ return record(K.pipe[P_NORM], bd, 3, p, sizeof *p, gx, gy, 1);
+}
+int vkc_rope(VkcBuf *x, VkcBuf *cs, const VkcRope *p) {
+ K.kind = PK_ROPE;
+ VkcBind bd[2] = {B(x, 1), B(cs, 0)};
+ uint32_t gx, gy; grid(((uint64_t)p->nseg * p->half_ + 63) / 64, &gx, &gy);
+ return record(K.pipe[P_ROPE], bd, 2, p, sizeof *p, gx, gy, 1);
+}
+int vkc_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf *sel, const VkcAttn *p) {
+ K.kind = PK_ATTN;
+ if (p->hd > 256 || p->H % p->KVH) return 0;
+ VkcBind bd[6] = {B(q, 0), B(kc, 0), B(vc, 0), B(o, 1), B(gate, 0), B(sel, 0)};
+ return record(K.pipe[P_ATTN], bd, 6, p, sizeof *p, (uint32_t)p->H, (uint32_t)p->S, 1);
+}
+int vkc_dnconv(VkcBuf *in, VkcBuf *w, VkcBuf *ring, VkcBuf *out, VkcBuf *snap, const VkcDnConv *p) {
+ K.kind = PK_DNCONV;
+ if (p->CK < 1 || p->CK > 9) return 0;
+ VkcBind bd[5] = {B(in, 0), B(w, 0), B(ring, 1), B(out, 1), B(snap, snap != NULL && snap != ring)};
+ if (snap == ring) bd[4].written = 0; /* the same buffer: tracked once, as written */
+ uint32_t gx, gy; grid(((uint64_t)p->CD + 63) / 64, &gx, &gy);
+ return record(K.pipe[P_DNCONV], bd, 5, p, sizeof *p, gx, gy, 1);
+}
+static VkPipeline dnrec_pipe(int KD) {
+ for (int i = 0; i < K.ndnrec; i++) if (K.dnrec_kd[i] == KD) return K.dnrec[i];
+ if (K.ndnrec == VKC_DNREC_MAX) return VK_NULL_HANDLE;
+ int32_t kd = KD;
+ VkSpecializationMapEntry me = {0, 0, 4};
+ VkSpecializationInfo si = {1, &me, 4, &kd};
+ VkPipeline p = make_pipe(K.mod_dnrec, &si);
+ if (!p) return VK_NULL_HANDLE;
+ K.dnrec[K.ndnrec] = p; K.dnrec_kd[K.ndnrec++] = KD;
+ return p;
+}
+int vkc_dnrec(int KD, VkcBuf *cv, VkcBuf *ab, VkcBuf *z, VkcBuf *st, VkcBuf *prm, VkcBuf *y, VkcBuf *snap, const VkcDnRec *p) {
+ K.kind = PK_DNREC;
+ if (p->VD > 128 || KD < 1 || KD > 256 || p->VH % p->KH) return 0;
+ VkPipeline pipe = dnrec_pipe(KD);
+ if (!pipe) return 0;
+ VkcBind bd[7] = {B(cv, 0), B(ab, 0), B(z, 0), B(st, 1), B(prm, 0), B(y, 1), B(snap, snap != NULL && snap != st)};
+ if (snap == st) bd[6].written = 0;
+ return record(pipe, bd, 7, p, sizeof *p, (uint32_t)p->VH, 1, 1);
+}
+int vkc_ew(VkcBuf *y, VkcBuf *a, VkcBuf *b, VkcBuf *c, VkcBuf *e, const VkcEw *p) {
+ K.kind = PK_EW;
+ VkcBind bd[5] = {B(y, 1), B(a == y ? NULL : a, 0), B(b == y ? NULL : b, 0), B(c == y ? NULL : c, 0), B(e == y ? NULL : e, 0)};
+ /* an input that is y itself reads through binding 0's alias: bind it there too */
+ if (a == y) bd[1] = (VkcBind){y->buf, 0, 0, 0};
+ if (b == y) bd[2] = (VkcBind){y->buf, 0, 0, 0};
+ if (c == y) bd[3] = (VkcBind){y->buf, 0, 0, 0};
+ if (e == y) bd[4] = (VkcBind){y->buf, 0, 0, 0};
+ uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);
+ return record(K.pipe[P_EW], bd, 5, p, sizeof *p, gx, gy, 1);
+}
+int vkc_qsa(VkcBuf *src, VkcBuf *w, VkcBuf *pk, VkcBuf *cs, VkcBuf *sc, VkcBuf *sel, const VkcQsa *p) {
+ K.kind = PK_QSA;
+ if (p->ID > 256) return 0;
+ if (p->mode == 0) {
+ VkcBind bd[4] = {B(src, 0), B(w, 0), B(pk, 1), B(cs, 0)};
+ return record(K.pipe[P_QSA], bd, 4, p, sizeof *p, (uint32_t)p->nb, 1, 1);
+ }
+ VkcBind bd[6] = {B(src, 0), {VK_NULL_HANDLE, 0, 0, 0}, B(pk, 0), {VK_NULL_HANDLE, 0, 0, 0}, B(sc, 1), B(sel, 1)};
+ return record(K.pipe[P_QSA], bd, 6, p, sizeof *p, (uint32_t)p->S, 1, 1);
+}
+int vkc_ple(VkcBuf *keys, VkcBuf *hyp, VkcBuf *val, VkcBuf *prm, VkcBuf *gated, VkcBuf *normv,
+ VkcBuf *conv, VkcBuf *ring, const VkcPle *p) {
+ K.kind = PK_PLE;
+ if (p->mode == 0) {
+ VkcBind bd[6] = {B(keys, 0), B(hyp, 0), B(val, 0), B(prm, 0), B(gated, 1), B(normv, 1)};
+ return record(K.pipe[P_PLE], bd, 6, p, sizeof *p, (uint32_t)(p->S * p->C), 1, 1);
+ }
+ if ((p->CK - 1) * p->NG > 32) return 0;
+ VkcBind bd[8] = {{VK_NULL_HANDLE, 0, 0, 0}, B(hyp, 1), {VK_NULL_HANDLE, 0, 0, 0}, {VK_NULL_HANDLE, 0, 0, 0},
+ B(gated, 0), B(normv, 0), B(conv, 0), B(ring, 1)};
+ uint32_t gx, gy; grid(((uint64_t)p->C * p->H + 255) / 256, &gx, &gy);
+ return record(K.pipe[P_PLE], bd, 8, p, sizeof *p, gx, gy, 1);
+}
+
+void vkc_stats(VkcStats *st) { *st = K.st; }
+void vkc_prof_print(void) {
+ if (!K.prof) return;
+ double tot = 0; for (int k = 0; k < PK_N; k++) tot += K.prof_ms[k];
+ fprintf(stderr, "[VK] chain profile: %.1f ms of device time", tot);
+ for (int k = 0; k < PK_N; k++)
+ if (K.prof_n[k]) fprintf(stderr, " | %s %.1f ms (%llu)", pk_name[k], K.prof_ms[k], K.prof_n[k]);
+ fprintf(stderr, "\n");
+}
+
+void vkc_shutdown(void) {
+ if (!K.ready) return;
+ if (!K.lost && coli_vk_available()) vkc_finish();
+ else vkDeviceWaitIdle(K.dev);
+ for (int i = 0; i < VKC_FRAMES; i++) {
+ VkcFrame *f = &K.fr[i];
+ frame_reclaim(f);
+ if (f->stage) buf_release(f->stage);
+ free(f->tmp);
+ for (int k = 0; k < f->npools; k++) vkDestroyDescriptorPool(K.dev, f->pools[k], NULL);
+ free(f->pools);
+ if (f->fence) vkDestroyFence(K.dev, f->fence, NULL);
+ if (f->qp) vkDestroyQueryPool(K.dev, f->qp, NULL);
+ }
+ if (K.dummy) buf_release(K.dummy);
+ for (int k = 0; k < 3; k++) {
+ VkaPool *P = &K.pool[k].p;
+ for (int b = 0; b < P->nb; b++) {
+ if (!P->b[b].present) continue;
+ VkcBlock *e = P->b[b].user;
+ if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }
+ }
+ vka_pool_destroy(P);
+ }
+ if (K.cpool) vkDestroyCommandPool(K.dev, K.cpool, NULL);
+ for (int i = 0; i < P_NPIPE; i++) {
+ if (K.pipe[i]) vkDestroyPipeline(K.dev, K.pipe[i], NULL);
+ if (K.mod[i]) vkDestroyShaderModule(K.dev, K.mod[i], NULL);
+ }
+ for (int i = 0; i < K.ngemm; i++) vkDestroyPipeline(K.dev, K.gemm[i], NULL);
+ for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);
+ if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);
+ if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);
+ if (K.mod_gemm) vkDestroyShaderModule(K.dev, K.mod_gemm, NULL);
+ if (K.mod_dnrec) vkDestroyShaderModule(K.dev, K.mod_dnrec, NULL);
+ if (K.pl) vkDestroyPipelineLayout(K.dev, K.pl, NULL);
+ if (K.dsl) vkDestroyDescriptorSetLayout(K.dev, K.dsl, NULL);
+ memset(&K, 0, sizeof K);
+ K.cur = -1; K.gemm_rows = -1;
+}
diff --git a/c/vk_chain.h b/c/vk_chain.h
new file mode 100644
index 00000000..5c177295
--- /dev/null
+++ b/c/vk_chain.h
@@ -0,0 +1,135 @@
+/* vk_chain.h -- a layer's dense chain on the Vulkan device, recorded into one submission.
+ *
+ * Why: the engines' dense part on Vulkan was one synchronous coli_vk_matmul per matrix
+ * (about 726 submits and host round trips per Qwen3.8 decode token), slower on an
+ * integrated GPU than the CPU. Here an engine records a whole layer -- norms,
+ * projections, RoPE, attention over a KV cache that lives on the device, the Gated
+ * DeltaNet recurrence with its state on the device, gates, the router logits, the
+ * shared expert, the residual add -- into one command buffer, and the residual stream
+ * stays on the device from one layer to the next. Only what the CPU needs crosses:
+ * the rows the CPU's routed experts read and the router logits, then the routed sum
+ * coming back.
+ *
+ * The pieces:
+ * - buffers (VkcBuf): VKC_DEV the device's own (state, scratch), VKC_UP written by the
+ * host and read by the device, VKC_DOWN written by the device and read by the host.
+ * They are sub-allocated from a few large memory blocks per kind (vk_alloc.h), so a
+ * submit references a handful of allocations, not hundreds.
+ * - recording: vkc_begin opens a command buffer (a ring of frames, each with its own
+ * fence and descriptor pool); every op records the barrier it needs (a buffer read or
+ * written after an earlier write, or written after a read, since the last barrier);
+ * vkc_submit(wait) sends it. Submissions run in order on the backend's main queue, a
+ * frame's first barrier orders it after everything submitted before it.
+ * - ops: vkc_matmul over the resident tensors the engines already upload
+ * (coli_vk_tensor_ensure), the GEMV per row or, from the backend's threshold, the
+ * fp32 tiled GEMM; and the chain's shaders (shaders/chain_*.comp), each documented
+ * at its top: chain_norm, chain_rope, chain_attn, chain_dnconv, chain_dnrec,
+ * chain_ew, chain_qsa, chain_ple. Offsets and strides are in floats.
+ *
+ * Threading: the engine thread only (the main queue is the backend's, used from the
+ * same thread by coli_vk_matmul; the expert tier submits on its own queue).
+ * A failed fence wait marks the device lost (coli_vk_mark_lost): vkc_lost() says so,
+ * and every later call returns 0.
+ *
+ * tests/test_vk_chain.c checks every op against a CPU reference (make vk-chain-check). */
+#ifndef COLI_VK_CHAIN_H
+#define COLI_VK_CHAIN_H
+#include <stddef.h>
+#include <stdint.h>
+#include "backend_vulkan.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct VkcBuf VkcBuf;
+#define VKC_DEV 0
+#define VKC_UP 1
+#define VKC_DOWN 2
+
+int vkc_init(void); /* after coli_vk_init; 1 = the chain's pipelines are up */
+int vkc_ready(void);
+int vkc_lost(void);
+void vkc_shutdown(void); /* before coli_vk_shutdown (register it with atexit after it) */
+
+VkcBuf *vkc_buf(size_t bytes, int kind); /* zero-filled; NULL when out of memory */
+void vkc_free(VkcBuf *b); /* waits for the frames that may read it */
+int vkc_reserve(VkcBuf **b, size_t bytes, int kind); /* at least `bytes`; growing drops the contents */
+void *vkc_ptr(const VkcBuf *b); /* host mapping (VKC_UP, VKC_DOWN; VKC_DEV when host-visible) */
+size_t vkc_bytes(const VkcBuf *b);
+
+/* recording */
+int vkc_begin(void);
+int vkc_submit(int wait);
+int vkc_finish(void); /* wait for every submitted frame */
+
+/* transfers, recorded into the open frame (offsets and counts in floats) */
+int vkc_copy(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t n);
+int vkc_zero(VkcBuf *dst, size_t off, size_t n);
+/* one copy command over n regions (a KV row per head, say) */
+typedef struct { size_t dst, src, n; } VkcRegion;
+int vkc_copy_regions(VkcBuf *dst, VkcBuf *src, const VkcRegion *r, int n);
+int vkc_write(VkcBuf *dst, size_t off, const void *src, size_t bytes); /* through the frame's staging */
+/* synchronous: finishes what is in flight, copies, waits (the open frame, if any, is submitted first) */
+int vkc_read(VkcBuf *src, size_t off, void *dst, size_t bytes);
+
+/* y[S][O] = x[S][I] @ W^T for a resident tensor (its fmt, I, O); x and y at float offsets */
+int vkc_matmul(ColiVkTensor *t, VkcBuf *x, size_t xo, VkcBuf *y, size_t yo, int S);
+/* Rows from which vkc_matmul takes the tiled GEMM: -1 = the backend's rule (S >= 2 and
+ * S*O >= 4096), 0 = never (an MTP verify, whose rows must get a decode step's bits). */
+void vkc_gemm_rows(int rows);
+
+/* chain_norm.comp */
+typedef struct { int nseg, D, per_row, x_off, x_row, x_seg, y_off, y_row, y_seg, w_off, w_mod, flags; float eps, post; } VkcNorm;
+#define VKC_NORM_ADD1 1
+#define VKC_NORM_NOW 2
+#define VKC_NORM_L2 4
+int vkc_norm(VkcBuf *x, VkcBuf *w, VkcBuf *y, const VkcNorm *p);
+/* chain_rope.comp */
+typedef struct { int nseg, per_row, x_off, x_row, x_seg, half_, cs_off, cs_row; } VkcRope;
+int vkc_rope(VkcBuf *x, VkcBuf *cs, const VkcRope *p);
+/* chain_attn.comp (k_off/v_off: where the layer's cache starts in kc/vc) */
+typedef struct { int S, H, KVH, hd, pos_base, cap, q_off, q_row, q_seg, g_off, g_row, g_seg, has_gate,
+ o_off, o_row, sel_off, sel_row; float scale; int k_off, v_off; } VkcAttn;
+int vkc_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf *sel, const VkcAttn *p);
+/* chain_dnconv.comp */
+typedef struct { int S, CD, CK, in_off, in_row, out_off, out_row, snap_row, order, w_off, ring_off, snap_off; } VkcDnConv;
+int vkc_dnconv(VkcBuf *in, VkcBuf *w, VkcBuf *ring, VkcBuf *out, VkcBuf *snap, const VkcDnConv *p);
+/* chain_dnrec.comp (KD a specialization constant, VD <= 128) */
+typedef struct { int S, VH, KH, VD, Ktot, cv_off, cv_row, b_off, b_row, a_off, a_row, z_off, z_row,
+ y_off, y_row, snap_row, flags; float eps, qscale; int st_off, snap_off, prm_off; } VkcDnRec;
+int vkc_dnrec(int KD, VkcBuf *cv, VkcBuf *ab, VkcBuf *z, VkcBuf *st, VkcBuf *prm, VkcBuf *y, VkcBuf *snap, const VkcDnRec *p);
+/* chain_ew.comp */
+#define VKC_EW_ADD 0
+#define VKC_EW_COMBINE 1
+#define VKC_EW_SWIGLU 2
+#define VKC_EW_HC_LOW 3
+#define VKC_EW_HC_MIX 4
+#define VKC_EW_HC_INJ 5
+#define VKC_EW_HC_APPLY 6
+typedef struct { int op, n, D, C, flags, e_row, y_off, a_off, b_off, c_off, e_off; float fc; } VkcEw;
+int vkc_ew(VkcBuf *y, VkcBuf *a, VkcBuf *b, VkcBuf *c, VkcBuf *e, const VkcEw *p);
+/* chain_qsa.comp (mode 0: nb block keys from b0; mode 1: S rows' selections) */
+typedef struct { int mode, ID, R, b0, half_, S, pos_base, budget, IQ, q_off, q_row, nbmax, sel_row; float eps;
+ int src_off, w_off, pk_off, nb; } VkcQsa;
+int vkc_qsa(VkcBuf *src, VkcBuf *w, VkcBuf *pk, VkcBuf *cs, VkcBuf *sc, VkcBuf *sel, const VkcQsa *p);
+/* chain_ple.comp (mode 0: the gate over S*C (row, stream) pairs; mode 1: the convolution) */
+typedef struct { int mode, S, C, H, CK, NG, keys_off, hyp_off, val_off, snap_row, snap_off; float eps;
+ int prm_off, conv_off, ring_off; } VkcPle;
+int vkc_ple(VkcBuf *keys, VkcBuf *hyp, VkcBuf *val, VkcBuf *prm, VkcBuf *gated, VkcBuf *normv,
+ VkcBuf *conv, VkcBuf *ring, const VkcPle *p);
+
+/* counters, for the engines' [VK] lines */
+typedef struct {
+ unsigned long long frames, waits, ops, matmuls, gemms, barriers, bytes_up, bytes_down;
+ double wait_ms; /* host time blocked in fence waits */
+ size_t dev_bytes; /* live chain buffers */
+} VkcStats;
+void vkc_stats(VkcStats *st);
+/* COLI_VK_CHAIN_PROF=1: one stderr line of device time per kind of op */
+void vkc_prof_print(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif