Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan chain: int8 decode GEMVs with rows a workgroup by the matrix's length

huppifluppd27edd703aae
diff --git a/c/Makefile b/c/Makefile
index d883728c..4fa9765e 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -721,6 +721,8 @@ VK_SPV += shaders/chain_norm.spv shaders/chain_rope.spv shaders/chain_attn.spv
VK_SPV += shaders/chain_mla.spv shaders/chain_hgemv.spv shaders/chain_dsa.spv
# Kimi Delta Attention and manifold-constrained hyper-connections (vk_chain.h)
VK_SPV += shaders/chain_kda.spv shaders/chain_mhc.spv
+# the int8 decode GEMV with per-matrix rows per workgroup (optional at run time)
+VK_SPV += shaders/chain_gemv2.spv
# DeepSeek V4.1 Flash and DeepSeek V4 attention (vk_chain.h: vkc_dsv4_*)
VK_SPV += shaders/chain_dsv4.spv
# Kimi K3's attention residuals and SiTU-GLU (vk_chain.h)
diff --git a/c/shaders/chain_gemv2.comp b/c/shaders/chain_gemv2.comp
new file mode 100644
index 00000000..5441be4d
--- /dev/null
+++ b/c/shaders/chain_gemv2.comp
@@ -0,0 +1,93 @@
+#version 450
+// The dense chain's decode GEMV (vk_chain.c, S <= 4 rows): y[s,o] = sum_i x[s,i] *
+// dequant(W[o,i]) for int8 per row (fmt 1), int4 per row (2) and int4 per (row, gs) (4),
+// the bindings and push constants of chain_gemv.comp plus rpw. A workgroup takes rpw
+// output rows of one input row (WorkGroupID.x the block, .y the row); x is staged in
+// shared memory once, each output row's 16-byte weight words are spread over lpr lanes
+// (a power of two up to 64), RPS rows in flight per lane group, the partial sums joined
+// with clustered subgroup adds. The host picks rpw: fewer rows a workgroup for a short
+// matrix (more workgroups to fill the device), more for a long one. nr rows of x (up to
+// 4: a speculative verify's) share a workgroup and every weight load; each row keeps
+// its own sums in the same order, so its bits are a decode step's (nr = 1).
+// Host: I % 32 == 0, nr * I <= 16384, rowWords % 4 == 0; fmt 4: gs % 32 == 0.
+#extension GL_KHR_shader_subgroup_arithmetic : require
+#extension GL_KHR_shader_subgroup_basic : require
+#extension GL_KHR_shader_subgroup_clustered : require
+#extension GL_EXT_control_flow_attributes : enable
+
+layout(local_size_x = 256) in;
+layout(std430, binding = 0) readonly buffer X { vec4 x4[]; };
+layout(std430, binding = 1) readonly buffer W { uvec4 wv[]; };
+layout(std430, binding = 2) readonly buffer Sc { float sc[]; };
+layout(std430, binding = 3) writeonly buffer Y { float y[]; };
+layout(push_constant) uniform P { int fmt, S, I, O, rowWords, gs, rpw, nr; } p;
+#define NRMAX 4
+
+#define RPS 4u
+shared vec4 xsh[4096];
+
+float i8(uint w, uint k) { return float(int(w << (24u - 8u * k)) >> 24); }
+float d8(uint w, vec4 a) { return a.x * i8(w, 0u) + a.y * i8(w, 1u) + a.z * i8(w, 2u) + a.w * i8(w, 3u); }
+float d4(uint w, vec4 a, vec4 b) {
+ return a.x * float(int(w & 15u) - 8) + a.y * float(int((w >> 4u) & 15u) - 8) +
+ a.z * float(int((w >> 8u) & 15u) - 8) + a.w * float(int((w >> 12u) & 15u) - 8) +
+ b.x * float(int((w >> 16u) & 15u) - 8) + b.y * float(int((w >> 20u) & 15u) - 8) +
+ b.z * float(int((w >> 24u) & 15u) - 8) + b.w * float(int(w >> 28u) - 8);
+}
+float red(float v, uint lpr) {
+ if (lpr == 64u) return subgroupClusteredAdd(v, 64u);
+ if (lpr == 32u) return subgroupClusteredAdd(v, 32u);
+ if (lpr == 16u) return subgroupClusteredAdd(v, 16u);
+ if (lpr == 8u) return subgroupClusteredAdd(v, 8u);
+ if (lpr == 4u) return subgroupClusteredAdd(v, 4u);
+ if (lpr == 2u) return subgroupClusteredAdd(v, 2u);
+ return v;
+}
+
+void main() {
+ uint nr = uint(max(p.nr, 1)), s0 = gl_WorkGroupID.y * nr, I = uint(p.I), I4 = I / 4u, O = uint(p.O);
+ for (uint r = 0u; r < nr; r++)
+ for (uint i = gl_LocalInvocationID.x; i < I4; i += 256u) xsh[r * I4 + i] = s0 + r < uint(p.S) ? x4[(s0 + r) * I4 + i] : vec4(0.0);
+ barrier();
+ int fmt = p.fmt;
+ uint rowWords = uint(p.rowWords), gs = uint(max(p.gs, 1)), ng = (I + gs - 1u) / gs;
+ uint W4 = fmt == 1 ? I / 16u : I / 32u;
+ uint cap = min(min(gl_SubgroupSize, W4), 64u), lpr = 1u;
+ while (lpr * 2u <= cap) lpr *= 2u;
+ uint rg = gl_SubgroupSize / lpr, rr = gl_SubgroupInvocationID / lpr, jj = gl_SubgroupInvocationID % lpr;
+ uint RW = uint(p.rpw), o0 = gl_WorkGroupID.x * RW;
+ uint rows_sg = (RW + gl_NumSubgroups - 1u) / gl_NumSubgroups;
+ for (uint base = 0u; base < rows_sg; base += RPS * rg) {
+ uint qbase = gl_SubgroupID * rows_sg + base + rr;
+ uint remaining = rows_sg - min(rows_sg, base + rr);
+ float acc[RPS * NRMAX];
+ [[unroll]] for (uint t = 0u; t < RPS * NRMAX; t++) acc[t] = 0.0;
+ for (uint j = jj; j < W4; j += lpr) {
+ uvec4 v[RPS]; float sv[RPS];
+ [[unroll]] for (uint t = 0u; t < RPS; t++) {
+ uint q = qbase + t * rg, o = o0 + q;
+ bool ok = t * rg < remaining && q < RW && o < O;
+ v[t] = ok ? wv[o * (rowWords / 4u) + j] : uvec4(0u);
+ sv[t] = fmt == 4 && ok ? sc[o * ng + (j * 32u) / gs] : 1.0;
+ }
+ [[unroll]] for (uint r = 0u; r < NRMAX; r++) if (r < nr) {
+ uint xb = r * I4;
+ [[unroll]] for (uint t = 0u; t < RPS; t++) {
+ float d = 0.0;
+ if (fmt == 1) {
+ [[unroll]] for (uint h = 0u; h < 4u; h++) d += d8(v[t][h], xsh[xb + j * 4u + h]);
+ } else {
+ [[unroll]] for (uint h = 0u; h < 4u; h++) d += d4(v[t][h], xsh[xb + j * 8u + h * 2u], xsh[xb + j * 8u + h * 2u + 1u]);
+ }
+ acc[r * RPS + t] += sv[t] * d;
+ }
+ }
+ }
+ [[unroll]] for (uint r = 0u; r < NRMAX; r++) [[unroll]] for (uint t = 0u; t < RPS; t++) {
+ uint q = qbase + t * rg, o = o0 + q;
+ bool writer = r < nr && s0 + r < uint(p.S) && jj == 0u && t * rg < remaining && q < RW && o < O;
+ float v2 = red(acc[r * RPS + t], lpr);
+ if (writer) y[(s0 + r) * O + o] = fmt == 4 ? v2 : v2 * sc[o];
+ }
+ }
+}
diff --git a/c/shaders/chain_gemv2.spv b/c/shaders/chain_gemv2.spv
new file mode 100644
index 00000000..f644c848
Binary files /dev/null and b/c/shaders/chain_gemv2.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index 6b6e4f0b..dad7dde5 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -2336,6 +2336,37 @@ static void bench_gemv(void) {
}
}
+/* A speculative verify's rows through the decode GEMV: each of S rows (2..4, one
+ * workgroup) must carry the bits a one-row call gives that row. */
+static void test_gemv_rows(int fmt, int S, int I, int O) {
+ float *x = fvec((size_t)S * I, 1.f);
+ void *codes = NULL; float *sc = NULL; int gs = 0;
+ if (fmt == 1) {
+ int8_t *q = malloc((size_t)O * I); sc = malloc(O * sizeof *sc);
+ for (size_t i = 0; i < (size_t)O * I; i++) q[i] = (int8_t)((int)(rnd() % 255) - 127);
+ for (int o = 0; o < O; o++) sc[o] = 0.01f + (rnd() % 100) / 5000.f;
+ codes = q;
+ } else {
+ gs = 64; int ng = I / 64;
+ uint8_t *q = malloc((size_t)O * (I / 2)); sc = malloc((size_t)O * ng * sizeof *sc);
+ for (size_t i = 0; i < (size_t)O * (I / 2); i++) q[i] = (uint8_t)rnd();
+ for (int i = 0; i < O * ng; i++) sc[i] = 0.02f + (rnd() % 100) / 3000.f;
+ codes = q;
+ }
+ ColiVkTensor *t = NULL;
+ if (!coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs)) { CHECK(0, "gemv rows fmt %d: upload", fmt); return; }
+ VkcBuf *xb = vkc_buf((size_t)S * I * 4, VKC_DEV), *ya = vkc_buf((size_t)S * O * 4, VKC_DOWN), *y1 = vkc_buf((size_t)S * O * 4, VKC_DOWN);
+ vkc_gemm_rows(0); /* every row on the GEMV, as a verify's */
+ vkc_begin(); vkc_write(xb, 0, x, (size_t)S * I * 4);
+ int ok = vkc_matmul(t, xb, 0, ya, 0, S);
+ for (int s = 0; s < S; s++) ok = ok && vkc_matmul(t, xb, (size_t)s * I, y1, (size_t)s * O, 1);
+ vkc_submit(1);
+ vkc_gemm_rows(-1);
+ int same = ok && !memcmp(vkc_ptr(ya), vkc_ptr(y1), (size_t)S * O * 4);
+ CHECK(same, "gemv rows fmt %d S %d I %d O %d: a row's bits depend on the rows beside it", fmt, S, I, O);
+ vkc_free(xb); vkc_free(ya); vkc_free(y1); coli_vk_tensor_free(t);
+ free(x); free(codes); free(sc);
+}
int main(int argc, char **argv) {
const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }
@@ -2353,6 +2384,10 @@ int main(int argc, char **argv) {
printf("matmul done\n");
test_norm(VKC_NORM_ADD1, 0); test_norm(VKC_NORM_ADD1, 1); test_norm(0, 0); test_norm(VKC_NORM_NOW, 1); test_norm(VKC_NORM_L2 | VKC_NORM_NOW, 0);
printf("norm done\n");
+ for (int k = 0; k < 2; k++) { /* a verify's rows: the decode step's bits each */
+ int f = k ? 4 : 1;
+ test_gemv_rows(f, 2, 2048, 512); test_gemv_rows(f, 3, 4096, 256); test_gemv_rows(f, 4, 2048, 300);
+ }
test_rope();
test_chunk_rows();
test_attn(1, 0, 16, 0); test_attn(1, 140, 32, 0); test_attn(5, 200, 64, 0); test_attn(6, 9, 256, 1); test_attn(130, 3, 24, 0);
diff --git a/c/vk_chain.c b/c/vk_chain.c
index c45c0f00..7c912c29 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -93,6 +93,7 @@ static struct {
VkDescriptorSetLayout dsl;
VkPipelineLayout pl;
VkShaderModule mod[P_NPIPE], mod_gemm, mod_dnrec;
+ VkShaderModule mod_gv2; VkPipeline gv2; /* chain_gemv2.comp: int8 decode GEMVs (COLI_VK_CHAIN_GEMV2) */
VkPipeline pipe[P_NPIPE];
VkPipeline gemm[VKC_GEMM_MAX]; int gemm_bm[VKC_GEMM_MAX], gemm_bn[VKC_GEMM_MAX], ngemm;
VkPipeline dnrec[VKC_DNREC_MAX]; int dnrec_kd[VKC_DNREC_MAX], ndnrec;
@@ -303,6 +304,17 @@ int vkc_init(void) {
if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;
}
}
+ /* the int8/int4 decode GEMV with lanes per row and rows per workgroup chosen per
+ * matrix (chain_gemv2.comp): needs clustered subgroup operations */
+ {
+ const char *e = getenv("COLI_VK_CHAIN_GEMV2");
+ VkPhysicalDeviceSubgroupProperties sgp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
+ VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &sgp};
+ vkGetPhysicalDeviceProperties2((VkPhysicalDevice)K.core.phys, &p2);
+ int ok = (sgp.supportedOperations & VK_SUBGROUP_FEATURE_CLUSTERED_BIT) && (sgp.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) &&
+ sgp.subgroupSize >= 16 && !(e && *e == '0');
+ if (ok && (K.mod_gv2 = load_module(K.core.spv_path, "chain_gemv2.spv"))) K.gv2 = make_pipe(K.mod_gv2, NULL);
+ }
/* the MLA, KDA and mHC shaders, optional: without one only its ops decline */
for (int i = 0; i < PM_N; i++) {
char path[1200];
@@ -682,7 +694,17 @@ static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcB
int v4 = path < 0 && K.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&
(ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= K.gemv4_xs;
K.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;
- if (v4) {
+ int g2 = path < 0 && K.gv2 && S <= 4 && ti->fmt == 1 && /* int4 measured slower than chain_gemv here */
+ ti->I % 32 == 0 && ti->I <= 16384 && ti->rowWords % 4 == 0;
+ if (g2) {
+ /* rows a workgroup: short matrices in small blocks, so they still fill the device */
+ int rpw = ti->O <= 1024 ? 16 : ti->O <= 4096 ? 32 : 64;
+ /* a speculative verify's rows (S <= 4) share each weight load */
+ int nr = S <= 4 && S * ti->I <= 16384 ? S : 1;
+ struct { int fmt, S, I, O, rowWords, gs, rpw, nr; } pc8 = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs, rpw, nr};
+ ok = record(K.gv2, bd, 4, &pc8, sizeof pc8, (uint32_t)((ti->O + rpw - 1) / rpw), (uint32_t)((S + nr - 1) / nr), 1);
+ }
+ else if (v4) {
/* each workgroup stages x once: give it enough rows that the staging does not
* rival the weights, while keeping enough workgroups to fill the device */
/* lanes per row: about sixteen 16-byte steps each (COLI_VK_CHAIN_GEMV_LPR overrides) */
@@ -1486,6 +1508,8 @@ void vkc_shutdown(void) {
for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);
if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);
if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);
+ if (K.gv2) vkDestroyPipeline(K.dev, K.gv2, NULL);
+ if (K.mod_gv2) vkDestroyShaderModule(K.dev, K.mod_gv2, NULL);
if (K.mod_gemm) vkDestroyShaderModule(K.dev, K.mod_gemm, NULL);
if (K.mod_dnrec) vkDestroyShaderModule(K.dev, K.mod_dnrec, NULL);
if (K.pl) vkDestroyPipelineLayout(K.dev, K.pl, NULL);
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 6d2d8025..f314f260 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -267,6 +267,7 @@ With `COLI_VULKAN=1` every MoE engine (qwen36, qwen38, inkling, olmoe, kimi_k3,
| `COLI_VK_CHAIN` (laya, gliner_decide) | on for a GPU, discrete or integrated; off on a CPU device | The decision engines' encoder forward (and Laya's head layers) recorded as one submission on the device, the activations staying there (`decide_vk.h`, [vulkan.md](vulkan.md#the-decision-engines)). `0`: matrix by matrix through the device (`COLI_VK_DENSE` decides, as above), the rest on the CPU; `1`: on. The `[VK] <engine>: forward on the device ...` line says which and why. Clef's is qwen36's chain, as above. |
| `COLI_VK_IMPORT` | on for a model without routed experts on a device sharing the CPU's RAM; off otherwise | qwen36: the device reads the int8 and f16 dense rows where the host keeps them (`VK_EXT_external_memory_host`, `coli_vk_tensor_import`) instead of holding a copy of each. `1` requests import, `0` copies. On an integrated GPU, imported rows remain shared even with `COLI_VK_DENSE_HOST=0`: they already have one physical copy, and their pages stay alive while Vulkan reads them. `COLI_VK_IMPORT=0` forces an independent device copy whose host pages can be released; only those released pages count as RAM given back. On other devices, `COLI_VK_DENSE_HOST=0` disables import. Unset, on for the dense 27B family (Qwen3.8-27B, Clef) on an integrated GPU or Lavapipe, whose trunk is the whole model and would be held twice (Clef's f16 trunk, 54 GB, twice past a 61 GB box); a copy elsewhere. Where the device cannot import (no extension, staged uploads) the rows are copied. See [vulkan.md](vulkan.md#the-decision-engines). |
| `COLI_VK_CHAIN_GEMV` | on | `0`: the chain's decode matrices take `qmatmul.comp`'s GEMV instead of `chain_gemv.comp`'s. |
+| `COLI_VK_CHAIN_GEMV2` | on where supported | The chain's int8 decode matrices on `chain_gemv2.comp`: rows a workgroup from the matrix's length (16 up to 1024 outputs, 32 up to 4096, else 64), a row's 16-byte words over a power-of-two lane group with clustered adds, up to four rows of x (a verify's) sharing each weight load with a decode step's bits. Needs clustered subgroup operations. `0` keeps `chain_gemv.comp`. |
| `COLI_VK_CHAIN_SPIN_US` | `2000` | How long a wait on a chain frame polls before blocking. |
| `COLI_VK_CHAIN_PROF` | off | `1`: device time per kind of chain op, one line per run. |
| `COLI_VK_KV_SPLIT` | on | Past the device's budget a chain engine keeps part of each attention layer's KV cache on the device and attends over the rest on the CPU, the two parts merged (DeepSeek stages only selected missing rows to preserve sparse-attention arithmetic) (see [vulkan.md](vulkan.md#a-kv-cache-past-the-devices-budget)). `0`: never split: the whole mirrors as before, and past the budget the chain declines. |
diff --git a/docs/vulkan.md b/docs/vulkan.md
index 3e1245e7..3704f287 100644
--- a/docs/vulkan.md
+++ b/docs/vulkan.md
@@ -1027,6 +1027,7 @@ f32 throughout, as the CPU's f32 path.
| `COLI_VK_CHAIN_ROWS` | from the budget, up to 8192 | Prompt rows per chunk: a longer prompt runs every layer chunk by chunk (the device's scratch is sized for one chunk). Unset or `auto`: the most rows whose buffers fit half the free device memory, up to `COLI_VK_CHAIN_ROWS_MAX` (8192); see [big prompt chunks](#big-prompt-chunks-and-expert-streaming). It was 512 before. |
| `COLI_VK_CHAIN_LAYERS` | what fits | The engines on [the partial chain](#a-partial-chain): the first N layers on the device, the CPU the rest and the head. Unset or `auto`: the most layers the device's free memory holds (every layer when they fit, the chain as before); `n`: n layers; `0`: the chain off. |
| `COLI_VK_CHAIN_GEMV` | on | `0`: the decode matrices take `qmatmul.comp`'s GEMV instead of `chain_gemv.comp`'s. |
+| `COLI_VK_CHAIN_GEMV2` | on where supported | `0`: the int8 decode matrices keep `chain_gemv.comp` instead of `chain_gemv2.comp` (rows a workgroup by the matrix's length). On a Radeon 8060S (Qwen3.6, a 256-token answer, int8 trunk) decode went from 36.8 to 39.0 tok/s; at int4 (`COLI_DENSE_BITS=4`) it measured slower, so it takes int8 only. |
| `COLI_VK_CHAIN_SPIN_US` | `2000` | How long a wait on a chain frame polls the fence before blocking. |
| `COLI_VK_CHAIN_PROF` | off | `1`: one `[VK] chain profile` line of device time per kind of op (timestamps). |
| `COLI_VK_KV_SPLIT` | on | `0`: never split the KV cache past the device's budget ([below](#a-kv-cache-past-the-devices-budget)); the whole mirrors, and past the budget the chain declines. |