Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan chain: int8/int4 prompt GEMMs on the matrix units

huppifluppd86804b1c92d
diff --git a/c/Makefile b/c/Makefile
index d883728c..4c3c3d5a 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -730,6 +730,8 @@ VK_SPV += shaders/chain_ares.spv
VK_SPV += shaders/chain_sconv.spv shaders/chain_relattn.spv
# blocked attention for prompt chunks (made on first use; vk_chain.c, chain_attnb.comp)
VK_SPV += shaders/chain_attnb.spv
+# the int8/int4 prompt GEMM on the matrix units (optional at run time)
+VK_SPV += shaders/chain_gemm.spv
# the KV cache split between the device and the host past the device's budget
# (vk_chain.h: vkc_kvs_*; vk_kvsplit.h)
VK_SPV += shaders/chain_kvs.spv
@@ -883,6 +885,12 @@ shaders/qmatmul_coop.spv: shaders/qmatmul_coop.comp
@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
$(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \
echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }
+# chain_gemm.comp needs the same glslc as qmatmul_coop.comp above, with the same
+# fallback; without its .spv the chain keeps its other GEMMs.
+shaders/chain_gemm.spv: shaders/chain_gemm.comp
+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \
+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }
# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.
backend_loader.o: backend_loader.c .build-config
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index dcd52a5d..b85ebb20 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -4463,6 +4463,7 @@ int coli_vk_core(ColiVkCore *o) {
}
o->gemm_min_s = G.gemm_min_s; o->gemm_min_so = G.gemm_min_so;
o->integrated = coli_vk_device_integrated(); o->shares_ram = coli_vk_device_shares_ram();
+ o->coop_sg = G.has_coop ? G.coop_sg : 0;
return 1;
}
int coli_vk_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *o) {
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index 882fb7d1..9fa97d2f 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -395,6 +395,7 @@ typedef struct {
int gemm_tiles, gemm_tile[4][6]; /* bm, bn, bk, tm, tn, pf */
int gemm_min_s, gemm_min_so;
int has_prio, integrated, shares_ram;
+ int coop_sg; /* subgroup size the cooperative-matrix shaders run at; 0 = none */
} ColiVkCore;
int coli_vk_core(ColiVkCore *out);
/* A resident tensor's buffers (VkBuffer as void *) and layout; 0 for a COLI_VK_DEV2 one. */
diff --git a/c/shaders/chain_gemm.comp b/c/shaders/chain_gemm.comp
new file mode 100644
index 00000000..9bcc5037
--- /dev/null
+++ b/c/shaders/chain_gemm.comp
@@ -0,0 +1,162 @@
+#version 450
+// The dense chain's prompt GEMM on the matrix units (vk_chain.c): y[s,o] = sum_i x[s,i] *
+// dequant(W[o,i]) for int8 per row (fmt 1), int4 per row (2) and int4 per (row, gs)
+// (4), the bindings and push constants of qmatmul_gemm.comp. 256 invocations = 4
+// subgroups of 64 (required), a block of 128 outputs x 16*TT rows; per 64-input step two
+// lanes dequantise one weight row to f16 in shared memory and x is rounded to f16 beside
+// it, the next step's words fetched into registers during the multiplies. Shared memory
+// holds each 16x16 operand tile contiguously (stride 16): strided cooperative-matrix
+// loads measured about 2x slower on RDNA3.5. A partial last row tile goes through a
+// per-subgroup patch, so rows past S are never written.
+// Unlike qmatmul_coop.comp (x split into hi and lo, two products), x is rounded to f16
+// once: half the multiplies, f16 inputs (as a GEMV over f16 activations would).
+// Grid: x = blocks of 16*TT rows, y = blocks of 128 outputs.
+// Host gates: I % 64 == 0, O % 128 == 0, rowWords % 4 == 0, fmt 4: gs % 8 == 0.
+#extension GL_KHR_cooperative_matrix : require
+#extension GL_KHR_memory_scope_semantics : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
+#extension GL_EXT_control_flow_attributes : enable
+#extension GL_KHR_shader_subgroup_basic : require
+layout(local_size_x = 256) in;
+layout(constant_id = 0) const uint TT = 4; // 16-row token tiles per workgroup
+
+layout(std430, binding = 0) readonly buffer X { float x[]; };
+layout(std430, binding = 1) readonly buffer W { uint wg[]; };
+layout(std430, binding = 2) readonly buffer Sc { float gscale[]; };
+layout(std430, binding = 3) writeonly buffer Y { float y[]; };
+layout(push_constant) uniform P { int fmt, S, I, O, rowWords, gs; } p;
+
+#define BO 128u // output rows per workgroup (8 tiles)
+#define TM (16u * TT) // token rows per workgroup
+shared uvec4 sx[TM * 8u]; // x slice, A tiles row-major, 8 f16 per uvec4
+shared uvec4 sg[BO * 8u]; // weights, B tiles COLUMN-major (= W[n][k], k contiguous)
+shared float spatch[4u * 256u]; // per-subgroup 16x16 patch for partial token tiles (4 x wave64)
+
+// Shared memory holds every 16x16 operand tile CONTIGUOUSLY (256 values, loaded
+// with stride 16): strided cooperative-matrix loads out of a wide row-major
+// array are what made the first version slow (RADV, cf. Kernel-Lab E020/E030).
+// x: A tile (token tile t, k tile kt), element (row r, col k) -> r*16 + k
+// W: B tile (out tile q, k tile kt), B(k, n) = W[n][k], row-major -> k*16 + n
+uint wix(uint n, uint k) { return ((n >> 4u) * 4u + (k >> 4u)) * 256u + (n & 15u) * 16u + (k & 15u); }
+uint xix(uint r, uint k) { return ((r >> 4u) * 4u + (k >> 4u)) * 256u + (r & 15u) * 16u + (k & 15u); }
+
+// Register double buffering: the raw words, scales and x values of the NEXT 64-input
+// step are fetched into registers while the matrix units work on the current step
+// out of shared memory; they are dequantised into shared memory after a barrier.
+#define XPL (TM * 64u / 256u) // x values per lane per step
+uint rwg[8];
+float rsg[4];
+float rx[XPL];
+void fetch_raw(uint s0, uint o0, uint k0, uint tid) {
+ uint K = uint(p.I), S = uint(p.S);
+ {
+ uint r = tid / (64u / XPL), kk = (tid % (64u / XPL)) * XPL;
+ bool ok = s0 + r < S;
+ [[unroll]] for (uint i = 0u; i < XPL; i++) rx[i] = ok ? x[(s0 + r) * K + k0 + kk + i] : 0.0;
+ }
+ uint n = tid >> 1u, hf = tid & 1u, o = o0 + n, kb = k0 + 32u * hf;
+ int ng = (p.I + p.gs - 1) / max(p.gs, 1);
+ if (p.fmt == 1) {
+ uint rb = o * uint(p.rowWords) + kb / 4u;
+ [[unroll]] for (uint j = 0u; j < 8u; j++) rwg[j] = wg[rb + j];
+ rsg[0] = gscale[o];
+ } else {
+ uint rb = o * uint(p.rowWords) + kb / 8u;
+ [[unroll]] for (uint j = 0u; j < 4u; j++) {
+ uint si = p.fmt == 4 ? o * uint(ng) + (kb + 8u * j) / uint(p.gs) : o;
+ rwg[j] = wg[rb + j]; rsg[j] = gscale[si];
+ }
+ }
+}
+uint pk(float a, float b) { return packFloat2x16(f16vec2(float16_t(a), float16_t(b))); }
+uint i8(uint w, uint k) { return uint(int(w << (24u - 8u * k)) >> 24); }
+void store_raw(uint tid) {
+ {
+ uint r = tid / (64u / XPL), kk = (tid % (64u / XPL)) * XPL;
+ [[unroll]] for (uint c = 0u; c < XPL / 8u; c++) {
+ uint k = kk + 8u * c;
+ uint base = ((r >> 4u) * 4u + (k >> 4u)) * 32u + (r & 15u) * 2u + ((k & 15u) >> 3u);
+ sx[base] = uvec4(pk(rx[8u*c], rx[8u*c+1u]), pk(rx[8u*c+2u], rx[8u*c+3u]), pk(rx[8u*c+4u], rx[8u*c+5u]), pk(rx[8u*c+6u], rx[8u*c+7u]));
+ }
+ }
+ uint n = tid >> 1u, hf = tid & 1u;
+ // lane's 32 values = k tiles 2hf, 2hf+1; 2 uvec4 per tile row
+ uint wb = ((n >> 4u) * 4u + 2u * hf) * 32u + (n & 15u) * 2u;
+ if (p.fmt == 1) {
+ [[unroll]] for (uint j = 0u; j < 8u; j += 2u) { // 2 words = 8 values = one uvec4
+ float a = rsg[0];
+ uvec4 v = uvec4(pk(float(int(i8(rwg[j],0u)))*a, float(int(i8(rwg[j],1u)))*a), pk(float(int(i8(rwg[j],2u)))*a, float(int(i8(rwg[j],3u)))*a),
+ pk(float(int(i8(rwg[j+1u],0u)))*a, float(int(i8(rwg[j+1u],1u)))*a), pk(float(int(i8(rwg[j+1u],2u)))*a, float(int(i8(rwg[j+1u],3u)))*a));
+ uint e = wb + (j >> 2u) * 32u + ((j >> 1u) & 1u);
+ sg[e] = v;
+ }
+ } else {
+ [[unroll]] for (uint j = 0u; j < 4u; j++) { // 1 word = 8 nibbles = one uvec4
+ float a = rsg[j]; uint w = rwg[j];
+ uvec4 v; [[unroll]] for (uint c = 0u; c < 4u; c++) v[c] = pk(float(int((w >> (8u*c)) & 0xfu) - 8) * a, float(int((w >> (8u*c+4u)) & 0xfu) - 8) * a);
+ uint e = wb + (j >> 1u) * 32u + (j & 1u);
+ sg[e] = v;
+ }
+ }
+}
+
+// Each subgroup owns output tiles q = sgi, sgi + nsg, ... (8 per workgroup) for
+// all TT token tiles: MAXQ tiles per subgroup at most (8 with one wave64 group of 4).
+#define MAXQ 2u
+
+void main() {
+ uint s0 = gl_WorkGroupID.x * TM;
+ uint o0 = gl_WorkGroupID.y * BO;
+ uint tid = gl_LocalInvocationID.x;
+ uint nsg = gl_NumSubgroups, sgi = gl_SubgroupID;
+ uint K = uint(p.I), S = uint(p.S);
+ uint nq = (8u + nsg - 1u) / nsg; // output tiles per subgroup: 2 (wave64) or 1 (wave32)
+
+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> G[MAXQ * TT];
+ [[unroll]] for (uint t = 0u; t < MAXQ * TT; t++) {
+ G[t] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);
+ }
+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> A[TT];
+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> B;
+
+ fetch_raw(s0, o0, 0u, tid);
+ store_raw(tid);
+ barrier();
+ for (uint k0 = 0u; k0 < K; k0 += 64u) {
+ bool nxt = k0 + 64u < K; // workgroup-uniform
+ if (nxt) fetch_raw(s0, o0, k0 + 64u, tid); // in flight during the multiply below
+ [[unroll]] for (uint kt = 0u; kt < 4u; kt++) {
+ [[unroll]] for (uint t = 0u; t < TT; t++)
+ coopMatLoad(A[t], sx, (t * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutRowMajor);
+ [[unroll]] for (uint qi = 0u; qi < MAXQ; qi++) {
+ uint q = sgi + qi * 4u; // wave64 assumed (requiredSubgroupSize = 64)
+ coopMatLoad(B, sg, (q * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutColumnMajor);
+ [[unroll]] for (uint t = 0u; t < TT; t++) G[qi * TT + t] = coopMatMulAdd(A[t], B, G[qi * TT + t]);
+ }
+ }
+ barrier();
+ if (nxt) { store_raw(tid); barrier(); }
+ }
+
+ uint lane = gl_SubgroupInvocationID, sgsz = gl_SubgroupSize;
+ [[unroll]] for (uint qi = 0u; qi < MAXQ; qi++) {
+ if (qi >= nq) break;
+ uint q = sgi + qi * nsg;
+ if (q >= 8u) break;
+ [[unroll]] for (uint t = 0u; t < TT; t++) {
+ uint r0 = s0 + t * 16u;
+ if (r0 >= S) continue; // subgroup-uniform
+ if (r0 + 16u <= S) {
+ coopMatStore(G[qi * TT + t], y, r0 * uint(p.O) + o0 + q * 16u, uint(p.O), gl_CooperativeMatrixLayoutRowMajor);
+ } else {
+ coopMatStore(G[qi * TT + t], spatch, sgi * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);
+ subgroupBarrier();
+ for (uint e = lane; e < 256u; e += sgsz) {
+ uint r = e / 16u, n = e % 16u;
+ if (r0 + r < S) y[(r0 + r) * uint(p.O) + o0 + q * 16u + n] = spatch[sgi * 256u + e];
+ }
+ subgroupBarrier();
+ }
+ }
+ }
+}
diff --git a/c/shaders/chain_gemm.spv b/c/shaders/chain_gemm.spv
new file mode 100644
index 00000000..8a18e113
Binary files /dev/null and b/c/shaders/chain_gemm.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index 6b6e4f0b..80dfdbdc 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -102,6 +102,7 @@ static float *down(VkcBuf *b, size_t off, size_t n) {
/* ---- matmul ----------------------------------------------------------------------- */
static uint16_t f2bf(float f) { uint32_t u; memcpy(&u, &f, 4); return (uint16_t)((u + 0x7fff + ((u >> 16) & 1)) >> 16); }
static float bf2f(uint16_t h) { uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }
+static double g_mm_tol = 2e-4; /* chain_gemm.comp's cases: x rounded to f16 */
static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {
float *x = fvec((size_t)S * I, 1.f), *W = malloc((size_t)O * I * sizeof(float));
void *codes = NULL; float *sc = NULL; int gs = 0;
@@ -140,7 +141,7 @@ static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {
vkc_submit(1);
float *y = (float *)vkc_ptr(yb) + yo;
double e = relerr(y, ref, (size_t)S * O, 1e-3);
- CHECK(ok && e < 2e-4 && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);
+ CHECK(ok && e < g_mm_tol && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);
vkc_free(xb); vkc_free(yb); coli_vk_tensor_free(t);
free(x); free(W); free(codes); free(sc); free(ref);
}
@@ -2350,6 +2351,21 @@ int main(int argc, char **argv) {
test_matmul(fmts[k], 1, 100, 33, 0, 0); /* rows that are not whole 16-byte steps */
test_matmul(fmts[k], 2, 3072, 40, 0, 0); /* a long row (chain_gemv's staging) */
}
+ /* int8 and int4 prompt GEMMs whose shapes chain_gemm.comp takes (I % 64, O % 128), on
+ * a device with cooperative matrices at subgroup size 64: full and partial row tiles,
+ * bindable offsets; its x is rounded to f16, hence the bound */
+ {
+ VkcStats s0; vkc_stats(&s0);
+ g_mm_tol = 2e-3;
+ for (int k = 0; k < 2; k++) {
+ test_matmul(fmts[k], 64, 256, 256, 0, 0);
+ test_matmul(fmts[k], 70, 512, 384, 64, 128);
+ test_matmul(fmts[k], 130, 192, 128, 0, 0);
+ }
+ g_mm_tol = 2e-4;
+ VkcStats s1; vkc_stats(&s1);
+ printf(" prompt GEMMs on chain_gemm: %llu\n", s1.tile_gemms - s0.tile_gemms);
+ }
printf("matmul done\n");
test_norm(VKC_NORM_ADD1, 0); test_norm(VKC_NORM_ADD1, 1); test_norm(0, 0); test_norm(VKC_NORM_NOW, 1); test_norm(VKC_NORM_L2 | VKC_NORM_NOW, 0);
printf("norm done\n");
diff --git a/c/vk_chain.c b/c/vk_chain.c
index c45c0f00..ac728c0b 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -97,6 +97,8 @@ static struct {
VkPipeline gemm[VKC_GEMM_MAX]; int gemm_bm[VKC_GEMM_MAX], gemm_bn[VKC_GEMM_MAX], ngemm;
VkPipeline dnrec[VKC_DNREC_MAX]; int dnrec_kd[VKC_DNREC_MAX], ndnrec;
VkShaderModule mod_gemv4; VkPipeline gemv4; int gemv4_xs; /* chain_gemv.comp: the vectorized decode GEMV */
+ /* chain_gemm.comp: int8/int4 prompt GEMMs, x rounded to f16 once (COLI_VK_CHAIN_GEMM) */
+ VkShaderModule mod_tg; VkPipeline tg;
VkShaderModule mmod[PM_N]; VkPipeline mpipe[PM_N]; int mla_ok; /* chain_mla, chain_hgemv, chain_dsa */
VkPipeline kdarec[VKC_KDA_MAX]; int kdarec_kd[VKC_KDA_MAX], nkdarec; /* chain_kda's recurrence per key dim */
VkCommandPool cpool;
@@ -318,6 +320,25 @@ int vkc_init(void) {
}
K.mla_ok = K.mpipe[PM_MLA] && K.mpipe[PM_HGEMV] && K.mpipe[PM_DSA];
dsv4_init();
+ /* the int8/int4 prompt GEMM on the matrix units (chain_gemm.comp, subgroups of 64);
+ * COLI_VK_CHAIN_GEMM=0 leaves those formats to the GEMMs below */
+ {
+ const char *e = getenv("COLI_VK_CHAIN_GEMM");
+ if (K.core.coop_sg == 64 && !(e && *e == '0') && (K.mod_tg = load_module(K.core.spv_path, "chain_gemm.spv"))) {
+ int32_t tt = 4;
+ VkSpecializationMapEntry me = {0, 0, 4};
+ VkSpecializationInfo si = {1, &me, 4, &tt};
+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,
+ .requiredSubgroupSize = 64};
+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,
+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,
+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = K.mod_tg, .pName = "main",
+ .pSpecializationInfo = &si}, .layout = K.pl};
+ if (vkCreateComputePipelines(K.dev, VK_NULL_HANDLE, 1, &ci, NULL, &K.tg) != VK_SUCCESS) K.tg = VK_NULL_HANDLE;
+ }
+ }
/* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */
K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;
for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {
@@ -695,6 +716,11 @@ static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcB
int wg = (ti->O + rows_wg - 1) / rows_wg; if (wg > 1024) wg = 1024;
ok = record(K.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);
}
+ else if (path >= 0 && K.tg && (ti->fmt == 1 || ti->fmt == 2 || (ti->fmt == 4 && ti->gs >= 8 && ti->gs % 8 == 0)) &&
+ ti->I % 64 == 0 && ti->O % 128 == 0 && ti->rowWords % 4 == 0) {
+ ok = record(K.tg, bd, 4, &pc, sizeof pc, (uint32_t)((S + 63) / 64), (uint32_t)(ti->O / 128), 1);
+ K.st.tile_gemms += ok;
+ }
else if (path >= 0)
ok = record(K.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + K.gemm_bm[path] - 1) / K.gemm_bm[path]),
(uint32_t)((S + K.gemm_bn[path] - 1) / K.gemm_bn[path]), 1);
@@ -1483,6 +1509,8 @@ void vkc_shutdown(void) {
if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);
}
for (int i = 0; i < K.ngemm; i++) vkDestroyPipeline(K.dev, K.gemm[i], NULL);
+ if (K.tg) vkDestroyPipeline(K.dev, K.tg, NULL);
+ if (K.mod_tg) vkDestroyShaderModule(K.dev, K.mod_tg, NULL);
for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);
if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);
if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);
diff --git a/c/vk_chain.h b/c/vk_chain.h
index f67041ae..1eb3b7b7 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -646,6 +646,7 @@ typedef struct {
size_t dev_bytes; /* live chain buffers */
unsigned long long attn_blocked; /* attention calls through chain_attnb (blocks of rows) */
unsigned long long attn_slices; /* extra submissions that cut a long attention (COLI_VK_ATTN_SLICE) */
+ unsigned long long tile_gemms; /* prompt matmuls through chain_gemm.comp (matrix units) */
} VkcStats;
void vkc_stats(VkcStats *st);
/* COLI_VK_CHAIN_PROF=1: one stderr line of device time per kind of op */
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 6d2d8025..b363b46b 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -226,6 +226,7 @@ Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRE
| `COLI_VK_GEMM_MIN_S` | measured | When a resident matmul (`coli_vk_matmul`, every engine) takes the tiled GEMM instead of the per-row GEMV. Unset: S ≥ 2 and S·O ≥ 4096, measured on a Radeon 780M (a narrow matrix at small S stays on the GEMV). `N`: every call with S ≥ N. `0`: the GEMV for every S. S = 1 (decode) always stays on the GEMV. See [vulkan.md](vulkan.md#prefill-the-tiled-gemms). |
| `COLI_VK_COOP` | on where supported | The cooperative-matrix GEMM (`VK_KHR_cooperative_matrix`, 16x16x16 fp16 → fp32) for the formats whose weights decode exactly to fp16: int8, int4, int3-g64, MXFP4 and fp8, the grouped ones with a group size that is a multiple of 32. `0` keeps every format on the fp32 GEMM. |
| `COLI_VK_COOP_SG` | `64` where allowed | Subgroup size the cooperative-matrix pipeline requires (RDNA3: wave64 measured fastest). Tuning only. |
+| `COLI_VK_CHAIN_GEMM` | on where supported | The dense chain's int8 and int4 prompt matmuls (`I` a multiple of 64, `O` of 128) on the matrix units (`chain_gemm.comp`: x rounded to f16 once) on a device with cooperative matrices at subgroup size 64. `0` keeps them on the chain's other GEMMs. |
| `COLI_VK_GEMM_TILE` | measured | `bm,bn,bk,tm,tn[,pf]`: one tile for every width of the fp32 GEMM instead of the measured pair. Tuning only (`COLI_VK_TEST_GEMM_BENCH`). |
| `COLI_VK_COOP_TILE` | measured | `bm,bn,wm,wn,bk`: one tile for every width of the cooperative-matrix GEMM. Tuning only. |
| `COLI_VK_STAGED` | auto | Staged uploads: resident data (the weights, the expert tier, the MLA KV mirror, `COLI_VK_DEV2`'s experts) copied into device-local memory the host does not map, through a host staging buffer, instead of written through a mapping of the host-visible device-local memory. `1` on, `0` off. Unset: on when the host-visible device-local heap is under a quarter of the largest device-local heap, as on a discrete card without Resizable BAR (about 256 MB of 8 GB, where the mapped path fails or spills to system RAM); off on a card with Resizable BAR, an integrated GPU or Lavapipe. See [vulkan.md](vulkan.md#memory-placement-without-resizable-bar). |
diff --git a/docs/vulkan.md b/docs/vulkan.md
index 3e1245e7..6dd89005 100644
--- a/docs/vulkan.md
+++ b/docs/vulkan.md
@@ -2049,6 +2049,7 @@ The run's line counts what streamed:
| `COLI_VK_CHAIN_ROWS` | from the budget, up to 8192 | Prompt rows per chain chunk; `auto` as unset. Set, it also keeps the engines' own prompt blocks (above). |
| `COLI_VK_CHAIN_ROWS_MAX` | `8192` | The most rows the budget's chunk takes. |
| `COLI_VK_ATTN_BLOCK` | `16` | Rows from which the chain's attention takes `chain_attnb.comp`; `0` never. |
+| `COLI_VK_CHAIN_GEMM` | on where supported | The chain's int8/int4 prompt matmuls (`I % 64`, `O % 128`) on `chain_gemm.comp` (cooperative matrices at subgroup size 64, x rounded to f16 once, 128 outputs x 64 rows a workgroup); `0` keeps the fp32 GEMM. On a Radeon 8060S (Qwen3.6, 1011-token prompt): 517-546 -> 282-287 ms of tiled GEMM, the first token 2.08 -> 1.85 s, perplexity unchanged (7.65). |
| `COLI_VK_ATTN_SLICE` | `4294967296` | Rows x positions x heads x head dim past which an attention is cut over several submissions; `0` never. |
| `COLI_VK_TIER_STREAM` | on | `0`: no streaming; the tier takes a prompt step in the engine's usual blocks and leaves the cold experts to the CPU, as before. |
| `COLI_VK_TIER_STREAM_SLOTS` | `64` | Staging slots (experts) for streaming, out of the tier's budget. |