Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan chain: int8/int4 prompt GEMMs on the matrix units
huppifluppd86804b1c92d
diff --git a/c/Makefile b/c/Makefileindex d883728c..4c3c3d5a 100644--- a/c/Makefile+++ b/c/Makefile@@ -730,6 +730,8 @@ VK_SPV += shaders/chain_ares.spvVK_SPV += shaders/chain_sconv.spv shaders/chain_relattn.spv# blocked attention for prompt chunks (made on first use; vk_chain.c, chain_attnb.comp)VK_SPV += shaders/chain_attnb.spv+# the int8/int4 prompt GEMM on the matrix units (optional at run time)+VK_SPV += shaders/chain_gemm.spv# the KV cache split between the device and the host past the device's budget# (vk_chain.h: vkc_kvs_*; vk_kvsplit.h)VK_SPV += shaders/chain_kvs.spv@@ -883,6 +885,12 @@ shaders/qmatmul_coop.spv: shaders/qmatmul_coop.comp@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }$(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }+# chain_gemm.comp needs the same glslc as qmatmul_coop.comp above, with the same+# fallback; without its .spv the chain keeps its other GEMMs.+shaders/chain_gemm.spv: shaders/chain_gemm.comp+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.backend_loader.o: backend_loader.c .build-configdiff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex dcd52a5d..b85ebb20 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -4463,6 +4463,7 @@ int coli_vk_core(ColiVkCore *o) {}o->gemm_min_s = G.gemm_min_s; o->gemm_min_so = G.gemm_min_so;o->integrated = coli_vk_device_integrated(); o->shares_ram = coli_vk_device_shares_ram();+ o->coop_sg = G.has_coop ? G.coop_sg : 0;return 1;}int coli_vk_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *o) {diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 882fb7d1..9fa97d2f 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -395,6 +395,7 @@ typedef struct {int gemm_tiles, gemm_tile[4][6]; /* bm, bn, bk, tm, tn, pf */int gemm_min_s, gemm_min_so;int has_prio, integrated, shares_ram;+ int coop_sg; /* subgroup size the cooperative-matrix shaders run at; 0 = none */} ColiVkCore;int coli_vk_core(ColiVkCore *out);/* A resident tensor's buffers (VkBuffer as void *) and layout; 0 for a COLI_VK_DEV2 one. */diff --git a/c/shaders/chain_gemm.comp b/c/shaders/chain_gemm.compnew file mode 100644index 00000000..9bcc5037--- /dev/null+++ b/c/shaders/chain_gemm.comp@@ -0,0 +1,162 @@+#version 450+// The dense chain's prompt GEMM on the matrix units (vk_chain.c): y[s,o] = sum_i x[s,i] *+// dequant(W[o,i]) for int8 per row (fmt 1), int4 per row (2) and int4 per (row, gs)+// (4), the bindings and push constants of qmatmul_gemm.comp. 256 invocations = 4+// subgroups of 64 (required), a block of 128 outputs x 16*TT rows; per 64-input step two+// lanes dequantise one weight row to f16 in shared memory and x is rounded to f16 beside+// it, the next step's words fetched into registers during the multiplies. Shared memory+// holds each 16x16 operand tile contiguously (stride 16): strided cooperative-matrix+// loads measured about 2x slower on RDNA3.5. A partial last row tile goes through a+// per-subgroup patch, so rows past S are never written.+// Unlike qmatmul_coop.comp (x split into hi and lo, two products), x is rounded to f16+// once: half the multiplies, f16 inputs (as a GEMV over f16 activations would).+// Grid: x = blocks of 16*TT rows, y = blocks of 128 outputs.+// Host gates: I % 64 == 0, O % 128 == 0, rowWords % 4 == 0, fmt 4: gs % 8 == 0.+#extension GL_KHR_cooperative_matrix : require+#extension GL_KHR_memory_scope_semantics : require+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require+#extension GL_EXT_control_flow_attributes : enable+#extension GL_KHR_shader_subgroup_basic : require+layout(local_size_x = 256) in;+layout(constant_id = 0) const uint TT = 4; // 16-row token tiles per workgroup++layout(std430, binding = 0) readonly buffer X { float x[]; };+layout(std430, binding = 1) readonly buffer W { uint wg[]; };+layout(std430, binding = 2) readonly buffer Sc { float gscale[]; };+layout(std430, binding = 3) writeonly buffer Y { float y[]; };+layout(push_constant) uniform P { int fmt, S, I, O, rowWords, gs; } p;++#define BO 128u // output rows per workgroup (8 tiles)+#define TM (16u * TT) // token rows per workgroup+shared uvec4 sx[TM * 8u]; // x slice, A tiles row-major, 8 f16 per uvec4+shared uvec4 sg[BO * 8u]; // weights, B tiles COLUMN-major (= W[n][k], k contiguous)+shared float spatch[4u * 256u]; // per-subgroup 16x16 patch for partial token tiles (4 x wave64)++// Shared memory holds every 16x16 operand tile CONTIGUOUSLY (256 values, loaded+// with stride 16): strided cooperative-matrix loads out of a wide row-major+// array are what made the first version slow (RADV, cf. Kernel-Lab E020/E030).+// x: A tile (token tile t, k tile kt), element (row r, col k) -> r*16 + k+// W: B tile (out tile q, k tile kt), B(k, n) = W[n][k], row-major -> k*16 + n+uint wix(uint n, uint k) { return ((n >> 4u) * 4u + (k >> 4u)) * 256u + (n & 15u) * 16u + (k & 15u); }+uint xix(uint r, uint k) { return ((r >> 4u) * 4u + (k >> 4u)) * 256u + (r & 15u) * 16u + (k & 15u); }++// Register double buffering: the raw words, scales and x values of the NEXT 64-input+// step are fetched into registers while the matrix units work on the current step+// out of shared memory; they are dequantised into shared memory after a barrier.+#define XPL (TM * 64u / 256u) // x values per lane per step+uint rwg[8];+float rsg[4];+float rx[XPL];+void fetch_raw(uint s0, uint o0, uint k0, uint tid) {+ uint K = uint(p.I), S = uint(p.S);+ {+ uint r = tid / (64u / XPL), kk = (tid % (64u / XPL)) * XPL;+ bool ok = s0 + r < S;+ [[unroll]] for (uint i = 0u; i < XPL; i++) rx[i] = ok ? x[(s0 + r) * K + k0 + kk + i] : 0.0;+ }+ uint n = tid >> 1u, hf = tid & 1u, o = o0 + n, kb = k0 + 32u * hf;+ int ng = (p.I + p.gs - 1) / max(p.gs, 1);+ if (p.fmt == 1) {+ uint rb = o * uint(p.rowWords) + kb / 4u;+ [[unroll]] for (uint j = 0u; j < 8u; j++) rwg[j] = wg[rb + j];+ rsg[0] = gscale[o];+ } else {+ uint rb = o * uint(p.rowWords) + kb / 8u;+ [[unroll]] for (uint j = 0u; j < 4u; j++) {+ uint si = p.fmt == 4 ? o * uint(ng) + (kb + 8u * j) / uint(p.gs) : o;+ rwg[j] = wg[rb + j]; rsg[j] = gscale[si];+ }+ }+}+uint pk(float a, float b) { return packFloat2x16(f16vec2(float16_t(a), float16_t(b))); }+uint i8(uint w, uint k) { return uint(int(w << (24u - 8u * k)) >> 24); }+void store_raw(uint tid) {+ {+ uint r = tid / (64u / XPL), kk = (tid % (64u / XPL)) * XPL;+ [[unroll]] for (uint c = 0u; c < XPL / 8u; c++) {+ uint k = kk + 8u * c;+ uint base = ((r >> 4u) * 4u + (k >> 4u)) * 32u + (r & 15u) * 2u + ((k & 15u) >> 3u);+ sx[base] = uvec4(pk(rx[8u*c], rx[8u*c+1u]), pk(rx[8u*c+2u], rx[8u*c+3u]), pk(rx[8u*c+4u], rx[8u*c+5u]), pk(rx[8u*c+6u], rx[8u*c+7u]));+ }+ }+ uint n = tid >> 1u, hf = tid & 1u;+ // lane's 32 values = k tiles 2hf, 2hf+1; 2 uvec4 per tile row+ uint wb = ((n >> 4u) * 4u + 2u * hf) * 32u + (n & 15u) * 2u;+ if (p.fmt == 1) {+ [[unroll]] for (uint j = 0u; j < 8u; j += 2u) { // 2 words = 8 values = one uvec4+ float a = rsg[0];+ uvec4 v = uvec4(pk(float(int(i8(rwg[j],0u)))*a, float(int(i8(rwg[j],1u)))*a), pk(float(int(i8(rwg[j],2u)))*a, float(int(i8(rwg[j],3u)))*a),+ pk(float(int(i8(rwg[j+1u],0u)))*a, float(int(i8(rwg[j+1u],1u)))*a), pk(float(int(i8(rwg[j+1u],2u)))*a, float(int(i8(rwg[j+1u],3u)))*a));+ uint e = wb + (j >> 2u) * 32u + ((j >> 1u) & 1u);+ sg[e] = v;+ }+ } else {+ [[unroll]] for (uint j = 0u; j < 4u; j++) { // 1 word = 8 nibbles = one uvec4+ float a = rsg[j]; uint w = rwg[j];+ uvec4 v; [[unroll]] for (uint c = 0u; c < 4u; c++) v[c] = pk(float(int((w >> (8u*c)) & 0xfu) - 8) * a, float(int((w >> (8u*c+4u)) & 0xfu) - 8) * a);+ uint e = wb + (j >> 1u) * 32u + (j & 1u);+ sg[e] = v;+ }+ }+}++// Each subgroup owns output tiles q = sgi, sgi + nsg, ... (8 per workgroup) for+// all TT token tiles: MAXQ tiles per subgroup at most (8 with one wave64 group of 4).+#define MAXQ 2u++void main() {+ uint s0 = gl_WorkGroupID.x * TM;+ uint o0 = gl_WorkGroupID.y * BO;+ uint tid = gl_LocalInvocationID.x;+ uint nsg = gl_NumSubgroups, sgi = gl_SubgroupID;+ uint K = uint(p.I), S = uint(p.S);+ uint nq = (8u + nsg - 1u) / nsg; // output tiles per subgroup: 2 (wave64) or 1 (wave32)++ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> G[MAXQ * TT];+ [[unroll]] for (uint t = 0u; t < MAXQ * TT; t++) {+ G[t] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);+ }+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> A[TT];+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> B;++ fetch_raw(s0, o0, 0u, tid);+ store_raw(tid);+ barrier();+ for (uint k0 = 0u; k0 < K; k0 += 64u) {+ bool nxt = k0 + 64u < K; // workgroup-uniform+ if (nxt) fetch_raw(s0, o0, k0 + 64u, tid); // in flight during the multiply below+ [[unroll]] for (uint kt = 0u; kt < 4u; kt++) {+ [[unroll]] for (uint t = 0u; t < TT; t++)+ coopMatLoad(A[t], sx, (t * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutRowMajor);+ [[unroll]] for (uint qi = 0u; qi < MAXQ; qi++) {+ uint q = sgi + qi * 4u; // wave64 assumed (requiredSubgroupSize = 64)+ coopMatLoad(B, sg, (q * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutColumnMajor);+ [[unroll]] for (uint t = 0u; t < TT; t++) G[qi * TT + t] = coopMatMulAdd(A[t], B, G[qi * TT + t]);+ }+ }+ barrier();+ if (nxt) { store_raw(tid); barrier(); }+ }++ uint lane = gl_SubgroupInvocationID, sgsz = gl_SubgroupSize;+ [[unroll]] for (uint qi = 0u; qi < MAXQ; qi++) {+ if (qi >= nq) break;+ uint q = sgi + qi * nsg;+ if (q >= 8u) break;+ [[unroll]] for (uint t = 0u; t < TT; t++) {+ uint r0 = s0 + t * 16u;+ if (r0 >= S) continue; // subgroup-uniform+ if (r0 + 16u <= S) {+ coopMatStore(G[qi * TT + t], y, r0 * uint(p.O) + o0 + q * 16u, uint(p.O), gl_CooperativeMatrixLayoutRowMajor);+ } else {+ coopMatStore(G[qi * TT + t], spatch, sgi * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);+ subgroupBarrier();+ for (uint e = lane; e < 256u; e += sgsz) {+ uint r = e / 16u, n = e % 16u;+ if (r0 + r < S) y[(r0 + r) * uint(p.O) + o0 + q * 16u + n] = spatch[sgi * 256u + e];+ }+ subgroupBarrier();+ }+ }+ }+}diff --git a/c/shaders/chain_gemm.spv b/c/shaders/chain_gemm.spvnew file mode 100644index 00000000..8a18e113Binary files /dev/null and b/c/shaders/chain_gemm.spv differdiff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.cindex 6b6e4f0b..80dfdbdc 100644--- a/c/tests/test_vk_chain.c+++ b/c/tests/test_vk_chain.c@@ -102,6 +102,7 @@ static float *down(VkcBuf *b, size_t off, size_t n) {/* ---- matmul ----------------------------------------------------------------------- */static uint16_t f2bf(float f) { uint32_t u; memcpy(&u, &f, 4); return (uint16_t)((u + 0x7fff + ((u >> 16) & 1)) >> 16); }static float bf2f(uint16_t h) { uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }+static double g_mm_tol = 2e-4; /* chain_gemm.comp's cases: x rounded to f16 */static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {float *x = fvec((size_t)S * I, 1.f), *W = malloc((size_t)O * I * sizeof(float));void *codes = NULL; float *sc = NULL; int gs = 0;@@ -140,7 +141,7 @@ static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {vkc_submit(1);float *y = (float *)vkc_ptr(yb) + yo;double e = relerr(y, ref, (size_t)S * O, 1e-3);- CHECK(ok && e < 2e-4 && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);+ CHECK(ok && e < g_mm_tol && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);vkc_free(xb); vkc_free(yb); coli_vk_tensor_free(t);free(x); free(W); free(codes); free(sc); free(ref);}@@ -2350,6 +2351,21 @@ int main(int argc, char **argv) {test_matmul(fmts[k], 1, 100, 33, 0, 0); /* rows that are not whole 16-byte steps */test_matmul(fmts[k], 2, 3072, 40, 0, 0); /* a long row (chain_gemv's staging) */}+ /* int8 and int4 prompt GEMMs whose shapes chain_gemm.comp takes (I % 64, O % 128), on+ * a device with cooperative matrices at subgroup size 64: full and partial row tiles,+ * bindable offsets; its x is rounded to f16, hence the bound */+ {+ VkcStats s0; vkc_stats(&s0);+ g_mm_tol = 2e-3;+ for (int k = 0; k < 2; k++) {+ test_matmul(fmts[k], 64, 256, 256, 0, 0);+ test_matmul(fmts[k], 70, 512, 384, 64, 128);+ test_matmul(fmts[k], 130, 192, 128, 0, 0);+ }+ g_mm_tol = 2e-4;+ VkcStats s1; vkc_stats(&s1);+ printf(" prompt GEMMs on chain_gemm: %llu\n", s1.tile_gemms - s0.tile_gemms);+ }printf("matmul done\n");test_norm(VKC_NORM_ADD1, 0); test_norm(VKC_NORM_ADD1, 1); test_norm(0, 0); test_norm(VKC_NORM_NOW, 1); test_norm(VKC_NORM_L2 | VKC_NORM_NOW, 0);printf("norm done\n");diff --git a/c/vk_chain.c b/c/vk_chain.cindex c45c0f00..ac728c0b 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -97,6 +97,8 @@ static struct {VkPipeline gemm[VKC_GEMM_MAX]; int gemm_bm[VKC_GEMM_MAX], gemm_bn[VKC_GEMM_MAX], ngemm;VkPipeline dnrec[VKC_DNREC_MAX]; int dnrec_kd[VKC_DNREC_MAX], ndnrec;VkShaderModule mod_gemv4; VkPipeline gemv4; int gemv4_xs; /* chain_gemv.comp: the vectorized decode GEMV */+ /* chain_gemm.comp: int8/int4 prompt GEMMs, x rounded to f16 once (COLI_VK_CHAIN_GEMM) */+ VkShaderModule mod_tg; VkPipeline tg;VkShaderModule mmod[PM_N]; VkPipeline mpipe[PM_N]; int mla_ok; /* chain_mla, chain_hgemv, chain_dsa */VkPipeline kdarec[VKC_KDA_MAX]; int kdarec_kd[VKC_KDA_MAX], nkdarec; /* chain_kda's recurrence per key dim */VkCommandPool cpool;@@ -318,6 +320,25 @@ int vkc_init(void) {}K.mla_ok = K.mpipe[PM_MLA] && K.mpipe[PM_HGEMV] && K.mpipe[PM_DSA];dsv4_init();+ /* the int8/int4 prompt GEMM on the matrix units (chain_gemm.comp, subgroups of 64);+ * COLI_VK_CHAIN_GEMM=0 leaves those formats to the GEMMs below */+ {+ const char *e = getenv("COLI_VK_CHAIN_GEMM");+ if (K.core.coop_sg == 64 && !(e && *e == '0') && (K.mod_tg = load_module(K.core.spv_path, "chain_gemm.spv"))) {+ int32_t tt = 4;+ VkSpecializationMapEntry me = {0, 0, 4};+ VkSpecializationInfo si = {1, &me, 4, &tt};+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,+ .requiredSubgroupSize = 64};+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = K.mod_tg, .pName = "main",+ .pSpecializationInfo = &si}, .layout = K.pl};+ if (vkCreateComputePipelines(K.dev, VK_NULL_HANDLE, 1, &ci, NULL, &K.tg) != VK_SUCCESS) K.tg = VK_NULL_HANDLE;+ }+ }/* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {@@ -695,6 +716,11 @@ static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcBint wg = (ti->O + rows_wg - 1) / rows_wg; if (wg > 1024) wg = 1024;ok = record(K.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);}+ else if (path >= 0 && K.tg && (ti->fmt == 1 || ti->fmt == 2 || (ti->fmt == 4 && ti->gs >= 8 && ti->gs % 8 == 0)) &&+ ti->I % 64 == 0 && ti->O % 128 == 0 && ti->rowWords % 4 == 0) {+ ok = record(K.tg, bd, 4, &pc, sizeof pc, (uint32_t)((S + 63) / 64), (uint32_t)(ti->O / 128), 1);+ K.st.tile_gemms += ok;+ }else if (path >= 0)ok = record(K.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + K.gemm_bm[path] - 1) / K.gemm_bm[path]),(uint32_t)((S + K.gemm_bn[path] - 1) / K.gemm_bn[path]), 1);@@ -1483,6 +1509,8 @@ void vkc_shutdown(void) {if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);}for (int i = 0; i < K.ngemm; i++) vkDestroyPipeline(K.dev, K.gemm[i], NULL);+ if (K.tg) vkDestroyPipeline(K.dev, K.tg, NULL);+ if (K.mod_tg) vkDestroyShaderModule(K.dev, K.mod_tg, NULL);for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);diff --git a/c/vk_chain.h b/c/vk_chain.hindex f67041ae..1eb3b7b7 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -646,6 +646,7 @@ typedef struct {size_t dev_bytes; /* live chain buffers */unsigned long long attn_blocked; /* attention calls through chain_attnb (blocks of rows) */unsigned long long attn_slices; /* extra submissions that cut a long attention (COLI_VK_ATTN_SLICE) */+ unsigned long long tile_gemms; /* prompt matmuls through chain_gemm.comp (matrix units) */} VkcStats;void vkc_stats(VkcStats *st);/* COLI_VK_CHAIN_PROF=1: one stderr line of device time per kind of op */diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 6d2d8025..b363b46b 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -226,6 +226,7 @@ Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRE| `COLI_VK_GEMM_MIN_S` | measured | When a resident matmul (`coli_vk_matmul`, every engine) takes the tiled GEMM instead of the per-row GEMV. Unset: S ≥ 2 and S·O ≥ 4096, measured on a Radeon 780M (a narrow matrix at small S stays on the GEMV). `N`: every call with S ≥ N. `0`: the GEMV for every S. S = 1 (decode) always stays on the GEMV. See [vulkan.md](vulkan.md#prefill-the-tiled-gemms). || `COLI_VK_COOP` | on where supported | The cooperative-matrix GEMM (`VK_KHR_cooperative_matrix`, 16x16x16 fp16 → fp32) for the formats whose weights decode exactly to fp16: int8, int4, int3-g64, MXFP4 and fp8, the grouped ones with a group size that is a multiple of 32. `0` keeps every format on the fp32 GEMM. || `COLI_VK_COOP_SG` | `64` where allowed | Subgroup size the cooperative-matrix pipeline requires (RDNA3: wave64 measured fastest). Tuning only. |+| `COLI_VK_CHAIN_GEMM` | on where supported | The dense chain's int8 and int4 prompt matmuls (`I` a multiple of 64, `O` of 128) on the matrix units (`chain_gemm.comp`: x rounded to f16 once) on a device with cooperative matrices at subgroup size 64. `0` keeps them on the chain's other GEMMs. || `COLI_VK_GEMM_TILE` | measured | `bm,bn,bk,tm,tn[,pf]`: one tile for every width of the fp32 GEMM instead of the measured pair. Tuning only (`COLI_VK_TEST_GEMM_BENCH`). || `COLI_VK_COOP_TILE` | measured | `bm,bn,wm,wn,bk`: one tile for every width of the cooperative-matrix GEMM. Tuning only. || `COLI_VK_STAGED` | auto | Staged uploads: resident data (the weights, the expert tier, the MLA KV mirror, `COLI_VK_DEV2`'s experts) copied into device-local memory the host does not map, through a host staging buffer, instead of written through a mapping of the host-visible device-local memory. `1` on, `0` off. Unset: on when the host-visible device-local heap is under a quarter of the largest device-local heap, as on a discrete card without Resizable BAR (about 256 MB of 8 GB, where the mapped path fails or spills to system RAM); off on a card with Resizable BAR, an integrated GPU or Lavapipe. See [vulkan.md](vulkan.md#memory-placement-without-resizable-bar). |diff --git a/docs/vulkan.md b/docs/vulkan.mdindex 3e1245e7..6dd89005 100644--- a/docs/vulkan.md+++ b/docs/vulkan.md@@ -2049,6 +2049,7 @@ The run's line counts what streamed:| `COLI_VK_CHAIN_ROWS` | from the budget, up to 8192 | Prompt rows per chain chunk; `auto` as unset. Set, it also keeps the engines' own prompt blocks (above). || `COLI_VK_CHAIN_ROWS_MAX` | `8192` | The most rows the budget's chunk takes. || `COLI_VK_ATTN_BLOCK` | `16` | Rows from which the chain's attention takes `chain_attnb.comp`; `0` never. |+| `COLI_VK_CHAIN_GEMM` | on where supported | The chain's int8/int4 prompt matmuls (`I % 64`, `O % 128`) on `chain_gemm.comp` (cooperative matrices at subgroup size 64, x rounded to f16 once, 128 outputs x 64 rows a workgroup); `0` keeps the fp32 GEMM. On a Radeon 8060S (Qwen3.6, 1011-token prompt): 517-546 -> 282-287 ms of tiled GEMM, the first token 2.08 -> 1.85 s, perplexity unchanged (7.65). || `COLI_VK_ATTN_SLICE` | `4294967296` | Rows x positions x heads x head dim past which an attention is cut over several submissions; `0` never. || `COLI_VK_TIER_STREAM` | on | `0`: no streaming; the tier takes a prompt step in the engine's usual blocks and leaves the cold experts to the CPU, as before. || `COLI_VK_TIER_STREAM_SLOTS` | `64` | Staging slots (experts) for streaming, out of the tier's budget. |