Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
vulkan: tiled GEMMs for prefill-sized S
JustVugg8b85c66112dd
diff --git a/c/Makefile b/c/Makefileindex d2f677d2..54354a75 100644--- a/c/Makefile+++ b/c/Makefile@@ -704,7 +704,8 @@ elseLDFLAGS += -lvulkanendifVK_OBJ = backend_vulkan.o-VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv+VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \+ shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spvendif# Optional AMD XDNA2 (Ryzen AI NPU) lane, Windows. Same opt-in shape as VK=1:@@ -841,6 +842,13 @@ backend_vulkan.o: backend_vulkan.c .build-configshaders/%.spv: shaders/%.comp@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }$(GLSLC) --target-env=vulkan1.2 $< -o $@+# The cooperative-matrix GEMM needs a glslc that knows GL_KHR_cooperative_matrix+# (shaderc 2023.6+). An older one keeps the committed .spv instead of failing the build;+# at run time the shader is optional anyway (the fp32 GEMM takes over without it).+shaders/qmatmul_coop.spv: shaders/qmatmul_coop.comp+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.backend_loader.o: backend_loader.c .build-configdiff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4index 20e17870..4e87b387 100644--- a/c/Makefile.deepseek-v4+++ b/c/Makefile.deepseek-v4@@ -235,7 +235,8 @@ LDFLAGS += -lvulkanendifV4_VK_OBJ = backend_vulkan_v4.oV4_VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv \- shaders/attention_absorb.spv shaders/rmsnorm.spv+ shaders/attention_absorb.spv shaders/rmsnorm.spv \+ shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spvendifinclude Makefile.deepseek-v4.units@@ -312,6 +313,11 @@ backend_vulkan_v4.o: backend_vulkan.c backend_vulkan.h $(V4_FLAGS_STAMP)shaders/%.spv: shaders/%.comp@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }$(GLSLC) --target-env=vulkan1.2 $< -o $@+# As in the parent: an older glslc keeps the committed cooperative-matrix .spv.+shaders/qmatmul_coop.spv: shaders/qmatmul_coop.comp+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }# Pluggable expert-store backend registry (standalone; not a deepseek_v4.c unit).# Linked into the binary so the engine can dispatch COLI_EXPERT_STORE backends.diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex 9d272e6c..9d07a729 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -34,6 +34,16 @@ typedef struct {VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;} Scratch;+/* Tile of the fp32 GEMM pipeline: BM outputs x BN activation rows per workgroup, BK+ * inputs per shared-memory step, TM x TN accumulators per thread; pf = fetch the next+ * step's words while this one's products run. */+typedef struct { int bm, bn, bk, tm, tn, pf; } VkGemmTile;+/* Tile of the cooperative-matrix pipeline: BM outputs x BN activation rows per+ * workgroup, WM x WN of it per subgroup (multiples of 16), BK inputs per staged step. */+typedef struct { int bm, bn, wm, wn, bk; } VkCoopTile;+#define VK_GEMM_SLOTS 2+#define VK_COOP_SLOTS 3+/* Persistent device-side KV latent/rope cache for one layer (MLA attention).* Host appends rows as tokens decode (absolute-position indexing); the absorb* kernel reads them in place. Allocated once at max_t rows, like the CUDA@@ -59,6 +69,19 @@ static struct {VkShaderModule shader;VkDescriptorPool dpool;VkDescriptorSet dset;+ /* Tiled GEMMs for prefill-sized S, with the GEMV's 4 bindings and push constants so+ * they bind through G.plyt/G.dset: the fp32 one (qmatmul_gemm.spv) and, when the+ * device has a 16x16x16 fp16 -> fp32 subgroup MMA and a settable subgroup size+ * (coop_sg), the cooperative-matrix one (qmatmul_coop.spv) for the formats whose+ * decoded weights are fp16 values. Each is built at a few tile widths (specialization+ * constants, slots of growing BN) and a call takes the narrowest that covers its S.+ * A call takes a GEMM when S >= gemm_min_s (0 = GEMV for every S) and S*O >=+ * gemm_min_so; coop_off is the harness's switch to the fp32 GEMM. */+ VkShaderModule shader_gemm, shader_coop;+ VkPipeline pipe_gemm[VK_GEMM_SLOTS], pipe_coop[VK_COOP_SLOTS];+ VkGemmTile gemm_t[VK_GEMM_SLOTS];+ VkCoopTile coop_t[VK_COOP_SLOTS];+ int gemm_min_s, gemm_min_so, has_coop, coop_sg, coop_off;/* fused dual gate+up+silu pipeline (6 bindings): x, Wg, gscale, Wu, uscale, hidden */VkShaderModule shader_gu; VkDescriptorSetLayout dsl_gu; VkPipelineLayout plyt_gu;VkPipeline pipe_gu; VkDescriptorPool dpool_gu; VkDescriptorSet dset_gu;@@ -96,7 +119,7 @@ static struct {* expert-called-repeatedly pattern). The synchronous fence wait each call means no* submission is ever in flight, so rebinding/re-recording only when something* actually changed is safe. */- ColiVkTensor *bound_tensor; int bound_S, bound_I, bound_O, cmd_ready;+ ColiVkTensor *bound_tensor; int bound_S, bound_I, bound_O, bound_gemm, cmd_ready;VkBuffer bound_xbuf, bound_ybuf;size_t used_bytes, tensor_count;/* VRAM pressure-proofing: with VK_EXT_memory_priority the attention working set@@ -307,6 +330,89 @@ static void derive_dir_file(const char *spv, const char *fname, char *out, size_else snprintf(out, n, "%s", fname);}+static int gemm_tile_ok(VkGemmTile t, const VkPhysicalDeviceLimits *lim) {+ if (t.bm < 1 || t.bn < 1 || t.tm < 1 || t.tn < 1 || t.bk < 8 || t.bk % 8 ||+ t.bm % t.tm || t.bn % t.tn) return 0;+ long threads = (long)(t.bm / t.tm) * (t.bn / t.tn);+ long lds = (long)(t.bm + t.bn) * (t.bk / 4 + 1) * 16; /* wsh + xsh, vec4 quads */+ return threads >= 32 && threads <= (long)lim->maxComputeWorkGroupInvocations &&+ threads <= (long)lim->maxComputeWorkGroupSize[0] && lds <= (long)lim->maxComputeSharedMemorySize;+}+static int gemm_pipeline(VkGemmTile t, int slot) {+ int32_t sv[7] = {t.bm, t.bn, t.bk, t.tm, t.tn, (t.bm / t.tm) * (t.bn / t.tn), t.pf ? 1 : 0};+ VkSpecializationMapEntry me[7];+ for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};+ VkSpecializationInfo si = {7, me, sizeof(sv), sv};+ VkComputePipelineCreateInfo cpi = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = G.shader_gemm, .pName = "main",+ .pSpecializationInfo = &si},+ .layout = G.plyt};+ VkPipeline pipe;+ VKCHECK(vkCreateComputePipelines(G.dev, VK_NULL_HANDLE, 1, &cpi, NULL, &pipe), "gemm pipeline");+ if (G.pipe_gemm[slot]) vkDestroyPipeline(G.dev, G.pipe_gemm[slot], NULL);+ G.pipe_gemm[slot] = pipe; G.gemm_t[slot] = t;+ G.cmd_ready = 0; /* a recorded GEMM dispatch names the old pipeline */+ return 1;+}++/* Subgroups of exactly G.coop_sg lanes; BK a multiple of 32 (group sizes are). */+static int coop_tile_ok(VkCoopTile t, const VkPhysicalDeviceLimits *lim) {+ if (t.wm < 16 || t.wn < 16 || t.wm % 16 || t.wn % 16 || t.bm % t.wm || t.bn % t.wn ||+ t.bk < 32 || t.bk % 32 || G.coop_sg < 16 || 256 % G.coop_sg) return 0;+ long nw = (long)(t.bm / t.wm) * (t.bn / t.wn), threads = nw * G.coop_sg;+ long lds = (long)(t.bm + 2 * t.bn) * (t.bk + 8) * 2 + (long)t.bm * 4 + nw * 16 * 17 * 4;+ return threads <= (long)lim->maxComputeWorkGroupInvocations &&+ threads <= (long)lim->maxComputeWorkGroupSize[0] && lds <= (long)lim->maxComputeSharedMemorySize;+}+static int coop_pipeline(VkCoopTile t, int slot) {+#ifdef VK_KHR_cooperative_matrix+ int32_t sv[7] = {t.bm, t.bn, t.wm, t.wn, t.bk, G.coop_sg, (t.bm / t.wm) * (t.bn / t.wn) * G.coop_sg};+ VkSpecializationMapEntry me[7];+ for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};+ VkSpecializationInfo si = {7, me, sizeof(sv), sv};+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,+ .requiredSubgroupSize = (uint32_t)G.coop_sg};+ VkComputePipelineCreateInfo cpi = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = G.shader_coop, .pName = "main",+ .pSpecializationInfo = &si},+ .layout = G.plyt};+ VkPipeline pipe;+ VKCHECK(vkCreateComputePipelines(G.dev, VK_NULL_HANDLE, 1, &cpi, NULL, &pipe), "coop pipeline");+ if (G.pipe_coop[slot]) vkDestroyPipeline(G.dev, G.pipe_coop[slot], NULL);+ G.pipe_coop[slot] = pipe; G.coop_t[slot] = t;+ G.cmd_ready = 0;+ return 1;+#else+ (void)t; (void)slot; return 0;+#endif+}++/* The tiles, narrowest first, measured on a Radeon 780M (RDNA3, RADV) over Qwen3.8's+ * shapes: BN = 32 serves a 32-row prefill chunk, the wider ones longer prompts.+ * COLI_VK_GEMM_TILE=bm,bn,bk,tm,tn[,pf] and COLI_VK_COOP_TILE=bm,bn,wm,wn,bk put one+ * tile in every slot (bench). */+static const VkGemmTile gemm_tiles[VK_GEMM_SLOTS] = {{128, 32, 32, 4, 4, 1}, {64, 64, 32, 4, 4, 0}};+static const VkCoopTile coop_tiles[VK_COOP_SLOTS] = {{128, 32, 32, 32, 32}, {128, 64, 32, 32, 32},+ {128, 128, 64, 32, 32}};+/* The narrowest slot whose BN covers S, else the widest. */+static int gemm_slot(int S) {+ int k = 0;+ while (k + 1 < VK_GEMM_SLOTS && G.pipe_gemm[k + 1] && S > G.gemm_t[k].bn) k++;+ return k;+}+static int coop_slot(int S) {+ int k = 0;+ while (k + 1 < VK_COOP_SLOTS && G.pipe_coop[k + 1] && S > G.coop_t[k].bn) k++;+ return k;+}++static void vk_prof_paths(void);+static int g_vk_prof;+int coli_vk_init(const char *spv_path) {if (G.ready) return 1;VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,@@ -348,7 +454,8 @@ int coli_vk_init(const char *spv_path) {/* Pressure-proofing extensions (both optional, detected at runtime):* memory_priority ranks allocations for the kernel's eviction order,* memory_budget exposes how much VRAM a new allocation can still take. */- const char *dext[2]; uint32_t ndext = 0;+ const char *dext[4]; uint32_t ndext = 0;+ int has_cm = 0, has_ssc = 0;{uint32_t ne = 0;vkEnumerateDeviceExtensionProperties(G.phys, NULL, &ne, NULL);@@ -361,11 +468,72 @@ int coli_vk_init(const char *spv_path) {#endif#ifdef VK_EXT_memory_budgetif (!strcmp(ep[i].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) G.has_budget = 1;+#endif+#ifdef VK_KHR_cooperative_matrix+ if (!strcmp(ep[i].extensionName, VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME)) has_cm = 1;+ if (!strcmp(ep[i].extensionName, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME)) has_ssc = 1;#endif}free(ep);}}+ /* Cooperative-matrix GEMM (optional, COLI_VK_COOP=0 turns it off): the KHR extension+ * with a 16x16x16 fp16 x fp16 -> fp32 subgroup shape, shaderFloat16, the Vulkan+ * memory model, and a subgroup size the pipeline can require. All of it or none. */+#ifdef VK_KHR_cooperative_matrix+ VkPhysicalDeviceCooperativeMatrixFeaturesKHR cmf = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR};+ VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sscf = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT, .pNext = &cmf};+ VkPhysicalDeviceVulkan12Features v12f = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, .pNext = &sscf};+ {+ const char *e = getenv("COLI_VK_COOP");+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties(G.phys, &pp);+ if (has_cm && has_ssc && pp.apiVersion >= VK_API_VERSION_1_2 && !(e && *e == '0')) {+ VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &v12f};+ vkGetPhysicalDeviceFeatures2(G.phys, &f2);+ VkPhysicalDeviceSubgroupSizeControlPropertiesEXT ssp = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT};+ VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &ssp};+ vkGetPhysicalDeviceProperties2(G.phys, &p2);+ int shape = 0;+ PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR cmp = (PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR)+ vkGetInstanceProcAddr(G.inst, "vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR");+ uint32_t nc = 0;+ if (cmp && cmp(G.phys, &nc, NULL) == VK_SUCCESS && nc) {+ VkCooperativeMatrixPropertiesKHR *cp = calloc(nc, sizeof(*cp));+ if (cp) {+ for (uint32_t i = 0; i < nc; i++) cp[i].sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR;+ if (cmp(G.phys, &nc, cp) == VK_SUCCESS)+ for (uint32_t i = 0; i < nc; i++)+ if (cp[i].MSize == 16 && cp[i].NSize == 16 && cp[i].KSize == 16 &&+ cp[i].AType == VK_COMPONENT_TYPE_FLOAT16_KHR && cp[i].BType == VK_COMPONENT_TYPE_FLOAT16_KHR &&+ cp[i].CType == VK_COMPONENT_TYPE_FLOAT32_KHR && cp[i].ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR &&+ cp[i].scope == VK_SCOPE_SUBGROUP_KHR) shape = 1;+ free(cp);+ }+ }+ /* Subgroups of 64 where the device allows (RDNA3: the wave64 MMA measured+ * fastest), else the largest it can require; COLI_VK_COOP_SG overrides. */+ int sgz = ssp.maxSubgroupSize >= 64 && ssp.minSubgroupSize <= 64 ? 64 : (int)ssp.maxSubgroupSize;+ const char *se = getenv("COLI_VK_COOP_SG");+ if (se && atoi(se) >= (int)ssp.minSubgroupSize && atoi(se) <= (int)ssp.maxSubgroupSize) sgz = atoi(se);+ if (shape && cmf.cooperativeMatrix && v12f.shaderFloat16 && v12f.vulkanMemoryModel &&+ sscf.subgroupSizeControl && sscf.computeFullSubgroups &&+ (ssp.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) && sgz >= 16) {+ G.has_coop = 1; G.coop_sg = sgz;+ }+ }+ }+ /* enable exactly what the cooperative-matrix shader needs, nothing else */+ memset(&v12f, 0, sizeof(v12f)); memset(&sscf, 0, sizeof(sscf)); memset(&cmf, 0, sizeof(cmf));+ cmf.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR; cmf.cooperativeMatrix = VK_TRUE;+ sscf.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT; sscf.pNext = &cmf;+ sscf.subgroupSizeControl = VK_TRUE; sscf.computeFullSubgroups = VK_TRUE;+ v12f.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; v12f.pNext = &sscf;+ v12f.shaderFloat16 = VK_TRUE; v12f.vulkanMemoryModel = VK_TRUE;+#endifVkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,.queueCreateInfoCount = 1, .pQueueCreateInfos = &qi};#ifdef VK_EXT_memory_priority@@ -379,6 +547,19 @@ int coli_vk_init(const char *spv_path) {#endifdi.enabledExtensionCount = ndext; di.ppEnabledExtensionNames = ndext ? dext : NULL;G.prio = 0.75f; /* default class: dense/resident weights */+#ifdef VK_KHR_cooperative_matrix+ if (G.has_coop) { /* a device that refuses the extra features still comes up without them */+ VkDeviceCreateInfo dc = di;+ const char *cext[4]; uint32_t nc = 0;+ for (uint32_t i = 0; i < ndext; i++) cext[nc++] = dext[i];+ cext[nc++] = VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME;+ cext[nc++] = VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME;+ cmf.pNext = (void *)di.pNext; dc.pNext = &v12f;+ dc.enabledExtensionCount = nc; dc.ppEnabledExtensionNames = cext;+ if (vkCreateDevice(G.phys, &dc, NULL, &G.dev) != VK_SUCCESS) { G.has_coop = 0; G.dev = VK_NULL_HANDLE; }+ }+ if (!G.dev)+#endifVKCHECK(vkCreateDevice(G.phys, &di, NULL, &G.dev), "vkCreateDevice");vkGetDeviceQueue(G.dev, G.qfam, 0, &G.queue);if (G.has_prio || G.has_budget)@@ -421,6 +602,61 @@ int coli_vk_init(const char *spv_path) {if (!G.shader) return 0;if (!build_pipeline(G.dev, 4, sizeof(struct PC), G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;+ /* Optional tiled GEMMs: an absent shader or a tile the device cannot hold leaves+ * that slot empty; without the fp32 one every S stays on the GEMV. The cooperative-+ * matrix one only comes up on devices that passed the checks above, and alongside+ * the fp32 one, which takes the formats it does not. */+ {+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties(G.phys, &pp);+ char gm_path[512]; derive_sibling(spv_path, "_gemm.spv", gm_path, sizeof(gm_path));+ G.shader_gemm = load_spv(G.dev, gm_path);+ const char *e = getenv("COLI_VK_GEMM_TILE");+ VkGemmTile u = {0, 0, 0, 0, 0, 0};+ int one = e && *e && sscanf(e, "%d,%d,%d,%d,%d,%d", &u.bm, &u.bn, &u.bk, &u.tm, &u.tn, &u.pf) >= 5 &&+ gemm_tile_ok(u, &pp.limits);+ if (e && *e && !one) fprintf(stderr, "[VK] COLI_VK_GEMM_TILE=%s rejected\n", e);+ for (int k = 0; G.shader_gemm && k < (one ? 1 : VK_GEMM_SLOTS); k++) {+ VkGemmTile t = one ? u : gemm_tiles[k];+ if (gemm_tile_ok(t, &pp.limits)) gemm_pipeline(t, k);+ }+ if (G.shader_gemm && !G.pipe_gemm[0]) {+ vkDestroyShaderModule(G.dev, G.shader_gemm, NULL); G.shader_gemm = VK_NULL_HANDLE;+ for (int k = 1; k < VK_GEMM_SLOTS; k++)+ if (G.pipe_gemm[k]) { vkDestroyPipeline(G.dev, G.pipe_gemm[k], NULL); G.pipe_gemm[k] = VK_NULL_HANDLE; }+ }+ if (G.has_coop && G.pipe_gemm[0]) {+ char cm_path[512]; derive_sibling(spv_path, "_coop.spv", cm_path, sizeof(cm_path));+ G.shader_coop = load_spv(G.dev, cm_path);+ e = getenv("COLI_VK_COOP_TILE");+ VkCoopTile c = {0, 0, 0, 0, 0};+ one = e && *e && sscanf(e, "%d,%d,%d,%d,%d", &c.bm, &c.bn, &c.wm, &c.wn, &c.bk) == 5 &&+ coop_tile_ok(c, &pp.limits);+ if (e && *e && !one) fprintf(stderr, "[VK] COLI_VK_COOP_TILE=%s rejected\n", e);+ /* every slot shares BK: coop_fmt checks group sizes against it */+ for (int k = 0; G.shader_coop && k < (one ? 1 : VK_COOP_SLOTS); k++) {+ VkCoopTile t = one ? c : coop_tiles[k];+ if (coop_tile_ok(t, &pp.limits)) coop_pipeline(t, k);+ }+ }+ if (!G.pipe_coop[0]) {+ for (int k = 1; k < VK_COOP_SLOTS; k++)+ if (G.pipe_coop[k]) { vkDestroyPipeline(G.dev, G.pipe_coop[k], NULL); G.pipe_coop[k] = VK_NULL_HANDLE; }+ if (G.shader_coop) vkDestroyShaderModule(G.dev, G.shader_coop, NULL);+ G.shader_coop = VK_NULL_HANDLE; G.has_coop = 0;+ }+ /* Where the GEMM overtakes the GEMV, measured on the 780M over Qwen3.8's shapes+ * (int8, int4-g64, bf16): from S = 2 on a wide matrix (O = 2560: 0.65 against+ * 0.90 ms in int8, 1.0 against 2.2 in bf16), but a narrow one fills too few+ * tiles (O = 48: the GEMV wins up to S = 32, loses from 128). The rule that+ * fits: S >= 2 and S*O >= 4096. COLI_VK_GEMM_MIN_S=N puts every S >= N on the+ * GEMM, 0 none; S == 1 (decode) always stays on the GEMV. */+ const char *m = getenv("COLI_VK_GEMM_MIN_S");+ G.gemm_min_s = !G.pipe_gemm[0] ? 0 : m && *m ? atoi(m) : 2;+ G.gemm_min_so = m && *m ? 0 : 4096;+ if (G.gemm_min_s == 1) G.gemm_min_s = 2;+ if (G.gemm_min_s < 0) G.gemm_min_s = 0;+ }+/* Optional fused gate+up pipeline: skip gracefully if its shader isn't present* (single-matmul path keeps working). */char gu_path[512]; derive_sibling(spv_path, "_gate_up.spv", gu_path, sizeof(gu_path));@@ -464,9 +700,13 @@ int coli_vk_init(const char *spv_path) {VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.eg_fence), "eg fence");G.ready = 1;+ if ((g_vk_prof = getenv("VK_PROF") != NULL)) atexit(vk_prof_paths);VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);- fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s%s\n", p.deviceName, G.qfam, G.memtype,+ fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s%s", p.deviceName, G.qfam, G.memtype,G.shader_gu ? ", fused gate+up" : "", G.shader_att ? ", absorb attention" : "");+ if (G.gemm_min_s) fprintf(stderr, ", tiled GEMM from S=%d%s%s", G.gemm_min_s,+ G.gemm_min_so ? " and S*O>=4096" : "", G.pipe_coop[0] ? " (cooperative matrix)" : "");+ fprintf(stderr, "\n");return 1;}@@ -577,21 +817,32 @@ int coli_vk_tensor_ensure(ColiVkTensor **tensor, const void *weights, const floa* then fall back to the blocking wait. The spinning thread is stalled on the* GPU result anyway. COLI_VK_SPIN_US=0 restores the pure blocking wait. */static long g_vk_spin_us = -1;-static VkResult vk_fence_wait(VkDevice dev, VkFence f) {- if (g_vk_spin_us < 0) {- const char *e = getenv("COLI_VK_SPIN_US");- g_vk_spin_us = e ? atol(e) : 300;- if (g_vk_spin_us < 0) g_vk_spin_us = 0;- }- if (g_vk_spin_us > 0) {+static VkResult vk_fence_wait_us(VkDevice dev, VkFence f, long spin_us) {+ if (spin_us > 0) {double t0 = vk_now();do {VkResult r = vkGetFenceStatus(dev, f);if (r != VK_NOT_READY) return r; /* VK_SUCCESS or a real error */- } while ((vk_now() - t0) * 1000.0 < (double)g_vk_spin_us);+ } while ((vk_now() - t0) * 1000.0 < (double)spin_us);}return vkWaitForFences(dev, 1, &f, VK_TRUE, 10000000000ULL);}+static long vk_spin_us(void) {+ if (g_vk_spin_us < 0) {+ const char *e = getenv("COLI_VK_SPIN_US");+ g_vk_spin_us = e ? atol(e) : 300;+ if (g_vk_spin_us < 0) g_vk_spin_us = 0;+ }+ return g_vk_spin_us;+}+static VkResult vk_fence_wait(VkDevice dev, VkFence f) { return vk_fence_wait_us(dev, f, vk_spin_us()); }+/* A tiled GEMM runs for milliseconds, past the spin budget, and the blocked wait then+ * wakes up to a millisecond late: measured on the 780M, a 0.6 ms S=32 GEMM took 0.6 or+ * 1.05 ms per call at random. Its wait spins through the dispatch, bounded at 50 ms;+ * COLI_VK_SPIN_US=0 still blocks. */+static VkResult vk_fence_wait_gemm(VkDevice dev, VkFence f) {+ return vk_fence_wait_us(dev, f, vk_spin_us() > 0 ? 50000 : 0);+}/* Global submit/wait totals across EVERY synchronous GPU path (VK_PROF=1) β the* per-path counters miss traffic that flows through the fused pair/absorb/group@@ -603,6 +854,48 @@ static void vkprof_tick(void) {}static unsigned long long g_vk_matmul_calls; /* successful coli_vk_matmul calls */+static unsigned long long g_vk_gemm_calls; /* ... of which through a tiled GEMM */+static unsigned long long g_vk_coop_calls; /* ... of which the cooperative-matrix one */++/* VK_PROF=1: at exit, how the matmuls and their wall time (upload to readback) split+ * between the GEMV and the two tiled GEMMs. */+static double g_vk_path_ms[3];+static unsigned long long g_vk_path_n[3];+static void vk_prof_paths(void) {+ fprintf(stderr, "[VK_PROF] %llu matmuls: GEMV %llu in %.0f ms, fp32 GEMM %llu in %.0f ms, "+ "cooperative-matrix GEMM %llu in %.0f ms\n", g_vk_matmul_calls,+ g_vk_path_n[0], g_vk_path_ms[0], g_vk_path_n[1], g_vk_path_ms[1], g_vk_path_n[2], g_vk_path_ms[2]);+}++/* Formats the cooperative-matrix GEMM takes: decoded weights exact in fp16, and group+ * boundaries on its staged steps (one group scale per step). */+static int coop_fmt(const ColiVkTensor *t) {+ int gs = t->fmt == 5 ? 64 : t->gs;+ return t->fmt == 1 || t->fmt == 2 ||+ ((t->fmt == 4 || t->fmt == 5 || t->fmt == 7 || t->fmt == 12) && gs % G.coop_t[0].bk == 0);+}+/* Its x upload: the rows, then their powers of two rs[s] = 2^(15-e) with+ * max|x[s,:]| < 2^e, so a scaled row stays under 2^15 (fp16 max 65504) and its hi/lo+ * fp16 split keeps 22 bits. One pass copies and takes the integer max of the magnitude+ * bits (vectorizes without fast-math); a row holding Inf or NaN keeps rs = 1 and its+ * non-finite result, as on the GEMV. */+static void coop_stage_x(float *restrict dst, const float *restrict x, int S, int I) {+ float *rs = dst + (size_t)S * I;+ for (int s = 0; s < S; s++) {+ const float *restrict r = x + (size_t)s * I;+ float *restrict d = dst + (size_t)s * I;+ uint32_t m = 0;+ for (int i = 0; i < I; i++) {+ uint32_t u; memcpy(&u, &r[i], 4);+ d[i] = r[i]; u &= 0x7fffffffu; m = u > m ? u : m;+ }+ int e = (int)(m >> 23) - 126; /* m < 2^e for a normal m */+ if (m >= 0x7f800000u) e = 15;+ if (e < -100) e = -100; /* zero and subnormal rows */+ uint32_t b = (uint32_t)(15 - e + 127) << 23;+ memcpy(&rs[s], &b, sizeof(b));+ }+}int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,const void *weights, const float *scales,@@ -613,11 +906,20 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,* separates our code (memcpy/desc/record) from the driver (submit) and the GPU* (fence wait) to localize the tier-size-linear tax. */static double p_x, p_desc, p_rec, p_sub, p_wait, p_y; static long p_n;- double t0 = G.eg_prof ? vk_now() : 0, tA;+ double t0 = G.eg_prof ? vk_now() : 0, tA, tp = g_vk_prof ? vk_now() : 0;+ /* Prefill-sized S takes a tiled GEMM: one BM x BN tile of y per workgroup, so each+ * weight is fetched ceil(S/BN) times instead of S; the cooperative-matrix one where+ * the device and the format allow (path 2), else the fp32 one (1). Decode (S = 1),+ * and a matrix too narrow to fill the device at this S, keep the GEMV (0). */+ int path = 0;+ if (G.gemm_min_s && S >= G.gemm_min_s && (int64_t)S * O >= G.gemm_min_so)+ path = G.pipe_coop[0] && !G.coop_off && coop_fmt(t) ? 2 : 1;size_t xb = (size_t)S * I * sizeof(float), yb = (size_t)S * O * sizeof(float);VkBuffer old_x = G.x.buf, old_y = G.y.buf;- if (!scratch_reserve(&G.x, xb) || !scratch_reserve_mt(&G.y, yb, G.memtype_cached)) return 0; /* y read back */- memcpy(G.x.ptr, x, xb);+ if (!scratch_reserve(&G.x, xb + (path == 2 ? (size_t)S * sizeof(float) : 0)) || /* + rs[S] */+ !scratch_reserve_mt(&G.y, yb, G.memtype_cached)) return 0; /* y read back */+ if (path == 2) coop_stage_x(G.x.ptr, x, S, I);+ else memcpy(G.x.ptr, x, xb);if (G.eg_prof) { tA = vk_now(); p_x += tA - t0; t0 = tA; }/* Rebind descriptors only when the tensor or a scratch buffer changed (a realloc@@ -643,19 +945,26 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,/* Re-record the command buffer only when the binding or the dispatch shape changed.* Recorded WITHOUT one-time-submit so the same buffer can be resubmitted verbatim β* for repeated calls to the same expert this drops setup to a bare submit+wait. */- if (rebind || !G.cmd_ready || G.bound_S != S || G.bound_I != I || G.bound_O != O) {+ if (rebind || !G.cmd_ready || G.bound_S != S || G.bound_I != I || G.bound_O != O || G.bound_gemm != path) {VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);+ int slot = path == 2 ? coop_slot(S) : path ? gemm_slot(S) : 0;+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE,+ path == 2 ? G.pipe_coop[slot] : path ? G.pipe_gemm[slot] : G.pipe);vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.dset, 0, NULL);struct PC pc = {fmt, S, I, O, t->rowWords, t->gs};vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- /* Grid-stride shader: one subgroup per output row (~8 rows/workgroup at wave32).- * Launch ~O/8 workgroups for occupancy; the shader loops to cover any O / wave width. */- vkCmdDispatch(G.cmd, (uint32_t)((O + 7) / 8), (uint32_t)S, 1);+ if (path) {+ int bm = path == 2 ? G.coop_t[slot].bm : G.gemm_t[slot].bm;+ int bn = path == 2 ? G.coop_t[slot].bn : G.gemm_t[slot].bn;+ vkCmdDispatch(G.cmd, (uint32_t)((O + bm - 1) / bm), (uint32_t)((S + bn - 1) / bn), 1);+ } else+ /* Grid-stride shader: one subgroup per output row (~8 rows/workgroup at wave32).+ * Launch ~O/8 workgroups for occupancy; the shader loops to cover any O / wave width. */+ vkCmdDispatch(G.cmd, (uint32_t)((O + 7) / 8), (uint32_t)S, 1);VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");- G.cmd_ready = 1; G.bound_S = S; G.bound_I = I; G.bound_O = O;+ G.cmd_ready = 1; G.bound_S = S; G.bound_I = I; G.bound_O = O; G.bound_gemm = path;}if (G.eg_prof) { tA = vk_now(); p_rec += tA - t0; t0 = tA; }@@ -667,7 +976,7 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,// Bounded wait: a GPU hang/TDR must never wedge the process. 10s is orders of// magnitude over a single-GEMV dispatch; on timeout/device-loss disable VK for// the rest of the run and fall back to CPU (the caller degrades on our 0 return).- VkResult wr = vk_fence_wait(G.dev, G.fence);+ VkResult wr = path ? vk_fence_wait_gemm(G.dev, G.fence) : vk_fence_wait(G.dev, G.fence);if (wr != VK_SUCCESS) {fprintf(stderr, "[VK] fence wait failed: %d β disabling GPU offload, staying on CPU\n", wr);G.ready = 0;@@ -682,6 +991,9 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,p_n, p_x, p_desc, p_rec, p_sub, p_wait, p_y);}g_vk_matmul_calls++;+ g_vk_gemm_calls += path != 0;+ g_vk_coop_calls += path == 2;+ if (g_vk_prof) { g_vk_path_ms[path] += vk_now() - tp; g_vk_path_n[path]++; }return 1;}@@ -1638,6 +1950,10 @@ void coli_vk_shutdown(void) {vkDestroyCommandPool(G.dev, G.cpool, NULL);vkDestroyDescriptorPool(G.dev, G.dpool, NULL);vkDestroyPipeline(G.dev, G.pipe, NULL);+ for (int k = 0; k < VK_GEMM_SLOTS; k++) if (G.pipe_gemm[k]) vkDestroyPipeline(G.dev, G.pipe_gemm[k], NULL);+ for (int k = 0; k < VK_COOP_SLOTS; k++) if (G.pipe_coop[k]) vkDestroyPipeline(G.dev, G.pipe_coop[k], NULL);+ if (G.shader_gemm) vkDestroyShaderModule(G.dev, G.shader_gemm, NULL);+ if (G.shader_coop) vkDestroyShaderModule(G.dev, G.shader_coop, NULL);vkDestroyPipelineLayout(G.dev, G.plyt, NULL);vkDestroyDescriptorSetLayout(G.dev, G.dsl, NULL);vkDestroyShaderModule(G.dev, G.shader, NULL);@@ -1682,7 +1998,7 @@ static size_t ref_rowbytes(int fmt, int I) {static size_t ref_scales(int fmt, int I, int O) { // scale COUNT (per-group for fmt 4/5)if (fmt == 10 || fmt == 11) return 1;if (fmt == 5) return (size_t)O * (size_t)((I + 63) / 64);- if (fmt == 4 || fmt == 12) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);+ if (fmt == 4 || fmt == 7 || fmt == 12) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);return (size_t)O;}static float deq(const uint8_t *row, int fmt, int i) {@@ -1698,16 +2014,19 @@ static float deq(const uint8_t *row, int fmt, int i) {const uint8_t *lo = row + (size_t)(i >> 6) * 24, *hi = lo + 16; int j = i & 63;unsigned u = ((lo[j >> 2] >> ((j & 3) * 2)) & 3u) | (((hi[j >> 3] >> (j & 7)) & 1u) << 2);return (float)((int)u - 4); }- uint8_t v = row[i >> 1]; int nib = (i & 1) ? (v >> 4) : (v & 15); return (float)(nib - 8);+ uint8_t v = row[i >> 1]; int nib = (i & 1) ? (v >> 4) : (v & 15);+ if (fmt == 7) { static const float e2m1[8] = {0, 0.5f, 1, 1.5f, 2, 3, 4, 6}; /* MXFP4 */+ return (nib & 8) ? -e2m1[nib & 7] : e2m1[nib & 7]; }+ return (float)(nib - 8);}static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,int fmt, int S, int I, int O) {size_t rb = ref_rowbytes(fmt, I);- int gw2 = (fmt == 4 || fmt == 12) ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;+ int gw2 = (fmt == 4 || fmt == 7 || fmt == 12) ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {double sum = 0; const uint8_t *row = w + (size_t)o * rb;- if (fmt == 5 || fmt == 4 || fmt == 12) { // per-group scales fold inside the sum+ if (fmt == 5 || fmt == 4 || fmt == 7 || fmt == 12) { // per-group scales fold inside the sumfor (int g = 0; g < ng; g++) {double a = 0; int end = (g + 1) * gw2 < I ? (g + 1) * gw2 : I;for (int i = g * gw2; i < end; i++) a += x[s * I + i] * deq(row, fmt, i);@@ -1739,28 +2058,36 @@ static double ref_dot(const float *x, const uint8_t *row, const float *scb, intreturn sum * scb[o];}-static int run_case(int fmt, int S, int I, int O, int iters) {+/* Random activations, weights and scales for one case: x[S,I] in [-1,1), raw weight+ * bytes (no NaN/Inf for the float and fp8 formats), scales in [0.01,0.02). */+static void case_fill(int fmt, int S, int I, int O, float **x, uint8_t **w, float **sc) {size_t rb = ref_rowbytes(fmt, I), nsc = ref_scales(fmt, I, O);- float *x = malloc((size_t)S * I * sizeof(float));- uint8_t *w = malloc(rb * O);- float *sc = malloc(nsc * sizeof(float));- float *yg = malloc((size_t)S * O * sizeof(float));- float *yc = malloc((size_t)S * O * sizeof(float));- for (int i = 0; i < S * I; i++) x[i] = (float)((rand() % 200 - 100) / 100.0);- for (size_t i = 0; i < rb * O; i++) w[i] = rand() & 0xff;+ *x = malloc((size_t)S * I * sizeof(float));+ *w = malloc(rb * O);+ *sc = malloc(nsc * sizeof(float));+ for (size_t i = 0; i < (size_t)S * I; i++) (*x)[i] = (float)((rand() % 200 - 100) / 100.0);+ for (size_t i = 0; i < rb * O; i++) (*w)[i] = rand() & 0xff;if (fmt == 12) /* no NaN bytes: they would poison the comparison */- for (size_t i = 0; i < rb * O; i++) if ((w[i] & 0x7f) == 0x7f) w[i] ^= 1;+ for (size_t i = 0; i < rb * O; i++) if (((*w)[i] & 0x7f) == 0x7f) (*w)[i] ^= 1;if (fmt == 10 || fmt == 11) /* float weights: random bytes could be NaN/Inf */for (size_t i = 0; i < (size_t)I * O; i++) {float f = (float)((rand() % 2001 - 1000) / 1000.0);- if (fmt == 10) memcpy(w + i * 4, &f, 4);- else { uint32_t u; memcpy(&u, &f, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(w + i * 2, &h, 2); }+ if (fmt == 10) memcpy(*w + i * 4, &f, 4);+ else { uint32_t u; memcpy(&u, &f, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(*w + i * 2, &h, 2); }}- for (size_t o = 0; o < nsc; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;+ for (size_t o = 0; o < nsc; o++) (*sc)[o] = 0.01f + (rand() % 100) / 10000.0f;+}++static int run_case(int fmt, int S, int I, int O, int iters) {+ float *x, *sc; uint8_t *w;+ case_fill(fmt, S, I, O, &x, &w, &sc);+ float *yg = malloc((size_t)S * O * sizeof(float));+ float *yc = malloc((size_t)S * O * sizeof(float));ColiVkTensor *t = NULL;if (!coli_vk_matmul(&t, yg, x, w, sc, fmt, S, I, O, g_ref_gs)) { printf("matmul failed\n"); return 1; }- cpu_ref(yc, x, w, sc, fmt, S, I, O);+ const char *path = G.bound_gemm == 2 ? "coop" : G.bound_gemm ? "gemm" : "gemv";+ double c0 = now(); cpu_ref(yc, x, w, sc, fmt, S, I, O); double cpu_ms = (now() - c0) * 1000;double maxerr = 0, maxrel = 0;for (int i = 0; i < S * O; i++) {double e = fabs(yg[i] - yc[i]); if (e > maxerr) maxerr = e;@@ -1770,15 +2097,114 @@ static int run_case(int fmt, int S, int I, int O, int iters) {double t0 = now();for (int k = 0; k < iters; k++) coli_vk_matmul(&t, yg, x, w, sc, fmt, S, I, O, g_ref_gs);double gpu_ms = (now() - t0) * 1000 / iters;- // microbench (CPU ref, 1 iter β it's slow)- double c0 = now(); cpu_ref(yc, x, w, sc, fmt, S, I, O); double cpu_ms = (now() - c0) * 1000;- printf("fmt=%d S=%d I=%d O=%d | maxerr=%.4g maxrel=%.4g | gpu=%.3f ms cpu_ref=%.3f ms\n",- fmt, S, I, O, maxerr, maxrel, gpu_ms, cpu_ms);+ char gsb[16] = "";+ if (fmt == 4 || fmt == 7 || fmt == 12) snprintf(gsb, sizeof(gsb), " gs=%d", g_ref_gs);+ printf("fmt=%d%s S=%d I=%d O=%d %s | maxerr=%.4g maxrel=%.4g | gpu=%.3f ms cpu_ref=%.3f ms\n",+ fmt, gsb, S, I, O, path, maxerr, maxrel, gpu_ms, cpu_ms);coli_vk_tensor_free(t);free(x); free(w); free(sc); free(yg); free(yc);return maxrel > 1e-3 ? 1 : 0;}+/* The tiled GEMMs against the same reference, forced for this S whatever the device+ * threshold, and required to have really run: a missing qmatmul_gemm.spv must fail+ * here, not pass on the GEMV. The fp32 GEMM always; then, on a device with the+ * cooperative-matrix pipeline, that one too for the formats it takes. gs is the group+ * size of fmt 4, 7 and 12. */+static int coop_takes(int fmt, int gs) {+ ColiVkTensor t = {.fmt = fmt, .gs = gs};+ return G.pipe_coop[0] && coop_fmt(&t);+}+static int run_gemm_case(int fmt, int S, int I, int O, int gs) {+ if (!G.pipe_gemm[0]) { printf("gemm fmt=%d: no tiled GEMM pipeline (qmatmul_gemm.spv missing?)\n", fmt); return 1; }+ int keep_s = G.gemm_min_s, keep_so = G.gemm_min_so, keep_gs = g_ref_gs, bad = 0;+ G.gemm_min_s = 2; G.gemm_min_so = 0; g_ref_gs = gs;+ G.coop_off = 1; /* the fp32 GEMM */+ unsigned long long n0 = g_vk_gemm_calls;+ bad |= run_case(fmt, S, I, O, 2);+ if (g_vk_gemm_calls == n0) { printf(" ^ did not take the tiled GEMM\n"); bad = 1; }+ G.coop_off = 0;+ if (coop_takes(fmt, gs)) { /* the cooperative-matrix one */+ n0 = g_vk_coop_calls;+ bad |= run_case(fmt, S, I, O, 2);+ if (g_vk_coop_calls == n0) { printf(" ^ did not take the cooperative-matrix GEMM\n"); bad = 1; }+ }+ G.gemm_min_s = keep_s; G.gemm_min_so = keep_so; g_ref_gs = keep_gs;+ return bad;+}++/* Seconds per coli_vk_matmul call with the GEMM threshold at min_s (0 = GEMV): x upload,+ * dispatch and y readback, what an engine pays; and, in *gpu, per resubmit of the+ * recorded command buffer, the dispatch alone plus the submit roundtrip. */+static double time_matmul(ColiVkTensor **t, float *y, const float *x, const void *w, const float *sc,+ int fmt, int S, int I, int O, int gs, int min_s, double *gpu) {+ int keep = G.gemm_min_s, keep_so = G.gemm_min_so; G.gemm_min_s = min_s; G.gemm_min_so = 0;+ coli_vk_matmul(t, y, x, w, sc, fmt, S, I, O, gs); /* record + warm */+ int it = 0; double t0 = now(), el;+ do { coli_vk_matmul(t, y, x, w, sc, fmt, S, I, O, gs); it++; } while ((el = now() - t0) < 0.25 && it < 500);+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};+ int ig = 0; double g0 = now(), gl;+ do { vkResetFences(G.dev, 1, &G.fence); vkQueueSubmit(G.queue, 1, &si, G.fence);+ vk_fence_wait_gemm(G.dev, G.fence); ig++;+ } while ((gl = now() - g0) < 0.25 && ig < 500);+ G.gemm_min_s = keep; G.gemm_min_so = keep_so;+ *gpu = gl / ig;+ return el / it;+}++/* COLI_VK_TEST_GEMM_BENCH=1: GEMV against the tiled GEMM, format by format, over S on+ * one engine-sized matrix (COLI_VK_TEST_GEMM_SHAPE=I,O; default 2560,6144, Qwen3.8's+ * DeltaNet z projection). GFLOP/s per call and per resubmit; the S where the GEMM+ * overtakes is what COLI_VK_GEMM_MIN_S defaults to. COLI_VK_TEST_GEMM_FMT=a,b,... and+ * COLI_VK_TEST_GEMM_S=a,b,... restrict the formats and the S values. Runs instead of+ * every other case. */+static void bench_gemm(void) {+ int I = 2560, O = 6144, fm[8] = {1, 2, 4, 5, 7, 10, 11, 12}, nf = 8;+ const char *e = getenv("COLI_VK_TEST_GEMM_SHAPE");+ if (e) sscanf(e, "%d,%d", &I, &O);+ if ((e = getenv("COLI_VK_TEST_GEMM_FMT"))) {+ nf = 0;+ for (const char *c = e; *c && nf < 8; ) { fm[nf++] = atoi(c); while (*c && *c != ',') c++; if (*c) c++; }+ }+ int Ss[16] = {2, 4, 8, 16, 32, 64, 128, 256, 512}, ns = 9, Smax = 0;+ if ((e = getenv("COLI_VK_TEST_GEMM_S"))) { /* the S values, a,b,... */+ ns = 0;+ for (const char *c = e; *c && ns < 16; ) { Ss[ns++] = atoi(c); while (*c && *c != ',') c++; if (*c) c++; }+ }+ for (int k = 0; k < ns; k++) if (Ss[k] > Smax) Smax = Ss[k];+ printf("GEMM bench I=%d O=%d, threshold now S>=%d and S*O>=%d; GEMM tiles", I, O, G.gemm_min_s, G.gemm_min_so);+ for (int k = 0; k < VK_GEMM_SLOTS; k++) if (G.pipe_gemm[k]) printf(" %dx%d", G.gemm_t[k].bm, G.gemm_t[k].bn);+ printf("; COOP tiles (subgroup %d)", G.coop_sg);+ for (int k = 0; k < VK_COOP_SLOTS; k++) if (G.pipe_coop[k]) printf(" %dx%d", G.coop_t[k].bm, G.coop_t[k].bn);+ printf("\n");+ for (int f = 0; f < nf; f++) {+ int fmt = fm[f];+ g_ref_gs = fmt == 7 ? 32 : 64;+ float *x, *sc; uint8_t *w;+ case_fill(fmt, Smax, I, O, &x, &w, &sc);+ float *y = malloc((size_t)Smax * O * sizeof(float));+ ColiVkTensor *t = NULL;+ if (!coli_vk_matmul(&t, y, x, w, sc, fmt, 1, I, O, g_ref_gs)) { printf("fmt=%d: upload failed\n", fmt); continue; }+ for (int k = 0; k < ns; k++) {+ int S = Ss[k];+ double gv, gm, gvg, gmg, cm = 0, cmg = 0;+ gv = time_matmul(&t, y, x, w, sc, fmt, S, I, O, g_ref_gs, 0, &gvg);+ G.coop_off = 1;+ gm = time_matmul(&t, y, x, w, sc, fmt, S, I, O, g_ref_gs, 2, &gmg);+ G.coop_off = 0;+ if (coop_takes(fmt, g_ref_gs)) cm = time_matmul(&t, y, x, w, sc, fmt, S, I, O, g_ref_gs, 2, &cmg);+ double fl = 2.0 * S * I * O / 1e9;+ printf("fmt=%-2d S=%-4d | GEMV %8.3f ms %7.1f GFLOP/s (gpu %8.3f) | GEMM %8.3f ms %7.1f GFLOP/s (gpu %8.3f)",+ fmt, S, gv * 1e3, fl / gv, gvg * 1e3, gm * 1e3, fl / gm, gmg * 1e3);+ if (cm > 0) printf(" | COOP %8.3f ms %7.1f GFLOP/s (gpu %8.3f)", cm * 1e3, fl / cm, cmg * 1e3);+ printf("\n");+ }+ coli_vk_tensor_free(t);+ free(x); free(w); free(sc); free(y);+ }+ g_ref_gs = 64;+}+/* Batched throughput: record N dispatches in ONE command buffer, one submit + one* fence wait β the amortized per-matmul cost with the submit roundtrip spread across* the batch (how the real expert tier would drive it). Reuses the descriptor binding@@ -2149,6 +2575,12 @@ int main(int argc, char **argv) {}if (nb) printf("ballast: %d x 4 MB idle allocations\n", nb);}+ if (getenv("COLI_VK_TEST_GEMM_BENCH") && atoi(getenv("COLI_VK_TEST_GEMM_BENCH"))) {+ bench_gemm();+ coli_vk_shutdown();+ return 0;+ }+ G.gemm_min_s = 0; /* every case below is the GEMV's, but for run_gemm_case's */bad |= run_case(1, 1, 6144, 1536, 50); // int8 expert gate/up shape (S=1 decode)bad |= run_case(2, 1, 6144, 1536, 50); // int4 expertbad |= run_case(1, 1, 1536, 6144, 50); // down proj shape@@ -2178,6 +2610,23 @@ int main(int argc, char **argv) {g_ref_gs = 32; bad |= run_case(12, 2, 4096, 256, 5);g_ref_gs = 128; bad |= run_case(12, 2, 4096, 256, 5);g_ref_gs = 64;+ /* Tiled GEMM (prefill-sized S), every format: S = 16, 64 and 512 against odd O and+ * odd or unaligned I, so the last output, row and input tiles are all partial, the+ * last group is a tail, and x rows come both on and off a 16-byte quad. */+ {+ static const int gf[8] = {1, 2, 4, 5, 7, 10, 11, 12};+ for (int f = 0; f < 8; f++) {+ int gs = gf[f] == 7 ? 32 : 64;+ bad |= run_gemm_case(gf[f], 16, 1001, 193, gs); // odd I: tail group, x off a quad+ bad |= run_gemm_case(gf[f], 64, 1536, 97, gs); // I % 4 == 0: x by quads+ bad |= run_gemm_case(gf[f], 512, 520, 131, gs); // 8-input tail group+ }+ bad |= run_gemm_case(1, 37, 999, 65, 0); // odd S, a 1-output last tile+ bad |= run_gemm_case(4, 33, 1001, 70, 8); // the smallest grouped-int4 group+ bad |= run_gemm_case(4, 33, 1001, 70, 24); // a group that is no power of two+ bad |= run_gemm_case(12, 33, 1001, 70, 12); // fp8: the two words of a unit in two groups+ bad |= run_gemm_case(12, 33, 1001, 70, 128);+ }/* COLI_VK_TEST_MATMUL_ONLY=1: stop after the per-format matmul cases above. CI runs* this on Lavapipe, where the benches below say nothing and take most of the time. */if (getenv("COLI_VK_TEST_MATMUL_ONLY") && atoi(getenv("COLI_VK_TEST_MATMUL_ONLY"))) {diff --git a/c/shaders/qmatmul_coop.comp b/c/shaders/qmatmul_coop.compnew file mode 100644index 00000000..bcae0690--- /dev/null+++ b/c/shaders/qmatmul_coop.comp@@ -0,0 +1,247 @@+#version 450+// Cooperative-matrix twin of qmatmul_gemm.comp (VK_KHR_cooperative_matrix, 16x16x16+// fp16 x fp16 -> fp32 per subgroup): the same y[s,o] = sum_i x[s,i] * dequant(W[o,i]),+// for the formats whose decoded weights are fp16 values: 1, 2, 4, 5, 7 and 12, the+// grouped ones with gs % BK == 0 (backend_vulkan.c routes the rest to the fp32 GEMM).+// Nothing is rounded to fp16, and the result stays as close to fp32 as the fp32+// GEMM's (the VK_TEST harness holds both to the same bound):+// - every decoded weight (int8, int4, int3, e2m1, e4m3) is exact in fp16, so the A+// operand carries no rounding; group and row scales stay fp32, outside the MMA;+// - each activation row arrives with a power of two rs[s] (written by the host after+// the S*I floats of x) that puts its largest |x| under 2^15; the scaled row is+// split into hi (its top 11 significant bits) and lo = x - hi, both exact in fp16+// but for lo's last bits, and the MMA runs on both (A*hi + A*lo);+// - every 16-input MMA product joins an fp32 accumulator with an fp32 add, times the+// group scale for the grouped formats.+// A workgroup owns a BM-output x BN-row tile of y, each subgroup WM x WN of it in+// 16 x 16 accumulators. Weights and activations are staged BK inputs at a time, the+// next step's words fetched into registers before this step's MMAs.+#extension GL_KHR_cooperative_matrix : require+#extension GL_KHR_memory_scope_semantics : require+#extension GL_KHR_shader_subgroup_basic : require+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require+#extension GL_EXT_control_flow_attributes : require++layout(constant_id = 0) const int BM = 64; // outputs per tile+layout(constant_id = 1) const int BN = 64; // activation rows per tile+layout(constant_id = 2) const int WM = 32; // outputs per subgroup, a multiple of 16+layout(constant_id = 3) const int WN = 32; // rows per subgroup, a multiple of 16+layout(constant_id = 4) const int BK = 32; // inputs per staged step, a multiple of 32+layout(constant_id = 5) const int SG = 64; // subgroup size the pipeline requires+layout(local_size_x_id = 6) in; // (BM/WM) * (BN/WN) * SG threads++layout(std430, binding = 0) readonly buffer X { float x[]; }; // x[S*I], then rs[S]+layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; };+layout(std430, binding = 1) readonly buffer W { uint w[]; };+layout(std430, binding = 1) readonly buffer W2 { uvec2 w2[]; };+layout(std430, binding = 2) readonly buffer Sc { float scale[]; };+layout(std430, binding = 3) writeonly buffer Y { float y[]; };++layout(push_constant) uniform P {+ int fmt; // 1, 2, 4, 5, 7, 12+ int S, I, O;+ int rowWords; // uint32 words per weight row+ int gs; // group size of fmt 4, 7 and 12 (a multiple of 32 here)+} p;++const int FM = WM / 16, FN = WN / 16; // 16 x 16 accumulators per subgroup+const int NW = (BM / WM) * (BN / WN); // subgroups+const int NT = NW * SG;+const int WST = BK + 8; // halves per staged row, 16 bytes of padding+const int EST = 17; // floats per epilogue row: odd, no bank conflicts+shared float16_t wsh[BM * WST]; // [o][i]: the MMA's A, row-major+shared float16_t xhs[BN * WST]; // [s][i]: its B, column-major; hi part+shared float16_t xls[BN * WST]; // lo part+shared float ssh[BM]; // the current group's scales+shared float esh[NW * 16 * EST]; // per subgroup: one accumulator on its way to y++#define ACC coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>+#define MATA coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA>+#define MATB coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB>++float i8(uint word, int lane) {+ int b = int((word >> (uint(lane) * 8u)) & 0xffu);+ if (b >= 128) b -= 256;+ return float(b);+}+float i4(uint word, int lane) {+ return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);+}+float mx4(uint word, int lane) { // e2m1, bit3 = sign (qmatmul.comp)+ uint n = (word >> (uint(lane) * 4u)) & 0xfu;+ const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);+ float v = lut[n & 7u];+ return (n & 8u) != 0u ? -v : v;+}+float e4m3(uint word, int lane) { // OCP E4M3-FN, arithmetic decode (qmatmul.comp)+ uint b = (word >> (uint(lane) * 8u)) & 0xffu;+ if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);+ uint e = (b >> 3) & 0xfu, m = b & 7u;+ float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);+ return (b & 0x80u) != 0u ? -v : v;+}++// The words of weights k..k+7 of row o (k % 8 == 0, k < I), as qmatmul_gemm.comp's+// fetch8 for these formats; no scale: it multiplies the MMA products instead.+uvec2 fetch8(int o, int k) {+ uint rb = uint(o) * uint(p.rowWords);+ uvec2 r = uvec2(0u);+ if (p.fmt == 1 || p.fmt == 12) { // 4 bytes per word+ if ((p.rowWords & 1) == 0) r = w2[(rb >> 1) + uint(k >> 3)];+ else { r.x = w[rb + uint(k >> 2)]; if (k + 4 < p.I) r.y = w[rb + uint(k >> 2) + 1u]; }+ if (k + 4 >= p.I) r.y = 0u;+ } else if (p.fmt == 5) { // int3-g64: 16B low plane + 8B high plane+ int g = k >> 6, j = k & 63;+ r.x = w[rb + uint(g * 6 + (j >> 4))] >> uint((j & 15) * 2);+ r.y = w[rb + uint(g * 6 + 4 + (j >> 5))] >> uint(j & 31);+ } else r.x = w[rb + uint(k >> 3)]; // nibbles: 2, 4, 7+ return r;+}+void decode8(uvec2 a, out vec4 lo, out vec4 hi) {+ if (p.fmt == 1) {+ lo = vec4(i8(a.x, 0), i8(a.x, 1), i8(a.x, 2), i8(a.x, 3));+ hi = vec4(i8(a.y, 0), i8(a.y, 1), i8(a.y, 2), i8(a.y, 3));+ } else if (p.fmt == 12) {+ lo = vec4(e4m3(a.x, 0), e4m3(a.x, 1), e4m3(a.x, 2), e4m3(a.x, 3));+ hi = vec4(e4m3(a.y, 0), e4m3(a.y, 1), e4m3(a.y, 2), e4m3(a.y, 3));+ } else if (p.fmt == 5) {+ float v[8];+ [[unroll]] for (int t = 0; t < 8; t++)+ v[t] = float(int(((a.x >> uint(2 * t)) & 3u) | (((a.y >> uint(t)) & 1u) << 2)) - 4);+ lo = vec4(v[0], v[1], v[2], v[3]); hi = vec4(v[4], v[5], v[6], v[7]);+ } else if (p.fmt == 7) {+ lo = vec4(mx4(a.x, 0), mx4(a.x, 1), mx4(a.x, 2), mx4(a.x, 3));+ hi = vec4(mx4(a.x, 4), mx4(a.x, 5), mx4(a.x, 6), mx4(a.x, 7));+ } else {+ lo = vec4(i4(a.x, 0), i4(a.x, 1), i4(a.x, 2), i4(a.x, 3));+ hi = vec4(i4(a.x, 4), i4(a.x, 5), i4(a.x, 6), i4(a.x, 7));+ }+}++const int WU = (BM * (BK / 8) + NT - 1) / NT; // weight units per thread and step+const int XU = (BN * (BK / 4) + NT - 1) / NT; // activation quads per thread and step+uvec2 rw[WU]; vec4 rx[XU];++// Fetch step k0 into registers: zeros past O and I (finite after decode) and past S.+void fetch_step(int tid, int o0, int s0, int k0) {+ [[unroll]] for (int r = 0; r < WU; r++) {+ int u = tid + r * NT, m = u / (BK / 8), k = k0 + (u % (BK / 8)) * 8;+ rw[r] = uvec2(0u);+ if (u < BM * (BK / 8) && o0 + m < p.O && k < p.I) rw[r] = fetch8(o0 + m, k);+ }+ [[unroll]] for (int r = 0; r < XU; r++) {+ int u = tid + r * NT, s = s0 + u / (BK / 4), k = k0 + (u % (BK / 4)) * 4;+ vec4 v = vec4(0.0);+ if (u < BN * (BK / 4) && s < p.S && k < p.I) {+ uint b = uint(s) * uint(p.I) + uint(k);+ if ((p.I & 3) == 0) v = x4[b >> 2];+ else {+ v.x = x[b];+ if (k + 1 < p.I) v.y = x[b + 1u];+ if (k + 2 < p.I) v.z = x[b + 2u];+ if (k + 3 < p.I) v.w = x[b + 3u];+ }+ v *= x[uint(p.S) * uint(p.I) + uint(s)]; // the row's power of two+ }+ rx[r] = v;+ }+}++void main() {+ int tid = int(gl_LocalInvocationID.x);+ int sg = int(gl_SubgroupID), lane = int(gl_SubgroupInvocationID);+ int wm = sg % (BM / WM), wn = sg / (BM / WM); // this subgroup's WM x WN+ int o0 = int(gl_WorkGroupID.x) * BM, s0 = int(gl_WorkGroupID.y) * BN;+ bool grouped = p.fmt != 1 && p.fmt != 2;+ int gsz = p.fmt == 5 ? 64 : p.gs; // a multiple of BK+ int ng = grouped ? (p.I + gsz - 1) / gsz : 1;++ ACC acc[FM][FN]; // fp32 sums; one declarator per line in this file: glslang drops+ // the matrix use from the second declarator of a list+ [[unroll]] for (int i = 0; i < FM; i++)+ [[unroll]] for (int j = 0; j < FN; j++) acc[i][j] = ACC(0.0);++ fetch_step(tid, o0, s0, 0);+ for (int k0 = 0; k0 < p.I; k0 += BK) {+ // the fetched step into shared memory: weights as fp16, x as hi and lo+ [[unroll]] for (int r = 0; r < WU; r++) {+ int u = tid + r * NT;+ if (u < BM * (BK / 8)) {+ int base = (u / (BK / 8)) * WST + (u % (BK / 8)) * 8;+ vec4 lo, hi;+ decode8(rw[r], lo, hi);+ f16vec4 a = f16vec4(lo), b = f16vec4(hi);+ [[unroll]] for (int t = 0; t < 4; t++) { wsh[base + t] = a[t]; wsh[base + 4 + t] = b[t]; }+ }+ }+ [[unroll]] for (int r = 0; r < XU; r++) {+ int u = tid + r * NT;+ if (u < BN * (BK / 4)) {+ int base = (u / (BK / 4)) * WST + (u % (BK / 4)) * 4;+ // hi keeps the top 11 significant bits by masking, not by an fp16 round+ // trip: the compiler folds vec4(f16vec4(v)) back to v, which zeroed lo+ // (measured on RADV: fp16-level error, 1000x the fp32 GEMM's). Both+ // conversions below are then exact, but for lo's last bits.+ vec4 h32 = uintBitsToFloat(floatBitsToUint(rx[r]) & 0xffffe000u);+ f16vec4 hi = f16vec4(h32), lo = f16vec4(rx[r] - h32);+ [[unroll]] for (int t = 0; t < 4; t++) { xhs[base + t] = hi[t]; xls[base + t] = lo[t]; }+ }+ }+ // a new group starts at this step: its scales+ if (grouped && k0 % gsz == 0)+ for (int u = tid; u < BM; u += NT)+ ssh[u] = o0 + u < p.O ? scale[uint(o0 + u) * uint(ng) + uint(k0 / gsz)] : 0.0;+ barrier();+ if (k0 + BK < p.I) fetch_step(tid, o0, s0, k0 + BK);+ ACC sm[FM]; // grouped formats: a 16 x 16 tile whose row m is output m's+ if (grouped) // scale, a column-major load with stride 0+ [[unroll]] for (int i = 0; i < FM; i++)+ coopMatLoad(sm[i], ssh, uint(wm * WM + i * 16), 0u, gl_CooperativeMatrixLayoutColumnMajor);+ [[unroll]] for (int kk = 0; kk < BK; kk += 16) {+ if (k0 + kk < p.I) {+ MATA a[FM];+ MATB bh[FN];+ MATB bl[FN];+ [[unroll]] for (int i = 0; i < FM; i++)+ coopMatLoad(a[i], wsh, uint((wm * WM + i * 16) * WST + kk), uint(WST),+ gl_CooperativeMatrixLayoutRowMajor);+ [[unroll]] for (int j = 0; j < FN; j++) {+ coopMatLoad(bh[j], xhs, uint((wn * WN + j * 16) * WST + kk), uint(WST),+ gl_CooperativeMatrixLayoutColumnMajor);+ coopMatLoad(bl[j], xls, uint((wn * WN + j * 16) * WST + kk), uint(WST),+ gl_CooperativeMatrixLayoutColumnMajor);+ }+ // Each 16-input product joins acc with an fp32 add (times the group scale+ // for the grouped formats): the MMA accumulates with less than fp32's+ // rounding, measured at 10x the fp32 GEMM's error when it summed 512+ // inputs before a flush.+ [[unroll]] for (int i = 0; i < FM; i++)+ [[unroll]] for (int j = 0; j < FN; j++) {+ ACC c = coopMatMulAdd(a[i], bh[j], ACC(0.0));+ c = coopMatMulAdd(a[i], bl[j], c);+ acc[i][j] = grouped ? acc[i][j] + c * sm[i] : acc[i][j] + c;+ }+ }+ }+ barrier();+ }++ // Each accumulator through this subgroup's 16 x 17 tile: lane l takes output row+ // l % 16 and every (SG/16)-th row of x from l / 16, times the per-row formats' scale+ // and 1/rs[s], stores masked to O and S.+ int eb = sg * 16 * EST, m = lane % 16;+ [[unroll]] for (int i = 0; i < FM; i++) {+ int o = o0 + wm * WM + i * 16 + m;+ float sc = grouped || o >= p.O ? 1.0 : scale[o];+ [[unroll]] for (int j = 0; j < FN; j++) {+ coopMatStore(acc[i][j], esh, uint(eb), uint(EST), gl_CooperativeMatrixLayoutRowMajor);+ subgroupMemoryBarrierShared(); subgroupBarrier();+ [[unroll]] for (int t = 0; t < 256 / SG; t++) {+ int n = lane / 16 + t * (SG / 16), s = s0 + wn * WN + j * 16 + n;+ if (o < p.O && s < p.S)+ y[uint(s) * uint(p.O) + uint(o)] = esh[eb + m * EST + n] * sc / x[uint(p.S) * uint(p.I) + uint(s)];+ }+ subgroupBarrier();+ }+ }+}diff --git a/c/shaders/qmatmul_coop.spv b/c/shaders/qmatmul_coop.spvnew file mode 100644index 00000000..8685197aBinary files /dev/null and b/c/shaders/qmatmul_coop.spv differdiff --git a/c/shaders/qmatmul_gemm.comp b/c/shaders/qmatmul_gemm.compnew file mode 100644index 00000000..4d824926--- /dev/null+++ b/c/shaders/qmatmul_gemm.comp@@ -0,0 +1,234 @@+#version 450+// Tiled GEMM twin of qmatmul.comp, for prefill-sized S:+// y[s,o] = sum_i x[s,i] * dequant(W[o,i]) (same fmt, layout and scales)+// qmatmul.comp runs one GEMV per activation row, so every weight is fetched and+// decoded S times. Here a workgroup owns a BM-output x BN-row tile of y and walks i+// in BK-wide steps: each step decodes its BM weight rows into shared memory ONCE+// and multiplies them against BN activation rows, staged in shared memory too, so a+// weight is fetched ceil(S/BN) times and decoded once per tile.+// - both tiles keep i contiguous, wsh[m][i] and xsh[n][i], as vec4 quads with one+// quad of padding per row: the vec4 reads of 8 consecutive rows land on+// distinct banks, and the staging writes stay plain quad stores;+// - each thread accumulates a TM x TN block whose rows are strided by BM/TM and+// BN/TN, so neighbouring threads read neighbouring rows and store neighbouring+// outputs (coalesced y writes);+// - the grouped formats (4, 5, 7, 12) fold their group scale into the weight as it+// enters shared memory, which frees the tile from group boundaries; the per-row+// formats (1, 2) multiply scale[o] at the end, as the GEMV does.+// The tile shape is a set of specialization constants; backend_vulkan.c picks it, and+// whether the next step's words are fetched during this step's products (PF): faster+// for the narrow BN = 32 tile on the 780M, slower for the 64 x 64 one (registers).+#extension GL_EXT_control_flow_attributes : require++layout(constant_id = 0) const int BM = 64; // outputs per tile+layout(constant_id = 1) const int BN = 64; // activation rows per tile+layout(constant_id = 2) const int BK = 32; // inputs per step, a multiple of 8+layout(constant_id = 3) const int TM = 4; // outputs per thread+layout(constant_id = 4) const int TN = 4; // activation rows per thread+layout(local_size_x_id = 5) in; // (BM/TM) * (BN/TN) threads+layout(constant_id = 6) const bool PF = true; // fetch the next step during the products++layout(std430, binding = 0) readonly buffer X { float x[]; };+layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; }; // same buffer, quads+layout(std430, binding = 1) readonly buffer W { uint w[]; }; // row o at o*rowWords+layout(std430, binding = 1) readonly buffer W2 { uvec2 w2[]; }; // same buffer, pairs+layout(std430, binding = 1) readonly buffer W4 { uvec4 w4[]; }; // same buffer, quads+layout(std430, binding = 2) readonly buffer Sc { float scale[]; };+layout(std430, binding = 3) writeonly buffer Y { float y[]; };++layout(push_constant) uniform P {+ int fmt; // as qmatmul.comp: 1, 2, 4, 5, 7, 10, 11, 12+ int S, I, O;+ int rowWords; // uint32 words per weight row+ int gs; // group size of fmt 4/7 (multiple of 8) and 12 (multiple of 4)+} p;++const int KQ = BK / 4 + 1; // quads per shared row, one of padding+const int NTM = BM / TM, NTN = BN / TN; // thread grid+shared vec4 wsh[BM * KQ];+shared vec4 xsh[BN * KQ];++float i8(uint word, int lane) {+ int b = int((word >> (uint(lane) * 8u)) & 0xffu);+ if (b >= 128) b -= 256;+ return float(b);+}+float i4(uint word, int lane) {+ return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);+}+float mx4(uint word, int lane) { // e2m1, bit3 = sign (qmatmul.comp)+ uint n = (word >> (uint(lane) * 4u)) & 0xfu;+ const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);+ float v = lut[n & 7u];+ return (n & 8u) != 0u ? -v : v;+}+float e4m3(uint word, int lane) { // OCP E4M3-FN, arithmetic decode (qmatmul.comp)+ uint b = (word >> (uint(lane) * 8u)) & 0xffu;+ if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);+ uint e = (b >> 3) & 0xfu, m = b & 7u;+ float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);+ return (b & 0x80u) != 0u ? -v : v;+}++// One staging unit is 8 consecutive weights of a row. A step's units are fetched into+// registers (with PF, before the previous step's products run, the loads in flight+// during them) and decoded into shared memory after: fetch8 reads the words and group+// scales, decode8 turns them into floats.+//+// fetch8: the words of weights k..k+7 of row o (k % 8 == 0, k < I) and their scales.+// Words past the row's last input are never read: they belong to the next row, or+// lie past the buffer for the last one. A lane past I decodes the row's zero padding+// to a finite value and meets a zero activation, so it adds nothing.+void fetch8(int o, int k, out uvec4 lo, out uvec4 hi, out vec2 sc) {+ uint rb = uint(o) * uint(p.rowWords);+ lo = uvec4(0u); hi = uvec4(0u); sc = vec2(0.0);+ if (p.fmt == 1 || p.fmt == 12) { // 4 bytes per word+ if ((p.rowWords & 1) == 0) lo.xy = w2[(rb >> 1) + uint(k >> 3)];+ else { lo.x = w[rb + uint(k >> 2)]; if (k + 4 < p.I) lo.y = w[rb + uint(k >> 2) + 1u]; }+ if (k + 4 >= p.I) lo.y = 0u;+ if (p.fmt == 12) { // gs % 4 == 0: a word never straddles+ uint sb = uint(o) * uint((p.I + p.gs - 1) / p.gs);+ sc.x = scale[sb + uint(k / p.gs)];+ if (k + 4 < p.I) sc.y = scale[sb + uint((k + 4) / p.gs)];+ }+ } else if (p.fmt == 10) { // f32, one per word+ if ((p.I & 7) == 0) { lo = w4[(rb + uint(k)) >> 2]; hi = w4[((rb + uint(k)) >> 2) + 1u]; }+ else [[unroll]] for (int j = 0; j < 4; j++) {+ if (k + j < p.I) lo[j] = w[rb + uint(k + j)];+ if (k + 4 + j < p.I) hi[j] = w[rb + uint(k + 4 + j)];+ }+ } else if (p.fmt == 11) { // bf16, two per word+ if ((p.I & 7) == 0) lo = w4[(rb + uint(k >> 1)) >> 2];+ else [[unroll]] for (int j = 0; j < 4; j++)+ if (k + 2 * j < p.I) lo[j] = w[rb + uint((k >> 1) + j)];+ } else if (p.fmt == 5) { // int3-g64: 16B low plane + 8B high plane+ int g = k >> 6, j = k & 63;+ lo.x = w[rb + uint(g * 6 + (j >> 4))] >> uint((j & 15) * 2);+ lo.y = w[rb + uint(g * 6 + 4 + (j >> 5))] >> uint(j & 31);+ sc.x = scale[uint(o) * uint((p.I + 63) >> 6) + uint(g)];+ } else { // nibbles: 2 int4, 4 grouped int4, 7 MXFP4+ lo.x = w[rb + uint(k >> 3)];+ if (p.fmt != 2) // gs % 8 == 0: the word is one group+ sc.x = scale[uint(o) * uint((p.I + p.gs - 1) / p.gs) + uint(k / p.gs)];+ }+}++// decode8: the 8 weights of a fetched unit, group scale folded in.+void decode8(uvec4 a, uvec4 b, vec2 sc, out vec4 lo, out vec4 hi) {+ if (p.fmt == 1) {+ lo = vec4(i8(a.x, 0), i8(a.x, 1), i8(a.x, 2), i8(a.x, 3));+ hi = vec4(i8(a.y, 0), i8(a.y, 1), i8(a.y, 2), i8(a.y, 3));+ } else if (p.fmt == 12) {+ lo = vec4(e4m3(a.x, 0), e4m3(a.x, 1), e4m3(a.x, 2), e4m3(a.x, 3)) * sc.x;+ hi = vec4(e4m3(a.y, 0), e4m3(a.y, 1), e4m3(a.y, 2), e4m3(a.y, 3)) * sc.y;+ } else if (p.fmt == 10) {+ lo = uintBitsToFloat(a); hi = uintBitsToFloat(b);+ } else if (p.fmt == 11) { // low half = even column+ lo = vec4(uintBitsToFloat(a.x << 16), uintBitsToFloat(a.x & 0xffff0000u),+ uintBitsToFloat(a.y << 16), uintBitsToFloat(a.y & 0xffff0000u));+ hi = vec4(uintBitsToFloat(a.z << 16), uintBitsToFloat(a.z & 0xffff0000u),+ uintBitsToFloat(a.w << 16), uintBitsToFloat(a.w & 0xffff0000u));+ } else if (p.fmt == 5) { // value = 2 low-plane bits | high bit << 2, - 4+ float v[8];+ [[unroll]] for (int t = 0; t < 8; t++)+ v[t] = float(int(((a.x >> uint(2 * t)) & 3u) | (((a.y >> uint(t)) & 1u) << 2)) - 4);+ lo = vec4(v[0], v[1], v[2], v[3]) * sc.x; hi = vec4(v[4], v[5], v[6], v[7]) * sc.x;+ } else if (p.fmt == 7) {+ lo = vec4(mx4(a.x, 0), mx4(a.x, 1), mx4(a.x, 2), mx4(a.x, 3)) * sc.x;+ hi = vec4(mx4(a.x, 4), mx4(a.x, 5), mx4(a.x, 6), mx4(a.x, 7)) * sc.x;+ } else {+ lo = vec4(i4(a.x, 0), i4(a.x, 1), i4(a.x, 2), i4(a.x, 3));+ hi = vec4(i4(a.x, 4), i4(a.x, 5), i4(a.x, 6), i4(a.x, 7));+ if (p.fmt == 4) { lo *= sc.x; hi *= sc.x; }+ }+}++const int NT = NTM * NTN;+const int WU = (BM * (BK / 8) + NT - 1) / NT; // weight units per thread and step+const int XU = (BN * (BK / 4) + NT - 1) / NT; // activation quads per thread and step+uvec4 rwa[WU], rwb[WU]; vec2 rws[WU]; vec4 rx[XU];++// Fetch step k0 of this tile into registers (zeros past O, S and I).+void fetch_step(int tid, int o0, int s0, int k0) {+ [[unroll]] for (int r = 0; r < WU; r++) {+ int u = tid + r * NT, m = u / (BK / 8), k = k0 + (u % (BK / 8)) * 8;+ rwa[r] = uvec4(0u); rwb[r] = uvec4(0u); rws[r] = vec2(0.0);+ if (u < BM * (BK / 8) && o0 + m < p.O && k < p.I) fetch8(o0 + m, k, rwa[r], rwb[r], rws[r]);+ }+ [[unroll]] for (int r = 0; r < XU; r++) {+ int u = tid + r * NT, n = u / (BK / 4), s = s0 + n, k = k0 + (u % (BK / 4)) * 4;+ vec4 v = vec4(0.0);+ if (u < BN * (BK / 4) && s < p.S && k < p.I) {+ uint b = uint(s) * uint(p.I) + uint(k);+ if ((p.I & 3) == 0) v = x4[b >> 2]; // rows of x start on a quad+ else {+ v.x = x[b];+ if (k + 1 < p.I) v.y = x[b + 1u];+ if (k + 2 < p.I) v.z = x[b + 2u];+ if (k + 3 < p.I) v.w = x[b + 3u];+ }+ }+ rx[r] = v;+ }+}++void main() {+ int tid = int(gl_LocalInvocationID.x);+ int tm = tid % NTM, tn = tid / NTM;+ int o0 = int(gl_WorkGroupID.x) * BM, s0 = int(gl_WorkGroupID.y) * BN;++ float acc[TM][TN];+ [[unroll]] for (int i = 0; i < TM; i++)+ [[unroll]] for (int j = 0; j < TN; j++) acc[i][j] = 0.0;++ if (PF) fetch_step(tid, o0, s0, 0);+ for (int k0 = 0; k0 < p.I; k0 += BK) {+ if (!PF) fetch_step(tid, o0, s0, k0);+ // decode the fetched step into shared memory+ [[unroll]] for (int r = 0; r < WU; r++) {+ int u = tid + r * NT;+ if (u < BM * (BK / 8)) {+ int m = u / (BK / 8), kq = (u % (BK / 8)) * 2;+ vec4 lo, hi;+ decode8(rwa[r], rwb[r], rws[r], lo, hi);+ wsh[m * KQ + kq] = lo;+ wsh[m * KQ + kq + 1] = hi;+ }+ }+ [[unroll]] for (int r = 0; r < XU; r++) {+ int u = tid + r * NT;+ if (u < BN * (BK / 4)) xsh[(u / (BK / 4)) * KQ + u % (BK / 4)] = rx[r];+ }+ barrier();+ if (PF && k0 + BK < p.I) fetch_step(tid, o0, s0, k0 + BK);+ // The step's BK products sum into a fresh partial that then joins acc: blocked+ // summation keeps the rounding error near (BK + I/BK) ulps instead of I, the+ // same order as the GEMV's per-lane sums plus subgroup tree.+ float part[TM][TN];+ [[unroll]] for (int i = 0; i < TM; i++)+ [[unroll]] for (int j = 0; j < TN; j++) part[i][j] = 0.0;+ [[unroll]] for (int q = 0; q < BK / 4; q++) {+ vec4 a[TM], b[TN];+ [[unroll]] for (int i = 0; i < TM; i++) a[i] = wsh[(tm + i * NTM) * KQ + q];+ [[unroll]] for (int j = 0; j < TN; j++) b[j] = xsh[(tn + j * NTN) * KQ + q];+ [[unroll]] for (int i = 0; i < TM; i++)+ [[unroll]] for (int j = 0; j < TN; j++)+ part[i][j] = fma(a[i].w, b[j].w, fma(a[i].z, b[j].z,+ fma(a[i].y, b[j].y, fma(a[i].x, b[j].x, part[i][j]))));+ }+ [[unroll]] for (int i = 0; i < TM; i++)+ [[unroll]] for (int j = 0; j < TN; j++) acc[i][j] += part[i][j];+ barrier();+ }++ bool perrow = p.fmt == 1 || p.fmt == 2;+ [[unroll]] for (int i = 0; i < TM; i++) {+ int o = o0 + tm + i * NTM;+ if (o >= p.O) continue;+ float sc = perrow ? scale[o] : 1.0;+ [[unroll]] for (int j = 0; j < TN; j++) {+ int s = s0 + tn + j * NTN;+ if (s < p.S) y[uint(s) * uint(p.O) + uint(o)] = acc[i][j] * sc;+ }+ }+}diff --git a/c/shaders/qmatmul_gemm.spv b/c/shaders/qmatmul_gemm.spvnew file mode 100644index 00000000..a5a89659Binary files /dev/null and b/c/shaders/qmatmul_gemm.spv differ