Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: tiled GEMMs for prefill-sized S

JustVugg8b85c66112dd
diff --git a/c/Makefile b/c/Makefile
index d2f677d2..54354a75 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -704,7 +704,8 @@ else
LDFLAGS += -lvulkan
endif
VK_OBJ = backend_vulkan.o
-VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv
+VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \
+ shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv
endif
# Optional AMD XDNA2 (Ryzen AI NPU) lane, Windows. Same opt-in shape as VK=1:
@@ -841,6 +842,13 @@ backend_vulkan.o: backend_vulkan.c .build-config
shaders/%.spv: shaders/%.comp
@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
$(GLSLC) --target-env=vulkan1.2 $< -o $@
+# The cooperative-matrix GEMM needs a glslc that knows GL_KHR_cooperative_matrix
+# (shaderc 2023.6+). An older one keeps the committed .spv instead of failing the build;
+# at run time the shader is optional anyway (the fp32 GEMM takes over without it).
+shaders/qmatmul_coop.spv: shaders/qmatmul_coop.comp
+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \
+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }
# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.
backend_loader.o: backend_loader.c .build-config
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4
index 20e17870..4e87b387 100644
--- a/c/Makefile.deepseek-v4
+++ b/c/Makefile.deepseek-v4
@@ -235,7 +235,8 @@ LDFLAGS += -lvulkan
endif
V4_VK_OBJ = backend_vulkan_v4.o
V4_VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv \
- shaders/attention_absorb.spv shaders/rmsnorm.spv
+ shaders/attention_absorb.spv shaders/rmsnorm.spv \
+ shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv
endif
include Makefile.deepseek-v4.units
@@ -312,6 +313,11 @@ backend_vulkan_v4.o: backend_vulkan.c backend_vulkan.h $(V4_FLAGS_STAMP)
shaders/%.spv: shaders/%.comp
@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
$(GLSLC) --target-env=vulkan1.2 $< -o $@
+# As in the parent: an older glslc keeps the committed cooperative-matrix .spv.
+shaders/qmatmul_coop.spv: shaders/qmatmul_coop.comp
+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \
+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }
# Pluggable expert-store backend registry (standalone; not a deepseek_v4.c unit).
# Linked into the binary so the engine can dispatch COLI_EXPERT_STORE backends.
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index 9d272e6c..9d07a729 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -34,6 +34,16 @@ typedef struct {
VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;
} Scratch;
+/* Tile of the fp32 GEMM pipeline: BM outputs x BN activation rows per workgroup, BK
+ * inputs per shared-memory step, TM x TN accumulators per thread; pf = fetch the next
+ * step's words while this one's products run. */
+typedef struct { int bm, bn, bk, tm, tn, pf; } VkGemmTile;
+/* Tile of the cooperative-matrix pipeline: BM outputs x BN activation rows per
+ * workgroup, WM x WN of it per subgroup (multiples of 16), BK inputs per staged step. */
+typedef struct { int bm, bn, wm, wn, bk; } VkCoopTile;
+#define VK_GEMM_SLOTS 2
+#define VK_COOP_SLOTS 3
+
/* Persistent device-side KV latent/rope cache for one layer (MLA attention).
* Host appends rows as tokens decode (absolute-position indexing); the absorb
* kernel reads them in place. Allocated once at max_t rows, like the CUDA
@@ -59,6 +69,19 @@ static struct {
VkShaderModule shader;
VkDescriptorPool dpool;
VkDescriptorSet dset;
+ /* Tiled GEMMs for prefill-sized S, with the GEMV's 4 bindings and push constants so
+ * they bind through G.plyt/G.dset: the fp32 one (qmatmul_gemm.spv) and, when the
+ * device has a 16x16x16 fp16 -> fp32 subgroup MMA and a settable subgroup size
+ * (coop_sg), the cooperative-matrix one (qmatmul_coop.spv) for the formats whose
+ * decoded weights are fp16 values. Each is built at a few tile widths (specialization
+ * constants, slots of growing BN) and a call takes the narrowest that covers its S.
+ * A call takes a GEMM when S >= gemm_min_s (0 = GEMV for every S) and S*O >=
+ * gemm_min_so; coop_off is the harness's switch to the fp32 GEMM. */
+ VkShaderModule shader_gemm, shader_coop;
+ VkPipeline pipe_gemm[VK_GEMM_SLOTS], pipe_coop[VK_COOP_SLOTS];
+ VkGemmTile gemm_t[VK_GEMM_SLOTS];
+ VkCoopTile coop_t[VK_COOP_SLOTS];
+ int gemm_min_s, gemm_min_so, has_coop, coop_sg, coop_off;
/* fused dual gate+up+silu pipeline (6 bindings): x, Wg, gscale, Wu, uscale, hidden */
VkShaderModule shader_gu; VkDescriptorSetLayout dsl_gu; VkPipelineLayout plyt_gu;
VkPipeline pipe_gu; VkDescriptorPool dpool_gu; VkDescriptorSet dset_gu;
@@ -96,7 +119,7 @@ static struct {
* expert-called-repeatedly pattern). The synchronous fence wait each call means no
* submission is ever in flight, so rebinding/re-recording only when something
* actually changed is safe. */
- ColiVkTensor *bound_tensor; int bound_S, bound_I, bound_O, cmd_ready;
+ ColiVkTensor *bound_tensor; int bound_S, bound_I, bound_O, bound_gemm, cmd_ready;
VkBuffer bound_xbuf, bound_ybuf;
size_t used_bytes, tensor_count;
/* VRAM pressure-proofing: with VK_EXT_memory_priority the attention working set
@@ -307,6 +330,89 @@ static void derive_dir_file(const char *spv, const char *fname, char *out, size_
else snprintf(out, n, "%s", fname);
}
+static int gemm_tile_ok(VkGemmTile t, const VkPhysicalDeviceLimits *lim) {
+ if (t.bm < 1 || t.bn < 1 || t.tm < 1 || t.tn < 1 || t.bk < 8 || t.bk % 8 ||
+ t.bm % t.tm || t.bn % t.tn) return 0;
+ long threads = (long)(t.bm / t.tm) * (t.bn / t.tn);
+ long lds = (long)(t.bm + t.bn) * (t.bk / 4 + 1) * 16; /* wsh + xsh, vec4 quads */
+ return threads >= 32 && threads <= (long)lim->maxComputeWorkGroupInvocations &&
+ threads <= (long)lim->maxComputeWorkGroupSize[0] && lds <= (long)lim->maxComputeSharedMemorySize;
+}
+static int gemm_pipeline(VkGemmTile t, int slot) {
+ int32_t sv[7] = {t.bm, t.bn, t.bk, t.tm, t.tn, (t.bm / t.tm) * (t.bn / t.tn), t.pf ? 1 : 0};
+ VkSpecializationMapEntry me[7];
+ for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};
+ VkSpecializationInfo si = {7, me, sizeof(sv), sv};
+ VkComputePipelineCreateInfo cpi = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = G.shader_gemm, .pName = "main",
+ .pSpecializationInfo = &si},
+ .layout = G.plyt};
+ VkPipeline pipe;
+ VKCHECK(vkCreateComputePipelines(G.dev, VK_NULL_HANDLE, 1, &cpi, NULL, &pipe), "gemm pipeline");
+ if (G.pipe_gemm[slot]) vkDestroyPipeline(G.dev, G.pipe_gemm[slot], NULL);
+ G.pipe_gemm[slot] = pipe; G.gemm_t[slot] = t;
+ G.cmd_ready = 0; /* a recorded GEMM dispatch names the old pipeline */
+ return 1;
+}
+
+/* Subgroups of exactly G.coop_sg lanes; BK a multiple of 32 (group sizes are). */
+static int coop_tile_ok(VkCoopTile t, const VkPhysicalDeviceLimits *lim) {
+ if (t.wm < 16 || t.wn < 16 || t.wm % 16 || t.wn % 16 || t.bm % t.wm || t.bn % t.wn ||
+ t.bk < 32 || t.bk % 32 || G.coop_sg < 16 || 256 % G.coop_sg) return 0;
+ long nw = (long)(t.bm / t.wm) * (t.bn / t.wn), threads = nw * G.coop_sg;
+ long lds = (long)(t.bm + 2 * t.bn) * (t.bk + 8) * 2 + (long)t.bm * 4 + nw * 16 * 17 * 4;
+ return threads <= (long)lim->maxComputeWorkGroupInvocations &&
+ threads <= (long)lim->maxComputeWorkGroupSize[0] && lds <= (long)lim->maxComputeSharedMemorySize;
+}
+static int coop_pipeline(VkCoopTile t, int slot) {
+#ifdef VK_KHR_cooperative_matrix
+ int32_t sv[7] = {t.bm, t.bn, t.wm, t.wn, t.bk, G.coop_sg, (t.bm / t.wm) * (t.bn / t.wn) * G.coop_sg};
+ VkSpecializationMapEntry me[7];
+ for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};
+ VkSpecializationInfo si = {7, me, sizeof(sv), sv};
+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,
+ .requiredSubgroupSize = (uint32_t)G.coop_sg};
+ VkComputePipelineCreateInfo cpi = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,
+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,
+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = G.shader_coop, .pName = "main",
+ .pSpecializationInfo = &si},
+ .layout = G.plyt};
+ VkPipeline pipe;
+ VKCHECK(vkCreateComputePipelines(G.dev, VK_NULL_HANDLE, 1, &cpi, NULL, &pipe), "coop pipeline");
+ if (G.pipe_coop[slot]) vkDestroyPipeline(G.dev, G.pipe_coop[slot], NULL);
+ G.pipe_coop[slot] = pipe; G.coop_t[slot] = t;
+ G.cmd_ready = 0;
+ return 1;
+#else
+ (void)t; (void)slot; return 0;
+#endif
+}
+
+/* The tiles, narrowest first, measured on a Radeon 780M (RDNA3, RADV) over Qwen3.8's
+ * shapes: BN = 32 serves a 32-row prefill chunk, the wider ones longer prompts.
+ * COLI_VK_GEMM_TILE=bm,bn,bk,tm,tn[,pf] and COLI_VK_COOP_TILE=bm,bn,wm,wn,bk put one
+ * tile in every slot (bench). */
+static const VkGemmTile gemm_tiles[VK_GEMM_SLOTS] = {{128, 32, 32, 4, 4, 1}, {64, 64, 32, 4, 4, 0}};
+static const VkCoopTile coop_tiles[VK_COOP_SLOTS] = {{128, 32, 32, 32, 32}, {128, 64, 32, 32, 32},
+ {128, 128, 64, 32, 32}};
+/* The narrowest slot whose BN covers S, else the widest. */
+static int gemm_slot(int S) {
+ int k = 0;
+ while (k + 1 < VK_GEMM_SLOTS && G.pipe_gemm[k + 1] && S > G.gemm_t[k].bn) k++;
+ return k;
+}
+static int coop_slot(int S) {
+ int k = 0;
+ while (k + 1 < VK_COOP_SLOTS && G.pipe_coop[k + 1] && S > G.coop_t[k].bn) k++;
+ return k;
+}
+
+static void vk_prof_paths(void);
+static int g_vk_prof;
+
int coli_vk_init(const char *spv_path) {
if (G.ready) return 1;
VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
@@ -348,7 +454,8 @@ int coli_vk_init(const char *spv_path) {
/* Pressure-proofing extensions (both optional, detected at runtime):
* memory_priority ranks allocations for the kernel's eviction order,
* memory_budget exposes how much VRAM a new allocation can still take. */
- const char *dext[2]; uint32_t ndext = 0;
+ const char *dext[4]; uint32_t ndext = 0;
+ int has_cm = 0, has_ssc = 0;
{
uint32_t ne = 0;
vkEnumerateDeviceExtensionProperties(G.phys, NULL, &ne, NULL);
@@ -361,11 +468,72 @@ int coli_vk_init(const char *spv_path) {
#endif
#ifdef VK_EXT_memory_budget
if (!strcmp(ep[i].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) G.has_budget = 1;
+#endif
+#ifdef VK_KHR_cooperative_matrix
+ if (!strcmp(ep[i].extensionName, VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME)) has_cm = 1;
+ if (!strcmp(ep[i].extensionName, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME)) has_ssc = 1;
#endif
}
free(ep);
}
}
+ /* Cooperative-matrix GEMM (optional, COLI_VK_COOP=0 turns it off): the KHR extension
+ * with a 16x16x16 fp16 x fp16 -> fp32 subgroup shape, shaderFloat16, the Vulkan
+ * memory model, and a subgroup size the pipeline can require. All of it or none. */
+#ifdef VK_KHR_cooperative_matrix
+ VkPhysicalDeviceCooperativeMatrixFeaturesKHR cmf = {
+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR};
+ VkPhysicalDeviceSubgroupSizeControlFeaturesEXT sscf = {
+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT, .pNext = &cmf};
+ VkPhysicalDeviceVulkan12Features v12f = {
+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, .pNext = &sscf};
+ {
+ const char *e = getenv("COLI_VK_COOP");
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties(G.phys, &pp);
+ if (has_cm && has_ssc && pp.apiVersion >= VK_API_VERSION_1_2 && !(e && *e == '0')) {
+ VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &v12f};
+ vkGetPhysicalDeviceFeatures2(G.phys, &f2);
+ VkPhysicalDeviceSubgroupSizeControlPropertiesEXT ssp = {
+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT};
+ VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &ssp};
+ vkGetPhysicalDeviceProperties2(G.phys, &p2);
+ int shape = 0;
+ PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR cmp = (PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR)
+ vkGetInstanceProcAddr(G.inst, "vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR");
+ uint32_t nc = 0;
+ if (cmp && cmp(G.phys, &nc, NULL) == VK_SUCCESS && nc) {
+ VkCooperativeMatrixPropertiesKHR *cp = calloc(nc, sizeof(*cp));
+ if (cp) {
+ for (uint32_t i = 0; i < nc; i++) cp[i].sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR;
+ if (cmp(G.phys, &nc, cp) == VK_SUCCESS)
+ for (uint32_t i = 0; i < nc; i++)
+ if (cp[i].MSize == 16 && cp[i].NSize == 16 && cp[i].KSize == 16 &&
+ cp[i].AType == VK_COMPONENT_TYPE_FLOAT16_KHR && cp[i].BType == VK_COMPONENT_TYPE_FLOAT16_KHR &&
+ cp[i].CType == VK_COMPONENT_TYPE_FLOAT32_KHR && cp[i].ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR &&
+ cp[i].scope == VK_SCOPE_SUBGROUP_KHR) shape = 1;
+ free(cp);
+ }
+ }
+ /* Subgroups of 64 where the device allows (RDNA3: the wave64 MMA measured
+ * fastest), else the largest it can require; COLI_VK_COOP_SG overrides. */
+ int sgz = ssp.maxSubgroupSize >= 64 && ssp.minSubgroupSize <= 64 ? 64 : (int)ssp.maxSubgroupSize;
+ const char *se = getenv("COLI_VK_COOP_SG");
+ if (se && atoi(se) >= (int)ssp.minSubgroupSize && atoi(se) <= (int)ssp.maxSubgroupSize) sgz = atoi(se);
+ if (shape && cmf.cooperativeMatrix && v12f.shaderFloat16 && v12f.vulkanMemoryModel &&
+ sscf.subgroupSizeControl && sscf.computeFullSubgroups &&
+ (ssp.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) && sgz >= 16) {
+ G.has_coop = 1; G.coop_sg = sgz;
+ }
+ }
+ }
+ /* enable exactly what the cooperative-matrix shader needs, nothing else */
+ memset(&v12f, 0, sizeof(v12f)); memset(&sscf, 0, sizeof(sscf)); memset(&cmf, 0, sizeof(cmf));
+ cmf.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR; cmf.cooperativeMatrix = VK_TRUE;
+ sscf.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT; sscf.pNext = &cmf;
+ sscf.subgroupSizeControl = VK_TRUE; sscf.computeFullSubgroups = VK_TRUE;
+ v12f.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; v12f.pNext = &sscf;
+ v12f.shaderFloat16 = VK_TRUE; v12f.vulkanMemoryModel = VK_TRUE;
+#endif
VkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
.queueCreateInfoCount = 1, .pQueueCreateInfos = &qi};
#ifdef VK_EXT_memory_priority
@@ -379,6 +547,19 @@ int coli_vk_init(const char *spv_path) {
#endif
di.enabledExtensionCount = ndext; di.ppEnabledExtensionNames = ndext ? dext : NULL;
G.prio = 0.75f; /* default class: dense/resident weights */
+#ifdef VK_KHR_cooperative_matrix
+ if (G.has_coop) { /* a device that refuses the extra features still comes up without them */
+ VkDeviceCreateInfo dc = di;
+ const char *cext[4]; uint32_t nc = 0;
+ for (uint32_t i = 0; i < ndext; i++) cext[nc++] = dext[i];
+ cext[nc++] = VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME;
+ cext[nc++] = VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME;
+ cmf.pNext = (void *)di.pNext; dc.pNext = &v12f;
+ dc.enabledExtensionCount = nc; dc.ppEnabledExtensionNames = cext;
+ if (vkCreateDevice(G.phys, &dc, NULL, &G.dev) != VK_SUCCESS) { G.has_coop = 0; G.dev = VK_NULL_HANDLE; }
+ }
+ if (!G.dev)
+#endif
VKCHECK(vkCreateDevice(G.phys, &di, NULL, &G.dev), "vkCreateDevice");
vkGetDeviceQueue(G.dev, G.qfam, 0, &G.queue);
if (G.has_prio || G.has_budget)
@@ -421,6 +602,61 @@ int coli_vk_init(const char *spv_path) {
if (!G.shader) return 0;
if (!build_pipeline(G.dev, 4, sizeof(struct PC), G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;
+ /* Optional tiled GEMMs: an absent shader or a tile the device cannot hold leaves
+ * that slot empty; without the fp32 one every S stays on the GEMV. The cooperative-
+ * matrix one only comes up on devices that passed the checks above, and alongside
+ * the fp32 one, which takes the formats it does not. */
+ {
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties(G.phys, &pp);
+ char gm_path[512]; derive_sibling(spv_path, "_gemm.spv", gm_path, sizeof(gm_path));
+ G.shader_gemm = load_spv(G.dev, gm_path);
+ const char *e = getenv("COLI_VK_GEMM_TILE");
+ VkGemmTile u = {0, 0, 0, 0, 0, 0};
+ int one = e && *e && sscanf(e, "%d,%d,%d,%d,%d,%d", &u.bm, &u.bn, &u.bk, &u.tm, &u.tn, &u.pf) >= 5 &&
+ gemm_tile_ok(u, &pp.limits);
+ if (e && *e && !one) fprintf(stderr, "[VK] COLI_VK_GEMM_TILE=%s rejected\n", e);
+ for (int k = 0; G.shader_gemm && k < (one ? 1 : VK_GEMM_SLOTS); k++) {
+ VkGemmTile t = one ? u : gemm_tiles[k];
+ if (gemm_tile_ok(t, &pp.limits)) gemm_pipeline(t, k);
+ }
+ if (G.shader_gemm && !G.pipe_gemm[0]) {
+ vkDestroyShaderModule(G.dev, G.shader_gemm, NULL); G.shader_gemm = VK_NULL_HANDLE;
+ for (int k = 1; k < VK_GEMM_SLOTS; k++)
+ if (G.pipe_gemm[k]) { vkDestroyPipeline(G.dev, G.pipe_gemm[k], NULL); G.pipe_gemm[k] = VK_NULL_HANDLE; }
+ }
+ if (G.has_coop && G.pipe_gemm[0]) {
+ char cm_path[512]; derive_sibling(spv_path, "_coop.spv", cm_path, sizeof(cm_path));
+ G.shader_coop = load_spv(G.dev, cm_path);
+ e = getenv("COLI_VK_COOP_TILE");
+ VkCoopTile c = {0, 0, 0, 0, 0};
+ one = e && *e && sscanf(e, "%d,%d,%d,%d,%d", &c.bm, &c.bn, &c.wm, &c.wn, &c.bk) == 5 &&
+ coop_tile_ok(c, &pp.limits);
+ if (e && *e && !one) fprintf(stderr, "[VK] COLI_VK_COOP_TILE=%s rejected\n", e);
+ /* every slot shares BK: coop_fmt checks group sizes against it */
+ for (int k = 0; G.shader_coop && k < (one ? 1 : VK_COOP_SLOTS); k++) {
+ VkCoopTile t = one ? c : coop_tiles[k];
+ if (coop_tile_ok(t, &pp.limits)) coop_pipeline(t, k);
+ }
+ }
+ if (!G.pipe_coop[0]) {
+ for (int k = 1; k < VK_COOP_SLOTS; k++)
+ if (G.pipe_coop[k]) { vkDestroyPipeline(G.dev, G.pipe_coop[k], NULL); G.pipe_coop[k] = VK_NULL_HANDLE; }
+ if (G.shader_coop) vkDestroyShaderModule(G.dev, G.shader_coop, NULL);
+ G.shader_coop = VK_NULL_HANDLE; G.has_coop = 0;
+ }
+ /* Where the GEMM overtakes the GEMV, measured on the 780M over Qwen3.8's shapes
+ * (int8, int4-g64, bf16): from S = 2 on a wide matrix (O = 2560: 0.65 against
+ * 0.90 ms in int8, 1.0 against 2.2 in bf16), but a narrow one fills too few
+ * tiles (O = 48: the GEMV wins up to S = 32, loses from 128). The rule that
+ * fits: S >= 2 and S*O >= 4096. COLI_VK_GEMM_MIN_S=N puts every S >= N on the
+ * GEMM, 0 none; S == 1 (decode) always stays on the GEMV. */
+ const char *m = getenv("COLI_VK_GEMM_MIN_S");
+ G.gemm_min_s = !G.pipe_gemm[0] ? 0 : m && *m ? atoi(m) : 2;
+ G.gemm_min_so = m && *m ? 0 : 4096;
+ if (G.gemm_min_s == 1) G.gemm_min_s = 2;
+ if (G.gemm_min_s < 0) G.gemm_min_s = 0;
+ }
+
/* Optional fused gate+up pipeline: skip gracefully if its shader isn't present
* (single-matmul path keeps working). */
char gu_path[512]; derive_sibling(spv_path, "_gate_up.spv", gu_path, sizeof(gu_path));
@@ -464,9 +700,13 @@ int coli_vk_init(const char *spv_path) {
VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.eg_fence), "eg fence");
G.ready = 1;
+ if ((g_vk_prof = getenv("VK_PROF") != NULL)) atexit(vk_prof_paths);
VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);
- fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s%s\n", p.deviceName, G.qfam, G.memtype,
+ fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s%s", p.deviceName, G.qfam, G.memtype,
G.shader_gu ? ", fused gate+up" : "", G.shader_att ? ", absorb attention" : "");
+ if (G.gemm_min_s) fprintf(stderr, ", tiled GEMM from S=%d%s%s", G.gemm_min_s,
+ G.gemm_min_so ? " and S*O>=4096" : "", G.pipe_coop[0] ? " (cooperative matrix)" : "");
+ fprintf(stderr, "\n");
return 1;
}
@@ -577,21 +817,32 @@ int coli_vk_tensor_ensure(ColiVkTensor **tensor, const void *weights, const floa
* then fall back to the blocking wait. The spinning thread is stalled on the
* GPU result anyway. COLI_VK_SPIN_US=0 restores the pure blocking wait. */
static long g_vk_spin_us = -1;
-static VkResult vk_fence_wait(VkDevice dev, VkFence f) {
- if (g_vk_spin_us < 0) {
- const char *e = getenv("COLI_VK_SPIN_US");
- g_vk_spin_us = e ? atol(e) : 300;
- if (g_vk_spin_us < 0) g_vk_spin_us = 0;
- }
- if (g_vk_spin_us > 0) {
+static VkResult vk_fence_wait_us(VkDevice dev, VkFence f, long spin_us) {
+ if (spin_us > 0) {
double t0 = vk_now();
do {
VkResult r = vkGetFenceStatus(dev, f);
if (r != VK_NOT_READY) return r; /* VK_SUCCESS or a real error */
- } while ((vk_now() - t0) * 1000.0 < (double)g_vk_spin_us);
+ } while ((vk_now() - t0) * 1000.0 < (double)spin_us);
}
return vkWaitForFences(dev, 1, &f, VK_TRUE, 10000000000ULL);
}
+static long vk_spin_us(void) {
+ if (g_vk_spin_us < 0) {
+ const char *e = getenv("COLI_VK_SPIN_US");
+ g_vk_spin_us = e ? atol(e) : 300;
+ if (g_vk_spin_us < 0) g_vk_spin_us = 0;
+ }
+ return g_vk_spin_us;
+}
+static VkResult vk_fence_wait(VkDevice dev, VkFence f) { return vk_fence_wait_us(dev, f, vk_spin_us()); }
+/* A tiled GEMM runs for milliseconds, past the spin budget, and the blocked wait then
+ * wakes up to a millisecond late: measured on the 780M, a 0.6 ms S=32 GEMM took 0.6 or
+ * 1.05 ms per call at random. Its wait spins through the dispatch, bounded at 50 ms;
+ * COLI_VK_SPIN_US=0 still blocks. */
+static VkResult vk_fence_wait_gemm(VkDevice dev, VkFence f) {
+ return vk_fence_wait_us(dev, f, vk_spin_us() > 0 ? 50000 : 0);
+}
/* Global submit/wait totals across EVERY synchronous GPU path (VK_PROF=1) β€” the
* per-path counters miss traffic that flows through the fused pair/absorb/group
@@ -603,6 +854,48 @@ static void vkprof_tick(void) {
}
static unsigned long long g_vk_matmul_calls; /* successful coli_vk_matmul calls */
+static unsigned long long g_vk_gemm_calls; /* ... of which through a tiled GEMM */
+static unsigned long long g_vk_coop_calls; /* ... of which the cooperative-matrix one */
+
+/* VK_PROF=1: at exit, how the matmuls and their wall time (upload to readback) split
+ * between the GEMV and the two tiled GEMMs. */
+static double g_vk_path_ms[3];
+static unsigned long long g_vk_path_n[3];
+static void vk_prof_paths(void) {
+ fprintf(stderr, "[VK_PROF] %llu matmuls: GEMV %llu in %.0f ms, fp32 GEMM %llu in %.0f ms, "
+ "cooperative-matrix GEMM %llu in %.0f ms\n", g_vk_matmul_calls,
+ g_vk_path_n[0], g_vk_path_ms[0], g_vk_path_n[1], g_vk_path_ms[1], g_vk_path_n[2], g_vk_path_ms[2]);
+}
+
+/* Formats the cooperative-matrix GEMM takes: decoded weights exact in fp16, and group
+ * boundaries on its staged steps (one group scale per step). */
+static int coop_fmt(const ColiVkTensor *t) {
+ int gs = t->fmt == 5 ? 64 : t->gs;
+ return t->fmt == 1 || t->fmt == 2 ||
+ ((t->fmt == 4 || t->fmt == 5 || t->fmt == 7 || t->fmt == 12) && gs % G.coop_t[0].bk == 0);
+}
+/* Its x upload: the rows, then their powers of two rs[s] = 2^(15-e) with
+ * max|x[s,:]| < 2^e, so a scaled row stays under 2^15 (fp16 max 65504) and its hi/lo
+ * fp16 split keeps 22 bits. One pass copies and takes the integer max of the magnitude
+ * bits (vectorizes without fast-math); a row holding Inf or NaN keeps rs = 1 and its
+ * non-finite result, as on the GEMV. */
+static void coop_stage_x(float *restrict dst, const float *restrict x, int S, int I) {
+ float *rs = dst + (size_t)S * I;
+ for (int s = 0; s < S; s++) {
+ const float *restrict r = x + (size_t)s * I;
+ float *restrict d = dst + (size_t)s * I;
+ uint32_t m = 0;
+ for (int i = 0; i < I; i++) {
+ uint32_t u; memcpy(&u, &r[i], 4);
+ d[i] = r[i]; u &= 0x7fffffffu; m = u > m ? u : m;
+ }
+ int e = (int)(m >> 23) - 126; /* m < 2^e for a normal m */
+ if (m >= 0x7f800000u) e = 15;
+ if (e < -100) e = -100; /* zero and subnormal rows */
+ uint32_t b = (uint32_t)(15 - e + 127) << 23;
+ memcpy(&rs[s], &b, sizeof(b));
+ }
+}
int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
@@ -613,11 +906,20 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
* separates our code (memcpy/desc/record) from the driver (submit) and the GPU
* (fence wait) to localize the tier-size-linear tax. */
static double p_x, p_desc, p_rec, p_sub, p_wait, p_y; static long p_n;
- double t0 = G.eg_prof ? vk_now() : 0, tA;
+ double t0 = G.eg_prof ? vk_now() : 0, tA, tp = g_vk_prof ? vk_now() : 0;
+ /* Prefill-sized S takes a tiled GEMM: one BM x BN tile of y per workgroup, so each
+ * weight is fetched ceil(S/BN) times instead of S; the cooperative-matrix one where
+ * the device and the format allow (path 2), else the fp32 one (1). Decode (S = 1),
+ * and a matrix too narrow to fill the device at this S, keep the GEMV (0). */
+ int path = 0;
+ if (G.gemm_min_s && S >= G.gemm_min_s && (int64_t)S * O >= G.gemm_min_so)
+ path = G.pipe_coop[0] && !G.coop_off && coop_fmt(t) ? 2 : 1;
size_t xb = (size_t)S * I * sizeof(float), yb = (size_t)S * O * sizeof(float);
VkBuffer old_x = G.x.buf, old_y = G.y.buf;
- if (!scratch_reserve(&G.x, xb) || !scratch_reserve_mt(&G.y, yb, G.memtype_cached)) return 0; /* y read back */
- memcpy(G.x.ptr, x, xb);
+ if (!scratch_reserve(&G.x, xb + (path == 2 ? (size_t)S * sizeof(float) : 0)) || /* + rs[S] */
+ !scratch_reserve_mt(&G.y, yb, G.memtype_cached)) return 0; /* y read back */
+ if (path == 2) coop_stage_x(G.x.ptr, x, S, I);
+ else memcpy(G.x.ptr, x, xb);
if (G.eg_prof) { tA = vk_now(); p_x += tA - t0; t0 = tA; }
/* Rebind descriptors only when the tensor or a scratch buffer changed (a realloc
@@ -643,19 +945,26 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
/* Re-record the command buffer only when the binding or the dispatch shape changed.
* Recorded WITHOUT one-time-submit so the same buffer can be resubmitted verbatim β€”
* for repeated calls to the same expert this drops setup to a bare submit+wait. */
- if (rebind || !G.cmd_ready || G.bound_S != S || G.bound_I != I || G.bound_O != O) {
+ if (rebind || !G.cmd_ready || G.bound_S != S || G.bound_I != I || G.bound_O != O || G.bound_gemm != path) {
VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");
VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");
- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
+ int slot = path == 2 ? coop_slot(S) : path ? gemm_slot(S) : 0;
+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE,
+ path == 2 ? G.pipe_coop[slot] : path ? G.pipe_gemm[slot] : G.pipe);
vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.dset, 0, NULL);
struct PC pc = {fmt, S, I, O, t->rowWords, t->gs};
vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
- /* Grid-stride shader: one subgroup per output row (~8 rows/workgroup at wave32).
- * Launch ~O/8 workgroups for occupancy; the shader loops to cover any O / wave width. */
- vkCmdDispatch(G.cmd, (uint32_t)((O + 7) / 8), (uint32_t)S, 1);
+ if (path) {
+ int bm = path == 2 ? G.coop_t[slot].bm : G.gemm_t[slot].bm;
+ int bn = path == 2 ? G.coop_t[slot].bn : G.gemm_t[slot].bn;
+ vkCmdDispatch(G.cmd, (uint32_t)((O + bm - 1) / bm), (uint32_t)((S + bn - 1) / bn), 1);
+ } else
+ /* Grid-stride shader: one subgroup per output row (~8 rows/workgroup at wave32).
+ * Launch ~O/8 workgroups for occupancy; the shader loops to cover any O / wave width. */
+ vkCmdDispatch(G.cmd, (uint32_t)((O + 7) / 8), (uint32_t)S, 1);
VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");
- G.cmd_ready = 1; G.bound_S = S; G.bound_I = I; G.bound_O = O;
+ G.cmd_ready = 1; G.bound_S = S; G.bound_I = I; G.bound_O = O; G.bound_gemm = path;
}
if (G.eg_prof) { tA = vk_now(); p_rec += tA - t0; t0 = tA; }
@@ -667,7 +976,7 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
// Bounded wait: a GPU hang/TDR must never wedge the process. 10s is orders of
// magnitude over a single-GEMV dispatch; on timeout/device-loss disable VK for
// the rest of the run and fall back to CPU (the caller degrades on our 0 return).
- VkResult wr = vk_fence_wait(G.dev, G.fence);
+ VkResult wr = path ? vk_fence_wait_gemm(G.dev, G.fence) : vk_fence_wait(G.dev, G.fence);
if (wr != VK_SUCCESS) {
fprintf(stderr, "[VK] fence wait failed: %d β€” disabling GPU offload, staying on CPU\n", wr);
G.ready = 0;
@@ -682,6 +991,9 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
p_n, p_x, p_desc, p_rec, p_sub, p_wait, p_y);
}
g_vk_matmul_calls++;
+ g_vk_gemm_calls += path != 0;
+ g_vk_coop_calls += path == 2;
+ if (g_vk_prof) { g_vk_path_ms[path] += vk_now() - tp; g_vk_path_n[path]++; }
return 1;
}
@@ -1638,6 +1950,10 @@ void coli_vk_shutdown(void) {
vkDestroyCommandPool(G.dev, G.cpool, NULL);
vkDestroyDescriptorPool(G.dev, G.dpool, NULL);
vkDestroyPipeline(G.dev, G.pipe, NULL);
+ for (int k = 0; k < VK_GEMM_SLOTS; k++) if (G.pipe_gemm[k]) vkDestroyPipeline(G.dev, G.pipe_gemm[k], NULL);
+ for (int k = 0; k < VK_COOP_SLOTS; k++) if (G.pipe_coop[k]) vkDestroyPipeline(G.dev, G.pipe_coop[k], NULL);
+ if (G.shader_gemm) vkDestroyShaderModule(G.dev, G.shader_gemm, NULL);
+ if (G.shader_coop) vkDestroyShaderModule(G.dev, G.shader_coop, NULL);
vkDestroyPipelineLayout(G.dev, G.plyt, NULL);
vkDestroyDescriptorSetLayout(G.dev, G.dsl, NULL);
vkDestroyShaderModule(G.dev, G.shader, NULL);
@@ -1682,7 +1998,7 @@ static size_t ref_rowbytes(int fmt, int I) {
static size_t ref_scales(int fmt, int I, int O) { // scale COUNT (per-group for fmt 4/5)
if (fmt == 10 || fmt == 11) return 1;
if (fmt == 5) return (size_t)O * (size_t)((I + 63) / 64);
- if (fmt == 4 || fmt == 12) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);
+ if (fmt == 4 || fmt == 7 || fmt == 12) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);
return (size_t)O;
}
static float deq(const uint8_t *row, int fmt, int i) {
@@ -1698,16 +2014,19 @@ static float deq(const uint8_t *row, int fmt, int i) {
const uint8_t *lo = row + (size_t)(i >> 6) * 24, *hi = lo + 16; int j = i & 63;
unsigned u = ((lo[j >> 2] >> ((j & 3) * 2)) & 3u) | (((hi[j >> 3] >> (j & 7)) & 1u) << 2);
return (float)((int)u - 4); }
- uint8_t v = row[i >> 1]; int nib = (i & 1) ? (v >> 4) : (v & 15); return (float)(nib - 8);
+ uint8_t v = row[i >> 1]; int nib = (i & 1) ? (v >> 4) : (v & 15);
+ if (fmt == 7) { static const float e2m1[8] = {0, 0.5f, 1, 1.5f, 2, 3, 4, 6}; /* MXFP4 */
+ return (nib & 8) ? -e2m1[nib & 7] : e2m1[nib & 7]; }
+ return (float)(nib - 8);
}
static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,
int fmt, int S, int I, int O) {
size_t rb = ref_rowbytes(fmt, I);
- int gw2 = (fmt == 4 || fmt == 12) ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;
+ int gw2 = (fmt == 4 || fmt == 7 || fmt == 12) ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;
for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {
double sum = 0; const uint8_t *row = w + (size_t)o * rb;
- if (fmt == 5 || fmt == 4 || fmt == 12) { // per-group scales fold inside the sum
+ if (fmt == 5 || fmt == 4 || fmt == 7 || fmt == 12) { // per-group scales fold inside the sum
for (int g = 0; g < ng; g++) {
double a = 0; int end = (g + 1) * gw2 < I ? (g + 1) * gw2 : I;
for (int i = g * gw2; i < end; i++) a += x[s * I + i] * deq(row, fmt, i);
@@ -1739,28 +2058,36 @@ static double ref_dot(const float *x, const uint8_t *row, const float *scb, int
return sum * scb[o];
}
-static int run_case(int fmt, int S, int I, int O, int iters) {
+/* Random activations, weights and scales for one case: x[S,I] in [-1,1), raw weight
+ * bytes (no NaN/Inf for the float and fp8 formats), scales in [0.01,0.02). */
+static void case_fill(int fmt, int S, int I, int O, float **x, uint8_t **w, float **sc) {
size_t rb = ref_rowbytes(fmt, I), nsc = ref_scales(fmt, I, O);
- float *x = malloc((size_t)S * I * sizeof(float));
- uint8_t *w = malloc(rb * O);
- float *sc = malloc(nsc * sizeof(float));
- float *yg = malloc((size_t)S * O * sizeof(float));
- float *yc = malloc((size_t)S * O * sizeof(float));
- for (int i = 0; i < S * I; i++) x[i] = (float)((rand() % 200 - 100) / 100.0);
- for (size_t i = 0; i < rb * O; i++) w[i] = rand() & 0xff;
+ *x = malloc((size_t)S * I * sizeof(float));
+ *w = malloc(rb * O);
+ *sc = malloc(nsc * sizeof(float));
+ for (size_t i = 0; i < (size_t)S * I; i++) (*x)[i] = (float)((rand() % 200 - 100) / 100.0);
+ for (size_t i = 0; i < rb * O; i++) (*w)[i] = rand() & 0xff;
if (fmt == 12) /* no NaN bytes: they would poison the comparison */
- for (size_t i = 0; i < rb * O; i++) if ((w[i] & 0x7f) == 0x7f) w[i] ^= 1;
+ for (size_t i = 0; i < rb * O; i++) if (((*w)[i] & 0x7f) == 0x7f) (*w)[i] ^= 1;
if (fmt == 10 || fmt == 11) /* float weights: random bytes could be NaN/Inf */
for (size_t i = 0; i < (size_t)I * O; i++) {
float f = (float)((rand() % 2001 - 1000) / 1000.0);
- if (fmt == 10) memcpy(w + i * 4, &f, 4);
- else { uint32_t u; memcpy(&u, &f, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(w + i * 2, &h, 2); }
+ if (fmt == 10) memcpy(*w + i * 4, &f, 4);
+ else { uint32_t u; memcpy(&u, &f, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(*w + i * 2, &h, 2); }
}
- for (size_t o = 0; o < nsc; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;
+ for (size_t o = 0; o < nsc; o++) (*sc)[o] = 0.01f + (rand() % 100) / 10000.0f;
+}
+
+static int run_case(int fmt, int S, int I, int O, int iters) {
+ float *x, *sc; uint8_t *w;
+ case_fill(fmt, S, I, O, &x, &w, &sc);
+ float *yg = malloc((size_t)S * O * sizeof(float));
+ float *yc = malloc((size_t)S * O * sizeof(float));
ColiVkTensor *t = NULL;
if (!coli_vk_matmul(&t, yg, x, w, sc, fmt, S, I, O, g_ref_gs)) { printf("matmul failed\n"); return 1; }
- cpu_ref(yc, x, w, sc, fmt, S, I, O);
+ const char *path = G.bound_gemm == 2 ? "coop" : G.bound_gemm ? "gemm" : "gemv";
+ double c0 = now(); cpu_ref(yc, x, w, sc, fmt, S, I, O); double cpu_ms = (now() - c0) * 1000;
double maxerr = 0, maxrel = 0;
for (int i = 0; i < S * O; i++) {
double e = fabs(yg[i] - yc[i]); if (e > maxerr) maxerr = e;
@@ -1770,15 +2097,114 @@ static int run_case(int fmt, int S, int I, int O, int iters) {
double t0 = now();
for (int k = 0; k < iters; k++) coli_vk_matmul(&t, yg, x, w, sc, fmt, S, I, O, g_ref_gs);
double gpu_ms = (now() - t0) * 1000 / iters;
- // microbench (CPU ref, 1 iter β€” it's slow)
- double c0 = now(); cpu_ref(yc, x, w, sc, fmt, S, I, O); double cpu_ms = (now() - c0) * 1000;
- printf("fmt=%d S=%d I=%d O=%d | maxerr=%.4g maxrel=%.4g | gpu=%.3f ms cpu_ref=%.3f ms\n",
- fmt, S, I, O, maxerr, maxrel, gpu_ms, cpu_ms);
+ char gsb[16] = "";
+ if (fmt == 4 || fmt == 7 || fmt == 12) snprintf(gsb, sizeof(gsb), " gs=%d", g_ref_gs);
+ printf("fmt=%d%s S=%d I=%d O=%d %s | maxerr=%.4g maxrel=%.4g | gpu=%.3f ms cpu_ref=%.3f ms\n",
+ fmt, gsb, S, I, O, path, maxerr, maxrel, gpu_ms, cpu_ms);
coli_vk_tensor_free(t);
free(x); free(w); free(sc); free(yg); free(yc);
return maxrel > 1e-3 ? 1 : 0;
}
+/* The tiled GEMMs against the same reference, forced for this S whatever the device
+ * threshold, and required to have really run: a missing qmatmul_gemm.spv must fail
+ * here, not pass on the GEMV. The fp32 GEMM always; then, on a device with the
+ * cooperative-matrix pipeline, that one too for the formats it takes. gs is the group
+ * size of fmt 4, 7 and 12. */
+static int coop_takes(int fmt, int gs) {
+ ColiVkTensor t = {.fmt = fmt, .gs = gs};
+ return G.pipe_coop[0] && coop_fmt(&t);
+}
+static int run_gemm_case(int fmt, int S, int I, int O, int gs) {
+ if (!G.pipe_gemm[0]) { printf("gemm fmt=%d: no tiled GEMM pipeline (qmatmul_gemm.spv missing?)\n", fmt); return 1; }
+ int keep_s = G.gemm_min_s, keep_so = G.gemm_min_so, keep_gs = g_ref_gs, bad = 0;
+ G.gemm_min_s = 2; G.gemm_min_so = 0; g_ref_gs = gs;
+ G.coop_off = 1; /* the fp32 GEMM */
+ unsigned long long n0 = g_vk_gemm_calls;
+ bad |= run_case(fmt, S, I, O, 2);
+ if (g_vk_gemm_calls == n0) { printf(" ^ did not take the tiled GEMM\n"); bad = 1; }
+ G.coop_off = 0;
+ if (coop_takes(fmt, gs)) { /* the cooperative-matrix one */
+ n0 = g_vk_coop_calls;
+ bad |= run_case(fmt, S, I, O, 2);
+ if (g_vk_coop_calls == n0) { printf(" ^ did not take the cooperative-matrix GEMM\n"); bad = 1; }
+ }
+ G.gemm_min_s = keep_s; G.gemm_min_so = keep_so; g_ref_gs = keep_gs;
+ return bad;
+}
+
+/* Seconds per coli_vk_matmul call with the GEMM threshold at min_s (0 = GEMV): x upload,
+ * dispatch and y readback, what an engine pays; and, in *gpu, per resubmit of the
+ * recorded command buffer, the dispatch alone plus the submit roundtrip. */
+static double time_matmul(ColiVkTensor **t, float *y, const float *x, const void *w, const float *sc,
+ int fmt, int S, int I, int O, int gs, int min_s, double *gpu) {
+ int keep = G.gemm_min_s, keep_so = G.gemm_min_so; G.gemm_min_s = min_s; G.gemm_min_so = 0;
+ coli_vk_matmul(t, y, x, w, sc, fmt, S, I, O, gs); /* record + warm */
+ int it = 0; double t0 = now(), el;
+ do { coli_vk_matmul(t, y, x, w, sc, fmt, S, I, O, gs); it++; } while ((el = now() - t0) < 0.25 && it < 500);
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};
+ int ig = 0; double g0 = now(), gl;
+ do { vkResetFences(G.dev, 1, &G.fence); vkQueueSubmit(G.queue, 1, &si, G.fence);
+ vk_fence_wait_gemm(G.dev, G.fence); ig++;
+ } while ((gl = now() - g0) < 0.25 && ig < 500);
+ G.gemm_min_s = keep; G.gemm_min_so = keep_so;
+ *gpu = gl / ig;
+ return el / it;
+}
+
+/* COLI_VK_TEST_GEMM_BENCH=1: GEMV against the tiled GEMM, format by format, over S on
+ * one engine-sized matrix (COLI_VK_TEST_GEMM_SHAPE=I,O; default 2560,6144, Qwen3.8's
+ * DeltaNet z projection). GFLOP/s per call and per resubmit; the S where the GEMM
+ * overtakes is what COLI_VK_GEMM_MIN_S defaults to. COLI_VK_TEST_GEMM_FMT=a,b,... and
+ * COLI_VK_TEST_GEMM_S=a,b,... restrict the formats and the S values. Runs instead of
+ * every other case. */
+static void bench_gemm(void) {
+ int I = 2560, O = 6144, fm[8] = {1, 2, 4, 5, 7, 10, 11, 12}, nf = 8;
+ const char *e = getenv("COLI_VK_TEST_GEMM_SHAPE");
+ if (e) sscanf(e, "%d,%d", &I, &O);
+ if ((e = getenv("COLI_VK_TEST_GEMM_FMT"))) {
+ nf = 0;
+ for (const char *c = e; *c && nf < 8; ) { fm[nf++] = atoi(c); while (*c && *c != ',') c++; if (*c) c++; }
+ }
+ int Ss[16] = {2, 4, 8, 16, 32, 64, 128, 256, 512}, ns = 9, Smax = 0;
+ if ((e = getenv("COLI_VK_TEST_GEMM_S"))) { /* the S values, a,b,... */
+ ns = 0;
+ for (const char *c = e; *c && ns < 16; ) { Ss[ns++] = atoi(c); while (*c && *c != ',') c++; if (*c) c++; }
+ }
+ for (int k = 0; k < ns; k++) if (Ss[k] > Smax) Smax = Ss[k];
+ printf("GEMM bench I=%d O=%d, threshold now S>=%d and S*O>=%d; GEMM tiles", I, O, G.gemm_min_s, G.gemm_min_so);
+ for (int k = 0; k < VK_GEMM_SLOTS; k++) if (G.pipe_gemm[k]) printf(" %dx%d", G.gemm_t[k].bm, G.gemm_t[k].bn);
+ printf("; COOP tiles (subgroup %d)", G.coop_sg);
+ for (int k = 0; k < VK_COOP_SLOTS; k++) if (G.pipe_coop[k]) printf(" %dx%d", G.coop_t[k].bm, G.coop_t[k].bn);
+ printf("\n");
+ for (int f = 0; f < nf; f++) {
+ int fmt = fm[f];
+ g_ref_gs = fmt == 7 ? 32 : 64;
+ float *x, *sc; uint8_t *w;
+ case_fill(fmt, Smax, I, O, &x, &w, &sc);
+ float *y = malloc((size_t)Smax * O * sizeof(float));
+ ColiVkTensor *t = NULL;
+ if (!coli_vk_matmul(&t, y, x, w, sc, fmt, 1, I, O, g_ref_gs)) { printf("fmt=%d: upload failed\n", fmt); continue; }
+ for (int k = 0; k < ns; k++) {
+ int S = Ss[k];
+ double gv, gm, gvg, gmg, cm = 0, cmg = 0;
+ gv = time_matmul(&t, y, x, w, sc, fmt, S, I, O, g_ref_gs, 0, &gvg);
+ G.coop_off = 1;
+ gm = time_matmul(&t, y, x, w, sc, fmt, S, I, O, g_ref_gs, 2, &gmg);
+ G.coop_off = 0;
+ if (coop_takes(fmt, g_ref_gs)) cm = time_matmul(&t, y, x, w, sc, fmt, S, I, O, g_ref_gs, 2, &cmg);
+ double fl = 2.0 * S * I * O / 1e9;
+ printf("fmt=%-2d S=%-4d | GEMV %8.3f ms %7.1f GFLOP/s (gpu %8.3f) | GEMM %8.3f ms %7.1f GFLOP/s (gpu %8.3f)",
+ fmt, S, gv * 1e3, fl / gv, gvg * 1e3, gm * 1e3, fl / gm, gmg * 1e3);
+ if (cm > 0) printf(" | COOP %8.3f ms %7.1f GFLOP/s (gpu %8.3f)", cm * 1e3, fl / cm, cmg * 1e3);
+ printf("\n");
+ }
+ coli_vk_tensor_free(t);
+ free(x); free(w); free(sc); free(y);
+ }
+ g_ref_gs = 64;
+}
+
/* Batched throughput: record N dispatches in ONE command buffer, one submit + one
* fence wait β€” the amortized per-matmul cost with the submit roundtrip spread across
* the batch (how the real expert tier would drive it). Reuses the descriptor binding
@@ -2149,6 +2575,12 @@ int main(int argc, char **argv) {
}
if (nb) printf("ballast: %d x 4 MB idle allocations\n", nb);
}
+ if (getenv("COLI_VK_TEST_GEMM_BENCH") && atoi(getenv("COLI_VK_TEST_GEMM_BENCH"))) {
+ bench_gemm();
+ coli_vk_shutdown();
+ return 0;
+ }
+ G.gemm_min_s = 0; /* every case below is the GEMV's, but for run_gemm_case's */
bad |= run_case(1, 1, 6144, 1536, 50); // int8 expert gate/up shape (S=1 decode)
bad |= run_case(2, 1, 6144, 1536, 50); // int4 expert
bad |= run_case(1, 1, 1536, 6144, 50); // down proj shape
@@ -2178,6 +2610,23 @@ int main(int argc, char **argv) {
g_ref_gs = 32; bad |= run_case(12, 2, 4096, 256, 5);
g_ref_gs = 128; bad |= run_case(12, 2, 4096, 256, 5);
g_ref_gs = 64;
+ /* Tiled GEMM (prefill-sized S), every format: S = 16, 64 and 512 against odd O and
+ * odd or unaligned I, so the last output, row and input tiles are all partial, the
+ * last group is a tail, and x rows come both on and off a 16-byte quad. */
+ {
+ static const int gf[8] = {1, 2, 4, 5, 7, 10, 11, 12};
+ for (int f = 0; f < 8; f++) {
+ int gs = gf[f] == 7 ? 32 : 64;
+ bad |= run_gemm_case(gf[f], 16, 1001, 193, gs); // odd I: tail group, x off a quad
+ bad |= run_gemm_case(gf[f], 64, 1536, 97, gs); // I % 4 == 0: x by quads
+ bad |= run_gemm_case(gf[f], 512, 520, 131, gs); // 8-input tail group
+ }
+ bad |= run_gemm_case(1, 37, 999, 65, 0); // odd S, a 1-output last tile
+ bad |= run_gemm_case(4, 33, 1001, 70, 8); // the smallest grouped-int4 group
+ bad |= run_gemm_case(4, 33, 1001, 70, 24); // a group that is no power of two
+ bad |= run_gemm_case(12, 33, 1001, 70, 12); // fp8: the two words of a unit in two groups
+ bad |= run_gemm_case(12, 33, 1001, 70, 128);
+ }
/* COLI_VK_TEST_MATMUL_ONLY=1: stop after the per-format matmul cases above. CI runs
* this on Lavapipe, where the benches below say nothing and take most of the time. */
if (getenv("COLI_VK_TEST_MATMUL_ONLY") && atoi(getenv("COLI_VK_TEST_MATMUL_ONLY"))) {
diff --git a/c/shaders/qmatmul_coop.comp b/c/shaders/qmatmul_coop.comp
new file mode 100644
index 00000000..bcae0690
--- /dev/null
+++ b/c/shaders/qmatmul_coop.comp
@@ -0,0 +1,247 @@
+#version 450
+// Cooperative-matrix twin of qmatmul_gemm.comp (VK_KHR_cooperative_matrix, 16x16x16
+// fp16 x fp16 -> fp32 per subgroup): the same y[s,o] = sum_i x[s,i] * dequant(W[o,i]),
+// for the formats whose decoded weights are fp16 values: 1, 2, 4, 5, 7 and 12, the
+// grouped ones with gs % BK == 0 (backend_vulkan.c routes the rest to the fp32 GEMM).
+// Nothing is rounded to fp16, and the result stays as close to fp32 as the fp32
+// GEMM's (the VK_TEST harness holds both to the same bound):
+// - every decoded weight (int8, int4, int3, e2m1, e4m3) is exact in fp16, so the A
+// operand carries no rounding; group and row scales stay fp32, outside the MMA;
+// - each activation row arrives with a power of two rs[s] (written by the host after
+// the S*I floats of x) that puts its largest |x| under 2^15; the scaled row is
+// split into hi (its top 11 significant bits) and lo = x - hi, both exact in fp16
+// but for lo's last bits, and the MMA runs on both (A*hi + A*lo);
+// - every 16-input MMA product joins an fp32 accumulator with an fp32 add, times the
+// group scale for the grouped formats.
+// A workgroup owns a BM-output x BN-row tile of y, each subgroup WM x WN of it in
+// 16 x 16 accumulators. Weights and activations are staged BK inputs at a time, the
+// next step's words fetched into registers before this step's MMAs.
+#extension GL_KHR_cooperative_matrix : require
+#extension GL_KHR_memory_scope_semantics : require
+#extension GL_KHR_shader_subgroup_basic : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
+#extension GL_EXT_control_flow_attributes : require
+
+layout(constant_id = 0) const int BM = 64; // outputs per tile
+layout(constant_id = 1) const int BN = 64; // activation rows per tile
+layout(constant_id = 2) const int WM = 32; // outputs per subgroup, a multiple of 16
+layout(constant_id = 3) const int WN = 32; // rows per subgroup, a multiple of 16
+layout(constant_id = 4) const int BK = 32; // inputs per staged step, a multiple of 32
+layout(constant_id = 5) const int SG = 64; // subgroup size the pipeline requires
+layout(local_size_x_id = 6) in; // (BM/WM) * (BN/WN) * SG threads
+
+layout(std430, binding = 0) readonly buffer X { float x[]; }; // x[S*I], then rs[S]
+layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; };
+layout(std430, binding = 1) readonly buffer W { uint w[]; };
+layout(std430, binding = 1) readonly buffer W2 { uvec2 w2[]; };
+layout(std430, binding = 2) readonly buffer Sc { float scale[]; };
+layout(std430, binding = 3) writeonly buffer Y { float y[]; };
+
+layout(push_constant) uniform P {
+ int fmt; // 1, 2, 4, 5, 7, 12
+ int S, I, O;
+ int rowWords; // uint32 words per weight row
+ int gs; // group size of fmt 4, 7 and 12 (a multiple of 32 here)
+} p;
+
+const int FM = WM / 16, FN = WN / 16; // 16 x 16 accumulators per subgroup
+const int NW = (BM / WM) * (BN / WN); // subgroups
+const int NT = NW * SG;
+const int WST = BK + 8; // halves per staged row, 16 bytes of padding
+const int EST = 17; // floats per epilogue row: odd, no bank conflicts
+shared float16_t wsh[BM * WST]; // [o][i]: the MMA's A, row-major
+shared float16_t xhs[BN * WST]; // [s][i]: its B, column-major; hi part
+shared float16_t xls[BN * WST]; // lo part
+shared float ssh[BM]; // the current group's scales
+shared float esh[NW * 16 * EST]; // per subgroup: one accumulator on its way to y
+
+#define ACC coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>
+#define MATA coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA>
+#define MATB coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB>
+
+float i8(uint word, int lane) {
+ int b = int((word >> (uint(lane) * 8u)) & 0xffu);
+ if (b >= 128) b -= 256;
+ return float(b);
+}
+float i4(uint word, int lane) {
+ return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);
+}
+float mx4(uint word, int lane) { // e2m1, bit3 = sign (qmatmul.comp)
+ uint n = (word >> (uint(lane) * 4u)) & 0xfu;
+ const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);
+ float v = lut[n & 7u];
+ return (n & 8u) != 0u ? -v : v;
+}
+float e4m3(uint word, int lane) { // OCP E4M3-FN, arithmetic decode (qmatmul.comp)
+ uint b = (word >> (uint(lane) * 8u)) & 0xffu;
+ if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);
+ uint e = (b >> 3) & 0xfu, m = b & 7u;
+ float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);
+ return (b & 0x80u) != 0u ? -v : v;
+}
+
+// The words of weights k..k+7 of row o (k % 8 == 0, k < I), as qmatmul_gemm.comp's
+// fetch8 for these formats; no scale: it multiplies the MMA products instead.
+uvec2 fetch8(int o, int k) {
+ uint rb = uint(o) * uint(p.rowWords);
+ uvec2 r = uvec2(0u);
+ if (p.fmt == 1 || p.fmt == 12) { // 4 bytes per word
+ if ((p.rowWords & 1) == 0) r = w2[(rb >> 1) + uint(k >> 3)];
+ else { r.x = w[rb + uint(k >> 2)]; if (k + 4 < p.I) r.y = w[rb + uint(k >> 2) + 1u]; }
+ if (k + 4 >= p.I) r.y = 0u;
+ } else if (p.fmt == 5) { // int3-g64: 16B low plane + 8B high plane
+ int g = k >> 6, j = k & 63;
+ r.x = w[rb + uint(g * 6 + (j >> 4))] >> uint((j & 15) * 2);
+ r.y = w[rb + uint(g * 6 + 4 + (j >> 5))] >> uint(j & 31);
+ } else r.x = w[rb + uint(k >> 3)]; // nibbles: 2, 4, 7
+ return r;
+}
+void decode8(uvec2 a, out vec4 lo, out vec4 hi) {
+ if (p.fmt == 1) {
+ lo = vec4(i8(a.x, 0), i8(a.x, 1), i8(a.x, 2), i8(a.x, 3));
+ hi = vec4(i8(a.y, 0), i8(a.y, 1), i8(a.y, 2), i8(a.y, 3));
+ } else if (p.fmt == 12) {
+ lo = vec4(e4m3(a.x, 0), e4m3(a.x, 1), e4m3(a.x, 2), e4m3(a.x, 3));
+ hi = vec4(e4m3(a.y, 0), e4m3(a.y, 1), e4m3(a.y, 2), e4m3(a.y, 3));
+ } else if (p.fmt == 5) {
+ float v[8];
+ [[unroll]] for (int t = 0; t < 8; t++)
+ v[t] = float(int(((a.x >> uint(2 * t)) & 3u) | (((a.y >> uint(t)) & 1u) << 2)) - 4);
+ lo = vec4(v[0], v[1], v[2], v[3]); hi = vec4(v[4], v[5], v[6], v[7]);
+ } else if (p.fmt == 7) {
+ lo = vec4(mx4(a.x, 0), mx4(a.x, 1), mx4(a.x, 2), mx4(a.x, 3));
+ hi = vec4(mx4(a.x, 4), mx4(a.x, 5), mx4(a.x, 6), mx4(a.x, 7));
+ } else {
+ lo = vec4(i4(a.x, 0), i4(a.x, 1), i4(a.x, 2), i4(a.x, 3));
+ hi = vec4(i4(a.x, 4), i4(a.x, 5), i4(a.x, 6), i4(a.x, 7));
+ }
+}
+
+const int WU = (BM * (BK / 8) + NT - 1) / NT; // weight units per thread and step
+const int XU = (BN * (BK / 4) + NT - 1) / NT; // activation quads per thread and step
+uvec2 rw[WU]; vec4 rx[XU];
+
+// Fetch step k0 into registers: zeros past O and I (finite after decode) and past S.
+void fetch_step(int tid, int o0, int s0, int k0) {
+ [[unroll]] for (int r = 0; r < WU; r++) {
+ int u = tid + r * NT, m = u / (BK / 8), k = k0 + (u % (BK / 8)) * 8;
+ rw[r] = uvec2(0u);
+ if (u < BM * (BK / 8) && o0 + m < p.O && k < p.I) rw[r] = fetch8(o0 + m, k);
+ }
+ [[unroll]] for (int r = 0; r < XU; r++) {
+ int u = tid + r * NT, s = s0 + u / (BK / 4), k = k0 + (u % (BK / 4)) * 4;
+ vec4 v = vec4(0.0);
+ if (u < BN * (BK / 4) && s < p.S && k < p.I) {
+ uint b = uint(s) * uint(p.I) + uint(k);
+ if ((p.I & 3) == 0) v = x4[b >> 2];
+ else {
+ v.x = x[b];
+ if (k + 1 < p.I) v.y = x[b + 1u];
+ if (k + 2 < p.I) v.z = x[b + 2u];
+ if (k + 3 < p.I) v.w = x[b + 3u];
+ }
+ v *= x[uint(p.S) * uint(p.I) + uint(s)]; // the row's power of two
+ }
+ rx[r] = v;
+ }
+}
+
+void main() {
+ int tid = int(gl_LocalInvocationID.x);
+ int sg = int(gl_SubgroupID), lane = int(gl_SubgroupInvocationID);
+ int wm = sg % (BM / WM), wn = sg / (BM / WM); // this subgroup's WM x WN
+ int o0 = int(gl_WorkGroupID.x) * BM, s0 = int(gl_WorkGroupID.y) * BN;
+ bool grouped = p.fmt != 1 && p.fmt != 2;
+ int gsz = p.fmt == 5 ? 64 : p.gs; // a multiple of BK
+ int ng = grouped ? (p.I + gsz - 1) / gsz : 1;
+
+ ACC acc[FM][FN]; // fp32 sums; one declarator per line in this file: glslang drops
+ // the matrix use from the second declarator of a list
+ [[unroll]] for (int i = 0; i < FM; i++)
+ [[unroll]] for (int j = 0; j < FN; j++) acc[i][j] = ACC(0.0);
+
+ fetch_step(tid, o0, s0, 0);
+ for (int k0 = 0; k0 < p.I; k0 += BK) {
+ // the fetched step into shared memory: weights as fp16, x as hi and lo
+ [[unroll]] for (int r = 0; r < WU; r++) {
+ int u = tid + r * NT;
+ if (u < BM * (BK / 8)) {
+ int base = (u / (BK / 8)) * WST + (u % (BK / 8)) * 8;
+ vec4 lo, hi;
+ decode8(rw[r], lo, hi);
+ f16vec4 a = f16vec4(lo), b = f16vec4(hi);
+ [[unroll]] for (int t = 0; t < 4; t++) { wsh[base + t] = a[t]; wsh[base + 4 + t] = b[t]; }
+ }
+ }
+ [[unroll]] for (int r = 0; r < XU; r++) {
+ int u = tid + r * NT;
+ if (u < BN * (BK / 4)) {
+ int base = (u / (BK / 4)) * WST + (u % (BK / 4)) * 4;
+ // hi keeps the top 11 significant bits by masking, not by an fp16 round
+ // trip: the compiler folds vec4(f16vec4(v)) back to v, which zeroed lo
+ // (measured on RADV: fp16-level error, 1000x the fp32 GEMM's). Both
+ // conversions below are then exact, but for lo's last bits.
+ vec4 h32 = uintBitsToFloat(floatBitsToUint(rx[r]) & 0xffffe000u);
+ f16vec4 hi = f16vec4(h32), lo = f16vec4(rx[r] - h32);
+ [[unroll]] for (int t = 0; t < 4; t++) { xhs[base + t] = hi[t]; xls[base + t] = lo[t]; }
+ }
+ }
+ // a new group starts at this step: its scales
+ if (grouped && k0 % gsz == 0)
+ for (int u = tid; u < BM; u += NT)
+ ssh[u] = o0 + u < p.O ? scale[uint(o0 + u) * uint(ng) + uint(k0 / gsz)] : 0.0;
+ barrier();
+ if (k0 + BK < p.I) fetch_step(tid, o0, s0, k0 + BK);
+ ACC sm[FM]; // grouped formats: a 16 x 16 tile whose row m is output m's
+ if (grouped) // scale, a column-major load with stride 0
+ [[unroll]] for (int i = 0; i < FM; i++)
+ coopMatLoad(sm[i], ssh, uint(wm * WM + i * 16), 0u, gl_CooperativeMatrixLayoutColumnMajor);
+ [[unroll]] for (int kk = 0; kk < BK; kk += 16) {
+ if (k0 + kk < p.I) {
+ MATA a[FM];
+ MATB bh[FN];
+ MATB bl[FN];
+ [[unroll]] for (int i = 0; i < FM; i++)
+ coopMatLoad(a[i], wsh, uint((wm * WM + i * 16) * WST + kk), uint(WST),
+ gl_CooperativeMatrixLayoutRowMajor);
+ [[unroll]] for (int j = 0; j < FN; j++) {
+ coopMatLoad(bh[j], xhs, uint((wn * WN + j * 16) * WST + kk), uint(WST),
+ gl_CooperativeMatrixLayoutColumnMajor);
+ coopMatLoad(bl[j], xls, uint((wn * WN + j * 16) * WST + kk), uint(WST),
+ gl_CooperativeMatrixLayoutColumnMajor);
+ }
+ // Each 16-input product joins acc with an fp32 add (times the group scale
+ // for the grouped formats): the MMA accumulates with less than fp32's
+ // rounding, measured at 10x the fp32 GEMM's error when it summed 512
+ // inputs before a flush.
+ [[unroll]] for (int i = 0; i < FM; i++)
+ [[unroll]] for (int j = 0; j < FN; j++) {
+ ACC c = coopMatMulAdd(a[i], bh[j], ACC(0.0));
+ c = coopMatMulAdd(a[i], bl[j], c);
+ acc[i][j] = grouped ? acc[i][j] + c * sm[i] : acc[i][j] + c;
+ }
+ }
+ }
+ barrier();
+ }
+
+ // Each accumulator through this subgroup's 16 x 17 tile: lane l takes output row
+ // l % 16 and every (SG/16)-th row of x from l / 16, times the per-row formats' scale
+ // and 1/rs[s], stores masked to O and S.
+ int eb = sg * 16 * EST, m = lane % 16;
+ [[unroll]] for (int i = 0; i < FM; i++) {
+ int o = o0 + wm * WM + i * 16 + m;
+ float sc = grouped || o >= p.O ? 1.0 : scale[o];
+ [[unroll]] for (int j = 0; j < FN; j++) {
+ coopMatStore(acc[i][j], esh, uint(eb), uint(EST), gl_CooperativeMatrixLayoutRowMajor);
+ subgroupMemoryBarrierShared(); subgroupBarrier();
+ [[unroll]] for (int t = 0; t < 256 / SG; t++) {
+ int n = lane / 16 + t * (SG / 16), s = s0 + wn * WN + j * 16 + n;
+ if (o < p.O && s < p.S)
+ y[uint(s) * uint(p.O) + uint(o)] = esh[eb + m * EST + n] * sc / x[uint(p.S) * uint(p.I) + uint(s)];
+ }
+ subgroupBarrier();
+ }
+ }
+}
diff --git a/c/shaders/qmatmul_coop.spv b/c/shaders/qmatmul_coop.spv
new file mode 100644
index 00000000..8685197a
Binary files /dev/null and b/c/shaders/qmatmul_coop.spv differ
diff --git a/c/shaders/qmatmul_gemm.comp b/c/shaders/qmatmul_gemm.comp
new file mode 100644
index 00000000..4d824926
--- /dev/null
+++ b/c/shaders/qmatmul_gemm.comp
@@ -0,0 +1,234 @@
+#version 450
+// Tiled GEMM twin of qmatmul.comp, for prefill-sized S:
+// y[s,o] = sum_i x[s,i] * dequant(W[o,i]) (same fmt, layout and scales)
+// qmatmul.comp runs one GEMV per activation row, so every weight is fetched and
+// decoded S times. Here a workgroup owns a BM-output x BN-row tile of y and walks i
+// in BK-wide steps: each step decodes its BM weight rows into shared memory ONCE
+// and multiplies them against BN activation rows, staged in shared memory too, so a
+// weight is fetched ceil(S/BN) times and decoded once per tile.
+// - both tiles keep i contiguous, wsh[m][i] and xsh[n][i], as vec4 quads with one
+// quad of padding per row: the vec4 reads of 8 consecutive rows land on
+// distinct banks, and the staging writes stay plain quad stores;
+// - each thread accumulates a TM x TN block whose rows are strided by BM/TM and
+// BN/TN, so neighbouring threads read neighbouring rows and store neighbouring
+// outputs (coalesced y writes);
+// - the grouped formats (4, 5, 7, 12) fold their group scale into the weight as it
+// enters shared memory, which frees the tile from group boundaries; the per-row
+// formats (1, 2) multiply scale[o] at the end, as the GEMV does.
+// The tile shape is a set of specialization constants; backend_vulkan.c picks it, and
+// whether the next step's words are fetched during this step's products (PF): faster
+// for the narrow BN = 32 tile on the 780M, slower for the 64 x 64 one (registers).
+#extension GL_EXT_control_flow_attributes : require
+
+layout(constant_id = 0) const int BM = 64; // outputs per tile
+layout(constant_id = 1) const int BN = 64; // activation rows per tile
+layout(constant_id = 2) const int BK = 32; // inputs per step, a multiple of 8
+layout(constant_id = 3) const int TM = 4; // outputs per thread
+layout(constant_id = 4) const int TN = 4; // activation rows per thread
+layout(local_size_x_id = 5) in; // (BM/TM) * (BN/TN) threads
+layout(constant_id = 6) const bool PF = true; // fetch the next step during the products
+
+layout(std430, binding = 0) readonly buffer X { float x[]; };
+layout(std430, binding = 0) readonly buffer X4 { vec4 x4[]; }; // same buffer, quads
+layout(std430, binding = 1) readonly buffer W { uint w[]; }; // row o at o*rowWords
+layout(std430, binding = 1) readonly buffer W2 { uvec2 w2[]; }; // same buffer, pairs
+layout(std430, binding = 1) readonly buffer W4 { uvec4 w4[]; }; // same buffer, quads
+layout(std430, binding = 2) readonly buffer Sc { float scale[]; };
+layout(std430, binding = 3) writeonly buffer Y { float y[]; };
+
+layout(push_constant) uniform P {
+ int fmt; // as qmatmul.comp: 1, 2, 4, 5, 7, 10, 11, 12
+ int S, I, O;
+ int rowWords; // uint32 words per weight row
+ int gs; // group size of fmt 4/7 (multiple of 8) and 12 (multiple of 4)
+} p;
+
+const int KQ = BK / 4 + 1; // quads per shared row, one of padding
+const int NTM = BM / TM, NTN = BN / TN; // thread grid
+shared vec4 wsh[BM * KQ];
+shared vec4 xsh[BN * KQ];
+
+float i8(uint word, int lane) {
+ int b = int((word >> (uint(lane) * 8u)) & 0xffu);
+ if (b >= 128) b -= 256;
+ return float(b);
+}
+float i4(uint word, int lane) {
+ return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);
+}
+float mx4(uint word, int lane) { // e2m1, bit3 = sign (qmatmul.comp)
+ uint n = (word >> (uint(lane) * 4u)) & 0xfu;
+ const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);
+ float v = lut[n & 7u];
+ return (n & 8u) != 0u ? -v : v;
+}
+float e4m3(uint word, int lane) { // OCP E4M3-FN, arithmetic decode (qmatmul.comp)
+ uint b = (word >> (uint(lane) * 8u)) & 0xffu;
+ if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);
+ uint e = (b >> 3) & 0xfu, m = b & 7u;
+ float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);
+ return (b & 0x80u) != 0u ? -v : v;
+}
+
+// One staging unit is 8 consecutive weights of a row. A step's units are fetched into
+// registers (with PF, before the previous step's products run, the loads in flight
+// during them) and decoded into shared memory after: fetch8 reads the words and group
+// scales, decode8 turns them into floats.
+//
+// fetch8: the words of weights k..k+7 of row o (k % 8 == 0, k < I) and their scales.
+// Words past the row's last input are never read: they belong to the next row, or
+// lie past the buffer for the last one. A lane past I decodes the row's zero padding
+// to a finite value and meets a zero activation, so it adds nothing.
+void fetch8(int o, int k, out uvec4 lo, out uvec4 hi, out vec2 sc) {
+ uint rb = uint(o) * uint(p.rowWords);
+ lo = uvec4(0u); hi = uvec4(0u); sc = vec2(0.0);
+ if (p.fmt == 1 || p.fmt == 12) { // 4 bytes per word
+ if ((p.rowWords & 1) == 0) lo.xy = w2[(rb >> 1) + uint(k >> 3)];
+ else { lo.x = w[rb + uint(k >> 2)]; if (k + 4 < p.I) lo.y = w[rb + uint(k >> 2) + 1u]; }
+ if (k + 4 >= p.I) lo.y = 0u;
+ if (p.fmt == 12) { // gs % 4 == 0: a word never straddles
+ uint sb = uint(o) * uint((p.I + p.gs - 1) / p.gs);
+ sc.x = scale[sb + uint(k / p.gs)];
+ if (k + 4 < p.I) sc.y = scale[sb + uint((k + 4) / p.gs)];
+ }
+ } else if (p.fmt == 10) { // f32, one per word
+ if ((p.I & 7) == 0) { lo = w4[(rb + uint(k)) >> 2]; hi = w4[((rb + uint(k)) >> 2) + 1u]; }
+ else [[unroll]] for (int j = 0; j < 4; j++) {
+ if (k + j < p.I) lo[j] = w[rb + uint(k + j)];
+ if (k + 4 + j < p.I) hi[j] = w[rb + uint(k + 4 + j)];
+ }
+ } else if (p.fmt == 11) { // bf16, two per word
+ if ((p.I & 7) == 0) lo = w4[(rb + uint(k >> 1)) >> 2];
+ else [[unroll]] for (int j = 0; j < 4; j++)
+ if (k + 2 * j < p.I) lo[j] = w[rb + uint((k >> 1) + j)];
+ } else if (p.fmt == 5) { // int3-g64: 16B low plane + 8B high plane
+ int g = k >> 6, j = k & 63;
+ lo.x = w[rb + uint(g * 6 + (j >> 4))] >> uint((j & 15) * 2);
+ lo.y = w[rb + uint(g * 6 + 4 + (j >> 5))] >> uint(j & 31);
+ sc.x = scale[uint(o) * uint((p.I + 63) >> 6) + uint(g)];
+ } else { // nibbles: 2 int4, 4 grouped int4, 7 MXFP4
+ lo.x = w[rb + uint(k >> 3)];
+ if (p.fmt != 2) // gs % 8 == 0: the word is one group
+ sc.x = scale[uint(o) * uint((p.I + p.gs - 1) / p.gs) + uint(k / p.gs)];
+ }
+}
+
+// decode8: the 8 weights of a fetched unit, group scale folded in.
+void decode8(uvec4 a, uvec4 b, vec2 sc, out vec4 lo, out vec4 hi) {
+ if (p.fmt == 1) {
+ lo = vec4(i8(a.x, 0), i8(a.x, 1), i8(a.x, 2), i8(a.x, 3));
+ hi = vec4(i8(a.y, 0), i8(a.y, 1), i8(a.y, 2), i8(a.y, 3));
+ } else if (p.fmt == 12) {
+ lo = vec4(e4m3(a.x, 0), e4m3(a.x, 1), e4m3(a.x, 2), e4m3(a.x, 3)) * sc.x;
+ hi = vec4(e4m3(a.y, 0), e4m3(a.y, 1), e4m3(a.y, 2), e4m3(a.y, 3)) * sc.y;
+ } else if (p.fmt == 10) {
+ lo = uintBitsToFloat(a); hi = uintBitsToFloat(b);
+ } else if (p.fmt == 11) { // low half = even column
+ lo = vec4(uintBitsToFloat(a.x << 16), uintBitsToFloat(a.x & 0xffff0000u),
+ uintBitsToFloat(a.y << 16), uintBitsToFloat(a.y & 0xffff0000u));
+ hi = vec4(uintBitsToFloat(a.z << 16), uintBitsToFloat(a.z & 0xffff0000u),
+ uintBitsToFloat(a.w << 16), uintBitsToFloat(a.w & 0xffff0000u));
+ } else if (p.fmt == 5) { // value = 2 low-plane bits | high bit << 2, - 4
+ float v[8];
+ [[unroll]] for (int t = 0; t < 8; t++)
+ v[t] = float(int(((a.x >> uint(2 * t)) & 3u) | (((a.y >> uint(t)) & 1u) << 2)) - 4);
+ lo = vec4(v[0], v[1], v[2], v[3]) * sc.x; hi = vec4(v[4], v[5], v[6], v[7]) * sc.x;
+ } else if (p.fmt == 7) {
+ lo = vec4(mx4(a.x, 0), mx4(a.x, 1), mx4(a.x, 2), mx4(a.x, 3)) * sc.x;
+ hi = vec4(mx4(a.x, 4), mx4(a.x, 5), mx4(a.x, 6), mx4(a.x, 7)) * sc.x;
+ } else {
+ lo = vec4(i4(a.x, 0), i4(a.x, 1), i4(a.x, 2), i4(a.x, 3));
+ hi = vec4(i4(a.x, 4), i4(a.x, 5), i4(a.x, 6), i4(a.x, 7));
+ if (p.fmt == 4) { lo *= sc.x; hi *= sc.x; }
+ }
+}
+
+const int NT = NTM * NTN;
+const int WU = (BM * (BK / 8) + NT - 1) / NT; // weight units per thread and step
+const int XU = (BN * (BK / 4) + NT - 1) / NT; // activation quads per thread and step
+uvec4 rwa[WU], rwb[WU]; vec2 rws[WU]; vec4 rx[XU];
+
+// Fetch step k0 of this tile into registers (zeros past O, S and I).
+void fetch_step(int tid, int o0, int s0, int k0) {
+ [[unroll]] for (int r = 0; r < WU; r++) {
+ int u = tid + r * NT, m = u / (BK / 8), k = k0 + (u % (BK / 8)) * 8;
+ rwa[r] = uvec4(0u); rwb[r] = uvec4(0u); rws[r] = vec2(0.0);
+ if (u < BM * (BK / 8) && o0 + m < p.O && k < p.I) fetch8(o0 + m, k, rwa[r], rwb[r], rws[r]);
+ }
+ [[unroll]] for (int r = 0; r < XU; r++) {
+ int u = tid + r * NT, n = u / (BK / 4), s = s0 + n, k = k0 + (u % (BK / 4)) * 4;
+ vec4 v = vec4(0.0);
+ if (u < BN * (BK / 4) && s < p.S && k < p.I) {
+ uint b = uint(s) * uint(p.I) + uint(k);
+ if ((p.I & 3) == 0) v = x4[b >> 2]; // rows of x start on a quad
+ else {
+ v.x = x[b];
+ if (k + 1 < p.I) v.y = x[b + 1u];
+ if (k + 2 < p.I) v.z = x[b + 2u];
+ if (k + 3 < p.I) v.w = x[b + 3u];
+ }
+ }
+ rx[r] = v;
+ }
+}
+
+void main() {
+ int tid = int(gl_LocalInvocationID.x);
+ int tm = tid % NTM, tn = tid / NTM;
+ int o0 = int(gl_WorkGroupID.x) * BM, s0 = int(gl_WorkGroupID.y) * BN;
+
+ float acc[TM][TN];
+ [[unroll]] for (int i = 0; i < TM; i++)
+ [[unroll]] for (int j = 0; j < TN; j++) acc[i][j] = 0.0;
+
+ if (PF) fetch_step(tid, o0, s0, 0);
+ for (int k0 = 0; k0 < p.I; k0 += BK) {
+ if (!PF) fetch_step(tid, o0, s0, k0);
+ // decode the fetched step into shared memory
+ [[unroll]] for (int r = 0; r < WU; r++) {
+ int u = tid + r * NT;
+ if (u < BM * (BK / 8)) {
+ int m = u / (BK / 8), kq = (u % (BK / 8)) * 2;
+ vec4 lo, hi;
+ decode8(rwa[r], rwb[r], rws[r], lo, hi);
+ wsh[m * KQ + kq] = lo;
+ wsh[m * KQ + kq + 1] = hi;
+ }
+ }
+ [[unroll]] for (int r = 0; r < XU; r++) {
+ int u = tid + r * NT;
+ if (u < BN * (BK / 4)) xsh[(u / (BK / 4)) * KQ + u % (BK / 4)] = rx[r];
+ }
+ barrier();
+ if (PF && k0 + BK < p.I) fetch_step(tid, o0, s0, k0 + BK);
+ // The step's BK products sum into a fresh partial that then joins acc: blocked
+ // summation keeps the rounding error near (BK + I/BK) ulps instead of I, the
+ // same order as the GEMV's per-lane sums plus subgroup tree.
+ float part[TM][TN];
+ [[unroll]] for (int i = 0; i < TM; i++)
+ [[unroll]] for (int j = 0; j < TN; j++) part[i][j] = 0.0;
+ [[unroll]] for (int q = 0; q < BK / 4; q++) {
+ vec4 a[TM], b[TN];
+ [[unroll]] for (int i = 0; i < TM; i++) a[i] = wsh[(tm + i * NTM) * KQ + q];
+ [[unroll]] for (int j = 0; j < TN; j++) b[j] = xsh[(tn + j * NTN) * KQ + q];
+ [[unroll]] for (int i = 0; i < TM; i++)
+ [[unroll]] for (int j = 0; j < TN; j++)
+ part[i][j] = fma(a[i].w, b[j].w, fma(a[i].z, b[j].z,
+ fma(a[i].y, b[j].y, fma(a[i].x, b[j].x, part[i][j]))));
+ }
+ [[unroll]] for (int i = 0; i < TM; i++)
+ [[unroll]] for (int j = 0; j < TN; j++) acc[i][j] += part[i][j];
+ barrier();
+ }
+
+ bool perrow = p.fmt == 1 || p.fmt == 2;
+ [[unroll]] for (int i = 0; i < TM; i++) {
+ int o = o0 + tm + i * NTM;
+ if (o >= p.O) continue;
+ float sc = perrow ? scale[o] : 1.0;
+ [[unroll]] for (int j = 0; j < TN; j++) {
+ int s = s0 + tn + j * NTN;
+ if (s < p.S) y[uint(s) * uint(p.O) + uint(o)] = acc[i][j] * sc;
+ }
+ }
+}
diff --git a/c/shaders/qmatmul_gemm.spv b/c/shaders/qmatmul_gemm.spv
new file mode 100644
index 00000000..a5a89659
Binary files /dev/null and b/c/shaders/qmatmul_gemm.spv differ