Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge #1939 into integ/release-batch-7

JustVugg88a26cbe9d04
diff --cc c/tests/test_vk_chain.c
index 60feaa14,dad7dde5..1a1e3fbb
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@@ -2394,67 -2336,37 +2394,98 @@@ static void bench_gemv(void)
}
}
+/* ---- a second device (vkc_device(1), COLI_VK_DEV2) -------------------------------------
+ * The chain's context on COLI_VK_DEV2's device: the ops against their references there,
+ * the matmul's bits equal to device 0's, and the boundary held (a buffer of one device
+ * bound or copied on the other fails; vkc_free takes it back on its own device). */
+static void test_dev2(const char *spv) {
+ const char *d2 = getenv("COLI_VK_DEV2");
+ if (!coli_vk_init_dev2(spv, strcmp(d2, "auto") ? atoi(d2) : -1)) { CHECK(0, "dev2: the second device did not open"); return; }
+ vkc_device(1);
+ if (!vkc_init()) { CHECK(0, "dev2: the chain's pipelines did not come up there"); vkc_device(0); return; }
+ vkc_device(0);
+ int fmts[4] = {1, 4, 10, 11};
+ for (int k = 0; k < 4; k++) {
+ int shapes[3][3] = {{1, 192, 70}, {40, 256, 160}, {2, 3072, 40}};
+ for (int j = 0; j < 3; j++) {
+ int S = shapes[j][0], I = shapes[j][1], O = shapes[j][2];
+ float *y0 = malloc((size_t)S * O * sizeof(float)), *y1 = malloc((size_t)S * O * sizeof(float));
+ unsigned seed = rng;
+ g_mm_out = y0; test_matmul(fmts[k], S, I, O, 0, 0);
+ rng = seed; vkc_device(1); g_test_dev = 1;
+ g_mm_out = y1; test_matmul(fmts[k], S, I, O, 0, 0);
+ g_test_dev = 0; vkc_device(0); g_mm_out = NULL;
+ CHECK(!memcmp(y0, y1, (size_t)S * O * sizeof(float)), "dev2: matmul fmt %d S %d: device 1's bits differ from device 0's", fmts[k], S);
+ free(y0); free(y1);
+ }
+ }
+ vkc_device(1);
+ test_norm(VKC_NORM_ADD1, 0); test_norm(0, 0);
+ test_rope();
+ test_attn(1, 140, 32, 0); test_attn(5, 200, 64, 0);
+ setenv("COLI_VK_ATTN_BLOCK", "16", 1); test_attn_hk(40, 7, 16, 2, 256, 0); unsetenv("COLI_VK_ATTN_BLOCK");
+ test_dnrec(8, 8, 8, 4, 0); test_dnrec(32, 100, 6, 3, 1);
+ if (vkc_kvs_ready()) test_kvs_steps();
+ /* the boundary: device 0's buffer on device 1 */
+ vkc_device(0);
+ float v[64]; for (int i = 0; i < 64; i++) v[i] = (float)i;
+ VkcBuf *a0 = up(v, 64);
+ vkc_device(1);
+ VkcBuf *a1 = up(v, 64), *b1 = vkc_buf(64 * 4, VKC_DEV);
+ vkc_begin();
+ int cross = vkc_copy(b1, 0, a0, 0, 64);
+ vkc_submit(1);
+ vkc_begin();
+ int same = vkc_copy(b1, 0, a1, 0, 64);
+ vkc_submit(1);
+ CHECK(!cross && same, "dev2: a copy across devices %s, one on device 1 %s", cross ? "ran" : "refused", same ? "ran" : "failed");
+ VkcNorm nm = {1, 64, 1, 0, 64, 64, 0, 64, 64, 0, 0, VKC_NORM_NOW, 1e-6f, 1.f};
+ vkc_begin();
+ int bound = vkc_norm(a0, a1, b1, &nm);
+ vkc_submit(1);
+ CHECK(!bound, "dev2: an op bound device 0's buffer on device 1");
+ vkc_free(a1); vkc_free(b1);
+ vkc_device(0);
+ vkc_free(a0);
+ vkc_device(1);
+ vkc_free(NULL);
+ VkcStats st; vkc_stats(&st);
+ printf("dev2: chain on the second device, %llu frames, %llu ops, %llu matmuls\n", st.frames, st.ops, st.matmuls);
+ vkc_shutdown();
+ vkc_device(0);
+}
+
+ /* A speculative verify's rows through the decode GEMV: each of S rows (2..4, one
+ * workgroup) must carry the bits a one-row call gives that row. */
+ static void test_gemv_rows(int fmt, int S, int I, int O) {
+ float *x = fvec((size_t)S * I, 1.f);
+ void *codes = NULL; float *sc = NULL; int gs = 0;
+ if (fmt == 1) {
+ int8_t *q = malloc((size_t)O * I); sc = malloc(O * sizeof *sc);
+ for (size_t i = 0; i < (size_t)O * I; i++) q[i] = (int8_t)((int)(rnd() % 255) - 127);
+ for (int o = 0; o < O; o++) sc[o] = 0.01f + (rnd() % 100) / 5000.f;
+ codes = q;
+ } else {
+ gs = 64; int ng = I / 64;
+ uint8_t *q = malloc((size_t)O * (I / 2)); sc = malloc((size_t)O * ng * sizeof *sc);
+ for (size_t i = 0; i < (size_t)O * (I / 2); i++) q[i] = (uint8_t)rnd();
+ for (int i = 0; i < O * ng; i++) sc[i] = 0.02f + (rnd() % 100) / 3000.f;
+ codes = q;
+ }
+ ColiVkTensor *t = NULL;
+ if (!coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs)) { CHECK(0, "gemv rows fmt %d: upload", fmt); return; }
+ VkcBuf *xb = vkc_buf((size_t)S * I * 4, VKC_DEV), *ya = vkc_buf((size_t)S * O * 4, VKC_DOWN), *y1 = vkc_buf((size_t)S * O * 4, VKC_DOWN);
+ vkc_gemm_rows(0); /* every row on the GEMV, as a verify's */
+ vkc_begin(); vkc_write(xb, 0, x, (size_t)S * I * 4);
+ int ok = vkc_matmul(t, xb, 0, ya, 0, S);
+ for (int s = 0; s < S; s++) ok = ok && vkc_matmul(t, xb, (size_t)s * I, y1, (size_t)s * O, 1);
+ vkc_submit(1);
+ vkc_gemm_rows(-1);
+ int same = ok && !memcmp(vkc_ptr(ya), vkc_ptr(y1), (size_t)S * O * 4);
+ CHECK(same, "gemv rows fmt %d S %d I %d O %d: a row's bits depend on the rows beside it", fmt, S, I, O);
+ vkc_free(xb); vkc_free(ya); vkc_free(y1); coli_vk_tensor_free(t);
+ free(x); free(codes); free(sc);
+ }
int main(int argc, char **argv) {
const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }
diff --cc c/tests/test_vk_tier.c
index 06d2e78a,ee59dbc9..40717624
--- a/c/tests/test_vk_tier.c
+++ b/c/tests/test_vk_tier.c
@@@ -410,94 -404,6 +410,95 @@@ static void sync_evict(void)
vkt_shutdown(); model_free();
}
+/* An extra layer: layer 0 is the model's (int4 v+8 gs64), layer 1 an MTP head's held
+ * as f32 (VktConfig.extra_layers), eight times the bytes. Its experts sit in a pool of
+ * their own with the extra layer's share of the budget; uploads awaited throughout. */
+static void extra(void) {
+ VktFmt fm = {VKT_SRC_I4U_PAIRS_GS, 64}, fx = {VKT_SRC_F32, 0};
+ layer_make(0, fm, fm); layer_make(1, fx, fx); g_act = VKT_ACT_SWIGLU; g_limit = 0;
+ size_t mb = vkt_expert_bytes(H, F, fm, fm), xb = vkt_expert_bytes(H, F, fx, fx);
+ int per = (int)((xb + mb - 1) / mb);
+ unsigned long long ev, failed;
+ int idx[4 * K]; float w[4 * K];
+ VktConfig vc = cfg_of(fm, fm, VKT_ACT_SWIGLU, 0);
+ vc.layers = 1; vc.extra_layers = 1; vc.extra_gate_up = fx; vc.extra_down = fx;
+ /* Half of a budget of 2 * per + 4 main experts is the extra layer's: one f32 expert,
+ * whose room leaves the main layer all ten of its own. Both layers route, the hot
+ * set moves halfway: each pool evicts in its own kind, every row is right. */
+ int units = 2 * per + 4;
+ set_budget(units, fm, fm);
+ setenv("COLI_VK_TIER_RATE", "64", 1);
+ setenv("COLI_VK_TIER_SYNC", "1", 1);
+ int on = vkt_init(&vc, NULL);
+ unsetenv("COLI_VK_TIER_SYNC");
+ CHECK(on && vkt_layers() == 2, "extra: the tier did not start with its extra layer (%d layers)", vkt_layers());
+ if (!on) { model_free(); return; }
+ Books b0 = {0, 0}, b1 = {0, 0}; double worst = 0;
+ for (int t = 0; t < 140; t++) /* the new hot set needs a while to outweigh the old one */
+ for (int l = 0; l < 2; l++) {
+ route(4, t < 40 ? 0 : 5, 5, idx, w);
+ double r = step(l, 4, idx, w, l ? &b1 : &b0);
+ if (r > worst) worst = r;
+ }
+ int r0 = 0, r1 = 0;
+ for (int e = 0; e < E; e++) { r0 += vkt_resident(0, e); r1 += vkt_resident(1, e); }
+ report_counts(&ev, &failed);
+ ColiVkPoolStats xp; coli_vk_pool_stats(4, &xp);
+ printf(" extra: an f32 expert is %d int4 ones; device main %llu of %llu, extra %llu of %llu; resident main %d, "
+ "extra %d (its pool %zu of %zu bytes), evictions %llu, failed %llu, worst relative error %.2e\n",
+ per, b0.dev, b0.routed, b1.dev, b1.routed, r0, r1, xp.used, xp.limit, ev, failed, worst);
+ CHECK(per > 1, "extra: the f32 form takes no more room than the int4 one (%zu, %zu bytes)", xb, mb);
+ CHECK(b0.dev > 0 && b1.dev > 0, "extra: a layer ran nothing on the device (main %llu, extra %llu)", b0.dev, b1.dev);
+ CHECK(r0 == E && r1 == 1, "extra: resident main %d (want %d), extra %d (want 1)", r0, E, r1);
+ CHECK(xp.used > 0 && xp.used <= xp.limit, "extra: the extra pool holds %zu bytes of %zu", xp.used, xp.limit);
+ CHECK(ev >= 1 && failed == 0, "extra: %llu evictions, %llu failed uploads", ev, failed);
+ CHECK(worst < 2e-3, "extra: relative error %.3g", worst);
+ vkt_shutdown();
+ /* A budget of per + 4: the extra layer's share is under one of its experts, so it
+ * gets one (two main ones still fit) and the main layer the four places left. The
+ * main layer fills them, then keeps routing and spends each forward's promotion
+ * (COLI_VK_TIER_RATE=1) before the extra layer runs: the extra layer's expert must
+ * still get in, on its own promotion. */
+ units = per + 4;
+ set_budget(units, fm, fm);
+ setenv("COLI_VK_TIER_RATE", "1", 1);
+ setenv("COLI_VK_TIER_SYNC", "1", 1);
+ on = vkt_init(&vc, NULL);
+ unsetenv("COLI_VK_TIER_SYNC");
+ CHECK(on && vkt_layers() == 2, "extra: the tier did not start for the full-tier case");
+ if (on) {
+ Books c0 = {0, 0}, c1 = {0, 0}; worst = 0;
+ for (int t = 0; t < 40; t++) { route(4, 0, E, idx, w); double r = step(0, 4, idx, w, &c0); if (r > worst) worst = r; }
+ int full = 0; for (int e = 0; e < E; e++) full += vkt_resident(0, e);
+ for (int t = 0; t < 60; t++) {
+ route(4, 0, E, idx, w); double r = step(0, 4, idx, w, &c0); if (r > worst) worst = r;
+ route(4, 0, 5, idx, w); r = step(1, 4, idx, w, &c1); if (r > worst) worst = r;
+ }
+ r0 = r1 = 0;
+ for (int e = 0; e < E; e++) { r0 += vkt_resident(0, e); r1 += vkt_resident(1, e); }
+ report_counts(&ev, &failed);
+ printf(" extra, full tier: main resident %d after the first phase; then main %d + extra %d, "
+ "extra device %llu of %llu, evictions %llu, failed %llu, worst %.2e\n",
+ full, r0, r1, c1.dev, c1.routed, ev, failed, worst);
+ CHECK(full == 4 && r0 == 4, "extra: the main layer holds %d then %d of its 4 places", full, r0);
+ CHECK(r1 == 1 && c1.dev > 0, "extra: no extra expert got in beside a full main pool (resident %d, device %llu)", r1, c1.dev);
+ CHECK(ev >= 1 && failed == 0, "extra, full tier: %llu evictions, %llu failed uploads", ev, failed);
+ CHECK(worst < 2e-3, "extra, full tier: relative error %.3g", worst);
+ vkt_shutdown();
+ }
+ /* an extra form with no device form: the main layers only */
+ vc.extra_gate_up = vc.extra_down = (VktFmt){VKT_SRC_NONE, 0};
+ on = vkt_init(&vc, NULL);
+ CHECK(on && vkt_layers() == 1, "extra: no device form, the tier has %d layers (want the main one)", vkt_layers());
+ if (on) {
- float x[H] = {0}; int id[K] = {0, 1, 2}; uint8_t taken[K];
++ float x[H]; memset(x, 0, sizeof x); /* H is a variable (grouped() widens it) */
++ int id[K] = {0, 1, 2}; uint8_t taken[K];
+ CHECK(vkt_issue(1, x, 1, K, id, taken) == 0, "extra: a layer past the tier's took rows");
+ vkt_shutdown();
+ }
+ model_free();
+}
+
/* DeepSeek V4's activation (VKT_ACT_SWIGLU_V4) through the tier: MXFP4 ue8m0 experts,
* x rounded to E4M3 per 128 here as the engine rounds it, the route weight applied on
* the device (vkt_issue_w), each device row rounded to bf16 and added with no weight
diff --cc c/vk_chain.c
index 0c50e273,7c912c29..0a3d3444
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@@ -351,9 -301,20 +352,20 @@@ int vkc_init(void)
int32_t v = xs;
VkSpecializationMapEntry me = {0, 0, 4};
VkSpecializationInfo si = {1, &me, 4, &v};
- if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;
+ if ((KC.gemv4 = make_pipe(KC.mod_gemv4, &si))) KC.gemv4_xs = xs;
}
}
+ /* the int8/int4 decode GEMV with lanes per row and rows per workgroup chosen per
+ * matrix (chain_gemv2.comp): needs clustered subgroup operations */
+ {
+ const char *e = getenv("COLI_VK_CHAIN_GEMV2");
+ VkPhysicalDeviceSubgroupProperties sgp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
+ VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &sgp};
- vkGetPhysicalDeviceProperties2((VkPhysicalDevice)K.core.phys, &p2);
++ vkGetPhysicalDeviceProperties2((VkPhysicalDevice)KC.core.phys, &p2);
+ int ok = (sgp.supportedOperations & VK_SUBGROUP_FEATURE_CLUSTERED_BIT) && (sgp.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) &&
+ sgp.subgroupSize >= 16 && !(e && *e == '0');
- if (ok && (K.mod_gv2 = load_module(K.core.spv_path, "chain_gemv2.spv"))) K.gv2 = make_pipe(K.mod_gv2, NULL);
++ if (ok && (KC.mod_gv2 = load_module(KC.core.spv_path, "chain_gemv2.spv"))) KC.gv2 = make_pipe(KC.mod_gv2, NULL);
+ }
/* the MLA, KDA and mHC shaders, optional: without one only its ops decline */
for (int i = 0; i < PM_N; i++) {
char path[1200];
@@@ -761,10 -691,20 +773,20 @@@ static int matmul_aligned(const ColiVkT
int ok;
/* the vectorized GEMV where the row is whole 16-byte steps and fits the staging */
int per = ti->fmt == 1 ? 16 : ti->fmt == 4 ? 32 : ti->fmt == 11 || ti->fmt == 14 ? 8 : ti->fmt == 10 ? 4 : 0;
- int v4 = path < 0 && K.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&
- (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= K.gemv4_xs;
- K.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;
- int g2 = path < 0 && K.gv2 && S <= 4 && ti->fmt == 1 && /* int4 measured slower than chain_gemv here */
+ int v4 = path < 0 && KC.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&
+ (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= KC.gemv4_xs;
+ KC.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;
- if (v4) {
++ int g2 = path < 0 && KC.gv2 && S <= 4 && ti->fmt == 1 && /* int4 measured slower than chain_gemv here */
+ ti->I % 32 == 0 && ti->I <= 16384 && ti->rowWords % 4 == 0;
+ if (g2) {
+ /* rows a workgroup: short matrices in small blocks, so they still fill the device */
+ int rpw = ti->O <= 1024 ? 16 : ti->O <= 4096 ? 32 : 64;
+ /* a speculative verify's rows (S <= 4) share each weight load */
+ int nr = S <= 4 && S * ti->I <= 16384 ? S : 1;
+ struct { int fmt, S, I, O, rowWords, gs, rpw, nr; } pc8 = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs, rpw, nr};
- ok = record(K.gv2, bd, 4, &pc8, sizeof pc8, (uint32_t)((ti->O + rpw - 1) / rpw), (uint32_t)((S + nr - 1) / nr), 1);
++ ok = record(KC.gv2, bd, 4, &pc8, sizeof pc8, (uint32_t)((ti->O + rpw - 1) / rpw), (uint32_t)((S + nr - 1) / nr), 1);
+ }
+ else if (v4) {
/* each workgroup stages x once: give it enough rows that the staging does not
* rival the weights, while keeping enough workgroups to fill the device */
/* lanes per row: about sixteen 16-byte steps each (COLI_VK_CHAIN_GEMV_LPR overrides) */
@@@ -1636,30 -1496,28 +1658,32 @@@ void vkc_shutdown(void)
kab_shutdown();
ke_shutdown();
for (int i = 0; i < P_NPIPE; i++) {
- if (K.pipe[i]) vkDestroyPipeline(K.dev, K.pipe[i], NULL);
- if (K.mod[i]) vkDestroyShaderModule(K.dev, K.mod[i], NULL);
+ if (KC.pipe[i]) vkDestroyPipeline(KC.dev, KC.pipe[i], NULL);
+ if (KC.mod[i]) vkDestroyShaderModule(KC.dev, KC.mod[i], NULL);
}
- for (int i = 0; i < K.nkdarec; i++) vkDestroyPipeline(K.dev, K.kdarec[i], NULL);
+ for (int i = 0; i < KC.nkdarec; i++) vkDestroyPipeline(KC.dev, KC.kdarec[i], NULL);
for (int i = 0; i < PM_N; i++) {
- if (K.mpipe[i]) vkDestroyPipeline(K.dev, K.mpipe[i], NULL);
- if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);
+ if (KC.mpipe[i]) vkDestroyPipeline(KC.dev, KC.mpipe[i], NULL);
+ if (KC.mmod[i]) vkDestroyShaderModule(KC.dev, KC.mmod[i], NULL);
}
- for (int i = 0; i < K.ngemm; i++) vkDestroyPipeline(K.dev, K.gemm[i], NULL);
- for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);
- if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);
- if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);
- if (K.gv2) vkDestroyPipeline(K.dev, K.gv2, NULL);
- if (K.mod_gv2) vkDestroyShaderModule(K.dev, K.mod_gv2, NULL);
- if (K.mod_gemm) vkDestroyShaderModule(K.dev, K.mod_gemm, NULL);
- if (K.mod_dnrec) vkDestroyShaderModule(K.dev, K.mod_dnrec, NULL);
- if (K.pl) vkDestroyPipelineLayout(K.dev, K.pl, NULL);
- if (K.dsl) vkDestroyDescriptorSetLayout(K.dev, K.dsl, NULL);
+ for (int i = 0; i < KC.ngemm; i++) vkDestroyPipeline(KC.dev, KC.gemm[i], NULL);
+ if (KC.tg) vkDestroyPipeline(KC.dev, KC.tg, NULL);
+ if (KC.mod_tg) vkDestroyShaderModule(KC.dev, KC.mod_tg, NULL);
+ for (int i = 0; i < KC.ndnrec; i++) vkDestroyPipeline(KC.dev, KC.dnrec[i], NULL);
+ if (KC.gemv4) vkDestroyPipeline(KC.dev, KC.gemv4, NULL);
+ if (KC.mod_gemv4) vkDestroyShaderModule(KC.dev, KC.mod_gemv4, NULL);
++ if (KC.gv2) vkDestroyPipeline(KC.dev, KC.gv2, NULL);
++ if (KC.mod_gv2) vkDestroyShaderModule(KC.dev, KC.mod_gv2, NULL);
+ if (KC.mod_gemm) vkDestroyShaderModule(KC.dev, KC.mod_gemm, NULL);
+ if (KC.mod_dnrec) vkDestroyShaderModule(KC.dev, KC.mod_dnrec, NULL);
+ for (int i = 0; i < KC.nfa; i++) vkDestroyPipeline(KC.dev, KC.fa[i], NULL);
+ if (KC.mod_fa) vkDestroyShaderModule(KC.dev, KC.mod_fa, NULL);
+ if (KC.pl) vkDestroyPipelineLayout(KC.dev, KC.pl, NULL);
+ if (KC.dsl) vkDestroyDescriptorSetLayout(KC.dev, KC.dsl, NULL);
dsv4_shutdown();
kvs_shutdown();
- memset(&K, 0, sizeof K);
- K.cur = -1; K.gemm_rows = -1;
+ memset(&KC, 0, sizeof KC);
+ KC.cur = -1; KC.gemm_rows = -1;
}
/* ---- DeepSeek V4.1 Flash and DeepSeek V4 attention (chain_dsv4.comp) ------------------
diff --cc docs/ENVIRONMENT.md
index a9da3002,f314f260..723fcdc6
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@@ -276,8 -266,8 +276,9 @@@ With `COLI_VULKAN=1` every MoE engine (
| `COLI_VK_ATTN_SLICE` | `4294967296` | Rows x positions x heads x head dim past which the chain's attention ops are recorded in slices of rows, a submission each, so no submission of a big prompt chunk runs past a driver's job timeout; `0` never. |
| `COLI_VK_CHAIN` (laya, gliner_decide) | on for a GPU, discrete or integrated; off on a CPU device | The decision engines' encoder forward (and Laya's head layers) recorded as one submission on the device, the activations staying there (`decide_vk.h`, [vulkan.md](vulkan.md#the-decision-engines)). `0`: matrix by matrix through the device (`COLI_VK_DENSE` decides, as above), the rest on the CPU; `1`: on. The `[VK] <engine>: forward on the device ...` line says which and why. Clef's is qwen36's chain, as above. |
| `COLI_VK_IMPORT` | on for a model without routed experts on a device sharing the CPU's RAM; off otherwise | qwen36: the device reads the int8 and f16 dense rows where the host keeps them (`VK_EXT_external_memory_host`, `coli_vk_tensor_import`) instead of holding a copy of each. `1` requests import, `0` copies. On an integrated GPU, imported rows remain shared even with `COLI_VK_DENSE_HOST=0`: they already have one physical copy, and their pages stay alive while Vulkan reads them. `COLI_VK_IMPORT=0` forces an independent device copy whose host pages can be released; only those released pages count as RAM given back. On other devices, `COLI_VK_DENSE_HOST=0` disables import. Unset, on for the dense 27B family (Qwen3.8-27B, Clef) on an integrated GPU or Lavapipe, whose trunk is the whole model and would be held twice (Clef's f16 trunk, 54 GB, twice past a 61 GB box); a copy elsewhere. Where the device cannot import (no extension, staged uploads) the rows are copied. See [vulkan.md](vulkan.md#the-decision-engines). |
+| `COLI_VK_CHAIN_FLASH` | `16` | Rows of a forward from which the chain's causal attention (no selection list, window, ring or sink, head dim a multiple of 64) runs on the matrix units (`chain_attn_flash.comp`, f16 operands) on a device with cooperative matrices at subgroup size 64. Ahead of `COLI_VK_ATTN_BLOCK`'s blocked shader. `0` never. |
| `COLI_VK_CHAIN_GEMV` | on | `0`: the chain's decode matrices take `qmatmul.comp`'s GEMV instead of `chain_gemv.comp`'s. |
+ | `COLI_VK_CHAIN_GEMV2` | on where supported | The chain's int8 decode matrices on `chain_gemv2.comp`: rows a workgroup from the matrix's length (16 up to 1024 outputs, 32 up to 4096, else 64), a row's 16-byte words over a power-of-two lane group with clustered adds, up to four rows of x (a verify's) sharing each weight load with a decode step's bits. Needs clustered subgroup operations. `0` keeps `chain_gemv.comp`. |
| `COLI_VK_CHAIN_SPIN_US` | `2000` | How long a wait on a chain frame polls before blocking. |
| `COLI_VK_CHAIN_PROF` | off | `1`: device time per kind of chain op, one line per run. |
| `COLI_VK_KV_SPLIT` | on | Past the device's budget a chain engine keeps part of each attention layer's KV cache on the device and attends over the rest on the CPU, the two parts merged (DeepSeek stages only selected missing rows to preserve sparse-attention arithmetic) (see [vulkan.md](vulkan.md#a-kv-cache-past-the-devices-budget)). `0`: never split: the whole mirrors as before, and past the budget the chain declines. |
diff --cc docs/vulkan.md
index 507b0544,3704f287..d5fbca7c
--- a/docs/vulkan.md
+++ b/docs/vulkan.md
@@@ -1103,8 -1026,8 +1103,9 @@@ f32 throughout, as the CPU's f32 path
| `COLI_VK_CHAIN` | on for a discrete GPU; on an integrated GPU with the expert tier, what the engine measured (qwen36 and olmoe on, qwen38 off; mimo, inkling, colibri, glm53, kimi_k3, deepseek_v41 and deepseek_v4 off: not measured); off on a CPU device | `1`: every layer's dense chain on the device; `2`: prompts only (forwards of more than two rows that are not a speculative verify; decode and verifies on the per-matrix path, the state moving between the two); `0`: the per-matrix path. The `[VK] <engine>: dense chain ...` line says which and why. |
| `COLI_VK_CHAIN_ROWS` | from the budget, up to 8192 | Prompt rows per chunk: a longer prompt runs every layer chunk by chunk (the device's scratch is sized for one chunk). Unset or `auto`: the most rows whose buffers fit half the free device memory, up to `COLI_VK_CHAIN_ROWS_MAX` (8192); see [big prompt chunks](#big-prompt-chunks-and-expert-streaming). It was 512 before. |
| `COLI_VK_CHAIN_LAYERS` | what fits | The engines on [the partial chain](#a-partial-chain): the first N layers on the device, the CPU the rest and the head. Unset or `auto`: the most layers the device's free memory holds (every layer when they fit, the chain as before); `n`: n layers; `0`: the chain off. |
+| `COLI_VK_CHAIN_FLASH` | `16` | Rows from which a forward's plain causal attention runs on the matrix units (`chain_attn_flash.comp`: query heads of one kv head folded into 16-row tiles, K/V staged once per block as f16, two passes); `0` never. Needs cooperative matrices at subgroup size 64; otherwise, and for a selection list, a window, a ring or a sink, the blocked shader or `chain_attn.comp`. On a Radeon 8060S (Qwen3.6, 1011-token prompt) the attention went from 186 ms (`chain_attnb`) to 22 ms, the first token from 2.08 to 1.85 s, perplexity unchanged (7.65). |
| `COLI_VK_CHAIN_GEMV` | on | `0`: the decode matrices take `qmatmul.comp`'s GEMV instead of `chain_gemv.comp`'s. |
+ | `COLI_VK_CHAIN_GEMV2` | on where supported | `0`: the int8 decode matrices keep `chain_gemv.comp` instead of `chain_gemv2.comp` (rows a workgroup by the matrix's length). On a Radeon 8060S (Qwen3.6, a 256-token answer, int8 trunk) decode went from 36.8 to 39.0 tok/s; at int4 (`COLI_DENSE_BITS=4`) it measured slower, so it takes int8 only. |
| `COLI_VK_CHAIN_SPIN_US` | `2000` | How long a wait on a chain frame polls the fence before blocking. |
| `COLI_VK_CHAIN_PROF` | off | `1`: one `[VK] chain profile` line of device time per kind of op (timestamps). |
| `COLI_VK_KV_SPLIT` | on | `0`: never split the KV cache past the device's budget ([below](#a-kv-cache-past-the-devices-budget)); the whole mirrors, and past the budget the chain declines. |