Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: the chain's context per device (vkc_device), for layers on COLI_VK_DEV2
JustVuggfb79bda81dbc
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex fc85d261..f1587027 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -4490,6 +4490,69 @@ int coli_vk_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *o) {}/* A fence the chain waited on failed: the device is gone, everyone falls back. */void coli_vk_mark_lost(void) { G.ready = 0; }++/* ---- the dense chain on either device (vk_chain.c keeps a context per device) --------+ * d = 0 is the primary device, what the calls without _dev answer; d = 1 is COLI_VK_DEV2's+ * (G2), which until now held only routed experts. The chain's layers on it run the same+ * shaders: the core below is G2's, with the primary's GEMM tiles (a tile is a speed+ * choice; every tile computes the same bits). */+int coli_vk_core_dev(int d, ColiVkCore *o) {+ if (d == 0) return coli_vk_core(o);+ if (d != 1 || !G2.ready || !G.ready || !o) return 0;+ if (!coli_vk_core(o)) return 0; /* the tiles, the shader path: the primary's */+ o->phys = (void *)G2.phys; o->device = (void *)G2.dev; o->queue = (void *)G2.queue; o->qfam = G2.qfam;+ o->memtype_host = g_up[1].on ? g_up[1].mt_stage : G2.memtype;+ o->memtype_cached = G2.memtype_cached; o->memtype_dev = G2.memtype_dev;+ o->ssbo_align = G2.ssbo_align; o->ssbo_range = G2.ssbo_range;+ o->has_prio = 0;+ o->integrated = G2.integrated; o->shares_ram = G2.shares_ram;+ return 1;+}+int coli_vk_available_dev(int d) { return d == 1 ? G2.ready : G.ready; }+void coli_vk_mark_lost_dev(int d) { if (d == 1) G2.ready = 0; else G.ready = 0; }+int coli_vk_queue_submit_dev(int d, void *queue, const void *submit_info, void *fence) {+ return (int)vk_submit(d == 1 ? 1 : 0, (VkQueue)queue, (const VkSubmitInfo *)submit_info, (VkFence)fence);+}+int coli_vk_tensor_info_dev(const ColiVkTensor *t, ColiVkTensorInfo *o, int *dev) {+ if (!t || !o || t->dev < 0 || t->dev > 1) return 0;+ o->wbuf = (void *)t->wbuf; o->sbuf = (void *)t->sbuf;+ o->fmt = t->fmt; o->I = t->I; o->O = t->O; o->rowWords = t->rowWords; o->gs = t->gs;+ if (dev) *dev = t->dev;+ return 1;+}+int coli_vk_mem_budget_dev(int d, double *used_gb, double *budget_gb) {+ return d == 1 ? coli_vk_mem_budget2(used_gb, budget_gb) : coli_vk_mem_budget(used_gb, budget_gb);+}+size_t coli_vk_device_used_dev(int d) { return (size_t)__atomic_load_n(&g_mem.used[d == 1 ? 1 : 0], __ATOMIC_RELAXED); }+size_t coli_vk_device_local_bytes_dev(int d) {+ if (d != 1) return coli_vk_device_local_bytes();+ if (!G2.phys) return 0;+ VkPhysicalDeviceMemoryProperties mp;+ vkGetPhysicalDeviceMemoryProperties(G2.phys, &mp);+ VkDeviceSize m = 0;+ for (uint32_t i = 0; i < mp.memoryHeapCount; i++)+ if ((mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && mp.memoryHeaps[i].size > m) m = mp.memoryHeaps[i].size;+ if (g_mem.cap[1] && g_mem.cap[1] < m) m = g_mem.cap[1]; /* COLI_VK_DEVICE_CAP_MB */+ return (size_t)m;+}+size_t coli_vk_free_bytes_dev(int d) {+ if (d != 1) return coli_vk_free_bytes();+ if (!G2.phys) return 0;+ if (g_mem.cap[1]) return (size_t)vk_mem_room(1);+ double used = 0, bud = 0;+ if (coli_vk_mem_budget2(&used, &bud)) return bud > used ? (size_t)((bud - used) * 1e9) : 0;+ size_t dl = coli_vk_device_local_bytes_dev(1), u = coli_vk_device_used_dev(1);+ return dl > u ? dl - u : 0;+}+void coli_vk_mem_info_dev(int d, size_t *used, size_t *count) {+ if (d != 1) { coli_vk_mem_info(used, count); return; }+ if (used) *used = (size_t)__atomic_load_n(&g_wpool2.bytes, __ATOMIC_RELAXED);+ if (count) *count = (size_t)__atomic_load_n(&g_wpool2.tensors, __ATOMIC_RELAXED);+}+size_t coli_vk_buffer_alignment_dev(int d) {+ if (d != 1) return coli_vk_buffer_alignment();+ return G2.buf_align ? G2.buf_align : 256;+}/* The chain's submits: through the lock the staged uploader takes on a shared queue. */int coli_vk_queue_submit(void *queue, const void *submit_info, void *fence) {return (int)vk_submit(0, (VkQueue)queue, (const VkSubmitInfo *)submit_info, (VkFence)fence);diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 30b41c92..457a13c5 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -411,6 +411,21 @@ void coli_vk_mark_lost(void);/* vkQueueSubmit(queue, 1, submit_info, fence) as the backend submits (a VkResult): the* staged uploader may share the main queue from its own thread. */int coli_vk_queue_submit(void *queue, const void *submit_info, void *fence);+/* The same for either device: d = 0 the primary (as the calls above), d = 1 COLI_VK_DEV2's,+ * whose context the chain opens for the layers it places there (vk_chain.c). The core of+ * device 1 carries the primary's shader path and GEMM tiles. coli_vk_tensor_info_dev+ * reports a tensor on either device and which (coli_vk_tensor_info: device 0's only). */+int coli_vk_core_dev(int d, ColiVkCore *out);+int coli_vk_available_dev(int d);+void coli_vk_mark_lost_dev(int d);+int coli_vk_queue_submit_dev(int d, void *queue, const void *submit_info, void *fence);+int coli_vk_tensor_info_dev(const ColiVkTensor *t, ColiVkTensorInfo *out, int *dev);+int coli_vk_mem_budget_dev(int d, double *used_gb, double *budget_gb);+size_t coli_vk_device_used_dev(int d);+size_t coli_vk_device_local_bytes_dev(int d);+size_t coli_vk_free_bytes_dev(int d);+size_t coli_vk_buffer_alignment_dev(int d);+void coli_vk_mem_info_dev(int d, size_t *used_bytes, size_t *tensor_count); /* dense weights on device d */#ifdef __cplusplus}diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.cindex 6b6e4f0b..1328c374 100644--- a/c/tests/test_vk_chain.c+++ b/c/tests/test_vk_chain.c@@ -102,6 +102,8 @@ static float *down(VkcBuf *b, size_t off, size_t n) {/* ---- matmul ----------------------------------------------------------------------- */static uint16_t f2bf(float f) { uint32_t u; memcpy(&u, &f, 4); return (uint16_t)((u + 0x7fff + ((u >> 16) & 1)) >> 16); }static float bf2f(uint16_t h) { uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }+static int g_test_dev; /* the device test_matmul uploads to (the chain's current one) */+static float *g_mm_out; /* test_matmul's output, kept when set to non-NULL (the two devices' bits) */static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {float *x = fvec((size_t)S * I, 1.f), *W = malloc((size_t)O * I * sizeof(float));void *codes = NULL; float *sc = NULL; int gs = 0;@@ -127,7 +129,8 @@ static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {codes = w;}ColiVkTensor *t = NULL;- if (!coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs)) { CHECK(0, "matmul fmt %d: upload", fmt); return; }+ if (!(g_test_dev ? coli_vk_tensor_ensure2(&t, codes, sc, fmt, I, O, gs) : coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs))) {+ CHECK(0, "matmul fmt %d: upload", fmt); return; }float *ref = malloc((size_t)S * O * sizeof *ref);for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {double a = 0; for (int i = 0; i < I; i++) a += (double)x[(size_t)s * I + i] * W[(size_t)o * I + i];@@ -141,6 +144,7 @@ static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {float *y = (float *)vkc_ptr(yb) + yo;double e = relerr(y, ref, (size_t)S * O, 1e-3);CHECK(ok && e < 2e-4 && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);+ if (g_mm_out) memcpy(g_mm_out, y, (size_t)S * O * sizeof(float));vkc_free(xb); vkc_free(yb); coli_vk_tensor_free(t);free(x); free(W); free(codes); free(sc); free(ref);}@@ -2336,6 +2340,67 @@ static void bench_gemv(void) {}}+/* ---- a second device (vkc_device(1), COLI_VK_DEV2) -------------------------------------+ * The chain's context on COLI_VK_DEV2's device: the ops against their references there,+ * the matmul's bits equal to device 0's, and the boundary held (a buffer of one device+ * bound or copied on the other fails; vkc_free takes it back on its own device). */+static void test_dev2(const char *spv) {+ const char *d2 = getenv("COLI_VK_DEV2");+ if (!coli_vk_init_dev2(spv, strcmp(d2, "auto") ? atoi(d2) : -1)) { CHECK(0, "dev2: the second device did not open"); return; }+ vkc_device(1);+ if (!vkc_init()) { CHECK(0, "dev2: the chain's pipelines did not come up there"); vkc_device(0); return; }+ vkc_device(0);+ int fmts[4] = {1, 4, 10, 11};+ for (int k = 0; k < 4; k++) {+ int shapes[3][3] = {{1, 192, 70}, {40, 256, 160}, {2, 3072, 40}};+ for (int j = 0; j < 3; j++) {+ int S = shapes[j][0], I = shapes[j][1], O = shapes[j][2];+ float *y0 = malloc((size_t)S * O * sizeof(float)), *y1 = malloc((size_t)S * O * sizeof(float));+ unsigned seed = rng;+ g_mm_out = y0; test_matmul(fmts[k], S, I, O, 0, 0);+ rng = seed; vkc_device(1); g_test_dev = 1;+ g_mm_out = y1; test_matmul(fmts[k], S, I, O, 0, 0);+ g_test_dev = 0; vkc_device(0); g_mm_out = NULL;+ CHECK(!memcmp(y0, y1, (size_t)S * O * sizeof(float)), "dev2: matmul fmt %d S %d: device 1's bits differ from device 0's", fmts[k], S);+ free(y0); free(y1);+ }+ }+ vkc_device(1);+ test_norm(VKC_NORM_ADD1, 0); test_norm(0, 0);+ test_rope();+ test_attn(1, 140, 32, 0); test_attn(5, 200, 64, 0);+ setenv("COLI_VK_ATTN_BLOCK", "16", 1); test_attn_hk(40, 7, 16, 2, 256, 0); unsetenv("COLI_VK_ATTN_BLOCK");+ test_dnrec(8, 8, 8, 4, 0); test_dnrec(32, 100, 6, 3, 1);+ if (vkc_kvs_ready()) test_kvs_steps();+ /* the boundary: device 0's buffer on device 1 */+ vkc_device(0);+ float v[64]; for (int i = 0; i < 64; i++) v[i] = (float)i;+ VkcBuf *a0 = up(v, 64);+ vkc_device(1);+ VkcBuf *a1 = up(v, 64), *b1 = vkc_buf(64 * 4, VKC_DEV);+ vkc_begin();+ int cross = vkc_copy(b1, 0, a0, 0, 64);+ vkc_submit(1);+ vkc_begin();+ int same = vkc_copy(b1, 0, a1, 0, 64);+ vkc_submit(1);+ CHECK(!cross && same, "dev2: a copy across devices %s, one on device 1 %s", cross ? "ran" : "refused", same ? "ran" : "failed");+ VkcNorm nm = {1, 64, 1, 0, 64, 64, 0, 64, 64, 0, 0, VKC_NORM_NOW, 1e-6f, 1.f};+ vkc_begin();+ int bound = vkc_norm(a0, a1, b1, &nm);+ vkc_submit(1);+ CHECK(!bound, "dev2: an op bound device 0's buffer on device 1");+ vkc_free(a1); vkc_free(b1);+ vkc_device(0);+ vkc_free(a0);+ vkc_device(1);+ vkc_free(NULL);+ VkcStats st; vkc_stats(&st);+ printf("dev2: chain on the second device, %llu frames, %llu ops, %llu matmuls\n", st.frames, st.ops, st.matmuls);+ vkc_shutdown();+ vkc_device(0);+}+int main(int argc, char **argv) {const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }@@ -2495,6 +2560,7 @@ int main(int argc, char **argv) {}if (!vkc_kvs_ready()) { fails++; printf("FAIL: the split KV shader did not load\n"); }else { test_kvs(); printf("kvs done\n"); }+ if (getenv("COLI_VK_DEV2")) { test_dev2(spv); printf("dev2 done\n"); }VkcStats st; vkc_stats(&st);printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);vkc_shutdown();diff --git a/c/vk_chain.c b/c/vk_chain.cindex c45c0f00..5b973b3f 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -72,6 +72,7 @@ struct VkcBuf {VkaRange r;uint8_t *ptr;unsigned long long last_frame; /* serial of the last frame that bound it */+ int d; /* the device (vkc_device) it was made on */};typedef struct {@@ -85,8 +86,9 @@ typedef struct {VkQueryPool qp; int nq; unsigned char kind[VKC_PROF_Q]; /* COLI_VK_CHAIN_PROF: a timestamp after each op */} VkcFrame;-static struct {+typedef struct {int ready, lost;+ int misuse; /* a bind of the other device's buffer: the record fails */ColiVkCore core;VkDevice dev;VkQueue queue;@@ -113,15 +115,29 @@ static struct {VkcStats st;int prof, kind; float ts_period; uint32_t ts_mask;double prof_ms[PK_N]; unsigned long long prof_n[PK_N];-} K = {.cur = -1, .gemm_rows = -1};+} VkcCtx;+/* One context per device (vkc_device): 0 the primary, 1 COLI_VK_DEV2's, for the layers an+ * engine places there. Every call works on the current device's; a buffer remembers its+ * device, and binding or copying one of the other device's is refused. */+static VkcCtx g_kctx[2] = {{.cur = -1, .gemm_rows = -1}, {.cur = -1, .gemm_rows = -1}};+static int g_kd;+#define KC (g_kctx[g_kd])++/* a weight tensor the current device holds (one on the other device is refused) */+static int vkc_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *ti) {+ int d = 0;+ return coli_vk_tensor_info_dev(t, ti, &d) && d == g_kd;+}-int vkc_ready(void) { return K.ready && !K.lost; }-int vkc_lost(void) { return K.lost; }+int vkc_ready(void) { return KC.ready && !KC.lost; }+int vkc_device(int d) { int was = g_kd; g_kd = d == 1 ? 1 : 0; return was; }+int vkc_device_now(void) { return g_kd; }+int vkc_lost(void) { return KC.lost; }static void lose(const char *what, VkResult r) {- if (!K.lost) fprintf(stderr, "[VK] chain: %s failed (%d): the device is lost\n", what, (int)r);- K.lost = 1;- coli_vk_mark_lost();+ if (!KC.lost) fprintf(stderr, "[VK] chain: %s failed (%d): the device is lost\n", what, (int)r);+ KC.lost = 1;+ coli_vk_mark_lost_dev(g_kd);}/* ---- blocks ---------------------------------------------------------------------- */@@ -133,95 +149,101 @@ static int pool_grow(VkcPool *P, uint64_t need) {#ifdef VK_EXT_memory_priorityVkMemoryPriorityAllocateInfoEXT pri = {.sType = VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT,.priority = 1.0f}; /* rides every chain submit: never evicted before weights */- if (K.core.has_prio) ai.pNext = &pri;+ if (KC.core.has_prio) ai.pNext = &pri;#endifVkcBlock *bk = calloc(1, sizeof *bk);if (!bk) return 0;- if (vkAllocateMemory(K.dev, &ai, NULL, &bk->mem) != VK_SUCCESS) { free(bk); return 0; }- if (P->mapped && vkMapMemory(K.dev, bk->mem, 0, cap, 0, (void **)&bk->map) != VK_SUCCESS) {- vkFreeMemory(K.dev, bk->mem, NULL); free(bk); return 0;+ if (vkAllocateMemory(KC.dev, &ai, NULL, &bk->mem) != VK_SUCCESS) { free(bk); return 0; }+ if (P->mapped && vkMapMemory(KC.dev, bk->mem, 0, cap, 0, (void **)&bk->map) != VK_SUCCESS) {+ vkFreeMemory(KC.dev, bk->mem, NULL); free(bk); return 0;}if (vka_pool_add_block(&P->p, cap, bk) < 0) {- if (bk->map) vkUnmapMemory(K.dev, bk->mem);- vkFreeMemory(K.dev, bk->mem, NULL); free(bk); return 0;+ if (bk->map) vkUnmapMemory(KC.dev, bk->mem);+ vkFreeMemory(KC.dev, bk->mem, NULL); free(bk); return 0;}return 1;}static uint32_t memtype_of(int kind) {- return kind == VKC_UP ? K.core.memtype_host : kind == VKC_DOWN ? K.core.memtype_cached : K.core.memtype_dev;+ return kind == VKC_UP ? KC.core.memtype_host : kind == VKC_DOWN ? KC.core.memtype_cached : KC.core.memtype_dev;}static int memtype_host_visible(uint32_t t) {VkPhysicalDeviceMemoryProperties mp;- vkGetPhysicalDeviceMemoryProperties((VkPhysicalDevice)K.core.phys, &mp);+ vkGetPhysicalDeviceMemoryProperties((VkPhysicalDevice)KC.core.phys, &mp);return (mp.memoryTypes[t].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;}-static void track_reset(void) { K.nwr = K.nrd = 0; }+static void track_reset(void) { KC.nwr = KC.nrd = 0; }static void zero_now(VkcBuf *b);VkcBuf *vkc_buf(size_t bytes, int kind) {if (!vkc_ready() || kind < 0 || kind > 2) return NULL;if (!bytes) bytes = 4;bytes = (bytes + 3) & ~(size_t)3;- if (K.core.ssbo_range && bytes > K.core.ssbo_range) return NULL; /* one binding could not address it */+ if (KC.core.ssbo_range && bytes > KC.core.ssbo_range) return NULL; /* one binding could not address it */VkcBuf *b = calloc(1, sizeof *b);if (!b) return NULL;- b->kind = kind; b->bytes = bytes;+ b->kind = kind; b->bytes = bytes; b->d = g_kd;VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bytes,.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,.sharingMode = VK_SHARING_MODE_EXCLUSIVE};- if (vkCreateBuffer(K.dev, &bi, NULL, &b->buf) != VK_SUCCESS) { free(b); return NULL; }+ if (vkCreateBuffer(KC.dev, &bi, NULL, &b->buf) != VK_SUCCESS) { free(b); return NULL; }VkMemoryRequirements req;- vkGetBufferMemoryRequirements(K.dev, b->buf, &req);- VkcPool *P = &K.pool[kind];- uint64_t align = req.alignment > K.align ? req.alignment : K.align;+ vkGetBufferMemoryRequirements(KC.dev, b->buf, &req);+ VkcPool *P = &KC.pool[kind];+ uint64_t align = req.alignment > KC.align ? req.alignment : KC.align;if (!(req.memoryTypeBits & (1u << P->memtype)) ||(!vka_alloc(&P->p, req.size, align, &b->r) &&!(pool_grow(P, req.size + align) && vka_alloc(&P->p, req.size, align, &b->r)))) {- vkDestroyBuffer(K.dev, b->buf, NULL); free(b); return NULL;+ vkDestroyBuffer(KC.dev, b->buf, NULL); free(b); return NULL;}VkcBlock *bk = P->p.b[b->r.block].user;- if (vkBindBufferMemory(K.dev, b->buf, bk->mem, b->r.off) != VK_SUCCESS) {+ if (vkBindBufferMemory(KC.dev, b->buf, bk->mem, b->r.off) != VK_SUCCESS) {if (vka_free(&P->p, b->r)) { VkcBlock *e = P->p.b[b->r.block].user;- if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }+ if (e) { if (e->map) vkUnmapMemory(KC.dev, e->mem); vkFreeMemory(KC.dev, e->mem, NULL); free(e); }vka_pool_drop_block(&P->p, b->r.block); }- vkDestroyBuffer(K.dev, b->buf, NULL); free(b); return NULL;+ vkDestroyBuffer(KC.dev, b->buf, NULL); free(b); return NULL;}b->ptr = bk->map ? bk->map + b->r.off : NULL;- K.st.dev_bytes += bytes;+ KC.st.dev_bytes += bytes;if (b->ptr) memset(b->ptr, 0, bytes);else zero_now(b);return b;}static void buf_release(VkcBuf *b) {- VkcPool *P = &K.pool[b->kind];- vkDestroyBuffer(K.dev, b->buf, NULL);+ VkcPool *P = &KC.pool[b->kind];+ vkDestroyBuffer(KC.dev, b->buf, NULL);if (vka_free(&P->p, b->r)) {VkcBlock *e = P->p.b[b->r.block].user;- if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }+ if (e) { if (e->map) vkUnmapMemory(KC.dev, e->mem); vkFreeMemory(KC.dev, e->mem, NULL); free(e); }vka_pool_drop_block(&P->p, b->r.block);}- K.st.dev_bytes -= b->bytes;+ KC.st.dev_bytes -= b->bytes;free(b);}/* a frame that may still read b must finish first */static int frame_wait(VkcFrame *f);-void vkc_free(VkcBuf *b) {- if (!b) return;- if (!K.dev) { free(b); return; }- if (!K.lost)+static void vkc_free_here(VkcBuf *b) {+ if (!KC.dev) { free(b); return; }+ if (!KC.lost)for (int i = 0; i < VKC_FRAMES; i++) {- VkcFrame *f = &K.fr[i];+ VkcFrame *f = &KC.fr[i];if ((f->inflight || f->open) && f->serial == b->last_frame) {if (f->open) vkc_submit(1); else frame_wait(f);}}- for (int i = 0; i < K.nwr; i++) if (K.wr[i] == b->buf) K.wr[i] = VK_NULL_HANDLE;- for (int i = 0; i < K.nrd; i++) if (K.rd[i] == b->buf) K.rd[i] = VK_NULL_HANDLE;+ for (int i = 0; i < KC.nwr; i++) if (KC.wr[i] == b->buf) KC.wr[i] = VK_NULL_HANDLE;+ for (int i = 0; i < KC.nrd; i++) if (KC.rd[i] == b->buf) KC.rd[i] = VK_NULL_HANDLE;buf_release(b);}+void vkc_free(VkcBuf *b) { /* on the device that made it, whichever is current */+ if (!b) return;+ int was = g_kd;+ g_kd = b->d;+ vkc_free_here(b);+ g_kd = was;+}int vkc_reserve(VkcBuf **b, size_t bytes, int kind) {if (*b && (*b)->bytes >= bytes && (*b)->kind == kind) return 1;size_t want = bytes;@@ -248,7 +270,7 @@ static VkShaderModule load_module(const char *dir_spv, const char *file) {fclose(f);VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)n, .pCode = code};VkShaderModule m = VK_NULL_HANDLE;- if (vkCreateShaderModule(K.dev, &ci, NULL, &m) != VK_SUCCESS) m = VK_NULL_HANDLE;+ if (vkCreateShaderModule(KC.dev, &ci, NULL, &m) != VK_SUCCESS) m = VK_NULL_HANDLE;free(code);if (!m) fprintf(stderr, "[VK] chain: cannot load %s\n", path);return m;@@ -256,9 +278,9 @@ static VkShaderModule load_module(const char *dir_spv, const char *file) {static VkPipeline make_pipe(VkShaderModule m, const VkSpecializationInfo *si) {VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT,- .module = m, .pName = "main", .pSpecializationInfo = si}, .layout = K.pl};+ .module = m, .pName = "main", .pSpecializationInfo = si}, .layout = KC.pl};VkPipeline p = VK_NULL_HANDLE;- if (vkCreateComputePipelines(K.dev, VK_NULL_HANDLE, 1, &ci, NULL, &p) != VK_SUCCESS) return VK_NULL_HANDLE;+ if (vkCreateComputePipelines(KC.dev, VK_NULL_HANDLE, 1, &ci, NULL, &p) != VK_SUCCESS) return VK_NULL_HANDLE;return p;}@@ -266,105 +288,105 @@ static void dsv4_init(void); /* chain_dsv4.comp (DeepSeek V4.1 / V4), belostatic void dsv4_shutdown(void);static void kvs_shutdown(void); /* chain_kvs.comp (the split KV cache), below */int vkc_init(void) {- if (K.ready) return !K.lost;- if (!coli_vk_core(&K.core)) return 0;- K.dev = (VkDevice)K.core.device; K.queue = (VkQueue)K.core.queue;- K.align = K.core.ssbo_align > 16 ? K.core.ssbo_align : 16;+ if (KC.ready) return !KC.lost;+ if (!coli_vk_core_dev(g_kd, &KC.core)) return 0;+ KC.dev = (VkDevice)KC.core.device; KC.queue = (VkQueue)KC.core.queue;+ KC.align = KC.core.ssbo_align > 16 ? KC.core.ssbo_align : 16;VkDescriptorSetLayoutBinding b[VKC_BIND];for (int i = 0; i < VKC_BIND; i++) b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i,.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,.bindingCount = VKC_BIND, .pBindings = b};VkPushConstantRange pr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = 128};- if (vkCreateDescriptorSetLayout(K.dev, &li, NULL, &K.dsl) != VK_SUCCESS) return 0;+ if (vkCreateDescriptorSetLayout(KC.dev, &li, NULL, &KC.dsl) != VK_SUCCESS) return 0;VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,- .setLayoutCount = 1, .pSetLayouts = &K.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};- if (vkCreatePipelineLayout(K.dev, &pi, NULL, &K.pl) != VK_SUCCESS) return 0;+ .setLayoutCount = 1, .pSetLayouts = &KC.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};+ if (vkCreatePipelineLayout(KC.dev, &pi, NULL, &KC.pl) != VK_SUCCESS) return 0;for (int i = 0; i < P_NPIPE; i++) {- K.mod[i] = load_module(K.core.spv_path, pipe_file[i]);- if (!K.mod[i] || !(K.pipe[i] = make_pipe(K.mod[i], NULL))) {+ KC.mod[i] = load_module(KC.core.spv_path, pipe_file[i]);+ if (!KC.mod[i] || !(KC.pipe[i] = make_pipe(KC.mod[i], NULL))) {fprintf(stderr, "[VK] chain: shader %s unavailable, the chain stays off\n", pipe_file[i]);return 0;}}- K.mod_dnrec = load_module(K.core.spv_path, "chain_dnrec.spv");- if (!K.mod_dnrec) return 0;+ KC.mod_dnrec = load_module(KC.core.spv_path, "chain_dnrec.spv");+ if (!KC.mod_dnrec) return 0;/* the vectorized decode GEMV, its x staging as large as the device's shared memory* allows (16 KiB floats at most); COLI_VK_CHAIN_GEMV=0 keeps qmatmul.comp's */{const char *e = getenv("COLI_VK_CHAIN_GEMV");- VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)KC.core.phys, &pp);int xs = (int)(pp.limits.maxComputeSharedMemorySize / 16);if (xs > 4096) xs = 4096;- if (!(e && *e == '0') && xs >= 256 && (K.mod_gemv4 = load_module(K.core.spv_path, "chain_gemv.spv"))) {+ if (!(e && *e == '0') && xs >= 256 && (KC.mod_gemv4 = load_module(KC.core.spv_path, "chain_gemv.spv"))) {int32_t v = xs;VkSpecializationMapEntry me = {0, 0, 4};VkSpecializationInfo si = {1, &me, 4, &v};- if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;+ if ((KC.gemv4 = make_pipe(KC.mod_gemv4, &si))) KC.gemv4_xs = xs;}}/* the MLA, KDA and mHC shaders, optional: without one only its ops decline */for (int i = 0; i < PM_N; i++) {char path[1200];- const char *sl = strrchr(K.core.spv_path, '/');- size_t pre = sl ? (size_t)(sl - K.core.spv_path) + 1 : 0;+ const char *sl = strrchr(KC.core.spv_path, '/');+ size_t pre = sl ? (size_t)(sl - KC.core.spv_path) + 1 : 0;FILE *f = NULL;if (pre + strlen(mla_file[i]) + 1 < sizeof path) {- memcpy(path, K.core.spv_path, pre); strcpy(path + pre, mla_file[i]);+ memcpy(path, KC.core.spv_path, pre); strcpy(path + pre, mla_file[i]);f = fopen(path, "rb");}if (f) fclose(f);- if (f && (K.mmod[i] = load_module(K.core.spv_path, mla_file[i]))) K.mpipe[i] = make_pipe(K.mmod[i], NULL);+ if (f && (KC.mmod[i] = load_module(KC.core.spv_path, mla_file[i]))) KC.mpipe[i] = make_pipe(KC.mmod[i], NULL);}- K.mla_ok = K.mpipe[PM_MLA] && K.mpipe[PM_HGEMV] && K.mpipe[PM_DSA];+ KC.mla_ok = KC.mpipe[PM_MLA] && KC.mpipe[PM_HGEMV] && KC.mpipe[PM_DSA];dsv4_init();/* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */- K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;- for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {- const int *t = K.core.gemm_tile[k];+ KC.mod_gemm = KC.core.gemm_tiles ? load_module(KC.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;+ for (int k = 0; KC.mod_gemm && k < KC.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {+ const int *t = KC.core.gemm_tile[k];int32_t sv[7] = {t[0], t[1], t[2], t[3], t[4], (t[0] / t[3]) * (t[1] / t[4]), t[5] ? 1 : 0};VkSpecializationMapEntry me[7];for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};VkSpecializationInfo si = {7, me, sizeof sv, sv};- if (!(K.gemm[k] = make_pipe(K.mod_gemm, &si))) break;- K.gemm_bm[k] = t[0]; K.gemm_bn[k] = t[1]; K.ngemm = k + 1;+ if (!(KC.gemm[k] = make_pipe(KC.mod_gemm, &si))) break;+ KC.gemm_bm[k] = t[0]; KC.gemm_bn[k] = t[1]; KC.ngemm = k + 1;}VkCommandPoolCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,- .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = K.core.qfam};- if (vkCreateCommandPool(K.dev, &cp, NULL, &K.cpool) != VK_SUCCESS) return 0;+ .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = KC.core.qfam};+ if (vkCreateCommandPool(KC.dev, &cp, NULL, &KC.cpool) != VK_SUCCESS) return 0;for (int i = 0; i < VKC_FRAMES; i++) {VkCommandBufferAllocateInfo ca = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,- .commandPool = K.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};+ .commandPool = KC.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};- if (vkAllocateCommandBuffers(K.dev, &ca, &K.fr[i].cmd) != VK_SUCCESS ||- vkCreateFence(K.dev, &fi, NULL, &K.fr[i].fence) != VK_SUCCESS) return 0;+ if (vkAllocateCommandBuffers(KC.dev, &ca, &KC.fr[i].cmd) != VK_SUCCESS ||+ vkCreateFence(KC.dev, &fi, NULL, &KC.fr[i].fence) != VK_SUCCESS) return 0;}{const char *e = getenv("COLI_VK_CHAIN_PROF");- uint32_t nqf = 0; vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)K.core.phys, &nqf, NULL);+ uint32_t nqf = 0; vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)KC.core.phys, &nqf, NULL);VkQueueFamilyProperties qf[16]; if (nqf > 16) nqf = 16;- vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)K.core.phys, &nqf, qf);- VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);- uint32_t bits = K.core.qfam < nqf ? qf[K.core.qfam].timestampValidBits : 0;+ vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)KC.core.phys, &nqf, qf);+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)KC.core.phys, &pp);+ uint32_t bits = KC.core.qfam < nqf ? qf[KC.core.qfam].timestampValidBits : 0;if (e && *e == '1' && bits && pp.limits.timestampPeriod > 0) {- K.prof = 1; K.ts_period = pp.limits.timestampPeriod;- K.ts_mask = bits >= 32 ? 0xffffffffu : (1u << bits) - 1;+ KC.prof = 1; KC.ts_period = pp.limits.timestampPeriod;+ KC.ts_mask = bits >= 32 ? 0xffffffffu : (1u << bits) - 1;for (int i = 0; i < VKC_FRAMES; i++) {VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,.queryType = VK_QUERY_TYPE_TIMESTAMP, .queryCount = VKC_PROF_Q};- if (vkCreateQueryPool(K.dev, &qi, NULL, &K.fr[i].qp) != VK_SUCCESS) K.prof = 0;+ if (vkCreateQueryPool(KC.dev, &qi, NULL, &KC.fr[i].qp) != VK_SUCCESS) KC.prof = 0;}}}for (int k = 0; k < 3; k++) {- K.pool[k].memtype = memtype_of(k);- K.pool[k].mapped = memtype_host_visible(K.pool[k].memtype);- vka_pool_init(&K.pool[k].p, coli_vk_block_bytes(VKC_BLOCK), 0); /* smaller under COLI_VK_DEVICE_CAP_MB */+ KC.pool[k].memtype = memtype_of(k);+ KC.pool[k].mapped = memtype_host_visible(KC.pool[k].memtype);+ vka_pool_init(&KC.pool[k].p, coli_vk_block_bytes_dev(g_kd, VKC_BLOCK), 0); /* smaller under COLI_VK_DEVICE_CAP_MB */}- if (!K.pool[VKC_UP].mapped || !K.pool[VKC_DOWN].mapped) return 0;- K.ready = 1;- K.dummy = vkc_buf(256, VKC_DEV);- if (!K.dummy) { K.ready = 0; return 0; }+ if (!KC.pool[VKC_UP].mapped || !KC.pool[VKC_DOWN].mapped) return 0;+ KC.ready = 1;+ KC.dummy = vkc_buf(256, VKC_DEV);+ if (!KC.dummy) { KC.ready = 0; return 0; }return 1;}@@ -373,7 +395,7 @@ static void frame_reclaim(VkcFrame *f) {for (int i = 0; i < f->ntmp; i++) buf_release(f->tmp[i]);f->ntmp = 0;f->stage_used = 0;- for (int i = 0; i < f->npools; i++) vkResetDescriptorPool(K.dev, f->pools[i], 0);+ for (int i = 0; i < f->npools; i++) vkResetDescriptorPool(KC.dev, f->pools[i], 0);f->cur_pool = 0;}/* A frame is milliseconds of work and the thread waiting on it has nothing else to do:@@ -389,18 +411,18 @@ static int frame_wait(VkcFrame *f) {double t0 = vkc_now_ms();VkResult r = VK_NOT_READY;for (long spin = chain_spin_us(); spin > 0 && r == VK_NOT_READY && (vkc_now_ms() - t0) * 1000.0 < spin; )- r = vkGetFenceStatus(K.dev, f->fence);- if (r == VK_NOT_READY) r = vkWaitForFences(K.dev, 1, &f->fence, VK_TRUE, 20000000000ULL);- K.st.wait_ms += vkc_now_ms() - t0; K.st.waits++;+ r = vkGetFenceStatus(KC.dev, f->fence);+ if (r == VK_NOT_READY) r = vkWaitForFences(KC.dev, 1, &f->fence, VK_TRUE, 20000000000ULL);+ KC.st.wait_ms += vkc_now_ms() - t0; KC.st.waits++;f->inflight = 0;if (r != VK_SUCCESS) { lose("fence wait", r); return 0; }- if (K.prof && f->qp && f->nq > 1) {+ if (KC.prof && f->qp && f->nq > 1) {uint64_t ts[VKC_PROF_Q];- if (vkGetQueryPoolResults(K.dev, f->qp, 0, (uint32_t)f->nq, sizeof ts, ts, sizeof ts[0],+ if (vkGetQueryPoolResults(KC.dev, f->qp, 0, (uint32_t)f->nq, sizeof ts, ts, sizeof ts[0],VK_QUERY_RESULT_64_BIT) == VK_SUCCESS)for (int i = 1; i < f->nq; i++) {- uint64_t d = (ts[i] - ts[i - 1]) & (K.ts_mask == 0xffffffffu ? ~0ull : (uint64_t)K.ts_mask);- K.prof_ms[f->kind[i]] += (double)d * K.ts_period / 1e6; K.prof_n[f->kind[i]]++;+ uint64_t d = (ts[i] - ts[i - 1]) & (KC.ts_mask == 0xffffffffu ? ~0ull : (uint64_t)KC.ts_mask);+ KC.prof_ms[f->kind[i]] += (double)d * KC.ts_period / 1e6; KC.prof_n[f->kind[i]]++;}f->nq = 0;}@@ -408,15 +430,15 @@ static int frame_wait(VkcFrame *f) {return 1;}static void prof_mark(VkcFrame *f, int kind) {- if (!K.prof || !f->qp || f->nq >= VKC_PROF_Q) return;+ if (!KC.prof || !f->qp || f->nq >= VKC_PROF_Q) return;vkCmdWriteTimestamp(f->cmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, f->qp, (uint32_t)f->nq);f->kind[f->nq++] = (unsigned char)kind;}int vkc_finish(void) {- if (!K.ready) return 0;- if (K.cur >= 0 && !vkc_submit(0)) return 0;- for (int i = 0; i < VKC_FRAMES; i++) if (!frame_wait(&K.fr[i])) return 0;- return !K.lost;+ if (!KC.ready) return 0;+ if (KC.cur >= 0 && !vkc_submit(0)) return 0;+ for (int i = 0; i < VKC_FRAMES; i++) if (!frame_wait(&KC.fr[i])) return 0;+ return !KC.lost;}static void barrier_now(VkCommandBuffer c, VkPipelineStageFlags dst_stage, VkAccessFlags dst) {@@ -424,7 +446,7 @@ static void barrier_now(VkCommandBuffer c, VkPipelineStageFlags dst_stage, VkAcc.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = dst};vkCmdPipelineBarrier(c, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, dst_stage,0, 1, &mb, 0, NULL, 0, NULL);- K.st.barriers++;+ KC.st.barriers++;}static const VkAccessFlags ALL_RW = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;@@ -433,25 +455,25 @@ static const VkAccessFlags ALL_RW = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADERint vkc_begin(void) {/* the backend found the device lost (a staged upload's fence): this frame fails as a* lost device's would, and the engine takes over on the CPU */- if (K.ready && !K.lost && !coli_vk_available()) lose("the backend's device", VK_ERROR_DEVICE_LOST);+ if (KC.ready && !KC.lost && !coli_vk_available_dev(g_kd)) lose("the backend's device", VK_ERROR_DEVICE_LOST);if (!vkc_ready()) return 0;- if (K.cur >= 0) return 1; /* already open */- int i = (int)(K.serial % VKC_FRAMES);- VkcFrame *f = &K.fr[i];+ if (KC.cur >= 0) return 1; /* already open */+ int i = (int)(KC.serial % VKC_FRAMES);+ VkcFrame *f = &KC.fr[i];if (!frame_wait(f)) return 0;frame_reclaim(f);if (vkResetCommandBuffer(f->cmd, 0) != VK_SUCCESS) { lose("command buffer reset", VK_ERROR_DEVICE_LOST); return 0; }VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};vkBeginCommandBuffer(f->cmd, &bi);- f->serial = ++K.serial;+ f->serial = ++KC.serial;f->open = 1;- K.cur = i;- K.bound = VK_NULL_HANDLE;+ KC.cur = i;+ KC.bound = VK_NULL_HANDLE;/* after everything submitted earlier on this queue (and the host's writes) */barrier_now(f->cmd, ALL_STAGES, ALL_RW);track_reset();- if (K.prof && f->qp) {+ if (KC.prof && f->qp) {vkCmdResetQueryPool(f->cmd, f->qp, 0, VKC_PROF_Q);f->nq = 0;vkCmdWriteTimestamp(f->cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, f->qp, 0);@@ -460,51 +482,51 @@ int vkc_begin(void) {return 1;}int vkc_submit(int wait) {- if (K.cur < 0) return vkc_ready();- VkcFrame *f = &K.fr[K.cur];+ if (KC.cur < 0) return vkc_ready();+ VkcFrame *f = &KC.fr[KC.cur];barrier_now(f->cmd, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_HOST_READ_BIT);vkEndCommandBuffer(f->cmd);- f->open = 0; K.cur = -1;- if (K.lost) return 0;+ f->open = 0; KC.cur = -1;+ if (KC.lost) return 0;{ /* COLI_VK_CHAIN_FAULT=n (tests): the n-th submission fails as a lost device would */static long fault = -2;if (fault == -2) { const char *e = getenv("COLI_VK_CHAIN_FAULT"); fault = e && *e ? atol(e) : -1; }- if (fault > 0 && (long)K.st.frames + 1 >= fault) { lose("queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }+ if (fault > 0 && (long)KC.st.frames + 1 >= fault) { lose("queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }}- vkResetFences(K.dev, 1, &f->fence);+ vkResetFences(KC.dev, 1, &f->fence);VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &f->cmd};- VkResult r = (VkResult)coli_vk_queue_submit(K.queue, &si, f->fence); /* the backend's lock, staged uploads */+ VkResult r = (VkResult)coli_vk_queue_submit_dev(g_kd, KC.queue, &si, f->fence); /* the backend's lock, staged uploads */if (r != VK_SUCCESS) { lose("queue submit", r); return 0; }f->inflight = 1;- K.st.frames++;+ KC.st.frames++;return wait ? frame_wait(f) : 1;}/* ---- recording helpers ----------------------------------------------------------- */-static VkcFrame *open_frame(void) { return K.cur >= 0 ? &K.fr[K.cur] : NULL; }+static VkcFrame *open_frame(void) { return KC.cur >= 0 ? &KC.fr[KC.cur] : NULL; }static int in_set(const VkBuffer *s, int n, VkBuffer b) { for (int i = 0; i < n; i++) if (s[i] == b) return 1; return 0; }/* a barrier before an op reading rd[] and writing wr[] when it conflicts with the ops* recorded since the last one */static void hazard(VkcFrame *f, const VkBuffer *rd, int nr, const VkBuffer *wr, int nw) {int need = 0;- for (int i = 0; i < nr && !need; i++) need = in_set(K.wr, K.nwr, rd[i]);- for (int i = 0; i < nw && !need; i++) need = in_set(K.wr, K.nwr, wr[i]) || in_set(K.rd, K.nrd, wr[i]);- if (need || K.nwr + nw > VKC_TRACK || K.nrd + nr > VKC_TRACK) {+ for (int i = 0; i < nr && !need; i++) need = in_set(KC.wr, KC.nwr, rd[i]);+ for (int i = 0; i < nw && !need; i++) need = in_set(KC.wr, KC.nwr, wr[i]) || in_set(KC.rd, KC.nrd, wr[i]);+ if (need || KC.nwr + nw > VKC_TRACK || KC.nrd + nr > VKC_TRACK) {barrier_now(f->cmd, ALL_STAGES, ALL_RW);track_reset();}- for (int i = 0; i < nr; i++) if (!in_set(K.rd, K.nrd, rd[i])) K.rd[K.nrd++] = rd[i];- for (int i = 0; i < nw; i++) if (!in_set(K.wr, K.nwr, wr[i])) K.wr[K.nwr++] = wr[i];+ for (int i = 0; i < nr; i++) if (!in_set(KC.rd, KC.nrd, rd[i])) KC.rd[KC.nrd++] = rd[i];+ for (int i = 0; i < nw; i++) if (!in_set(KC.wr, KC.nwr, wr[i])) KC.wr[KC.nwr++] = wr[i];}static VkDescriptorSet alloc_set(VkcFrame *f) {for (;;) {if (f->cur_pool < f->npools) {VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,- .descriptorPool = f->pools[f->cur_pool], .descriptorSetCount = 1, .pSetLayouts = &K.dsl};+ .descriptorPool = f->pools[f->cur_pool], .descriptorSetCount = 1, .pSetLayouts = &KC.dsl};VkDescriptorSet s;- if (vkAllocateDescriptorSets(K.dev, &da, &s) == VK_SUCCESS) return s;+ if (vkAllocateDescriptorSets(KC.dev, &da, &s) == VK_SUCCESS) return s;f->cur_pool++;continue;}@@ -514,7 +536,7 @@ static VkDescriptorSet alloc_set(VkcFrame *f) {VkDescriptorPool *n = realloc(f->pools, (size_t)(f->npools + 1) * sizeof *n);if (!n) return VK_NULL_HANDLE;f->pools = n;- if (vkCreateDescriptorPool(K.dev, &pc, NULL, &f->pools[f->npools]) != VK_SUCCESS) return VK_NULL_HANDLE;+ if (vkCreateDescriptorPool(KC.dev, &pc, NULL, &f->pools[f->npools]) != VK_SUCCESS) return VK_NULL_HANDLE;f->npools++;}}@@ -524,7 +546,8 @@ typedef struct { VkBuffer buf; VkDeviceSize off, range; int written; } VkcBind;static int record(VkPipeline pipe, const VkcBind *bd, int n, const void *pc, size_t pcb,uint32_t gx, uint32_t gy, uint32_t gz) {VkcFrame *f = open_frame();- if (!f || K.lost || !gx || !gy || !gz) return f && !K.lost;+ if (KC.misuse) { KC.misuse = 0; return 0; }+ if (!f || KC.lost || !gx || !gy || !gz) return f && !KC.lost;VkBuffer rd[VKC_BIND], wr[VKC_BIND]; int nr = 0, nw = 0;VkDescriptorBufferInfo bi[VKC_BIND];VkWriteDescriptorSet w[VKC_BIND];@@ -534,23 +557,28 @@ static int record(VkPipeline pipe, const VkcBind *bd, int n, const void *pc, sizif (i < n && bd[i].buf) {bi[i] = (VkDescriptorBufferInfo){bd[i].buf, bd[i].off, bd[i].range ? bd[i].range : VK_WHOLE_SIZE};if (bd[i].written) wr[nw++] = bd[i].buf; else rd[nr++] = bd[i].buf;- } else bi[i] = (VkDescriptorBufferInfo){K.dummy->buf, 0, VK_WHOLE_SIZE};+ } else bi[i] = (VkDescriptorBufferInfo){KC.dummy->buf, 0, VK_WHOLE_SIZE};w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = set,.dstBinding = (uint32_t)i, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,.pBufferInfo = &bi[i]};}- vkUpdateDescriptorSets(K.dev, VKC_BIND, w, 0, NULL);+ vkUpdateDescriptorSets(KC.dev, VKC_BIND, w, 0, NULL);hazard(f, rd, nr, wr, nw);- if (K.bound != pipe) { vkCmdBindPipeline(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); K.bound = pipe; }- vkCmdBindDescriptorSets(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, K.pl, 0, 1, &set, 0, NULL);- if (pcb) vkCmdPushConstants(f->cmd, K.pl, VK_SHADER_STAGE_COMPUTE_BIT, 0, (uint32_t)pcb, pc);+ if (KC.bound != pipe) { vkCmdBindPipeline(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); KC.bound = pipe; }+ vkCmdBindDescriptorSets(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, KC.pl, 0, 1, &set, 0, NULL);+ if (pcb) vkCmdPushConstants(f->cmd, KC.pl, VK_SHADER_STAGE_COMPUTE_BIT, 0, (uint32_t)pcb, pc);vkCmdDispatch(f->cmd, gx, gy, gz);- K.st.ops++;- prof_mark(f, K.kind);+ KC.st.ops++;+ prof_mark(f, KC.kind);return 1;}static VkcBind B(VkcBuf *b, int written) {- if (b) b->last_frame = K.serial;+ if (b && b->d != g_kd) { /* the other device's: the record that binds it fails */+ if (!KC.misuse) fprintf(stderr, "[VK] chain: a buffer of device %d bound on device %d\n", b->d, g_kd);+ KC.misuse = 1;+ return (VkcBind){VK_NULL_HANDLE, 0, 0, written};+ }+ if (b) b->last_frame = KC.serial;return (VkcBind){b ? b->buf : VK_NULL_HANDLE, 0, 0, written};}/* groups over x, spilling into y past the 65535 limit */@@ -563,15 +591,16 @@ static void grid(uint64_t groups, uint32_t *gx, uint32_t *gy) {/* ---- transfers ------------------------------------------------------------------- */static int copy_bytes(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t bytes) {VkcFrame *f = open_frame();- if (!f || K.lost) return 0;+ if (!f || KC.lost) return 0;if (!bytes) return 1;if (doff + bytes > dst->bytes || soff + bytes > src->bytes) { fprintf(stderr, "[VK] chain: copy out of range\n"); return 0; }+ if (dst->d != g_kd || src->d != g_kd) { fprintf(stderr, "[VK] chain: a copy across devices\n"); return 0; }VkBuffer r = src->buf, w = dst->buf;hazard(f, &r, 1, &w, 1);VkBufferCopy c = {soff, doff, bytes};vkCmdCopyBuffer(f->cmd, src->buf, dst->buf, 1, &c);prof_mark(f, PK_COPY);- src->last_frame = dst->last_frame = K.serial;+ src->last_frame = dst->last_frame = KC.serial;return 1;}int vkc_copy(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t n) {@@ -579,7 +608,7 @@ int vkc_copy(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t n) {}int vkc_copy_regions(VkcBuf *dst, VkcBuf *src, const VkcRegion *r, int n) {VkcFrame *f = open_frame();- if (!f || K.lost) return 0;+ if (!f || KC.lost) return 0;if (n < 1) return 1;VkBufferCopy *c = malloc((size_t)n * sizeof *c);if (!c) return 0;@@ -593,31 +622,31 @@ int vkc_copy_regions(VkcBuf *dst, VkcBuf *src, const VkcRegion *r, int n) {hazard(f, &rb, 1, &wb, 1);vkCmdCopyBuffer(f->cmd, src->buf, dst->buf, (uint32_t)n, c);prof_mark(f, PK_COPY);- src->last_frame = dst->last_frame = K.serial;+ src->last_frame = dst->last_frame = KC.serial;free(c);return 1;}int vkc_zero(VkcBuf *dst, size_t off, size_t n) {VkcFrame *f = open_frame();- if (!f || K.lost) return 0;+ if (!f || KC.lost) return 0;if (!n) return 1;if ((off + n) * 4 > dst->bytes) return 0;VkBuffer w = dst->buf;hazard(f, NULL, 0, &w, 1);vkCmdFillBuffer(f->cmd, dst->buf, off * 4, n * 4, 0);- dst->last_frame = K.serial;+ dst->last_frame = KC.serial;return 1;}static void zero_now(VkcBuf *b) {/* a device-only buffer starts zeroed too: its own little frame */- int was_open = K.cur >= 0;+ int was_open = KC.cur >= 0;if (!was_open && !vkc_begin()) return;vkc_zero(b, 0, b->bytes / 4);if (!was_open) vkc_submit(1);}int vkc_write(VkcBuf *dst, size_t off, const void *src, size_t bytes) {VkcFrame *f = open_frame();- if (!f || K.lost) return 0;+ if (!f || KC.lost) return 0;if (!bytes) return 1;/* always through the staging, even into host-visible memory: the copy runs in* frame order, after the ops recorded before it (a direct memcpy would not) */@@ -641,7 +670,7 @@ int vkc_write(VkcBuf *dst, size_t off, const void *src, size_t bytes) {}memcpy(f->stage->ptr + at, src, bytes);f->stage_used = at + bytes;- K.st.bytes_up += bytes;+ KC.st.bytes_up += bytes;return copy_bytes(dst, off * 4, f->stage, at, bytes);}int vkc_read(VkcBuf *src, size_t off, void *dst, size_t bytes) {@@ -650,38 +679,38 @@ int vkc_read(VkcBuf *src, size_t off, void *dst, size_t bytes) {if (!vkc_finish()) return 0;if (src->ptr && src->kind != VKC_UP) { /* the device wrote it into host-readable memory */memcpy(dst, src->ptr + off * 4, bytes);- K.st.bytes_down += bytes;+ KC.st.bytes_down += bytes;return 1;}VkcBuf *rb = vkc_buf(bytes, VKC_DOWN);if (!rb) return 0;int ok = vkc_begin() && copy_bytes(rb, 0, src, off * 4, bytes) && vkc_submit(1);- if (ok) { memcpy(dst, rb->ptr, bytes); K.st.bytes_down += bytes; }+ if (ok) { memcpy(dst, rb->ptr, bytes); KC.st.bytes_down += bytes; }vkc_free(rb);return ok;}/* ---- matmul ---------------------------------------------------------------------- */-void vkc_gemm_rows(int rows) { K.gemm_rows = rows; }+void vkc_gemm_rows(int rows) { KC.gemm_rows = rows; }struct VkcPC { int fmt, S, I, O, rowWords, gs; };static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcBuf *y, size_t yb, int S) {int path = -1; /* GEMM slot, -1 the GEMV */- if (K.ngemm && S >= 2) {- int take = K.gemm_rows < 0 ? (K.core.gemm_min_s && S >= K.core.gemm_min_s &&- (int64_t)S * ti->O >= K.core.gemm_min_so)- : (K.gemm_rows > 0 && S >= K.gemm_rows);- if (take) { path = 0; while (path + 1 < K.ngemm && S > K.gemm_bn[path]) path++; }+ if (KC.ngemm && S >= 2) {+ int take = KC.gemm_rows < 0 ? (KC.core.gemm_min_s && S >= KC.core.gemm_min_s &&+ (int64_t)S * ti->O >= KC.core.gemm_min_so)+ : (KC.gemm_rows > 0 && S >= KC.gemm_rows);+ if (take) { path = 0; while (path + 1 < KC.ngemm && S > KC.gemm_bn[path]) path++; }}VkcBind bd[4] = {{x->buf, xb, 0, 0}, {(VkBuffer)ti->wbuf, 0, 0, 0}, {(VkBuffer)ti->sbuf, 0, 0, 0},{y->buf, yb, 0, 1}};- x->last_frame = y->last_frame = K.serial;+ x->last_frame = y->last_frame = KC.serial;struct VkcPC pc = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs};int ok;/* the vectorized GEMV where the row is whole 16-byte steps and fits the staging */int per = ti->fmt == 1 ? 16 : ti->fmt == 4 ? 32 : ti->fmt == 11 || ti->fmt == 14 ? 8 : ti->fmt == 10 ? 4 : 0;- int v4 = path < 0 && K.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&- (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= K.gemv4_xs;- K.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;+ int v4 = path < 0 && KC.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&+ (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= KC.gemv4_xs;+ KC.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;if (v4) {/* each workgroup stages x once: give it enough rows that the staging does not* rival the weights, while keeping enough workgroups to fill the device */@@ -693,21 +722,21 @@ static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcBstruct { int fmt, S, I, O, rowWords, gs, lpr; } pc7 = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs, lpr};int rows_wg = 256 / lpr; /* at least, at subgroups of 64 */int wg = (ti->O + rows_wg - 1) / rows_wg; if (wg > 1024) wg = 1024;- ok = record(K.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);+ ok = record(KC.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);}else if (path >= 0)- ok = record(K.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + K.gemm_bm[path] - 1) / K.gemm_bm[path]),- (uint32_t)((S + K.gemm_bn[path] - 1) / K.gemm_bn[path]), 1);- else ok = record(K.pipe[P_GEMV], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + 7) / 8), (uint32_t)S, 1);- K.st.matmuls += ok; K.st.gemms += ok && path >= 0;+ ok = record(KC.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + KC.gemm_bm[path] - 1) / KC.gemm_bm[path]),+ (uint32_t)((S + KC.gemm_bn[path] - 1) / KC.gemm_bn[path]), 1);+ else ok = record(KC.pipe[P_GEMV], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + 7) / 8), (uint32_t)S, 1);+ KC.st.matmuls += ok; KC.st.gemms += ok && path >= 0;return ok;}int vkc_matmul(ColiVkTensor *t, VkcBuf *x, size_t xo, VkcBuf *y, size_t yo, int S) {ColiVkTensorInfo ti;- if (!open_frame() || K.lost || S < 1 || S > 65535 || !coli_vk_tensor_info(t, &ti)) return 0;+ if (!open_frame() || KC.lost || S < 1 || S > 65535 || !vkc_tensor_info(t, &ti)) return 0;size_t xb = xo * 4, yb = yo * 4, xn = (size_t)S * ti.I * 4, yn = (size_t)S * ti.O * 4;if (xb + xn > x->bytes || yb + yn > y->bytes) { fprintf(stderr, "[VK] chain: matmul out of range\n"); return 0; }- if (xb % K.align == 0 && yb % K.align == 0) return matmul_aligned(&ti, x, xb, y, yb, S);+ if (xb % KC.align == 0 && yb % KC.align == 0) return matmul_aligned(&ti, x, xb, y, yb, S);/* an offset the device cannot bind: through temporaries that live with the frame */VkcFrame *f = open_frame();VkcBuf *tx = vkc_buf(xn, VKC_DEV), *ty = vkc_buf(yn, VKC_DEV);@@ -724,16 +753,16 @@ int vkc_matmul(ColiVkTensor *t, VkcBuf *x, size_t xo, VkcBuf *y, size_t yo, int/* ---- the chain's shaders ----------------------------------------------------------- */int vkc_norm(VkcBuf *x, VkcBuf *w, VkcBuf *y, const VkcNorm *p) {- K.kind = PK_NORM;+ KC.kind = PK_NORM;VkcBind bd[3] = {B(x, x == y), B(w, 0), B(y, 1)};uint32_t gx, gy; grid((uint64_t)p->nseg, &gx, &gy);- return record(K.pipe[P_NORM], bd, 3, p, sizeof *p, gx, gy, 1);+ return record(KC.pipe[P_NORM], bd, 3, p, sizeof *p, gx, gy, 1);}int vkc_rope(VkcBuf *x, VkcBuf *cs, const VkcRope *p) {- K.kind = PK_ROPE;+ KC.kind = PK_ROPE;VkcBind bd[2] = {B(x, 1), B(cs, 0)};uint32_t gx, gy; grid(((uint64_t)p->nseg * p->half_ + 63) / 64, &gx, &gy);- return record(K.pipe[P_ROPE], bd, 2, p, sizeof *p, gx, gy, 1);+ return record(KC.pipe[P_ROPE], bd, 2, p, sizeof *p, gx, gy, 1);}int vkc_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf *sel, const VkcAttn *p) {VkcAttnW w;@@ -747,17 +776,18 @@ int vkc_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf* Radeon 780M. From COLI_VK_ATTN_BLOCK rows (16; 0 = never) a call takes chain_attnb,* one workgroup per KV head and block of rows: each K and V row read once per block.* Below it, and for decode and an MTP verify, chain_attn's bits as before. */-static struct { VkShaderModule mod; VkPipeline pipe; int tried, bc; } KAB;+static struct { VkShaderModule mod; VkPipeline pipe; int tried, bc; } g_kab[2];+#define KAB (g_kab[g_kd])static VkPipeline kab_pipe(void) {if (KAB.pipe || KAB.tried || !vkc_ready()) return KAB.pipe;KAB.tried = 1;/* the tile of positions as the device's shared memory allows: K and V rows of 256* floats, the partial dots and the scores, about 3.2 KiB a position */- VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)KC.core.phys, &pp);int bc = 16;while (bc >= 4 && (size_t)bc * (2 * 256 + 32 * 8 + 32 + 1) * 4 > pp.limits.maxComputeSharedMemorySize) bc /= 2;if (bc < 4) return VK_NULL_HANDLE;- if ((KAB.mod = load_module(K.core.spv_path, "chain_attnb.spv"))) {+ if ((KAB.mod = load_module(KC.core.spv_path, "chain_attnb.spv"))) {int32_t v = bc;VkSpecializationMapEntry me = {0, 0, 4};VkSpecializationInfo si = {1, &me, 4, &v};@@ -766,8 +796,8 @@ static VkPipeline kab_pipe(void) {return KAB.pipe;}static void kab_shutdown(void) {- if (KAB.pipe) vkDestroyPipeline(K.dev, KAB.pipe, NULL);- if (KAB.mod) vkDestroyShaderModule(K.dev, KAB.mod, NULL);+ if (KAB.pipe) vkDestroyPipeline(KC.dev, KAB.pipe, NULL);+ if (KAB.mod) vkDestroyShaderModule(KC.dev, KAB.mod, NULL);memset(&KAB, 0, sizeof KAB);}/* ---- attention over a prompt chunk, cut over several submissions ----------------@@ -796,7 +826,7 @@ static int attn_slice_rows(int S, int pos_base, double width, int align) {return rr >= S ? S : (int)rr;}static int attn_slice_next(void) { /* the open frame out, a new one in */- K.st.attn_slices++;+ KC.st.attn_slices++;return vkc_submit(0) && vkc_begin();}int vkc_attn_block_rows(void) {@@ -806,7 +836,7 @@ int vkc_attn_block_rows(void) {}int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf *sel, VkcBuf *snk,const VkcAttnW *p) {- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcAttnW w = *p; /* the shader's push constants: VkcAttn's fields, then these */if (w.vd <= 0) w.vd = w.a.hd;if (w.a.hd > 256 || w.vd > 256 || w.a.H % w.a.KVH || w.win < 0 || w.ring < 0 || (w.sink && !snk)) return 0;@@ -825,32 +855,32 @@ int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuif (blocked) {struct { VkcAttnW w; int br; } pc = {x, br};ok = record(KAB.pipe, bd, 7, &pc, sizeof pc, (uint32_t)x.a.KVH, (uint32_t)((x.a.S + br - 1) / br), 1);- K.st.attn_blocked += ok;- } else ok = record(K.pipe[P_ATTN], bd, 7, &x, sizeof x, (uint32_t)x.a.H, (uint32_t)x.a.S, 1);+ KC.st.attn_blocked += ok;+ } else ok = record(KC.pipe[P_ATTN], bd, 7, &x, sizeof x, (uint32_t)x.a.H, (uint32_t)x.a.S, 1);}return ok;}int vkc_dnconv(VkcBuf *in, VkcBuf *w, VkcBuf *ring, VkcBuf *out, VkcBuf *snap, const VkcDnConv *p) {- K.kind = PK_DNCONV;+ KC.kind = PK_DNCONV;if (p->CK < 1 || p->CK > 9) return 0;VkcBind bd[5] = {B(in, 0), B(w, 0), B(ring, 1), B(out, 1), B(snap, snap != NULL && snap != ring)};if (snap == ring) bd[4].written = 0; /* the same buffer: tracked once, as written */uint32_t gx, gy; grid(((uint64_t)p->CD + 63) / 64, &gx, &gy);- return record(K.pipe[P_DNCONV], bd, 5, p, sizeof *p, gx, gy, 1);+ return record(KC.pipe[P_DNCONV], bd, 5, p, sizeof *p, gx, gy, 1);}static VkPipeline dnrec_pipe(int KD) {- for (int i = 0; i < K.ndnrec; i++) if (K.dnrec_kd[i] == KD) return K.dnrec[i];- if (K.ndnrec == VKC_DNREC_MAX) return VK_NULL_HANDLE;+ for (int i = 0; i < KC.ndnrec; i++) if (KC.dnrec_kd[i] == KD) return KC.dnrec[i];+ if (KC.ndnrec == VKC_DNREC_MAX) return VK_NULL_HANDLE;int32_t kd = KD;VkSpecializationMapEntry me = {0, 0, 4};VkSpecializationInfo si = {1, &me, 4, &kd};- VkPipeline p = make_pipe(K.mod_dnrec, &si);+ VkPipeline p = make_pipe(KC.mod_dnrec, &si);if (!p) return VK_NULL_HANDLE;- K.dnrec[K.ndnrec] = p; K.dnrec_kd[K.ndnrec++] = KD;+ KC.dnrec[KC.ndnrec] = p; KC.dnrec_kd[KC.ndnrec++] = KD;return p;}int vkc_dnrec(int KD, VkcBuf *cv, VkcBuf *ab, VkcBuf *z, VkcBuf *st, VkcBuf *prm, VkcBuf *y, VkcBuf *snap, const VkcDnRec *p) {- K.kind = PK_DNREC;+ KC.kind = PK_DNREC;if (p->VD > 128 || KD < 1 || KD > 256 || p->VH % p->KH) return 0;VkPipeline pipe = dnrec_pipe(KD);if (!pipe) return 0;@@ -859,7 +889,7 @@ int vkc_dnrec(int KD, VkcBuf *cv, VkcBuf *ab, VkcBuf *z, VkcBuf *st, VkcBuf *prmreturn record(pipe, bd, 7, p, sizeof *p, (uint32_t)p->VH, 1, 1);}int vkc_ew(VkcBuf *y, VkcBuf *a, VkcBuf *b, VkcBuf *c, VkcBuf *e, const VkcEw *p) {- K.kind = PK_EW;+ KC.kind = PK_EW;VkcBind bd[5] = {B(y, 1), B(a == y ? NULL : a, 0), B(b == y ? NULL : b, 0), B(c == y ? NULL : c, 0), B(e == y ? NULL : e, 0)};/* an input that is y itself reads through binding 0's alias: bind it there too */if (a == y) bd[1] = (VkcBind){y->buf, 0, 0, 0};@@ -867,38 +897,38 @@ int vkc_ew(VkcBuf *y, VkcBuf *a, VkcBuf *b, VkcBuf *c, VkcBuf *e, const VkcEw *pif (c == y) bd[3] = (VkcBind){y->buf, 0, 0, 0};if (e == y) bd[4] = (VkcBind){y->buf, 0, 0, 0};uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);- return record(K.pipe[P_EW], bd, 5, p, sizeof *p, gx, gy, 1);+ return record(KC.pipe[P_EW], bd, 5, p, sizeof *p, gx, gy, 1);}int vkc_qsa(VkcBuf *src, VkcBuf *w, VkcBuf *pk, VkcBuf *cs, VkcBuf *sc, VkcBuf *sel, const VkcQsa *p) {- K.kind = PK_QSA;+ KC.kind = PK_QSA;if (p->ID > 256) return 0;if (p->mode == 0) {VkcBind bd[4] = {B(src, 0), B(w, 0), B(pk, 1), B(cs, 0)};- return record(K.pipe[P_QSA], bd, 4, p, sizeof *p, (uint32_t)p->nb, 1, 1);+ return record(KC.pipe[P_QSA], bd, 4, p, sizeof *p, (uint32_t)p->nb, 1, 1);}VkcBind bd[6] = {B(src, 0), {VK_NULL_HANDLE, 0, 0, 0}, B(pk, 0), {VK_NULL_HANDLE, 0, 0, 0}, B(sc, 1), B(sel, 1)};- return record(K.pipe[P_QSA], bd, 6, p, sizeof *p, (uint32_t)p->S, 1, 1);+ return record(KC.pipe[P_QSA], bd, 6, p, sizeof *p, (uint32_t)p->S, 1, 1);}int vkc_ple(VkcBuf *keys, VkcBuf *hyp, VkcBuf *val, VkcBuf *prm, VkcBuf *gated, VkcBuf *normv,VkcBuf *conv, VkcBuf *ring, const VkcPle *p) {- K.kind = PK_PLE;+ KC.kind = PK_PLE;if (p->mode == 0) {VkcBind bd[6] = {B(keys, 0), B(hyp, 0), B(val, 0), B(prm, 0), B(gated, 1), B(normv, 1)};- return record(K.pipe[P_PLE], bd, 6, p, sizeof *p, (uint32_t)(p->S * p->C), 1, 1);+ return record(KC.pipe[P_PLE], bd, 6, p, sizeof *p, (uint32_t)(p->S * p->C), 1, 1);}if ((p->CK - 1) * p->NG > 32) return 0;VkcBind bd[8] = {{VK_NULL_HANDLE, 0, 0, 0}, B(hyp, 1), {VK_NULL_HANDLE, 0, 0, 0}, {VK_NULL_HANDLE, 0, 0, 0},B(gated, 0), B(normv, 0), B(conv, 0), B(ring, 1)};uint32_t gx, gy; grid(((uint64_t)p->C * p->H + 255) / 256, &gx, &gy);- return record(K.pipe[P_PLE], bd, 8, p, sizeof *p, gx, gy, 1);+ return record(KC.pipe[P_PLE], bd, 8, p, sizeof *p, gx, gy, 1);}/* ---- multi-head latent attention ------------------------------------------------- */-int vkc_mla_ready(void) { return vkc_ready() && K.mla_ok; }+int vkc_mla_ready(void) { return vkc_ready() && KC.mla_ok; }int vkc_mla_core(VkcBuf *qabs, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sel, VkcBuf *clat, const VkcMlaCore *p) {- K.kind = PK_MLA;- if (!K.mla_ok || p->S < 1 || p->H < 1 || p->K < 1 || p->K > 1024 || p->R < 0 || p->R > 128 ||+ KC.kind = PK_MLA;+ if (!KC.mla_ok || p->S < 1 || p->H < 1 || p->K < 1 || p->K > 1024 || p->R < 0 || p->R > 128 ||(p->R > 0 && !rope) || p->pos_base < 0 || p->kv_start < 0 || p->kv_start > p->pos_base) return 0;typedef struct { int mode; VkcMlaCore b; } CorePC;CorePC pc = {0, *p};@@ -913,27 +943,27 @@ int vkc_mla_core(VkcBuf *qabs, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sex.b.qa_off += r0 * p->qa_row; x.b.qr_off += r0 * p->qr_row; x.b.o_off += r0 * p->o_row;if (pc.b.sel_row > 0) x.b.sel_off += r0 * pc.b.sel_row;if (r0 && !attn_slice_next()) return 0;- ok = record(K.mpipe[PM_MLA], bd, 6, &x, sizeof x, (uint32_t)p->H, (uint32_t)x.b.S, 1);+ ok = record(KC.mpipe[PM_MLA], bd, 6, &x, sizeof x, (uint32_t)p->H, (uint32_t)x.b.S, 1);}return ok;}static int mla_rowop(int mode, VkcBuf *x, VkcBuf *aux, VkcBuf *y, const VkcMlaRow *p) {- if (!K.mla_ok || !open_frame() || K.lost || p->nseg < 0) return 0;+ if (!KC.mla_ok || !open_frame() || KC.lost || p->nseg < 0) return 0;if (p->nseg == 0) return 1;if (mode == 1 && (p->rd < 2 || p->rd > 256 || (p->rd & 1) || p->rd > p->seg_len)) return 0;if (mode == 2 && p->seg_len < 1) return 0;struct { int mode; VkcMlaRow b; int copy_rest; } pc = {mode, *p, x != y};VkcBind bd[6] = {B(x, x == y), B(aux, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};uint32_t gx, gy; grid((uint64_t)p->nseg, &gx, &gy);- return record(K.mpipe[PM_MLA], bd, 6, &pc, sizeof pc, gx, gy, 1);+ return record(KC.mpipe[PM_MLA], bd, 6, &pc, sizeof pc, gx, gy, 1);}-int vkc_mla_rope(VkcBuf *x, VkcBuf *cs, VkcBuf *y, const VkcMlaRow *p) { K.kind = PK_ROPE; return mla_rowop(1, x, cs, y, p); }-int vkc_mla_lnorm(VkcBuf *x, VkcBuf *prm, VkcBuf *y, const VkcMlaRow *p) { K.kind = PK_NORM; return mla_rowop(2, x, prm, y, p); }+int vkc_mla_rope(VkcBuf *x, VkcBuf *cs, VkcBuf *y, const VkcMlaRow *p) { KC.kind = PK_ROPE; return mla_rowop(1, x, cs, y, p); }+int vkc_mla_lnorm(VkcBuf *x, VkcBuf *prm, VkcBuf *y, const VkcMlaRow *p) { KC.kind = PK_NORM; return mla_rowop(2, x, prm, y, p); }int vkc_mla_hgemv(ColiVkTensor *t, VkcBuf *x, VkcBuf *y, VkcBuf *gate, const VkcHgemv *p) {- K.kind = PK_MLAW;+ KC.kind = PK_MLAW;ColiVkTensorInfo ti;- if (!K.mla_ok || !open_frame() || K.lost || !coli_vk_tensor_info(t, &ti) || p->S < 1 || p->S > 65535 ||+ if (!KC.mla_ok || !open_frame() || KC.lost || !vkc_tensor_info(t, &ti) || p->S < 1 || p->S > 65535 ||p->H < 1 || p->H > 65535 || p->n < 1 || (p->trans && p->n > 1024)) return 0;int f = ti.fmt;if (!(f == 1 || f == 2 || f == 4 || f == 5 || f == 7 || f == 10 || f == 11 || f == 12 || f == 13)) return 0;@@ -943,61 +973,61 @@ int vkc_mla_hgemv(ColiVkTensor *t, VkcBuf *x, VkcBuf *y, VkcBuf *gate, const Vkcg_off, g_row, has_gate; } pc = {p->trans, f, ti.I, ti.rowWords, ti.gs, p->S, p->H, p->n, p->hstride, p->hoff,p->x_off, p->x_row, p->x_seg, p->y_off, p->y_row, p->y_seg, p->g_off, p->g_row, gate && p->has_gate};VkcBind bd[5] = {B(x, 0), {(VkBuffer)ti.wbuf, 0, 0, 0}, {(VkBuffer)ti.sbuf, 0, 0, 0}, B(y, 1), B(pc.has_gate ? gate : NULL, 0)};- if (p->trans) return record(K.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);+ if (p->trans) return record(KC.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);/* rows: a subgroup each, grid-stride (at least four subgroups a workgroup) */int64_t rows = (int64_t)p->H * p->n, wg = (rows + 7) / 8;if (wg > 4096) wg = 4096;- return record(K.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)wg, (uint32_t)p->S, 1);+ return record(KC.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)wg, (uint32_t)p->S, 1);}int vkc_dsa_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *sc, VkcBuf *sel, const VkcDsa *p) {- K.kind = PK_DSA;- if (!K.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||+ KC.kind = PK_DSA;+ if (!KC.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||p->topk < 1 || p->pos_base < 0 || p->sc_row < p->pos_base + p->S ||p->sel_row < 1 + (p->topk < p->pos_base + p->S ? p->topk : p->pos_base + p->S)) return 0;struct { int mode; VkcDsa b; } pc = {0, *p};VkcBind bd[5] = {B(iq, 0), B(hw, 0), B(keys, 0), B(sc, 1), B(sel, 1)};- return record(K.mpipe[PM_DSA], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);+ return record(KC.mpipe[PM_DSA], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);}/* k-pooling (chain_dsa.comp modes 1 and 2) */int vkc_dsa_pool_keys(VkcBuf *keys, VkcBuf *gates, VkcBuf *prm, VkcBuf *pk, const VkcDsaPool *p) {- K.kind = PK_DSA;- if (!K.mla_ok || p->np < 0 || p->pool < 1 || p->p0 < 0 || p->ID < 1) return 0;- if (p->np == 0) return open_frame() && !K.lost;+ KC.kind = PK_DSA;+ if (!KC.mla_ok || p->np < 0 || p->pool < 1 || p->p0 < 0 || p->ID < 1) return 0;+ if (p->np == 0) return open_frame() && !KC.lost;struct { int mode; VkcDsaPool b; } pc = {1, *p};VkcBind bd[8] = {B(NULL, 0), B(NULL, 0), B(keys, 0), B(NULL, 0), B(NULL, 0), B(pk, 1), B(gates, 0), B(prm, 0)};- return record(K.mpipe[PM_DSA], bd, 8, &pc, sizeof pc, (uint32_t)p->np, 1, 1);+ return record(KC.mpipe[PM_DSA], bd, 8, &pc, sizeof pc, (uint32_t)p->np, 1, 1);}int vkc_dsa_pool_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *pk, VkcBuf *sc, VkcBuf *sel, const VkcDsaPick *p) {- K.kind = PK_DSA;- if (!K.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||+ KC.kind = PK_DSA;+ if (!KC.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||p->pool < 1 || p->topk < p->pool || p->topk % p->pool || p->topk / p->pool > 1024 || p->pos_base < 0 ||p->sc_row < (p->pos_base + p->S) / p->pool || p->sel_row < 1 + p->topk + (p->tail ? p->pool - 1 : 0)) return 0;struct { int mode; VkcDsaPick b; } pc = {2, *p};VkcBind bd[6] = {B(iq, 0), B(hw, 0), B(NULL, 0), B(sc, 1), B(sel, 1), B(pk, 0)};- return record(K.mpipe[PM_DSA], bd, 6, &pc, sizeof pc, (uint32_t)p->S, 1, 1);+ return record(KC.mpipe[PM_DSA], bd, 6, &pc, sizeof pc, (uint32_t)p->S, 1, 1);}/* ---- Kimi Delta Attention (chain_kda.comp) ------------------------------------------ */-int vkc_kda_ready(void) { return vkc_ready() && K.mpipe[PM_KDA]; }+int vkc_kda_ready(void) { return vkc_ready() && KC.mpipe[PM_KDA]; }int vkc_kda_conv(VkcBuf *in, VkcBuf *w, VkcBuf *win, VkcBuf *out, const VkcKdaConv *p) {- K.kind = PK_KDA;- if (!K.mpipe[PM_KDA] || p->K < 1 || p->K > 8 || p->P < 1 || p->C < 1 || p->S < 1) return 0;+ KC.kind = PK_KDA;+ if (!KC.mpipe[PM_KDA] || p->K < 1 || p->K > 8 || p->P < 1 || p->C < 1 || p->S < 1) return 0;struct { int mode; VkcKdaConv b; } pc = {0, *p};VkcBind bd[4] = {B(in, 0), B(w, 0), B(win, 1), B(out, 1)};uint32_t gx, gy; grid(((uint64_t)p->C + 127) / 128, &gx, &gy);- return record(K.mpipe[PM_KDA], bd, 4, &pc, sizeof pc, gx, gy, 1);+ return record(KC.mpipe[PM_KDA], bd, 4, &pc, sizeof pc, gx, gy, 1);}static VkPipeline kda_pipe(int KD) {- for (int i = 0; i < K.nkdarec; i++) if (K.kdarec_kd[i] == KD) return K.kdarec[i];- if (K.nkdarec == VKC_KDA_MAX || !K.mmod[PM_KDA]) return VK_NULL_HANDLE;+ for (int i = 0; i < KC.nkdarec; i++) if (KC.kdarec_kd[i] == KD) return KC.kdarec[i];+ if (KC.nkdarec == VKC_KDA_MAX || !KC.mmod[PM_KDA]) return VK_NULL_HANDLE;int32_t kd = KD;VkSpecializationMapEntry me = {0, 0, 4};VkSpecializationInfo si = {1, &me, 4, &kd};- VkPipeline p = make_pipe(K.mmod[PM_KDA], &si);+ VkPipeline p = make_pipe(KC.mmod[PM_KDA], &si);if (!p) return VK_NULL_HANDLE;- K.kdarec[K.nkdarec] = p; K.kdarec_kd[K.nkdarec++] = KD;+ KC.kdarec[KC.nkdarec] = p; KC.kdarec_kd[KC.nkdarec++] = KD;return p;}int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y, const VkcKdaRec *p) {@@ -1005,8 +1035,8 @@ int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm,}int vkc_kda_rec_flags(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y,const VkcKdaRec *p, int flags) {- K.kind = PK_KDA;- if (!K.mpipe[PM_KDA] || KD < 1 || KD > 256 || p->VD < 1 || p->VD > 128 || p->H < 1 || p->S < 1 ||+ KC.kind = PK_KDA;+ if (!KC.mpipe[PM_KDA] || KD < 1 || KD > 256 || p->VD < 1 || p->VD > 128 || p->H < 1 || p->S < 1 ||(flags & ~(VKC_KDA_EXP_A | VKC_KDA_K3))) return 0;VkPipeline pipe = kda_pipe(KD);if (!pipe) return 0;@@ -1016,39 +1046,39 @@ int vkc_kda_rec_flags(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf}/* ---- manifold-constrained hyper-connections (chain_mhc.comp) ------------------------ */-int vkc_mhc_ready(void) { return vkc_ready() && K.mpipe[PM_MHC]; }+int vkc_mhc_ready(void) { return vkc_ready() && KC.mpipe[PM_MHC]; }int vkc_mhc(int mode, VkcBuf *x, VkcBuf *m, VkcBuf *hp, VkcBuf *prm, VkcBuf *y, const VkcMhc *p) {- K.kind = PK_MHC;- if (!K.mpipe[PM_MHC] || mode < 0 || mode > 4 || p->S < 0 || (mode != 4 && (p->H < 1 || p->H > 8 || p->D < 1))) return 0;+ KC.kind = PK_MHC;+ if (!KC.mpipe[PM_MHC] || mode < 0 || mode > 4 || p->S < 0 || (mode != 4 && (p->H < 1 || p->H > 8 || p->D < 1))) return 0;if (mode == 0 && p->iters < 1) return 0;struct { int mode; VkcMhc b; } pc = {mode, *p};VkcBind bd[5] = {B(x, 0), B(m, 0), B(hp, mode == 0), B(prm, 0), B(y, mode != 0)};uint64_t n = mode == 4 ? (uint64_t)p->n : mode == 2 ? (uint64_t)p->S * p->H * p->D : (uint64_t)p->S * p->D;- if (mode == 0) return p->S == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);+ if (mode == 0) return p->S == 0 || record(KC.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);uint32_t gx, gy; grid((n + 255) / 256, &gx, &gy);- return n == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, gx, gy, 1);+ return n == 0 || record(KC.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, gx, gy, 1);}/* ---- attention residuals and SiTU-GLU (chain_ares.comp) ---------------------------- */-int vkc_ares_ready(void) { return vkc_ready() && K.mpipe[PM_ARES]; }+int vkc_ares_ready(void) { return vkc_ready() && KC.mpipe[PM_ARES]; }int vkc_ares_mix(VkcBuf *x, VkcBuf *blk, VkcBuf *prm, VkcBuf *y, const VkcAres *p) {- K.kind = PK_ARES;- if (!K.mpipe[PM_ARES] || p->S < 0 || p->D < 1 || p->nb < 0 || p->nb > 15 || (p->nb > 0 && !blk) || y == x || y == blk) return 0;- if (p->S == 0) return open_frame() && !K.lost;+ KC.kind = PK_ARES;+ if (!KC.mpipe[PM_ARES] || p->S < 0 || p->D < 1 || p->nb < 0 || p->nb > 15 || (p->nb > 0 && !blk) || y == x || y == blk) return 0;+ if (p->S == 0) return open_frame() && !KC.lost;struct { int mode; VkcAres b; } pc = {0, *p};VkcBind bd[4] = {B(x, 0), B(p->nb > 0 ? blk : NULL, 0), B(prm, 0), B(y, 1)};uint32_t gx, gy; grid((uint64_t)p->S, &gx, &gy);- return record(K.mpipe[PM_ARES], bd, 4, &pc, sizeof pc, gx, gy, 1);+ return record(KC.mpipe[PM_ARES], bd, 4, &pc, sizeof pc, gx, gy, 1);}int vkc_situ(VkcBuf *g, VkcBuf *u, VkcBuf *y, const VkcSitu *p) {- K.kind = PK_ARES;- if (!K.mpipe[PM_ARES] || p->n < 0) return 0;- if (p->n == 0) return open_frame() && !K.lost;+ KC.kind = PK_ARES;+ if (!KC.mpipe[PM_ARES] || p->n < 0) return 0;+ if (p->n == 0) return open_frame() && !KC.lost;int32_t pc[13] = {1, p->n, 0, 0, p->g_off, 0, p->u_off, 0, 0, p->y_off, 0, 0, 0};memcpy(&pc[11], &p->b1, 4); memcpy(&pc[12], &p->b2, 4);VkcBind bd[4] = {B(g, 0), B(u, 0), B(NULL, 0), B(y, 1)};uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);- return record(K.mpipe[PM_ARES], bd, 4, pc, sizeof pc, gx, gy, 1);+ return record(KC.mpipe[PM_ARES], bd, 4, pc, sizeof pc, gx, gy, 1);}int vkc_mla_scratch(VkcMlaScratch *s, const VkcMla *m, int rows) {@@ -1070,12 +1100,12 @@ void vkc_mla_scratch_free(VkcMlaScratch *s) {/* the tensor is there and has the shape [O x I] */static int mla_shape(ColiVkTensor *t, int O, int I) {ColiVkTensorInfo ti;- return t && coli_vk_tensor_info(t, &ti) && ti.I == I && ti.O == O;+ return t && vkc_tensor_info(t, &ti) && ti.I == I && ti.O == O;}int vkc_mla_qkv(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int S, int pos_base,VkcBuf *cs, VkcMlaCache *c, VkcBuf *down, size_t down_off) {int H = m->H, QR = m->Q + m->R, KR = m->K + m->R;- if (!K.mla_ok || S < 1 || S > s->rows || pos_base < 0 || (int64_t)pos_base + S > c->cap || m->R < 0 ||+ if (!KC.mla_ok || S < 1 || S > s->rows || pos_base < 0 || (int64_t)pos_base + S > c->cap || m->R < 0 ||(m->R & 1) || m->R > 128 || m->K < 1 || m->K > 1024 || (m->R > 0 && (!cs || !c->rope)) ||!mla_shape(m->q_b, H * QR, m->q_lora > 0 ? m->q_lora : m->D) || !mla_shape(m->kv_a, KR, m->D) ||(m->q_lora > 0 && !mla_shape(m->q_a, m->q_lora, m->D))) return 0;@@ -1101,7 +1131,7 @@ int vkc_mla_qkv(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, intint vkc_mla_attn(const VkcMla *m, VkcMlaScratch *s, int S, int pos_base, int kv_start, VkcMlaCache *c,VkcBuf *sel, size_t sel_off, int sel_row, VkcBuf *gate, size_t gate_off, VkcBuf *out, size_t out_off) {int H = m->H, QR = m->Q + m->R, HK = H * m->K, HV = H * m->V;- if (!K.mla_ok || S < 1 || S > s->rows || (int64_t)pos_base + S > c->cap) return 0;+ if (!KC.mla_ok || S < 1 || S > s->rows || (int64_t)pos_base + S > c->cap) return 0;if (m->kv_b ? !mla_shape(m->kv_b, H * (m->Q + m->V), m->K): !(mla_shape(m->k_abs, HK, m->Q) && mla_shape(m->v_abs, HV, m->K))) return 0;if (m->o && out && !mla_shape(m->o, m->D, HV)) return 0;@@ -1128,18 +1158,19 @@ int vkc_mla(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int S, ivkc_mla_attn(m, s, S, pos_base, kv_start, c, NULL, 0, 0, NULL, 0, out, out_off);}/* ---- inkling's ops: chain_sconv.comp and chain_relattn.comp, made on first use ---- */-static struct { VkShaderModule mod[2]; VkPipeline pipe[2]; int tried[2]; } KX;+static struct { VkShaderModule mod[2]; VkPipeline pipe[2]; int tried[2]; } g_kx[2];+#define KX (g_kx[g_kd])static VkPipeline kx_pipe(int k) {static const char *const file[2] = {"chain_sconv.spv", "chain_relattn.spv"};if (KX.pipe[k] || KX.tried[k] || !vkc_ready()) return KX.pipe[k];KX.tried[k] = 1;- if ((KX.mod[k] = load_module(K.core.spv_path, file[k]))) KX.pipe[k] = make_pipe(KX.mod[k], NULL);+ if ((KX.mod[k] = load_module(KC.core.spv_path, file[k]))) KX.pipe[k] = make_pipe(KX.mod[k], NULL);return KX.pipe[k];}static void kx_shutdown(void) {for (int k = 0; k < 2; k++) {- if (KX.pipe[k]) vkDestroyPipeline(K.dev, KX.pipe[k], NULL);- if (KX.mod[k]) vkDestroyShaderModule(K.dev, KX.mod[k], NULL);+ if (KX.pipe[k]) vkDestroyPipeline(KC.dev, KX.pipe[k], NULL);+ if (KX.mod[k]) vkDestroyShaderModule(KC.dev, KX.mod[k], NULL);}memset(&KX, 0, sizeof KX);}@@ -1148,7 +1179,7 @@ int vkc_relattn_ready(void) { return kx_pipe(1) != VK_NULL_HANDLE; }int vkc_sconv(VkcBuf *x, VkcBuf *w, VkcBuf *ring, const VkcSconv *p) {VkPipeline pipe = kx_pipe(0);if (!pipe || (p->mode == 0 && (p->CK < 1 || p->CK > 9))) return 0;- K.kind = p->mode == 0 ? PK_DNCONV : PK_EW;+ KC.kind = p->mode == 0 ? PK_DNCONV : PK_EW;uint32_t gx, gy;if (p->mode == 0) {VkcBind bd[3] = {B(x, 1), B(w, 0), B(ring, 1)};@@ -1163,7 +1194,7 @@ int vkc_relattn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *kvs, VkcBuconst VkcRelAttn *p) {VkPipeline pipe = kx_pipe(1);if (!pipe || p->hd > 256 || p->d_rel > 64 || p->d_rel < 0 || p->KVH < 1 || p->H % p->KVH || p->cap < 1) return 0;- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[8] = {B(q, 0), B(kc, 0), B(vc, 0), B(o, 1), B(kvs, 0), B(r, 0), B(relp, 0), B(tau, 0)};/* a long prompt chunk in slices of rows, a submission each (attn_slice_rows): the* shader's s0 is the slice's first row, pos_base stays the step's */@@ -1179,16 +1210,17 @@ int vkc_relattn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *kvs, VkcBu}/* ---- the decision engines' encoders: chain_enc.comp, made on first use ---- */-static struct { VkShaderModule mod; VkPipeline pipe; int tried; } KE;+static struct { VkShaderModule mod; VkPipeline pipe; int tried; } g_ke[2];+#define KE (g_ke[g_kd])static VkPipeline ke_pipe(void) {if (KE.pipe || KE.tried || !vkc_ready()) return KE.pipe;KE.tried = 1;- if ((KE.mod = load_module(K.core.spv_path, "chain_enc.spv"))) KE.pipe = make_pipe(KE.mod, NULL);+ if ((KE.mod = load_module(KC.core.spv_path, "chain_enc.spv"))) KE.pipe = make_pipe(KE.mod, NULL);return KE.pipe;}static void ke_shutdown(void) {- if (KE.pipe) vkDestroyPipeline(K.dev, KE.pipe, NULL);- if (KE.mod) vkDestroyShaderModule(K.dev, KE.mod, NULL);+ if (KE.pipe) vkDestroyPipeline(KC.dev, KE.pipe, NULL);+ if (KE.mod) vkDestroyShaderModule(KC.dev, KE.mod, NULL);memset(&KE, 0, sizeof KE);}int vkc_enc_ready(void) { return ke_pipe() != VK_NULL_HANDLE; }@@ -1196,7 +1228,7 @@ static int ke_ew(int kind, int mode, void *pc, size_t bytes, const VkcBind *bd,VkPipeline pipe = ke_pipe();if (!pipe) return 0;*(int *)pc = mode;- K.kind = kind;+ KC.kind = kind;uint32_t gx, gy; grid((n + 255) / 256, &gx, &gy);return record(pipe, bd, nb, pc, bytes, gx, gy, 1);}@@ -1205,7 +1237,7 @@ int vkc_enc_norm(VkcBuf *a, VkcBuf *b, VkcBuf *prm, VkcBuf *y, const VkcEncNormif (!pipe || p->D < 1 || p->D > 4096) return 0;VkcEncNorm c = *p;c.mode = 0;- K.kind = PK_NORM;+ KC.kind = PK_NORM;VkcBind bd[4] = {B(a, (p->flags & VKC_ENC_SUM) != 0), B((p->flags & VKC_ENC_ADD) ? b : NULL, 0), B(prm, 0), B(y, 1)};uint32_t gx, gy; grid((uint64_t)p->rows, &gx, &gy);return record(pipe, bd, 4, &c, sizeof c, gx, gy, 1);@@ -1237,7 +1269,7 @@ int vkc_enc_attn(VkcBuf *qkv, VkcBuf *c2p, VkcBuf *p2c, VkcBuf *o, VkcBuf *rng,if (!pipe || p->hd < 1 || p->hd > 128 || p->S < 1 || p->H < 1) return 0;VkcEncAttn c = *p;c.mode = 5;- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[7] = {B(qkv, 0), B(c2p, 0), B(p2c, 0), B(o, 1), B(rng, 0), B(ridx, 0), B(pos, 0)};return record(pipe, bd, 7, &c, sizeof c, (uint32_t)p->H, (uint32_t)((p->S + 15) / 16), 1);}@@ -1246,12 +1278,12 @@ int vkc_enc_rel(VkcBuf *x, VkcBuf *pt, VkcBuf *y, const VkcEncRel *p) {if (!pipe || p->hd < 1 || p->S < 1 || p->H < 1 || p->nr < 1) return 0;VkcEncRel c = *p;c.mode = 6;- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[4] = {B(x, 0), B(pt, 0), B(NULL, 0), B(y, 1)};return record(pipe, bd, 4, &c, sizeof c, (uint32_t)((p->nr + 63) / 64), (uint32_t)((p->S + 15) / 16), (uint32_t)p->H);}-void vkc_stats(VkcStats *st) { *st = K.st; }+void vkc_stats(VkcStats *st) { *st = KC.st; }/* ---- big prefill chunks ------------------------------------------------------------* The rows of a prompt chunk (vk_chain.h, vkc_chunk_rows). The free device memory is@@ -1267,15 +1299,15 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes) {int v = atoi(e);return v < 1 ? 1 : v > 65535 ? 65535 : v;}- static char said[16][24]; static int nsaid, rows_said[16];- for (int i = 0; i < nsaid; i++) if (!strcmp(said[i], engine ? engine : "")) return rows_said[i];+ static char said[16][24]; static int nsaid, rows_said[16], said_d[16];+ for (int i = 0; i < nsaid; i++) if (!strcmp(said[i], engine ? engine : "") && said_d[i] == g_kd) return rows_said[i];const char *mx = getenv("COLI_VK_CHAIN_ROWS_MAX");int cap = mx && *mx ? atoi(mx) : 8192;if (cap < 1) cap = 1;if (cap > 65535) cap = 65535;double used = 0, bud = 0, free_b;- int have = coli_vk_mem_budget(&used, &bud);- free_b = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes() / 4;+ int have = coli_vk_mem_budget_dev(g_kd, &used, &bud);+ free_b = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes_dev(g_kd) / 4;/* Expert outputs and CPU fallback scratch also grow with the chunk on a* discrete device. Bound both memories, including Windows commit headroom. */double avail = compat_mem_available_gb() * 1e9;@@ -1287,7 +1319,7 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes) {if (rows < 1) rows = 1;if (nsaid < 16) {snprintf(said[nsaid], sizeof said[nsaid], "%s", engine ? engine : "");- rows_said[nsaid++] = rows;+ said_d[nsaid] = g_kd; rows_said[nsaid++] = rows;fprintf(stderr, "[VK] %s chain: prompt chunks of up to %d rows (%.0f KiB a row; %.2f GiB free on the device%s, ""half of it at most; COLI_VK_CHAIN_ROWS sets it, COLI_VK_CHAIN_ROWS_MAX caps it at %d)\n",engine ? engine : "engine", rows, row_bytes / 1024.0, free_b / 1073741824.0,@@ -1310,7 +1342,7 @@ size_t vkc_fit_reserve(void) {/* A weight pool's block, a block of each of the chain's three pools, and the frames'* first staging buffers: what the pools may hold beyond the bytes asked for. */size_t vkc_fit_pools(void) {- return coli_vk_block_bytes((size_t)256 << 20) + 3 * (size_t)coli_vk_block_bytes(VKC_BLOCK) ++ return coli_vk_block_bytes_dev(g_kd, (size_t)256 << 20) + 3 * (size_t)coli_vk_block_bytes_dev(g_kd, VKC_BLOCK) +(size_t)VKC_FRAMES * ((size_t)4 << 20);}int vkc_fit_rows(int def) {@@ -1322,13 +1354,13 @@ int vkc_fit_rows(int def) {return def < 1 ? 1 : def;}size_t vkc_fit_tensor(int fmt, int I, int O, int gs) {- size_t a = coli_vk_buffer_alignment();+ size_t a = coli_vk_buffer_alignment_dev(g_kd);size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * sizeof(float);return fit_up(rows ? rows : 4, a) + fit_up(sc ? sc : 4, a);}size_t vkc_fit_buf(size_t bytes) {- size_t a = K.align > 16 ? K.align : 16, b = coli_vk_buffer_alignment();+ size_t a = KC.align > 16 ? KC.align : 16, b = coli_vk_buffer_alignment_dev(g_kd);if (b > a) a = b;size_t v = bytes ? bytes : 4;return fit_up((v + 3) & ~(size_t)3, a);@@ -1348,7 +1380,7 @@ int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *if (fit->mat && matrix_bytes) fit->mat[i] = matrix_bytes[i];all += b;}- fit->free_b = coli_vk_free_bytes();+ fit->free_b = coli_vk_free_bytes_dev(g_kd);fit->reserve = vkc_fit_reserve();fit->pools = vkc_fit_pools();fit->engine_fixed = fixed_bytes;@@ -1417,11 +1449,11 @@ void vkc_fit_mark(VkcFit *fit, int layer) {void vkc_fit_placed(const char *engine, const VkcFit *fit) {if (!fit) return;size_t w = 0, nt = 0, m = 0;- coli_vk_mem_info(&w, &nt);+ coli_vk_mem_info_dev(g_kd, &w, &nt);for (int i = 0; i < fit->n && fit->mat; i++) m += fit->mat[i];fprintf(stderr, "[VK] %s chain: %d of %d layers placed: %zu B of matrices on the device (the fit counted %zu B for these ""layers), chain buffers %zu B, device memory held %zu B", engine ? engine : "engine", fit->n, fit->L, w, m,- K.ready ? K.st.dev_bytes : (size_t)0, coli_vk_device_used());+ KC.ready ? KC.st.dev_bytes : (size_t)0, coli_vk_device_used_dev(g_kd));if (coli_vk_fault_set() && fit->marks) {fprintf(stderr, "; COLI_VK_STAGED_FAULT's point reached");for (int i = 0; i < fit->L; i++) fprintf(stderr, "%s%llu", i ? "," : " ", fit->marks[i]);@@ -1437,83 +1469,84 @@ void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb)}void vkc_prof_print(void) {- if (!K.prof) return;- double tot = 0; for (int k = 0; k < PK_N; k++) tot += K.prof_ms[k];+ if (!KC.prof) return;+ double tot = 0; for (int k = 0; k < PK_N; k++) tot += KC.prof_ms[k];fprintf(stderr, "[VK] chain profile: %.1f ms of device time", tot);for (int k = 0; k < PK_N; k++)- if (K.prof_n[k]) fprintf(stderr, " | %s %.1f ms (%llu)", pk_name[k], K.prof_ms[k], K.prof_n[k]);+ if (KC.prof_n[k]) fprintf(stderr, " | %s %.1f ms (%llu)", pk_name[k], KC.prof_ms[k], KC.prof_n[k]);fprintf(stderr, "\n");}void vkc_shutdown(void) {- if (!K.ready) return;- if (!K.lost && coli_vk_available()) vkc_finish();- else vkDeviceWaitIdle(K.dev);+ if (!KC.ready) return;+ if (!KC.lost && coli_vk_available_dev(g_kd)) vkc_finish();+ else vkDeviceWaitIdle(KC.dev);for (int i = 0; i < VKC_FRAMES; i++) {- VkcFrame *f = &K.fr[i];+ VkcFrame *f = &KC.fr[i];frame_reclaim(f);if (f->stage) buf_release(f->stage);free(f->tmp);- for (int k = 0; k < f->npools; k++) vkDestroyDescriptorPool(K.dev, f->pools[k], NULL);+ for (int k = 0; k < f->npools; k++) vkDestroyDescriptorPool(KC.dev, f->pools[k], NULL);free(f->pools);- if (f->fence) vkDestroyFence(K.dev, f->fence, NULL);- if (f->qp) vkDestroyQueryPool(K.dev, f->qp, NULL);+ if (f->fence) vkDestroyFence(KC.dev, f->fence, NULL);+ if (f->qp) vkDestroyQueryPool(KC.dev, f->qp, NULL);}- if (K.dummy) buf_release(K.dummy);+ if (KC.dummy) buf_release(KC.dummy);for (int k = 0; k < 3; k++) {- VkaPool *P = &K.pool[k].p;+ VkaPool *P = &KC.pool[k].p;for (int b = 0; b < P->nb; b++) {if (!P->b[b].present) continue;VkcBlock *e = P->b[b].user;- if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }+ if (e) { if (e->map) vkUnmapMemory(KC.dev, e->mem); vkFreeMemory(KC.dev, e->mem, NULL); free(e); }}vka_pool_destroy(P);}- if (K.cpool) vkDestroyCommandPool(K.dev, K.cpool, NULL);+ if (KC.cpool) vkDestroyCommandPool(KC.dev, KC.cpool, NULL);kx_shutdown();kab_shutdown();ke_shutdown();for (int i = 0; i < P_NPIPE; i++) {- if (K.pipe[i]) vkDestroyPipeline(K.dev, K.pipe[i], NULL);- if (K.mod[i]) vkDestroyShaderModule(K.dev, K.mod[i], NULL);+ if (KC.pipe[i]) vkDestroyPipeline(KC.dev, KC.pipe[i], NULL);+ if (KC.mod[i]) vkDestroyShaderModule(KC.dev, KC.mod[i], NULL);}- for (int i = 0; i < K.nkdarec; i++) vkDestroyPipeline(K.dev, K.kdarec[i], NULL);+ for (int i = 0; i < KC.nkdarec; i++) vkDestroyPipeline(KC.dev, KC.kdarec[i], NULL);for (int i = 0; i < PM_N; i++) {- if (K.mpipe[i]) vkDestroyPipeline(K.dev, K.mpipe[i], NULL);- if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);+ if (KC.mpipe[i]) vkDestroyPipeline(KC.dev, KC.mpipe[i], NULL);+ if (KC.mmod[i]) vkDestroyShaderModule(KC.dev, KC.mmod[i], NULL);}- for (int i = 0; i < K.ngemm; i++) vkDestroyPipeline(K.dev, K.gemm[i], NULL);- for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);- if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);- if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);- if (K.mod_gemm) vkDestroyShaderModule(K.dev, K.mod_gemm, NULL);- if (K.mod_dnrec) vkDestroyShaderModule(K.dev, K.mod_dnrec, NULL);- if (K.pl) vkDestroyPipelineLayout(K.dev, K.pl, NULL);- if (K.dsl) vkDestroyDescriptorSetLayout(K.dev, K.dsl, NULL);+ for (int i = 0; i < KC.ngemm; i++) vkDestroyPipeline(KC.dev, KC.gemm[i], NULL);+ for (int i = 0; i < KC.ndnrec; i++) vkDestroyPipeline(KC.dev, KC.dnrec[i], NULL);+ if (KC.gemv4) vkDestroyPipeline(KC.dev, KC.gemv4, NULL);+ if (KC.mod_gemv4) vkDestroyShaderModule(KC.dev, KC.mod_gemv4, NULL);+ if (KC.mod_gemm) vkDestroyShaderModule(KC.dev, KC.mod_gemm, NULL);+ if (KC.mod_dnrec) vkDestroyShaderModule(KC.dev, KC.mod_dnrec, NULL);+ if (KC.pl) vkDestroyPipelineLayout(KC.dev, KC.pl, NULL);+ if (KC.dsl) vkDestroyDescriptorSetLayout(KC.dev, KC.dsl, NULL);dsv4_shutdown();kvs_shutdown();- memset(&K, 0, sizeof K);- K.cur = -1; K.gemm_rows = -1;+ memset(&KC, 0, sizeof KC);+ KC.cur = -1; KC.gemm_rows = -1;}/* ---- DeepSeek V4.1 Flash and DeepSeek V4 attention (chain_dsv4.comp) ------------------* Its own optional pipeline: without the shader only these ops decline. */-static struct { VkShaderModule mod; VkPipeline pipe; } D4;+static struct { VkShaderModule mod; VkPipeline pipe; } g_d4[2];+#define D4 (g_d4[g_kd])static void dsv4_init(void) {char path[1200];- const char *sl = strrchr(K.core.spv_path, '/');- size_t pre = sl ? (size_t)(sl - K.core.spv_path) + 1 : 0;+ const char *sl = strrchr(KC.core.spv_path, '/');+ size_t pre = sl ? (size_t)(sl - KC.core.spv_path) + 1 : 0;const char *file = "chain_dsv4.spv";if (pre + strlen(file) + 1 >= sizeof path) return;- memcpy(path, K.core.spv_path, pre); strcpy(path + pre, file);+ memcpy(path, KC.core.spv_path, pre); strcpy(path + pre, file);FILE *f = fopen(path, "rb");if (!f) return;fclose(f);- if ((D4.mod = load_module(K.core.spv_path, file))) D4.pipe = make_pipe(D4.mod, NULL);+ if ((D4.mod = load_module(KC.core.spv_path, file))) D4.pipe = make_pipe(D4.mod, NULL);}static void dsv4_shutdown(void) {- if (D4.pipe) vkDestroyPipeline(K.dev, D4.pipe, NULL);- if (D4.mod) vkDestroyShaderModule(K.dev, D4.mod, NULL);+ if (D4.pipe) vkDestroyPipeline(KC.dev, D4.pipe, NULL);+ if (D4.mod) vkDestroyShaderModule(KC.dev, D4.mod, NULL);memset(&D4, 0, sizeof D4);}int vkc_dsv4_ready(void) { return vkc_ready() && D4.pipe; }@@ -1521,10 +1554,10 @@ int vkc_dsv4_ready(void) { return vkc_ready() && D4.pipe; }/* the shader reads one block of 32 words: the mode, then the op's fields */typedef struct { int mode; int w[31]; } Dsv4PC;static int dsv4_rec(int kind, int mode, const void *b, size_t bytes, VkcBind *bd, int nb, uint32_t gx, uint32_t gy) {- if (!D4.pipe || !open_frame() || K.lost || bytes > sizeof(int) * 31) return 0;+ if (!D4.pipe || !open_frame() || KC.lost || bytes > sizeof(int) * 31) return 0;Dsv4PC pc; memset(&pc, 0, sizeof pc);pc.mode = mode; memcpy(pc.w, b, bytes);- K.kind = kind;+ KC.kind = kind;return record(D4.pipe, bd, nb, &pc, sizeof pc, gx, gy, 1);}int vkc_dsv4_attn(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *list, VkcBuf *prm, VkcBuf *out, const VkcDsAttn *p) {@@ -1536,14 +1569,14 @@ int vkc_dsv4_attn(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *list, VkcBuf *prm}int vkc_dsv4_rope(VkcBuf *x, VkcBuf *cs, const VkcDsRope *p) {if (p->nseg < 0 || p->per_row < 1 || p->rd < 2 || (p->rd & 1)) return 0;- if (p->nseg == 0) return open_frame() && !K.lost;+ if (p->nseg == 0) return open_frame() && !KC.lost;VkcBind bd[6] = {B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(cs, 0), B(x, 1)};uint32_t gx, gy; grid(((uint64_t)p->nseg * (p->rd / 2) + 255) / 256, &gx, &gy);return dsv4_rec(PK_ROPE, 1, p, sizeof *p, bd, 6, gx, gy);}int vkc_dsv4_compress(VkcBuf *kv, VkcBuf *sc, VkcBuf *ring, VkcBuf *prm, VkcBuf *out, const VkcDsComp *p) {if (p->S < 0 || p->ratio < 1 || p->P < 1 || p->D < 1 || p->D > p->P || (p->overlap && p->P < 2 * p->D)) return 0;- if (p->S == 0) return open_frame() && !K.lost;+ if (p->S == 0) return open_frame() && !KC.lost;VkcBind bd[8] = {B(kv, 0), B(sc, 0), B(NULL, 0), B(NULL, 0), B(p->ape_off >= 0 ? prm : NULL, 0), B(out, 1), B(NULL, 0),B(ring, 1)};return dsv4_rec(PK_DSA, 2, p, sizeof *p, bd, 8, 1, 1);@@ -1553,7 +1586,7 @@ int vkc_dsv4_score(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *mask, VkcBuf *sint wide = p->IH > 64 || p->IH * p->ID > 4096;if (p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 4096 || p->ID < 1 || p->ratio < 1 ||p->width < 0 || p->sc_row < p->width || (p->mask_row > 0 && !mask)) return 0;- if (p->width == 0) return open_frame() && !K.lost;+ if (p->width == 0) return open_frame() && !KC.lost;VkcBind bd[6] = {B(iq, 0), B(hw, 0), B(keys, 0), B(p->mask_row > 0 ? mask : NULL, 0), B(NULL, 0), B(sc, 1)};return dsv4_rec(PK_DSA, wide ? 9 : 3, p, sizeof *p, bd, 6, (uint32_t)p->S, 1);}@@ -1565,7 +1598,7 @@ int vkc_dsv4_cand(VkcBuf *sc, VkcBuf *mask, const VkcDsCand *p) {}int vkc_dsv4_topk(VkcBuf *sc, VkcBuf *list, const VkcDsTopk *p) {if (p->S < 1 || p->S > 65535 || p->topk < 0 || p->width < 0 || (p->order && p->topk > 4096)) return 0;- if (p->topk == 0) return open_frame() && !K.lost;+ if (p->topk == 0) return open_frame() && !KC.lost;VkcBind bd[7] = {B(sc, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(list, 1)};return dsv4_rec(PK_DSA, 5, p, sizeof *p, bd, 7, (uint32_t)p->S, 1);}@@ -1579,7 +1612,7 @@ int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p) {return 0;if ((p->kind == VKC_DS_E4M3 || p->kind == VKC_DS_E2M1) && (p->block < 1 || p->block > 256)) return 0;if (p->kind == VKC_DS_HADAMARD && (p->len > 4096 || (p->len & (p->len - 1)))) return 0;- if (p->nseg == 0) return open_frame() && !K.lost;+ if (p->nseg == 0) return open_frame() && !KC.lost;/* the fields, then whether it runs in place (x is read where y is written), then hscale */int w[14];memcpy(w, p, 12 * sizeof(int));@@ -1594,7 +1627,7 @@ int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p) {}int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p) {if (!a || !b || !y || p->n < 0) return 0;- if (p->n == 0) return open_frame() && !K.lost;+ if (p->n == 0) return open_frame() && !KC.lost;uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);VkcBind bd[6] = {B(a, 0), B(b, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};return dsv4_rec(PK_EW, 8, p, sizeof *p, bd, 6, gx, gy);@@ -1603,25 +1636,26 @@ int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p) {/* ---- a KV cache split between the device and the host (chain_kvs.comp) ------------* Its own optional pipeline, as DeepSeek's: without the shader only these ops decline* (and an engine's split with them: past the budget its chain declines as before). */-static struct { VkShaderModule mod; VkPipeline pipe; int tried; } KS;+static struct { VkShaderModule mod; VkPipeline pipe; int tried; } g_ks[2];+#define KS (g_ks[g_kd])static VkPipeline kvs_pipe(void) {if (KS.pipe || KS.tried || !vkc_ready()) return KS.pipe;KS.tried = 1;char path[1200];- const char *sl = strrchr(K.core.spv_path, '/');- size_t pre = sl ? (size_t)(sl - K.core.spv_path) + 1 : 0;+ const char *sl = strrchr(KC.core.spv_path, '/');+ size_t pre = sl ? (size_t)(sl - KC.core.spv_path) + 1 : 0;const char *file = "chain_kvs.spv";if (pre + strlen(file) + 1 >= sizeof path) return VK_NULL_HANDLE;- memcpy(path, K.core.spv_path, pre); strcpy(path + pre, file);+ memcpy(path, KC.core.spv_path, pre); strcpy(path + pre, file);FILE *f = fopen(path, "rb");if (!f) return VK_NULL_HANDLE;fclose(f);- if ((KS.mod = load_module(K.core.spv_path, file))) KS.pipe = make_pipe(KS.mod, NULL);+ if ((KS.mod = load_module(KC.core.spv_path, file))) KS.pipe = make_pipe(KS.mod, NULL);return KS.pipe;}static void kvs_shutdown(void) {- if (KS.pipe) vkDestroyPipeline(K.dev, KS.pipe, NULL);- if (KS.mod) vkDestroyShaderModule(K.dev, KS.mod, NULL);+ if (KS.pipe) vkDestroyPipeline(KC.dev, KS.pipe, NULL);+ if (KS.mod) vkDestroyShaderModule(KC.dev, KS.mod, NULL);memset(&KS, 0, sizeof KS);}int vkc_kvs_ready(void) { return kvs_pipe() != VK_NULL_HANDLE; }@@ -1633,7 +1667,7 @@ int vkc_kvs_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *out, VkcBuf *gate, Vp->nblk < 0 || p->anchor < 0 || p->win < 0 || (p->sink && !snk) || (p->sel_row > 0 && !sel) ||((p->flags & VKC_KVS_GATE) && (!gate || !(p->flags & VKC_KVS_FIN)))) return 0;struct { int mode; VkcKvsAttn b; } pc = {0, *p};- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[8] = {B(q, 0), B(kc, 0), B(vc, 0), B(out, 1), B(p->flags & VKC_KVS_GATE ? gate : NULL, 0),B(p->sel_row > 0 ? sel : NULL, 0), B(p->sink ? snk : NULL, 0), B(tab, 0)};return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);@@ -1645,7 +1679,7 @@ int vkc_kvs_mla(VkcBuf *qa, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sel,p->anchor < 0 ||p->kv_start < 0 || (p->sel_row > 0 && !sel)) return 0;struct { int mode; VkcKvsMla b; } pc = {1, *p};- K.kind = PK_MLA;+ KC.kind = PK_MLA;VkcBind bd[8] = {B(qa, 0), B(p->R > 0 ? qr : NULL, 0), B(lat, 0), B(p->R > 0 ? rope : NULL, 0), B(NULL, 0),B(p->sel_row > 0 ? sel : NULL, 0), B(out, 1), B(tab, 0)};return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);@@ -1653,9 +1687,9 @@ int vkc_kvs_mla(VkcBuf *qa, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sel,int vkc_kvs_merge(VkcBuf *dev, VkcBuf *cpu, VkcBuf *gate, VkcBuf *out, const VkcKvsMerge *p) {VkPipeline pipe = kvs_pipe();if (!pipe || !dev || !cpu || !out || p->n < 0 || p->H < 1 || p->d < 1 || ((p->flags & VKC_KVS_GATE) && !gate)) return 0;- if (p->n == 0) return open_frame() && !K.lost;+ if (p->n == 0) return open_frame() && !KC.lost;struct { int mode; VkcKvsMerge b; } pc = {2, *p};- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[4] = {B(dev, 0), B(cpu, 0), B(p->flags & VKC_KVS_GATE ? gate : NULL, 0), B(out, 1)};uint32_t gx, gy; grid((uint64_t)p->n, &gx, &gy);return record(pipe, bd, 4, &pc, sizeof pc, gx, gy, 1);@@ -1667,7 +1701,7 @@ int vkc_kvs_rel(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *out, VkcBuf *r, VkcBup->KVH < 1 || p->H % p->KVH || p->hd < 1 || p->hd > 256 || p->d_rel < 0 || p->d_rel > 64 ||(p->d_rel > 0 && (!r || !relp)) || p->B < 1 || p->nblk < 0 || p->anchor < 0) return 0;struct { int mode; VkcKvsRel b; } pc = {3, *p};- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[8] = {B(q, 0), B(kc, 0), B(vc, 0), B(out, 1), B(p->d_rel > 0 ? r : NULL, 0), B(tau, 0),B(p->d_rel > 0 ? relp : NULL, 0), B(tab, 0)};return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);@@ -1679,7 +1713,7 @@ int vkc_kvs_ds_cold(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *cold, VkcBuf *o((p->flags & VKC_KVS_DSCOLD) && (!cold || !(p->flags & VKC_KVS_DSFIN))) ||((p->flags & VKC_KVS_V4) && !(p->flags & VKC_KVS_DSFIN))) return 0;struct { int mode; VkcKvsDs b; } pc = {4, *p};- K.kind = PK_ATTN;+ KC.kind = PK_ATTN;VkcBind bd[8] = {B(q, 0), B(win, 0), B(cmp ? cmp : win, 0), B(out, 1), B(prm, 0), B(list, 0), B(cold, 0), B(tab, 0)};return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);}@@ -1689,14 +1723,14 @@ int vkc_kvs_ds(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *out, VkcBuf *prm, Vkunsigned long long vkc_serial(void) {unsigned long long s = 0;for (int i = 0; i < VKC_FRAMES; i++)- if (!K.fr[i].open && K.fr[i].serial > s) s = K.fr[i].serial;+ if (!KC.fr[i].open && KC.fr[i].serial > s) s = KC.fr[i].serial;return s;}int vkc_wait_serial(unsigned long long serial) {if (!vkc_ready()) return 0;for (int i = 0; i < VKC_FRAMES; i++) {- VkcFrame *f = &K.fr[i];- if (f->serial == serial && !f->open) return frame_wait(f) && !K.lost;+ VkcFrame *f = &KC.fr[i];+ if (f->serial == serial && !f->open) return frame_wait(f) && !KC.lost;}- return !K.lost; /* reused since: it completed long ago */+ return !KC.lost; /* reused since: it completed long ago */}diff --git a/c/vk_chain.h b/c/vk_chain.hindex f67041ae..6a83bf2e 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -58,6 +58,12 @@ int vkc_init(void); /* after coli_vk_init; 1 = the chain's pipelines aint vkc_ready(void);int vkc_lost(void);void vkc_shutdown(void); /* before coli_vk_shutdown (register it with atexit after it) */+/* The device the calls go to: 0 the primary, 1 COLI_VK_DEV2's (coli_vk_core_dev), each+ * with a context of its own that vkc_init opens and vkc_shutdown closes while it is+ * current. A buffer belongs to the device it was made on: bound or copied on the other+ * the op fails; vkc_free takes it back on its own device. Returns the previous device. */+int vkc_device(int d);+int vkc_device_now(void);VkcBuf *vkc_buf(size_t bytes, int kind); /* zero-filled; NULL when out of memory */void vkc_free(VkcBuf *b); /* waits for the frames that may read it */diff --git a/c/vk_kvsplit.h b/c/vk_kvsplit.hindex 1c7b6687..8c7a1bea 100644--- a/c/vk_kvsplit.h+++ b/c/vk_kvsplit.h@@ -149,11 +149,11 @@ static int vkc_kv_plan_need(VkcKvSplit *ks, const char *engine, int nl, size_t r} else {double used = 0, bud = 0;size_t avail;- if (coli_vk_mem_budget(&used, &bud)) avail = bud > used ? (size_t)((bud - used) * 1e9) : 0;+ if (coli_vk_mem_budget_dev(vkc_device_now(), &used, &bud)) avail = bud > used ? (size_t)((bud - used) * 1e9) : 0;else {- size_t dev = coli_vk_device_local_bytes(), w = 0, n = 0;+ size_t dev = coli_vk_device_local_bytes_dev(vkc_device_now()), w = 0, n = 0;VkcStats st; vkc_stats(&st);- coli_vk_mem_info(&w, &n);+ coli_vk_mem_info_dev(vkc_device_now(), &w, &n);avail = dev > w + st.dev_bytes ? dev - w - st.dev_bytes : 0;}avail += held;@@ -895,10 +895,10 @@ static int vkc_kv_sparse_reserve(VkcBuf **b, size_t bytes) {if (old >= bytes) return 1;double used = 0, budget = 0;size_t free_bytes;- if (coli_vk_mem_budget(&used, &budget)) free_bytes = budget > used ? (size_t)((budget - used) * 1e9) : 0;+ if (coli_vk_mem_budget_dev(vkc_device_now(), &used, &budget)) free_bytes = budget > used ? (size_t)((budget - used) * 1e9) : 0;else {- size_t total = coli_vk_device_local_bytes(), weights = 0, count = 0;- VkcStats st; vkc_stats(&st); coli_vk_mem_info(&weights, &count);+ size_t total = coli_vk_device_local_bytes_dev(vkc_device_now()), weights = 0, count = 0;+ VkcStats st; vkc_stats(&st); coli_vk_mem_info_dev(vkc_device_now(), &weights, &count);free_bytes = total > weights + st.dev_bytes ? total - weights - st.dev_bytes : 0;}if (bytes - old > free_bytes) return 0;