Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: the split KV cache's host part on the device (COLI_VK_KV_COLD=device)
JustVugg323da5aaf4bc
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 4d5710ca..3eb54ee4 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -362,6 +362,13 @@ jobs:- family: kv-split-sanitizereq: c/tools/requirements-qwen38-tiny.txtextra: numpy pillow+ # the same with the host's part on the device too (COLI_VK_KV_COLD=device)+ - family: kv-split-cold+ req: c/tools/requirements-qwen38-tiny.txt+ extra: numpy pillow+ - family: kv-split-cold-sanitize+ req: c/tools/requirements-qwen38-tiny.txt+ extra: numpy pillow- family: kv-split-deepseekreq: c/tools/requirements-deepseek-v4-tiny.txtextra: sympy pillowdiff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex a77011dd..904941e2 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -1569,6 +1569,53 @@ size_t coli_vk_import_alignment(void) {#endif}size_t coli_vk_imported_bytes(void) { return __atomic_load_n(&g_import_bytes, __ATOMIC_RELAXED); }+/* A host range read in place by a shader (the chain's VKC_HOST): the pages around it,+ * so the caller's allocation need not be aligned; the pages past its ends are the+ * process's own and are never written. */+int coli_vk_host_buffer(const void *ptr, size_t bytes, void **buf_out, void **mem_out, size_t *off) {+#ifdef VK_EXT_external_memory_host+ size_t al = coli_vk_import_alignment();+ if (!al || !ptr || !bytes) return 0;+ uintptr_t p = (uintptr_t)ptr, base = p / al * al, end = (p + bytes + al - 1) / al * al;+ size_t sz = end - base;+ if (sz > G.ssbo_range) return 0;+ static PFN_vkGetMemoryHostPointerPropertiesEXT gp;+ if (!gp) gp = (PFN_vkGetMemoryHostPointerPropertiesEXT)vkGetDeviceProcAddr(G.dev, "vkGetMemoryHostPointerPropertiesEXT");+ VkMemoryHostPointerPropertiesEXT mp = {.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};+ if (!gp || gp(G.dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, (void *)base, &mp) != VK_SUCCESS ||+ !mp.memoryTypeBits) return 0;+ VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,+ .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT};+ VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = sz,+ .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE};+ VkBuffer b;+ if (vkCreateBuffer(G.dev, &bi, NULL, &b) != VK_SUCCESS) return 0;+ VkMemoryRequirements req;+ vkGetBufferMemoryRequirements(G.dev, b, &req);+ uint32_t bits = mp.memoryTypeBits & req.memoryTypeBits, mt = 0;+ if (!bits || req.size > sz) { vkDestroyBuffer(G.dev, b, NULL); return 0; }+ while (!(bits & (1u << mt))) mt++;+ VkImportMemoryHostPointerInfoEXT imp = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,+ .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, .pHostPointer = (void *)base};+ VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &imp, .allocationSize = sz,+ .memoryTypeIndex = mt};+ VkDeviceMemory m;+ if (vkAllocateMemory(G.dev, &ai, NULL, &m) != VK_SUCCESS) { vkDestroyBuffer(G.dev, b, NULL); return 0; }+ if (vkBindBufferMemory(G.dev, b, m, 0) != VK_SUCCESS) { vkDestroyBuffer(G.dev, b, NULL); vkFreeMemory(G.dev, m, NULL); return 0; }+ *buf_out = (void *)b; *mem_out = (void *)m; *off = (size_t)(p - base);+ __atomic_add_fetch(&g_import_bytes, sz, __ATOMIC_RELAXED);+ return 1;+#else+ (void)ptr; (void)bytes; (void)buf_out; (void)mem_out; (void)off;+ return 0;+#endif+}+void coli_vk_host_buffer_free(void *buf, void *mem, size_t bytes) {+ if (!G.dev) return;+ if (buf) vkDestroyBuffer(G.dev, (VkBuffer)buf, NULL);+ if (mem) vkFreeMemory(G.dev, (VkDeviceMemory)mem, NULL);+ if (bytes) __atomic_sub_fetch(&g_import_bytes, bytes, __ATOMIC_RELAXED);+}int coli_vk_tensor_import(ColiVkTensor **tensor, const void *weights, size_t alloc_bytes, const float *scales,int fmt, int I, int O, int gs) {if (*tensor) return (*tensor)->fmt == fmt && (*tensor)->I == I && (*tensor)->O == O;diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 882fb7d1..19d7bdad 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -111,6 +111,14 @@ int coli_vk_tensor_ensure(ColiVkTensor **tensor, const void *weights, const floint coli_vk_tensor_import(ColiVkTensor **tensor, const void *weights, size_t alloc_bytes, const float *scales,int fmt, int I, int O, int gs);size_t coli_vk_import_alignment(void);+/* Host memory a shader reads in place (VK_EXT_external_memory_host, where+ * coli_vk_import_alignment() is not 0): the pages around [ptr, ptr + bytes) as a storage+ * buffer of the primary device (VkBuffer, VkDeviceMemory as void *), *off the byte+ * offset of ptr in it. 0 when the device refuses. The pages must stay mapped until+ * coli_vk_host_buffer_free (bytes: the buffer's, ptr's page-rounded range), after the+ * last submission that read them. */+int coli_vk_host_buffer(const void *ptr, size_t bytes, void **buf, void **mem, size_t *off);+void coli_vk_host_buffer_free(void *buf, void *mem, size_t bytes);size_t coli_vk_imported_bytes(void);/* SECOND DEVICE (COLI_VK_DEV2): a self-contained context on another Vulkan GPU thatdiff --git a/c/shaders/chain_attnb.comp b/c/shaders/chain_attnb.compindex 599ecda9..5244c242 100644--- a/c/shaders/chain_attnb.comp+++ b/c/shaders/chain_attnb.comp@@ -20,6 +20,18 @@// per row (sel_row > 0): then BR = 1, the row's list is the workgroup's.// The order of the sums differs from chain_attn's and the CPU's; all agree to rounding.// Push constants: chain_attn's, then br (rows per workgroup, br * H / KVH <= 32).+// part != 0 (vk_kvsplit.h's host part on the device, from its shadow of the host's+// rows): a row's positions are its causal ones before its share of a split cache, the+// blocks pos/B - anchor and on, so [first, (pos/B - anchor)*B); the output is the part+// (the value sum unnormalized at o_off + s*o_row + h*vd, (m, l) at st_off + (s*H + h)*2)+// for vkc_kvs_merge. Its tiles start at multiples of BC: a row's sums then depend on its+// own positions only, whatever rows share its workgroup (a tile it does not see changes+// nothing), so a row gets the same bits however a forward is cut into steps. No lists,+// no sink, no ring there. With g_off > 0 the positions are cut into chunks of g_off (a+// multiple of BC), workgroup z taking [z*g_off, (z+1)*g_off): its part at o_off ++// z*g_row + ... and its (m, l) at st_off + z*g_seg + ..., for chain_kvs.comp's join (a+// decode row's thousands of positions over many workgroups; the chunks are fixed by+// position, so the bits still are the row's own).// Limits: hd, vd <= 256.layout(local_size_x = 256) in;layout(constant_id = 0) const int BC = 8;@@ -42,6 +54,7 @@ layout(push_constant) uniform P {int k_off, v_off;int win, ring, vd, kv_pm, sink, sink_off;int br;+ int B, anchor, part, st_off;} p;shared float ks[BC * 256]; // the tile's keys [j][d]@@ -73,6 +86,18 @@ void main() {int first = p.win > 0 ? max(0, pos - p.win + 1) : 0; // this pair's rowint bfirst = p.win > 0 ? max(0, p.pos_base + s0 - p.win + 1) : 0;int n = p.pos_base + slast + 1 - bfirst;+ int end = pos + 1; // this pair's row: positions before end+ if (p.part != 0) {+ end = (pos / p.B - p.anchor) * p.B;+ int bend = ((p.pos_base + slast) / p.B - p.anchor) * p.B; // the block's last row's+ bfirst -= bfirst % BC;+ if (p.g_off > 0) {+ int c0 = int(gl_WorkGroupID.z) * p.g_off, c1 = c0 + p.g_off;+ bfirst = max(bfirst, c0); bend = min(bend, c1);+ first = max(first, c0); end = min(end, c1);+ }+ n = max(0, bend - bfirst);+ }bool list = false;int lb = 0;if (p.sel_row > 0) {@@ -122,7 +147,7 @@ void main() {for (int j = lane; j < BC; j += 8) {int t = tpos[j];float sc = -3.0e38;- if (live && t >= 0 && (list || (t >= first && t <= pos))) {+ if (live && t >= 0 && (list || (t >= first && t < end))) {float a = 0.0;for (int k = 0; k < 8; k++) a += part[(pair * 8 + k) * BC + j];sc = a * p.scale;@@ -150,8 +175,19 @@ void main() {barrier();}if (!live) return;- float inv = l > 0.0 ? 1.0 / l : 0.0;int ob = p.o_off + s * p.o_row + h * p.vd;+ if (p.part != 0) { // the part (a chunk's, with g_off), for the merge (or the join)+ int z = p.g_off > 0 ? int(gl_WorkGroupID.z) : 0;+ int obz = ob + z * p.g_row, sb = p.st_off + z * p.g_seg + (s * p.H + h) * 2;+ for (int i = 0; i < 32; i++) {+ int d = lane + 8 * i;+ if (d >= p.vd) break;+ o[obz + d] = av[i];+ }+ if (lane == 0) { o[sb] = m; o[sb + 1] = l; }+ return;+ }+ float inv = l > 0.0 ? 1.0 / l : 0.0;int gb = p.g_off + s * p.g_row + h * p.g_seg;for (int i = 0; i < 32; i++) {int d = lane + 8 * i;diff --git a/c/shaders/chain_attnb.spv b/c/shaders/chain_attnb.spvindex 697ed5c9..5d3d39a7 100644Binary files a/c/shaders/chain_attnb.spv and b/c/shaders/chain_attnb.spv differdiff --git a/c/shaders/chain_kvs.comp b/c/shaders/chain_kvs.compindex bb460029..8113794d 100644--- a/c/shaders/chain_kvs.comp+++ b/c/shaders/chain_kvs.comp@@ -40,6 +40,12 @@// rope key at rope_off + r*rope_row. flags 1: the normalized clat at o_off + s*o_row// + h*o_seg, as chain_mla writes it; else acc there and (m, l) at st_off + (s*H+h)*2.// Limits: K <= 1024, R <= 128.+// flags 32 (cold), modes 0, 1 and 3: the host's positions of the row instead of the+// device's, read from a buffer in the host cache's own layout (an import of its pages:+// position t is row t, `rows` its positions for head-major K/V), so the device computes+// the part the CPU would: causal, [the row's first position, its share's first block);+// listed, the entries older than the share and not in a pinned block. The table only+// says what the device holds; a part's (m, l, acc) as without the flag. No sink.// mode 2, the merge: segment g = (s, h) of d floats, s = g / H. The device's part at// a_off + s*a_row + h*a_seg with its (m, l) at sa_off + 2g (buffer 0), the host's// packed at c_off + g*d with (m, l) at sc_off + 2g (buffer 1); the result at o_off +@@ -96,6 +102,14 @@ int slot_of(int t, int B, int bt_off, int nblk) {int j = b < nblk ? tab[bt_off + b] : -1;return j < 0 ? -1 : (j & 0x3fffffff);}+// a listed position the host takes (flags 32): older than the row's share, not pinned+bool cold_pos(int t, int pos, int B, int bt_off, int nblk, int anchor) {+ if (t < 0 || t > pos) return false;+ int b = t / B;+ if (b >= pos / B - anchor) return false;+ int j = b < nblk ? tab[bt_off + b] : -1;+ return j < 0 || (j & 0x40000000) == 0;+}// the device row of a listed position, or -1 (the host's): the anchored blocks of the// row at pos, and pinned onesint dev_row(int t, int pos, int B, int bt_off, int nblk, int anchor) {@@ -121,9 +135,11 @@ void gqa() {int qb = q_off + s * q_row + h * q_seg;for (int d = tid; d < hd; d += 256) qs[d] = b0[qb + d];int pos = pos_base + s;- int first = max(win > 0 ? max(0, pos - win + 1) : 0, (pos / B - anchor) * B);+ bool cold = (flags & 32) != 0;+ int start = win > 0 ? max(0, pos - win + 1) : 0, share0 = (pos / B - anchor) * B;+ int first = cold ? start : max(start, share0);bool list = false;- int lb = 0, n = pos - first + 1;+ int lb = 0, n = cold ? max(0, min(pos + 1, share0) - start) : pos - first + 1;if (sel_row > 0) {lb = sel_off + s * sel_row;int c = sel[lb];@@ -134,7 +150,8 @@ void gqa() {for (int t0 = 0; t0 < n; t0 += 256) {int i = t0 + tid, r = -1;if (i < n) {- if (list) r = dev_row(sel[lb + 1 + i], pos, B, bt_off, nblk, anchor);+ if (list) { int t = sel[lb + 1 + i]; r = cold ? (cold_pos(t, pos, B, bt_off, nblk, anchor) ? t : -1) : dev_row(t, pos, B, bt_off, nblk, anchor); }+ else if (cold) r = first + i;else { int t = first + i, j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; }}float sv = -3.0e38;@@ -205,9 +222,11 @@ void mla() {for (int i = tid; i < K; i += 256) qs[i] = b0[qb + i];for (int i = tid; i < R; i += 256) qrs[i] = b1[rbq + i];int pos = pos_base + s;- int first = max(kv_start, (pos / B - anchor) * B);+ bool cold = (flags & 32) != 0;+ int share0 = (pos / B - anchor) * B;+ int first = cold ? kv_start : max(kv_start, share0);bool list = false;- int lb = 0, n = max(0, pos - first + 1);+ int lb = 0, n = cold ? max(0, min(pos + 1, share0) - kv_start) : max(0, pos - first + 1);if (sel_row > 0) {lb = sel_off + s * sel_row;int c = sel[lb];@@ -222,7 +241,8 @@ void mla() {int r = -1;if (tid < cnt) {int i = t0 + tid;- if (list) r = dev_row(sel[lb + 1 + i], pos, B, bt_off, nblk, anchor);+ if (list) { int t = sel[lb + 1 + i]; r = cold ? (cold_pos(t, pos, B, bt_off, nblk, anchor) ? t : -1) : dev_row(t, pos, B, bt_off, nblk, anchor); }+ else if (cold) r = first + i;else { int t = first + i, j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; }}pr[tid] = r;@@ -321,13 +341,15 @@ void rel() {int qb = q_off + s * q_row + h * hd;for (int d = tid; d < hd; d += 256) qs[d] = b0[qb + d];for (int j = tid; j < d_rel; j += 256) qrs[j] = b4[r_off + s * r_row + h * d_rel + j];- int pos = pos_base + s, first = max(0, (pos / B - anchor) * B), n = pos - first + 1;+ bool cold = (flags & 32) != 0;+ int pos = pos_base + s, share0 = (pos / B - anchor) * B;+ int first = cold ? 0 : max(0, share0), n = cold ? max(0, min(pos + 1, share0)) : pos - first + 1;float ts = intBitsToFloat(sel[tau_off + s]);barrier();float m = -3.0e38, l = 0.0, acc = 0.0;for (int t0 = 0; t0 < n; t0 += 256) {int i = t0 + tid, r = -1, t = first + i;- if (i < n) { int j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; }+ if (i < n) { if (cold) r = t; else { int j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; } }float sv = -3.0e38;if (r >= 0) {int kb = k_off + (kvh * rows + r) * hd;@@ -473,10 +495,43 @@ void dsa() {if (tid == 0 && (flags & 8) == 0) { b3[st_off + (s * H + h) * 2] = best; b3[st_off + (s * H + h) * 2 + 1] = den; }}+// ---- mode 5 ----------------------------------------------------------------------+// The chunks of a part joined (chain_attnb's split over positions): segment g of d+// floats, chunk z's sum at a_off + z*a_z + g*d and its (m, l) at sa_off + z*sa_z + 2g+// (buffer 0), joined in chunk order as the host joins its chunks (vkc_kv_join): the part+// at o_off + g*d, (m, l) at so_off + 2g (buffer 3). A chunk with no position weighs nothing.+void join() {+ int n = p.v[1], d = p.v[2], nz = p.v[3], a_off = p.v[4], a_z = p.v[5], sa_off = p.v[6], sa_z = p.v[7];+ int o_off = p.v[8], so_off = p.v[9];+ int g = int(gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x), tid = int(gl_LocalInvocationID.x);+ if (g >= n) return;+ float m = -3.0e38, l = 0.0;+ for (int z = 0; z < nz; z++) {+ float mz = b0[sa_off + z * sa_z + 2 * g], lz = b0[sa_off + z * sa_z + 2 * g + 1];+ if (mz <= -1.0e38) continue;+ float mn = max(m, mz);+ l = l * (m > -1.0e38 ? exp(m - mn) : 0.0) + lz * exp(mz - mn);+ m = mn;+ }+ for (int i = tid; i < d; i += 256) {+ float acc = 0.0, mm = -3.0e38;+ for (int z = 0; z < nz; z++) {+ float mz = b0[sa_off + z * sa_z + 2 * g];+ if (mz <= -1.0e38) continue;+ float mn = max(mm, mz);+ acc = acc * (mm > -1.0e38 ? exp(mm - mn) : 0.0) + b0[a_off + z * a_z + g * d + i] * exp(mz - mn);+ mm = mn;+ }+ b3[o_off + g * d + i] = acc;+ }+ if (tid == 0) { b3[so_off + 2 * g] = m; b3[so_off + 2 * g + 1] = l; }+}+void main() {if (MODE == 0) gqa();else if (MODE == 1) mla();else if (MODE == 2) merge();else if (MODE == 3) rel();+ else if (MODE == 5) join();else dsa();}diff --git a/c/shaders/chain_kvs.spv b/c/shaders/chain_kvs.spvindex 96d2a0c2..92c1e69f 100644Binary files a/c/shaders/chain_kvs.spv and b/c/shaders/chain_kvs.spv differdiff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.cindex 6b6e4f0b..2c18409e 100644--- a/c/tests/test_vk_chain.c+++ b/c/tests/test_vk_chain.c@@ -1870,6 +1870,9 @@ static void kvs_arrive(float *dst, const float *src, int nseg, size_t seg, int lstatic void kvs_keep_rows(const float *o, int pb, int S) { if (kvs_keep) memcpy(kvs_keep + (size_t)pb * kvs_keep_row, o, (size_t)S * kvs_keep_row * 4); }/* GQA: steps of `step` rows from position 0 to T, the device holding `rows` positions. */+/* COLI_VK_KV_COLD=device (test_kvs_cold, or a family run with it): every split step's host part must run on+ * the device, but where pins are on (they need the CPU's lists) */+static int kvs_cold_wanted(void) { const char *c = getenv("COLI_VK_KV_COLD"); return c && !strcmp(c, "device"); }static void test_kvs_gqa(int T, int step, int H, int KVH, int hd, int vd, int B, int rows, int win, int sink, int kv_pm,int gated, int lists, int selects) {char rv[32]; snprintf(rv, sizeof rv, "%d", rows); setenv("COLI_VK_KV_DEVICE_ROWS", rv, 1);@@ -1965,8 +1968,9 @@ static void test_kvs_gqa(int T, int step, int H, int KVH, int hd, int vd, int B,}CHECK(ok && worst < 2e-5 && splits > 0, "kvs gqa T %d step %d H %d/%d hd %d vd %d B %d rows %d win %d sink %d pm %d gate %d lists %d: ""err %.2e (%d split steps, %d whole)", T, step, H, KVH, hd, vd, B, rows, win, sink, kv_pm, gated, lists, worst, splits, fins);- printf(" kvs gqa T %d step %d rows %d lists %d: rel err %.2e, %d split steps, %d whole, %llu host positions, %llu pinned\n",- T, step, rows, lists, worst, splits, fins, ks.host_pos, ks.pins);+ printf(" kvs gqa T %d step %d rows %d lists %d: rel err %.2e, %d split steps, %d whole, %llu host positions, %llu pinned%s\n",+ T, step, rows, lists, worst, splits, fins, ks.host_pos, ks.pins, ks.cold_dev ? ", the host's part on the device" : "");+ CHECK(!kvs_cold_wanted() || ks.np || (ks.cold_dev && ks.dev_cold > 0 && ks.host_pos == 0), "kvs gqa: the host's part did not run on the device");CHECK(!ks.np || !selects || ks.pins > 0, "kvs gqa lists: no block was pinned");vkc_free(kc); vkc_free(vc); vkc_free(qb); vkc_free(nk); vkc_free(nv); vkc_free(ob); vkc_free(sb); vkc_free(lb);vkc_kv_free(&ks);@@ -2054,9 +2058,10 @@ static void test_kvs_mla(int T, int step, int H, int K, int R, int B, int rows,}CHECK(ok && worst < 2e-5 && splits > 0, "kvs mla T %d step %d H %d K %d R %d B %d rows %d start %d lists %d: err %.2e (%d split steps)",T, step, H, K, R, B, rows, kv_start, lists, worst, splits);- printf(" kvs mla T %d step %d K %d rows %d lists %d: rel err %.2e, %d split steps, %llu host positions, %llu pinned\n",- T, step, K, rows, lists, worst, splits, ks.host_pos, ks.pins);- CHECK(!lists || ks.pins > 0, "kvs mla lists: no block was pinned");+ printf(" kvs mla T %d step %d K %d rows %d lists %d: rel err %.2e, %d split steps, %llu host positions, %llu pinned%s\n",+ T, step, K, rows, lists, worst, splits, ks.host_pos, ks.pins, ks.cold_dev ? ", the host's part on the device" : "");+ CHECK(!lists || !ks.np || ks.pins > 0, "kvs mla lists: no block was pinned");+ CHECK(!kvs_cold_wanted() || ks.np || (ks.cold_dev && ks.dev_cold > 0 && ks.host_pos == 0), "kvs mla: the host's part did not run on the device");vkc_free(lat); vkc_free(rope); vkc_free(qab); vkc_free(nl); vkc_free(ob); vkc_free(lb);vkc_kv_free(&ks);free(Lh); free(Rh); free(Lx); free(Rx); free(posl); free(sel); free(ref); free(nh);@@ -2137,7 +2142,9 @@ static void test_kvs_rel(int T, int step, int H, int KVH, int hd, int ext, int d}CHECK(ok && worst < 2e-5 && splits > 0, "kvs rel T %d step %d H %d/%d hd %d ext %d d_rel %d B %d rows %d: err %.2e (%d split steps)",T, step, H, KVH, hd, ext, d_rel, B, rows, worst, splits);- printf(" kvs rel T %d step %d rows %d ext %d: rel err %.2e, %d split steps, %llu host positions\n", T, step, rows, ext, worst, splits, ks.host_pos);+ printf(" kvs rel T %d step %d rows %d ext %d: rel err %.2e, %d split steps, %llu host positions%s\n", T, step, rows, ext, worst,+ splits, ks.host_pos, ks.cold_dev ? ", the host's part on the device" : "");+ CHECK(!kvs_cold_wanted() || ks.np || (ks.cold_dev && ks.dev_cold > 0 && ks.host_pos == 0), "kvs rel: the host's part did not run on the device");vkc_free(kc); vkc_free(vc); vkc_free(qb); vkc_free(rb); vkc_free(nk); vkc_free(nv); vkc_free(ob); vkc_free(tb); vkc_free(pb_);vkc_kv_free(&ks);free(Kh); free(Vh); free(Kx); free(Vx); free(relp); free(ref); free(nkh); free(nvh); free(tau);@@ -2305,6 +2312,49 @@ static void test_kvs(void) {kvs_start = 0; kvs_keep = NULL; free(a); free(b);}+/* COLI_VK_KV_COLD=device: the host's part on the device, from the shadow of the host's+ * rows. Every case without pins against the same double-precision reference (sinks,+ * gates, MiMo's position-major rows, windows, lists, MLA with and without a rope key and+ * a nonzero start, Inkling's bias reaching into the host's rows), each split step's host+ * part on the device and none on the CPU, and the same bits for every position however+ * the forward is cut (steps of 1 to 4, slices of rows, crossing 64K). */+static void test_kvs_cold(void) {+ if (!coli_vk_import_alignment()) { printf(" kvs cold: no host memory the device reads in place here, skipped\n"); return; }+ setenv("COLI_VK_KV_COLD", "device", 1);+ /* T step H KVH hd vd B rows win sink pm gate lists selects */+ test_kvs_gqa(40, 1, 4, 2, 32, 32, 4, 12, 0, 0, 0, 1, 0, 0);+ test_kvs_gqa(41, 3, 4, 2, 32, 32, 4, 8, 0, 0, 0, 1, 0, 0);+ test_kvs_gqa(70, 7, 4, 2, 48, 32, 8, 32, 0, 1, 1, 0, 0, 0);+ test_kvs_gqa(300, 1, 8, 2, 64, 64, 16, 64, 0, 0, 0, 1, 0, 0);+ test_kvs_gqa(600, 50, 4, 4, 16, 16, 16, 128, 0, 0, 0, 0, 0, 0);+ test_kvs_gqa(60, 3, 4, 2, 32, 32, 4, 24, 0, 0, 0, 1, 1, 1); /* lists, no pins */+ test_kvs_gqa(50, 2, 4, 2, 256, 256, 4, 16, 0, 0, 0, 1, 1, 1);+ test_kvs_gqa(60, 2, 4, 2, 32, 24, 4, 16, 20, 1, 1, 0, 0, 0);+ test_kvs_gqa(48, 12, 4, 2, 32, 32, 4, 40, 0, 0, 0, 1, 0, 0);+ /* T step H K R B rows kv_start lists */+ test_kvs_mla(40, 1, 4, 32, 8, 4, 12, 0, 0);+ test_kvs_mla(90, 5, 2, 64, 16, 8, 40, 0, 1);+ test_kvs_mla(60, 4, 3, 48, 0, 4, 16, 0, 0);+ test_kvs_mla(70, 2, 2, 512, 64, 8, 32, 5, 0);+ test_kvs_mla(40, 3, 1, 1024, 8, 4, 16, 0, 1);+ /* T step H KVH hd ext d_rel B rows */+ test_kvs_rel(60, 1, 4, 2, 32, 8, 4, 4, 16);+ test_kvs_rel(80, 5, 4, 2, 16, 40, 16, 8, 32);+ test_kvs_rel(50, 3, 2, 1, 128, 16, 64, 4, 24);+ test_kvs_steps();+ int T = 65543, first = 65531, H = 2, hd = 16;+ float *a = calloc((size_t)T * H * hd, sizeof(float));+ float *b = calloc((size_t)T * H * hd, sizeof(float));+ kvs_start = first; kvs_keep_row = (size_t)H * hd;+ kvs_keep = a; test_kvs_gqa(T, 1, H, 1, hd, hd, 64, 512, 0, 0, 0, 0, 0, 0);+ kvs_keep = b; test_kvs_gqa(T, 7, H, 1, hd, hd, 64, 512, 0, 0, 0, 0, 0, 0);+ CHECK(!memcmp(a + (size_t)first * H * hd, b + (size_t)first * H * hd,+ (size_t)(T - first) * H * hd * sizeof(float)), "kvs cold: rows crossing 64K differ with chunk size");+ kvs_start = 0; kvs_keep = NULL; free(a); free(b);+ unsetenv("COLI_VK_KV_COLD");+ printf(" kvs cold: every case with the host's part on the device, the same bits however the forward is cut\n");+}+/* COLI_VK_CHAIN_BENCH=1: decode-shaped GEMVs back to back in one frame (the chain's* situation: no host gap between matrices), per weight format, in GB/s of weights. */#include <time.h>@@ -2494,7 +2544,7 @@ int main(int argc, char **argv) {printf("dsv4 rounding done\n");}if (!vkc_kvs_ready()) { fails++; printf("FAIL: the split KV shader did not load\n"); }- else { test_kvs(); printf("kvs done\n"); }+ else { test_kvs(); test_kvs_cold(); printf("kvs done\n"); }VkcStats st; vkc_stats(&st);printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);vkc_shutdown();diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex 1546106f..d9b65504 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -3279,6 +3279,24 @@ kv_hostparts() {n=$(sed -n "s/^\[VK\] $1 chain: KV split: [0-9]* layer steps, \([0-9]*\) with a host part.*/\1/p" "$2" | tail -1)echo "${n:-0}"}+# kv_devcold <engine> <log>: N from "| the host's part on the device: N layer steps"+# (COLI_VK_KV_COLD=device), 0 without it+kv_devcold() {+ local n+ n=$(sed -n "s/^\[VK\] $1 chain: KV split: .* the host's part on the device: \([0-9]*\) layer steps.*/\1/p" "$2" | tail -1)+ echo "${n:-0}"+}+# kv_cold_check <engine> <tag> <log>: with COLI_VK_KV_COLD=device exported (the+# kv-split-cold families), the host's part must have run on the device; with read-based+# pins (COLI_VK_KV_PIN=1) it stays on the CPU by design, and the engine says so+kv_cold_check() {+ [ "${COLI_VK_KV_COLD:-}" = device ] || return 0+ if [ "${COLI_VK_KV_PIN:-0}" = 1 ]; then+ grep -qa "COLI_VK_KV_COLD=device needs .* no pins; the CPU computes the host's part" "$3" || { cat "$3"; fail "$2: pins on, and no line that the CPU computes the host's part"; }+ return 0+ fi+ [ "$(kv_devcold "$1" "$3")" -gt 0 ] || { cat "$3"; fail "$2: the host's part never ran on the device"; }+}# kv_gate <engine> <tag> <tol 0|1> <rows> <block> <env...> -- <argv...>: chain_gate (the# CPU's tokens, logits within 1e-4 of the largest with tol 1) with the split on, which# must have run steps with a host part.@@ -3286,6 +3304,7 @@ kv_gate() {local eng=$1 tag=$2 tol=$3 rows=$4 blk=$5; shift 5chain_gate "$eng" "$tag" "$tol" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"[ "$(kv_hostparts "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }+ kv_cold_check "$eng" "$tag" vk.logecho " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"}# kv_san <engine> <tag> <env and argv...>: a sanitized run under the split (the build is@@ -3296,6 +3315,7 @@ kv_san() {env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || trueif grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi[ "$(kv_hostparts "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the split never ran a host part"; }+ kv_cold_check "$eng" "$tag" san.logecho "OK $tag: sanitizers clean, $(kv_hostparts "$eng" san.log) layer steps with a host part"}@@ -3627,6 +3647,7 @@ kv_mla_gate() {mla_gate "$eng" "$tag" "$tol" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"[ "$(kv_hostparts "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }if [ "${PINNED:-0}" = 1 ]; then grep -qa 'KV split: .* [1-9][0-9]* blocks pinned by reads' vk.log || { cat vk.log; fail "$tag: no block was pinned"; }; fi+ kv_cold_check "$eng" "$tag" vk.logecho " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"}# colibri (GLM-5.2): the 32-token oracle over 8 or 16 rows on the device; DSA's lists@@ -3954,6 +3975,20 @@ family_kv_split_sanitize() {kv_split_mimo_sanitize; kv_split_colibri_sanitize; kv_split_glm53_sanitize; kv_split_kimi_sanitizemake clean >/dev/null 2>&1 || true}+# COLI_VK_KV_COLD=device: every gate of the two families above again, with the host's+# part of each split layer's attention on the device too (vkc_kv_shadow). The tokens and+# logits are held to the CPU's as before, and each gate must have run the host's part on+# the device (kv_cold_check). DeepSeek's sparse forms keep it on the CPU.+family_kv_split_cold() {+ export COLI_VK_KV_COLD=device+ family_kv_split+ unset COLI_VK_KV_COLD+}+family_kv_split_cold_sanitize() {+ export COLI_VK_KV_COLD=device+ family_kv_split_sanitize+ unset COLI_VK_KV_COLD+}family_kv_split_deepseek() {export OMP_NUM_THREADS=2make deepseek_v41 deepseek-v4 VK=1@@ -4083,6 +4118,8 @@ case "${1:-}" inprefill-deepseek) family_prefill_deepseek ;;kv-split) family_kv_split ;;kv-split-sanitize) family_kv_split_sanitize ;;+ kv-split-cold) family_kv_split_cold ;;+ kv-split-cold-sanitize) family_kv_split_cold_sanitize ;;kv-split-deepseek) family_kv_split_deepseek ;;kv-split-deepseek-sanitize) family_kv_split_deepseek_sanitize ;;staged) family_staged ;;@@ -4103,5 +4140,5 @@ case "${1:-}" inpartial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }"$fn" ;;- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;+ *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-cold|kv-split-cold-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;esacdiff --git a/c/vk_chain.c b/c/vk_chain.cindex c45c0f00..7674e0a6 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -67,6 +67,7 @@ typedef struct { VkDeviceMemory mem; uint8_t *map; } VkcBlock;typedef struct { VkaPool p; uint32_t memtype; int mapped; } VkcPool;struct VkcBuf {VkBuffer buf;+ VkDeviceMemory mem; /* VKC_HOST: the imported pages */int kind;size_t bytes;VkaRange r;@@ -195,6 +196,7 @@ VkcBuf *vkc_buf(size_t bytes, int kind) {}static void buf_release(VkcBuf *b) {+ if (b->kind == VKC_HOST) { coli_vk_host_buffer_free((void *)b->buf, (void *)b->mem, b->bytes); free(b); return; }VkcPool *P = &K.pool[b->kind];vkDestroyBuffer(K.dev, b->buf, NULL);if (vka_free(&P->p, b->r)) {@@ -231,6 +233,19 @@ int vkc_reserve(VkcBuf **b, size_t bytes, int kind) {}void *vkc_ptr(const VkcBuf *b) { return b ? b->ptr : NULL; }size_t vkc_bytes(const VkcBuf *b) { return b ? b->bytes : 0; }+VkcBuf *vkc_host(const void *ptr, size_t bytes, size_t *off) {+ if (!vkc_ready() || !ptr || !bytes) return NULL;+ void *buf = NULL, *mem = NULL;+ size_t o = 0;+ if (!coli_vk_host_buffer(ptr, bytes, &buf, &mem, &o)) return NULL;+ size_t al = coli_vk_import_alignment(), sz = (o + bytes + al - 1) / al * al; /* the pages, as imported */+ VkcBuf *b = calloc(1, sizeof *b);+ if (!b) { coli_vk_host_buffer_free(buf, mem, sz); return NULL; }+ b->kind = VKC_HOST; b->buf = (VkBuffer)buf; b->mem = (VkDeviceMemory)mem;+ b->bytes = sz;+ *off = o;+ return b;+}/* ---- init ------------------------------------------------------------------------ */static VkShaderModule load_module(const char *dir_spv, const char *file) {@@ -823,13 +838,44 @@ int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuif (w.a.sel_row > 0) x.a.sel_off += r0 * w.a.sel_row;if (r0 && !attn_slice_next()) return 0;if (blocked) {- struct { VkcAttnW w; int br; } pc = {x, br};+ struct { VkcAttnW w; int br, B, anchor, part, st_off; } pc = {x, br, 0, 0, 0, 0};ok = record(KAB.pipe, bd, 7, &pc, sizeof pc, (uint32_t)x.a.KVH, (uint32_t)((x.a.S + br - 1) / br), 1);K.st.attn_blocked += ok;} else ok = record(K.pipe[P_ATTN], bd, 7, &x, sizeof x, (uint32_t)x.a.H, (uint32_t)x.a.S, 1);}return ok;}+static int attn_part_rec(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off,+ int ch, int nz, int o_z, int st_z) {+ K.kind = PK_ATTN;+ VkcAttnW w = *p;+ if (w.vd <= 0) w.vd = w.a.hd;+ int G = w.a.KVH > 0 ? w.a.H / w.a.KVH : 0;+ if (w.a.hd > 256 || w.vd > 256 || G < 1 || G > 32 || w.a.H % w.a.KVH || w.a.sel_row > 0 || w.sink || w.ring || blk < 1 ||+ anchor < 0 || ch < 0 || (ch > 0 && (ch % 16 || nz < 1 || nz > 65535)) || !kab_pipe()) return 0;+ w.a.has_gate = 0; w.a.g_off = ch; w.a.g_row = o_z; w.a.g_seg = st_z; /* the chunks (unused by part otherwise) */+ VkcBind bd[7] = {B(q, 0), B(kc, 0), B(vc, 0), B(o, 1), B(NULL, 0), B(NULL, 0), B(NULL, 0)};+ int br = 32 / G, S = w.a.S, rr = attn_slice_rows(S, w.a.pos_base, (double)w.a.H * (w.a.hd > w.vd ? w.a.hd : w.vd), br);+ int ok = 1;+ for (int r0 = 0; ok && r0 < S; r0 += rr) {+ VkcAttnW x = w;+ int n = S - r0 < rr ? S - r0 : rr;+ x.a.S = n; x.a.pos_base = w.a.pos_base + r0;+ x.a.q_off += r0 * w.a.q_row; x.a.o_off += r0 * w.a.o_row;+ if (r0 && !attn_slice_next()) return 0;+ struct { VkcAttnW w; int br, B, anchor, part, st_off; } pc = {x, br, blk, anchor, 1, st_off + r0 * w.a.H * 2};+ ok = record(KAB.pipe, bd, 7, &pc, sizeof pc, (uint32_t)x.a.KVH, (uint32_t)((x.a.S + br - 1) / br), (uint32_t)(ch > 0 ? nz : 1));+ K.st.attn_blocked += ok;+ }+ return ok;+}+int vkc_attn_part(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off) {+ return attn_part_rec(q, kc, vc, o, p, blk, anchor, st_off, 0, 1, 0, 0);+}+int vkc_attn_part_chunks(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off,+ int ch, int nz, int o_z, int st_z) {+ return ch > 0 && attn_part_rec(q, kc, vc, o, p, blk, anchor, st_off, ch, nz, o_z, st_z);+}int vkc_dnconv(VkcBuf *in, VkcBuf *w, VkcBuf *ring, VkcBuf *out, VkcBuf *snap, const VkcDnConv *p) {K.kind = PK_DNCONV;if (p->CK < 1 || p->CK > 9) return 0;@@ -1660,6 +1706,16 @@ int vkc_kvs_merge(VkcBuf *dev, VkcBuf *cpu, VkcBuf *gate, VkcBuf *out, const Vkcuint32_t gx, gy; grid((uint64_t)p->n, &gx, &gy);return record(pipe, bd, 4, &pc, sizeof pc, gx, gy, 1);}+int vkc_kvs_join(VkcBuf *parts, VkcBuf *out, const VkcKvsJoin *p) {+ VkPipeline pipe = kvs_pipe();+ if (!pipe || !parts || !out || p->n < 0 || p->d < 1 || p->nz < 0) return 0;+ if (p->n == 0) return open_frame() && !K.lost;+ struct { int mode; VkcKvsJoin b; } pc = {5, *p};+ K.kind = PK_ATTN;+ VkcBind bd[4] = {B(parts, 0), B(NULL, 0), B(NULL, 0), B(out, 1)};+ uint32_t gx, gy; grid((uint64_t)p->n, &gx, &gy);+ return record(pipe, bd, 4, &pc, sizeof pc, gx, gy, 1);+}int vkc_kvs_rel(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *out, VkcBuf *r, VkcBuf *tau, VkcBuf *relp, VkcBuf *tab,const VkcKvsRel *p) {VkPipeline pipe = kvs_pipe();diff --git a/c/vk_chain.h b/c/vk_chain.hindex f67041ae..1b154de1 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -53,6 +53,7 @@ typedef struct VkcBuf VkcBuf;#define VKC_DEV 0#define VKC_UP 1#define VKC_DOWN 2+#define VKC_HOST 3 /* host memory read in place (vkc_host): bound read-only, never written */int vkc_init(void); /* after coli_vk_init; 1 = the chain's pipelines are up */int vkc_ready(void);@@ -64,6 +65,10 @@ void vkc_free(VkcBuf *b); /* waits for the framesint vkc_reserve(VkcBuf **b, size_t bytes, int kind); /* at least `bytes`; growing drops the contents */void *vkc_ptr(const VkcBuf *b); /* host mapping (VKC_UP, VKC_DOWN; VKC_DEV when host-visible) */size_t vkc_bytes(const VkcBuf *b);+/* The pages around a host range as a buffer the device reads in place (coli_vk_host_buffer:+ * a device that shares the RAM, without staged uploads); *off: ptr's byte offset in it.+ * NULL when the device cannot. vkc_free releases it after the frames that read it. */+VkcBuf *vkc_host(const void *ptr, size_t bytes, size_t *off);/* recording */int vkc_begin(void);@@ -112,6 +117,23 @@ int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcB* chain_attnb.comp: a workgroup per KV head and block of rows, each K and V row read once* per block instead of once per (head, row); below it chain_attn as before. */int vkc_attn_block_rows(void);+/* chain_attnb's part mode (vk_kvsplit.h, the host's part on the device): each row's causal+ * positions before its share of a split cache, [first, (pos/B - anchor)*B), from a cache+ * in the host's head-major layout (p->a.cap positions a KV head), the value sum+ * unnormalized at o_off + s*o_row + h*vd and (m, l) at st_off + (s*H + h)*2. Blocked for+ * any S, its tiles at multiples of their size: a row's bits do not depend on the step+ * around it. No lists, sink or ring; 0 without the blocked pipeline. */+int vkc_attn_part(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off);+/* The same over chunks of `ch` positions (a multiple of 16), nz of them, chunk z's part at+ * o_off + z*o_z + s*o_row + h*vd and (m, l) at st_off + z*st_z + (s*H + h)*2 (vkc_kvs_join+ * joins them): a decode row's positions over many workgroups. */+int vkc_attn_part_chunks(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off,+ int ch, int nz, int o_z, int st_z);+/* chain_kvs.comp mode 5: the chunks of a part joined in order (vkc_attn_part_chunks): segment+ * g of d floats, chunk z's sum at a_off + z*a_z + g*d and (m, l) at sa_off + z*sa_z + 2g of+ * parts, into out at o_off + g*d and so_off + 2g. */+typedef struct { int n, d, nz, a_off, a_z, sa_off, sa_z, o_off, so_off; } VkcKvsJoin;+int vkc_kvs_join(VkcBuf *parts, VkcBuf *out, const VkcKvsJoin *p);/* The attention ops (vkc_attn, vkc_attn_w, vkc_mla_core, vkc_relattn) whose rows x positions* x heads x head dim pass COLI_VK_ATTN_SLICE (2^32; 0 = never) record their rows in slices,* each ending its frame (submitted, not waited for) and the next in a new one: no single@@ -541,6 +563,7 @@ void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nbint vkc_kvs_ready(void);#define VKC_KVS_FIN 1#define VKC_KVS_GATE 2+#define VKC_KVS_COLD 32 /* vkc_kvs_attn/_mla/_rel: the host's positions, from a buffer in the host's layout (row t = position t) */typedef struct { int S, H, KVH, hd, vd, pos_base, rows, q_off, q_row, q_seg, sel_off, sel_row; float scale;int k_off, v_off, win, kv_pm, sink, sink_off, B, ns, bt_off, nblk, anchor, o_off, o_row, st_off, flags,g_off, g_row, g_seg; } VkcKvsAttn;diff --git a/c/vk_kvsplit.h b/c/vk_kvsplit.hindex 1c7b6687..d6c3777c 100644--- a/c/vk_kvsplit.h+++ b/c/vk_kvsplit.h@@ -33,6 +33,12 @@* COLI_VK_KV_BLOCK=B positions per block (default 64)* COLI_VK_KV_PIN=1 opt in to read-based pins (GQA/MLA rounding can then* depend on read history; the default is deterministic)+ * COLI_VK_KV_COLD=device the host's part on the device too (vkc_kv_shadow): each+ * split layer keeps a copy of the host's rows in memory the+ * device reads in place, kept up to date row by row, and the+ * device computes the same positions the CPU would; on a+ * device that can import host memory (not with staged+ * uploads), without pins. Unset: the CPU, as before** The residency (VkcKvTab, per layer): bt (block -> slot or -1) and rb (slot -> block* or -1), uploaded to `tab` when they change; `valid`, the watermark: every resident@@ -72,6 +78,14 @@ typedef struct {VkcBuf *dev; size_t dev_off;} VkcKvPart;+/* COLI_VK_KV_COLD=device: a split layer's host rows in memory the device reads in place,+ * per array (K and V, or the latent and the rope key): nseg segments of cap positions of+ * seglen floats, head-major (segment g, position t at (g*cap + t)*seglen), page-aligned and+ * imported once (b, off floats into it); rows below `valid` equal the host's. */+typedef struct {+ float *p[2]; VkcBuf *b[2]; size_t off[2], n[2];+ int nseg[2], seglen[2], cap, valid;+} VkcKvShadow;typedef struct {int on; /* split: every split layer holds `rows` rows on the device */int B, ns, nw, np, rows, nl; /* block size, slots (window, pinned), rows = ns*B, layers */@@ -86,12 +100,26 @@ typedef struct {float *hpart; size_t hpart_n;int *cold, *lo, *cnt, *causal; size_t cold_n, lo_n;unsigned long long steps, splits, host_pos, pins, staged_rows; double host_ms, wait_ms;+ int cold_dev; /* COLI_VK_KV_COLD=device: the host's part on the device, from sh */+ VkcKvShadow *sh; /* per split layer (cold_dev) */+ VkcBuf *cscr; /* its chunks' parts (vkc_attn_part_chunks), joined into cpart */+ unsigned long long dev_cold, shadow_rows; size_t shadow_bytes; double shadow_ms;const char *engine;} VkcKvSplit;static double vkc_kv_now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }+static void vkc_kv_shadow_free(VkcKvSplit *ks) {+ if (ks->sh) for (int i = 0; i < ks->nl; i++)+ for (int k = 0; k < 2; k++) {+ vkc_free(ks->sh[i].b[k]); /* after the frames that read it, before its pages go */+ free(ks->sh[i].p[k]);+ }+ free(ks->sh); ks->sh = NULL; ks->shadow_bytes = 0;+ vkc_free(ks->cscr); ks->cscr = NULL;+}static void vkc_kv_free(VkcKvSplit *ks) {+ vkc_kv_shadow_free(ks);if (ks->t) for (int i = 0; i < ks->nl; i++) {VkcKvTab *t = &ks->t[i];free(t->bt); free(t->rb); free(t->pin); free(t->fresh); free(t->hits);@@ -114,6 +142,7 @@ static void vkc_kv_reset(VkcKvSplit *ks) {for (int j = 0; j < ks->ns; j++) { t->rb[j] = -1; t->pin[j] = 0; t->fresh[j] = 0; }memset(t->hits, 0, (size_t)t->nblk * sizeof *t->hits);t->valid = 0; t->dirty = 1; t->wb0 = t->wb1 = 0; t->counted = 0;+ if (ks->sh) ks->sh[i].valid = 0;}}@@ -194,13 +223,19 @@ static int vkc_kv_plan_need(VkcKvSplit *ks, const char *engine, int nl, size_t r}ks->tab = vkc_buf((size_t)nl * ks->tab_stride * sizeof(int), VKC_DEV);if (!ks->tab) { vkc_kv_free(ks); return 0; }+ { const char *cm = getenv("COLI_VK_KV_COLD");+ ks->cold_dev = cm && !strcmp(cm, "device") && np == 0 && coli_vk_import_alignment() > 0;+ if (cm && !strcmp(cm, "device") && !ks->cold_dev)+ fprintf(stderr, "[VK] %s chain: COLI_VK_KV_COLD=device needs host memory the device reads in place and no pins; "+ "the CPU computes the host's part\n", engine);+ if (ks->cold_dev && !(ks->sh = (VkcKvShadow *)calloc((size_t)nl, sizeof *ks->sh))) { vkc_kv_free(ks); return 0; } }ks->on = 1;vkc_kv_reset(ks);fprintf(stderr, "[VK] %s chain: the KV cache split past the device's budget: %d of %d positions a layer on the device ""(%d blocks of %d, %d for pins by reads; a row's newest %d positions on the device), the rest in RAM "- "(%.1f MiB on the device instead of %.1f)\n",+ "(%.1f MiB on the device instead of %.1f)%s\n",engine, ks->rows, cap, ns, B, np, (ks->anchor + 1) * B, (double)nl * ks->rows * row_bytes / 1048576.0,- need / 1048576.0);+ need / 1048576.0, ks->cold_dev ? ", its attention on the device too (COLI_VK_KV_COLD=device)" : "");return 2;}static int vkc_kv_plan(VkcKvSplit *ks, const char *engine, int nl, size_t row_bytes, int cap, int chunk, int selects,@@ -229,6 +264,9 @@ static void vkc_kv_assign(VkcKvTab *t, int j, int b, int pinned) {* blocks and the ones its rows' shares reach back to get slots. */static void vkc_kv_place(VkcKvSplit *ks, int li, int pos_base, int S) {VkcKvTab *t = &ks->t[li];+ /* the shadow too: rows from pos_base on may be rewritten from here (this step's, or a+ * new request's after a rewind, whether or not this step has a host part) */+ if (ks->sh && ks->sh[li].valid > pos_base) ks->sh[li].valid = pos_base > 0 ? pos_base : 0;if (pos_base + S <= 0) { t->wb0 = t->wb1 = 0; return; }int B = ks->B, wb1 = (pos_base + S - 1) / B, wb0 = wb1 - ks->nw + 1;if (wb0 < 0) wb0 = 0;@@ -348,7 +386,10 @@ static int vkc_kv_store(VkcKvSplit *ks, int li, const VkcKvPart *pt, VkcBuf *srcreturn ok;}static void vkc_kv_done(VkcKvSplit *ks, int li, int end) { if (ks->on) ks->t[li].valid = end; }-static void vkc_kv_lower(VkcKvSplit *ks, int li, int pos_base) { if (ks->on && ks->t[li].valid > pos_base) ks->t[li].valid = pos_base; }+static void vkc_kv_lower(VkcKvSplit *ks, int li, int pos_base) {+ if (ks->on && ks->t[li].valid > pos_base) ks->t[li].valid = pos_base;+ if (ks->on && ks->sh && ks->sh[li].valid > pos_base) ks->sh[li].valid = pos_base;+}/* 1 when a row of the step may read a position outside the device's share: causal* rows see [first, pos] (first = pos - win + 1 with a window, else kv_start), the share@@ -671,6 +712,55 @@ static int vkc_kv_run(VkcKvSplit *ks, VkcBuf *q, size_t q_off, size_t n_q, VkcBureturn vkc_kv_run2(ks, q, q_off, n_q, NULL, 0, 0, sel, sel_off, n_sel, dev, host, u, part_n);}+/* ---- the host's part on the device (COLI_VK_KV_COLD=device) ------------------------+ * Layer li's shadow up to row `upto` (exclusive: a step's cold positions are all below its+ * first row, which the host holds): nparts arrays, part k's segment g of position t at+ * host[k] + g*grp[k] + t*pstride[k], seglen[k] floats. Made at the first use, sized to the+ * tables' positions; rows from `valid` on are copied (a step lowers `valid` to its first+ * row: the rows from there may be rewritten, by it or by a rollback). NULL when the memory+ * or the import fails (the CPU's part then, for good on this layer). */+static VkcKvShadow *vkc_kv_shadow(VkcKvSplit *ks, int li, int nparts, const float *const *host, const size_t *grp,+ const size_t *pstride, const int *nseg, const int *seglen, int upto) {+ VkcKvShadow *sh = &ks->sh[li];+ if (sh->cap < 0) return NULL;+ if (!sh->cap) {+ size_t al = coli_vk_import_alignment();+ for (int k = 0; k < nparts; k++) {+ size_t n = (size_t)nseg[k] * ks->cap * seglen[k], bytes = (n * sizeof(float) + al - 1) / al * al;+ void *p = NULL;+ if (!al || posix_memalign(&p, al, bytes)) p = NULL;+ sh->p[k] = (float *)p;+ if (p) { memset(p, 0, bytes); sh->b[k] = vkc_host(p, bytes, &sh->off[k]); }+ if (!p || !sh->b[k]) {+ fprintf(stderr, "[VK] %s chain: no shadow for layer %d's host rows (%.1f MiB), the CPU computes its part\n",+ ks->engine, li, bytes / 1048576.0);+ for (int j = 0; j <= k; j++) { vkc_free(sh->b[j]); free(sh->p[j]); sh->b[j] = NULL; sh->p[j] = NULL; }+ sh->cap = -1;+ return NULL;+ }+ sh->off[k] /= sizeof(float); sh->n[k] = n; sh->nseg[k] = nseg[k]; sh->seglen[k] = seglen[k];+ ks->shadow_bytes += bytes;+ }+ sh->cap = ks->cap; sh->valid = 0;+ }+ if (upto > sh->cap) upto = sh->cap;+ if (sh->valid > upto) sh->valid = upto;+ if (sh->valid < upto) {+ double t0 = vkc_kv_now();+ long rows = upto - sh->valid;+ for (int k = 0; k < nparts; k++) {+ int G = nseg[k], L = seglen[k];+ #pragma omp parallel for collapse(2) schedule(static) if (rows * G * L > (1 << 20))+ for (int g = 0; g < G; g++)+ for (long t = sh->valid; t < upto; t++)+ memcpy(sh->p[k] + ((size_t)g * sh->cap + t) * L, host[k] + g * grp[k] + t * pstride[k], (size_t)L * sizeof(float));+ }+ ks->shadow_rows += (unsigned long long)rows; ks->shadow_ms += vkc_kv_now() - t0;+ sh->valid = upto;+ }+ return sh;+}+/* ---- one layer's attention over the split cache, the forms the engines use ----------- *//* The device's attention of S rows in slices of rows, each slice in a submission of its* own (the rows are independent: the same bits), so that no one submission runs long@@ -715,6 +805,48 @@ static int vkc_kv_gqa_rec(VkcKvGqa *a, int s0, int ns, int fin) {return vkc_kvs_attn(a->q, a->kc, a->vc, fin ? a->out : ks->dpart, fin ? a->gate : NULL, a->sel, a->snk, ks->tab, &p);}static int vkc_kv_gqa_part(void *u, int s0, int ns) { return vkc_kv_gqa_rec((VkcKvGqa *)u, s0, ns, 0); }+/* rows [s0, s0 + ns) of the host's part on the device, from the shadow, into ks->cpart */+static int vkc_kv_gqa_cold(void *u, int s0, int ns) {+ VkcKvGqa *a = (VkcKvGqa *)u; VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];+ int bt = (int)((size_t)a->li * ks->tab_stride), nb = ks->t[a->li].nblk;+ VkcKvsAttn p = {ns, a->H, a->KVH, a->hd, a->vd, a->pos_base + s0, sh->cap, s0 * a->q_row, a->q_row, a->q_seg,+ a->sel_off + s0 * a->sel_row, a->sel_row, a->scale, (int)sh->off[0], (int)sh->off[1], a->win, 0, 0, 0,+ ks->B, ks->ns, bt, nb, ks->anchor, s0 * a->H * a->vd, a->H * a->vd, a->S * a->H * a->vd + s0 * a->H * 2,+ VKC_KVS_COLD, 0, 0, 0};+ return vkc_kvs_attn(a->q, sh->b[0], sh->b[1], ks->cpart, NULL, a->sel, NULL, ks->tab, &p);+}+/* The same without lists through the blocked attention (vkc_attn_part_chunks): the+ * shadow's K and V rows read once per block of rows, not once per (head, row), and the+ * positions in chunks of VKC_KV_CCH, one workgroup each, joined in order (vkc_kvs_join):+ * a decode row's thousands of positions over many workgroups. The chunks are fixed by+ * position, so a row's bits do not depend on the step around it. Rows go in slices whose+ * chunk parts fit 64 MiB. */+#define VKC_KV_CCH 512+static int vkc_kv_gqa_cold_blocked(VkcKvGqa *a) {+ VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];+ int H = a->H, vd = a->vd, last = a->pos_base + a->S - 1;+ int end = (last / ks->B - ks->anchor) * ks->B; /* the last row's cold positions end there */+ int nz = end > 0 ? (end + VKC_KV_CCH - 1) / VKC_KV_CCH : 0;+ size_t per_row = (size_t)(nz > 0 ? nz : 1) * H * (vd + 2) * sizeof(float);+ int rows = (int)(((size_t)64 << 20) / per_row);+ if (rows < 1) rows = 1;+ if (rows > a->S) rows = a->S;+ if (!vkc_reserve(&ks->cscr, (size_t)rows * per_row, VKC_DEV)) return 0;+ for (int r0 = 0; r0 < a->S; r0 += rows) {+ int n = a->S - r0 < rows ? a->S - r0 : rows, o_z = n * H * vd, st_z = n * H * 2, sa_off = nz * o_z;+ if (nz > 0) {+ VkcAttnW w; memset(&w, 0, sizeof w);+ w.a = (VkcAttn){n, H, a->KVH, a->hd, a->pos_base + r0, sh->cap, r0 * a->q_row, a->q_row, a->q_seg, 0, 0, 0, 0,+ 0, H * vd, 0, 0, a->scale, (int)sh->off[0], (int)sh->off[1]};+ w.win = a->win; w.vd = vd;+ if (!vkc_attn_part_chunks(a->q, sh->b[0], sh->b[1], ks->cscr, &w, ks->B, ks->anchor, sa_off, VKC_KV_CCH, nz,+ o_z, st_z)) return 0;+ }+ VkcKvsJoin j = {n * H, vd, nz, 0, o_z, sa_off, st_z, r0 * H * vd, a->S * H * vd + r0 * H * 2};+ if (!vkc_kvs_join(ks->cscr, ks->cpart, &j)) return 0;+ }+ return 1;+}static int vkc_kv_gqa_fin(void *u, int s0, int ns) { return vkc_kv_gqa_rec((VkcKvGqa *)u, s0, ns, 1); }static long vkc_kv_span(const VkcKvSplit *ks, int win, int sel_row) { /* the most positions a row's device part reads */long n = (long)(ks->anchor + 1) * ks->B;@@ -747,6 +879,20 @@ static int vkc_kv_gqa(VkcKvGqa *a) {if (!vkc_kv_cold(ks, a->li, a->pos_base, a->S, a->win, 0, a->sel_row > 0))return vkc_kv_sliced(a->S, a->H, vkc_kv_span(ks, a->win, a->sel_row), a->hd + a->vd, vkc_kv_gqa_fin, a);size_t part = (size_t)a->S * a->H * (a->vd + 2);+ if (ks->cold_dev) { /* the host's part on the device too: one frame, no round trip */+ const float *host[2] = {a->Kh, a->Vh};+ size_t grp[2] = {a->k_grp, a->v_grp}, ps[2] = {a->k_pos, a->v_pos};+ int nseg[2] = {a->KVH, a->KVH}, sl[2] = {a->hd, a->vd};+ if (vkc_kv_shadow(ks, a->li, 2, host, grp, ps, nseg, sl, a->pos_base)) {+ ks->dev_cold++; ks->splits++; /* a step with a host part, on the device */+ VkcKvsMerge mg = {a->S * a->H, a->H, a->vd, 0, a->H * a->vd, a->vd, a->S * a->H * a->vd, 0, a->S * a->H * a->vd,+ 0, a->o_row, a->vd, a->gated ? VKC_KVS_GATE : 0, a->g_off, a->g_row, a->g_seg};+ return vkc_kv_parts(ks, part) && vkc_kv_gqa_dev(a) &&+ (a->sel_row > 0 ? vkc_kv_sliced(a->S, a->H, (long)a->pos_base + a->S, a->hd + a->vd, vkc_kv_gqa_cold, a)+ : vkc_kv_gqa_cold_blocked(a)) &&+ vkc_kvs_merge(ks->dpart, ks->cpart, a->gate, a->out, &mg);+ }+ }if (!vkc_kv_parts(ks, part) ||!vkc_kv_run(ks, a->q, 0, (size_t)a->S * a->q_row, a->sel_row > 0 ? a->sel : NULL, (size_t)a->sel_off,(size_t)a->S * a->sel_row, vkc_kv_gqa_dev, vkc_kv_gqa_host, a, part)) return 0;@@ -781,6 +927,15 @@ static int vkc_kv_mla_rec(VkcKvMla *a, int s0, int ns, int fin) {return vkc_kvs_mla(a->qa, a->qr, a->lat, a->rope, a->sel, fin ? a->out : ks->dpart, ks->tab, &p);}static int vkc_kv_mla_part(void *u, int s0, int ns) { return vkc_kv_mla_rec((VkcKvMla *)u, s0, ns, 0); }+static int vkc_kv_mla_cold(void *u, int s0, int ns) {+ VkcKvMla *a = (VkcKvMla *)u; VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];+ int bt = (int)((size_t)a->li * ks->tab_stride), nb = ks->t[a->li].nblk;+ VkcKvsMla p = {ns, a->H, a->K, a->R, a->pos_base + s0, a->kv_start, a->qa_off + s0 * a->qa_row, a->qa_row, a->qa_seg,+ a->qr_off + s0 * a->qr_row, a->qr_row, a->qr_seg, (int)sh->off[0], a->K, (int)sh->off[1], a->R,+ a->sel_off + s0 * a->sel_row, a->sel_row, s0 * a->H * a->K, a->H * a->K, a->K, a->scale, ks->B, ks->ns,+ bt, nb, ks->anchor, VKC_KVS_COLD, a->S * a->H * a->K + s0 * a->H * 2};+ return vkc_kvs_mla(a->qa, a->qr, sh->b[0], a->R > 0 ? sh->b[1] : NULL, a->sel, ks->cpart, ks->tab, &p);+}static int vkc_kv_mla_fin(void *u, int s0, int ns) { return vkc_kv_mla_rec((VkcKvMla *)u, s0, ns, 1); }static int vkc_kv_mla_dev(void *u) {VkcKvMla *a = (VkcKvMla *)u;@@ -805,6 +960,19 @@ static int vkc_kv_mla(VkcKvMla *a) {if (!vkc_kv_cold(ks, a->li, a->pos_base, a->S, 0, a->kv_start, a->sel_row > 0))return vkc_kv_sliced(a->S, a->H, vkc_kv_span(ks, 0, a->sel_row), 2 * a->K + a->R, vkc_kv_mla_fin, a);size_t part = (size_t)a->S * a->H * (a->K + 2);+ if (ks->cold_dev) { /* the host's part on the device too */+ const float *host[2] = {a->Lh, a->Rh};+ size_t grp[2] = {0, 0}, ps[2] = {(size_t)a->K, (size_t)a->R};+ int nseg[2] = {1, 1}, sl[2] = {a->K, a->R};+ if (vkc_kv_shadow(ks, a->li, a->R > 0 ? 2 : 1, host, grp, ps, nseg, sl, a->pos_base)) {+ ks->dev_cold++; ks->splits++; /* a step with a host part, on the device */+ VkcKvsMerge mg = {a->S * a->H, a->H, a->K, 0, a->H * a->K, a->K, a->S * a->H * a->K, 0, a->S * a->H * a->K,+ a->o_off, a->o_row, a->o_seg, 0, 0, 0, 0};+ return vkc_kv_parts(ks, part) && vkc_kv_mla_dev(a) &&+ vkc_kv_sliced(a->S, a->H, (long)a->pos_base + a->S, 2 * a->K + a->R, vkc_kv_mla_cold, a) &&+ vkc_kvs_merge(ks->dpart, ks->cpart, NULL, a->out, &mg);+ }+ }a->nqa = (size_t)(a->S - 1) * a->qa_row + (size_t)(a->H - 1) * a->qa_seg + a->K;size_t nqr = a->R > 0 ? (size_t)(a->S - 1) * a->qr_row + (size_t)(a->H - 1) * a->qr_seg + a->R : 0;if (!vkc_kv_parts(ks, part) ||@@ -841,6 +1009,14 @@ static int vkc_kv_rel_rec(VkcKvRel *a, int s0, int ns, int fin) {return vkc_kvs_rel(a->q, a->kc, a->vc, fin ? a->out : ks->dpart, a->r, a->tau, a->relp, ks->tab, &p);}static int vkc_kv_rel_part(void *u, int s0, int ns) { return vkc_kv_rel_rec((VkcKvRel *)u, s0, ns, 0); }+static int vkc_kv_rel_cold(void *u, int s0, int ns) {+ VkcKvRel *a = (VkcKvRel *)u; VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];+ int bt = (int)((size_t)a->li * ks->tab_stride), nb = ks->t[a->li].nblk;+ VkcKvsRel p = {ns, a->H, a->KVH, a->hd, a->pos_base + s0, sh->cap, s0 * a->q_row, a->q_row, a->scale, (int)sh->off[0],+ (int)sh->off[1], a->ext, a->d_rel, s0 * a->r_row, a->r_row, a->relp_off, a->tau_off + s0, ks->B, bt, nb,+ ks->anchor, s0 * a->H * a->hd, a->H * a->hd, a->S * a->H * a->hd + s0 * a->H * 2, VKC_KVS_COLD};+ return vkc_kvs_rel(a->q, sh->b[0], sh->b[1], ks->cpart, a->r, a->tau, a->relp, ks->tab, &p);+}static int vkc_kv_rel_fin(void *u, int s0, int ns) { return vkc_kv_rel_rec((VkcKvRel *)u, s0, ns, 1); }static int vkc_kv_rel_dev(void *u) {VkcKvRel *a = (VkcKvRel *)u;@@ -867,6 +1043,19 @@ static int vkc_kv_rel(VkcKvRel *a) {if (!vkc_kv_cold(ks, a->li, a->pos_base, a->S, 0, 0, 0))return vkc_kv_sliced(a->S, a->H, vkc_kv_span(ks, 0, 0), 2 * a->hd + a->d_rel, vkc_kv_rel_fin, a);size_t part = (size_t)a->S * a->H * (a->hd + 2);+ if (ks->cold_dev) { /* the host's part on the device too */+ const float *host[2] = {a->Kh, a->Vh};+ size_t grp[2] = {a->k_grp, a->v_grp}, ps[2] = {a->k_pos, a->v_pos};+ int nseg[2] = {a->KVH, a->KVH}, sl[2] = {a->hd, a->hd};+ if (vkc_kv_shadow(ks, a->li, 2, host, grp, ps, nseg, sl, a->pos_base)) {+ ks->dev_cold++; ks->splits++; /* a step with a host part, on the device */+ VkcKvsMerge mg = {a->S * a->H, a->H, a->hd, 0, a->H * a->hd, a->hd, a->S * a->H * a->hd, 0, a->S * a->H * a->hd,+ 0, a->o_row, a->hd, 0, 0, 0, 0};+ return vkc_kv_parts(ks, part) && vkc_kv_rel_dev(a) &&+ vkc_kv_sliced(a->S, a->H, (long)a->pos_base + a->S, 2 * a->hd + a->d_rel, vkc_kv_rel_cold, a) &&+ vkc_kvs_merge(ks->dpart, ks->cpart, NULL, a->out, &mg);+ }+ }a->nq = (size_t)a->S * a->q_row;if (!vkc_kv_parts(ks, part) ||!vkc_kv_run2(ks, a->q, 0, a->nq, a->d_rel > 0 ? a->r : NULL, 0, (size_t)a->S * a->r_row, NULL, 0, 0,@@ -1016,7 +1205,11 @@ static int vkc_kv_mla_attn(VkcKvSplit *ks, int li, const VkcMla *m, VkcMlaScratcstatic void vkc_kv_report(const VkcKvSplit *ks) {if (!ks->on || !ks->steps) return;fprintf(stderr, "[VK] %s chain: KV split: %llu layer steps, %llu with a host part (%.1f ms waiting for the queries, "- "%.1f ms of host work over %llu positions, %llu sparse rows staged), %llu blocks pinned by reads\n",+ "%.1f ms of host work over %llu positions, %llu sparse rows staged), %llu blocks pinned by reads",ks->engine, ks->steps, ks->splits, ks->wait_ms, ks->host_ms, ks->host_pos, ks->staged_rows, ks->pins);+ if (ks->cold_dev)+ fprintf(stderr, " | the host's part on the device: %llu layer steps, %llu rows copied to the shadow in %.1f ms "+ "(%.1f MiB held)", ks->dev_cold, ks->shadow_rows, ks->shadow_ms, ks->shadow_bytes / 1048576.0);+ fprintf(stderr, "\n");}#endifdiff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex a014f033..dc0a6295 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -272,6 +272,7 @@ With `COLI_VULKAN=1` every MoE engine (qwen36, qwen38, inkling, olmoe, kimi_k3,| `COLI_VK_KV_DEVICE_ROWS` | from the device's budget | The positions a split layer keeps on the device; set, the cache splits whenever it is longer (the tests' small device, a measurement's split). || `COLI_VK_KV_BLOCK` | `64` | Positions per block of the split cache's block table. || `COLI_VK_KV_PIN` | off | `1`: pin blocks by sparse-selection reads. For QSA/DSA/pooled MLA this can change rounding with read history; the default keeps the partition independent of chunking and prefix reuse. DeepSeek stages missing selected rows and preserves its arithmetic in either mode. |+| `COLI_VK_KV_COLD` | unset | `device`: the host's part of a split KV cache attended on the device too, from a shadow of the host's rows (GQA, MLA and Inkling forms; DeepSeek keeps it on the CPU). Off by default: on the measured integrated GPU it made the prompt 4-6% faster and decode 5.7x slower (see [vulkan.md](vulkan.md#a-kv-cache-past-the-devices-budget)). || `COLI_VK_CHAIN_FAULT` | off | Tests: the n-th chain frame fails as a lost device would; the engine rebuilds the state on the CPU and runs there (colibri, mimo and the DeepSeek engines have none to rebuild: the CPU continues from its host caches). || `DUMP` | unset | colibri, glm53, deepseek_v41 and deepseek_v4 in a `VK=1` build: every logits row the forward computes, appended to this file as raw f32 (the Vulkan gates compare the device's with the CPU's). |diff --git a/docs/vulkan.md b/docs/vulkan.mdindex de6ef842..7bf7eae0 100644--- a/docs/vulkan.md+++ b/docs/vulkan.md@@ -1001,6 +1001,7 @@ f32 throughout, as the CPU's f32 path.| `COLI_VK_KV_DEVICE_ROWS` | from the budget | The positions a split layer keeps on the device; set, the cache splits whenever it is longer. || `COLI_VK_KV_BLOCK` | `64` | Positions per block of the split's block table. || `COLI_VK_KV_PIN` | off | `1`: enable read-based block pins; QSA/DSA/pooled MLA rounding can then depend on read history. DeepSeek preserves its sparse arithmetic in either mode. |+| `COLI_VK_KV_COLD` | unset | `device`: a split layer's host part attended on the device too, from a shadow of the host's rows ([below](#a-kv-cache-past-the-devices-budget)); off by default, because on the measured integrated GPU it slowed decode. |Each run and serve turn prints `[VK] <engine> chain: N forwards, F frames (ops,matmuls, tiled GEMM), the time spent waiting for the device, the routed experts' host@@ -2063,6 +2064,38 @@ round trip is paid only on layers with a host part.| `vkc_kvs_ds` | `chain_kvs` (mode 4) | DeepSeek's sparse attention with its sink over the window rows and the resident compressed rows; with every listed row resident, `vkc_dsv4_attn`'s bits || `vkc_kvs_merge` | `chain_kvs` (mode 2) | the two parts joined, a gate or V4's bf16 rounding after |+**The host's part on the device (`COLI_VK_KV_COLD=device`, off by default).** The+host's rows can also be attended on the device, so a step with a host part runs in one+frame with no round trip:+- **The copy.** Each split layer keeps a shadow of the host's rows in page-aligned memory+ the device reads in place (an imported host allocation, `VK_EXT_external_memory_host`).+ The shadow follows the host's cache row by row and is lowered with it on a rewind or a+ rollback.+- **The ops.** `chain_kvs` reads the shadow with its COLD flag (the GQA, MLA and Inkling+ forms; position t is row t). The prompt's rows take the blocked attention+ (`chain_attnb`'s part mode) in chunks of 512 positions, one workgroup each, and mode 5+ of `chain_kvs` joins them in order.+- **The bits.** The chunks are fixed by position, so a row's bits do not depend on how a+ forward is cut into steps, as on the CPU path.+- **Where it applies.** It needs a device that imports host memory, no read-based pins+ and no staged uploads. DeepSeek's sparse forms keep the host's part on the CPU. Where+ it cannot run, the line says so and the CPU computes the host's part.++The run's report adds `| the host's part on the device: N layer steps, R rows copied+to the shadow in T ms (M MiB held)`.++| Qwen3.6-35B-A3B, 7579-token prompt, Radeon 780M, the chain | first token after | 31 decode tokens |+|---|---|---|+| 1024 positions a layer on the device, the host's part on the CPU | 121 s | 3.5 s |+| the same, the host's part on the device | 114 s | 20 s |+| 4096 positions a layer, on the CPU | 143 s | |+| the same, on the device | 137 s | |++The prompt gains 4 to 6%. Decode is 5.7 times slower: with the default, the CPU+computes the host's part while the device runs its own, and on this integrated GPU the+decode attention is slower than the CPU's. Hence the default. A dedicated GPU, whose+attention outruns the CPU's by more, has not been measured.+`vk_kvsplit.h` carries the rest: the plan, the tables, the uploads and the stores of astep's rows, the host's part (`vkc_kv_host_attn`, with the same bias, tau and V4rounding), and one call per form for an engine (`vkc_kv_gqa`, `vkc_kv_mla`,@@ -2132,7 +2165,9 @@ part): the CPU's tokens, logits within each family's tolerance, chunks of 3, proonly, the split off, a lost device, pins, MTP and n-gram drafts, serve sessions withpins, prompt-cache extensions and divergent prompts, and the prefix-reuse tests;`kv-split-deepseek` the same for deepseek_v41 and deepseek_v4; `kv-split-sanitize` and-`kv-split-deepseek-sanitize` a set of each under ASan and UBSan.+`kv-split-deepseek-sanitize` a set of each under ASan and UBSan. `kv-split-cold` and+`kv-split-cold-sanitize` run `kv-split` and `kv-split-sanitize` again with+`COLI_VK_KV_COLD=device`, and every gate must have run the host's part on the device.Validation on the local Lavapipe device covers numerical correctness, not hardwarethroughput. A discrete GPU has not been measured for this change.