Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: the split KV cache's host part on the device (COLI_VK_KV_COLD=device)

JustVugg323da5aaf4bc
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4d5710ca..3eb54ee4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -362,6 +362,13 @@ jobs:
- family: kv-split-sanitize
req: c/tools/requirements-qwen38-tiny.txt
extra: numpy pillow
+ # the same with the host's part on the device too (COLI_VK_KV_COLD=device)
+ - family: kv-split-cold
+ req: c/tools/requirements-qwen38-tiny.txt
+ extra: numpy pillow
+ - family: kv-split-cold-sanitize
+ req: c/tools/requirements-qwen38-tiny.txt
+ extra: numpy pillow
- family: kv-split-deepseek
req: c/tools/requirements-deepseek-v4-tiny.txt
extra: sympy pillow
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index a77011dd..904941e2 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -1569,6 +1569,53 @@ size_t coli_vk_import_alignment(void) {
#endif
}
size_t coli_vk_imported_bytes(void) { return __atomic_load_n(&g_import_bytes, __ATOMIC_RELAXED); }
+/* A host range read in place by a shader (the chain's VKC_HOST): the pages around it,
+ * so the caller's allocation need not be aligned; the pages past its ends are the
+ * process's own and are never written. */
+int coli_vk_host_buffer(const void *ptr, size_t bytes, void **buf_out, void **mem_out, size_t *off) {
+#ifdef VK_EXT_external_memory_host
+ size_t al = coli_vk_import_alignment();
+ if (!al || !ptr || !bytes) return 0;
+ uintptr_t p = (uintptr_t)ptr, base = p / al * al, end = (p + bytes + al - 1) / al * al;
+ size_t sz = end - base;
+ if (sz > G.ssbo_range) return 0;
+ static PFN_vkGetMemoryHostPointerPropertiesEXT gp;
+ if (!gp) gp = (PFN_vkGetMemoryHostPointerPropertiesEXT)vkGetDeviceProcAddr(G.dev, "vkGetMemoryHostPointerPropertiesEXT");
+ VkMemoryHostPointerPropertiesEXT mp = {.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
+ if (!gp || gp(G.dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, (void *)base, &mp) != VK_SUCCESS ||
+ !mp.memoryTypeBits) return 0;
+ VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
+ .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT};
+ VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = sz,
+ .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
+ VkBuffer b;
+ if (vkCreateBuffer(G.dev, &bi, NULL, &b) != VK_SUCCESS) return 0;
+ VkMemoryRequirements req;
+ vkGetBufferMemoryRequirements(G.dev, b, &req);
+ uint32_t bits = mp.memoryTypeBits & req.memoryTypeBits, mt = 0;
+ if (!bits || req.size > sz) { vkDestroyBuffer(G.dev, b, NULL); return 0; }
+ while (!(bits & (1u << mt))) mt++;
+ VkImportMemoryHostPointerInfoEXT imp = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
+ .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, .pHostPointer = (void *)base};
+ VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &imp, .allocationSize = sz,
+ .memoryTypeIndex = mt};
+ VkDeviceMemory m;
+ if (vkAllocateMemory(G.dev, &ai, NULL, &m) != VK_SUCCESS) { vkDestroyBuffer(G.dev, b, NULL); return 0; }
+ if (vkBindBufferMemory(G.dev, b, m, 0) != VK_SUCCESS) { vkDestroyBuffer(G.dev, b, NULL); vkFreeMemory(G.dev, m, NULL); return 0; }
+ *buf_out = (void *)b; *mem_out = (void *)m; *off = (size_t)(p - base);
+ __atomic_add_fetch(&g_import_bytes, sz, __ATOMIC_RELAXED);
+ return 1;
+#else
+ (void)ptr; (void)bytes; (void)buf_out; (void)mem_out; (void)off;
+ return 0;
+#endif
+}
+void coli_vk_host_buffer_free(void *buf, void *mem, size_t bytes) {
+ if (!G.dev) return;
+ if (buf) vkDestroyBuffer(G.dev, (VkBuffer)buf, NULL);
+ if (mem) vkFreeMemory(G.dev, (VkDeviceMemory)mem, NULL);
+ if (bytes) __atomic_sub_fetch(&g_import_bytes, bytes, __ATOMIC_RELAXED);
+}
int coli_vk_tensor_import(ColiVkTensor **tensor, const void *weights, size_t alloc_bytes, const float *scales,
int fmt, int I, int O, int gs) {
if (*tensor) return (*tensor)->fmt == fmt && (*tensor)->I == I && (*tensor)->O == O;
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index 882fb7d1..19d7bdad 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -111,6 +111,14 @@ int coli_vk_tensor_ensure(ColiVkTensor **tensor, const void *weights, const flo
int coli_vk_tensor_import(ColiVkTensor **tensor, const void *weights, size_t alloc_bytes, const float *scales,
int fmt, int I, int O, int gs);
size_t coli_vk_import_alignment(void);
+/* Host memory a shader reads in place (VK_EXT_external_memory_host, where
+ * coli_vk_import_alignment() is not 0): the pages around [ptr, ptr + bytes) as a storage
+ * buffer of the primary device (VkBuffer, VkDeviceMemory as void *), *off the byte
+ * offset of ptr in it. 0 when the device refuses. The pages must stay mapped until
+ * coli_vk_host_buffer_free (bytes: the buffer's, ptr's page-rounded range), after the
+ * last submission that read them. */
+int coli_vk_host_buffer(const void *ptr, size_t bytes, void **buf, void **mem, size_t *off);
+void coli_vk_host_buffer_free(void *buf, void *mem, size_t bytes);
size_t coli_vk_imported_bytes(void);
/* SECOND DEVICE (COLI_VK_DEV2): a self-contained context on another Vulkan GPU that
diff --git a/c/shaders/chain_attnb.comp b/c/shaders/chain_attnb.comp
index 599ecda9..5244c242 100644
--- a/c/shaders/chain_attnb.comp
+++ b/c/shaders/chain_attnb.comp
@@ -20,6 +20,18 @@
// per row (sel_row > 0): then BR = 1, the row's list is the workgroup's.
// The order of the sums differs from chain_attn's and the CPU's; all agree to rounding.
// Push constants: chain_attn's, then br (rows per workgroup, br * H / KVH <= 32).
+// part != 0 (vk_kvsplit.h's host part on the device, from its shadow of the host's
+// rows): a row's positions are its causal ones before its share of a split cache, the
+// blocks pos/B - anchor and on, so [first, (pos/B - anchor)*B); the output is the part
+// (the value sum unnormalized at o_off + s*o_row + h*vd, (m, l) at st_off + (s*H + h)*2)
+// for vkc_kvs_merge. Its tiles start at multiples of BC: a row's sums then depend on its
+// own positions only, whatever rows share its workgroup (a tile it does not see changes
+// nothing), so a row gets the same bits however a forward is cut into steps. No lists,
+// no sink, no ring there. With g_off > 0 the positions are cut into chunks of g_off (a
+// multiple of BC), workgroup z taking [z*g_off, (z+1)*g_off): its part at o_off +
+// z*g_row + ... and its (m, l) at st_off + z*g_seg + ..., for chain_kvs.comp's join (a
+// decode row's thousands of positions over many workgroups; the chunks are fixed by
+// position, so the bits still are the row's own).
// Limits: hd, vd <= 256.
layout(local_size_x = 256) in;
layout(constant_id = 0) const int BC = 8;
@@ -42,6 +54,7 @@ layout(push_constant) uniform P {
int k_off, v_off;
int win, ring, vd, kv_pm, sink, sink_off;
int br;
+ int B, anchor, part, st_off;
} p;
shared float ks[BC * 256]; // the tile's keys [j][d]
@@ -73,6 +86,18 @@ void main() {
int first = p.win > 0 ? max(0, pos - p.win + 1) : 0; // this pair's row
int bfirst = p.win > 0 ? max(0, p.pos_base + s0 - p.win + 1) : 0;
int n = p.pos_base + slast + 1 - bfirst;
+ int end = pos + 1; // this pair's row: positions before end
+ if (p.part != 0) {
+ end = (pos / p.B - p.anchor) * p.B;
+ int bend = ((p.pos_base + slast) / p.B - p.anchor) * p.B; // the block's last row's
+ bfirst -= bfirst % BC;
+ if (p.g_off > 0) {
+ int c0 = int(gl_WorkGroupID.z) * p.g_off, c1 = c0 + p.g_off;
+ bfirst = max(bfirst, c0); bend = min(bend, c1);
+ first = max(first, c0); end = min(end, c1);
+ }
+ n = max(0, bend - bfirst);
+ }
bool list = false;
int lb = 0;
if (p.sel_row > 0) {
@@ -122,7 +147,7 @@ void main() {
for (int j = lane; j < BC; j += 8) {
int t = tpos[j];
float sc = -3.0e38;
- if (live && t >= 0 && (list || (t >= first && t <= pos))) {
+ if (live && t >= 0 && (list || (t >= first && t < end))) {
float a = 0.0;
for (int k = 0; k < 8; k++) a += part[(pair * 8 + k) * BC + j];
sc = a * p.scale;
@@ -150,8 +175,19 @@ void main() {
barrier();
}
if (!live) return;
- float inv = l > 0.0 ? 1.0 / l : 0.0;
int ob = p.o_off + s * p.o_row + h * p.vd;
+ if (p.part != 0) { // the part (a chunk's, with g_off), for the merge (or the join)
+ int z = p.g_off > 0 ? int(gl_WorkGroupID.z) : 0;
+ int obz = ob + z * p.g_row, sb = p.st_off + z * p.g_seg + (s * p.H + h) * 2;
+ for (int i = 0; i < 32; i++) {
+ int d = lane + 8 * i;
+ if (d >= p.vd) break;
+ o[obz + d] = av[i];
+ }
+ if (lane == 0) { o[sb] = m; o[sb + 1] = l; }
+ return;
+ }
+ float inv = l > 0.0 ? 1.0 / l : 0.0;
int gb = p.g_off + s * p.g_row + h * p.g_seg;
for (int i = 0; i < 32; i++) {
int d = lane + 8 * i;
diff --git a/c/shaders/chain_attnb.spv b/c/shaders/chain_attnb.spv
index 697ed5c9..5d3d39a7 100644
Binary files a/c/shaders/chain_attnb.spv and b/c/shaders/chain_attnb.spv differ
diff --git a/c/shaders/chain_kvs.comp b/c/shaders/chain_kvs.comp
index bb460029..8113794d 100644
--- a/c/shaders/chain_kvs.comp
+++ b/c/shaders/chain_kvs.comp
@@ -40,6 +40,12 @@
// rope key at rope_off + r*rope_row. flags 1: the normalized clat at o_off + s*o_row
// + h*o_seg, as chain_mla writes it; else acc there and (m, l) at st_off + (s*H+h)*2.
// Limits: K <= 1024, R <= 128.
+// flags 32 (cold), modes 0, 1 and 3: the host's positions of the row instead of the
+// device's, read from a buffer in the host cache's own layout (an import of its pages:
+// position t is row t, `rows` its positions for head-major K/V), so the device computes
+// the part the CPU would: causal, [the row's first position, its share's first block);
+// listed, the entries older than the share and not in a pinned block. The table only
+// says what the device holds; a part's (m, l, acc) as without the flag. No sink.
// mode 2, the merge: segment g = (s, h) of d floats, s = g / H. The device's part at
// a_off + s*a_row + h*a_seg with its (m, l) at sa_off + 2g (buffer 0), the host's
// packed at c_off + g*d with (m, l) at sc_off + 2g (buffer 1); the result at o_off +
@@ -96,6 +102,14 @@ int slot_of(int t, int B, int bt_off, int nblk) {
int j = b < nblk ? tab[bt_off + b] : -1;
return j < 0 ? -1 : (j & 0x3fffffff);
}
+// a listed position the host takes (flags 32): older than the row's share, not pinned
+bool cold_pos(int t, int pos, int B, int bt_off, int nblk, int anchor) {
+ if (t < 0 || t > pos) return false;
+ int b = t / B;
+ if (b >= pos / B - anchor) return false;
+ int j = b < nblk ? tab[bt_off + b] : -1;
+ return j < 0 || (j & 0x40000000) == 0;
+}
// the device row of a listed position, or -1 (the host's): the anchored blocks of the
// row at pos, and pinned ones
int dev_row(int t, int pos, int B, int bt_off, int nblk, int anchor) {
@@ -121,9 +135,11 @@ void gqa() {
int qb = q_off + s * q_row + h * q_seg;
for (int d = tid; d < hd; d += 256) qs[d] = b0[qb + d];
int pos = pos_base + s;
- int first = max(win > 0 ? max(0, pos - win + 1) : 0, (pos / B - anchor) * B);
+ bool cold = (flags & 32) != 0;
+ int start = win > 0 ? max(0, pos - win + 1) : 0, share0 = (pos / B - anchor) * B;
+ int first = cold ? start : max(start, share0);
bool list = false;
- int lb = 0, n = pos - first + 1;
+ int lb = 0, n = cold ? max(0, min(pos + 1, share0) - start) : pos - first + 1;
if (sel_row > 0) {
lb = sel_off + s * sel_row;
int c = sel[lb];
@@ -134,7 +150,8 @@ void gqa() {
for (int t0 = 0; t0 < n; t0 += 256) {
int i = t0 + tid, r = -1;
if (i < n) {
- if (list) r = dev_row(sel[lb + 1 + i], pos, B, bt_off, nblk, anchor);
+ if (list) { int t = sel[lb + 1 + i]; r = cold ? (cold_pos(t, pos, B, bt_off, nblk, anchor) ? t : -1) : dev_row(t, pos, B, bt_off, nblk, anchor); }
+ else if (cold) r = first + i;
else { int t = first + i, j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; }
}
float sv = -3.0e38;
@@ -205,9 +222,11 @@ void mla() {
for (int i = tid; i < K; i += 256) qs[i] = b0[qb + i];
for (int i = tid; i < R; i += 256) qrs[i] = b1[rbq + i];
int pos = pos_base + s;
- int first = max(kv_start, (pos / B - anchor) * B);
+ bool cold = (flags & 32) != 0;
+ int share0 = (pos / B - anchor) * B;
+ int first = cold ? kv_start : max(kv_start, share0);
bool list = false;
- int lb = 0, n = max(0, pos - first + 1);
+ int lb = 0, n = cold ? max(0, min(pos + 1, share0) - kv_start) : max(0, pos - first + 1);
if (sel_row > 0) {
lb = sel_off + s * sel_row;
int c = sel[lb];
@@ -222,7 +241,8 @@ void mla() {
int r = -1;
if (tid < cnt) {
int i = t0 + tid;
- if (list) r = dev_row(sel[lb + 1 + i], pos, B, bt_off, nblk, anchor);
+ if (list) { int t = sel[lb + 1 + i]; r = cold ? (cold_pos(t, pos, B, bt_off, nblk, anchor) ? t : -1) : dev_row(t, pos, B, bt_off, nblk, anchor); }
+ else if (cold) r = first + i;
else { int t = first + i, j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; }
}
pr[tid] = r;
@@ -321,13 +341,15 @@ void rel() {
int qb = q_off + s * q_row + h * hd;
for (int d = tid; d < hd; d += 256) qs[d] = b0[qb + d];
for (int j = tid; j < d_rel; j += 256) qrs[j] = b4[r_off + s * r_row + h * d_rel + j];
- int pos = pos_base + s, first = max(0, (pos / B - anchor) * B), n = pos - first + 1;
+ bool cold = (flags & 32) != 0;
+ int pos = pos_base + s, share0 = (pos / B - anchor) * B;
+ int first = cold ? 0 : max(0, share0), n = cold ? max(0, min(pos + 1, share0)) : pos - first + 1;
float ts = intBitsToFloat(sel[tau_off + s]);
barrier();
float m = -3.0e38, l = 0.0, acc = 0.0;
for (int t0 = 0; t0 < n; t0 += 256) {
int i = t0 + tid, r = -1, t = first + i;
- if (i < n) { int j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; }
+ if (i < n) { if (cold) r = t; else { int j = slot_of(t, B, bt_off, nblk); r = j >= 0 ? j * B + t % B : -1; } }
float sv = -3.0e38;
if (r >= 0) {
int kb = k_off + (kvh * rows + r) * hd;
@@ -473,10 +495,43 @@ void dsa() {
if (tid == 0 && (flags & 8) == 0) { b3[st_off + (s * H + h) * 2] = best; b3[st_off + (s * H + h) * 2 + 1] = den; }
}
+// ---- mode 5 ----------------------------------------------------------------------
+// The chunks of a part joined (chain_attnb's split over positions): segment g of d
+// floats, chunk z's sum at a_off + z*a_z + g*d and its (m, l) at sa_off + z*sa_z + 2g
+// (buffer 0), joined in chunk order as the host joins its chunks (vkc_kv_join): the part
+// at o_off + g*d, (m, l) at so_off + 2g (buffer 3). A chunk with no position weighs nothing.
+void join() {
+ int n = p.v[1], d = p.v[2], nz = p.v[3], a_off = p.v[4], a_z = p.v[5], sa_off = p.v[6], sa_z = p.v[7];
+ int o_off = p.v[8], so_off = p.v[9];
+ int g = int(gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x), tid = int(gl_LocalInvocationID.x);
+ if (g >= n) return;
+ float m = -3.0e38, l = 0.0;
+ for (int z = 0; z < nz; z++) {
+ float mz = b0[sa_off + z * sa_z + 2 * g], lz = b0[sa_off + z * sa_z + 2 * g + 1];
+ if (mz <= -1.0e38) continue;
+ float mn = max(m, mz);
+ l = l * (m > -1.0e38 ? exp(m - mn) : 0.0) + lz * exp(mz - mn);
+ m = mn;
+ }
+ for (int i = tid; i < d; i += 256) {
+ float acc = 0.0, mm = -3.0e38;
+ for (int z = 0; z < nz; z++) {
+ float mz = b0[sa_off + z * sa_z + 2 * g];
+ if (mz <= -1.0e38) continue;
+ float mn = max(mm, mz);
+ acc = acc * (mm > -1.0e38 ? exp(mm - mn) : 0.0) + b0[a_off + z * a_z + g * d + i] * exp(mz - mn);
+ mm = mn;
+ }
+ b3[o_off + g * d + i] = acc;
+ }
+ if (tid == 0) { b3[so_off + 2 * g] = m; b3[so_off + 2 * g + 1] = l; }
+}
+
void main() {
if (MODE == 0) gqa();
else if (MODE == 1) mla();
else if (MODE == 2) merge();
else if (MODE == 3) rel();
+ else if (MODE == 5) join();
else dsa();
}
diff --git a/c/shaders/chain_kvs.spv b/c/shaders/chain_kvs.spv
index 96d2a0c2..92c1e69f 100644
Binary files a/c/shaders/chain_kvs.spv and b/c/shaders/chain_kvs.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index 6b6e4f0b..2c18409e 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -1870,6 +1870,9 @@ static void kvs_arrive(float *dst, const float *src, int nseg, size_t seg, int l
static void kvs_keep_rows(const float *o, int pb, int S) { if (kvs_keep) memcpy(kvs_keep + (size_t)pb * kvs_keep_row, o, (size_t)S * kvs_keep_row * 4); }
/* GQA: steps of `step` rows from position 0 to T, the device holding `rows` positions. */
+/* COLI_VK_KV_COLD=device (test_kvs_cold, or a family run with it): every split step's host part must run on
+ * the device, but where pins are on (they need the CPU's lists) */
+static int kvs_cold_wanted(void) { const char *c = getenv("COLI_VK_KV_COLD"); return c && !strcmp(c, "device"); }
static void test_kvs_gqa(int T, int step, int H, int KVH, int hd, int vd, int B, int rows, int win, int sink, int kv_pm,
int gated, int lists, int selects) {
char rv[32]; snprintf(rv, sizeof rv, "%d", rows); setenv("COLI_VK_KV_DEVICE_ROWS", rv, 1);
@@ -1965,8 +1968,9 @@ static void test_kvs_gqa(int T, int step, int H, int KVH, int hd, int vd, int B,
}
CHECK(ok && worst < 2e-5 && splits > 0, "kvs gqa T %d step %d H %d/%d hd %d vd %d B %d rows %d win %d sink %d pm %d gate %d lists %d: "
"err %.2e (%d split steps, %d whole)", T, step, H, KVH, hd, vd, B, rows, win, sink, kv_pm, gated, lists, worst, splits, fins);
- printf(" kvs gqa T %d step %d rows %d lists %d: rel err %.2e, %d split steps, %d whole, %llu host positions, %llu pinned\n",
- T, step, rows, lists, worst, splits, fins, ks.host_pos, ks.pins);
+ printf(" kvs gqa T %d step %d rows %d lists %d: rel err %.2e, %d split steps, %d whole, %llu host positions, %llu pinned%s\n",
+ T, step, rows, lists, worst, splits, fins, ks.host_pos, ks.pins, ks.cold_dev ? ", the host's part on the device" : "");
+ CHECK(!kvs_cold_wanted() || ks.np || (ks.cold_dev && ks.dev_cold > 0 && ks.host_pos == 0), "kvs gqa: the host's part did not run on the device");
CHECK(!ks.np || !selects || ks.pins > 0, "kvs gqa lists: no block was pinned");
vkc_free(kc); vkc_free(vc); vkc_free(qb); vkc_free(nk); vkc_free(nv); vkc_free(ob); vkc_free(sb); vkc_free(lb);
vkc_kv_free(&ks);
@@ -2054,9 +2058,10 @@ static void test_kvs_mla(int T, int step, int H, int K, int R, int B, int rows,
}
CHECK(ok && worst < 2e-5 && splits > 0, "kvs mla T %d step %d H %d K %d R %d B %d rows %d start %d lists %d: err %.2e (%d split steps)",
T, step, H, K, R, B, rows, kv_start, lists, worst, splits);
- printf(" kvs mla T %d step %d K %d rows %d lists %d: rel err %.2e, %d split steps, %llu host positions, %llu pinned\n",
- T, step, K, rows, lists, worst, splits, ks.host_pos, ks.pins);
- CHECK(!lists || ks.pins > 0, "kvs mla lists: no block was pinned");
+ printf(" kvs mla T %d step %d K %d rows %d lists %d: rel err %.2e, %d split steps, %llu host positions, %llu pinned%s\n",
+ T, step, K, rows, lists, worst, splits, ks.host_pos, ks.pins, ks.cold_dev ? ", the host's part on the device" : "");
+ CHECK(!lists || !ks.np || ks.pins > 0, "kvs mla lists: no block was pinned");
+ CHECK(!kvs_cold_wanted() || ks.np || (ks.cold_dev && ks.dev_cold > 0 && ks.host_pos == 0), "kvs mla: the host's part did not run on the device");
vkc_free(lat); vkc_free(rope); vkc_free(qab); vkc_free(nl); vkc_free(ob); vkc_free(lb);
vkc_kv_free(&ks);
free(Lh); free(Rh); free(Lx); free(Rx); free(posl); free(sel); free(ref); free(nh);
@@ -2137,7 +2142,9 @@ static void test_kvs_rel(int T, int step, int H, int KVH, int hd, int ext, int d
}
CHECK(ok && worst < 2e-5 && splits > 0, "kvs rel T %d step %d H %d/%d hd %d ext %d d_rel %d B %d rows %d: err %.2e (%d split steps)",
T, step, H, KVH, hd, ext, d_rel, B, rows, worst, splits);
- printf(" kvs rel T %d step %d rows %d ext %d: rel err %.2e, %d split steps, %llu host positions\n", T, step, rows, ext, worst, splits, ks.host_pos);
+ printf(" kvs rel T %d step %d rows %d ext %d: rel err %.2e, %d split steps, %llu host positions%s\n", T, step, rows, ext, worst,
+ splits, ks.host_pos, ks.cold_dev ? ", the host's part on the device" : "");
+ CHECK(!kvs_cold_wanted() || ks.np || (ks.cold_dev && ks.dev_cold > 0 && ks.host_pos == 0), "kvs rel: the host's part did not run on the device");
vkc_free(kc); vkc_free(vc); vkc_free(qb); vkc_free(rb); vkc_free(nk); vkc_free(nv); vkc_free(ob); vkc_free(tb); vkc_free(pb_);
vkc_kv_free(&ks);
free(Kh); free(Vh); free(Kx); free(Vx); free(relp); free(ref); free(nkh); free(nvh); free(tau);
@@ -2305,6 +2312,49 @@ static void test_kvs(void) {
kvs_start = 0; kvs_keep = NULL; free(a); free(b);
}
+/* COLI_VK_KV_COLD=device: the host's part on the device, from the shadow of the host's
+ * rows. Every case without pins against the same double-precision reference (sinks,
+ * gates, MiMo's position-major rows, windows, lists, MLA with and without a rope key and
+ * a nonzero start, Inkling's bias reaching into the host's rows), each split step's host
+ * part on the device and none on the CPU, and the same bits for every position however
+ * the forward is cut (steps of 1 to 4, slices of rows, crossing 64K). */
+static void test_kvs_cold(void) {
+ if (!coli_vk_import_alignment()) { printf(" kvs cold: no host memory the device reads in place here, skipped\n"); return; }
+ setenv("COLI_VK_KV_COLD", "device", 1);
+ /* T step H KVH hd vd B rows win sink pm gate lists selects */
+ test_kvs_gqa(40, 1, 4, 2, 32, 32, 4, 12, 0, 0, 0, 1, 0, 0);
+ test_kvs_gqa(41, 3, 4, 2, 32, 32, 4, 8, 0, 0, 0, 1, 0, 0);
+ test_kvs_gqa(70, 7, 4, 2, 48, 32, 8, 32, 0, 1, 1, 0, 0, 0);
+ test_kvs_gqa(300, 1, 8, 2, 64, 64, 16, 64, 0, 0, 0, 1, 0, 0);
+ test_kvs_gqa(600, 50, 4, 4, 16, 16, 16, 128, 0, 0, 0, 0, 0, 0);
+ test_kvs_gqa(60, 3, 4, 2, 32, 32, 4, 24, 0, 0, 0, 1, 1, 1); /* lists, no pins */
+ test_kvs_gqa(50, 2, 4, 2, 256, 256, 4, 16, 0, 0, 0, 1, 1, 1);
+ test_kvs_gqa(60, 2, 4, 2, 32, 24, 4, 16, 20, 1, 1, 0, 0, 0);
+ test_kvs_gqa(48, 12, 4, 2, 32, 32, 4, 40, 0, 0, 0, 1, 0, 0);
+ /* T step H K R B rows kv_start lists */
+ test_kvs_mla(40, 1, 4, 32, 8, 4, 12, 0, 0);
+ test_kvs_mla(90, 5, 2, 64, 16, 8, 40, 0, 1);
+ test_kvs_mla(60, 4, 3, 48, 0, 4, 16, 0, 0);
+ test_kvs_mla(70, 2, 2, 512, 64, 8, 32, 5, 0);
+ test_kvs_mla(40, 3, 1, 1024, 8, 4, 16, 0, 1);
+ /* T step H KVH hd ext d_rel B rows */
+ test_kvs_rel(60, 1, 4, 2, 32, 8, 4, 4, 16);
+ test_kvs_rel(80, 5, 4, 2, 16, 40, 16, 8, 32);
+ test_kvs_rel(50, 3, 2, 1, 128, 16, 64, 4, 24);
+ test_kvs_steps();
+ int T = 65543, first = 65531, H = 2, hd = 16;
+ float *a = calloc((size_t)T * H * hd, sizeof(float));
+ float *b = calloc((size_t)T * H * hd, sizeof(float));
+ kvs_start = first; kvs_keep_row = (size_t)H * hd;
+ kvs_keep = a; test_kvs_gqa(T, 1, H, 1, hd, hd, 64, 512, 0, 0, 0, 0, 0, 0);
+ kvs_keep = b; test_kvs_gqa(T, 7, H, 1, hd, hd, 64, 512, 0, 0, 0, 0, 0, 0);
+ CHECK(!memcmp(a + (size_t)first * H * hd, b + (size_t)first * H * hd,
+ (size_t)(T - first) * H * hd * sizeof(float)), "kvs cold: rows crossing 64K differ with chunk size");
+ kvs_start = 0; kvs_keep = NULL; free(a); free(b);
+ unsetenv("COLI_VK_KV_COLD");
+ printf(" kvs cold: every case with the host's part on the device, the same bits however the forward is cut\n");
+}
+
/* COLI_VK_CHAIN_BENCH=1: decode-shaped GEMVs back to back in one frame (the chain's
* situation: no host gap between matrices), per weight format, in GB/s of weights. */
#include <time.h>
@@ -2494,7 +2544,7 @@ int main(int argc, char **argv) {
printf("dsv4 rounding done\n");
}
if (!vkc_kvs_ready()) { fails++; printf("FAIL: the split KV shader did not load\n"); }
- else { test_kvs(); printf("kvs done\n"); }
+ else { test_kvs(); test_kvs_cold(); printf("kvs done\n"); }
VkcStats st; vkc_stats(&st);
printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);
vkc_shutdown();
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index 1546106f..d9b65504 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -3279,6 +3279,24 @@ kv_hostparts() {
n=$(sed -n "s/^\[VK\] $1 chain: KV split: [0-9]* layer steps, \([0-9]*\) with a host part.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
+# kv_devcold <engine> <log>: N from "| the host's part on the device: N layer steps"
+# (COLI_VK_KV_COLD=device), 0 without it
+kv_devcold() {
+ local n
+ n=$(sed -n "s/^\[VK\] $1 chain: KV split: .* the host's part on the device: \([0-9]*\) layer steps.*/\1/p" "$2" | tail -1)
+ echo "${n:-0}"
+}
+# kv_cold_check <engine> <tag> <log>: with COLI_VK_KV_COLD=device exported (the
+# kv-split-cold families), the host's part must have run on the device; with read-based
+# pins (COLI_VK_KV_PIN=1) it stays on the CPU by design, and the engine says so
+kv_cold_check() {
+ [ "${COLI_VK_KV_COLD:-}" = device ] || return 0
+ if [ "${COLI_VK_KV_PIN:-0}" = 1 ]; then
+ grep -qa "COLI_VK_KV_COLD=device needs .* no pins; the CPU computes the host's part" "$3" || { cat "$3"; fail "$2: pins on, and no line that the CPU computes the host's part"; }
+ return 0
+ fi
+ [ "$(kv_devcold "$1" "$3")" -gt 0 ] || { cat "$3"; fail "$2: the host's part never ran on the device"; }
+}
# kv_gate <engine> <tag> <tol 0|1> <rows> <block> <env...> -- <argv...>: chain_gate (the
# CPU's tokens, logits within 1e-4 of the largest with tol 1) with the split on, which
# must have run steps with a host part.
@@ -3286,6 +3304,7 @@ kv_gate() {
local eng=$1 tag=$2 tol=$3 rows=$4 blk=$5; shift 5
chain_gate "$eng" "$tag" "$tol" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"
[ "$(kv_hostparts "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }
+ kv_cold_check "$eng" "$tag" vk.log
echo " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"
}
# kv_san <engine> <tag> <env and argv...>: a sanitized run under the split (the build is
@@ -3296,6 +3315,7 @@ kv_san() {
env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(kv_hostparts "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the split never ran a host part"; }
+ kv_cold_check "$eng" "$tag" san.log
echo "OK $tag: sanitizers clean, $(kv_hostparts "$eng" san.log) layer steps with a host part"
}
@@ -3627,6 +3647,7 @@ kv_mla_gate() {
mla_gate "$eng" "$tag" "$tol" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"
[ "$(kv_hostparts "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }
if [ "${PINNED:-0}" = 1 ]; then grep -qa 'KV split: .* [1-9][0-9]* blocks pinned by reads' vk.log || { cat vk.log; fail "$tag: no block was pinned"; }; fi
+ kv_cold_check "$eng" "$tag" vk.log
echo " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"
}
# colibri (GLM-5.2): the 32-token oracle over 8 or 16 rows on the device; DSA's lists
@@ -3954,6 +3975,20 @@ family_kv_split_sanitize() {
kv_split_mimo_sanitize; kv_split_colibri_sanitize; kv_split_glm53_sanitize; kv_split_kimi_sanitize
make clean >/dev/null 2>&1 || true
}
+# COLI_VK_KV_COLD=device: every gate of the two families above again, with the host's
+# part of each split layer's attention on the device too (vkc_kv_shadow). The tokens and
+# logits are held to the CPU's as before, and each gate must have run the host's part on
+# the device (kv_cold_check). DeepSeek's sparse forms keep it on the CPU.
+family_kv_split_cold() {
+ export COLI_VK_KV_COLD=device
+ family_kv_split
+ unset COLI_VK_KV_COLD
+}
+family_kv_split_cold_sanitize() {
+ export COLI_VK_KV_COLD=device
+ family_kv_split_sanitize
+ unset COLI_VK_KV_COLD
+}
family_kv_split_deepseek() {
export OMP_NUM_THREADS=2
make deepseek_v41 deepseek-v4 VK=1
@@ -4083,6 +4118,8 @@ case "${1:-}" in
prefill-deepseek) family_prefill_deepseek ;;
kv-split) family_kv_split ;;
kv-split-sanitize) family_kv_split_sanitize ;;
+ kv-split-cold) family_kv_split_cold ;;
+ kv-split-cold-sanitize) family_kv_split_cold_sanitize ;;
kv-split-deepseek) family_kv_split_deepseek ;;
kv-split-deepseek-sanitize) family_kv_split_deepseek_sanitize ;;
staged) family_staged ;;
@@ -4103,5 +4140,5 @@ case "${1:-}" in
partial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}
declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }
"$fn" ;;
- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;
+ *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-cold|kv-split-cold-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;
esac
diff --git a/c/vk_chain.c b/c/vk_chain.c
index c45c0f00..7674e0a6 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -67,6 +67,7 @@ typedef struct { VkDeviceMemory mem; uint8_t *map; } VkcBlock;
typedef struct { VkaPool p; uint32_t memtype; int mapped; } VkcPool;
struct VkcBuf {
VkBuffer buf;
+ VkDeviceMemory mem; /* VKC_HOST: the imported pages */
int kind;
size_t bytes;
VkaRange r;
@@ -195,6 +196,7 @@ VkcBuf *vkc_buf(size_t bytes, int kind) {
}
static void buf_release(VkcBuf *b) {
+ if (b->kind == VKC_HOST) { coli_vk_host_buffer_free((void *)b->buf, (void *)b->mem, b->bytes); free(b); return; }
VkcPool *P = &K.pool[b->kind];
vkDestroyBuffer(K.dev, b->buf, NULL);
if (vka_free(&P->p, b->r)) {
@@ -231,6 +233,19 @@ int vkc_reserve(VkcBuf **b, size_t bytes, int kind) {
}
void *vkc_ptr(const VkcBuf *b) { return b ? b->ptr : NULL; }
size_t vkc_bytes(const VkcBuf *b) { return b ? b->bytes : 0; }
+VkcBuf *vkc_host(const void *ptr, size_t bytes, size_t *off) {
+ if (!vkc_ready() || !ptr || !bytes) return NULL;
+ void *buf = NULL, *mem = NULL;
+ size_t o = 0;
+ if (!coli_vk_host_buffer(ptr, bytes, &buf, &mem, &o)) return NULL;
+ size_t al = coli_vk_import_alignment(), sz = (o + bytes + al - 1) / al * al; /* the pages, as imported */
+ VkcBuf *b = calloc(1, sizeof *b);
+ if (!b) { coli_vk_host_buffer_free(buf, mem, sz); return NULL; }
+ b->kind = VKC_HOST; b->buf = (VkBuffer)buf; b->mem = (VkDeviceMemory)mem;
+ b->bytes = sz;
+ *off = o;
+ return b;
+}
/* ---- init ------------------------------------------------------------------------ */
static VkShaderModule load_module(const char *dir_spv, const char *file) {
@@ -823,13 +838,44 @@ int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBu
if (w.a.sel_row > 0) x.a.sel_off += r0 * w.a.sel_row;
if (r0 && !attn_slice_next()) return 0;
if (blocked) {
- struct { VkcAttnW w; int br; } pc = {x, br};
+ struct { VkcAttnW w; int br, B, anchor, part, st_off; } pc = {x, br, 0, 0, 0, 0};
ok = record(KAB.pipe, bd, 7, &pc, sizeof pc, (uint32_t)x.a.KVH, (uint32_t)((x.a.S + br - 1) / br), 1);
K.st.attn_blocked += ok;
} else ok = record(K.pipe[P_ATTN], bd, 7, &x, sizeof x, (uint32_t)x.a.H, (uint32_t)x.a.S, 1);
}
return ok;
}
+static int attn_part_rec(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off,
+ int ch, int nz, int o_z, int st_z) {
+ K.kind = PK_ATTN;
+ VkcAttnW w = *p;
+ if (w.vd <= 0) w.vd = w.a.hd;
+ int G = w.a.KVH > 0 ? w.a.H / w.a.KVH : 0;
+ if (w.a.hd > 256 || w.vd > 256 || G < 1 || G > 32 || w.a.H % w.a.KVH || w.a.sel_row > 0 || w.sink || w.ring || blk < 1 ||
+ anchor < 0 || ch < 0 || (ch > 0 && (ch % 16 || nz < 1 || nz > 65535)) || !kab_pipe()) return 0;
+ w.a.has_gate = 0; w.a.g_off = ch; w.a.g_row = o_z; w.a.g_seg = st_z; /* the chunks (unused by part otherwise) */
+ VkcBind bd[7] = {B(q, 0), B(kc, 0), B(vc, 0), B(o, 1), B(NULL, 0), B(NULL, 0), B(NULL, 0)};
+ int br = 32 / G, S = w.a.S, rr = attn_slice_rows(S, w.a.pos_base, (double)w.a.H * (w.a.hd > w.vd ? w.a.hd : w.vd), br);
+ int ok = 1;
+ for (int r0 = 0; ok && r0 < S; r0 += rr) {
+ VkcAttnW x = w;
+ int n = S - r0 < rr ? S - r0 : rr;
+ x.a.S = n; x.a.pos_base = w.a.pos_base + r0;
+ x.a.q_off += r0 * w.a.q_row; x.a.o_off += r0 * w.a.o_row;
+ if (r0 && !attn_slice_next()) return 0;
+ struct { VkcAttnW w; int br, B, anchor, part, st_off; } pc = {x, br, blk, anchor, 1, st_off + r0 * w.a.H * 2};
+ ok = record(KAB.pipe, bd, 7, &pc, sizeof pc, (uint32_t)x.a.KVH, (uint32_t)((x.a.S + br - 1) / br), (uint32_t)(ch > 0 ? nz : 1));
+ K.st.attn_blocked += ok;
+ }
+ return ok;
+}
+int vkc_attn_part(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off) {
+ return attn_part_rec(q, kc, vc, o, p, blk, anchor, st_off, 0, 1, 0, 0);
+}
+int vkc_attn_part_chunks(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off,
+ int ch, int nz, int o_z, int st_z) {
+ return ch > 0 && attn_part_rec(q, kc, vc, o, p, blk, anchor, st_off, ch, nz, o_z, st_z);
+}
int vkc_dnconv(VkcBuf *in, VkcBuf *w, VkcBuf *ring, VkcBuf *out, VkcBuf *snap, const VkcDnConv *p) {
K.kind = PK_DNCONV;
if (p->CK < 1 || p->CK > 9) return 0;
@@ -1660,6 +1706,16 @@ int vkc_kvs_merge(VkcBuf *dev, VkcBuf *cpu, VkcBuf *gate, VkcBuf *out, const Vkc
uint32_t gx, gy; grid((uint64_t)p->n, &gx, &gy);
return record(pipe, bd, 4, &pc, sizeof pc, gx, gy, 1);
}
+int vkc_kvs_join(VkcBuf *parts, VkcBuf *out, const VkcKvsJoin *p) {
+ VkPipeline pipe = kvs_pipe();
+ if (!pipe || !parts || !out || p->n < 0 || p->d < 1 || p->nz < 0) return 0;
+ if (p->n == 0) return open_frame() && !K.lost;
+ struct { int mode; VkcKvsJoin b; } pc = {5, *p};
+ K.kind = PK_ATTN;
+ VkcBind bd[4] = {B(parts, 0), B(NULL, 0), B(NULL, 0), B(out, 1)};
+ uint32_t gx, gy; grid((uint64_t)p->n, &gx, &gy);
+ return record(pipe, bd, 4, &pc, sizeof pc, gx, gy, 1);
+}
int vkc_kvs_rel(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *out, VkcBuf *r, VkcBuf *tau, VkcBuf *relp, VkcBuf *tab,
const VkcKvsRel *p) {
VkPipeline pipe = kvs_pipe();
diff --git a/c/vk_chain.h b/c/vk_chain.h
index f67041ae..1b154de1 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -53,6 +53,7 @@ typedef struct VkcBuf VkcBuf;
#define VKC_DEV 0
#define VKC_UP 1
#define VKC_DOWN 2
+#define VKC_HOST 3 /* host memory read in place (vkc_host): bound read-only, never written */
int vkc_init(void); /* after coli_vk_init; 1 = the chain's pipelines are up */
int vkc_ready(void);
@@ -64,6 +65,10 @@ void vkc_free(VkcBuf *b); /* waits for the frames
int vkc_reserve(VkcBuf **b, size_t bytes, int kind); /* at least `bytes`; growing drops the contents */
void *vkc_ptr(const VkcBuf *b); /* host mapping (VKC_UP, VKC_DOWN; VKC_DEV when host-visible) */
size_t vkc_bytes(const VkcBuf *b);
+/* The pages around a host range as a buffer the device reads in place (coli_vk_host_buffer:
+ * a device that shares the RAM, without staged uploads); *off: ptr's byte offset in it.
+ * NULL when the device cannot. vkc_free releases it after the frames that read it. */
+VkcBuf *vkc_host(const void *ptr, size_t bytes, size_t *off);
/* recording */
int vkc_begin(void);
@@ -112,6 +117,23 @@ int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcB
* chain_attnb.comp: a workgroup per KV head and block of rows, each K and V row read once
* per block instead of once per (head, row); below it chain_attn as before. */
int vkc_attn_block_rows(void);
+/* chain_attnb's part mode (vk_kvsplit.h, the host's part on the device): each row's causal
+ * positions before its share of a split cache, [first, (pos/B - anchor)*B), from a cache
+ * in the host's head-major layout (p->a.cap positions a KV head), the value sum
+ * unnormalized at o_off + s*o_row + h*vd and (m, l) at st_off + (s*H + h)*2. Blocked for
+ * any S, its tiles at multiples of their size: a row's bits do not depend on the step
+ * around it. No lists, sink or ring; 0 without the blocked pipeline. */
+int vkc_attn_part(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off);
+/* The same over chunks of `ch` positions (a multiple of 16), nz of them, chunk z's part at
+ * o_off + z*o_z + s*o_row + h*vd and (m, l) at st_off + z*st_z + (s*H + h)*2 (vkc_kvs_join
+ * joins them): a decode row's positions over many workgroups. */
+int vkc_attn_part_chunks(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, const VkcAttnW *p, int blk, int anchor, int st_off,
+ int ch, int nz, int o_z, int st_z);
+/* chain_kvs.comp mode 5: the chunks of a part joined in order (vkc_attn_part_chunks): segment
+ * g of d floats, chunk z's sum at a_off + z*a_z + g*d and (m, l) at sa_off + z*sa_z + 2g of
+ * parts, into out at o_off + g*d and so_off + 2g. */
+typedef struct { int n, d, nz, a_off, a_z, sa_off, sa_z, o_off, so_off; } VkcKvsJoin;
+int vkc_kvs_join(VkcBuf *parts, VkcBuf *out, const VkcKvsJoin *p);
/* The attention ops (vkc_attn, vkc_attn_w, vkc_mla_core, vkc_relattn) whose rows x positions
* x heads x head dim pass COLI_VK_ATTN_SLICE (2^32; 0 = never) record their rows in slices,
* each ending its frame (submitted, not waited for) and the next in a new one: no single
@@ -541,6 +563,7 @@ void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb
int vkc_kvs_ready(void);
#define VKC_KVS_FIN 1
#define VKC_KVS_GATE 2
+#define VKC_KVS_COLD 32 /* vkc_kvs_attn/_mla/_rel: the host's positions, from a buffer in the host's layout (row t = position t) */
typedef struct { int S, H, KVH, hd, vd, pos_base, rows, q_off, q_row, q_seg, sel_off, sel_row; float scale;
int k_off, v_off, win, kv_pm, sink, sink_off, B, ns, bt_off, nblk, anchor, o_off, o_row, st_off, flags,
g_off, g_row, g_seg; } VkcKvsAttn;
diff --git a/c/vk_kvsplit.h b/c/vk_kvsplit.h
index 1c7b6687..d6c3777c 100644
--- a/c/vk_kvsplit.h
+++ b/c/vk_kvsplit.h
@@ -33,6 +33,12 @@
* COLI_VK_KV_BLOCK=B positions per block (default 64)
* COLI_VK_KV_PIN=1 opt in to read-based pins (GQA/MLA rounding can then
* depend on read history; the default is deterministic)
+ * COLI_VK_KV_COLD=device the host's part on the device too (vkc_kv_shadow): each
+ * split layer keeps a copy of the host's rows in memory the
+ * device reads in place, kept up to date row by row, and the
+ * device computes the same positions the CPU would; on a
+ * device that can import host memory (not with staged
+ * uploads), without pins. Unset: the CPU, as before
*
* The residency (VkcKvTab, per layer): bt (block -> slot or -1) and rb (slot -> block
* or -1), uploaded to `tab` when they change; `valid`, the watermark: every resident
@@ -72,6 +78,14 @@ typedef struct {
VkcBuf *dev; size_t dev_off;
} VkcKvPart;
+/* COLI_VK_KV_COLD=device: a split layer's host rows in memory the device reads in place,
+ * per array (K and V, or the latent and the rope key): nseg segments of cap positions of
+ * seglen floats, head-major (segment g, position t at (g*cap + t)*seglen), page-aligned and
+ * imported once (b, off floats into it); rows below `valid` equal the host's. */
+typedef struct {
+ float *p[2]; VkcBuf *b[2]; size_t off[2], n[2];
+ int nseg[2], seglen[2], cap, valid;
+} VkcKvShadow;
typedef struct {
int on; /* split: every split layer holds `rows` rows on the device */
int B, ns, nw, np, rows, nl; /* block size, slots (window, pinned), rows = ns*B, layers */
@@ -86,12 +100,26 @@ typedef struct {
float *hpart; size_t hpart_n;
int *cold, *lo, *cnt, *causal; size_t cold_n, lo_n;
unsigned long long steps, splits, host_pos, pins, staged_rows; double host_ms, wait_ms;
+ int cold_dev; /* COLI_VK_KV_COLD=device: the host's part on the device, from sh */
+ VkcKvShadow *sh; /* per split layer (cold_dev) */
+ VkcBuf *cscr; /* its chunks' parts (vkc_attn_part_chunks), joined into cpart */
+ unsigned long long dev_cold, shadow_rows; size_t shadow_bytes; double shadow_ms;
const char *engine;
} VkcKvSplit;
static double vkc_kv_now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }
+static void vkc_kv_shadow_free(VkcKvSplit *ks) {
+ if (ks->sh) for (int i = 0; i < ks->nl; i++)
+ for (int k = 0; k < 2; k++) {
+ vkc_free(ks->sh[i].b[k]); /* after the frames that read it, before its pages go */
+ free(ks->sh[i].p[k]);
+ }
+ free(ks->sh); ks->sh = NULL; ks->shadow_bytes = 0;
+ vkc_free(ks->cscr); ks->cscr = NULL;
+}
static void vkc_kv_free(VkcKvSplit *ks) {
+ vkc_kv_shadow_free(ks);
if (ks->t) for (int i = 0; i < ks->nl; i++) {
VkcKvTab *t = &ks->t[i];
free(t->bt); free(t->rb); free(t->pin); free(t->fresh); free(t->hits);
@@ -114,6 +142,7 @@ static void vkc_kv_reset(VkcKvSplit *ks) {
for (int j = 0; j < ks->ns; j++) { t->rb[j] = -1; t->pin[j] = 0; t->fresh[j] = 0; }
memset(t->hits, 0, (size_t)t->nblk * sizeof *t->hits);
t->valid = 0; t->dirty = 1; t->wb0 = t->wb1 = 0; t->counted = 0;
+ if (ks->sh) ks->sh[i].valid = 0;
}
}
@@ -194,13 +223,19 @@ static int vkc_kv_plan_need(VkcKvSplit *ks, const char *engine, int nl, size_t r
}
ks->tab = vkc_buf((size_t)nl * ks->tab_stride * sizeof(int), VKC_DEV);
if (!ks->tab) { vkc_kv_free(ks); return 0; }
+ { const char *cm = getenv("COLI_VK_KV_COLD");
+ ks->cold_dev = cm && !strcmp(cm, "device") && np == 0 && coli_vk_import_alignment() > 0;
+ if (cm && !strcmp(cm, "device") && !ks->cold_dev)
+ fprintf(stderr, "[VK] %s chain: COLI_VK_KV_COLD=device needs host memory the device reads in place and no pins; "
+ "the CPU computes the host's part\n", engine);
+ if (ks->cold_dev && !(ks->sh = (VkcKvShadow *)calloc((size_t)nl, sizeof *ks->sh))) { vkc_kv_free(ks); return 0; } }
ks->on = 1;
vkc_kv_reset(ks);
fprintf(stderr, "[VK] %s chain: the KV cache split past the device's budget: %d of %d positions a layer on the device "
"(%d blocks of %d, %d for pins by reads; a row's newest %d positions on the device), the rest in RAM "
- "(%.1f MiB on the device instead of %.1f)\n",
+ "(%.1f MiB on the device instead of %.1f)%s\n",
engine, ks->rows, cap, ns, B, np, (ks->anchor + 1) * B, (double)nl * ks->rows * row_bytes / 1048576.0,
- need / 1048576.0);
+ need / 1048576.0, ks->cold_dev ? ", its attention on the device too (COLI_VK_KV_COLD=device)" : "");
return 2;
}
static int vkc_kv_plan(VkcKvSplit *ks, const char *engine, int nl, size_t row_bytes, int cap, int chunk, int selects,
@@ -229,6 +264,9 @@ static void vkc_kv_assign(VkcKvTab *t, int j, int b, int pinned) {
* blocks and the ones its rows' shares reach back to get slots. */
static void vkc_kv_place(VkcKvSplit *ks, int li, int pos_base, int S) {
VkcKvTab *t = &ks->t[li];
+ /* the shadow too: rows from pos_base on may be rewritten from here (this step's, or a
+ * new request's after a rewind, whether or not this step has a host part) */
+ if (ks->sh && ks->sh[li].valid > pos_base) ks->sh[li].valid = pos_base > 0 ? pos_base : 0;
if (pos_base + S <= 0) { t->wb0 = t->wb1 = 0; return; }
int B = ks->B, wb1 = (pos_base + S - 1) / B, wb0 = wb1 - ks->nw + 1;
if (wb0 < 0) wb0 = 0;
@@ -348,7 +386,10 @@ static int vkc_kv_store(VkcKvSplit *ks, int li, const VkcKvPart *pt, VkcBuf *src
return ok;
}
static void vkc_kv_done(VkcKvSplit *ks, int li, int end) { if (ks->on) ks->t[li].valid = end; }
-static void vkc_kv_lower(VkcKvSplit *ks, int li, int pos_base) { if (ks->on && ks->t[li].valid > pos_base) ks->t[li].valid = pos_base; }
+static void vkc_kv_lower(VkcKvSplit *ks, int li, int pos_base) {
+ if (ks->on && ks->t[li].valid > pos_base) ks->t[li].valid = pos_base;
+ if (ks->on && ks->sh && ks->sh[li].valid > pos_base) ks->sh[li].valid = pos_base;
+}
/* 1 when a row of the step may read a position outside the device's share: causal
* rows see [first, pos] (first = pos - win + 1 with a window, else kv_start), the share
@@ -671,6 +712,55 @@ static int vkc_kv_run(VkcKvSplit *ks, VkcBuf *q, size_t q_off, size_t n_q, VkcBu
return vkc_kv_run2(ks, q, q_off, n_q, NULL, 0, 0, sel, sel_off, n_sel, dev, host, u, part_n);
}
+/* ---- the host's part on the device (COLI_VK_KV_COLD=device) ------------------------
+ * Layer li's shadow up to row `upto` (exclusive: a step's cold positions are all below its
+ * first row, which the host holds): nparts arrays, part k's segment g of position t at
+ * host[k] + g*grp[k] + t*pstride[k], seglen[k] floats. Made at the first use, sized to the
+ * tables' positions; rows from `valid` on are copied (a step lowers `valid` to its first
+ * row: the rows from there may be rewritten, by it or by a rollback). NULL when the memory
+ * or the import fails (the CPU's part then, for good on this layer). */
+static VkcKvShadow *vkc_kv_shadow(VkcKvSplit *ks, int li, int nparts, const float *const *host, const size_t *grp,
+ const size_t *pstride, const int *nseg, const int *seglen, int upto) {
+ VkcKvShadow *sh = &ks->sh[li];
+ if (sh->cap < 0) return NULL;
+ if (!sh->cap) {
+ size_t al = coli_vk_import_alignment();
+ for (int k = 0; k < nparts; k++) {
+ size_t n = (size_t)nseg[k] * ks->cap * seglen[k], bytes = (n * sizeof(float) + al - 1) / al * al;
+ void *p = NULL;
+ if (!al || posix_memalign(&p, al, bytes)) p = NULL;
+ sh->p[k] = (float *)p;
+ if (p) { memset(p, 0, bytes); sh->b[k] = vkc_host(p, bytes, &sh->off[k]); }
+ if (!p || !sh->b[k]) {
+ fprintf(stderr, "[VK] %s chain: no shadow for layer %d's host rows (%.1f MiB), the CPU computes its part\n",
+ ks->engine, li, bytes / 1048576.0);
+ for (int j = 0; j <= k; j++) { vkc_free(sh->b[j]); free(sh->p[j]); sh->b[j] = NULL; sh->p[j] = NULL; }
+ sh->cap = -1;
+ return NULL;
+ }
+ sh->off[k] /= sizeof(float); sh->n[k] = n; sh->nseg[k] = nseg[k]; sh->seglen[k] = seglen[k];
+ ks->shadow_bytes += bytes;
+ }
+ sh->cap = ks->cap; sh->valid = 0;
+ }
+ if (upto > sh->cap) upto = sh->cap;
+ if (sh->valid > upto) sh->valid = upto;
+ if (sh->valid < upto) {
+ double t0 = vkc_kv_now();
+ long rows = upto - sh->valid;
+ for (int k = 0; k < nparts; k++) {
+ int G = nseg[k], L = seglen[k];
+ #pragma omp parallel for collapse(2) schedule(static) if (rows * G * L > (1 << 20))
+ for (int g = 0; g < G; g++)
+ for (long t = sh->valid; t < upto; t++)
+ memcpy(sh->p[k] + ((size_t)g * sh->cap + t) * L, host[k] + g * grp[k] + t * pstride[k], (size_t)L * sizeof(float));
+ }
+ ks->shadow_rows += (unsigned long long)rows; ks->shadow_ms += vkc_kv_now() - t0;
+ sh->valid = upto;
+ }
+ return sh;
+}
+
/* ---- one layer's attention over the split cache, the forms the engines use ----------- */
/* The device's attention of S rows in slices of rows, each slice in a submission of its
* own (the rows are independent: the same bits), so that no one submission runs long
@@ -715,6 +805,48 @@ static int vkc_kv_gqa_rec(VkcKvGqa *a, int s0, int ns, int fin) {
return vkc_kvs_attn(a->q, a->kc, a->vc, fin ? a->out : ks->dpart, fin ? a->gate : NULL, a->sel, a->snk, ks->tab, &p);
}
static int vkc_kv_gqa_part(void *u, int s0, int ns) { return vkc_kv_gqa_rec((VkcKvGqa *)u, s0, ns, 0); }
+/* rows [s0, s0 + ns) of the host's part on the device, from the shadow, into ks->cpart */
+static int vkc_kv_gqa_cold(void *u, int s0, int ns) {
+ VkcKvGqa *a = (VkcKvGqa *)u; VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];
+ int bt = (int)((size_t)a->li * ks->tab_stride), nb = ks->t[a->li].nblk;
+ VkcKvsAttn p = {ns, a->H, a->KVH, a->hd, a->vd, a->pos_base + s0, sh->cap, s0 * a->q_row, a->q_row, a->q_seg,
+ a->sel_off + s0 * a->sel_row, a->sel_row, a->scale, (int)sh->off[0], (int)sh->off[1], a->win, 0, 0, 0,
+ ks->B, ks->ns, bt, nb, ks->anchor, s0 * a->H * a->vd, a->H * a->vd, a->S * a->H * a->vd + s0 * a->H * 2,
+ VKC_KVS_COLD, 0, 0, 0};
+ return vkc_kvs_attn(a->q, sh->b[0], sh->b[1], ks->cpart, NULL, a->sel, NULL, ks->tab, &p);
+}
+/* The same without lists through the blocked attention (vkc_attn_part_chunks): the
+ * shadow's K and V rows read once per block of rows, not once per (head, row), and the
+ * positions in chunks of VKC_KV_CCH, one workgroup each, joined in order (vkc_kvs_join):
+ * a decode row's thousands of positions over many workgroups. The chunks are fixed by
+ * position, so a row's bits do not depend on the step around it. Rows go in slices whose
+ * chunk parts fit 64 MiB. */
+#define VKC_KV_CCH 512
+static int vkc_kv_gqa_cold_blocked(VkcKvGqa *a) {
+ VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];
+ int H = a->H, vd = a->vd, last = a->pos_base + a->S - 1;
+ int end = (last / ks->B - ks->anchor) * ks->B; /* the last row's cold positions end there */
+ int nz = end > 0 ? (end + VKC_KV_CCH - 1) / VKC_KV_CCH : 0;
+ size_t per_row = (size_t)(nz > 0 ? nz : 1) * H * (vd + 2) * sizeof(float);
+ int rows = (int)(((size_t)64 << 20) / per_row);
+ if (rows < 1) rows = 1;
+ if (rows > a->S) rows = a->S;
+ if (!vkc_reserve(&ks->cscr, (size_t)rows * per_row, VKC_DEV)) return 0;
+ for (int r0 = 0; r0 < a->S; r0 += rows) {
+ int n = a->S - r0 < rows ? a->S - r0 : rows, o_z = n * H * vd, st_z = n * H * 2, sa_off = nz * o_z;
+ if (nz > 0) {
+ VkcAttnW w; memset(&w, 0, sizeof w);
+ w.a = (VkcAttn){n, H, a->KVH, a->hd, a->pos_base + r0, sh->cap, r0 * a->q_row, a->q_row, a->q_seg, 0, 0, 0, 0,
+ 0, H * vd, 0, 0, a->scale, (int)sh->off[0], (int)sh->off[1]};
+ w.win = a->win; w.vd = vd;
+ if (!vkc_attn_part_chunks(a->q, sh->b[0], sh->b[1], ks->cscr, &w, ks->B, ks->anchor, sa_off, VKC_KV_CCH, nz,
+ o_z, st_z)) return 0;
+ }
+ VkcKvsJoin j = {n * H, vd, nz, 0, o_z, sa_off, st_z, r0 * H * vd, a->S * H * vd + r0 * H * 2};
+ if (!vkc_kvs_join(ks->cscr, ks->cpart, &j)) return 0;
+ }
+ return 1;
+}
static int vkc_kv_gqa_fin(void *u, int s0, int ns) { return vkc_kv_gqa_rec((VkcKvGqa *)u, s0, ns, 1); }
static long vkc_kv_span(const VkcKvSplit *ks, int win, int sel_row) { /* the most positions a row's device part reads */
long n = (long)(ks->anchor + 1) * ks->B;
@@ -747,6 +879,20 @@ static int vkc_kv_gqa(VkcKvGqa *a) {
if (!vkc_kv_cold(ks, a->li, a->pos_base, a->S, a->win, 0, a->sel_row > 0))
return vkc_kv_sliced(a->S, a->H, vkc_kv_span(ks, a->win, a->sel_row), a->hd + a->vd, vkc_kv_gqa_fin, a);
size_t part = (size_t)a->S * a->H * (a->vd + 2);
+ if (ks->cold_dev) { /* the host's part on the device too: one frame, no round trip */
+ const float *host[2] = {a->Kh, a->Vh};
+ size_t grp[2] = {a->k_grp, a->v_grp}, ps[2] = {a->k_pos, a->v_pos};
+ int nseg[2] = {a->KVH, a->KVH}, sl[2] = {a->hd, a->vd};
+ if (vkc_kv_shadow(ks, a->li, 2, host, grp, ps, nseg, sl, a->pos_base)) {
+ ks->dev_cold++; ks->splits++; /* a step with a host part, on the device */
+ VkcKvsMerge mg = {a->S * a->H, a->H, a->vd, 0, a->H * a->vd, a->vd, a->S * a->H * a->vd, 0, a->S * a->H * a->vd,
+ 0, a->o_row, a->vd, a->gated ? VKC_KVS_GATE : 0, a->g_off, a->g_row, a->g_seg};
+ return vkc_kv_parts(ks, part) && vkc_kv_gqa_dev(a) &&
+ (a->sel_row > 0 ? vkc_kv_sliced(a->S, a->H, (long)a->pos_base + a->S, a->hd + a->vd, vkc_kv_gqa_cold, a)
+ : vkc_kv_gqa_cold_blocked(a)) &&
+ vkc_kvs_merge(ks->dpart, ks->cpart, a->gate, a->out, &mg);
+ }
+ }
if (!vkc_kv_parts(ks, part) ||
!vkc_kv_run(ks, a->q, 0, (size_t)a->S * a->q_row, a->sel_row > 0 ? a->sel : NULL, (size_t)a->sel_off,
(size_t)a->S * a->sel_row, vkc_kv_gqa_dev, vkc_kv_gqa_host, a, part)) return 0;
@@ -781,6 +927,15 @@ static int vkc_kv_mla_rec(VkcKvMla *a, int s0, int ns, int fin) {
return vkc_kvs_mla(a->qa, a->qr, a->lat, a->rope, a->sel, fin ? a->out : ks->dpart, ks->tab, &p);
}
static int vkc_kv_mla_part(void *u, int s0, int ns) { return vkc_kv_mla_rec((VkcKvMla *)u, s0, ns, 0); }
+static int vkc_kv_mla_cold(void *u, int s0, int ns) {
+ VkcKvMla *a = (VkcKvMla *)u; VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];
+ int bt = (int)((size_t)a->li * ks->tab_stride), nb = ks->t[a->li].nblk;
+ VkcKvsMla p = {ns, a->H, a->K, a->R, a->pos_base + s0, a->kv_start, a->qa_off + s0 * a->qa_row, a->qa_row, a->qa_seg,
+ a->qr_off + s0 * a->qr_row, a->qr_row, a->qr_seg, (int)sh->off[0], a->K, (int)sh->off[1], a->R,
+ a->sel_off + s0 * a->sel_row, a->sel_row, s0 * a->H * a->K, a->H * a->K, a->K, a->scale, ks->B, ks->ns,
+ bt, nb, ks->anchor, VKC_KVS_COLD, a->S * a->H * a->K + s0 * a->H * 2};
+ return vkc_kvs_mla(a->qa, a->qr, sh->b[0], a->R > 0 ? sh->b[1] : NULL, a->sel, ks->cpart, ks->tab, &p);
+}
static int vkc_kv_mla_fin(void *u, int s0, int ns) { return vkc_kv_mla_rec((VkcKvMla *)u, s0, ns, 1); }
static int vkc_kv_mla_dev(void *u) {
VkcKvMla *a = (VkcKvMla *)u;
@@ -805,6 +960,19 @@ static int vkc_kv_mla(VkcKvMla *a) {
if (!vkc_kv_cold(ks, a->li, a->pos_base, a->S, 0, a->kv_start, a->sel_row > 0))
return vkc_kv_sliced(a->S, a->H, vkc_kv_span(ks, 0, a->sel_row), 2 * a->K + a->R, vkc_kv_mla_fin, a);
size_t part = (size_t)a->S * a->H * (a->K + 2);
+ if (ks->cold_dev) { /* the host's part on the device too */
+ const float *host[2] = {a->Lh, a->Rh};
+ size_t grp[2] = {0, 0}, ps[2] = {(size_t)a->K, (size_t)a->R};
+ int nseg[2] = {1, 1}, sl[2] = {a->K, a->R};
+ if (vkc_kv_shadow(ks, a->li, a->R > 0 ? 2 : 1, host, grp, ps, nseg, sl, a->pos_base)) {
+ ks->dev_cold++; ks->splits++; /* a step with a host part, on the device */
+ VkcKvsMerge mg = {a->S * a->H, a->H, a->K, 0, a->H * a->K, a->K, a->S * a->H * a->K, 0, a->S * a->H * a->K,
+ a->o_off, a->o_row, a->o_seg, 0, 0, 0, 0};
+ return vkc_kv_parts(ks, part) && vkc_kv_mla_dev(a) &&
+ vkc_kv_sliced(a->S, a->H, (long)a->pos_base + a->S, 2 * a->K + a->R, vkc_kv_mla_cold, a) &&
+ vkc_kvs_merge(ks->dpart, ks->cpart, NULL, a->out, &mg);
+ }
+ }
a->nqa = (size_t)(a->S - 1) * a->qa_row + (size_t)(a->H - 1) * a->qa_seg + a->K;
size_t nqr = a->R > 0 ? (size_t)(a->S - 1) * a->qr_row + (size_t)(a->H - 1) * a->qr_seg + a->R : 0;
if (!vkc_kv_parts(ks, part) ||
@@ -841,6 +1009,14 @@ static int vkc_kv_rel_rec(VkcKvRel *a, int s0, int ns, int fin) {
return vkc_kvs_rel(a->q, a->kc, a->vc, fin ? a->out : ks->dpart, a->r, a->tau, a->relp, ks->tab, &p);
}
static int vkc_kv_rel_part(void *u, int s0, int ns) { return vkc_kv_rel_rec((VkcKvRel *)u, s0, ns, 0); }
+static int vkc_kv_rel_cold(void *u, int s0, int ns) {
+ VkcKvRel *a = (VkcKvRel *)u; VkcKvSplit *ks = a->ks; VkcKvShadow *sh = &ks->sh[a->li];
+ int bt = (int)((size_t)a->li * ks->tab_stride), nb = ks->t[a->li].nblk;
+ VkcKvsRel p = {ns, a->H, a->KVH, a->hd, a->pos_base + s0, sh->cap, s0 * a->q_row, a->q_row, a->scale, (int)sh->off[0],
+ (int)sh->off[1], a->ext, a->d_rel, s0 * a->r_row, a->r_row, a->relp_off, a->tau_off + s0, ks->B, bt, nb,
+ ks->anchor, s0 * a->H * a->hd, a->H * a->hd, a->S * a->H * a->hd + s0 * a->H * 2, VKC_KVS_COLD};
+ return vkc_kvs_rel(a->q, sh->b[0], sh->b[1], ks->cpart, a->r, a->tau, a->relp, ks->tab, &p);
+}
static int vkc_kv_rel_fin(void *u, int s0, int ns) { return vkc_kv_rel_rec((VkcKvRel *)u, s0, ns, 1); }
static int vkc_kv_rel_dev(void *u) {
VkcKvRel *a = (VkcKvRel *)u;
@@ -867,6 +1043,19 @@ static int vkc_kv_rel(VkcKvRel *a) {
if (!vkc_kv_cold(ks, a->li, a->pos_base, a->S, 0, 0, 0))
return vkc_kv_sliced(a->S, a->H, vkc_kv_span(ks, 0, 0), 2 * a->hd + a->d_rel, vkc_kv_rel_fin, a);
size_t part = (size_t)a->S * a->H * (a->hd + 2);
+ if (ks->cold_dev) { /* the host's part on the device too */
+ const float *host[2] = {a->Kh, a->Vh};
+ size_t grp[2] = {a->k_grp, a->v_grp}, ps[2] = {a->k_pos, a->v_pos};
+ int nseg[2] = {a->KVH, a->KVH}, sl[2] = {a->hd, a->hd};
+ if (vkc_kv_shadow(ks, a->li, 2, host, grp, ps, nseg, sl, a->pos_base)) {
+ ks->dev_cold++; ks->splits++; /* a step with a host part, on the device */
+ VkcKvsMerge mg = {a->S * a->H, a->H, a->hd, 0, a->H * a->hd, a->hd, a->S * a->H * a->hd, 0, a->S * a->H * a->hd,
+ 0, a->o_row, a->hd, 0, 0, 0, 0};
+ return vkc_kv_parts(ks, part) && vkc_kv_rel_dev(a) &&
+ vkc_kv_sliced(a->S, a->H, (long)a->pos_base + a->S, 2 * a->hd + a->d_rel, vkc_kv_rel_cold, a) &&
+ vkc_kvs_merge(ks->dpart, ks->cpart, NULL, a->out, &mg);
+ }
+ }
a->nq = (size_t)a->S * a->q_row;
if (!vkc_kv_parts(ks, part) ||
!vkc_kv_run2(ks, a->q, 0, a->nq, a->d_rel > 0 ? a->r : NULL, 0, (size_t)a->S * a->r_row, NULL, 0, 0,
@@ -1016,7 +1205,11 @@ static int vkc_kv_mla_attn(VkcKvSplit *ks, int li, const VkcMla *m, VkcMlaScratc
static void vkc_kv_report(const VkcKvSplit *ks) {
if (!ks->on || !ks->steps) return;
fprintf(stderr, "[VK] %s chain: KV split: %llu layer steps, %llu with a host part (%.1f ms waiting for the queries, "
- "%.1f ms of host work over %llu positions, %llu sparse rows staged), %llu blocks pinned by reads\n",
+ "%.1f ms of host work over %llu positions, %llu sparse rows staged), %llu blocks pinned by reads",
ks->engine, ks->steps, ks->splits, ks->wait_ms, ks->host_ms, ks->host_pos, ks->staged_rows, ks->pins);
+ if (ks->cold_dev)
+ fprintf(stderr, " | the host's part on the device: %llu layer steps, %llu rows copied to the shadow in %.1f ms "
+ "(%.1f MiB held)", ks->dev_cold, ks->shadow_rows, ks->shadow_ms, ks->shadow_bytes / 1048576.0);
+ fprintf(stderr, "\n");
}
#endif
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index a014f033..dc0a6295 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -272,6 +272,7 @@ With `COLI_VULKAN=1` every MoE engine (qwen36, qwen38, inkling, olmoe, kimi_k3,
| `COLI_VK_KV_DEVICE_ROWS` | from the device's budget | The positions a split layer keeps on the device; set, the cache splits whenever it is longer (the tests' small device, a measurement's split). |
| `COLI_VK_KV_BLOCK` | `64` | Positions per block of the split cache's block table. |
| `COLI_VK_KV_PIN` | off | `1`: pin blocks by sparse-selection reads. For QSA/DSA/pooled MLA this can change rounding with read history; the default keeps the partition independent of chunking and prefix reuse. DeepSeek stages missing selected rows and preserves its arithmetic in either mode. |
+| `COLI_VK_KV_COLD` | unset | `device`: the host's part of a split KV cache attended on the device too, from a shadow of the host's rows (GQA, MLA and Inkling forms; DeepSeek keeps it on the CPU). Off by default: on the measured integrated GPU it made the prompt 4-6% faster and decode 5.7x slower (see [vulkan.md](vulkan.md#a-kv-cache-past-the-devices-budget)). |
| `COLI_VK_CHAIN_FAULT` | off | Tests: the n-th chain frame fails as a lost device would; the engine rebuilds the state on the CPU and runs there (colibri, mimo and the DeepSeek engines have none to rebuild: the CPU continues from its host caches). |
| `DUMP` | unset | colibri, glm53, deepseek_v41 and deepseek_v4 in a `VK=1` build: every logits row the forward computes, appended to this file as raw f32 (the Vulkan gates compare the device's with the CPU's). |
diff --git a/docs/vulkan.md b/docs/vulkan.md
index de6ef842..7bf7eae0 100644
--- a/docs/vulkan.md
+++ b/docs/vulkan.md
@@ -1001,6 +1001,7 @@ f32 throughout, as the CPU's f32 path.
| `COLI_VK_KV_DEVICE_ROWS` | from the budget | The positions a split layer keeps on the device; set, the cache splits whenever it is longer. |
| `COLI_VK_KV_BLOCK` | `64` | Positions per block of the split's block table. |
| `COLI_VK_KV_PIN` | off | `1`: enable read-based block pins; QSA/DSA/pooled MLA rounding can then depend on read history. DeepSeek preserves its sparse arithmetic in either mode. |
+| `COLI_VK_KV_COLD` | unset | `device`: a split layer's host part attended on the device too, from a shadow of the host's rows ([below](#a-kv-cache-past-the-devices-budget)); off by default, because on the measured integrated GPU it slowed decode. |
Each run and serve turn prints `[VK] <engine> chain: N forwards, F frames (ops,
matmuls, tiled GEMM), the time spent waiting for the device, the routed experts' host
@@ -2063,6 +2064,38 @@ round trip is paid only on layers with a host part.
| `vkc_kvs_ds` | `chain_kvs` (mode 4) | DeepSeek's sparse attention with its sink over the window rows and the resident compressed rows; with every listed row resident, `vkc_dsv4_attn`'s bits |
| `vkc_kvs_merge` | `chain_kvs` (mode 2) | the two parts joined, a gate or V4's bf16 rounding after |
+**The host's part on the device (`COLI_VK_KV_COLD=device`, off by default).** The
+host's rows can also be attended on the device, so a step with a host part runs in one
+frame with no round trip:
+- **The copy.** Each split layer keeps a shadow of the host's rows in page-aligned memory
+ the device reads in place (an imported host allocation, `VK_EXT_external_memory_host`).
+ The shadow follows the host's cache row by row and is lowered with it on a rewind or a
+ rollback.
+- **The ops.** `chain_kvs` reads the shadow with its COLD flag (the GQA, MLA and Inkling
+ forms; position t is row t). The prompt's rows take the blocked attention
+ (`chain_attnb`'s part mode) in chunks of 512 positions, one workgroup each, and mode 5
+ of `chain_kvs` joins them in order.
+- **The bits.** The chunks are fixed by position, so a row's bits do not depend on how a
+ forward is cut into steps, as on the CPU path.
+- **Where it applies.** It needs a device that imports host memory, no read-based pins
+ and no staged uploads. DeepSeek's sparse forms keep the host's part on the CPU. Where
+ it cannot run, the line says so and the CPU computes the host's part.
+
+The run's report adds `| the host's part on the device: N layer steps, R rows copied
+to the shadow in T ms (M MiB held)`.
+
+| Qwen3.6-35B-A3B, 7579-token prompt, Radeon 780M, the chain | first token after | 31 decode tokens |
+|---|---|---|
+| 1024 positions a layer on the device, the host's part on the CPU | 121 s | 3.5 s |
+| the same, the host's part on the device | 114 s | 20 s |
+| 4096 positions a layer, on the CPU | 143 s | |
+| the same, on the device | 137 s | |
+
+The prompt gains 4 to 6%. Decode is 5.7 times slower: with the default, the CPU
+computes the host's part while the device runs its own, and on this integrated GPU the
+decode attention is slower than the CPU's. Hence the default. A dedicated GPU, whose
+attention outruns the CPU's by more, has not been measured.
+
`vk_kvsplit.h` carries the rest: the plan, the tables, the uploads and the stores of a
step's rows, the host's part (`vkc_kv_host_attn`, with the same bias, tau and V4
rounding), and one call per form for an engine (`vkc_kv_gqa`, `vkc_kv_mla`,
@@ -2132,7 +2165,9 @@ part): the CPU's tokens, logits within each family's tolerance, chunks of 3, pro
only, the split off, a lost device, pins, MTP and n-gram drafts, serve sessions with
pins, prompt-cache extensions and divergent prompts, and the prefix-reuse tests;
`kv-split-deepseek` the same for deepseek_v41 and deepseek_v4; `kv-split-sanitize` and
-`kv-split-deepseek-sanitize` a set of each under ASan and UBSan.
+`kv-split-deepseek-sanitize` a set of each under ASan and UBSan. `kv-split-cold` and
+`kv-split-cold-sanitize` run `kv-split` and `kv-split-sanitize` again with
+`COLI_VK_KV_COLD=device`, and every gate must have run the host's part on the device.
Validation on the local Lavapipe device covers numerical correctness, not hardware
throughput. A discrete GPU has not been measured for this change.