Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: the chain's context per device (vkc_device), for layers on COLI_VK_DEV2

JustVuggfb79bda81dbc
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index fc85d261..f1587027 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -4490,6 +4490,69 @@ int coli_vk_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *o) {
}
/* A fence the chain waited on failed: the device is gone, everyone falls back. */
void coli_vk_mark_lost(void) { G.ready = 0; }
+
+/* ---- the dense chain on either device (vk_chain.c keeps a context per device) --------
+ * d = 0 is the primary device, what the calls without _dev answer; d = 1 is COLI_VK_DEV2's
+ * (G2), which until now held only routed experts. The chain's layers on it run the same
+ * shaders: the core below is G2's, with the primary's GEMM tiles (a tile is a speed
+ * choice; every tile computes the same bits). */
+int coli_vk_core_dev(int d, ColiVkCore *o) {
+ if (d == 0) return coli_vk_core(o);
+ if (d != 1 || !G2.ready || !G.ready || !o) return 0;
+ if (!coli_vk_core(o)) return 0; /* the tiles, the shader path: the primary's */
+ o->phys = (void *)G2.phys; o->device = (void *)G2.dev; o->queue = (void *)G2.queue; o->qfam = G2.qfam;
+ o->memtype_host = g_up[1].on ? g_up[1].mt_stage : G2.memtype;
+ o->memtype_cached = G2.memtype_cached; o->memtype_dev = G2.memtype_dev;
+ o->ssbo_align = G2.ssbo_align; o->ssbo_range = G2.ssbo_range;
+ o->has_prio = 0;
+ o->integrated = G2.integrated; o->shares_ram = G2.shares_ram;
+ return 1;
+}
+int coli_vk_available_dev(int d) { return d == 1 ? G2.ready : G.ready; }
+void coli_vk_mark_lost_dev(int d) { if (d == 1) G2.ready = 0; else G.ready = 0; }
+int coli_vk_queue_submit_dev(int d, void *queue, const void *submit_info, void *fence) {
+ return (int)vk_submit(d == 1 ? 1 : 0, (VkQueue)queue, (const VkSubmitInfo *)submit_info, (VkFence)fence);
+}
+int coli_vk_tensor_info_dev(const ColiVkTensor *t, ColiVkTensorInfo *o, int *dev) {
+ if (!t || !o || t->dev < 0 || t->dev > 1) return 0;
+ o->wbuf = (void *)t->wbuf; o->sbuf = (void *)t->sbuf;
+ o->fmt = t->fmt; o->I = t->I; o->O = t->O; o->rowWords = t->rowWords; o->gs = t->gs;
+ if (dev) *dev = t->dev;
+ return 1;
+}
+int coli_vk_mem_budget_dev(int d, double *used_gb, double *budget_gb) {
+ return d == 1 ? coli_vk_mem_budget2(used_gb, budget_gb) : coli_vk_mem_budget(used_gb, budget_gb);
+}
+size_t coli_vk_device_used_dev(int d) { return (size_t)__atomic_load_n(&g_mem.used[d == 1 ? 1 : 0], __ATOMIC_RELAXED); }
+size_t coli_vk_device_local_bytes_dev(int d) {
+ if (d != 1) return coli_vk_device_local_bytes();
+ if (!G2.phys) return 0;
+ VkPhysicalDeviceMemoryProperties mp;
+ vkGetPhysicalDeviceMemoryProperties(G2.phys, &mp);
+ VkDeviceSize m = 0;
+ for (uint32_t i = 0; i < mp.memoryHeapCount; i++)
+ if ((mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && mp.memoryHeaps[i].size > m) m = mp.memoryHeaps[i].size;
+ if (g_mem.cap[1] && g_mem.cap[1] < m) m = g_mem.cap[1]; /* COLI_VK_DEVICE_CAP_MB */
+ return (size_t)m;
+}
+size_t coli_vk_free_bytes_dev(int d) {
+ if (d != 1) return coli_vk_free_bytes();
+ if (!G2.phys) return 0;
+ if (g_mem.cap[1]) return (size_t)vk_mem_room(1);
+ double used = 0, bud = 0;
+ if (coli_vk_mem_budget2(&used, &bud)) return bud > used ? (size_t)((bud - used) * 1e9) : 0;
+ size_t dl = coli_vk_device_local_bytes_dev(1), u = coli_vk_device_used_dev(1);
+ return dl > u ? dl - u : 0;
+}
+void coli_vk_mem_info_dev(int d, size_t *used, size_t *count) {
+ if (d != 1) { coli_vk_mem_info(used, count); return; }
+ if (used) *used = (size_t)__atomic_load_n(&g_wpool2.bytes, __ATOMIC_RELAXED);
+ if (count) *count = (size_t)__atomic_load_n(&g_wpool2.tensors, __ATOMIC_RELAXED);
+}
+size_t coli_vk_buffer_alignment_dev(int d) {
+ if (d != 1) return coli_vk_buffer_alignment();
+ return G2.buf_align ? G2.buf_align : 256;
+}
/* The chain's submits: through the lock the staged uploader takes on a shared queue. */
int coli_vk_queue_submit(void *queue, const void *submit_info, void *fence) {
return (int)vk_submit(0, (VkQueue)queue, (const VkSubmitInfo *)submit_info, (VkFence)fence);
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index 30b41c92..457a13c5 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -411,6 +411,21 @@ void coli_vk_mark_lost(void);
/* vkQueueSubmit(queue, 1, submit_info, fence) as the backend submits (a VkResult): the
* staged uploader may share the main queue from its own thread. */
int coli_vk_queue_submit(void *queue, const void *submit_info, void *fence);
+/* The same for either device: d = 0 the primary (as the calls above), d = 1 COLI_VK_DEV2's,
+ * whose context the chain opens for the layers it places there (vk_chain.c). The core of
+ * device 1 carries the primary's shader path and GEMM tiles. coli_vk_tensor_info_dev
+ * reports a tensor on either device and which (coli_vk_tensor_info: device 0's only). */
+int coli_vk_core_dev(int d, ColiVkCore *out);
+int coli_vk_available_dev(int d);
+void coli_vk_mark_lost_dev(int d);
+int coli_vk_queue_submit_dev(int d, void *queue, const void *submit_info, void *fence);
+int coli_vk_tensor_info_dev(const ColiVkTensor *t, ColiVkTensorInfo *out, int *dev);
+int coli_vk_mem_budget_dev(int d, double *used_gb, double *budget_gb);
+size_t coli_vk_device_used_dev(int d);
+size_t coli_vk_device_local_bytes_dev(int d);
+size_t coli_vk_free_bytes_dev(int d);
+size_t coli_vk_buffer_alignment_dev(int d);
+void coli_vk_mem_info_dev(int d, size_t *used_bytes, size_t *tensor_count); /* dense weights on device d */
#ifdef __cplusplus
}
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index 6b6e4f0b..1328c374 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -102,6 +102,8 @@ static float *down(VkcBuf *b, size_t off, size_t n) {
/* ---- matmul ----------------------------------------------------------------------- */
static uint16_t f2bf(float f) { uint32_t u; memcpy(&u, &f, 4); return (uint16_t)((u + 0x7fff + ((u >> 16) & 1)) >> 16); }
static float bf2f(uint16_t h) { uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }
+static int g_test_dev; /* the device test_matmul uploads to (the chain's current one) */
+static float *g_mm_out; /* test_matmul's output, kept when set to non-NULL (the two devices' bits) */
static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {
float *x = fvec((size_t)S * I, 1.f), *W = malloc((size_t)O * I * sizeof(float));
void *codes = NULL; float *sc = NULL; int gs = 0;
@@ -127,7 +129,8 @@ static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {
codes = w;
}
ColiVkTensor *t = NULL;
- if (!coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs)) { CHECK(0, "matmul fmt %d: upload", fmt); return; }
+ if (!(g_test_dev ? coli_vk_tensor_ensure2(&t, codes, sc, fmt, I, O, gs) : coli_vk_tensor_ensure(&t, codes, sc, fmt, I, O, gs))) {
+ CHECK(0, "matmul fmt %d: upload", fmt); return; }
float *ref = malloc((size_t)S * O * sizeof *ref);
for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {
double a = 0; for (int i = 0; i < I; i++) a += (double)x[(size_t)s * I + i] * W[(size_t)o * I + i];
@@ -141,6 +144,7 @@ static void test_matmul(int fmt, int S, int I, int O, size_t xo, size_t yo) {
float *y = (float *)vkc_ptr(yb) + yo;
double e = relerr(y, ref, (size_t)S * O, 1e-3);
CHECK(ok && e < 2e-4 && !bad(y, (size_t)S * O), "matmul fmt %d S %d I %d O %d xo %zu yo %zu: ok %d err %.2e", fmt, S, I, O, xo, yo, ok, e);
+ if (g_mm_out) memcpy(g_mm_out, y, (size_t)S * O * sizeof(float));
vkc_free(xb); vkc_free(yb); coli_vk_tensor_free(t);
free(x); free(W); free(codes); free(sc); free(ref);
}
@@ -2336,6 +2340,67 @@ static void bench_gemv(void) {
}
}
+/* ---- a second device (vkc_device(1), COLI_VK_DEV2) -------------------------------------
+ * The chain's context on COLI_VK_DEV2's device: the ops against their references there,
+ * the matmul's bits equal to device 0's, and the boundary held (a buffer of one device
+ * bound or copied on the other fails; vkc_free takes it back on its own device). */
+static void test_dev2(const char *spv) {
+ const char *d2 = getenv("COLI_VK_DEV2");
+ if (!coli_vk_init_dev2(spv, strcmp(d2, "auto") ? atoi(d2) : -1)) { CHECK(0, "dev2: the second device did not open"); return; }
+ vkc_device(1);
+ if (!vkc_init()) { CHECK(0, "dev2: the chain's pipelines did not come up there"); vkc_device(0); return; }
+ vkc_device(0);
+ int fmts[4] = {1, 4, 10, 11};
+ for (int k = 0; k < 4; k++) {
+ int shapes[3][3] = {{1, 192, 70}, {40, 256, 160}, {2, 3072, 40}};
+ for (int j = 0; j < 3; j++) {
+ int S = shapes[j][0], I = shapes[j][1], O = shapes[j][2];
+ float *y0 = malloc((size_t)S * O * sizeof(float)), *y1 = malloc((size_t)S * O * sizeof(float));
+ unsigned seed = rng;
+ g_mm_out = y0; test_matmul(fmts[k], S, I, O, 0, 0);
+ rng = seed; vkc_device(1); g_test_dev = 1;
+ g_mm_out = y1; test_matmul(fmts[k], S, I, O, 0, 0);
+ g_test_dev = 0; vkc_device(0); g_mm_out = NULL;
+ CHECK(!memcmp(y0, y1, (size_t)S * O * sizeof(float)), "dev2: matmul fmt %d S %d: device 1's bits differ from device 0's", fmts[k], S);
+ free(y0); free(y1);
+ }
+ }
+ vkc_device(1);
+ test_norm(VKC_NORM_ADD1, 0); test_norm(0, 0);
+ test_rope();
+ test_attn(1, 140, 32, 0); test_attn(5, 200, 64, 0);
+ setenv("COLI_VK_ATTN_BLOCK", "16", 1); test_attn_hk(40, 7, 16, 2, 256, 0); unsetenv("COLI_VK_ATTN_BLOCK");
+ test_dnrec(8, 8, 8, 4, 0); test_dnrec(32, 100, 6, 3, 1);
+ if (vkc_kvs_ready()) test_kvs_steps();
+ /* the boundary: device 0's buffer on device 1 */
+ vkc_device(0);
+ float v[64]; for (int i = 0; i < 64; i++) v[i] = (float)i;
+ VkcBuf *a0 = up(v, 64);
+ vkc_device(1);
+ VkcBuf *a1 = up(v, 64), *b1 = vkc_buf(64 * 4, VKC_DEV);
+ vkc_begin();
+ int cross = vkc_copy(b1, 0, a0, 0, 64);
+ vkc_submit(1);
+ vkc_begin();
+ int same = vkc_copy(b1, 0, a1, 0, 64);
+ vkc_submit(1);
+ CHECK(!cross && same, "dev2: a copy across devices %s, one on device 1 %s", cross ? "ran" : "refused", same ? "ran" : "failed");
+ VkcNorm nm = {1, 64, 1, 0, 64, 64, 0, 64, 64, 0, 0, VKC_NORM_NOW, 1e-6f, 1.f};
+ vkc_begin();
+ int bound = vkc_norm(a0, a1, b1, &nm);
+ vkc_submit(1);
+ CHECK(!bound, "dev2: an op bound device 0's buffer on device 1");
+ vkc_free(a1); vkc_free(b1);
+ vkc_device(0);
+ vkc_free(a0);
+ vkc_device(1);
+ vkc_free(NULL);
+ VkcStats st; vkc_stats(&st);
+ printf("dev2: chain on the second device, %llu frames, %llu ops, %llu matmuls\n", st.frames, st.ops, st.matmuls);
+ vkc_shutdown();
+ vkc_device(0);
+}
+
int main(int argc, char **argv) {
const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }
@@ -2495,6 +2560,7 @@ int main(int argc, char **argv) {
}
if (!vkc_kvs_ready()) { fails++; printf("FAIL: the split KV shader did not load\n"); }
else { test_kvs(); printf("kvs done\n"); }
+ if (getenv("COLI_VK_DEV2")) { test_dev2(spv); printf("dev2 done\n"); }
VkcStats st; vkc_stats(&st);
printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);
vkc_shutdown();
diff --git a/c/vk_chain.c b/c/vk_chain.c
index c45c0f00..5b973b3f 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -72,6 +72,7 @@ struct VkcBuf {
VkaRange r;
uint8_t *ptr;
unsigned long long last_frame; /* serial of the last frame that bound it */
+ int d; /* the device (vkc_device) it was made on */
};
typedef struct {
@@ -85,8 +86,9 @@ typedef struct {
VkQueryPool qp; int nq; unsigned char kind[VKC_PROF_Q]; /* COLI_VK_CHAIN_PROF: a timestamp after each op */
} VkcFrame;
-static struct {
+typedef struct {
int ready, lost;
+ int misuse; /* a bind of the other device's buffer: the record fails */
ColiVkCore core;
VkDevice dev;
VkQueue queue;
@@ -113,15 +115,29 @@ static struct {
VkcStats st;
int prof, kind; float ts_period; uint32_t ts_mask;
double prof_ms[PK_N]; unsigned long long prof_n[PK_N];
-} K = {.cur = -1, .gemm_rows = -1};
+} VkcCtx;
+/* One context per device (vkc_device): 0 the primary, 1 COLI_VK_DEV2's, for the layers an
+ * engine places there. Every call works on the current device's; a buffer remembers its
+ * device, and binding or copying one of the other device's is refused. */
+static VkcCtx g_kctx[2] = {{.cur = -1, .gemm_rows = -1}, {.cur = -1, .gemm_rows = -1}};
+static int g_kd;
+#define KC (g_kctx[g_kd])
+
+/* a weight tensor the current device holds (one on the other device is refused) */
+static int vkc_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *ti) {
+ int d = 0;
+ return coli_vk_tensor_info_dev(t, ti, &d) && d == g_kd;
+}
-int vkc_ready(void) { return K.ready && !K.lost; }
-int vkc_lost(void) { return K.lost; }
+int vkc_ready(void) { return KC.ready && !KC.lost; }
+int vkc_device(int d) { int was = g_kd; g_kd = d == 1 ? 1 : 0; return was; }
+int vkc_device_now(void) { return g_kd; }
+int vkc_lost(void) { return KC.lost; }
static void lose(const char *what, VkResult r) {
- if (!K.lost) fprintf(stderr, "[VK] chain: %s failed (%d): the device is lost\n", what, (int)r);
- K.lost = 1;
- coli_vk_mark_lost();
+ if (!KC.lost) fprintf(stderr, "[VK] chain: %s failed (%d): the device is lost\n", what, (int)r);
+ KC.lost = 1;
+ coli_vk_mark_lost_dev(g_kd);
}
/* ---- blocks ---------------------------------------------------------------------- */
@@ -133,95 +149,101 @@ static int pool_grow(VkcPool *P, uint64_t need) {
#ifdef VK_EXT_memory_priority
VkMemoryPriorityAllocateInfoEXT pri = {.sType = VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT,
.priority = 1.0f}; /* rides every chain submit: never evicted before weights */
- if (K.core.has_prio) ai.pNext = &pri;
+ if (KC.core.has_prio) ai.pNext = &pri;
#endif
VkcBlock *bk = calloc(1, sizeof *bk);
if (!bk) return 0;
- if (vkAllocateMemory(K.dev, &ai, NULL, &bk->mem) != VK_SUCCESS) { free(bk); return 0; }
- if (P->mapped && vkMapMemory(K.dev, bk->mem, 0, cap, 0, (void **)&bk->map) != VK_SUCCESS) {
- vkFreeMemory(K.dev, bk->mem, NULL); free(bk); return 0;
+ if (vkAllocateMemory(KC.dev, &ai, NULL, &bk->mem) != VK_SUCCESS) { free(bk); return 0; }
+ if (P->mapped && vkMapMemory(KC.dev, bk->mem, 0, cap, 0, (void **)&bk->map) != VK_SUCCESS) {
+ vkFreeMemory(KC.dev, bk->mem, NULL); free(bk); return 0;
}
if (vka_pool_add_block(&P->p, cap, bk) < 0) {
- if (bk->map) vkUnmapMemory(K.dev, bk->mem);
- vkFreeMemory(K.dev, bk->mem, NULL); free(bk); return 0;
+ if (bk->map) vkUnmapMemory(KC.dev, bk->mem);
+ vkFreeMemory(KC.dev, bk->mem, NULL); free(bk); return 0;
}
return 1;
}
static uint32_t memtype_of(int kind) {
- return kind == VKC_UP ? K.core.memtype_host : kind == VKC_DOWN ? K.core.memtype_cached : K.core.memtype_dev;
+ return kind == VKC_UP ? KC.core.memtype_host : kind == VKC_DOWN ? KC.core.memtype_cached : KC.core.memtype_dev;
}
static int memtype_host_visible(uint32_t t) {
VkPhysicalDeviceMemoryProperties mp;
- vkGetPhysicalDeviceMemoryProperties((VkPhysicalDevice)K.core.phys, &mp);
+ vkGetPhysicalDeviceMemoryProperties((VkPhysicalDevice)KC.core.phys, &mp);
return (mp.memoryTypes[t].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;
}
-static void track_reset(void) { K.nwr = K.nrd = 0; }
+static void track_reset(void) { KC.nwr = KC.nrd = 0; }
static void zero_now(VkcBuf *b);
VkcBuf *vkc_buf(size_t bytes, int kind) {
if (!vkc_ready() || kind < 0 || kind > 2) return NULL;
if (!bytes) bytes = 4;
bytes = (bytes + 3) & ~(size_t)3;
- if (K.core.ssbo_range && bytes > K.core.ssbo_range) return NULL; /* one binding could not address it */
+ if (KC.core.ssbo_range && bytes > KC.core.ssbo_range) return NULL; /* one binding could not address it */
VkcBuf *b = calloc(1, sizeof *b);
if (!b) return NULL;
- b->kind = kind; b->bytes = bytes;
+ b->kind = kind; b->bytes = bytes; b->d = g_kd;
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = bytes,
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE};
- if (vkCreateBuffer(K.dev, &bi, NULL, &b->buf) != VK_SUCCESS) { free(b); return NULL; }
+ if (vkCreateBuffer(KC.dev, &bi, NULL, &b->buf) != VK_SUCCESS) { free(b); return NULL; }
VkMemoryRequirements req;
- vkGetBufferMemoryRequirements(K.dev, b->buf, &req);
- VkcPool *P = &K.pool[kind];
- uint64_t align = req.alignment > K.align ? req.alignment : K.align;
+ vkGetBufferMemoryRequirements(KC.dev, b->buf, &req);
+ VkcPool *P = &KC.pool[kind];
+ uint64_t align = req.alignment > KC.align ? req.alignment : KC.align;
if (!(req.memoryTypeBits & (1u << P->memtype)) ||
(!vka_alloc(&P->p, req.size, align, &b->r) &&
!(pool_grow(P, req.size + align) && vka_alloc(&P->p, req.size, align, &b->r)))) {
- vkDestroyBuffer(K.dev, b->buf, NULL); free(b); return NULL;
+ vkDestroyBuffer(KC.dev, b->buf, NULL); free(b); return NULL;
}
VkcBlock *bk = P->p.b[b->r.block].user;
- if (vkBindBufferMemory(K.dev, b->buf, bk->mem, b->r.off) != VK_SUCCESS) {
+ if (vkBindBufferMemory(KC.dev, b->buf, bk->mem, b->r.off) != VK_SUCCESS) {
if (vka_free(&P->p, b->r)) { VkcBlock *e = P->p.b[b->r.block].user;
- if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }
+ if (e) { if (e->map) vkUnmapMemory(KC.dev, e->mem); vkFreeMemory(KC.dev, e->mem, NULL); free(e); }
vka_pool_drop_block(&P->p, b->r.block); }
- vkDestroyBuffer(K.dev, b->buf, NULL); free(b); return NULL;
+ vkDestroyBuffer(KC.dev, b->buf, NULL); free(b); return NULL;
}
b->ptr = bk->map ? bk->map + b->r.off : NULL;
- K.st.dev_bytes += bytes;
+ KC.st.dev_bytes += bytes;
if (b->ptr) memset(b->ptr, 0, bytes);
else zero_now(b);
return b;
}
static void buf_release(VkcBuf *b) {
- VkcPool *P = &K.pool[b->kind];
- vkDestroyBuffer(K.dev, b->buf, NULL);
+ VkcPool *P = &KC.pool[b->kind];
+ vkDestroyBuffer(KC.dev, b->buf, NULL);
if (vka_free(&P->p, b->r)) {
VkcBlock *e = P->p.b[b->r.block].user;
- if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }
+ if (e) { if (e->map) vkUnmapMemory(KC.dev, e->mem); vkFreeMemory(KC.dev, e->mem, NULL); free(e); }
vka_pool_drop_block(&P->p, b->r.block);
}
- K.st.dev_bytes -= b->bytes;
+ KC.st.dev_bytes -= b->bytes;
free(b);
}
/* a frame that may still read b must finish first */
static int frame_wait(VkcFrame *f);
-void vkc_free(VkcBuf *b) {
- if (!b) return;
- if (!K.dev) { free(b); return; }
- if (!K.lost)
+static void vkc_free_here(VkcBuf *b) {
+ if (!KC.dev) { free(b); return; }
+ if (!KC.lost)
for (int i = 0; i < VKC_FRAMES; i++) {
- VkcFrame *f = &K.fr[i];
+ VkcFrame *f = &KC.fr[i];
if ((f->inflight || f->open) && f->serial == b->last_frame) {
if (f->open) vkc_submit(1); else frame_wait(f);
}
}
- for (int i = 0; i < K.nwr; i++) if (K.wr[i] == b->buf) K.wr[i] = VK_NULL_HANDLE;
- for (int i = 0; i < K.nrd; i++) if (K.rd[i] == b->buf) K.rd[i] = VK_NULL_HANDLE;
+ for (int i = 0; i < KC.nwr; i++) if (KC.wr[i] == b->buf) KC.wr[i] = VK_NULL_HANDLE;
+ for (int i = 0; i < KC.nrd; i++) if (KC.rd[i] == b->buf) KC.rd[i] = VK_NULL_HANDLE;
buf_release(b);
}
+void vkc_free(VkcBuf *b) { /* on the device that made it, whichever is current */
+ if (!b) return;
+ int was = g_kd;
+ g_kd = b->d;
+ vkc_free_here(b);
+ g_kd = was;
+}
int vkc_reserve(VkcBuf **b, size_t bytes, int kind) {
if (*b && (*b)->bytes >= bytes && (*b)->kind == kind) return 1;
size_t want = bytes;
@@ -248,7 +270,7 @@ static VkShaderModule load_module(const char *dir_spv, const char *file) {
fclose(f);
VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)n, .pCode = code};
VkShaderModule m = VK_NULL_HANDLE;
- if (vkCreateShaderModule(K.dev, &ci, NULL, &m) != VK_SUCCESS) m = VK_NULL_HANDLE;
+ if (vkCreateShaderModule(KC.dev, &ci, NULL, &m) != VK_SUCCESS) m = VK_NULL_HANDLE;
free(code);
if (!m) fprintf(stderr, "[VK] chain: cannot load %s\n", path);
return m;
@@ -256,9 +278,9 @@ static VkShaderModule load_module(const char *dir_spv, const char *file) {
static VkPipeline make_pipe(VkShaderModule m, const VkSpecializationInfo *si) {
VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT,
- .module = m, .pName = "main", .pSpecializationInfo = si}, .layout = K.pl};
+ .module = m, .pName = "main", .pSpecializationInfo = si}, .layout = KC.pl};
VkPipeline p = VK_NULL_HANDLE;
- if (vkCreateComputePipelines(K.dev, VK_NULL_HANDLE, 1, &ci, NULL, &p) != VK_SUCCESS) return VK_NULL_HANDLE;
+ if (vkCreateComputePipelines(KC.dev, VK_NULL_HANDLE, 1, &ci, NULL, &p) != VK_SUCCESS) return VK_NULL_HANDLE;
return p;
}
@@ -266,105 +288,105 @@ static void dsv4_init(void); /* chain_dsv4.comp (DeepSeek V4.1 / V4), belo
static void dsv4_shutdown(void);
static void kvs_shutdown(void); /* chain_kvs.comp (the split KV cache), below */
int vkc_init(void) {
- if (K.ready) return !K.lost;
- if (!coli_vk_core(&K.core)) return 0;
- K.dev = (VkDevice)K.core.device; K.queue = (VkQueue)K.core.queue;
- K.align = K.core.ssbo_align > 16 ? K.core.ssbo_align : 16;
+ if (KC.ready) return !KC.lost;
+ if (!coli_vk_core_dev(g_kd, &KC.core)) return 0;
+ KC.dev = (VkDevice)KC.core.device; KC.queue = (VkQueue)KC.core.queue;
+ KC.align = KC.core.ssbo_align > 16 ? KC.core.ssbo_align : 16;
VkDescriptorSetLayoutBinding b[VKC_BIND];
for (int i = 0; i < VKC_BIND; i++) b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.bindingCount = VKC_BIND, .pBindings = b};
VkPushConstantRange pr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = 128};
- if (vkCreateDescriptorSetLayout(K.dev, &li, NULL, &K.dsl) != VK_SUCCESS) return 0;
+ if (vkCreateDescriptorSetLayout(KC.dev, &li, NULL, &KC.dsl) != VK_SUCCESS) return 0;
VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
- .setLayoutCount = 1, .pSetLayouts = &K.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
- if (vkCreatePipelineLayout(K.dev, &pi, NULL, &K.pl) != VK_SUCCESS) return 0;
+ .setLayoutCount = 1, .pSetLayouts = &KC.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr};
+ if (vkCreatePipelineLayout(KC.dev, &pi, NULL, &KC.pl) != VK_SUCCESS) return 0;
for (int i = 0; i < P_NPIPE; i++) {
- K.mod[i] = load_module(K.core.spv_path, pipe_file[i]);
- if (!K.mod[i] || !(K.pipe[i] = make_pipe(K.mod[i], NULL))) {
+ KC.mod[i] = load_module(KC.core.spv_path, pipe_file[i]);
+ if (!KC.mod[i] || !(KC.pipe[i] = make_pipe(KC.mod[i], NULL))) {
fprintf(stderr, "[VK] chain: shader %s unavailable, the chain stays off\n", pipe_file[i]);
return 0;
}
}
- K.mod_dnrec = load_module(K.core.spv_path, "chain_dnrec.spv");
- if (!K.mod_dnrec) return 0;
+ KC.mod_dnrec = load_module(KC.core.spv_path, "chain_dnrec.spv");
+ if (!KC.mod_dnrec) return 0;
/* the vectorized decode GEMV, its x staging as large as the device's shared memory
* allows (16 KiB floats at most); COLI_VK_CHAIN_GEMV=0 keeps qmatmul.comp's */
{
const char *e = getenv("COLI_VK_CHAIN_GEMV");
- VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)KC.core.phys, &pp);
int xs = (int)(pp.limits.maxComputeSharedMemorySize / 16);
if (xs > 4096) xs = 4096;
- if (!(e && *e == '0') && xs >= 256 && (K.mod_gemv4 = load_module(K.core.spv_path, "chain_gemv.spv"))) {
+ if (!(e && *e == '0') && xs >= 256 && (KC.mod_gemv4 = load_module(KC.core.spv_path, "chain_gemv.spv"))) {
int32_t v = xs;
VkSpecializationMapEntry me = {0, 0, 4};
VkSpecializationInfo si = {1, &me, 4, &v};
- if ((K.gemv4 = make_pipe(K.mod_gemv4, &si))) K.gemv4_xs = xs;
+ if ((KC.gemv4 = make_pipe(KC.mod_gemv4, &si))) KC.gemv4_xs = xs;
}
}
/* the MLA, KDA and mHC shaders, optional: without one only its ops decline */
for (int i = 0; i < PM_N; i++) {
char path[1200];
- const char *sl = strrchr(K.core.spv_path, '/');
- size_t pre = sl ? (size_t)(sl - K.core.spv_path) + 1 : 0;
+ const char *sl = strrchr(KC.core.spv_path, '/');
+ size_t pre = sl ? (size_t)(sl - KC.core.spv_path) + 1 : 0;
FILE *f = NULL;
if (pre + strlen(mla_file[i]) + 1 < sizeof path) {
- memcpy(path, K.core.spv_path, pre); strcpy(path + pre, mla_file[i]);
+ memcpy(path, KC.core.spv_path, pre); strcpy(path + pre, mla_file[i]);
f = fopen(path, "rb");
}
if (f) fclose(f);
- if (f && (K.mmod[i] = load_module(K.core.spv_path, mla_file[i]))) K.mpipe[i] = make_pipe(K.mmod[i], NULL);
+ if (f && (KC.mmod[i] = load_module(KC.core.spv_path, mla_file[i]))) KC.mpipe[i] = make_pipe(KC.mmod[i], NULL);
}
- K.mla_ok = K.mpipe[PM_MLA] && K.mpipe[PM_HGEMV] && K.mpipe[PM_DSA];
+ KC.mla_ok = KC.mpipe[PM_MLA] && KC.mpipe[PM_HGEMV] && KC.mpipe[PM_DSA];
dsv4_init();
/* the fp32 tiled GEMM at the backend's tiles; none = every S on the GEMV */
- K.mod_gemm = K.core.gemm_tiles ? load_module(K.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;
- for (int k = 0; K.mod_gemm && k < K.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {
- const int *t = K.core.gemm_tile[k];
+ KC.mod_gemm = KC.core.gemm_tiles ? load_module(KC.core.spv_path, "qmatmul_gemm.spv") : VK_NULL_HANDLE;
+ for (int k = 0; KC.mod_gemm && k < KC.core.gemm_tiles && k < VKC_GEMM_MAX; k++) {
+ const int *t = KC.core.gemm_tile[k];
int32_t sv[7] = {t[0], t[1], t[2], t[3], t[4], (t[0] / t[3]) * (t[1] / t[4]), t[5] ? 1 : 0};
VkSpecializationMapEntry me[7];
for (int i = 0; i < 7; i++) me[i] = (VkSpecializationMapEntry){(uint32_t)i, (uint32_t)(i * 4), 4};
VkSpecializationInfo si = {7, me, sizeof sv, sv};
- if (!(K.gemm[k] = make_pipe(K.mod_gemm, &si))) break;
- K.gemm_bm[k] = t[0]; K.gemm_bn[k] = t[1]; K.ngemm = k + 1;
+ if (!(KC.gemm[k] = make_pipe(KC.mod_gemm, &si))) break;
+ KC.gemm_bm[k] = t[0]; KC.gemm_bn[k] = t[1]; KC.ngemm = k + 1;
}
VkCommandPoolCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
- .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = K.core.qfam};
- if (vkCreateCommandPool(K.dev, &cp, NULL, &K.cpool) != VK_SUCCESS) return 0;
+ .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = KC.core.qfam};
+ if (vkCreateCommandPool(KC.dev, &cp, NULL, &KC.cpool) != VK_SUCCESS) return 0;
for (int i = 0; i < VKC_FRAMES; i++) {
VkCommandBufferAllocateInfo ca = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
- .commandPool = K.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
+ .commandPool = KC.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
- if (vkAllocateCommandBuffers(K.dev, &ca, &K.fr[i].cmd) != VK_SUCCESS ||
- vkCreateFence(K.dev, &fi, NULL, &K.fr[i].fence) != VK_SUCCESS) return 0;
+ if (vkAllocateCommandBuffers(KC.dev, &ca, &KC.fr[i].cmd) != VK_SUCCESS ||
+ vkCreateFence(KC.dev, &fi, NULL, &KC.fr[i].fence) != VK_SUCCESS) return 0;
}
{
const char *e = getenv("COLI_VK_CHAIN_PROF");
- uint32_t nqf = 0; vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)K.core.phys, &nqf, NULL);
+ uint32_t nqf = 0; vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)KC.core.phys, &nqf, NULL);
VkQueueFamilyProperties qf[16]; if (nqf > 16) nqf = 16;
- vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)K.core.phys, &nqf, qf);
- VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);
- uint32_t bits = K.core.qfam < nqf ? qf[K.core.qfam].timestampValidBits : 0;
+ vkGetPhysicalDeviceQueueFamilyProperties((VkPhysicalDevice)KC.core.phys, &nqf, qf);
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)KC.core.phys, &pp);
+ uint32_t bits = KC.core.qfam < nqf ? qf[KC.core.qfam].timestampValidBits : 0;
if (e && *e == '1' && bits && pp.limits.timestampPeriod > 0) {
- K.prof = 1; K.ts_period = pp.limits.timestampPeriod;
- K.ts_mask = bits >= 32 ? 0xffffffffu : (1u << bits) - 1;
+ KC.prof = 1; KC.ts_period = pp.limits.timestampPeriod;
+ KC.ts_mask = bits >= 32 ? 0xffffffffu : (1u << bits) - 1;
for (int i = 0; i < VKC_FRAMES; i++) {
VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
.queryType = VK_QUERY_TYPE_TIMESTAMP, .queryCount = VKC_PROF_Q};
- if (vkCreateQueryPool(K.dev, &qi, NULL, &K.fr[i].qp) != VK_SUCCESS) K.prof = 0;
+ if (vkCreateQueryPool(KC.dev, &qi, NULL, &KC.fr[i].qp) != VK_SUCCESS) KC.prof = 0;
}
}
}
for (int k = 0; k < 3; k++) {
- K.pool[k].memtype = memtype_of(k);
- K.pool[k].mapped = memtype_host_visible(K.pool[k].memtype);
- vka_pool_init(&K.pool[k].p, coli_vk_block_bytes(VKC_BLOCK), 0); /* smaller under COLI_VK_DEVICE_CAP_MB */
+ KC.pool[k].memtype = memtype_of(k);
+ KC.pool[k].mapped = memtype_host_visible(KC.pool[k].memtype);
+ vka_pool_init(&KC.pool[k].p, coli_vk_block_bytes_dev(g_kd, VKC_BLOCK), 0); /* smaller under COLI_VK_DEVICE_CAP_MB */
}
- if (!K.pool[VKC_UP].mapped || !K.pool[VKC_DOWN].mapped) return 0;
- K.ready = 1;
- K.dummy = vkc_buf(256, VKC_DEV);
- if (!K.dummy) { K.ready = 0; return 0; }
+ if (!KC.pool[VKC_UP].mapped || !KC.pool[VKC_DOWN].mapped) return 0;
+ KC.ready = 1;
+ KC.dummy = vkc_buf(256, VKC_DEV);
+ if (!KC.dummy) { KC.ready = 0; return 0; }
return 1;
}
@@ -373,7 +395,7 @@ static void frame_reclaim(VkcFrame *f) {
for (int i = 0; i < f->ntmp; i++) buf_release(f->tmp[i]);
f->ntmp = 0;
f->stage_used = 0;
- for (int i = 0; i < f->npools; i++) vkResetDescriptorPool(K.dev, f->pools[i], 0);
+ for (int i = 0; i < f->npools; i++) vkResetDescriptorPool(KC.dev, f->pools[i], 0);
f->cur_pool = 0;
}
/* A frame is milliseconds of work and the thread waiting on it has nothing else to do:
@@ -389,18 +411,18 @@ static int frame_wait(VkcFrame *f) {
double t0 = vkc_now_ms();
VkResult r = VK_NOT_READY;
for (long spin = chain_spin_us(); spin > 0 && r == VK_NOT_READY && (vkc_now_ms() - t0) * 1000.0 < spin; )
- r = vkGetFenceStatus(K.dev, f->fence);
- if (r == VK_NOT_READY) r = vkWaitForFences(K.dev, 1, &f->fence, VK_TRUE, 20000000000ULL);
- K.st.wait_ms += vkc_now_ms() - t0; K.st.waits++;
+ r = vkGetFenceStatus(KC.dev, f->fence);
+ if (r == VK_NOT_READY) r = vkWaitForFences(KC.dev, 1, &f->fence, VK_TRUE, 20000000000ULL);
+ KC.st.wait_ms += vkc_now_ms() - t0; KC.st.waits++;
f->inflight = 0;
if (r != VK_SUCCESS) { lose("fence wait", r); return 0; }
- if (K.prof && f->qp && f->nq > 1) {
+ if (KC.prof && f->qp && f->nq > 1) {
uint64_t ts[VKC_PROF_Q];
- if (vkGetQueryPoolResults(K.dev, f->qp, 0, (uint32_t)f->nq, sizeof ts, ts, sizeof ts[0],
+ if (vkGetQueryPoolResults(KC.dev, f->qp, 0, (uint32_t)f->nq, sizeof ts, ts, sizeof ts[0],
VK_QUERY_RESULT_64_BIT) == VK_SUCCESS)
for (int i = 1; i < f->nq; i++) {
- uint64_t d = (ts[i] - ts[i - 1]) & (K.ts_mask == 0xffffffffu ? ~0ull : (uint64_t)K.ts_mask);
- K.prof_ms[f->kind[i]] += (double)d * K.ts_period / 1e6; K.prof_n[f->kind[i]]++;
+ uint64_t d = (ts[i] - ts[i - 1]) & (KC.ts_mask == 0xffffffffu ? ~0ull : (uint64_t)KC.ts_mask);
+ KC.prof_ms[f->kind[i]] += (double)d * KC.ts_period / 1e6; KC.prof_n[f->kind[i]]++;
}
f->nq = 0;
}
@@ -408,15 +430,15 @@ static int frame_wait(VkcFrame *f) {
return 1;
}
static void prof_mark(VkcFrame *f, int kind) {
- if (!K.prof || !f->qp || f->nq >= VKC_PROF_Q) return;
+ if (!KC.prof || !f->qp || f->nq >= VKC_PROF_Q) return;
vkCmdWriteTimestamp(f->cmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, f->qp, (uint32_t)f->nq);
f->kind[f->nq++] = (unsigned char)kind;
}
int vkc_finish(void) {
- if (!K.ready) return 0;
- if (K.cur >= 0 && !vkc_submit(0)) return 0;
- for (int i = 0; i < VKC_FRAMES; i++) if (!frame_wait(&K.fr[i])) return 0;
- return !K.lost;
+ if (!KC.ready) return 0;
+ if (KC.cur >= 0 && !vkc_submit(0)) return 0;
+ for (int i = 0; i < VKC_FRAMES; i++) if (!frame_wait(&KC.fr[i])) return 0;
+ return !KC.lost;
}
static void barrier_now(VkCommandBuffer c, VkPipelineStageFlags dst_stage, VkAccessFlags dst) {
@@ -424,7 +446,7 @@ static void barrier_now(VkCommandBuffer c, VkPipelineStageFlags dst_stage, VkAcc
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = dst};
vkCmdPipelineBarrier(c, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, dst_stage,
0, 1, &mb, 0, NULL, 0, NULL);
- K.st.barriers++;
+ KC.st.barriers++;
}
static const VkAccessFlags ALL_RW = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
@@ -433,25 +455,25 @@ static const VkAccessFlags ALL_RW = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER
int vkc_begin(void) {
/* the backend found the device lost (a staged upload's fence): this frame fails as a
* lost device's would, and the engine takes over on the CPU */
- if (K.ready && !K.lost && !coli_vk_available()) lose("the backend's device", VK_ERROR_DEVICE_LOST);
+ if (KC.ready && !KC.lost && !coli_vk_available_dev(g_kd)) lose("the backend's device", VK_ERROR_DEVICE_LOST);
if (!vkc_ready()) return 0;
- if (K.cur >= 0) return 1; /* already open */
- int i = (int)(K.serial % VKC_FRAMES);
- VkcFrame *f = &K.fr[i];
+ if (KC.cur >= 0) return 1; /* already open */
+ int i = (int)(KC.serial % VKC_FRAMES);
+ VkcFrame *f = &KC.fr[i];
if (!frame_wait(f)) return 0;
frame_reclaim(f);
if (vkResetCommandBuffer(f->cmd, 0) != VK_SUCCESS) { lose("command buffer reset", VK_ERROR_DEVICE_LOST); return 0; }
VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
vkBeginCommandBuffer(f->cmd, &bi);
- f->serial = ++K.serial;
+ f->serial = ++KC.serial;
f->open = 1;
- K.cur = i;
- K.bound = VK_NULL_HANDLE;
+ KC.cur = i;
+ KC.bound = VK_NULL_HANDLE;
/* after everything submitted earlier on this queue (and the host's writes) */
barrier_now(f->cmd, ALL_STAGES, ALL_RW);
track_reset();
- if (K.prof && f->qp) {
+ if (KC.prof && f->qp) {
vkCmdResetQueryPool(f->cmd, f->qp, 0, VKC_PROF_Q);
f->nq = 0;
vkCmdWriteTimestamp(f->cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, f->qp, 0);
@@ -460,51 +482,51 @@ int vkc_begin(void) {
return 1;
}
int vkc_submit(int wait) {
- if (K.cur < 0) return vkc_ready();
- VkcFrame *f = &K.fr[K.cur];
+ if (KC.cur < 0) return vkc_ready();
+ VkcFrame *f = &KC.fr[KC.cur];
barrier_now(f->cmd, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_HOST_READ_BIT);
vkEndCommandBuffer(f->cmd);
- f->open = 0; K.cur = -1;
- if (K.lost) return 0;
+ f->open = 0; KC.cur = -1;
+ if (KC.lost) return 0;
{ /* COLI_VK_CHAIN_FAULT=n (tests): the n-th submission fails as a lost device would */
static long fault = -2;
if (fault == -2) { const char *e = getenv("COLI_VK_CHAIN_FAULT"); fault = e && *e ? atol(e) : -1; }
- if (fault > 0 && (long)K.st.frames + 1 >= fault) { lose("queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }
+ if (fault > 0 && (long)KC.st.frames + 1 >= fault) { lose("queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }
}
- vkResetFences(K.dev, 1, &f->fence);
+ vkResetFences(KC.dev, 1, &f->fence);
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &f->cmd};
- VkResult r = (VkResult)coli_vk_queue_submit(K.queue, &si, f->fence); /* the backend's lock, staged uploads */
+ VkResult r = (VkResult)coli_vk_queue_submit_dev(g_kd, KC.queue, &si, f->fence); /* the backend's lock, staged uploads */
if (r != VK_SUCCESS) { lose("queue submit", r); return 0; }
f->inflight = 1;
- K.st.frames++;
+ KC.st.frames++;
return wait ? frame_wait(f) : 1;
}
/* ---- recording helpers ----------------------------------------------------------- */
-static VkcFrame *open_frame(void) { return K.cur >= 0 ? &K.fr[K.cur] : NULL; }
+static VkcFrame *open_frame(void) { return KC.cur >= 0 ? &KC.fr[KC.cur] : NULL; }
static int in_set(const VkBuffer *s, int n, VkBuffer b) { for (int i = 0; i < n; i++) if (s[i] == b) return 1; return 0; }
/* a barrier before an op reading rd[] and writing wr[] when it conflicts with the ops
* recorded since the last one */
static void hazard(VkcFrame *f, const VkBuffer *rd, int nr, const VkBuffer *wr, int nw) {
int need = 0;
- for (int i = 0; i < nr && !need; i++) need = in_set(K.wr, K.nwr, rd[i]);
- for (int i = 0; i < nw && !need; i++) need = in_set(K.wr, K.nwr, wr[i]) || in_set(K.rd, K.nrd, wr[i]);
- if (need || K.nwr + nw > VKC_TRACK || K.nrd + nr > VKC_TRACK) {
+ for (int i = 0; i < nr && !need; i++) need = in_set(KC.wr, KC.nwr, rd[i]);
+ for (int i = 0; i < nw && !need; i++) need = in_set(KC.wr, KC.nwr, wr[i]) || in_set(KC.rd, KC.nrd, wr[i]);
+ if (need || KC.nwr + nw > VKC_TRACK || KC.nrd + nr > VKC_TRACK) {
barrier_now(f->cmd, ALL_STAGES, ALL_RW);
track_reset();
}
- for (int i = 0; i < nr; i++) if (!in_set(K.rd, K.nrd, rd[i])) K.rd[K.nrd++] = rd[i];
- for (int i = 0; i < nw; i++) if (!in_set(K.wr, K.nwr, wr[i])) K.wr[K.nwr++] = wr[i];
+ for (int i = 0; i < nr; i++) if (!in_set(KC.rd, KC.nrd, rd[i])) KC.rd[KC.nrd++] = rd[i];
+ for (int i = 0; i < nw; i++) if (!in_set(KC.wr, KC.nwr, wr[i])) KC.wr[KC.nwr++] = wr[i];
}
static VkDescriptorSet alloc_set(VkcFrame *f) {
for (;;) {
if (f->cur_pool < f->npools) {
VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
- .descriptorPool = f->pools[f->cur_pool], .descriptorSetCount = 1, .pSetLayouts = &K.dsl};
+ .descriptorPool = f->pools[f->cur_pool], .descriptorSetCount = 1, .pSetLayouts = &KC.dsl};
VkDescriptorSet s;
- if (vkAllocateDescriptorSets(K.dev, &da, &s) == VK_SUCCESS) return s;
+ if (vkAllocateDescriptorSets(KC.dev, &da, &s) == VK_SUCCESS) return s;
f->cur_pool++;
continue;
}
@@ -514,7 +536,7 @@ static VkDescriptorSet alloc_set(VkcFrame *f) {
VkDescriptorPool *n = realloc(f->pools, (size_t)(f->npools + 1) * sizeof *n);
if (!n) return VK_NULL_HANDLE;
f->pools = n;
- if (vkCreateDescriptorPool(K.dev, &pc, NULL, &f->pools[f->npools]) != VK_SUCCESS) return VK_NULL_HANDLE;
+ if (vkCreateDescriptorPool(KC.dev, &pc, NULL, &f->pools[f->npools]) != VK_SUCCESS) return VK_NULL_HANDLE;
f->npools++;
}
}
@@ -524,7 +546,8 @@ typedef struct { VkBuffer buf; VkDeviceSize off, range; int written; } VkcBind;
static int record(VkPipeline pipe, const VkcBind *bd, int n, const void *pc, size_t pcb,
uint32_t gx, uint32_t gy, uint32_t gz) {
VkcFrame *f = open_frame();
- if (!f || K.lost || !gx || !gy || !gz) return f && !K.lost;
+ if (KC.misuse) { KC.misuse = 0; return 0; }
+ if (!f || KC.lost || !gx || !gy || !gz) return f && !KC.lost;
VkBuffer rd[VKC_BIND], wr[VKC_BIND]; int nr = 0, nw = 0;
VkDescriptorBufferInfo bi[VKC_BIND];
VkWriteDescriptorSet w[VKC_BIND];
@@ -534,23 +557,28 @@ static int record(VkPipeline pipe, const VkcBind *bd, int n, const void *pc, siz
if (i < n && bd[i].buf) {
bi[i] = (VkDescriptorBufferInfo){bd[i].buf, bd[i].off, bd[i].range ? bd[i].range : VK_WHOLE_SIZE};
if (bd[i].written) wr[nw++] = bd[i].buf; else rd[nr++] = bd[i].buf;
- } else bi[i] = (VkDescriptorBufferInfo){K.dummy->buf, 0, VK_WHOLE_SIZE};
+ } else bi[i] = (VkDescriptorBufferInfo){KC.dummy->buf, 0, VK_WHOLE_SIZE};
w[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = set,
.dstBinding = (uint32_t)i, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &bi[i]};
}
- vkUpdateDescriptorSets(K.dev, VKC_BIND, w, 0, NULL);
+ vkUpdateDescriptorSets(KC.dev, VKC_BIND, w, 0, NULL);
hazard(f, rd, nr, wr, nw);
- if (K.bound != pipe) { vkCmdBindPipeline(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); K.bound = pipe; }
- vkCmdBindDescriptorSets(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, K.pl, 0, 1, &set, 0, NULL);
- if (pcb) vkCmdPushConstants(f->cmd, K.pl, VK_SHADER_STAGE_COMPUTE_BIT, 0, (uint32_t)pcb, pc);
+ if (KC.bound != pipe) { vkCmdBindPipeline(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); KC.bound = pipe; }
+ vkCmdBindDescriptorSets(f->cmd, VK_PIPELINE_BIND_POINT_COMPUTE, KC.pl, 0, 1, &set, 0, NULL);
+ if (pcb) vkCmdPushConstants(f->cmd, KC.pl, VK_SHADER_STAGE_COMPUTE_BIT, 0, (uint32_t)pcb, pc);
vkCmdDispatch(f->cmd, gx, gy, gz);
- K.st.ops++;
- prof_mark(f, K.kind);
+ KC.st.ops++;
+ prof_mark(f, KC.kind);
return 1;
}
static VkcBind B(VkcBuf *b, int written) {
- if (b) b->last_frame = K.serial;
+ if (b && b->d != g_kd) { /* the other device's: the record that binds it fails */
+ if (!KC.misuse) fprintf(stderr, "[VK] chain: a buffer of device %d bound on device %d\n", b->d, g_kd);
+ KC.misuse = 1;
+ return (VkcBind){VK_NULL_HANDLE, 0, 0, written};
+ }
+ if (b) b->last_frame = KC.serial;
return (VkcBind){b ? b->buf : VK_NULL_HANDLE, 0, 0, written};
}
/* groups over x, spilling into y past the 65535 limit */
@@ -563,15 +591,16 @@ static void grid(uint64_t groups, uint32_t *gx, uint32_t *gy) {
/* ---- transfers ------------------------------------------------------------------- */
static int copy_bytes(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t bytes) {
VkcFrame *f = open_frame();
- if (!f || K.lost) return 0;
+ if (!f || KC.lost) return 0;
if (!bytes) return 1;
if (doff + bytes > dst->bytes || soff + bytes > src->bytes) { fprintf(stderr, "[VK] chain: copy out of range\n"); return 0; }
+ if (dst->d != g_kd || src->d != g_kd) { fprintf(stderr, "[VK] chain: a copy across devices\n"); return 0; }
VkBuffer r = src->buf, w = dst->buf;
hazard(f, &r, 1, &w, 1);
VkBufferCopy c = {soff, doff, bytes};
vkCmdCopyBuffer(f->cmd, src->buf, dst->buf, 1, &c);
prof_mark(f, PK_COPY);
- src->last_frame = dst->last_frame = K.serial;
+ src->last_frame = dst->last_frame = KC.serial;
return 1;
}
int vkc_copy(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t n) {
@@ -579,7 +608,7 @@ int vkc_copy(VkcBuf *dst, size_t doff, VkcBuf *src, size_t soff, size_t n) {
}
int vkc_copy_regions(VkcBuf *dst, VkcBuf *src, const VkcRegion *r, int n) {
VkcFrame *f = open_frame();
- if (!f || K.lost) return 0;
+ if (!f || KC.lost) return 0;
if (n < 1) return 1;
VkBufferCopy *c = malloc((size_t)n * sizeof *c);
if (!c) return 0;
@@ -593,31 +622,31 @@ int vkc_copy_regions(VkcBuf *dst, VkcBuf *src, const VkcRegion *r, int n) {
hazard(f, &rb, 1, &wb, 1);
vkCmdCopyBuffer(f->cmd, src->buf, dst->buf, (uint32_t)n, c);
prof_mark(f, PK_COPY);
- src->last_frame = dst->last_frame = K.serial;
+ src->last_frame = dst->last_frame = KC.serial;
free(c);
return 1;
}
int vkc_zero(VkcBuf *dst, size_t off, size_t n) {
VkcFrame *f = open_frame();
- if (!f || K.lost) return 0;
+ if (!f || KC.lost) return 0;
if (!n) return 1;
if ((off + n) * 4 > dst->bytes) return 0;
VkBuffer w = dst->buf;
hazard(f, NULL, 0, &w, 1);
vkCmdFillBuffer(f->cmd, dst->buf, off * 4, n * 4, 0);
- dst->last_frame = K.serial;
+ dst->last_frame = KC.serial;
return 1;
}
static void zero_now(VkcBuf *b) {
/* a device-only buffer starts zeroed too: its own little frame */
- int was_open = K.cur >= 0;
+ int was_open = KC.cur >= 0;
if (!was_open && !vkc_begin()) return;
vkc_zero(b, 0, b->bytes / 4);
if (!was_open) vkc_submit(1);
}
int vkc_write(VkcBuf *dst, size_t off, const void *src, size_t bytes) {
VkcFrame *f = open_frame();
- if (!f || K.lost) return 0;
+ if (!f || KC.lost) return 0;
if (!bytes) return 1;
/* always through the staging, even into host-visible memory: the copy runs in
* frame order, after the ops recorded before it (a direct memcpy would not) */
@@ -641,7 +670,7 @@ int vkc_write(VkcBuf *dst, size_t off, const void *src, size_t bytes) {
}
memcpy(f->stage->ptr + at, src, bytes);
f->stage_used = at + bytes;
- K.st.bytes_up += bytes;
+ KC.st.bytes_up += bytes;
return copy_bytes(dst, off * 4, f->stage, at, bytes);
}
int vkc_read(VkcBuf *src, size_t off, void *dst, size_t bytes) {
@@ -650,38 +679,38 @@ int vkc_read(VkcBuf *src, size_t off, void *dst, size_t bytes) {
if (!vkc_finish()) return 0;
if (src->ptr && src->kind != VKC_UP) { /* the device wrote it into host-readable memory */
memcpy(dst, src->ptr + off * 4, bytes);
- K.st.bytes_down += bytes;
+ KC.st.bytes_down += bytes;
return 1;
}
VkcBuf *rb = vkc_buf(bytes, VKC_DOWN);
if (!rb) return 0;
int ok = vkc_begin() && copy_bytes(rb, 0, src, off * 4, bytes) && vkc_submit(1);
- if (ok) { memcpy(dst, rb->ptr, bytes); K.st.bytes_down += bytes; }
+ if (ok) { memcpy(dst, rb->ptr, bytes); KC.st.bytes_down += bytes; }
vkc_free(rb);
return ok;
}
/* ---- matmul ---------------------------------------------------------------------- */
-void vkc_gemm_rows(int rows) { K.gemm_rows = rows; }
+void vkc_gemm_rows(int rows) { KC.gemm_rows = rows; }
struct VkcPC { int fmt, S, I, O, rowWords, gs; };
static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcBuf *y, size_t yb, int S) {
int path = -1; /* GEMM slot, -1 the GEMV */
- if (K.ngemm && S >= 2) {
- int take = K.gemm_rows < 0 ? (K.core.gemm_min_s && S >= K.core.gemm_min_s &&
- (int64_t)S * ti->O >= K.core.gemm_min_so)
- : (K.gemm_rows > 0 && S >= K.gemm_rows);
- if (take) { path = 0; while (path + 1 < K.ngemm && S > K.gemm_bn[path]) path++; }
+ if (KC.ngemm && S >= 2) {
+ int take = KC.gemm_rows < 0 ? (KC.core.gemm_min_s && S >= KC.core.gemm_min_s &&
+ (int64_t)S * ti->O >= KC.core.gemm_min_so)
+ : (KC.gemm_rows > 0 && S >= KC.gemm_rows);
+ if (take) { path = 0; while (path + 1 < KC.ngemm && S > KC.gemm_bn[path]) path++; }
}
VkcBind bd[4] = {{x->buf, xb, 0, 0}, {(VkBuffer)ti->wbuf, 0, 0, 0}, {(VkBuffer)ti->sbuf, 0, 0, 0},
{y->buf, yb, 0, 1}};
- x->last_frame = y->last_frame = K.serial;
+ x->last_frame = y->last_frame = KC.serial;
struct VkcPC pc = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs};
int ok;
/* the vectorized GEMV where the row is whole 16-byte steps and fits the staging */
int per = ti->fmt == 1 ? 16 : ti->fmt == 4 ? 32 : ti->fmt == 11 || ti->fmt == 14 ? 8 : ti->fmt == 10 ? 4 : 0;
- int v4 = path < 0 && K.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&
- (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= K.gemv4_xs;
- K.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;
+ int v4 = path < 0 && KC.gemv4 && per && ti->rowWords % 4 == 0 && ti->I % 4 == 0 &&
+ (ti->fmt != 4 || (ti->gs > 0 && ti->gs % 32 == 0)) && (ti->rowWords / 4) * per / 4 <= KC.gemv4_xs;
+ KC.kind = path >= 0 ? PK_GEMM : ti->fmt == 1 ? PK_GEMV8 : PK_GEMV;
if (v4) {
/* each workgroup stages x once: give it enough rows that the staging does not
* rival the weights, while keeping enough workgroups to fill the device */
@@ -693,21 +722,21 @@ static int matmul_aligned(const ColiVkTensorInfo *ti, VkcBuf *x, size_t xb, VkcB
struct { int fmt, S, I, O, rowWords, gs, lpr; } pc7 = {ti->fmt, S, ti->I, ti->O, ti->rowWords, ti->gs, lpr};
int rows_wg = 256 / lpr; /* at least, at subgroups of 64 */
int wg = (ti->O + rows_wg - 1) / rows_wg; if (wg > 1024) wg = 1024;
- ok = record(K.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);
+ ok = record(KC.gemv4, bd, 4, &pc7, sizeof pc7, (uint32_t)wg, (uint32_t)S, 1);
}
else if (path >= 0)
- ok = record(K.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + K.gemm_bm[path] - 1) / K.gemm_bm[path]),
- (uint32_t)((S + K.gemm_bn[path] - 1) / K.gemm_bn[path]), 1);
- else ok = record(K.pipe[P_GEMV], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + 7) / 8), (uint32_t)S, 1);
- K.st.matmuls += ok; K.st.gemms += ok && path >= 0;
+ ok = record(KC.gemm[path], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + KC.gemm_bm[path] - 1) / KC.gemm_bm[path]),
+ (uint32_t)((S + KC.gemm_bn[path] - 1) / KC.gemm_bn[path]), 1);
+ else ok = record(KC.pipe[P_GEMV], bd, 4, &pc, sizeof pc, (uint32_t)((ti->O + 7) / 8), (uint32_t)S, 1);
+ KC.st.matmuls += ok; KC.st.gemms += ok && path >= 0;
return ok;
}
int vkc_matmul(ColiVkTensor *t, VkcBuf *x, size_t xo, VkcBuf *y, size_t yo, int S) {
ColiVkTensorInfo ti;
- if (!open_frame() || K.lost || S < 1 || S > 65535 || !coli_vk_tensor_info(t, &ti)) return 0;
+ if (!open_frame() || KC.lost || S < 1 || S > 65535 || !vkc_tensor_info(t, &ti)) return 0;
size_t xb = xo * 4, yb = yo * 4, xn = (size_t)S * ti.I * 4, yn = (size_t)S * ti.O * 4;
if (xb + xn > x->bytes || yb + yn > y->bytes) { fprintf(stderr, "[VK] chain: matmul out of range\n"); return 0; }
- if (xb % K.align == 0 && yb % K.align == 0) return matmul_aligned(&ti, x, xb, y, yb, S);
+ if (xb % KC.align == 0 && yb % KC.align == 0) return matmul_aligned(&ti, x, xb, y, yb, S);
/* an offset the device cannot bind: through temporaries that live with the frame */
VkcFrame *f = open_frame();
VkcBuf *tx = vkc_buf(xn, VKC_DEV), *ty = vkc_buf(yn, VKC_DEV);
@@ -724,16 +753,16 @@ int vkc_matmul(ColiVkTensor *t, VkcBuf *x, size_t xo, VkcBuf *y, size_t yo, int
/* ---- the chain's shaders ----------------------------------------------------------- */
int vkc_norm(VkcBuf *x, VkcBuf *w, VkcBuf *y, const VkcNorm *p) {
- K.kind = PK_NORM;
+ KC.kind = PK_NORM;
VkcBind bd[3] = {B(x, x == y), B(w, 0), B(y, 1)};
uint32_t gx, gy; grid((uint64_t)p->nseg, &gx, &gy);
- return record(K.pipe[P_NORM], bd, 3, p, sizeof *p, gx, gy, 1);
+ return record(KC.pipe[P_NORM], bd, 3, p, sizeof *p, gx, gy, 1);
}
int vkc_rope(VkcBuf *x, VkcBuf *cs, const VkcRope *p) {
- K.kind = PK_ROPE;
+ KC.kind = PK_ROPE;
VkcBind bd[2] = {B(x, 1), B(cs, 0)};
uint32_t gx, gy; grid(((uint64_t)p->nseg * p->half_ + 63) / 64, &gx, &gy);
- return record(K.pipe[P_ROPE], bd, 2, p, sizeof *p, gx, gy, 1);
+ return record(KC.pipe[P_ROPE], bd, 2, p, sizeof *p, gx, gy, 1);
}
int vkc_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf *sel, const VkcAttn *p) {
VkcAttnW w;
@@ -747,17 +776,18 @@ int vkc_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf
* Radeon 780M. From COLI_VK_ATTN_BLOCK rows (16; 0 = never) a call takes chain_attnb,
* one workgroup per KV head and block of rows: each K and V row read once per block.
* Below it, and for decode and an MTP verify, chain_attn's bits as before. */
-static struct { VkShaderModule mod; VkPipeline pipe; int tried, bc; } KAB;
+static struct { VkShaderModule mod; VkPipeline pipe; int tried, bc; } g_kab[2];
+#define KAB (g_kab[g_kd])
static VkPipeline kab_pipe(void) {
if (KAB.pipe || KAB.tried || !vkc_ready()) return KAB.pipe;
KAB.tried = 1;
/* the tile of positions as the device's shared memory allows: K and V rows of 256
* floats, the partial dots and the scores, about 3.2 KiB a position */
- VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)K.core.phys, &pp);
+ VkPhysicalDeviceProperties pp; vkGetPhysicalDeviceProperties((VkPhysicalDevice)KC.core.phys, &pp);
int bc = 16;
while (bc >= 4 && (size_t)bc * (2 * 256 + 32 * 8 + 32 + 1) * 4 > pp.limits.maxComputeSharedMemorySize) bc /= 2;
if (bc < 4) return VK_NULL_HANDLE;
- if ((KAB.mod = load_module(K.core.spv_path, "chain_attnb.spv"))) {
+ if ((KAB.mod = load_module(KC.core.spv_path, "chain_attnb.spv"))) {
int32_t v = bc;
VkSpecializationMapEntry me = {0, 0, 4};
VkSpecializationInfo si = {1, &me, 4, &v};
@@ -766,8 +796,8 @@ static VkPipeline kab_pipe(void) {
return KAB.pipe;
}
static void kab_shutdown(void) {
- if (KAB.pipe) vkDestroyPipeline(K.dev, KAB.pipe, NULL);
- if (KAB.mod) vkDestroyShaderModule(K.dev, KAB.mod, NULL);
+ if (KAB.pipe) vkDestroyPipeline(KC.dev, KAB.pipe, NULL);
+ if (KAB.mod) vkDestroyShaderModule(KC.dev, KAB.mod, NULL);
memset(&KAB, 0, sizeof KAB);
}
/* ---- attention over a prompt chunk, cut over several submissions ----------------
@@ -796,7 +826,7 @@ static int attn_slice_rows(int S, int pos_base, double width, int align) {
return rr >= S ? S : (int)rr;
}
static int attn_slice_next(void) { /* the open frame out, a new one in */
- K.st.attn_slices++;
+ KC.st.attn_slices++;
return vkc_submit(0) && vkc_begin();
}
int vkc_attn_block_rows(void) {
@@ -806,7 +836,7 @@ int vkc_attn_block_rows(void) {
}
int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBuf *sel, VkcBuf *snk,
const VkcAttnW *p) {
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcAttnW w = *p; /* the shader's push constants: VkcAttn's fields, then these */
if (w.vd <= 0) w.vd = w.a.hd;
if (w.a.hd > 256 || w.vd > 256 || w.a.H % w.a.KVH || w.win < 0 || w.ring < 0 || (w.sink && !snk)) return 0;
@@ -825,32 +855,32 @@ int vkc_attn_w(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *gate, VkcBu
if (blocked) {
struct { VkcAttnW w; int br; } pc = {x, br};
ok = record(KAB.pipe, bd, 7, &pc, sizeof pc, (uint32_t)x.a.KVH, (uint32_t)((x.a.S + br - 1) / br), 1);
- K.st.attn_blocked += ok;
- } else ok = record(K.pipe[P_ATTN], bd, 7, &x, sizeof x, (uint32_t)x.a.H, (uint32_t)x.a.S, 1);
+ KC.st.attn_blocked += ok;
+ } else ok = record(KC.pipe[P_ATTN], bd, 7, &x, sizeof x, (uint32_t)x.a.H, (uint32_t)x.a.S, 1);
}
return ok;
}
int vkc_dnconv(VkcBuf *in, VkcBuf *w, VkcBuf *ring, VkcBuf *out, VkcBuf *snap, const VkcDnConv *p) {
- K.kind = PK_DNCONV;
+ KC.kind = PK_DNCONV;
if (p->CK < 1 || p->CK > 9) return 0;
VkcBind bd[5] = {B(in, 0), B(w, 0), B(ring, 1), B(out, 1), B(snap, snap != NULL && snap != ring)};
if (snap == ring) bd[4].written = 0; /* the same buffer: tracked once, as written */
uint32_t gx, gy; grid(((uint64_t)p->CD + 63) / 64, &gx, &gy);
- return record(K.pipe[P_DNCONV], bd, 5, p, sizeof *p, gx, gy, 1);
+ return record(KC.pipe[P_DNCONV], bd, 5, p, sizeof *p, gx, gy, 1);
}
static VkPipeline dnrec_pipe(int KD) {
- for (int i = 0; i < K.ndnrec; i++) if (K.dnrec_kd[i] == KD) return K.dnrec[i];
- if (K.ndnrec == VKC_DNREC_MAX) return VK_NULL_HANDLE;
+ for (int i = 0; i < KC.ndnrec; i++) if (KC.dnrec_kd[i] == KD) return KC.dnrec[i];
+ if (KC.ndnrec == VKC_DNREC_MAX) return VK_NULL_HANDLE;
int32_t kd = KD;
VkSpecializationMapEntry me = {0, 0, 4};
VkSpecializationInfo si = {1, &me, 4, &kd};
- VkPipeline p = make_pipe(K.mod_dnrec, &si);
+ VkPipeline p = make_pipe(KC.mod_dnrec, &si);
if (!p) return VK_NULL_HANDLE;
- K.dnrec[K.ndnrec] = p; K.dnrec_kd[K.ndnrec++] = KD;
+ KC.dnrec[KC.ndnrec] = p; KC.dnrec_kd[KC.ndnrec++] = KD;
return p;
}
int vkc_dnrec(int KD, VkcBuf *cv, VkcBuf *ab, VkcBuf *z, VkcBuf *st, VkcBuf *prm, VkcBuf *y, VkcBuf *snap, const VkcDnRec *p) {
- K.kind = PK_DNREC;
+ KC.kind = PK_DNREC;
if (p->VD > 128 || KD < 1 || KD > 256 || p->VH % p->KH) return 0;
VkPipeline pipe = dnrec_pipe(KD);
if (!pipe) return 0;
@@ -859,7 +889,7 @@ int vkc_dnrec(int KD, VkcBuf *cv, VkcBuf *ab, VkcBuf *z, VkcBuf *st, VkcBuf *prm
return record(pipe, bd, 7, p, sizeof *p, (uint32_t)p->VH, 1, 1);
}
int vkc_ew(VkcBuf *y, VkcBuf *a, VkcBuf *b, VkcBuf *c, VkcBuf *e, const VkcEw *p) {
- K.kind = PK_EW;
+ KC.kind = PK_EW;
VkcBind bd[5] = {B(y, 1), B(a == y ? NULL : a, 0), B(b == y ? NULL : b, 0), B(c == y ? NULL : c, 0), B(e == y ? NULL : e, 0)};
/* an input that is y itself reads through binding 0's alias: bind it there too */
if (a == y) bd[1] = (VkcBind){y->buf, 0, 0, 0};
@@ -867,38 +897,38 @@ int vkc_ew(VkcBuf *y, VkcBuf *a, VkcBuf *b, VkcBuf *c, VkcBuf *e, const VkcEw *p
if (c == y) bd[3] = (VkcBind){y->buf, 0, 0, 0};
if (e == y) bd[4] = (VkcBind){y->buf, 0, 0, 0};
uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);
- return record(K.pipe[P_EW], bd, 5, p, sizeof *p, gx, gy, 1);
+ return record(KC.pipe[P_EW], bd, 5, p, sizeof *p, gx, gy, 1);
}
int vkc_qsa(VkcBuf *src, VkcBuf *w, VkcBuf *pk, VkcBuf *cs, VkcBuf *sc, VkcBuf *sel, const VkcQsa *p) {
- K.kind = PK_QSA;
+ KC.kind = PK_QSA;
if (p->ID > 256) return 0;
if (p->mode == 0) {
VkcBind bd[4] = {B(src, 0), B(w, 0), B(pk, 1), B(cs, 0)};
- return record(K.pipe[P_QSA], bd, 4, p, sizeof *p, (uint32_t)p->nb, 1, 1);
+ return record(KC.pipe[P_QSA], bd, 4, p, sizeof *p, (uint32_t)p->nb, 1, 1);
}
VkcBind bd[6] = {B(src, 0), {VK_NULL_HANDLE, 0, 0, 0}, B(pk, 0), {VK_NULL_HANDLE, 0, 0, 0}, B(sc, 1), B(sel, 1)};
- return record(K.pipe[P_QSA], bd, 6, p, sizeof *p, (uint32_t)p->S, 1, 1);
+ return record(KC.pipe[P_QSA], bd, 6, p, sizeof *p, (uint32_t)p->S, 1, 1);
}
int vkc_ple(VkcBuf *keys, VkcBuf *hyp, VkcBuf *val, VkcBuf *prm, VkcBuf *gated, VkcBuf *normv,
VkcBuf *conv, VkcBuf *ring, const VkcPle *p) {
- K.kind = PK_PLE;
+ KC.kind = PK_PLE;
if (p->mode == 0) {
VkcBind bd[6] = {B(keys, 0), B(hyp, 0), B(val, 0), B(prm, 0), B(gated, 1), B(normv, 1)};
- return record(K.pipe[P_PLE], bd, 6, p, sizeof *p, (uint32_t)(p->S * p->C), 1, 1);
+ return record(KC.pipe[P_PLE], bd, 6, p, sizeof *p, (uint32_t)(p->S * p->C), 1, 1);
}
if ((p->CK - 1) * p->NG > 32) return 0;
VkcBind bd[8] = {{VK_NULL_HANDLE, 0, 0, 0}, B(hyp, 1), {VK_NULL_HANDLE, 0, 0, 0}, {VK_NULL_HANDLE, 0, 0, 0},
B(gated, 0), B(normv, 0), B(conv, 0), B(ring, 1)};
uint32_t gx, gy; grid(((uint64_t)p->C * p->H + 255) / 256, &gx, &gy);
- return record(K.pipe[P_PLE], bd, 8, p, sizeof *p, gx, gy, 1);
+ return record(KC.pipe[P_PLE], bd, 8, p, sizeof *p, gx, gy, 1);
}
/* ---- multi-head latent attention ------------------------------------------------- */
-int vkc_mla_ready(void) { return vkc_ready() && K.mla_ok; }
+int vkc_mla_ready(void) { return vkc_ready() && KC.mla_ok; }
int vkc_mla_core(VkcBuf *qabs, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sel, VkcBuf *clat, const VkcMlaCore *p) {
- K.kind = PK_MLA;
- if (!K.mla_ok || p->S < 1 || p->H < 1 || p->K < 1 || p->K > 1024 || p->R < 0 || p->R > 128 ||
+ KC.kind = PK_MLA;
+ if (!KC.mla_ok || p->S < 1 || p->H < 1 || p->K < 1 || p->K > 1024 || p->R < 0 || p->R > 128 ||
(p->R > 0 && !rope) || p->pos_base < 0 || p->kv_start < 0 || p->kv_start > p->pos_base) return 0;
typedef struct { int mode; VkcMlaCore b; } CorePC;
CorePC pc = {0, *p};
@@ -913,27 +943,27 @@ int vkc_mla_core(VkcBuf *qabs, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *se
x.b.qa_off += r0 * p->qa_row; x.b.qr_off += r0 * p->qr_row; x.b.o_off += r0 * p->o_row;
if (pc.b.sel_row > 0) x.b.sel_off += r0 * pc.b.sel_row;
if (r0 && !attn_slice_next()) return 0;
- ok = record(K.mpipe[PM_MLA], bd, 6, &x, sizeof x, (uint32_t)p->H, (uint32_t)x.b.S, 1);
+ ok = record(KC.mpipe[PM_MLA], bd, 6, &x, sizeof x, (uint32_t)p->H, (uint32_t)x.b.S, 1);
}
return ok;
}
static int mla_rowop(int mode, VkcBuf *x, VkcBuf *aux, VkcBuf *y, const VkcMlaRow *p) {
- if (!K.mla_ok || !open_frame() || K.lost || p->nseg < 0) return 0;
+ if (!KC.mla_ok || !open_frame() || KC.lost || p->nseg < 0) return 0;
if (p->nseg == 0) return 1;
if (mode == 1 && (p->rd < 2 || p->rd > 256 || (p->rd & 1) || p->rd > p->seg_len)) return 0;
if (mode == 2 && p->seg_len < 1) return 0;
struct { int mode; VkcMlaRow b; int copy_rest; } pc = {mode, *p, x != y};
VkcBind bd[6] = {B(x, x == y), B(aux, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};
uint32_t gx, gy; grid((uint64_t)p->nseg, &gx, &gy);
- return record(K.mpipe[PM_MLA], bd, 6, &pc, sizeof pc, gx, gy, 1);
+ return record(KC.mpipe[PM_MLA], bd, 6, &pc, sizeof pc, gx, gy, 1);
}
-int vkc_mla_rope(VkcBuf *x, VkcBuf *cs, VkcBuf *y, const VkcMlaRow *p) { K.kind = PK_ROPE; return mla_rowop(1, x, cs, y, p); }
-int vkc_mla_lnorm(VkcBuf *x, VkcBuf *prm, VkcBuf *y, const VkcMlaRow *p) { K.kind = PK_NORM; return mla_rowop(2, x, prm, y, p); }
+int vkc_mla_rope(VkcBuf *x, VkcBuf *cs, VkcBuf *y, const VkcMlaRow *p) { KC.kind = PK_ROPE; return mla_rowop(1, x, cs, y, p); }
+int vkc_mla_lnorm(VkcBuf *x, VkcBuf *prm, VkcBuf *y, const VkcMlaRow *p) { KC.kind = PK_NORM; return mla_rowop(2, x, prm, y, p); }
int vkc_mla_hgemv(ColiVkTensor *t, VkcBuf *x, VkcBuf *y, VkcBuf *gate, const VkcHgemv *p) {
- K.kind = PK_MLAW;
+ KC.kind = PK_MLAW;
ColiVkTensorInfo ti;
- if (!K.mla_ok || !open_frame() || K.lost || !coli_vk_tensor_info(t, &ti) || p->S < 1 || p->S > 65535 ||
+ if (!KC.mla_ok || !open_frame() || KC.lost || !vkc_tensor_info(t, &ti) || p->S < 1 || p->S > 65535 ||
p->H < 1 || p->H > 65535 || p->n < 1 || (p->trans && p->n > 1024)) return 0;
int f = ti.fmt;
if (!(f == 1 || f == 2 || f == 4 || f == 5 || f == 7 || f == 10 || f == 11 || f == 12 || f == 13)) return 0;
@@ -943,61 +973,61 @@ int vkc_mla_hgemv(ColiVkTensor *t, VkcBuf *x, VkcBuf *y, VkcBuf *gate, const Vkc
g_off, g_row, has_gate; } pc = {p->trans, f, ti.I, ti.rowWords, ti.gs, p->S, p->H, p->n, p->hstride, p->hoff,
p->x_off, p->x_row, p->x_seg, p->y_off, p->y_row, p->y_seg, p->g_off, p->g_row, gate && p->has_gate};
VkcBind bd[5] = {B(x, 0), {(VkBuffer)ti.wbuf, 0, 0, 0}, {(VkBuffer)ti.sbuf, 0, 0, 0}, B(y, 1), B(pc.has_gate ? gate : NULL, 0)};
- if (p->trans) return record(K.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);
+ if (p->trans) return record(KC.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);
/* rows: a subgroup each, grid-stride (at least four subgroups a workgroup) */
int64_t rows = (int64_t)p->H * p->n, wg = (rows + 7) / 8;
if (wg > 4096) wg = 4096;
- return record(K.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)wg, (uint32_t)p->S, 1);
+ return record(KC.mpipe[PM_HGEMV], bd, 5, &pc, sizeof pc, (uint32_t)wg, (uint32_t)p->S, 1);
}
int vkc_dsa_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *sc, VkcBuf *sel, const VkcDsa *p) {
- K.kind = PK_DSA;
- if (!K.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||
+ KC.kind = PK_DSA;
+ if (!KC.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||
p->topk < 1 || p->pos_base < 0 || p->sc_row < p->pos_base + p->S ||
p->sel_row < 1 + (p->topk < p->pos_base + p->S ? p->topk : p->pos_base + p->S)) return 0;
struct { int mode; VkcDsa b; } pc = {0, *p};
VkcBind bd[5] = {B(iq, 0), B(hw, 0), B(keys, 0), B(sc, 1), B(sel, 1)};
- return record(K.mpipe[PM_DSA], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
+ return record(KC.mpipe[PM_DSA], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
}
/* k-pooling (chain_dsa.comp modes 1 and 2) */
int vkc_dsa_pool_keys(VkcBuf *keys, VkcBuf *gates, VkcBuf *prm, VkcBuf *pk, const VkcDsaPool *p) {
- K.kind = PK_DSA;
- if (!K.mla_ok || p->np < 0 || p->pool < 1 || p->p0 < 0 || p->ID < 1) return 0;
- if (p->np == 0) return open_frame() && !K.lost;
+ KC.kind = PK_DSA;
+ if (!KC.mla_ok || p->np < 0 || p->pool < 1 || p->p0 < 0 || p->ID < 1) return 0;
+ if (p->np == 0) return open_frame() && !KC.lost;
struct { int mode; VkcDsaPool b; } pc = {1, *p};
VkcBind bd[8] = {B(NULL, 0), B(NULL, 0), B(keys, 0), B(NULL, 0), B(NULL, 0), B(pk, 1), B(gates, 0), B(prm, 0)};
- return record(K.mpipe[PM_DSA], bd, 8, &pc, sizeof pc, (uint32_t)p->np, 1, 1);
+ return record(KC.mpipe[PM_DSA], bd, 8, &pc, sizeof pc, (uint32_t)p->np, 1, 1);
}
int vkc_dsa_pool_select(VkcBuf *iq, VkcBuf *hw, VkcBuf *pk, VkcBuf *sc, VkcBuf *sel, const VkcDsaPick *p) {
- K.kind = PK_DSA;
- if (!K.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||
+ KC.kind = PK_DSA;
+ if (!KC.mla_ok || p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 64 || p->ID < 1 || p->IH * p->ID > 4096 ||
p->pool < 1 || p->topk < p->pool || p->topk % p->pool || p->topk / p->pool > 1024 || p->pos_base < 0 ||
p->sc_row < (p->pos_base + p->S) / p->pool || p->sel_row < 1 + p->topk + (p->tail ? p->pool - 1 : 0)) return 0;
struct { int mode; VkcDsaPick b; } pc = {2, *p};
VkcBind bd[6] = {B(iq, 0), B(hw, 0), B(NULL, 0), B(sc, 1), B(sel, 1), B(pk, 0)};
- return record(K.mpipe[PM_DSA], bd, 6, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
+ return record(KC.mpipe[PM_DSA], bd, 6, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
}
/* ---- Kimi Delta Attention (chain_kda.comp) ------------------------------------------ */
-int vkc_kda_ready(void) { return vkc_ready() && K.mpipe[PM_KDA]; }
+int vkc_kda_ready(void) { return vkc_ready() && KC.mpipe[PM_KDA]; }
int vkc_kda_conv(VkcBuf *in, VkcBuf *w, VkcBuf *win, VkcBuf *out, const VkcKdaConv *p) {
- K.kind = PK_KDA;
- if (!K.mpipe[PM_KDA] || p->K < 1 || p->K > 8 || p->P < 1 || p->C < 1 || p->S < 1) return 0;
+ KC.kind = PK_KDA;
+ if (!KC.mpipe[PM_KDA] || p->K < 1 || p->K > 8 || p->P < 1 || p->C < 1 || p->S < 1) return 0;
struct { int mode; VkcKdaConv b; } pc = {0, *p};
VkcBind bd[4] = {B(in, 0), B(w, 0), B(win, 1), B(out, 1)};
uint32_t gx, gy; grid(((uint64_t)p->C + 127) / 128, &gx, &gy);
- return record(K.mpipe[PM_KDA], bd, 4, &pc, sizeof pc, gx, gy, 1);
+ return record(KC.mpipe[PM_KDA], bd, 4, &pc, sizeof pc, gx, gy, 1);
}
static VkPipeline kda_pipe(int KD) {
- for (int i = 0; i < K.nkdarec; i++) if (K.kdarec_kd[i] == KD) return K.kdarec[i];
- if (K.nkdarec == VKC_KDA_MAX || !K.mmod[PM_KDA]) return VK_NULL_HANDLE;
+ for (int i = 0; i < KC.nkdarec; i++) if (KC.kdarec_kd[i] == KD) return KC.kdarec[i];
+ if (KC.nkdarec == VKC_KDA_MAX || !KC.mmod[PM_KDA]) return VK_NULL_HANDLE;
int32_t kd = KD;
VkSpecializationMapEntry me = {0, 0, 4};
VkSpecializationInfo si = {1, &me, 4, &kd};
- VkPipeline p = make_pipe(K.mmod[PM_KDA], &si);
+ VkPipeline p = make_pipe(KC.mmod[PM_KDA], &si);
if (!p) return VK_NULL_HANDLE;
- K.kdarec[K.nkdarec] = p; K.kdarec_kd[K.nkdarec++] = KD;
+ KC.kdarec[KC.nkdarec] = p; KC.kdarec_kd[KC.nkdarec++] = KD;
return p;
}
int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y, const VkcKdaRec *p) {
@@ -1005,8 +1035,8 @@ int vkc_kda_rec(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm,
}
int vkc_kda_rec_flags(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf *prm, VkcBuf *st, VkcBuf *y,
const VkcKdaRec *p, int flags) {
- K.kind = PK_KDA;
- if (!K.mpipe[PM_KDA] || KD < 1 || KD > 256 || p->VD < 1 || p->VD > 128 || p->H < 1 || p->S < 1 ||
+ KC.kind = PK_KDA;
+ if (!KC.mpipe[PM_KDA] || KD < 1 || KD > 256 || p->VD < 1 || p->VD > 128 || p->H < 1 || p->S < 1 ||
(flags & ~(VKC_KDA_EXP_A | VKC_KDA_K3))) return 0;
VkPipeline pipe = kda_pipe(KD);
if (!pipe) return 0;
@@ -1016,39 +1046,39 @@ int vkc_kda_rec_flags(int KD, VkcBuf *m, VkcBuf *f, VkcBuf *b, VkcBuf *g, VkcBuf
}
/* ---- manifold-constrained hyper-connections (chain_mhc.comp) ------------------------ */
-int vkc_mhc_ready(void) { return vkc_ready() && K.mpipe[PM_MHC]; }
+int vkc_mhc_ready(void) { return vkc_ready() && KC.mpipe[PM_MHC]; }
int vkc_mhc(int mode, VkcBuf *x, VkcBuf *m, VkcBuf *hp, VkcBuf *prm, VkcBuf *y, const VkcMhc *p) {
- K.kind = PK_MHC;
- if (!K.mpipe[PM_MHC] || mode < 0 || mode > 4 || p->S < 0 || (mode != 4 && (p->H < 1 || p->H > 8 || p->D < 1))) return 0;
+ KC.kind = PK_MHC;
+ if (!KC.mpipe[PM_MHC] || mode < 0 || mode > 4 || p->S < 0 || (mode != 4 && (p->H < 1 || p->H > 8 || p->D < 1))) return 0;
if (mode == 0 && p->iters < 1) return 0;
struct { int mode; VkcMhc b; } pc = {mode, *p};
VkcBind bd[5] = {B(x, 0), B(m, 0), B(hp, mode == 0), B(prm, 0), B(y, mode != 0)};
uint64_t n = mode == 4 ? (uint64_t)p->n : mode == 2 ? (uint64_t)p->S * p->H * p->D : (uint64_t)p->S * p->D;
- if (mode == 0) return p->S == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
+ if (mode == 0) return p->S == 0 || record(KC.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, (uint32_t)p->S, 1, 1);
uint32_t gx, gy; grid((n + 255) / 256, &gx, &gy);
- return n == 0 || record(K.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, gx, gy, 1);
+ return n == 0 || record(KC.mpipe[PM_MHC], bd, 5, &pc, sizeof pc, gx, gy, 1);
}
/* ---- attention residuals and SiTU-GLU (chain_ares.comp) ---------------------------- */
-int vkc_ares_ready(void) { return vkc_ready() && K.mpipe[PM_ARES]; }
+int vkc_ares_ready(void) { return vkc_ready() && KC.mpipe[PM_ARES]; }
int vkc_ares_mix(VkcBuf *x, VkcBuf *blk, VkcBuf *prm, VkcBuf *y, const VkcAres *p) {
- K.kind = PK_ARES;
- if (!K.mpipe[PM_ARES] || p->S < 0 || p->D < 1 || p->nb < 0 || p->nb > 15 || (p->nb > 0 && !blk) || y == x || y == blk) return 0;
- if (p->S == 0) return open_frame() && !K.lost;
+ KC.kind = PK_ARES;
+ if (!KC.mpipe[PM_ARES] || p->S < 0 || p->D < 1 || p->nb < 0 || p->nb > 15 || (p->nb > 0 && !blk) || y == x || y == blk) return 0;
+ if (p->S == 0) return open_frame() && !KC.lost;
struct { int mode; VkcAres b; } pc = {0, *p};
VkcBind bd[4] = {B(x, 0), B(p->nb > 0 ? blk : NULL, 0), B(prm, 0), B(y, 1)};
uint32_t gx, gy; grid((uint64_t)p->S, &gx, &gy);
- return record(K.mpipe[PM_ARES], bd, 4, &pc, sizeof pc, gx, gy, 1);
+ return record(KC.mpipe[PM_ARES], bd, 4, &pc, sizeof pc, gx, gy, 1);
}
int vkc_situ(VkcBuf *g, VkcBuf *u, VkcBuf *y, const VkcSitu *p) {
- K.kind = PK_ARES;
- if (!K.mpipe[PM_ARES] || p->n < 0) return 0;
- if (p->n == 0) return open_frame() && !K.lost;
+ KC.kind = PK_ARES;
+ if (!KC.mpipe[PM_ARES] || p->n < 0) return 0;
+ if (p->n == 0) return open_frame() && !KC.lost;
int32_t pc[13] = {1, p->n, 0, 0, p->g_off, 0, p->u_off, 0, 0, p->y_off, 0, 0, 0};
memcpy(&pc[11], &p->b1, 4); memcpy(&pc[12], &p->b2, 4);
VkcBind bd[4] = {B(g, 0), B(u, 0), B(NULL, 0), B(y, 1)};
uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);
- return record(K.mpipe[PM_ARES], bd, 4, pc, sizeof pc, gx, gy, 1);
+ return record(KC.mpipe[PM_ARES], bd, 4, pc, sizeof pc, gx, gy, 1);
}
int vkc_mla_scratch(VkcMlaScratch *s, const VkcMla *m, int rows) {
@@ -1070,12 +1100,12 @@ void vkc_mla_scratch_free(VkcMlaScratch *s) {
/* the tensor is there and has the shape [O x I] */
static int mla_shape(ColiVkTensor *t, int O, int I) {
ColiVkTensorInfo ti;
- return t && coli_vk_tensor_info(t, &ti) && ti.I == I && ti.O == O;
+ return t && vkc_tensor_info(t, &ti) && ti.I == I && ti.O == O;
}
int vkc_mla_qkv(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int S, int pos_base,
VkcBuf *cs, VkcMlaCache *c, VkcBuf *down, size_t down_off) {
int H = m->H, QR = m->Q + m->R, KR = m->K + m->R;
- if (!K.mla_ok || S < 1 || S > s->rows || pos_base < 0 || (int64_t)pos_base + S > c->cap || m->R < 0 ||
+ if (!KC.mla_ok || S < 1 || S > s->rows || pos_base < 0 || (int64_t)pos_base + S > c->cap || m->R < 0 ||
(m->R & 1) || m->R > 128 || m->K < 1 || m->K > 1024 || (m->R > 0 && (!cs || !c->rope)) ||
!mla_shape(m->q_b, H * QR, m->q_lora > 0 ? m->q_lora : m->D) || !mla_shape(m->kv_a, KR, m->D) ||
(m->q_lora > 0 && !mla_shape(m->q_a, m->q_lora, m->D))) return 0;
@@ -1101,7 +1131,7 @@ int vkc_mla_qkv(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int
int vkc_mla_attn(const VkcMla *m, VkcMlaScratch *s, int S, int pos_base, int kv_start, VkcMlaCache *c,
VkcBuf *sel, size_t sel_off, int sel_row, VkcBuf *gate, size_t gate_off, VkcBuf *out, size_t out_off) {
int H = m->H, QR = m->Q + m->R, HK = H * m->K, HV = H * m->V;
- if (!K.mla_ok || S < 1 || S > s->rows || (int64_t)pos_base + S > c->cap) return 0;
+ if (!KC.mla_ok || S < 1 || S > s->rows || (int64_t)pos_base + S > c->cap) return 0;
if (m->kv_b ? !mla_shape(m->kv_b, H * (m->Q + m->V), m->K)
: !(mla_shape(m->k_abs, HK, m->Q) && mla_shape(m->v_abs, HV, m->K))) return 0;
if (m->o && out && !mla_shape(m->o, m->D, HV)) return 0;
@@ -1128,18 +1158,19 @@ int vkc_mla(const VkcMla *m, VkcMlaScratch *s, VkcBuf *x, size_t x_off, int S, i
vkc_mla_attn(m, s, S, pos_base, kv_start, c, NULL, 0, 0, NULL, 0, out, out_off);
}
/* ---- inkling's ops: chain_sconv.comp and chain_relattn.comp, made on first use ---- */
-static struct { VkShaderModule mod[2]; VkPipeline pipe[2]; int tried[2]; } KX;
+static struct { VkShaderModule mod[2]; VkPipeline pipe[2]; int tried[2]; } g_kx[2];
+#define KX (g_kx[g_kd])
static VkPipeline kx_pipe(int k) {
static const char *const file[2] = {"chain_sconv.spv", "chain_relattn.spv"};
if (KX.pipe[k] || KX.tried[k] || !vkc_ready()) return KX.pipe[k];
KX.tried[k] = 1;
- if ((KX.mod[k] = load_module(K.core.spv_path, file[k]))) KX.pipe[k] = make_pipe(KX.mod[k], NULL);
+ if ((KX.mod[k] = load_module(KC.core.spv_path, file[k]))) KX.pipe[k] = make_pipe(KX.mod[k], NULL);
return KX.pipe[k];
}
static void kx_shutdown(void) {
for (int k = 0; k < 2; k++) {
- if (KX.pipe[k]) vkDestroyPipeline(K.dev, KX.pipe[k], NULL);
- if (KX.mod[k]) vkDestroyShaderModule(K.dev, KX.mod[k], NULL);
+ if (KX.pipe[k]) vkDestroyPipeline(KC.dev, KX.pipe[k], NULL);
+ if (KX.mod[k]) vkDestroyShaderModule(KC.dev, KX.mod[k], NULL);
}
memset(&KX, 0, sizeof KX);
}
@@ -1148,7 +1179,7 @@ int vkc_relattn_ready(void) { return kx_pipe(1) != VK_NULL_HANDLE; }
int vkc_sconv(VkcBuf *x, VkcBuf *w, VkcBuf *ring, const VkcSconv *p) {
VkPipeline pipe = kx_pipe(0);
if (!pipe || (p->mode == 0 && (p->CK < 1 || p->CK > 9))) return 0;
- K.kind = p->mode == 0 ? PK_DNCONV : PK_EW;
+ KC.kind = p->mode == 0 ? PK_DNCONV : PK_EW;
uint32_t gx, gy;
if (p->mode == 0) {
VkcBind bd[3] = {B(x, 1), B(w, 0), B(ring, 1)};
@@ -1163,7 +1194,7 @@ int vkc_relattn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *kvs, VkcBu
const VkcRelAttn *p) {
VkPipeline pipe = kx_pipe(1);
if (!pipe || p->hd > 256 || p->d_rel > 64 || p->d_rel < 0 || p->KVH < 1 || p->H % p->KVH || p->cap < 1) return 0;
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[8] = {B(q, 0), B(kc, 0), B(vc, 0), B(o, 1), B(kvs, 0), B(r, 0), B(relp, 0), B(tau, 0)};
/* a long prompt chunk in slices of rows, a submission each (attn_slice_rows): the
* shader's s0 is the slice's first row, pos_base stays the step's */
@@ -1179,16 +1210,17 @@ int vkc_relattn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *o, VkcBuf *kvs, VkcBu
}
/* ---- the decision engines' encoders: chain_enc.comp, made on first use ---- */
-static struct { VkShaderModule mod; VkPipeline pipe; int tried; } KE;
+static struct { VkShaderModule mod; VkPipeline pipe; int tried; } g_ke[2];
+#define KE (g_ke[g_kd])
static VkPipeline ke_pipe(void) {
if (KE.pipe || KE.tried || !vkc_ready()) return KE.pipe;
KE.tried = 1;
- if ((KE.mod = load_module(K.core.spv_path, "chain_enc.spv"))) KE.pipe = make_pipe(KE.mod, NULL);
+ if ((KE.mod = load_module(KC.core.spv_path, "chain_enc.spv"))) KE.pipe = make_pipe(KE.mod, NULL);
return KE.pipe;
}
static void ke_shutdown(void) {
- if (KE.pipe) vkDestroyPipeline(K.dev, KE.pipe, NULL);
- if (KE.mod) vkDestroyShaderModule(K.dev, KE.mod, NULL);
+ if (KE.pipe) vkDestroyPipeline(KC.dev, KE.pipe, NULL);
+ if (KE.mod) vkDestroyShaderModule(KC.dev, KE.mod, NULL);
memset(&KE, 0, sizeof KE);
}
int vkc_enc_ready(void) { return ke_pipe() != VK_NULL_HANDLE; }
@@ -1196,7 +1228,7 @@ static int ke_ew(int kind, int mode, void *pc, size_t bytes, const VkcBind *bd,
VkPipeline pipe = ke_pipe();
if (!pipe) return 0;
*(int *)pc = mode;
- K.kind = kind;
+ KC.kind = kind;
uint32_t gx, gy; grid((n + 255) / 256, &gx, &gy);
return record(pipe, bd, nb, pc, bytes, gx, gy, 1);
}
@@ -1205,7 +1237,7 @@ int vkc_enc_norm(VkcBuf *a, VkcBuf *b, VkcBuf *prm, VkcBuf *y, const VkcEncNorm
if (!pipe || p->D < 1 || p->D > 4096) return 0;
VkcEncNorm c = *p;
c.mode = 0;
- K.kind = PK_NORM;
+ KC.kind = PK_NORM;
VkcBind bd[4] = {B(a, (p->flags & VKC_ENC_SUM) != 0), B((p->flags & VKC_ENC_ADD) ? b : NULL, 0), B(prm, 0), B(y, 1)};
uint32_t gx, gy; grid((uint64_t)p->rows, &gx, &gy);
return record(pipe, bd, 4, &c, sizeof c, gx, gy, 1);
@@ -1237,7 +1269,7 @@ int vkc_enc_attn(VkcBuf *qkv, VkcBuf *c2p, VkcBuf *p2c, VkcBuf *o, VkcBuf *rng,
if (!pipe || p->hd < 1 || p->hd > 128 || p->S < 1 || p->H < 1) return 0;
VkcEncAttn c = *p;
c.mode = 5;
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[7] = {B(qkv, 0), B(c2p, 0), B(p2c, 0), B(o, 1), B(rng, 0), B(ridx, 0), B(pos, 0)};
return record(pipe, bd, 7, &c, sizeof c, (uint32_t)p->H, (uint32_t)((p->S + 15) / 16), 1);
}
@@ -1246,12 +1278,12 @@ int vkc_enc_rel(VkcBuf *x, VkcBuf *pt, VkcBuf *y, const VkcEncRel *p) {
if (!pipe || p->hd < 1 || p->S < 1 || p->H < 1 || p->nr < 1) return 0;
VkcEncRel c = *p;
c.mode = 6;
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[4] = {B(x, 0), B(pt, 0), B(NULL, 0), B(y, 1)};
return record(pipe, bd, 4, &c, sizeof c, (uint32_t)((p->nr + 63) / 64), (uint32_t)((p->S + 15) / 16), (uint32_t)p->H);
}
-void vkc_stats(VkcStats *st) { *st = K.st; }
+void vkc_stats(VkcStats *st) { *st = KC.st; }
/* ---- big prefill chunks ------------------------------------------------------------
* The rows of a prompt chunk (vk_chain.h, vkc_chunk_rows). The free device memory is
@@ -1267,15 +1299,15 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes) {
int v = atoi(e);
return v < 1 ? 1 : v > 65535 ? 65535 : v;
}
- static char said[16][24]; static int nsaid, rows_said[16];
- for (int i = 0; i < nsaid; i++) if (!strcmp(said[i], engine ? engine : "")) return rows_said[i];
+ static char said[16][24]; static int nsaid, rows_said[16], said_d[16];
+ for (int i = 0; i < nsaid; i++) if (!strcmp(said[i], engine ? engine : "") && said_d[i] == g_kd) return rows_said[i];
const char *mx = getenv("COLI_VK_CHAIN_ROWS_MAX");
int cap = mx && *mx ? atoi(mx) : 8192;
if (cap < 1) cap = 1;
if (cap > 65535) cap = 65535;
double used = 0, bud = 0, free_b;
- int have = coli_vk_mem_budget(&used, &bud);
- free_b = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes() / 4;
+ int have = coli_vk_mem_budget_dev(g_kd, &used, &bud);
+ free_b = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes_dev(g_kd) / 4;
/* Expert outputs and CPU fallback scratch also grow with the chunk on a
* discrete device. Bound both memories, including Windows commit headroom. */
double avail = compat_mem_available_gb() * 1e9;
@@ -1287,7 +1319,7 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes) {
if (rows < 1) rows = 1;
if (nsaid < 16) {
snprintf(said[nsaid], sizeof said[nsaid], "%s", engine ? engine : "");
- rows_said[nsaid++] = rows;
+ said_d[nsaid] = g_kd; rows_said[nsaid++] = rows;
fprintf(stderr, "[VK] %s chain: prompt chunks of up to %d rows (%.0f KiB a row; %.2f GiB free on the device%s, "
"half of it at most; COLI_VK_CHAIN_ROWS sets it, COLI_VK_CHAIN_ROWS_MAX caps it at %d)\n",
engine ? engine : "engine", rows, row_bytes / 1024.0, free_b / 1073741824.0,
@@ -1310,7 +1342,7 @@ size_t vkc_fit_reserve(void) {
/* A weight pool's block, a block of each of the chain's three pools, and the frames'
* first staging buffers: what the pools may hold beyond the bytes asked for. */
size_t vkc_fit_pools(void) {
- return coli_vk_block_bytes((size_t)256 << 20) + 3 * (size_t)coli_vk_block_bytes(VKC_BLOCK) +
+ return coli_vk_block_bytes_dev(g_kd, (size_t)256 << 20) + 3 * (size_t)coli_vk_block_bytes_dev(g_kd, VKC_BLOCK) +
(size_t)VKC_FRAMES * ((size_t)4 << 20);
}
int vkc_fit_rows(int def) {
@@ -1322,13 +1354,13 @@ int vkc_fit_rows(int def) {
return def < 1 ? 1 : def;
}
size_t vkc_fit_tensor(int fmt, int I, int O, int gs) {
- size_t a = coli_vk_buffer_alignment();
+ size_t a = coli_vk_buffer_alignment_dev(g_kd);
size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;
size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * sizeof(float);
return fit_up(rows ? rows : 4, a) + fit_up(sc ? sc : 4, a);
}
size_t vkc_fit_buf(size_t bytes) {
- size_t a = K.align > 16 ? K.align : 16, b = coli_vk_buffer_alignment();
+ size_t a = KC.align > 16 ? KC.align : 16, b = coli_vk_buffer_alignment_dev(g_kd);
if (b > a) a = b;
size_t v = bytes ? bytes : 4;
return fit_up((v + 3) & ~(size_t)3, a);
@@ -1348,7 +1380,7 @@ int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *
if (fit->mat && matrix_bytes) fit->mat[i] = matrix_bytes[i];
all += b;
}
- fit->free_b = coli_vk_free_bytes();
+ fit->free_b = coli_vk_free_bytes_dev(g_kd);
fit->reserve = vkc_fit_reserve();
fit->pools = vkc_fit_pools();
fit->engine_fixed = fixed_bytes;
@@ -1417,11 +1449,11 @@ void vkc_fit_mark(VkcFit *fit, int layer) {
void vkc_fit_placed(const char *engine, const VkcFit *fit) {
if (!fit) return;
size_t w = 0, nt = 0, m = 0;
- coli_vk_mem_info(&w, &nt);
+ coli_vk_mem_info_dev(g_kd, &w, &nt);
for (int i = 0; i < fit->n && fit->mat; i++) m += fit->mat[i];
fprintf(stderr, "[VK] %s chain: %d of %d layers placed: %zu B of matrices on the device (the fit counted %zu B for these "
"layers), chain buffers %zu B, device memory held %zu B", engine ? engine : "engine", fit->n, fit->L, w, m,
- K.ready ? K.st.dev_bytes : (size_t)0, coli_vk_device_used());
+ KC.ready ? KC.st.dev_bytes : (size_t)0, coli_vk_device_used_dev(g_kd));
if (coli_vk_fault_set() && fit->marks) {
fprintf(stderr, "; COLI_VK_STAGED_FAULT's point reached");
for (int i = 0; i < fit->L; i++) fprintf(stderr, "%s%llu", i ? "," : " ", fit->marks[i]);
@@ -1437,83 +1469,84 @@ void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb)
}
void vkc_prof_print(void) {
- if (!K.prof) return;
- double tot = 0; for (int k = 0; k < PK_N; k++) tot += K.prof_ms[k];
+ if (!KC.prof) return;
+ double tot = 0; for (int k = 0; k < PK_N; k++) tot += KC.prof_ms[k];
fprintf(stderr, "[VK] chain profile: %.1f ms of device time", tot);
for (int k = 0; k < PK_N; k++)
- if (K.prof_n[k]) fprintf(stderr, " | %s %.1f ms (%llu)", pk_name[k], K.prof_ms[k], K.prof_n[k]);
+ if (KC.prof_n[k]) fprintf(stderr, " | %s %.1f ms (%llu)", pk_name[k], KC.prof_ms[k], KC.prof_n[k]);
fprintf(stderr, "\n");
}
void vkc_shutdown(void) {
- if (!K.ready) return;
- if (!K.lost && coli_vk_available()) vkc_finish();
- else vkDeviceWaitIdle(K.dev);
+ if (!KC.ready) return;
+ if (!KC.lost && coli_vk_available_dev(g_kd)) vkc_finish();
+ else vkDeviceWaitIdle(KC.dev);
for (int i = 0; i < VKC_FRAMES; i++) {
- VkcFrame *f = &K.fr[i];
+ VkcFrame *f = &KC.fr[i];
frame_reclaim(f);
if (f->stage) buf_release(f->stage);
free(f->tmp);
- for (int k = 0; k < f->npools; k++) vkDestroyDescriptorPool(K.dev, f->pools[k], NULL);
+ for (int k = 0; k < f->npools; k++) vkDestroyDescriptorPool(KC.dev, f->pools[k], NULL);
free(f->pools);
- if (f->fence) vkDestroyFence(K.dev, f->fence, NULL);
- if (f->qp) vkDestroyQueryPool(K.dev, f->qp, NULL);
+ if (f->fence) vkDestroyFence(KC.dev, f->fence, NULL);
+ if (f->qp) vkDestroyQueryPool(KC.dev, f->qp, NULL);
}
- if (K.dummy) buf_release(K.dummy);
+ if (KC.dummy) buf_release(KC.dummy);
for (int k = 0; k < 3; k++) {
- VkaPool *P = &K.pool[k].p;
+ VkaPool *P = &KC.pool[k].p;
for (int b = 0; b < P->nb; b++) {
if (!P->b[b].present) continue;
VkcBlock *e = P->b[b].user;
- if (e) { if (e->map) vkUnmapMemory(K.dev, e->mem); vkFreeMemory(K.dev, e->mem, NULL); free(e); }
+ if (e) { if (e->map) vkUnmapMemory(KC.dev, e->mem); vkFreeMemory(KC.dev, e->mem, NULL); free(e); }
}
vka_pool_destroy(P);
}
- if (K.cpool) vkDestroyCommandPool(K.dev, K.cpool, NULL);
+ if (KC.cpool) vkDestroyCommandPool(KC.dev, KC.cpool, NULL);
kx_shutdown();
kab_shutdown();
ke_shutdown();
for (int i = 0; i < P_NPIPE; i++) {
- if (K.pipe[i]) vkDestroyPipeline(K.dev, K.pipe[i], NULL);
- if (K.mod[i]) vkDestroyShaderModule(K.dev, K.mod[i], NULL);
+ if (KC.pipe[i]) vkDestroyPipeline(KC.dev, KC.pipe[i], NULL);
+ if (KC.mod[i]) vkDestroyShaderModule(KC.dev, KC.mod[i], NULL);
}
- for (int i = 0; i < K.nkdarec; i++) vkDestroyPipeline(K.dev, K.kdarec[i], NULL);
+ for (int i = 0; i < KC.nkdarec; i++) vkDestroyPipeline(KC.dev, KC.kdarec[i], NULL);
for (int i = 0; i < PM_N; i++) {
- if (K.mpipe[i]) vkDestroyPipeline(K.dev, K.mpipe[i], NULL);
- if (K.mmod[i]) vkDestroyShaderModule(K.dev, K.mmod[i], NULL);
+ if (KC.mpipe[i]) vkDestroyPipeline(KC.dev, KC.mpipe[i], NULL);
+ if (KC.mmod[i]) vkDestroyShaderModule(KC.dev, KC.mmod[i], NULL);
}
- for (int i = 0; i < K.ngemm; i++) vkDestroyPipeline(K.dev, K.gemm[i], NULL);
- for (int i = 0; i < K.ndnrec; i++) vkDestroyPipeline(K.dev, K.dnrec[i], NULL);
- if (K.gemv4) vkDestroyPipeline(K.dev, K.gemv4, NULL);
- if (K.mod_gemv4) vkDestroyShaderModule(K.dev, K.mod_gemv4, NULL);
- if (K.mod_gemm) vkDestroyShaderModule(K.dev, K.mod_gemm, NULL);
- if (K.mod_dnrec) vkDestroyShaderModule(K.dev, K.mod_dnrec, NULL);
- if (K.pl) vkDestroyPipelineLayout(K.dev, K.pl, NULL);
- if (K.dsl) vkDestroyDescriptorSetLayout(K.dev, K.dsl, NULL);
+ for (int i = 0; i < KC.ngemm; i++) vkDestroyPipeline(KC.dev, KC.gemm[i], NULL);
+ for (int i = 0; i < KC.ndnrec; i++) vkDestroyPipeline(KC.dev, KC.dnrec[i], NULL);
+ if (KC.gemv4) vkDestroyPipeline(KC.dev, KC.gemv4, NULL);
+ if (KC.mod_gemv4) vkDestroyShaderModule(KC.dev, KC.mod_gemv4, NULL);
+ if (KC.mod_gemm) vkDestroyShaderModule(KC.dev, KC.mod_gemm, NULL);
+ if (KC.mod_dnrec) vkDestroyShaderModule(KC.dev, KC.mod_dnrec, NULL);
+ if (KC.pl) vkDestroyPipelineLayout(KC.dev, KC.pl, NULL);
+ if (KC.dsl) vkDestroyDescriptorSetLayout(KC.dev, KC.dsl, NULL);
dsv4_shutdown();
kvs_shutdown();
- memset(&K, 0, sizeof K);
- K.cur = -1; K.gemm_rows = -1;
+ memset(&KC, 0, sizeof KC);
+ KC.cur = -1; KC.gemm_rows = -1;
}
/* ---- DeepSeek V4.1 Flash and DeepSeek V4 attention (chain_dsv4.comp) ------------------
* Its own optional pipeline: without the shader only these ops decline. */
-static struct { VkShaderModule mod; VkPipeline pipe; } D4;
+static struct { VkShaderModule mod; VkPipeline pipe; } g_d4[2];
+#define D4 (g_d4[g_kd])
static void dsv4_init(void) {
char path[1200];
- const char *sl = strrchr(K.core.spv_path, '/');
- size_t pre = sl ? (size_t)(sl - K.core.spv_path) + 1 : 0;
+ const char *sl = strrchr(KC.core.spv_path, '/');
+ size_t pre = sl ? (size_t)(sl - KC.core.spv_path) + 1 : 0;
const char *file = "chain_dsv4.spv";
if (pre + strlen(file) + 1 >= sizeof path) return;
- memcpy(path, K.core.spv_path, pre); strcpy(path + pre, file);
+ memcpy(path, KC.core.spv_path, pre); strcpy(path + pre, file);
FILE *f = fopen(path, "rb");
if (!f) return;
fclose(f);
- if ((D4.mod = load_module(K.core.spv_path, file))) D4.pipe = make_pipe(D4.mod, NULL);
+ if ((D4.mod = load_module(KC.core.spv_path, file))) D4.pipe = make_pipe(D4.mod, NULL);
}
static void dsv4_shutdown(void) {
- if (D4.pipe) vkDestroyPipeline(K.dev, D4.pipe, NULL);
- if (D4.mod) vkDestroyShaderModule(K.dev, D4.mod, NULL);
+ if (D4.pipe) vkDestroyPipeline(KC.dev, D4.pipe, NULL);
+ if (D4.mod) vkDestroyShaderModule(KC.dev, D4.mod, NULL);
memset(&D4, 0, sizeof D4);
}
int vkc_dsv4_ready(void) { return vkc_ready() && D4.pipe; }
@@ -1521,10 +1554,10 @@ int vkc_dsv4_ready(void) { return vkc_ready() && D4.pipe; }
/* the shader reads one block of 32 words: the mode, then the op's fields */
typedef struct { int mode; int w[31]; } Dsv4PC;
static int dsv4_rec(int kind, int mode, const void *b, size_t bytes, VkcBind *bd, int nb, uint32_t gx, uint32_t gy) {
- if (!D4.pipe || !open_frame() || K.lost || bytes > sizeof(int) * 31) return 0;
+ if (!D4.pipe || !open_frame() || KC.lost || bytes > sizeof(int) * 31) return 0;
Dsv4PC pc; memset(&pc, 0, sizeof pc);
pc.mode = mode; memcpy(pc.w, b, bytes);
- K.kind = kind;
+ KC.kind = kind;
return record(D4.pipe, bd, nb, &pc, sizeof pc, gx, gy, 1);
}
int vkc_dsv4_attn(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *list, VkcBuf *prm, VkcBuf *out, const VkcDsAttn *p) {
@@ -1536,14 +1569,14 @@ int vkc_dsv4_attn(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *list, VkcBuf *prm
}
int vkc_dsv4_rope(VkcBuf *x, VkcBuf *cs, const VkcDsRope *p) {
if (p->nseg < 0 || p->per_row < 1 || p->rd < 2 || (p->rd & 1)) return 0;
- if (p->nseg == 0) return open_frame() && !K.lost;
+ if (p->nseg == 0) return open_frame() && !KC.lost;
VkcBind bd[6] = {B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(cs, 0), B(x, 1)};
uint32_t gx, gy; grid(((uint64_t)p->nseg * (p->rd / 2) + 255) / 256, &gx, &gy);
return dsv4_rec(PK_ROPE, 1, p, sizeof *p, bd, 6, gx, gy);
}
int vkc_dsv4_compress(VkcBuf *kv, VkcBuf *sc, VkcBuf *ring, VkcBuf *prm, VkcBuf *out, const VkcDsComp *p) {
if (p->S < 0 || p->ratio < 1 || p->P < 1 || p->D < 1 || p->D > p->P || (p->overlap && p->P < 2 * p->D)) return 0;
- if (p->S == 0) return open_frame() && !K.lost;
+ if (p->S == 0) return open_frame() && !KC.lost;
VkcBind bd[8] = {B(kv, 0), B(sc, 0), B(NULL, 0), B(NULL, 0), B(p->ape_off >= 0 ? prm : NULL, 0), B(out, 1), B(NULL, 0),
B(ring, 1)};
return dsv4_rec(PK_DSA, 2, p, sizeof *p, bd, 8, 1, 1);
@@ -1553,7 +1586,7 @@ int vkc_dsv4_score(VkcBuf *iq, VkcBuf *hw, VkcBuf *keys, VkcBuf *mask, VkcBuf *s
int wide = p->IH > 64 || p->IH * p->ID > 4096;
if (p->S < 1 || p->S > 65535 || p->IH < 1 || p->IH > 4096 || p->ID < 1 || p->ratio < 1 ||
p->width < 0 || p->sc_row < p->width || (p->mask_row > 0 && !mask)) return 0;
- if (p->width == 0) return open_frame() && !K.lost;
+ if (p->width == 0) return open_frame() && !KC.lost;
VkcBind bd[6] = {B(iq, 0), B(hw, 0), B(keys, 0), B(p->mask_row > 0 ? mask : NULL, 0), B(NULL, 0), B(sc, 1)};
return dsv4_rec(PK_DSA, wide ? 9 : 3, p, sizeof *p, bd, 6, (uint32_t)p->S, 1);
}
@@ -1565,7 +1598,7 @@ int vkc_dsv4_cand(VkcBuf *sc, VkcBuf *mask, const VkcDsCand *p) {
}
int vkc_dsv4_topk(VkcBuf *sc, VkcBuf *list, const VkcDsTopk *p) {
if (p->S < 1 || p->S > 65535 || p->topk < 0 || p->width < 0 || (p->order && p->topk > 4096)) return 0;
- if (p->topk == 0) return open_frame() && !K.lost;
+ if (p->topk == 0) return open_frame() && !KC.lost;
VkcBind bd[7] = {B(sc, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(list, 1)};
return dsv4_rec(PK_DSA, 5, p, sizeof *p, bd, 7, (uint32_t)p->S, 1);
}
@@ -1579,7 +1612,7 @@ int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p) {
return 0;
if ((p->kind == VKC_DS_E4M3 || p->kind == VKC_DS_E2M1) && (p->block < 1 || p->block > 256)) return 0;
if (p->kind == VKC_DS_HADAMARD && (p->len > 4096 || (p->len & (p->len - 1)))) return 0;
- if (p->nseg == 0) return open_frame() && !K.lost;
+ if (p->nseg == 0) return open_frame() && !KC.lost;
/* the fields, then whether it runs in place (x is read where y is written), then hscale */
int w[14];
memcpy(w, p, 12 * sizeof(int));
@@ -1594,7 +1627,7 @@ int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p) {
}
int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p) {
if (!a || !b || !y || p->n < 0) return 0;
- if (p->n == 0) return open_frame() && !K.lost;
+ if (p->n == 0) return open_frame() && !KC.lost;
uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);
VkcBind bd[6] = {B(a, 0), B(b, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};
return dsv4_rec(PK_EW, 8, p, sizeof *p, bd, 6, gx, gy);
@@ -1603,25 +1636,26 @@ int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p) {
/* ---- a KV cache split between the device and the host (chain_kvs.comp) ------------
* Its own optional pipeline, as DeepSeek's: without the shader only these ops decline
* (and an engine's split with them: past the budget its chain declines as before). */
-static struct { VkShaderModule mod; VkPipeline pipe; int tried; } KS;
+static struct { VkShaderModule mod; VkPipeline pipe; int tried; } g_ks[2];
+#define KS (g_ks[g_kd])
static VkPipeline kvs_pipe(void) {
if (KS.pipe || KS.tried || !vkc_ready()) return KS.pipe;
KS.tried = 1;
char path[1200];
- const char *sl = strrchr(K.core.spv_path, '/');
- size_t pre = sl ? (size_t)(sl - K.core.spv_path) + 1 : 0;
+ const char *sl = strrchr(KC.core.spv_path, '/');
+ size_t pre = sl ? (size_t)(sl - KC.core.spv_path) + 1 : 0;
const char *file = "chain_kvs.spv";
if (pre + strlen(file) + 1 >= sizeof path) return VK_NULL_HANDLE;
- memcpy(path, K.core.spv_path, pre); strcpy(path + pre, file);
+ memcpy(path, KC.core.spv_path, pre); strcpy(path + pre, file);
FILE *f = fopen(path, "rb");
if (!f) return VK_NULL_HANDLE;
fclose(f);
- if ((KS.mod = load_module(K.core.spv_path, file))) KS.pipe = make_pipe(KS.mod, NULL);
+ if ((KS.mod = load_module(KC.core.spv_path, file))) KS.pipe = make_pipe(KS.mod, NULL);
return KS.pipe;
}
static void kvs_shutdown(void) {
- if (KS.pipe) vkDestroyPipeline(K.dev, KS.pipe, NULL);
- if (KS.mod) vkDestroyShaderModule(K.dev, KS.mod, NULL);
+ if (KS.pipe) vkDestroyPipeline(KC.dev, KS.pipe, NULL);
+ if (KS.mod) vkDestroyShaderModule(KC.dev, KS.mod, NULL);
memset(&KS, 0, sizeof KS);
}
int vkc_kvs_ready(void) { return kvs_pipe() != VK_NULL_HANDLE; }
@@ -1633,7 +1667,7 @@ int vkc_kvs_attn(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *out, VkcBuf *gate, V
p->nblk < 0 || p->anchor < 0 || p->win < 0 || (p->sink && !snk) || (p->sel_row > 0 && !sel) ||
((p->flags & VKC_KVS_GATE) && (!gate || !(p->flags & VKC_KVS_FIN)))) return 0;
struct { int mode; VkcKvsAttn b; } pc = {0, *p};
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[8] = {B(q, 0), B(kc, 0), B(vc, 0), B(out, 1), B(p->flags & VKC_KVS_GATE ? gate : NULL, 0),
B(p->sel_row > 0 ? sel : NULL, 0), B(p->sink ? snk : NULL, 0), B(tab, 0)};
return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);
@@ -1645,7 +1679,7 @@ int vkc_kvs_mla(VkcBuf *qa, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sel,
p->anchor < 0 ||
p->kv_start < 0 || (p->sel_row > 0 && !sel)) return 0;
struct { int mode; VkcKvsMla b; } pc = {1, *p};
- K.kind = PK_MLA;
+ KC.kind = PK_MLA;
VkcBind bd[8] = {B(qa, 0), B(p->R > 0 ? qr : NULL, 0), B(lat, 0), B(p->R > 0 ? rope : NULL, 0), B(NULL, 0),
B(p->sel_row > 0 ? sel : NULL, 0), B(out, 1), B(tab, 0)};
return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);
@@ -1653,9 +1687,9 @@ int vkc_kvs_mla(VkcBuf *qa, VkcBuf *qr, VkcBuf *lat, VkcBuf *rope, VkcBuf *sel,
int vkc_kvs_merge(VkcBuf *dev, VkcBuf *cpu, VkcBuf *gate, VkcBuf *out, const VkcKvsMerge *p) {
VkPipeline pipe = kvs_pipe();
if (!pipe || !dev || !cpu || !out || p->n < 0 || p->H < 1 || p->d < 1 || ((p->flags & VKC_KVS_GATE) && !gate)) return 0;
- if (p->n == 0) return open_frame() && !K.lost;
+ if (p->n == 0) return open_frame() && !KC.lost;
struct { int mode; VkcKvsMerge b; } pc = {2, *p};
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[4] = {B(dev, 0), B(cpu, 0), B(p->flags & VKC_KVS_GATE ? gate : NULL, 0), B(out, 1)};
uint32_t gx, gy; grid((uint64_t)p->n, &gx, &gy);
return record(pipe, bd, 4, &pc, sizeof pc, gx, gy, 1);
@@ -1667,7 +1701,7 @@ int vkc_kvs_rel(VkcBuf *q, VkcBuf *kc, VkcBuf *vc, VkcBuf *out, VkcBuf *r, VkcBu
p->KVH < 1 || p->H % p->KVH || p->hd < 1 || p->hd > 256 || p->d_rel < 0 || p->d_rel > 64 ||
(p->d_rel > 0 && (!r || !relp)) || p->B < 1 || p->nblk < 0 || p->anchor < 0) return 0;
struct { int mode; VkcKvsRel b; } pc = {3, *p};
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[8] = {B(q, 0), B(kc, 0), B(vc, 0), B(out, 1), B(p->d_rel > 0 ? r : NULL, 0), B(tau, 0),
B(p->d_rel > 0 ? relp : NULL, 0), B(tab, 0)};
return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);
@@ -1679,7 +1713,7 @@ int vkc_kvs_ds_cold(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *cold, VkcBuf *o
((p->flags & VKC_KVS_DSCOLD) && (!cold || !(p->flags & VKC_KVS_DSFIN))) ||
((p->flags & VKC_KVS_V4) && !(p->flags & VKC_KVS_DSFIN))) return 0;
struct { int mode; VkcKvsDs b; } pc = {4, *p};
- K.kind = PK_ATTN;
+ KC.kind = PK_ATTN;
VkcBind bd[8] = {B(q, 0), B(win, 0), B(cmp ? cmp : win, 0), B(out, 1), B(prm, 0), B(list, 0), B(cold, 0), B(tab, 0)};
return record(pipe, bd, 8, &pc, sizeof pc, (uint32_t)p->H, (uint32_t)p->S, 1);
}
@@ -1689,14 +1723,14 @@ int vkc_kvs_ds(VkcBuf *q, VkcBuf *win, VkcBuf *cmp, VkcBuf *out, VkcBuf *prm, Vk
unsigned long long vkc_serial(void) {
unsigned long long s = 0;
for (int i = 0; i < VKC_FRAMES; i++)
- if (!K.fr[i].open && K.fr[i].serial > s) s = K.fr[i].serial;
+ if (!KC.fr[i].open && KC.fr[i].serial > s) s = KC.fr[i].serial;
return s;
}
int vkc_wait_serial(unsigned long long serial) {
if (!vkc_ready()) return 0;
for (int i = 0; i < VKC_FRAMES; i++) {
- VkcFrame *f = &K.fr[i];
- if (f->serial == serial && !f->open) return frame_wait(f) && !K.lost;
+ VkcFrame *f = &KC.fr[i];
+ if (f->serial == serial && !f->open) return frame_wait(f) && !KC.lost;
}
- return !K.lost; /* reused since: it completed long ago */
+ return !KC.lost; /* reused since: it completed long ago */
}
diff --git a/c/vk_chain.h b/c/vk_chain.h
index f67041ae..6a83bf2e 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -58,6 +58,12 @@ int vkc_init(void); /* after coli_vk_init; 1 = the chain's pipelines a
int vkc_ready(void);
int vkc_lost(void);
void vkc_shutdown(void); /* before coli_vk_shutdown (register it with atexit after it) */
+/* The device the calls go to: 0 the primary, 1 COLI_VK_DEV2's (coli_vk_core_dev), each
+ * with a context of its own that vkc_init opens and vkc_shutdown closes while it is
+ * current. A buffer belongs to the device it was made on: bound or copied on the other
+ * the op fails; vkc_free takes it back on its own device. Returns the previous device. */
+int vkc_device(int d);
+int vkc_device_now(void);
VkcBuf *vkc_buf(size_t bytes, int kind); /* zero-filled; NULL when out of memory */
void vkc_free(VkcBuf *b); /* waits for the frames that may read it */
diff --git a/c/vk_kvsplit.h b/c/vk_kvsplit.h
index 1c7b6687..8c7a1bea 100644
--- a/c/vk_kvsplit.h
+++ b/c/vk_kvsplit.h
@@ -149,11 +149,11 @@ static int vkc_kv_plan_need(VkcKvSplit *ks, const char *engine, int nl, size_t r
} else {
double used = 0, bud = 0;
size_t avail;
- if (coli_vk_mem_budget(&used, &bud)) avail = bud > used ? (size_t)((bud - used) * 1e9) : 0;
+ if (coli_vk_mem_budget_dev(vkc_device_now(), &used, &bud)) avail = bud > used ? (size_t)((bud - used) * 1e9) : 0;
else {
- size_t dev = coli_vk_device_local_bytes(), w = 0, n = 0;
+ size_t dev = coli_vk_device_local_bytes_dev(vkc_device_now()), w = 0, n = 0;
VkcStats st; vkc_stats(&st);
- coli_vk_mem_info(&w, &n);
+ coli_vk_mem_info_dev(vkc_device_now(), &w, &n);
avail = dev > w + st.dev_bytes ? dev - w - st.dev_bytes : 0;
}
avail += held;
@@ -895,10 +895,10 @@ static int vkc_kv_sparse_reserve(VkcBuf **b, size_t bytes) {
if (old >= bytes) return 1;
double used = 0, budget = 0;
size_t free_bytes;
- if (coli_vk_mem_budget(&used, &budget)) free_bytes = budget > used ? (size_t)((budget - used) * 1e9) : 0;
+ if (coli_vk_mem_budget_dev(vkc_device_now(), &used, &budget)) free_bytes = budget > used ? (size_t)((budget - used) * 1e9) : 0;
else {
- size_t total = coli_vk_device_local_bytes(), weights = 0, count = 0;
- VkcStats st; vkc_stats(&st); coli_vk_mem_info(&weights, &count);
+ size_t total = coli_vk_device_local_bytes_dev(vkc_device_now()), weights = 0, count = 0;
+ VkcStats st; vkc_stats(&st); coli_vk_mem_info_dev(vkc_device_now(), &weights, &count);
free_bytes = total > weights + st.dev_bytes ? total - weights - st.dev_bytes : 0;
}
if (bytes - old > free_bytes) return 0;