Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(vulkan): device memory books, a device cap for tests, and the partial-chain fit
JustVugg061a9f33e26e
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex bfbe0124..b9074747 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -24,6 +24,128 @@ static double vk_now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &#define VKCHECK(x, what) do { VkResult _r = (x); if (_r != VK_SUCCESS) { \fprintf(stderr, "[VK] %s failed: %d\n", what, _r); return 0; } } while (0)+/* ---- device memory books, and COLI_VK_DEVICE_CAP_MB (docs/vulkan.md, "A partial chain") --+ * Every device memory allocation of the process goes through vk_mem_alloc / vk_mem_free+ * (the macros below rename the Vulkan calls in this file; vk_chain.c reaches them through+ * coli_vk_mem_alloc / coli_vk_mem_free): the bytes in a device-local heap are counted per+ * device (0 = G, 1 = G2). COLI_VK_DEVICE_CAP_MB=n (tests; a fraction is taken) makes a+ * device hold at most n MiB of them: an allocation past it fails as+ * VK_ERROR_OUT_OF_DEVICE_MEMORY, as a full card's does, and the budget the engines read+ * (coli_vk_mem_budget, _budget2, coli_vk_free_bytes, coli_vk_device_local_bytes, the+ * expert batch's heap room) is the cap and those bytes. So Lavapipe behaves like a small+ * card. Host memory imported in place (coli_vk_tensor_import) is the host's and is not+ * counted. Under the cap the pools take small blocks (coli_vk_block_bytes) so that their+ * granularity does not decide what fits. */+typedef struct { uint64_t key, bytes; int dev; } VkMemRec;+static struct {+ pthread_mutex_t mx;+ VkDevice dev[2];+ uint32_t dl_types[2]; /* memory types whose heap is device-local */+ uint64_t cap[2], used[2], peak[2];+ unsigned long long refused[2];+ VkMemRec *rec; size_t nrec, crec;+} g_mem = {.mx = PTHREAD_MUTEX_INITIALIZER};+static uint64_t vk_mem_key(VkDeviceMemory m) { uint64_t k = 0; memcpy(&k, &m, sizeof m); return k; }+static int vk_mem_dev(VkDevice d) { return d && d == g_mem.dev[1] ? 1 : 0; }+static int vk_mem_imported(const VkMemoryAllocateInfo *ai) {+ for (const VkBaseInStructure *p = ai->pNext; p; p = p->pNext)+ if (p->sType == VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT) return 1;+ return 0;+}+static VkResult vk_mem_alloc(VkDevice d, const VkMemoryAllocateInfo *ai, const VkAllocationCallbacks *cb,+ VkDeviceMemory *m) {+ int k = vk_mem_dev(d);+ int counted = ai->memoryTypeIndex < 32 && (g_mem.dl_types[k] >> ai->memoryTypeIndex & 1u) && !vk_mem_imported(ai);+ uint64_t n = ai->allocationSize;+ if (counted) {+ pthread_mutex_lock(&g_mem.mx);+ if (g_mem.cap[k] && g_mem.used[k] + n > g_mem.cap[k]) {+ g_mem.refused[k]++;+ pthread_mutex_unlock(&g_mem.mx);+ return VK_ERROR_OUT_OF_DEVICE_MEMORY;+ }+ g_mem.used[k] += n;+ if (g_mem.used[k] > g_mem.peak[k]) g_mem.peak[k] = g_mem.used[k];+ pthread_mutex_unlock(&g_mem.mx);+ }+ VkResult r = vkAllocateMemory(d, ai, cb, m);+ if (!counted) return r;+ pthread_mutex_lock(&g_mem.mx);+ if (r != VK_SUCCESS) g_mem.used[k] -= n;+ else {+ if (g_mem.nrec == g_mem.crec) {+ size_t c = g_mem.crec ? 2 * g_mem.crec : 256;+ VkMemRec *nr = realloc(g_mem.rec, c * sizeof *nr);+ if (nr) { g_mem.rec = nr; g_mem.crec = c; }+ }+ if (g_mem.nrec < g_mem.crec) g_mem.rec[g_mem.nrec++] = (VkMemRec){vk_mem_key(*m), n, k};+ }+ pthread_mutex_unlock(&g_mem.mx);+ return r;+}+static void vk_mem_free(VkDevice d, VkDeviceMemory m, const VkAllocationCallbacks *cb) {+ if (m != VK_NULL_HANDLE) {+ uint64_t key = vk_mem_key(m);+ int k = vk_mem_dev(d);+ pthread_mutex_lock(&g_mem.mx);+ for (size_t i = g_mem.nrec; i-- > 0;)+ if (g_mem.rec[i].key == key && g_mem.rec[i].dev == k) {+ g_mem.used[k] -= g_mem.rec[i].bytes;+ g_mem.rec[i] = g_mem.rec[--g_mem.nrec];+ break;+ }+ pthread_mutex_unlock(&g_mem.mx);+ }+ vkFreeMemory(d, m, cb);+}+#define vkAllocateMemory vk_mem_alloc+#define vkFreeMemory vk_mem_free+/* A device just created: which of its memory types count, and the cap. */+static void vk_mem_device(int k, VkPhysicalDevice phys, VkDevice d) {+ VkPhysicalDeviceMemoryProperties mp;+ vkGetPhysicalDeviceMemoryProperties(phys, &mp);+ uint32_t mask = 0;+ for (uint32_t i = 0; i < mp.memoryTypeCount && i < 32; i++)+ if (mp.memoryHeaps[mp.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) mask |= 1u << i;+ const char *e = getenv("COLI_VK_DEVICE_CAP_MB");+ double mb = e && *e ? atof(e) : 0.0;+ pthread_mutex_lock(&g_mem.mx);+ g_mem.dev[k] = d; g_mem.dl_types[k] = mask;+ g_mem.cap[k] = mb > 0 ? (uint64_t)(mb * 1048576.0) : 0;+ g_mem.used[k] = g_mem.peak[k] = 0; g_mem.refused[k] = 0;+ pthread_mutex_unlock(&g_mem.mx);+ if (g_mem.cap[k])+ fprintf(stderr, "[VK] COLI_VK_DEVICE_CAP_MB=%s: %s takes at most %.3f MiB of device-local memory\n",+ e, k ? "the second device" : "the device", (double)g_mem.cap[k] / 1048576.0);+}+/* The device is gone (shutdown): its books go with it. */+static void vk_mem_forget(int k) {+ pthread_mutex_lock(&g_mem.mx);+ for (size_t i = g_mem.nrec; i-- > 0;)+ if (g_mem.rec[i].dev == k) g_mem.rec[i] = g_mem.rec[--g_mem.nrec];+ g_mem.dev[k] = VK_NULL_HANDLE; g_mem.dl_types[k] = 0; g_mem.used[k] = 0;+ pthread_mutex_unlock(&g_mem.mx);+}+static uint64_t vk_mem_room(int k) { /* lock-free read is enough for a budget */+ uint64_t c = g_mem.cap[k], u = __atomic_load_n(&g_mem.used[k], __ATOMIC_RELAXED);+ return c > u ? c - u : 0;+}+static void vk_pool_blocks(int k); /* the weight pools' block size, below */+size_t coli_vk_device_cap(void) { return (size_t)g_mem.cap[0]; }+size_t coli_vk_device_used(void) { return (size_t)__atomic_load_n(&g_mem.used[0], __ATOMIC_RELAXED); }+size_t coli_vk_block_bytes(size_t def) {+ if (!g_mem.cap[0]) return def;+ size_t b = (size_t)64 << 10; /* at most about 4096 blocks fill the cap */+ while ((uint64_t)b < g_mem.cap[0] / 4096) b <<= 1;+ return b < def ? b : def;+}+int coli_vk_mem_alloc(void *device, const void *info, void *memory) {+ return (int)vk_mem_alloc((VkDevice)device, (const VkMemoryAllocateInfo *)info, NULL, (VkDeviceMemory *)memory);+}+void coli_vk_mem_free(void *device, const void *memory) {+ vk_mem_free((VkDevice)device, *(const VkDeviceMemory *)memory, NULL);+}+typedef struct VkWPool VkWPool;struct ColiVkTensor {VkBuffer wbuf, sbuf;@@ -103,6 +225,9 @@ static int up_fault(const char *point) {fprintf(stderr, "[VK] COLI_VK_STAGED_FAULT: %s #%ld fails\n", point, g_fault_at);return 1;}+/* How many times the COLI_VK_STAGED_FAULT point was reached so far (0 when unset). */+unsigned long long coli_vk_fault_reached(void) { return (unsigned long long)__atomic_load_n(&g_fault_n, __ATOMIC_RELAXED); }+int coli_vk_fault_set(void) { return g_fault_point[0] != 0; }typedef struct {VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;@@ -391,6 +516,11 @@ void coli_vk_alloc_priority(float p) { G.prio = p < 0 ? 0 : p > 1 ? 1 : p; }/* Device-local heap usage/budget in GB (VK_EXT_memory_budget). Returns 0 when the* extension is absent — callers then keep their count-based caps unchanged. */int coli_vk_mem_budget(double *used_gb, double *budget_gb) {+ if (g_mem.cap[0] && G.phys) { /* COLI_VK_DEVICE_CAP_MB: the cap and this process's bytes */+ if (used_gb) *used_gb = (double)__atomic_load_n(&g_mem.used[0], __ATOMIC_RELAXED) / 1e9;+ if (budget_gb) *budget_gb = (double)g_mem.cap[0] / 1e9;+ return 1;+ }#ifdef VK_EXT_memory_budgetif (!G.has_budget || !G.phys) return 0;VkPhysicalDeviceMemoryBudgetPropertiesEXT bud = {@@ -410,6 +540,17 @@ int coli_vk_mem_budget(double *used_gb, double *budget_gb) {(void)used_gb; (void)budget_gb; return 0;#endif}+/* Free device memory now: the cap less what this process holds (COLI_VK_DEVICE_CAP_MB), else+ * VK_EXT_memory_budget's budget less its usage, else the largest device-local heap less+ * what this process holds. */+size_t coli_vk_free_bytes(void) {+ if (!G.phys) return 0;+ if (g_mem.cap[0]) return (size_t)vk_mem_room(0);+ double used = 0, bud = 0;+ if (coli_vk_mem_budget(&used, &bud)) return bud > used ? (size_t)((bud - used) * 1e9) : 0;+ size_t dl = coli_vk_device_local_bytes(), u = coli_vk_device_used();+ return dl > u ? dl - u : 0;+}static int alloc_hostvis(size_t bytes, VkBuffer *buf, VkDeviceMemory *mem, void **ptr) {return alloc_hostvis_mt(bytes, buf, mem, ptr, G.memtype);}@@ -807,6 +948,8 @@ int coli_vk_init(const char *spv_path) {if (!G.dev)#endifVKCHECK(vkCreateDevice(G.phys, &di, NULL, &G.dev), "vkCreateDevice");+ vk_mem_device(0, G.phys, G.dev); /* the device memory books, and COLI_VK_DEVICE_CAP_MB */+ vk_pool_blocks(0);vkGetDeviceQueue(G.dev, G.qfam, 0, &G.queue);vkGetDeviceQueue(G.dev, G.tq_fam, G.tq_idx, &G.tqueue);G.tq_shared = G.tqueue == G.queue;@@ -1038,6 +1181,13 @@ struct VkWPool {static VkWPool g_wpool = {.p = {.block_bytes = VK_WBLOCK}, .mx = PTHREAD_MUTEX_INITIALIZER, .dev = 0, .prio = -1.f, .counted = 1};static VkWPool g_tpool = {.p = {.block_bytes = VK_WBLOCK}, .mx = PTHREAD_MUTEX_INITIALIZER, .dev = 0, .prio = 0.4f};static VkWPool g_wpool2 = {.p = {.block_bytes = VK_WBLOCK}, .mx = PTHREAD_MUTEX_INITIALIZER, .dev = 1, .prio = -1.f};+/* Device k's pools take VK_WBLOCK blocks, smaller ones under COLI_VK_DEVICE_CAP_MB. */+static void vk_pool_blocks(int k) {+ size_t bb = VK_WBLOCK;+ if (g_mem.cap[k]) { bb = (size_t)64 << 10; while ((uint64_t)bb < g_mem.cap[k] / 4096 && bb < VK_WBLOCK) bb <<= 1; }+ if (k == 0) { g_wpool.p.block_bytes = bb; g_tpool.p.block_bytes = bb; }+ else g_wpool2.p.block_bytes = bb;+}static VkDevice pool_device(const VkWPool *P);static uint32_t pool_memtype(const VkWPool *P);static int pool_has_prio(const VkWPool *P);@@ -1711,6 +1861,17 @@ int coli_vk_dense(void) { return g_dense_on; }static int g_dho, g_dho_atexit, g_dho_on_device;static char g_dho_engine[32];static unsigned long long g_dho_drop_n, g_dho_drop_b, g_dho_load_n, g_dho_load_b;+static int g_dho_lay_dev, g_dho_lay_all; /* a partial chain: the first lay_dev of lay_all layers dropped theirs */+/* " (the n of L layers ...)" for the lines, empty unless the engine said a partial chain */+static const char *dho_layers_note(char *buf, size_t n, int placed) {+ buf[0] = 0;+ if (g_dho_lay_all > 0 && g_dho_lay_dev < g_dho_lay_all) {+ if (placed) snprintf(buf, n, " (the %d of %d layers on the device; the %d on the CPU keep theirs)",+ g_dho_lay_dev, g_dho_lay_all, g_dho_lay_all - g_dho_lay_dev);+ else snprintf(buf, n, ", the %d of %d layers on the device", g_dho_lay_dev, g_dho_lay_all);+ }+ return buf;+}static double dho_gib(unsigned long long b) { return (double)b / 1073741824.0; }/* A size for the [VK] lines: GiB from 1 GiB, MiB below (a test fixture's KiB show as 0.0 MiB). */static const char *dho_size(unsigned long long b, char *buf, size_t n) {@@ -1738,10 +1899,11 @@ static void dho_exit_report(void) {fprintf(stderr, "[VK] %s: dense weights at exit: on the device and in host RAM%s\n", g_dho_engine, r);return;}- char a[32], b[32];+ char a[32], b[32], ln[96];fprintf(stderr, "[VK] %s: dense weights at exit: %llu matrices on the device only (%s of host copies "- "dropped), %llu read back from disk for the CPU (%s)%s\n", g_dho_engine,+ "dropped%s), %llu read back from disk for the CPU (%s)%s\n", g_dho_engine,__atomic_load_n(&g_dho_drop_n, __ATOMIC_RELAXED), dho_size(__atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED), a, sizeof a),+ dho_layers_note(ln, sizeof ln, 0),__atomic_load_n(&g_dho_load_n, __ATOMIC_RELAXED), dho_size(__atomic_load_n(&g_dho_load_b, __ATOMIC_RELAXED), b, sizeof b), r);}static int dho_choice(int dense_on_device, size_t dense_bytes, char *why, size_t n) {@@ -1800,11 +1962,14 @@ void coli_vk_dense_host_placed(const char *engine, const char *kept) {double rss = dho_rss_gib();char r[48] = "";if (rss >= 0) snprintf(r, sizeof r, "; RSS now %.2f GiB", rss);- char a[32];- fprintf(stderr, "[VK] %s: %llu dense matrices on the device only, %s of host RAM given back%s%s%s\n",+ char a[32], ln[96];+ fprintf(stderr, "[VK] %s: %llu dense matrices on the device only, %s of host RAM given back%s%s%s%s\n",engine, __atomic_load_n(&g_dho_drop_n, __ATOMIC_RELAXED),- dho_size(__atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED), a, sizeof a), kept && *kept ? "; kept on the host: " : "",- kept && *kept ? kept : "", r);+ dho_size(__atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED), a, sizeof a), dho_layers_note(ln, sizeof ln, 1),+ kept && *kept ? "; kept on the host: " : "", kept && *kept ? kept : "", r);+}+void coli_vk_dense_host_layers(int on_device, int layers) {+ g_dho_lay_dev = on_device < 0 ? 0 : on_device; g_dho_lay_all = layers < 0 ? 0 : layers;}unsigned long long coli_vk_dense_host_dropped_bytes(void) { return __atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED); }@@ -2172,6 +2337,8 @@ int coli_vk_init_dev2(const char *spv_path, int devidx) {.queueCreateInfoCount = 1, .pQueueCreateInfos = &qi,.enabledExtensionCount = ndext, .ppEnabledExtensionNames = ndext ? dext : NULL};VKCHECK(vkCreateDevice(G2.phys, &di, NULL, &G2.dev), "d2 vkCreateDevice");+ vk_mem_device(1, G2.phys, G2.dev);+ vk_pool_blocks(1);vkGetDeviceQueue(G2.dev, G2.qfam, 0, &G2.queue);int mt = pick_memtype(G2.phys);if (mt < 0) { fprintf(stderr, "[VK] dev2: no host-visible memory\n"); return 0; }@@ -2215,6 +2382,11 @@ int coli_vk_dev2_available(void) { return G2.ready; }int coli_vk_tensor_dev(const ColiVkTensor *t) { return t ? t->dev : 0; }int coli_vk_mem_budget2(double *used_gb, double *budget_gb) {+ if (g_mem.cap[1] && G2.phys) { /* COLI_VK_DEVICE_CAP_MB, as coli_vk_mem_budget */+ if (used_gb) *used_gb = (double)__atomic_load_n(&g_mem.used[1], __ATOMIC_RELAXED) / 1e9;+ if (budget_gb) *budget_gb = (double)g_mem.cap[1] / 1e9;+ return 1;+ }#ifdef VK_EXT_memory_budgetif (!G2.has_budget || !G2.phys) return 0;VkPhysicalDeviceMemoryBudgetPropertiesEXT bud = {@@ -2885,7 +3057,8 @@ void coli_vk_tier_pool_limit(size_t bytes) {pthread_mutex_lock(&g_tpool.mx);g_tpool.p.limit = bytes;/* blocks no larger than the budget, so a small budget is not one refused block */- g_tpool.p.block_bytes = bytes && bytes < VK_WBLOCK ? ((bytes + 4095) & ~(size_t)4095) : VK_WBLOCK;+ size_t bb = coli_vk_block_bytes(VK_WBLOCK);+ g_tpool.p.block_bytes = bytes && bytes < bb ? ((bytes + 4095) & ~(size_t)4095) : bb;/* the empty block the pool keeps as a spare (vka_free) goes too: a tier started* again after a shutdown would otherwise fill it past its new, smaller budget */for (int k = 0; k < g_tpool.p.nb; k++)@@ -2922,6 +3095,7 @@ size_t coli_vk_device_local_bytes(void) {VkDeviceSize m = 0;for (uint32_t i = 0; i < mp.memoryHeapCount; i++)if ((mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && mp.memoryHeaps[i].size > m) m = mp.memoryHeaps[i].size;+ if (g_mem.cap[0] && g_mem.cap[0] < m) m = g_mem.cap[0]; /* COLI_VK_DEVICE_CAP_MB */return (size_t)m;}const char *coli_vk_device_name(void) {@@ -3588,6 +3762,8 @@ int coli_vk_xb_sub_fit(int rows, int experts, size_t extra_budget) {#ifdef VK_EXT_memory_budgetif (G.has_budget) room[h] = bud.heapBudget[h] > bud.heapUsage[h] ? (size_t)((bud.heapBudget[h] - bud.heapUsage[h]) / 2) : 0;#endif+ if (g_mem.cap[0] && (mp.memoryProperties.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) &&+ room[h] > vk_mem_room(0) / 2) room[h] = (size_t)(vk_mem_room(0) / 2); /* COLI_VK_DEVICE_CAP_MB */}uint32_t xheap = mp.memoryProperties.memoryTypes[G.memtype].heapIndex;uint32_t iheap = mp.memoryProperties.memoryTypes[G.memtype_dev].heapIndex;@@ -3788,6 +3964,10 @@ int coli_vk_tensor_refill(ColiVkTensor *t, uint8_t **rows, size_t *stride, float}int coli_vk_staged(void) { return G.ready && g_up[0].on; }size_t coli_vk_tensor_row_bytes(int fmt, int I) { return cpu_row_bytes(fmt, I); }+size_t coli_vk_tensor_payload(int fmt, int I, int O, int gs) {+ if (fmt == 4 || fmt == 7 || fmt == 12 || fmt == 13) { if (gs < 1) return 0; } else gs = 0;+ return (size_t)rowwords(fmt, I) * 4 * (size_t)O + scale_floats(fmt, I, O, gs) * sizeof(float);+}size_t coli_vk_buffer_alignment(void) { return G.buf_align ? G.buf_align : 256; }size_t coli_vk_tensor_scale_count(int fmt, int I, int O, int gs) { return scale_floats(fmt, I, O, gs); }@@ -3886,6 +4066,8 @@ void coli_vk_shutdown(void) {vkDestroyDevice(G.dev, NULL);vkDestroyInstance(G.inst, NULL);memset(&G, 0, sizeof(G));+ vk_mem_forget(0);+ vk_pool_blocks(0);}/* ---- the dense chain's view of the device (vk_chain.c) -------------------------diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex a76f0405..0608bc06 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -33,6 +33,32 @@ void coli_vk_set_swiglu_limit(float limit);void coli_vk_alloc_priority(float p);int coli_vk_mem_budget(double *used_gb, double *budget_gb);+/* Device memory books (docs/vulkan.md, "A partial chain"). Every device memory allocation+ * of the backend and of the chain is counted per device when its heap is device-local.+ * COLI_VK_DEVICE_CAP_MB=n (tests; n may be a fraction) makes the device hold at most n MiB:+ * an allocation past it fails as out of device memory, and coli_vk_mem_budget (and+ * _budget2, coli_vk_free_bytes, coli_vk_device_local_bytes) report the cap and this+ * process's bytes, so a CPU device behaves like a small card.+ * coli_vk_device_cap the cap in bytes (0: none)+ * coli_vk_device_used device-local bytes this process holds on the device+ * coli_vk_free_bytes free device memory now: the cap less what is held, else the+ * budget less its usage (VK_EXT_memory_budget), else the largest+ * device-local heap less what is held+ * coli_vk_block_bytes the block size a memory pool of `def` bytes takes (smaller+ * under the cap, at most about 4096 blocks to fill it)+ * coli_vk_mem_alloc/free vkAllocateMemory / vkFreeMemory through the books (vk_chain.c):+ * info is a VkMemoryAllocateInfo *, memory a VkDeviceMemory *;+ * the result is a VkResult */+size_t coli_vk_device_cap(void);+size_t coli_vk_device_used(void);+size_t coli_vk_free_bytes(void);+size_t coli_vk_block_bytes(size_t def);+int coli_vk_mem_alloc(void *device, const void *info, void *memory);+void coli_vk_mem_free(void *device, const void *memory);+/* COLI_VK_STAGED_FAULT (tests): whether a point is set, and how many times it was reached. */+int coli_vk_fault_set(void);+unsigned long long coli_vk_fault_reached(void);+/* y[S,O] = (x[S,I] @ dequant(W[O,I])^T) * scale[O].* fmt matches QT in glm.c: 1=int8, 2=int4. (0=f32,3=int2 fall back to CPU.)* fmt 10 = plain f32 weights and 11 = bf16 weights (low half = even column): no@@ -179,6 +205,9 @@ int coli_vk_tier_tensor(ColiVkTensor **t, int fmt, int I, int O, int gs,int coli_vk_tensor_commit(ColiVkTensor *const *t, int n);int coli_vk_staged(void); /* 1 = resident data goes to the device through staged uploads */size_t coli_vk_tensor_row_bytes(int fmt, int I);+/* What coli_vk_tensor_ensure(fmt, I, O, gs) adds to coli_vk_mem_info's bytes: the rows+ * at their padded stride and the scales (0 for a grouped format without a group size). */+size_t coli_vk_tensor_payload(int fmt, int I, int O, int gs);size_t coli_vk_buffer_alignment(void); /* where a weight range may start (bytes) */size_t coli_vk_tensor_scale_count(int fmt, int I, int O, int gs);@@ -298,6 +327,11 @@ void coli_vk_dense_host_reloaded(size_t bytes);/* a host copy read back from dis* only, the RAM given back, and what stays on the host (kept: NULL or a short note). */void coli_vk_dense_host_placed(const char *engine, const char *kept);unsigned long long coli_vk_dense_host_dropped_bytes(void);+/* A partial chain (vk_chain.h, vkc_fit): only the first `on_device` of `layers` layers drop+ * their host copies; the placed and exit lines then say so (nothing changes when+ * on_device == layers, or before any call). Call it before coli_vk_dense_host_placed. The+ * bytes the engine passes coli_vk_dense_host_decide are those layers' only. */+void coli_vk_dense_host_layers(int on_device, int layers);/* COLI_VK_SHADERS (the .spv or its directory), else shaders/ next to the binary, else* shaders/ in the working directory. buf holds the result when it is not a literal. */const char *coli_vk_shader_path(char *buf, size_t n);diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex b3aa5490..5c2fde57 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -24,6 +24,9 @@# bash tests/vulkan_engines.sh staged-faults | staged-faults-sanitize # staged uploads failing# bash tests/vulkan_engines.sh dense-only-<group> | dense-only-<group>-sanitize # the dense weights# on the device only (COLI_VK_DENSE_HOST=0), tests/vulkan_dense_only_<group>.sh+# bash tests/vulkan_engines.sh partial-<group> | partial-<group>-sanitize # the partial chain:+# the first N layers on the device (COLI_VK_CHAIN_LAYERS, COLI_VK_DEVICE_CAP_MB),+# tests/vulkan_partial_<group>.sh# bash tests/vulkan_engines.sh decide | decide-sanitize # Laya, GLiNER2.5-Decide and Clef's DECIDE## Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of@@ -2883,6 +2886,86 @@ PYecho "OK $tag: $(grep -o 'given back.*' vk.log | head -1)"}for f in tests/vulkan_dense_only_*.sh; do [ -e "$f" ] && . "$f"; done++# ---- the partial chain (docs/vulkan.md, "A partial chain"): the first N layers on the+# device, the rest and the head on the CPU. One file per engine group,+# tests/vulkan_partial_<group>.sh, each defining ptl_family_<group> and+# ptl_family_<group>_sanitize (run as partial-<group> and partial-<group>-sanitize);+# tests/vulkan_partial_deepseek.sh is the template. The helpers read the engine's lines:+# [VK] <engine> chain fit: free F B, reserve R B, fixed X B (...), tail T B, layers b0 .. B, matrices m0 .. B+# [VK] <engine> chain: N of L layers on the device ... (the last one is the final N)+# [VK] <engine> chain: N of L layers placed: M B of matrices on the device (the fit counted M' B ...), ...+#+# ptl_n <engine> <log>: the final N (-1 without a line)+ptl_n() {+ local n; n=$(sed -n "s/^\[VK\] $1 chain: \([0-9][0-9]*\) of [0-9][0-9]* layers on the device.*/\1/p" "$2" | tail -1)+ echo "${n:--1}"+}+# ptl_L <engine> <log>: L, the model's layers, from the same line+ptl_L() {+ local n; n=$(sed -n "s/^\[VK\] $1 chain: [0-9][0-9]* of \([0-9][0-9]*\) layers on the device.*/\1/p" "$2" | tail -1)+ echo "${n:--1}"+}+# ptl_calc <what> <engine> <log> [k]: arithmetic on the fit line (a probe run's for "cap")+# predict N as vkc_fit's rule gives it from the line's numbers+# cap the COLI_VK_DEVICE_CAP_MB (MiB, a fraction) under which exactly k layers fit:+# the bytes the probe held at the fit (its cap PTL_PROBE_CAP_MB less its free),+# the reserve, fixed and k layers, and half of layer k (k < L) or the tail too+# (k = L)+# fault the COLI_VK_STAGED_FAULT count to aim inside layer k's setup: the second time+# the point is reached within it (the placed line's marks; a first one there may+# be a fresh block's zero fill, which only skips the fill)+# matrices M and M' of the placed line, "M M'"+ptl_calc() {+ $PY - "$@" <<'PY'+import os, re, sys+what, eng, log = sys.argv[1], sys.argv[2], sys.argv[3]+k = int(sys.argv[4]) if len(sys.argv) > 4 else 0+lines = open(log, errors="replace").read().splitlines()+fit = [l for l in lines if l.startswith(f"[VK] {eng} chain fit: ")]+if what in ("predict", "cap"):+ if not fit: sys.exit("no fit line")+ f = fit[-1]+ num = lambda key: int(re.search(key + r" (\d+) B", f)[1])+ free, res, fixed, tail = num("free"), num("reserve"), num("fixed"), num("tail")+ layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]+ L, room = len(layers), max(0, free - res)+ if what == "predict":+ if fixed + sum(layers) <= room: print(L); sys.exit(0)+ n, acc = 0, fixed+ while n < L and acc + layers[n] <= room: acc += layers[n]; n += 1+ print(n); sys.exit(0)+ held = int(float(os.environ["PTL_PROBE_CAP_MB"]) * 1048576) - free+ need = res + fixed + sum(layers[:k]) + (layers[k] // 2 if k < L else tail + 4096)+ print(f"{(held + need) / 1048576:.6f}")+elif what == "fault":+ p = [l for l in lines if re.match(rf"\[VK\] {eng} chain: \d+ of \d+ layers placed", l)]+ m = re.search(r"reached ([\d,]+) times", p[-1] if p else "")+ if not m: sys.exit("no marks on the placed line")+ c = [int(x) for x in m[1].split(",")]+ lo = c[k - 1] if k else 0+ if c[k] - lo < 2: sys.exit(f"layer {k} reaches the point {c[k] - lo} times")+ print(lo + 2)+elif what == "matrices":+ p = [l for l in lines if re.match(rf"\[VK\] {eng} chain: \d+ of \d+ layers placed", l)]+ m = re.search(r"placed: (\d+) B of matrices on the device \(the fit counted (\d+) B", p[-1] if p else "")+ if not m: sys.exit("no placed line")+ print(m[1], m[2])+PY+}+# ptl_check_n <engine> <log> <want> <tag>: the final N is want, and the line says L+ptl_check_n() {+ local n; n=$(ptl_n "$1" "$2")+ [ "$n" = "$3" ] || { grep -E "^\[VK\] $1 chain" "$2"; fail "$4: N is $n, not $3"; }+}+# ptl_check_placed <engine> <log> <tag>: the matrices on the device after setup are the N+# layers' as the fit counted them, nothing of a layer that did not complete+ptl_check_placed() {+ local mm; mm=$(ptl_calc matrices "$1" "$2") || { grep -E "^\[VK\] $1 chain" "$2"; fail "$3: no placed line"; }+ set -- $mm "$3"+ [ "$1" = "$2" ] || fail "$3: $1 B of matrices on the device after setup, the N layers' are $2 B"+}+for f in tests/vulkan_partial_*.sh; do [ -e "$f" ] && . "$f"; done# ---- big prompt chunks and expert streaming (docs/vulkan.md, "Big prompt chunks and# expert streaming") ----# Every engine with the chain, on a prompt longer than its usual block, against its CPU@@ -3993,5 +4076,8 @@ case "${1:-}" indense-only-*) g=${1#dense-only-}; fn=dho_family_${g//-/_}declare -F "$fn" >/dev/null || { echo "no dense-only group ${g}" >&2; exit 2; }"$fn" ;;- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;+ partial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}+ declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }+ "$fn" ;;+ *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;esacdiff --git a/c/vk_chain.c b/c/vk_chain.cindex 805b9b32..c45c0f00 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -21,6 +21,20 @@#include <time.h>#include "compat.h"+/* Every device memory block goes through the backend's books (COLI_VK_DEVICE_CAP_MB,+ * coli_vk_mem_alloc): the chain's buffers count against the device's budget as its+ * weights and the tier's experts do. */+static VkResult vkc_mem_alloc(VkDevice d, const VkMemoryAllocateInfo *ai, const VkAllocationCallbacks *cb, VkDeviceMemory *m) {+ (void)cb;+ return (VkResult)coli_vk_mem_alloc((void *)d, ai, m);+}+static void vkc_mem_free(VkDevice d, VkDeviceMemory m, const VkAllocationCallbacks *cb) {+ (void)cb;+ coli_vk_mem_free((void *)d, &m);+}+#define vkAllocateMemory vkc_mem_alloc+#define vkFreeMemory vkc_mem_free+static double vkc_now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }/* ---- pipelines ------------------------------------------------------------------ */@@ -345,7 +359,7 @@ int vkc_init(void) {for (int k = 0; k < 3; k++) {K.pool[k].memtype = memtype_of(k);K.pool[k].mapped = memtype_host_visible(K.pool[k].memtype);- vka_pool_init(&K.pool[k].p, VKC_BLOCK, 0);+ vka_pool_init(&K.pool[k].p, coli_vk_block_bytes(VKC_BLOCK), 0); /* smaller under COLI_VK_DEVICE_CAP_MB */}if (!K.pool[VKC_UP].mapped || !K.pool[VKC_DOWN].mapped) return 0;K.ready = 1;@@ -1281,6 +1295,147 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes) {}return rows;}+/* ---- a partial chain (vk_chain.h, vkc_fit) ------------------------------------------- */+static const char *fit_size(size_t b, char *buf, size_t n) {+ if (b >= ((size_t)1 << 30)) snprintf(buf, n, "%.2f GiB", (double)b / 1073741824.0);+ else snprintf(buf, n, "%.1f MiB", (double)b / 1048576.0);+ return buf;+}+static size_t fit_up(size_t b, size_t a) { return a > 1 ? (b + a - 1) / a * a : b; }+size_t vkc_fit_reserve(void) {+ const char *e = getenv("COLI_VK_TIER_RESERVE_GB");+ double g = e && *e ? atof(e) : 1.0;+ return g > 0 ? (size_t)(g * 1073741824.0) : 0;+}+/* A weight pool's block, a block of each of the chain's three pools, and the frames'+ * first staging buffers: what the pools may hold beyond the bytes asked for. */+size_t vkc_fit_pools(void) {+ return coli_vk_block_bytes((size_t)256 << 20) + 3 * (size_t)coli_vk_block_bytes(VKC_BLOCK) ++ (size_t)VKC_FRAMES * ((size_t)4 << 20);+}+int vkc_fit_rows(int def) {+ const char *e = getenv("COLI_VK_CHAIN_ROWS");+ if (e && *e && strcmp(e, "auto") != 0) {+ int v = atoi(e);+ return v < 1 ? 1 : v > 65535 ? 65535 : v;+ }+ return def < 1 ? 1 : def;+}+size_t vkc_fit_tensor(int fmt, int I, int O, int gs) {+ size_t a = coli_vk_buffer_alignment();+ size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;+ size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * sizeof(float);+ return fit_up(rows ? rows : 4, a) + fit_up(sc ? sc : 4, a);+}+size_t vkc_fit_buf(size_t bytes) {+ size_t a = K.align > 16 ? K.align : 16, b = coli_vk_buffer_alignment();+ if (b > a) a = b;+ size_t v = bytes ? bytes : 4;+ return fit_up((v + 3) & ~(size_t)3, a);+}+int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *matrix_bytes,+ size_t fixed_bytes, size_t tail_bytes, VkcFit *fit) {+ const char *eng = engine ? engine : "engine";+ memset(fit, 0, sizeof *fit);+ fit->L = L < 0 ? 0 : L;+ fit->per = calloc((size_t)(fit->L ? fit->L : 1), sizeof(size_t));+ fit->mat = calloc((size_t)(fit->L ? fit->L : 1), sizeof(size_t));+ fit->marks = calloc((size_t)(fit->L ? fit->L : 1), sizeof(unsigned long long));+ size_t all = 0;+ for (int i = 0; i < fit->L; i++) {+ size_t b = layer_bytes ? layer_bytes[i] : 0;+ if (fit->per) fit->per[i] = b;+ if (fit->mat && matrix_bytes) fit->mat[i] = matrix_bytes[i];+ all += b;+ }+ fit->free_b = coli_vk_free_bytes();+ fit->reserve = vkc_fit_reserve();+ fit->pools = vkc_fit_pools();+ fit->engine_fixed = fixed_bytes;+ fit->fixed = fixed_bytes + fit->pools;+ fit->tail_b = tail_bytes;+ size_t room = fit->free_b > fit->reserve ? fit->free_b - fit->reserve : 0;+ const char *e = getenv("COLI_VK_CHAIN_LAYERS");+ char forced[48] = "";+ if (e && *e && strcmp(e, "auto") != 0) {+ int v = atoi(e);+ fit->n = v < 0 ? 0 : v > fit->L ? fit->L : v;+ fit->forced = 1;+ fit->tail = fit->n == fit->L && fit->L > 0;+ snprintf(forced, sizeof forced, "COLI_VK_CHAIN_LAYERS=%s", e);+ } else if (fit->fixed + all <= room) {+ fit->n = fit->L;+ fit->tail = fit->fixed + all + tail_bytes <= room;+ } else {+ size_t acc = fit->fixed;+ while (fit->n < fit->L && layer_bytes && acc + layer_bytes[fit->n] <= room) acc += layer_bytes[fit->n++];+ }+ for (int i = 0; i < fit->n; i++) fit->used_b += layer_bytes ? layer_bytes[i] : 0;+ /* the numbers, exactly (a test predicts N from them) */+ fprintf(stderr, "[VK] %s chain fit: free %zu B, reserve %zu B, fixed %zu B (the engine's %zu B, the pools' %zu B), tail %zu B, layers",+ eng, fit->free_b, fit->reserve, fit->fixed, fit->engine_fixed, fit->pools, fit->tail_b);+ for (int i = 0; i < fit->L; i++) fprintf(stderr, " %zu", layer_bytes ? layer_bytes[i] : (size_t)0);+ fprintf(stderr, " B");+ if (matrix_bytes) {+ fprintf(stderr, ", matrices");+ for (int i = 0; i < fit->L; i++) fprintf(stderr, " %zu", matrix_bytes[i]);+ fprintf(stderr, " B");+ }+ fprintf(stderr, "\n");+ char a[32], f[32], r[32], t[32], x[32], why[160];+ if (fit->forced) snprintf(why, sizeof why, "%s", forced);+ else snprintf(why, sizeof why, "free %s, reserve %s", fit_size(fit->free_b, f, sizeof f), fit_size(fit->reserve, r, sizeof r));+ if (fit->n == 0)+ fprintf(stderr, "[VK] %s chain: 0 of %d layers on the device: the chain stays off (%s%s%s%s%s)\n", eng, fit->L, why,+ fit->forced || !fit->L ? "" : "; layer 0 takes ", fit->forced || !fit->L ? "" : fit_size(fit->per ? fit->per[0] : 0, a, sizeof a),+ fit->forced || !fit->L ? "" : " beside ", fit->forced || !fit->L ? "" : fit_size(fit->fixed, x, sizeof x));+ else+ fprintf(stderr, "[VK] %s chain: %d of %d layers on the device (%s), %d on the CPU%s%s%s (%s)\n", eng, fit->n, fit->L,+ fit_size(fit->used_b, a, sizeof a), fit->L - fit->n,+ !fit->tail && tail_bytes ? ", the head and what goes with it (" : "",+ !fit->tail && tail_bytes ? fit_size(tail_bytes, t, sizeof t) : "", !fit->tail && tail_bytes ? ") on the CPU" : "", why);+ return fit->n;+}+void vkc_fit_shrink(const char *engine, VkcFit *fit, int n, const char *why) {+ if (!fit || n >= fit->n) return;+ if (n < 0) n = 0;+ const char *eng = engine ? engine : "engine";+ fit->n = n; fit->tail = 0; fit->used_b = 0;+ for (int i = 0; i < n && fit->per; i++) fit->used_b += fit->per[i];+ char a[32];+ if (n == 0)+ fprintf(stderr, "[VK] %s chain: 0 of %d layers on the device: the chain stays off (layer 0 did not reach the device: %s; "+ "what it had placed was freed)\n", eng, fit->L, why ? why : "refused");+ else+ fprintf(stderr, "[VK] %s chain: %d of %d layers on the device (%s), %d on the CPU (layer %d did not reach the device: %s; "+ "what it had placed was freed)\n", eng, n, fit->L, fit_size(fit->used_b, a, sizeof a), fit->L - n, n,+ why ? why : "refused");+}+void vkc_fit_mark(VkcFit *fit, int layer) {+ if (fit && fit->marks && layer >= 0 && layer < fit->L) fit->marks[layer] = coli_vk_fault_reached();+}+void vkc_fit_placed(const char *engine, const VkcFit *fit) {+ if (!fit) return;+ size_t w = 0, nt = 0, m = 0;+ coli_vk_mem_info(&w, &nt);+ for (int i = 0; i < fit->n && fit->mat; i++) m += fit->mat[i];+ fprintf(stderr, "[VK] %s chain: %d of %d layers placed: %zu B of matrices on the device (the fit counted %zu B for these "+ "layers), chain buffers %zu B, device memory held %zu B", engine ? engine : "engine", fit->n, fit->L, w, m,+ K.ready ? K.st.dev_bytes : (size_t)0, coli_vk_device_used());+ if (coli_vk_fault_set() && fit->marks) {+ fprintf(stderr, "; COLI_VK_STAGED_FAULT's point reached");+ for (int i = 0; i < fit->L; i++) fprintf(stderr, "%s%llu", i ? "," : " ", fit->marks[i]);+ fprintf(stderr, " times by the end of each layer");+ }+ fprintf(stderr, "\n");+}+int vkc_fit_partial(const VkcFit *fit) { return fit && fit->L > 0 && (fit->n < fit->L || !fit->tail); }+void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb) {+ if (vkc_ready()) vkc_finish(); /* no frame may still read them */+ for (int i = 0; i < nt; i++) if (t[i] && *t[i]) { coli_vk_tensor_free(*t[i]); *t[i] = NULL; }+ for (int i = 0; i < nb; i++) if (b[i] && *b[i]) { vkc_free(*b[i]); *b[i] = NULL; }+}+void vkc_prof_print(void) {if (!K.prof) return;double tot = 0; for (int k = 0; k < PK_N; k++) tot += K.prof_ms[k];diff --git a/c/vk_chain.h b/c/vk_chain.hindex b3749e1e..f67041ae 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -439,6 +439,73 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes);* V4_PREFILL_CHUNK) then hands the chain blocks of the chain's chunk; with* COLI_VK_CHAIN_ROWS set it keeps its own blocks, as before. */int vkc_chunk_auto(void);++/* ---- a partial chain: the first N layers on the device (docs/vulkan.md, "A partial chain")+ * When the dense layers do not all fit the device, the chain takes a contiguous prefix of+ * N layers (their matrices, the residual and their state, as for every layer before) and+ * the CPU runs the other L - N layers and the head; the residual (and an engine's per-token+ * stream state) moves to the host once per forward, after layer N - 1. N = L is the full+ * chain; N = 0 is the chain off, nothing uploaded, every byte left to the expert tier.+ *+ * vkc_fit decides N at startup, before any upload and before the RAM plan sizes the expert+ * cache, from the device's free memory (coli_vk_free_bytes) less:+ * - the reserve, COLI_VK_TIER_RESERVE_GB (1 GiB): what the tier leaves for scratch, KV+ * mirrors and the driver;+ * - fixed: the engine's fixed_bytes (its parameter buffers, the scratch of one prompt+ * chunk of vkc_fit_rows(..) rows, buffers every layer shares) plus the pools' own+ * granularity (vkc_fit_pools: a weight block, a block of each chain pool, the frames'+ * staging);+ * - each layer's bytes, layer_bytes[i]: its matrices (vkc_fit_tensor each) and its state+ * at its starting size (vkc_fit_buf each; the KV split covers the growth);+ * - tail_bytes: what the engine puts on the device only with every layer there (the head,+ * a final norm, matrices the per-matrix path would upload lazily).+ * The rule: N = L when fixed + every layer fits; the tail goes up too (fit->tail) when it+ * fits beside them; else N = the most layers from layer 0 with fixed + their bytes within+ * the room, the tail on the CPU. COLI_VK_CHAIN_LAYERS=n forces N (capped at L; 0 = the chain+ * off; the tail goes up when n = L). When everything fits the result is the full chain, as+ * before. Two lines on stderr:+ * [VK] <engine> chain fit: free F B, reserve R B, fixed X B (the engine's E B, the pools' P B), tail T B, layers b0 b1 .. B, matrices m0 m1 .. B+ * [VK] <engine> chain: N of L layers on the device (S), L-N on the CPU (free F, reserve R)+ * (the second with "(COLI_VK_CHAIN_LAYERS=n)" when forced; with N = 0 it says the chain stays+ * off). matrix_bytes[i] (NULL: not printed) is the payload of layer i's matrices as+ * coli_vk_mem_info counts it (coli_vk_tensor_payload each), for vkc_fit_placed's check.+ * fit->per and fit->mat are copies the fit keeps. Returns N.+ *+ * vkc_fit_shrink: a layer that did not fully reach the device during setup (a driver+ * refusing what the budget promised, an upload that failed): the engine freed everything+ * of that layer (and of the layers after it, if any were placed) and continues with N = n,+ * the layers before it. The line, in the same format:+ * [VK] <engine> chain: n of L layers on the device (S), L-n on the CPU (layer n did not reach the device: why; what it had placed was freed)+ * vkc_fit_mark: the engine finished layer i's setup (COLI_VK_STAGED_FAULT's point count is+ * noted for the placed line, so a test can aim a fault inside a given layer).+ * vkc_fit_placed: after setup, before the tier sizes itself:+ * [VK] <engine> chain: N of L layers placed: M B of matrices on the device (the fit counted M' B for these layers), chain buffers C B, device memory held H B[; COLI_VK_STAGED_FAULT's point reached c0,c1,.. times by the end of each layer]+ * vkc_fit_partial: 1 when something the full chain would place stays on the CPU (N < L, or+ * the tail kept off): the engine's per-matrix path must then not upload matrices that are+ * not on the device already (the CPU layers and the tail stay on the CPU).+ * vkc_layer_free: frees a layer's tensors (coli_vk_tensor_free) and chain buffers (vkc_free)+ * after the frames that may read them, setting each pointer to NULL. */+typedef struct {+ int L, n; /* the model's layers; the first n run on the device */+ int forced, tail; /* COLI_VK_CHAIN_LAYERS decided; the tail goes on the device */+ size_t free_b, reserve, fixed, engine_fixed, pools, tail_b, used_b; /* used_b: the n layers' bytes */+ size_t *per, *mat; /* each layer's bytes and its matrices' payload */+ unsigned long long *marks; /* COLI_VK_STAGED_FAULT's count at the end of each layer's setup */+} VkcFit;+int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *matrix_bytes,+ size_t fixed_bytes, size_t tail_bytes, VkcFit *fit);+void vkc_fit_shrink(const char *engine, VkcFit *fit, int n, const char *why);+void vkc_fit_mark(VkcFit *fit, int layer);+void vkc_fit_placed(const char *engine, const VkcFit *fit);+int vkc_fit_partial(const VkcFit *fit);+/* COLI_VK_CHAIN_ROWS when set to a number, else def: the rows the fit's scratch is for */+int vkc_fit_rows(int def);+size_t vkc_fit_tensor(int fmt, int I, int O, int gs); /* a resident tensor's device bytes, its two ranges aligned */+size_t vkc_fit_buf(size_t bytes); /* a chain buffer's (vkc_buf) device bytes */+size_t vkc_fit_reserve(void); /* COLI_VK_TIER_RESERVE_GB in bytes */+size_t vkc_fit_pools(void); /* the pools' granularity the fit adds to fixed */+void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb);+/* ---- a KV cache split between the device and the host (chain_kvs.comp) ----------------* Past the device's budget a layer's cache keeps `ns` blocks of B positions on the* device (the recent window, and where an engine selects positions, the blocks readdiff --git a/c/vk_tier.c b/c/vk_tier.cindex 4624de22..24e9d8a4 100644--- a/c/vk_tier.c+++ b/c/vk_tier.c@@ -1211,7 +1211,7 @@ int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {if (want > room) want = room;if (want < 0) want = 0;/* whole experts per pool block, so the count matches what the blocks can hold */- size_t blk = (size_t)256 << 20;+ size_t blk = coli_vk_block_bytes((size_t)256 << 20); /* the pool's blocks (smaller under COLI_VK_DEVICE_CAP_MB) */if ((size_t)want < blk) blk = (size_t)want;long long fit = 0;if (T.exp_bytes && blk >= T.exp_bytes) {@@ -1268,7 +1268,7 @@ int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {* block for a few experts. */size_t lim = T.budget, need = (size_t)(all + 1 + (T.st_ok ? T.st_slots : 0)) * T.exp_bytes;if (fit + (T.st_ok ? T.st_slots : 0) >= all && fit_raw >= all + (T.st_ok ? T.st_slots : 0) &&- need < lim && need <= ((size_t)256 << 20)) lim = need;+ need < lim && need <= coli_vk_block_bytes((size_t)256 << 20)) lim = need;coli_vk_tier_pool_limit(lim);const char *bal = getenv("COLI_VK_TIER_BALANCE");T.balance = T.c.in_ram != NULL && !(bal && *bal == '0');