Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(vulkan): device memory books, a device cap for tests, and the partial-chain fit

JustVugg061a9f33e26e
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index bfbe0124..b9074747 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -24,6 +24,128 @@ static double vk_now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &
#define VKCHECK(x, what) do { VkResult _r = (x); if (_r != VK_SUCCESS) { \
fprintf(stderr, "[VK] %s failed: %d\n", what, _r); return 0; } } while (0)
+/* ---- device memory books, and COLI_VK_DEVICE_CAP_MB (docs/vulkan.md, "A partial chain") --
+ * Every device memory allocation of the process goes through vk_mem_alloc / vk_mem_free
+ * (the macros below rename the Vulkan calls in this file; vk_chain.c reaches them through
+ * coli_vk_mem_alloc / coli_vk_mem_free): the bytes in a device-local heap are counted per
+ * device (0 = G, 1 = G2). COLI_VK_DEVICE_CAP_MB=n (tests; a fraction is taken) makes a
+ * device hold at most n MiB of them: an allocation past it fails as
+ * VK_ERROR_OUT_OF_DEVICE_MEMORY, as a full card's does, and the budget the engines read
+ * (coli_vk_mem_budget, _budget2, coli_vk_free_bytes, coli_vk_device_local_bytes, the
+ * expert batch's heap room) is the cap and those bytes. So Lavapipe behaves like a small
+ * card. Host memory imported in place (coli_vk_tensor_import) is the host's and is not
+ * counted. Under the cap the pools take small blocks (coli_vk_block_bytes) so that their
+ * granularity does not decide what fits. */
+typedef struct { uint64_t key, bytes; int dev; } VkMemRec;
+static struct {
+ pthread_mutex_t mx;
+ VkDevice dev[2];
+ uint32_t dl_types[2]; /* memory types whose heap is device-local */
+ uint64_t cap[2], used[2], peak[2];
+ unsigned long long refused[2];
+ VkMemRec *rec; size_t nrec, crec;
+} g_mem = {.mx = PTHREAD_MUTEX_INITIALIZER};
+static uint64_t vk_mem_key(VkDeviceMemory m) { uint64_t k = 0; memcpy(&k, &m, sizeof m); return k; }
+static int vk_mem_dev(VkDevice d) { return d && d == g_mem.dev[1] ? 1 : 0; }
+static int vk_mem_imported(const VkMemoryAllocateInfo *ai) {
+ for (const VkBaseInStructure *p = ai->pNext; p; p = p->pNext)
+ if (p->sType == VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT) return 1;
+ return 0;
+}
+static VkResult vk_mem_alloc(VkDevice d, const VkMemoryAllocateInfo *ai, const VkAllocationCallbacks *cb,
+ VkDeviceMemory *m) {
+ int k = vk_mem_dev(d);
+ int counted = ai->memoryTypeIndex < 32 && (g_mem.dl_types[k] >> ai->memoryTypeIndex & 1u) && !vk_mem_imported(ai);
+ uint64_t n = ai->allocationSize;
+ if (counted) {
+ pthread_mutex_lock(&g_mem.mx);
+ if (g_mem.cap[k] && g_mem.used[k] + n > g_mem.cap[k]) {
+ g_mem.refused[k]++;
+ pthread_mutex_unlock(&g_mem.mx);
+ return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+ }
+ g_mem.used[k] += n;
+ if (g_mem.used[k] > g_mem.peak[k]) g_mem.peak[k] = g_mem.used[k];
+ pthread_mutex_unlock(&g_mem.mx);
+ }
+ VkResult r = vkAllocateMemory(d, ai, cb, m);
+ if (!counted) return r;
+ pthread_mutex_lock(&g_mem.mx);
+ if (r != VK_SUCCESS) g_mem.used[k] -= n;
+ else {
+ if (g_mem.nrec == g_mem.crec) {
+ size_t c = g_mem.crec ? 2 * g_mem.crec : 256;
+ VkMemRec *nr = realloc(g_mem.rec, c * sizeof *nr);
+ if (nr) { g_mem.rec = nr; g_mem.crec = c; }
+ }
+ if (g_mem.nrec < g_mem.crec) g_mem.rec[g_mem.nrec++] = (VkMemRec){vk_mem_key(*m), n, k};
+ }
+ pthread_mutex_unlock(&g_mem.mx);
+ return r;
+}
+static void vk_mem_free(VkDevice d, VkDeviceMemory m, const VkAllocationCallbacks *cb) {
+ if (m != VK_NULL_HANDLE) {
+ uint64_t key = vk_mem_key(m);
+ int k = vk_mem_dev(d);
+ pthread_mutex_lock(&g_mem.mx);
+ for (size_t i = g_mem.nrec; i-- > 0;)
+ if (g_mem.rec[i].key == key && g_mem.rec[i].dev == k) {
+ g_mem.used[k] -= g_mem.rec[i].bytes;
+ g_mem.rec[i] = g_mem.rec[--g_mem.nrec];
+ break;
+ }
+ pthread_mutex_unlock(&g_mem.mx);
+ }
+ vkFreeMemory(d, m, cb);
+}
+#define vkAllocateMemory vk_mem_alloc
+#define vkFreeMemory vk_mem_free
+/* A device just created: which of its memory types count, and the cap. */
+static void vk_mem_device(int k, VkPhysicalDevice phys, VkDevice d) {
+ VkPhysicalDeviceMemoryProperties mp;
+ vkGetPhysicalDeviceMemoryProperties(phys, &mp);
+ uint32_t mask = 0;
+ for (uint32_t i = 0; i < mp.memoryTypeCount && i < 32; i++)
+ if (mp.memoryHeaps[mp.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) mask |= 1u << i;
+ const char *e = getenv("COLI_VK_DEVICE_CAP_MB");
+ double mb = e && *e ? atof(e) : 0.0;
+ pthread_mutex_lock(&g_mem.mx);
+ g_mem.dev[k] = d; g_mem.dl_types[k] = mask;
+ g_mem.cap[k] = mb > 0 ? (uint64_t)(mb * 1048576.0) : 0;
+ g_mem.used[k] = g_mem.peak[k] = 0; g_mem.refused[k] = 0;
+ pthread_mutex_unlock(&g_mem.mx);
+ if (g_mem.cap[k])
+ fprintf(stderr, "[VK] COLI_VK_DEVICE_CAP_MB=%s: %s takes at most %.3f MiB of device-local memory\n",
+ e, k ? "the second device" : "the device", (double)g_mem.cap[k] / 1048576.0);
+}
+/* The device is gone (shutdown): its books go with it. */
+static void vk_mem_forget(int k) {
+ pthread_mutex_lock(&g_mem.mx);
+ for (size_t i = g_mem.nrec; i-- > 0;)
+ if (g_mem.rec[i].dev == k) g_mem.rec[i] = g_mem.rec[--g_mem.nrec];
+ g_mem.dev[k] = VK_NULL_HANDLE; g_mem.dl_types[k] = 0; g_mem.used[k] = 0;
+ pthread_mutex_unlock(&g_mem.mx);
+}
+static uint64_t vk_mem_room(int k) { /* lock-free read is enough for a budget */
+ uint64_t c = g_mem.cap[k], u = __atomic_load_n(&g_mem.used[k], __ATOMIC_RELAXED);
+ return c > u ? c - u : 0;
+}
+static void vk_pool_blocks(int k); /* the weight pools' block size, below */
+size_t coli_vk_device_cap(void) { return (size_t)g_mem.cap[0]; }
+size_t coli_vk_device_used(void) { return (size_t)__atomic_load_n(&g_mem.used[0], __ATOMIC_RELAXED); }
+size_t coli_vk_block_bytes(size_t def) {
+ if (!g_mem.cap[0]) return def;
+ size_t b = (size_t)64 << 10; /* at most about 4096 blocks fill the cap */
+ while ((uint64_t)b < g_mem.cap[0] / 4096) b <<= 1;
+ return b < def ? b : def;
+}
+int coli_vk_mem_alloc(void *device, const void *info, void *memory) {
+ return (int)vk_mem_alloc((VkDevice)device, (const VkMemoryAllocateInfo *)info, NULL, (VkDeviceMemory *)memory);
+}
+void coli_vk_mem_free(void *device, const void *memory) {
+ vk_mem_free((VkDevice)device, *(const VkDeviceMemory *)memory, NULL);
+}
+
typedef struct VkWPool VkWPool;
struct ColiVkTensor {
VkBuffer wbuf, sbuf;
@@ -103,6 +225,9 @@ static int up_fault(const char *point) {
fprintf(stderr, "[VK] COLI_VK_STAGED_FAULT: %s #%ld fails\n", point, g_fault_at);
return 1;
}
+/* How many times the COLI_VK_STAGED_FAULT point was reached so far (0 when unset). */
+unsigned long long coli_vk_fault_reached(void) { return (unsigned long long)__atomic_load_n(&g_fault_n, __ATOMIC_RELAXED); }
+int coli_vk_fault_set(void) { return g_fault_point[0] != 0; }
typedef struct {
VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;
@@ -391,6 +516,11 @@ void coli_vk_alloc_priority(float p) { G.prio = p < 0 ? 0 : p > 1 ? 1 : p; }
/* Device-local heap usage/budget in GB (VK_EXT_memory_budget). Returns 0 when the
* extension is absent — callers then keep their count-based caps unchanged. */
int coli_vk_mem_budget(double *used_gb, double *budget_gb) {
+ if (g_mem.cap[0] && G.phys) { /* COLI_VK_DEVICE_CAP_MB: the cap and this process's bytes */
+ if (used_gb) *used_gb = (double)__atomic_load_n(&g_mem.used[0], __ATOMIC_RELAXED) / 1e9;
+ if (budget_gb) *budget_gb = (double)g_mem.cap[0] / 1e9;
+ return 1;
+ }
#ifdef VK_EXT_memory_budget
if (!G.has_budget || !G.phys) return 0;
VkPhysicalDeviceMemoryBudgetPropertiesEXT bud = {
@@ -410,6 +540,17 @@ int coli_vk_mem_budget(double *used_gb, double *budget_gb) {
(void)used_gb; (void)budget_gb; return 0;
#endif
}
+/* Free device memory now: the cap less what this process holds (COLI_VK_DEVICE_CAP_MB), else
+ * VK_EXT_memory_budget's budget less its usage, else the largest device-local heap less
+ * what this process holds. */
+size_t coli_vk_free_bytes(void) {
+ if (!G.phys) return 0;
+ if (g_mem.cap[0]) return (size_t)vk_mem_room(0);
+ double used = 0, bud = 0;
+ if (coli_vk_mem_budget(&used, &bud)) return bud > used ? (size_t)((bud - used) * 1e9) : 0;
+ size_t dl = coli_vk_device_local_bytes(), u = coli_vk_device_used();
+ return dl > u ? dl - u : 0;
+}
static int alloc_hostvis(size_t bytes, VkBuffer *buf, VkDeviceMemory *mem, void **ptr) {
return alloc_hostvis_mt(bytes, buf, mem, ptr, G.memtype);
}
@@ -807,6 +948,8 @@ int coli_vk_init(const char *spv_path) {
if (!G.dev)
#endif
VKCHECK(vkCreateDevice(G.phys, &di, NULL, &G.dev), "vkCreateDevice");
+ vk_mem_device(0, G.phys, G.dev); /* the device memory books, and COLI_VK_DEVICE_CAP_MB */
+ vk_pool_blocks(0);
vkGetDeviceQueue(G.dev, G.qfam, 0, &G.queue);
vkGetDeviceQueue(G.dev, G.tq_fam, G.tq_idx, &G.tqueue);
G.tq_shared = G.tqueue == G.queue;
@@ -1038,6 +1181,13 @@ struct VkWPool {
static VkWPool g_wpool = {.p = {.block_bytes = VK_WBLOCK}, .mx = PTHREAD_MUTEX_INITIALIZER, .dev = 0, .prio = -1.f, .counted = 1};
static VkWPool g_tpool = {.p = {.block_bytes = VK_WBLOCK}, .mx = PTHREAD_MUTEX_INITIALIZER, .dev = 0, .prio = 0.4f};
static VkWPool g_wpool2 = {.p = {.block_bytes = VK_WBLOCK}, .mx = PTHREAD_MUTEX_INITIALIZER, .dev = 1, .prio = -1.f};
+/* Device k's pools take VK_WBLOCK blocks, smaller ones under COLI_VK_DEVICE_CAP_MB. */
+static void vk_pool_blocks(int k) {
+ size_t bb = VK_WBLOCK;
+ if (g_mem.cap[k]) { bb = (size_t)64 << 10; while ((uint64_t)bb < g_mem.cap[k] / 4096 && bb < VK_WBLOCK) bb <<= 1; }
+ if (k == 0) { g_wpool.p.block_bytes = bb; g_tpool.p.block_bytes = bb; }
+ else g_wpool2.p.block_bytes = bb;
+}
static VkDevice pool_device(const VkWPool *P);
static uint32_t pool_memtype(const VkWPool *P);
static int pool_has_prio(const VkWPool *P);
@@ -1711,6 +1861,17 @@ int coli_vk_dense(void) { return g_dense_on; }
static int g_dho, g_dho_atexit, g_dho_on_device;
static char g_dho_engine[32];
static unsigned long long g_dho_drop_n, g_dho_drop_b, g_dho_load_n, g_dho_load_b;
+static int g_dho_lay_dev, g_dho_lay_all; /* a partial chain: the first lay_dev of lay_all layers dropped theirs */
+/* " (the n of L layers ...)" for the lines, empty unless the engine said a partial chain */
+static const char *dho_layers_note(char *buf, size_t n, int placed) {
+ buf[0] = 0;
+ if (g_dho_lay_all > 0 && g_dho_lay_dev < g_dho_lay_all) {
+ if (placed) snprintf(buf, n, " (the %d of %d layers on the device; the %d on the CPU keep theirs)",
+ g_dho_lay_dev, g_dho_lay_all, g_dho_lay_all - g_dho_lay_dev);
+ else snprintf(buf, n, ", the %d of %d layers on the device", g_dho_lay_dev, g_dho_lay_all);
+ }
+ return buf;
+}
static double dho_gib(unsigned long long b) { return (double)b / 1073741824.0; }
/* A size for the [VK] lines: GiB from 1 GiB, MiB below (a test fixture's KiB show as 0.0 MiB). */
static const char *dho_size(unsigned long long b, char *buf, size_t n) {
@@ -1738,10 +1899,11 @@ static void dho_exit_report(void) {
fprintf(stderr, "[VK] %s: dense weights at exit: on the device and in host RAM%s\n", g_dho_engine, r);
return;
}
- char a[32], b[32];
+ char a[32], b[32], ln[96];
fprintf(stderr, "[VK] %s: dense weights at exit: %llu matrices on the device only (%s of host copies "
- "dropped), %llu read back from disk for the CPU (%s)%s\n", g_dho_engine,
+ "dropped%s), %llu read back from disk for the CPU (%s)%s\n", g_dho_engine,
__atomic_load_n(&g_dho_drop_n, __ATOMIC_RELAXED), dho_size(__atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED), a, sizeof a),
+ dho_layers_note(ln, sizeof ln, 0),
__atomic_load_n(&g_dho_load_n, __ATOMIC_RELAXED), dho_size(__atomic_load_n(&g_dho_load_b, __ATOMIC_RELAXED), b, sizeof b), r);
}
static int dho_choice(int dense_on_device, size_t dense_bytes, char *why, size_t n) {
@@ -1800,11 +1962,14 @@ void coli_vk_dense_host_placed(const char *engine, const char *kept) {
double rss = dho_rss_gib();
char r[48] = "";
if (rss >= 0) snprintf(r, sizeof r, "; RSS now %.2f GiB", rss);
- char a[32];
- fprintf(stderr, "[VK] %s: %llu dense matrices on the device only, %s of host RAM given back%s%s%s\n",
+ char a[32], ln[96];
+ fprintf(stderr, "[VK] %s: %llu dense matrices on the device only, %s of host RAM given back%s%s%s%s\n",
engine, __atomic_load_n(&g_dho_drop_n, __ATOMIC_RELAXED),
- dho_size(__atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED), a, sizeof a), kept && *kept ? "; kept on the host: " : "",
- kept && *kept ? kept : "", r);
+ dho_size(__atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED), a, sizeof a), dho_layers_note(ln, sizeof ln, 1),
+ kept && *kept ? "; kept on the host: " : "", kept && *kept ? kept : "", r);
+}
+void coli_vk_dense_host_layers(int on_device, int layers) {
+ g_dho_lay_dev = on_device < 0 ? 0 : on_device; g_dho_lay_all = layers < 0 ? 0 : layers;
}
unsigned long long coli_vk_dense_host_dropped_bytes(void) { return __atomic_load_n(&g_dho_drop_b, __ATOMIC_RELAXED); }
@@ -2172,6 +2337,8 @@ int coli_vk_init_dev2(const char *spv_path, int devidx) {
.queueCreateInfoCount = 1, .pQueueCreateInfos = &qi,
.enabledExtensionCount = ndext, .ppEnabledExtensionNames = ndext ? dext : NULL};
VKCHECK(vkCreateDevice(G2.phys, &di, NULL, &G2.dev), "d2 vkCreateDevice");
+ vk_mem_device(1, G2.phys, G2.dev);
+ vk_pool_blocks(1);
vkGetDeviceQueue(G2.dev, G2.qfam, 0, &G2.queue);
int mt = pick_memtype(G2.phys);
if (mt < 0) { fprintf(stderr, "[VK] dev2: no host-visible memory\n"); return 0; }
@@ -2215,6 +2382,11 @@ int coli_vk_dev2_available(void) { return G2.ready; }
int coli_vk_tensor_dev(const ColiVkTensor *t) { return t ? t->dev : 0; }
int coli_vk_mem_budget2(double *used_gb, double *budget_gb) {
+ if (g_mem.cap[1] && G2.phys) { /* COLI_VK_DEVICE_CAP_MB, as coli_vk_mem_budget */
+ if (used_gb) *used_gb = (double)__atomic_load_n(&g_mem.used[1], __ATOMIC_RELAXED) / 1e9;
+ if (budget_gb) *budget_gb = (double)g_mem.cap[1] / 1e9;
+ return 1;
+ }
#ifdef VK_EXT_memory_budget
if (!G2.has_budget || !G2.phys) return 0;
VkPhysicalDeviceMemoryBudgetPropertiesEXT bud = {
@@ -2885,7 +3057,8 @@ void coli_vk_tier_pool_limit(size_t bytes) {
pthread_mutex_lock(&g_tpool.mx);
g_tpool.p.limit = bytes;
/* blocks no larger than the budget, so a small budget is not one refused block */
- g_tpool.p.block_bytes = bytes && bytes < VK_WBLOCK ? ((bytes + 4095) & ~(size_t)4095) : VK_WBLOCK;
+ size_t bb = coli_vk_block_bytes(VK_WBLOCK);
+ g_tpool.p.block_bytes = bytes && bytes < bb ? ((bytes + 4095) & ~(size_t)4095) : bb;
/* the empty block the pool keeps as a spare (vka_free) goes too: a tier started
* again after a shutdown would otherwise fill it past its new, smaller budget */
for (int k = 0; k < g_tpool.p.nb; k++)
@@ -2922,6 +3095,7 @@ size_t coli_vk_device_local_bytes(void) {
VkDeviceSize m = 0;
for (uint32_t i = 0; i < mp.memoryHeapCount; i++)
if ((mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && mp.memoryHeaps[i].size > m) m = mp.memoryHeaps[i].size;
+ if (g_mem.cap[0] && g_mem.cap[0] < m) m = g_mem.cap[0]; /* COLI_VK_DEVICE_CAP_MB */
return (size_t)m;
}
const char *coli_vk_device_name(void) {
@@ -3588,6 +3762,8 @@ int coli_vk_xb_sub_fit(int rows, int experts, size_t extra_budget) {
#ifdef VK_EXT_memory_budget
if (G.has_budget) room[h] = bud.heapBudget[h] > bud.heapUsage[h] ? (size_t)((bud.heapBudget[h] - bud.heapUsage[h]) / 2) : 0;
#endif
+ if (g_mem.cap[0] && (mp.memoryProperties.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) &&
+ room[h] > vk_mem_room(0) / 2) room[h] = (size_t)(vk_mem_room(0) / 2); /* COLI_VK_DEVICE_CAP_MB */
}
uint32_t xheap = mp.memoryProperties.memoryTypes[G.memtype].heapIndex;
uint32_t iheap = mp.memoryProperties.memoryTypes[G.memtype_dev].heapIndex;
@@ -3788,6 +3964,10 @@ int coli_vk_tensor_refill(ColiVkTensor *t, uint8_t **rows, size_t *stride, float
}
int coli_vk_staged(void) { return G.ready && g_up[0].on; }
size_t coli_vk_tensor_row_bytes(int fmt, int I) { return cpu_row_bytes(fmt, I); }
+size_t coli_vk_tensor_payload(int fmt, int I, int O, int gs) {
+ if (fmt == 4 || fmt == 7 || fmt == 12 || fmt == 13) { if (gs < 1) return 0; } else gs = 0;
+ return (size_t)rowwords(fmt, I) * 4 * (size_t)O + scale_floats(fmt, I, O, gs) * sizeof(float);
+}
size_t coli_vk_buffer_alignment(void) { return G.buf_align ? G.buf_align : 256; }
size_t coli_vk_tensor_scale_count(int fmt, int I, int O, int gs) { return scale_floats(fmt, I, O, gs); }
@@ -3886,6 +4066,8 @@ void coli_vk_shutdown(void) {
vkDestroyDevice(G.dev, NULL);
vkDestroyInstance(G.inst, NULL);
memset(&G, 0, sizeof(G));
+ vk_mem_forget(0);
+ vk_pool_blocks(0);
}
/* ---- the dense chain's view of the device (vk_chain.c) -------------------------
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index a76f0405..0608bc06 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -33,6 +33,32 @@ void coli_vk_set_swiglu_limit(float limit);
void coli_vk_alloc_priority(float p);
int coli_vk_mem_budget(double *used_gb, double *budget_gb);
+/* Device memory books (docs/vulkan.md, "A partial chain"). Every device memory allocation
+ * of the backend and of the chain is counted per device when its heap is device-local.
+ * COLI_VK_DEVICE_CAP_MB=n (tests; n may be a fraction) makes the device hold at most n MiB:
+ * an allocation past it fails as out of device memory, and coli_vk_mem_budget (and
+ * _budget2, coli_vk_free_bytes, coli_vk_device_local_bytes) report the cap and this
+ * process's bytes, so a CPU device behaves like a small card.
+ * coli_vk_device_cap the cap in bytes (0: none)
+ * coli_vk_device_used device-local bytes this process holds on the device
+ * coli_vk_free_bytes free device memory now: the cap less what is held, else the
+ * budget less its usage (VK_EXT_memory_budget), else the largest
+ * device-local heap less what is held
+ * coli_vk_block_bytes the block size a memory pool of `def` bytes takes (smaller
+ * under the cap, at most about 4096 blocks to fill it)
+ * coli_vk_mem_alloc/free vkAllocateMemory / vkFreeMemory through the books (vk_chain.c):
+ * info is a VkMemoryAllocateInfo *, memory a VkDeviceMemory *;
+ * the result is a VkResult */
+size_t coli_vk_device_cap(void);
+size_t coli_vk_device_used(void);
+size_t coli_vk_free_bytes(void);
+size_t coli_vk_block_bytes(size_t def);
+int coli_vk_mem_alloc(void *device, const void *info, void *memory);
+void coli_vk_mem_free(void *device, const void *memory);
+/* COLI_VK_STAGED_FAULT (tests): whether a point is set, and how many times it was reached. */
+int coli_vk_fault_set(void);
+unsigned long long coli_vk_fault_reached(void);
+
/* y[S,O] = (x[S,I] @ dequant(W[O,I])^T) * scale[O].
* fmt matches QT in glm.c: 1=int8, 2=int4. (0=f32,3=int2 fall back to CPU.)
* fmt 10 = plain f32 weights and 11 = bf16 weights (low half = even column): no
@@ -179,6 +205,9 @@ int coli_vk_tier_tensor(ColiVkTensor **t, int fmt, int I, int O, int gs,
int coli_vk_tensor_commit(ColiVkTensor *const *t, int n);
int coli_vk_staged(void); /* 1 = resident data goes to the device through staged uploads */
size_t coli_vk_tensor_row_bytes(int fmt, int I);
+/* What coli_vk_tensor_ensure(fmt, I, O, gs) adds to coli_vk_mem_info's bytes: the rows
+ * at their padded stride and the scales (0 for a grouped format without a group size). */
+size_t coli_vk_tensor_payload(int fmt, int I, int O, int gs);
size_t coli_vk_buffer_alignment(void); /* where a weight range may start (bytes) */
size_t coli_vk_tensor_scale_count(int fmt, int I, int O, int gs);
@@ -298,6 +327,11 @@ void coli_vk_dense_host_reloaded(size_t bytes);/* a host copy read back from dis
* only, the RAM given back, and what stays on the host (kept: NULL or a short note). */
void coli_vk_dense_host_placed(const char *engine, const char *kept);
unsigned long long coli_vk_dense_host_dropped_bytes(void);
+/* A partial chain (vk_chain.h, vkc_fit): only the first `on_device` of `layers` layers drop
+ * their host copies; the placed and exit lines then say so (nothing changes when
+ * on_device == layers, or before any call). Call it before coli_vk_dense_host_placed. The
+ * bytes the engine passes coli_vk_dense_host_decide are those layers' only. */
+void coli_vk_dense_host_layers(int on_device, int layers);
/* COLI_VK_SHADERS (the .spv or its directory), else shaders/ next to the binary, else
* shaders/ in the working directory. buf holds the result when it is not a literal. */
const char *coli_vk_shader_path(char *buf, size_t n);
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index b3aa5490..5c2fde57 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -24,6 +24,9 @@
# bash tests/vulkan_engines.sh staged-faults | staged-faults-sanitize # staged uploads failing
# bash tests/vulkan_engines.sh dense-only-<group> | dense-only-<group>-sanitize # the dense weights
# on the device only (COLI_VK_DENSE_HOST=0), tests/vulkan_dense_only_<group>.sh
+# bash tests/vulkan_engines.sh partial-<group> | partial-<group>-sanitize # the partial chain:
+# the first N layers on the device (COLI_VK_CHAIN_LAYERS, COLI_VK_DEVICE_CAP_MB),
+# tests/vulkan_partial_<group>.sh
# bash tests/vulkan_engines.sh decide | decide-sanitize # Laya, GLiNER2.5-Decide and Clef's DECIDE
#
# Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of
@@ -2883,6 +2886,86 @@ PY
echo "OK $tag: $(grep -o 'given back.*' vk.log | head -1)"
}
for f in tests/vulkan_dense_only_*.sh; do [ -e "$f" ] && . "$f"; done
+
+# ---- the partial chain (docs/vulkan.md, "A partial chain"): the first N layers on the
+# device, the rest and the head on the CPU. One file per engine group,
+# tests/vulkan_partial_<group>.sh, each defining ptl_family_<group> and
+# ptl_family_<group>_sanitize (run as partial-<group> and partial-<group>-sanitize);
+# tests/vulkan_partial_deepseek.sh is the template. The helpers read the engine's lines:
+# [VK] <engine> chain fit: free F B, reserve R B, fixed X B (...), tail T B, layers b0 .. B, matrices m0 .. B
+# [VK] <engine> chain: N of L layers on the device ... (the last one is the final N)
+# [VK] <engine> chain: N of L layers placed: M B of matrices on the device (the fit counted M' B ...), ...
+#
+# ptl_n <engine> <log>: the final N (-1 without a line)
+ptl_n() {
+ local n; n=$(sed -n "s/^\[VK\] $1 chain: \([0-9][0-9]*\) of [0-9][0-9]* layers on the device.*/\1/p" "$2" | tail -1)
+ echo "${n:--1}"
+}
+# ptl_L <engine> <log>: L, the model's layers, from the same line
+ptl_L() {
+ local n; n=$(sed -n "s/^\[VK\] $1 chain: [0-9][0-9]* of \([0-9][0-9]*\) layers on the device.*/\1/p" "$2" | tail -1)
+ echo "${n:--1}"
+}
+# ptl_calc <what> <engine> <log> [k]: arithmetic on the fit line (a probe run's for "cap")
+# predict N as vkc_fit's rule gives it from the line's numbers
+# cap the COLI_VK_DEVICE_CAP_MB (MiB, a fraction) under which exactly k layers fit:
+# the bytes the probe held at the fit (its cap PTL_PROBE_CAP_MB less its free),
+# the reserve, fixed and k layers, and half of layer k (k < L) or the tail too
+# (k = L)
+# fault the COLI_VK_STAGED_FAULT count to aim inside layer k's setup: the second time
+# the point is reached within it (the placed line's marks; a first one there may
+# be a fresh block's zero fill, which only skips the fill)
+# matrices M and M' of the placed line, "M M'"
+ptl_calc() {
+ $PY - "$@" <<'PY'
+import os, re, sys
+what, eng, log = sys.argv[1], sys.argv[2], sys.argv[3]
+k = int(sys.argv[4]) if len(sys.argv) > 4 else 0
+lines = open(log, errors="replace").read().splitlines()
+fit = [l for l in lines if l.startswith(f"[VK] {eng} chain fit: ")]
+if what in ("predict", "cap"):
+ if not fit: sys.exit("no fit line")
+ f = fit[-1]
+ num = lambda key: int(re.search(key + r" (\d+) B", f)[1])
+ free, res, fixed, tail = num("free"), num("reserve"), num("fixed"), num("tail")
+ layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]
+ L, room = len(layers), max(0, free - res)
+ if what == "predict":
+ if fixed + sum(layers) <= room: print(L); sys.exit(0)
+ n, acc = 0, fixed
+ while n < L and acc + layers[n] <= room: acc += layers[n]; n += 1
+ print(n); sys.exit(0)
+ held = int(float(os.environ["PTL_PROBE_CAP_MB"]) * 1048576) - free
+ need = res + fixed + sum(layers[:k]) + (layers[k] // 2 if k < L else tail + 4096)
+ print(f"{(held + need) / 1048576:.6f}")
+elif what == "fault":
+ p = [l for l in lines if re.match(rf"\[VK\] {eng} chain: \d+ of \d+ layers placed", l)]
+ m = re.search(r"reached ([\d,]+) times", p[-1] if p else "")
+ if not m: sys.exit("no marks on the placed line")
+ c = [int(x) for x in m[1].split(",")]
+ lo = c[k - 1] if k else 0
+ if c[k] - lo < 2: sys.exit(f"layer {k} reaches the point {c[k] - lo} times")
+ print(lo + 2)
+elif what == "matrices":
+ p = [l for l in lines if re.match(rf"\[VK\] {eng} chain: \d+ of \d+ layers placed", l)]
+ m = re.search(r"placed: (\d+) B of matrices on the device \(the fit counted (\d+) B", p[-1] if p else "")
+ if not m: sys.exit("no placed line")
+ print(m[1], m[2])
+PY
+}
+# ptl_check_n <engine> <log> <want> <tag>: the final N is want, and the line says L
+ptl_check_n() {
+ local n; n=$(ptl_n "$1" "$2")
+ [ "$n" = "$3" ] || { grep -E "^\[VK\] $1 chain" "$2"; fail "$4: N is $n, not $3"; }
+}
+# ptl_check_placed <engine> <log> <tag>: the matrices on the device after setup are the N
+# layers' as the fit counted them, nothing of a layer that did not complete
+ptl_check_placed() {
+ local mm; mm=$(ptl_calc matrices "$1" "$2") || { grep -E "^\[VK\] $1 chain" "$2"; fail "$3: no placed line"; }
+ set -- $mm "$3"
+ [ "$1" = "$2" ] || fail "$3: $1 B of matrices on the device after setup, the N layers' are $2 B"
+}
+for f in tests/vulkan_partial_*.sh; do [ -e "$f" ] && . "$f"; done
# ---- big prompt chunks and expert streaming (docs/vulkan.md, "Big prompt chunks and
# expert streaming") ----
# Every engine with the chain, on a prompt longer than its usual block, against its CPU
@@ -3993,5 +4076,8 @@ case "${1:-}" in
dense-only-*) g=${1#dense-only-}; fn=dho_family_${g//-/_}
declare -F "$fn" >/dev/null || { echo "no dense-only group ${g}" >&2; exit 2; }
"$fn" ;;
- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;
+ partial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}
+ declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }
+ "$fn" ;;
+ *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize" >&2; exit 2 ;;
esac
diff --git a/c/vk_chain.c b/c/vk_chain.c
index 805b9b32..c45c0f00 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -21,6 +21,20 @@
#include <time.h>
#include "compat.h"
+/* Every device memory block goes through the backend's books (COLI_VK_DEVICE_CAP_MB,
+ * coli_vk_mem_alloc): the chain's buffers count against the device's budget as its
+ * weights and the tier's experts do. */
+static VkResult vkc_mem_alloc(VkDevice d, const VkMemoryAllocateInfo *ai, const VkAllocationCallbacks *cb, VkDeviceMemory *m) {
+ (void)cb;
+ return (VkResult)coli_vk_mem_alloc((void *)d, ai, m);
+}
+static void vkc_mem_free(VkDevice d, VkDeviceMemory m, const VkAllocationCallbacks *cb) {
+ (void)cb;
+ coli_vk_mem_free((void *)d, &m);
+}
+#define vkAllocateMemory vkc_mem_alloc
+#define vkFreeMemory vkc_mem_free
+
static double vkc_now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }
/* ---- pipelines ------------------------------------------------------------------ */
@@ -345,7 +359,7 @@ int vkc_init(void) {
for (int k = 0; k < 3; k++) {
K.pool[k].memtype = memtype_of(k);
K.pool[k].mapped = memtype_host_visible(K.pool[k].memtype);
- vka_pool_init(&K.pool[k].p, VKC_BLOCK, 0);
+ vka_pool_init(&K.pool[k].p, coli_vk_block_bytes(VKC_BLOCK), 0); /* smaller under COLI_VK_DEVICE_CAP_MB */
}
if (!K.pool[VKC_UP].mapped || !K.pool[VKC_DOWN].mapped) return 0;
K.ready = 1;
@@ -1281,6 +1295,147 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes) {
}
return rows;
}
+/* ---- a partial chain (vk_chain.h, vkc_fit) ------------------------------------------- */
+static const char *fit_size(size_t b, char *buf, size_t n) {
+ if (b >= ((size_t)1 << 30)) snprintf(buf, n, "%.2f GiB", (double)b / 1073741824.0);
+ else snprintf(buf, n, "%.1f MiB", (double)b / 1048576.0);
+ return buf;
+}
+static size_t fit_up(size_t b, size_t a) { return a > 1 ? (b + a - 1) / a * a : b; }
+size_t vkc_fit_reserve(void) {
+ const char *e = getenv("COLI_VK_TIER_RESERVE_GB");
+ double g = e && *e ? atof(e) : 1.0;
+ return g > 0 ? (size_t)(g * 1073741824.0) : 0;
+}
+/* A weight pool's block, a block of each of the chain's three pools, and the frames'
+ * first staging buffers: what the pools may hold beyond the bytes asked for. */
+size_t vkc_fit_pools(void) {
+ return coli_vk_block_bytes((size_t)256 << 20) + 3 * (size_t)coli_vk_block_bytes(VKC_BLOCK) +
+ (size_t)VKC_FRAMES * ((size_t)4 << 20);
+}
+int vkc_fit_rows(int def) {
+ const char *e = getenv("COLI_VK_CHAIN_ROWS");
+ if (e && *e && strcmp(e, "auto") != 0) {
+ int v = atoi(e);
+ return v < 1 ? 1 : v > 65535 ? 65535 : v;
+ }
+ return def < 1 ? 1 : def;
+}
+size_t vkc_fit_tensor(int fmt, int I, int O, int gs) {
+ size_t a = coli_vk_buffer_alignment();
+ size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;
+ size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * sizeof(float);
+ return fit_up(rows ? rows : 4, a) + fit_up(sc ? sc : 4, a);
+}
+size_t vkc_fit_buf(size_t bytes) {
+ size_t a = K.align > 16 ? K.align : 16, b = coli_vk_buffer_alignment();
+ if (b > a) a = b;
+ size_t v = bytes ? bytes : 4;
+ return fit_up((v + 3) & ~(size_t)3, a);
+}
+int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *matrix_bytes,
+ size_t fixed_bytes, size_t tail_bytes, VkcFit *fit) {
+ const char *eng = engine ? engine : "engine";
+ memset(fit, 0, sizeof *fit);
+ fit->L = L < 0 ? 0 : L;
+ fit->per = calloc((size_t)(fit->L ? fit->L : 1), sizeof(size_t));
+ fit->mat = calloc((size_t)(fit->L ? fit->L : 1), sizeof(size_t));
+ fit->marks = calloc((size_t)(fit->L ? fit->L : 1), sizeof(unsigned long long));
+ size_t all = 0;
+ for (int i = 0; i < fit->L; i++) {
+ size_t b = layer_bytes ? layer_bytes[i] : 0;
+ if (fit->per) fit->per[i] = b;
+ if (fit->mat && matrix_bytes) fit->mat[i] = matrix_bytes[i];
+ all += b;
+ }
+ fit->free_b = coli_vk_free_bytes();
+ fit->reserve = vkc_fit_reserve();
+ fit->pools = vkc_fit_pools();
+ fit->engine_fixed = fixed_bytes;
+ fit->fixed = fixed_bytes + fit->pools;
+ fit->tail_b = tail_bytes;
+ size_t room = fit->free_b > fit->reserve ? fit->free_b - fit->reserve : 0;
+ const char *e = getenv("COLI_VK_CHAIN_LAYERS");
+ char forced[48] = "";
+ if (e && *e && strcmp(e, "auto") != 0) {
+ int v = atoi(e);
+ fit->n = v < 0 ? 0 : v > fit->L ? fit->L : v;
+ fit->forced = 1;
+ fit->tail = fit->n == fit->L && fit->L > 0;
+ snprintf(forced, sizeof forced, "COLI_VK_CHAIN_LAYERS=%s", e);
+ } else if (fit->fixed + all <= room) {
+ fit->n = fit->L;
+ fit->tail = fit->fixed + all + tail_bytes <= room;
+ } else {
+ size_t acc = fit->fixed;
+ while (fit->n < fit->L && layer_bytes && acc + layer_bytes[fit->n] <= room) acc += layer_bytes[fit->n++];
+ }
+ for (int i = 0; i < fit->n; i++) fit->used_b += layer_bytes ? layer_bytes[i] : 0;
+ /* the numbers, exactly (a test predicts N from them) */
+ fprintf(stderr, "[VK] %s chain fit: free %zu B, reserve %zu B, fixed %zu B (the engine's %zu B, the pools' %zu B), tail %zu B, layers",
+ eng, fit->free_b, fit->reserve, fit->fixed, fit->engine_fixed, fit->pools, fit->tail_b);
+ for (int i = 0; i < fit->L; i++) fprintf(stderr, " %zu", layer_bytes ? layer_bytes[i] : (size_t)0);
+ fprintf(stderr, " B");
+ if (matrix_bytes) {
+ fprintf(stderr, ", matrices");
+ for (int i = 0; i < fit->L; i++) fprintf(stderr, " %zu", matrix_bytes[i]);
+ fprintf(stderr, " B");
+ }
+ fprintf(stderr, "\n");
+ char a[32], f[32], r[32], t[32], x[32], why[160];
+ if (fit->forced) snprintf(why, sizeof why, "%s", forced);
+ else snprintf(why, sizeof why, "free %s, reserve %s", fit_size(fit->free_b, f, sizeof f), fit_size(fit->reserve, r, sizeof r));
+ if (fit->n == 0)
+ fprintf(stderr, "[VK] %s chain: 0 of %d layers on the device: the chain stays off (%s%s%s%s%s)\n", eng, fit->L, why,
+ fit->forced || !fit->L ? "" : "; layer 0 takes ", fit->forced || !fit->L ? "" : fit_size(fit->per ? fit->per[0] : 0, a, sizeof a),
+ fit->forced || !fit->L ? "" : " beside ", fit->forced || !fit->L ? "" : fit_size(fit->fixed, x, sizeof x));
+ else
+ fprintf(stderr, "[VK] %s chain: %d of %d layers on the device (%s), %d on the CPU%s%s%s (%s)\n", eng, fit->n, fit->L,
+ fit_size(fit->used_b, a, sizeof a), fit->L - fit->n,
+ !fit->tail && tail_bytes ? ", the head and what goes with it (" : "",
+ !fit->tail && tail_bytes ? fit_size(tail_bytes, t, sizeof t) : "", !fit->tail && tail_bytes ? ") on the CPU" : "", why);
+ return fit->n;
+}
+void vkc_fit_shrink(const char *engine, VkcFit *fit, int n, const char *why) {
+ if (!fit || n >= fit->n) return;
+ if (n < 0) n = 0;
+ const char *eng = engine ? engine : "engine";
+ fit->n = n; fit->tail = 0; fit->used_b = 0;
+ for (int i = 0; i < n && fit->per; i++) fit->used_b += fit->per[i];
+ char a[32];
+ if (n == 0)
+ fprintf(stderr, "[VK] %s chain: 0 of %d layers on the device: the chain stays off (layer 0 did not reach the device: %s; "
+ "what it had placed was freed)\n", eng, fit->L, why ? why : "refused");
+ else
+ fprintf(stderr, "[VK] %s chain: %d of %d layers on the device (%s), %d on the CPU (layer %d did not reach the device: %s; "
+ "what it had placed was freed)\n", eng, n, fit->L, fit_size(fit->used_b, a, sizeof a), fit->L - n, n,
+ why ? why : "refused");
+}
+void vkc_fit_mark(VkcFit *fit, int layer) {
+ if (fit && fit->marks && layer >= 0 && layer < fit->L) fit->marks[layer] = coli_vk_fault_reached();
+}
+void vkc_fit_placed(const char *engine, const VkcFit *fit) {
+ if (!fit) return;
+ size_t w = 0, nt = 0, m = 0;
+ coli_vk_mem_info(&w, &nt);
+ for (int i = 0; i < fit->n && fit->mat; i++) m += fit->mat[i];
+ fprintf(stderr, "[VK] %s chain: %d of %d layers placed: %zu B of matrices on the device (the fit counted %zu B for these "
+ "layers), chain buffers %zu B, device memory held %zu B", engine ? engine : "engine", fit->n, fit->L, w, m,
+ K.ready ? K.st.dev_bytes : (size_t)0, coli_vk_device_used());
+ if (coli_vk_fault_set() && fit->marks) {
+ fprintf(stderr, "; COLI_VK_STAGED_FAULT's point reached");
+ for (int i = 0; i < fit->L; i++) fprintf(stderr, "%s%llu", i ? "," : " ", fit->marks[i]);
+ fprintf(stderr, " times by the end of each layer");
+ }
+ fprintf(stderr, "\n");
+}
+int vkc_fit_partial(const VkcFit *fit) { return fit && fit->L > 0 && (fit->n < fit->L || !fit->tail); }
+void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb) {
+ if (vkc_ready()) vkc_finish(); /* no frame may still read them */
+ for (int i = 0; i < nt; i++) if (t[i] && *t[i]) { coli_vk_tensor_free(*t[i]); *t[i] = NULL; }
+ for (int i = 0; i < nb; i++) if (b[i] && *b[i]) { vkc_free(*b[i]); *b[i] = NULL; }
+}
+
void vkc_prof_print(void) {
if (!K.prof) return;
double tot = 0; for (int k = 0; k < PK_N; k++) tot += K.prof_ms[k];
diff --git a/c/vk_chain.h b/c/vk_chain.h
index b3749e1e..f67041ae 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -439,6 +439,73 @@ int vkc_chunk_rows(const char *engine, size_t row_bytes);
* V4_PREFILL_CHUNK) then hands the chain blocks of the chain's chunk; with
* COLI_VK_CHAIN_ROWS set it keeps its own blocks, as before. */
int vkc_chunk_auto(void);
+
+/* ---- a partial chain: the first N layers on the device (docs/vulkan.md, "A partial chain")
+ * When the dense layers do not all fit the device, the chain takes a contiguous prefix of
+ * N layers (their matrices, the residual and their state, as for every layer before) and
+ * the CPU runs the other L - N layers and the head; the residual (and an engine's per-token
+ * stream state) moves to the host once per forward, after layer N - 1. N = L is the full
+ * chain; N = 0 is the chain off, nothing uploaded, every byte left to the expert tier.
+ *
+ * vkc_fit decides N at startup, before any upload and before the RAM plan sizes the expert
+ * cache, from the device's free memory (coli_vk_free_bytes) less:
+ * - the reserve, COLI_VK_TIER_RESERVE_GB (1 GiB): what the tier leaves for scratch, KV
+ * mirrors and the driver;
+ * - fixed: the engine's fixed_bytes (its parameter buffers, the scratch of one prompt
+ * chunk of vkc_fit_rows(..) rows, buffers every layer shares) plus the pools' own
+ * granularity (vkc_fit_pools: a weight block, a block of each chain pool, the frames'
+ * staging);
+ * - each layer's bytes, layer_bytes[i]: its matrices (vkc_fit_tensor each) and its state
+ * at its starting size (vkc_fit_buf each; the KV split covers the growth);
+ * - tail_bytes: what the engine puts on the device only with every layer there (the head,
+ * a final norm, matrices the per-matrix path would upload lazily).
+ * The rule: N = L when fixed + every layer fits; the tail goes up too (fit->tail) when it
+ * fits beside them; else N = the most layers from layer 0 with fixed + their bytes within
+ * the room, the tail on the CPU. COLI_VK_CHAIN_LAYERS=n forces N (capped at L; 0 = the chain
+ * off; the tail goes up when n = L). When everything fits the result is the full chain, as
+ * before. Two lines on stderr:
+ * [VK] <engine> chain fit: free F B, reserve R B, fixed X B (the engine's E B, the pools' P B), tail T B, layers b0 b1 .. B, matrices m0 m1 .. B
+ * [VK] <engine> chain: N of L layers on the device (S), L-N on the CPU (free F, reserve R)
+ * (the second with "(COLI_VK_CHAIN_LAYERS=n)" when forced; with N = 0 it says the chain stays
+ * off). matrix_bytes[i] (NULL: not printed) is the payload of layer i's matrices as
+ * coli_vk_mem_info counts it (coli_vk_tensor_payload each), for vkc_fit_placed's check.
+ * fit->per and fit->mat are copies the fit keeps. Returns N.
+ *
+ * vkc_fit_shrink: a layer that did not fully reach the device during setup (a driver
+ * refusing what the budget promised, an upload that failed): the engine freed everything
+ * of that layer (and of the layers after it, if any were placed) and continues with N = n,
+ * the layers before it. The line, in the same format:
+ * [VK] <engine> chain: n of L layers on the device (S), L-n on the CPU (layer n did not reach the device: why; what it had placed was freed)
+ * vkc_fit_mark: the engine finished layer i's setup (COLI_VK_STAGED_FAULT's point count is
+ * noted for the placed line, so a test can aim a fault inside a given layer).
+ * vkc_fit_placed: after setup, before the tier sizes itself:
+ * [VK] <engine> chain: N of L layers placed: M B of matrices on the device (the fit counted M' B for these layers), chain buffers C B, device memory held H B[; COLI_VK_STAGED_FAULT's point reached c0,c1,.. times by the end of each layer]
+ * vkc_fit_partial: 1 when something the full chain would place stays on the CPU (N < L, or
+ * the tail kept off): the engine's per-matrix path must then not upload matrices that are
+ * not on the device already (the CPU layers and the tail stay on the CPU).
+ * vkc_layer_free: frees a layer's tensors (coli_vk_tensor_free) and chain buffers (vkc_free)
+ * after the frames that may read them, setting each pointer to NULL. */
+typedef struct {
+ int L, n; /* the model's layers; the first n run on the device */
+ int forced, tail; /* COLI_VK_CHAIN_LAYERS decided; the tail goes on the device */
+ size_t free_b, reserve, fixed, engine_fixed, pools, tail_b, used_b; /* used_b: the n layers' bytes */
+ size_t *per, *mat; /* each layer's bytes and its matrices' payload */
+ unsigned long long *marks; /* COLI_VK_STAGED_FAULT's count at the end of each layer's setup */
+} VkcFit;
+int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *matrix_bytes,
+ size_t fixed_bytes, size_t tail_bytes, VkcFit *fit);
+void vkc_fit_shrink(const char *engine, VkcFit *fit, int n, const char *why);
+void vkc_fit_mark(VkcFit *fit, int layer);
+void vkc_fit_placed(const char *engine, const VkcFit *fit);
+int vkc_fit_partial(const VkcFit *fit);
+/* COLI_VK_CHAIN_ROWS when set to a number, else def: the rows the fit's scratch is for */
+int vkc_fit_rows(int def);
+size_t vkc_fit_tensor(int fmt, int I, int O, int gs); /* a resident tensor's device bytes, its two ranges aligned */
+size_t vkc_fit_buf(size_t bytes); /* a chain buffer's (vkc_buf) device bytes */
+size_t vkc_fit_reserve(void); /* COLI_VK_TIER_RESERVE_GB in bytes */
+size_t vkc_fit_pools(void); /* the pools' granularity the fit adds to fixed */
+void vkc_layer_free(ColiVkTensor **const *t, int nt, VkcBuf **const *b, int nb);
+
/* ---- a KV cache split between the device and the host (chain_kvs.comp) ----------------
* Past the device's budget a layer's cache keeps `ns` blocks of B positions on the
* device (the recent window, and where an engine selects positions, the blocks read
diff --git a/c/vk_tier.c b/c/vk_tier.c
index 4624de22..24e9d8a4 100644
--- a/c/vk_tier.c
+++ b/c/vk_tier.c
@@ -1211,7 +1211,7 @@ int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {
if (want > room) want = room;
if (want < 0) want = 0;
/* whole experts per pool block, so the count matches what the blocks can hold */
- size_t blk = (size_t)256 << 20;
+ size_t blk = coli_vk_block_bytes((size_t)256 << 20); /* the pool's blocks (smaller under COLI_VK_DEVICE_CAP_MB) */
if ((size_t)want < blk) blk = (size_t)want;
long long fit = 0;
if (T.exp_bytes && blk >= T.exp_bytes) {
@@ -1268,7 +1268,7 @@ int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {
* block for a few experts. */
size_t lim = T.budget, need = (size_t)(all + 1 + (T.st_ok ? T.st_slots : 0)) * T.exp_bytes;
if (fit + (T.st_ok ? T.st_slots : 0) >= all && fit_raw >= all + (T.st_ok ? T.st_slots : 0) &&
- need < lim && need <= ((size_t)256 << 20)) lim = need;
+ need < lim && need <= coli_vk_block_bytes((size_t)256 << 20)) lim = need;
coli_vk_tier_pool_limit(lim);
const char *bal = getenv("COLI_VK_TIER_BALANCE");
T.balance = T.c.in_ram != NULL && !(bal && *bal == '0');