Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: reserve the expert tier's RAM on integrated GPUs (#653 for the VK path)

huppiflupp772f9616f7ab
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index d779607d..e9a8185b 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -1533,6 +1533,19 @@ void coli_vk_tensor_free(ColiVkTensor *t) {
size_t coli_vk_tensor_bytes(const ColiVkTensor *t) { return t ? t->wbytes : 0; }
+/* Is the device we picked an integrated GPU? The backend already computes this
+ * at device selection (see the deviceType ranking above) but only uses it to
+ * rank candidates. The RAM planner needs it too: on an integrated GPU our
+ * HOST_VISIBLE|DEVICE_LOCAL allocations are the same physical memory the host
+ * expert cache draws from, so whoever sizes that cache has to know. Mirrors
+ * coli_cuda_device_integrated() (#653) for the Vulkan path. */
+int coli_vk_device_integrated(void) {
+ if (!G.phys) return 0;
+ VkPhysicalDeviceProperties p;
+ vkGetPhysicalDeviceProperties(G.phys, &p);
+ return p.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 1 : 0;
+}
+
void coli_vk_shutdown(void) {
if (!G.ready) return;
vkDeviceWaitIdle(G.dev);
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index 30b617d6..60673b10 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -127,6 +127,10 @@ int coli_vk_attention_absorb_project(ColiVkTensor **kvb, const void *w, const f
void coli_vk_tensor_free(ColiVkTensor *t);
size_t coli_vk_tensor_bytes(const ColiVkTensor *t);
+/* 1 if the selected device is an integrated GPU (shares physical memory with
+ * the host), 0 otherwise or when no device is selected. */
+int coli_vk_device_integrated(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/c/colibri.c b/c/colibri.c
index baa2394f..7ae98fc0 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -11396,6 +11396,29 @@ int main(int argc, char **argv){
{ double ram_env = getenv("RAM_GB")?atof(getenv("RAM_GB")):0.0;
int est_ctx = getenv("CTX")?atoi(getenv("CTX")):4096; /* stesso default di run_serve */
snprintf(g_usage_path,sizeof(g_usage_path),"%s/.coli_usage",snap);
+#ifdef COLI_VULKAN
+ /* #653's correction, for the Vulkan tier. On an integrated GPU the tier's
+ * HOST_VISIBLE|DEVICE_LOCAL allocation is the SAME physical RAM that
+ * expert_avail()/cap_for_ram() below hand to the pin set and the LRU.
+ * Unlike the CUDA tier this one cannot be subtracted after the fact:
+ * vk_registry_fill() runs at the END of init, long after both decisions
+ * are made, so the planned size has to be reserved here instead. Sized
+ * from a routed layer's row width x the configured expert count.
+ * Discrete GPUs have their own pool -> deviceType is not INTEGRATED and
+ * this is a no-op, as with #653. */
+ if(g_vulkan && g_vk_budget>0 && g_mem_avail_boot>0 && coli_vk_device_integrated()){
+ int probe_l = m.c.n_layers>1 ? m.c.n_layers/2 : 0;
+ double per = (double)expert_bytes_row(&m,probe_l,m.ebits);
+ double tier_gb = per>0 ? (double)g_vk_budget*per/1e9 : 0.0;
+ if(tier_gb>0){
+ g_mem_avail_boot -= tier_gb;
+ if(g_mem_avail_boot < 1.0) g_mem_avail_boot = 1.0;
+ fprintf(stderr,"[VK] integrated/unified memory: expert tier will share physical RAM; "
+ "RAM budget snapshot reduced by %.2f GB (%d experts planned) -> MemAvailable=%.1f GB\n",
+ tier_gb, g_vk_budget, g_mem_avail_boot);
+ }
+ }
+#endif
int64_t hist = usage_load(&m,g_usage_path);
if(hist>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",(long long)hist,g_usage_path);
int autopin = getenv("AUTOPIN")?atoi(getenv("AUTOPIN")):1;