Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
glm53: size the expert cache around the model, not around MemAvailable
Ronald Klompcf7288aff549
diff --git a/c/compat.h b/c/compat.hindex 0c5db65b..f074f54b 100644--- a/c/compat.h+++ b/c/compat.h@@ -641,28 +641,71 @@ static inline void coli_print_launcher_help(const char *engine)* 0 = non misurabile; e' il chiamante a decidere il fallback. */#ifdef __APPLE__#include <mach/mach.h>+#include <sys/sysctl.h>#endif#include <unistd.h>-static inline double compat_mem_available_gb(void){++/* Total AND available in one call, one pass.+ *+ * "Available" alone was consolidated here by #1375 because there were two+ * copies of it and one was wrong. "Total" is now in the same position: every+ * caller that wants it hand-rolls its own #ifdef ladder (telemetry.h uses+ * sysctl hw.memsize on macOS, olmoe.c uses sysconf(_SC_PHYS_PAGES), glm53.c+ * read /proc/meminfo unconditionally), and those definitions do not agree.+ * A budget computed from a total and an available that came from two+ * different definitions is not a budget, it is a coincidence.+ *+ * One pass also matters on Linux specifically: MemTotal and MemAvailable are+ * two lines of the same file, and reading it twice to get them is both a+ * second open and a second chance to read a file that changed underneath.+ *+ * 0 means "not measurable" for either field; the caller decides the fallback. */+static inline void compat_meminfo_gb(double *total_gb, double *avail_gb){+ double total = 0, avail = 0;#ifdef __APPLE__+ uint64_t memsize = 0; size_t len = sizeof memsize;+ if(sysctlbyname("hw.memsize", &memsize, &len, NULL, 0) == 0) total = (double)memsize / 1e9;mach_msg_type_number_t cnt = HOST_VM_INFO64_COUNT;vm_statistics64_data_t vm;- if(host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vm, &cnt) != KERN_SUCCESS) return 0;- return ((double)vm.free_count + (double)vm.inactive_count + (double)vm.purgeable_count)- * (double)sysconf(_SC_PAGESIZE) / 1e9;+ if(host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vm, &cnt) == KERN_SUCCESS)+ avail = ((double)vm.free_count + (double)vm.inactive_count + (double)vm.purgeable_count)+ * (double)sysconf(_SC_PAGESIZE) / 1e9;#elif defined(_WIN32)MEMORYSTATUSEX msx = {0};msx.dwLength = sizeof(msx);- if(!GlobalMemoryStatusEx(&msx)) return 0;- double phys = (double)msx.ullAvailPhys / 1e9;- double commit = (double)msx.ullAvailPageFile / 1e9;- return commit > 0 && commit < phys ? commit : phys;+ if(GlobalMemoryStatusEx(&msx)){+ total = (double)msx.ullTotalPhys / 1e9;+ double phys = (double)msx.ullAvailPhys / 1e9;+ double commit = (double)msx.ullAvailPageFile / 1e9;+ avail = commit > 0 && commit < phys ? commit : phys;+ }#else- FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;- char ln[256]; double kb = 0;- while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemAvailable: %lf", &kb) == 1) break;- fclose(f); return kb / 1e6;+ FILE *f = fopen("/proc/meminfo", "r");+ if(f){+ char ln[256]; double kb;+ /* /proc/meminfo's "kB" is KiB (1024 B), so a GB is kb*1024/1e9, not+ * kb/1e6. The old kb/1e6 understated by 2.3% -- harmless while the+ * number was only ever compared against itself, but glm53 now weighs+ * it against a model size computed from byte counts (/1e9, true GB),+ * and a budget that subtracts true GB from understated GB is wrong in+ * the direction that matters: it hands back less than it should. */+ /* MemTotal precedes MemAvailable in /proc/meminfo, but do not rely on+ * the order: stop only once both have been seen. */+ while((total == 0 || avail == 0) && fgets(ln, sizeof ln, f)){+ if(total == 0 && sscanf(ln, "MemTotal: %lf", &kb) == 1){ total = kb * 1024.0 / 1e9; continue; }+ if(avail == 0 && sscanf(ln, "MemAvailable: %lf", &kb) == 1) avail = kb * 1024.0 / 1e9;+ }+ fclose(f);+ }#endif+ if(total_gb) *total_gb = total;+ if(avail_gb) *avail_gb = avail;+}++static inline double compat_mem_available_gb(void){+ double avail = 0;+ compat_meminfo_gb(NULL, &avail);+ return avail;}#endif /* COMPAT_H */diff --git a/c/glm53.c b/c/glm53.cindex 6f5ddc69..420d0a0d 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -1275,15 +1275,6 @@ static void expert_table_init(GModel *m) {/* Quanti slot per layer: il budget diviso i layer sparsi. Il pavimento e' 1 e* non topk, perche' un pavimento a topk impegnerebbe topk*layer slot comunque,* cioe' molti GB, a dispetto del budget chiesto. */-/* Quanta RAM il sistema dice di poter dare adesso. MemAvailable e non MemFree:- * la seconda ignora la page cache riutilizzabile e farebbe stimare molto meno- * di quello che c'e'. */-static double memory_available_gb(void) {- /* #1375: era una lettura di /proc/meminfo, che su Windows e macOS non- * esiste: 0 -> budget 1 GB -> uno slot per layer, in silenzio. */- return compat_mem_available_gb();-}-static void expert_cache_init(GModel *m) {const Cfg *c = &m->c;const char *setting = getenv("GLM53_EXPERT_GB");@@ -1293,19 +1284,39 @@ static void expert_cache_init(GModel *m) {* versi -- su una macchina piccola va in OOM, su una grande lascia RAM* inutilizzata mentre il disco fa tutto il lavoro, che e' esattamente* quello che e' successo alla prima esecuzione vera. */+ const int from = c->first_dense > m->layer_begin ? c->first_dense : m->layer_begin;+ int sparse = m->layer_end - from;+ if (sparse < 0) sparse = 0;double budget;if (setting) budget = atof(setting);else {- const double free_now = memory_available_gb();- budget = free_now - 3.0;+ /* MemAvailable counts reclaimable page cache as free. Sizing this LRU+ * from it means allocating, as anonymous memory, the very pages the+ * next slot miss would have been served from: both caches hold the+ * same bytes, the model is paid for twice, and the cheap copy is the+ * one that loses. So leave the model room to stay in page cache and+ * take only what is left over. */+ double total = 0.0, free_now = 0.0;+ compat_meminfo_gb(&total, &free_now); /* one pass, both fields */+ /* the routed experts dominate; the rest of the weights are ~7% */+ const double model_gb = (double)sparse * c->n_experts * (double)m->e_slot / 1e9 * 1.07;+ double margin = total * 0.08;+ if (margin < 4.0) margin = 4.0;+ if (total <= 0.0 || model_gb >= total - margin) {+ /* The model does not fit in RAM anyway, so page cache cannot help:+ * keep as many slots as possible, exactly as before. `total <= 0`+ * is "could not measure" and takes the same safe path. */+ budget = free_now - 3.0;+ } else {+ budget = total - model_gb - margin;+ if (budget > free_now - 3.0) budget = free_now - 3.0;+ }if (budget < 1.0) budget = 1.0;if (getenv("GLM53_VERBOSE"))- fprintf(stderr, "expert budget: %.1f GB (%.1f available, 3 GB reserved)\n",- budget, free_now);+ fprintf(stderr, "expert budget: %.1f GB (%.1f total, %.1f model, "+ "%.1f margin, %.1f available)\n",+ budget, total, model_gb, margin, free_now);}- const int from = c->first_dense > m->layer_begin ? c->first_dense : m->layer_begin;- int sparse = m->layer_end - from;- if (sparse < 0) sparse = 0;int cap = (int)((budget * 1e9) / ((double)m->e_slot * (sparse > 0 ? sparse : 1)));if (g_cap_override > 0) cap = g_cap_override; /* scelta esplicita: vince */if (cap < 1) cap = 1;diff --git a/c/tests/test_mem_available.c b/c/tests/test_mem_available.cindex 8b05f7cb..a98b415f 100644--- a/c/tests/test_mem_available.c+++ b/c/tests/test_mem_available.c@@ -18,21 +18,23 @@static int fails;static void check(int ok, const char *what){ if(!ok){ printf(" FAIL: %s\n", what); fails++; } }+/* Il totale arriva ora dalla stessa funzione condivisa della disponibile+ * (compat_meminfo_gb), su tutte e tre le piattaforme. Prima questo test+ * tornava 0 su macOS -- "nessun totale a portata senza sysctl" -- e il+ * vincolo "disponibile <= RAM fisica" saltava proprio sulla piattaforma dove+ * la disponibile e' una stima. Ora non salta piu' da nessuna parte. */static double total_gb(void){-#ifdef _WIN32- double t, a; compat_meminfo(&t, &a); return t;-#elif defined(__APPLE__)- return 0; /* nessun totale a portata senza sysctl: il vincolo superiore salta */-#else- FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;- char ln[256]; double kb = 0;- while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemTotal: %lf", &kb) == 1) break;- fclose(f); return kb / 1e6;-#endif+ double t = 0, a = 0;+ compat_meminfo_gb(&t, &a);+ return t;}int main(void){double avail = compat_mem_available_gb();+ /* compat_mem_available_gb() e' ora un wrapper su compat_meminfo_gb(): le+ * due devono dare lo stesso numero, o il wrapper ha perso qualcosa. */+ { double t2 = 0, a2 = 0; compat_meminfo_gb(&t2, &a2);+ check(a2 == avail, "compat_meminfo_gb e compat_mem_available_gb non concordano"); }printf(" disponibile: %.2f GB\n", avail);check(avail > 0.0, "la misura vale 0: la piattaforma non e' coperta (era il bug di Windows)");double total = total_gb();