Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

glm53: size the expert cache around the model, not around MemAvailable

Ronald Klompcf7288aff549
diff --git a/c/compat.h b/c/compat.h
index 0c5db65b..f074f54b 100644
--- a/c/compat.h
+++ b/c/compat.h
@@ -641,28 +641,71 @@ static inline void coli_print_launcher_help(const char *engine)
* 0 = non misurabile; e' il chiamante a decidere il fallback. */
#ifdef __APPLE__
#include <mach/mach.h>
+#include <sys/sysctl.h>
#endif
#include <unistd.h>
-static inline double compat_mem_available_gb(void){
+
+/* Total AND available in one call, one pass.
+ *
+ * "Available" alone was consolidated here by #1375 because there were two
+ * copies of it and one was wrong. "Total" is now in the same position: every
+ * caller that wants it hand-rolls its own #ifdef ladder (telemetry.h uses
+ * sysctl hw.memsize on macOS, olmoe.c uses sysconf(_SC_PHYS_PAGES), glm53.c
+ * read /proc/meminfo unconditionally), and those definitions do not agree.
+ * A budget computed from a total and an available that came from two
+ * different definitions is not a budget, it is a coincidence.
+ *
+ * One pass also matters on Linux specifically: MemTotal and MemAvailable are
+ * two lines of the same file, and reading it twice to get them is both a
+ * second open and a second chance to read a file that changed underneath.
+ *
+ * 0 means "not measurable" for either field; the caller decides the fallback. */
+static inline void compat_meminfo_gb(double *total_gb, double *avail_gb){
+ double total = 0, avail = 0;
#ifdef __APPLE__
+ uint64_t memsize = 0; size_t len = sizeof memsize;
+ if(sysctlbyname("hw.memsize", &memsize, &len, NULL, 0) == 0) total = (double)memsize / 1e9;
mach_msg_type_number_t cnt = HOST_VM_INFO64_COUNT;
vm_statistics64_data_t vm;
- if(host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vm, &cnt) != KERN_SUCCESS) return 0;
- return ((double)vm.free_count + (double)vm.inactive_count + (double)vm.purgeable_count)
- * (double)sysconf(_SC_PAGESIZE) / 1e9;
+ if(host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vm, &cnt) == KERN_SUCCESS)
+ avail = ((double)vm.free_count + (double)vm.inactive_count + (double)vm.purgeable_count)
+ * (double)sysconf(_SC_PAGESIZE) / 1e9;
#elif defined(_WIN32)
MEMORYSTATUSEX msx = {0};
msx.dwLength = sizeof(msx);
- if(!GlobalMemoryStatusEx(&msx)) return 0;
- double phys = (double)msx.ullAvailPhys / 1e9;
- double commit = (double)msx.ullAvailPageFile / 1e9;
- return commit > 0 && commit < phys ? commit : phys;
+ if(GlobalMemoryStatusEx(&msx)){
+ total = (double)msx.ullTotalPhys / 1e9;
+ double phys = (double)msx.ullAvailPhys / 1e9;
+ double commit = (double)msx.ullAvailPageFile / 1e9;
+ avail = commit > 0 && commit < phys ? commit : phys;
+ }
#else
- FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;
- char ln[256]; double kb = 0;
- while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemAvailable: %lf", &kb) == 1) break;
- fclose(f); return kb / 1e6;
+ FILE *f = fopen("/proc/meminfo", "r");
+ if(f){
+ char ln[256]; double kb;
+ /* /proc/meminfo's "kB" is KiB (1024 B), so a GB is kb*1024/1e9, not
+ * kb/1e6. The old kb/1e6 understated by 2.3% -- harmless while the
+ * number was only ever compared against itself, but glm53 now weighs
+ * it against a model size computed from byte counts (/1e9, true GB),
+ * and a budget that subtracts true GB from understated GB is wrong in
+ * the direction that matters: it hands back less than it should. */
+ /* MemTotal precedes MemAvailable in /proc/meminfo, but do not rely on
+ * the order: stop only once both have been seen. */
+ while((total == 0 || avail == 0) && fgets(ln, sizeof ln, f)){
+ if(total == 0 && sscanf(ln, "MemTotal: %lf", &kb) == 1){ total = kb * 1024.0 / 1e9; continue; }
+ if(avail == 0 && sscanf(ln, "MemAvailable: %lf", &kb) == 1) avail = kb * 1024.0 / 1e9;
+ }
+ fclose(f);
+ }
#endif
+ if(total_gb) *total_gb = total;
+ if(avail_gb) *avail_gb = avail;
+}
+
+static inline double compat_mem_available_gb(void){
+ double avail = 0;
+ compat_meminfo_gb(NULL, &avail);
+ return avail;
}
#endif /* COMPAT_H */
diff --git a/c/glm53.c b/c/glm53.c
index 6f5ddc69..420d0a0d 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -1275,15 +1275,6 @@ static void expert_table_init(GModel *m) {
/* Quanti slot per layer: il budget diviso i layer sparsi. Il pavimento e' 1 e
* non topk, perche' un pavimento a topk impegnerebbe topk*layer slot comunque,
* cioe' molti GB, a dispetto del budget chiesto. */
-/* Quanta RAM il sistema dice di poter dare adesso. MemAvailable e non MemFree:
- * la seconda ignora la page cache riutilizzabile e farebbe stimare molto meno
- * di quello che c'e'. */
-static double memory_available_gb(void) {
- /* #1375: era una lettura di /proc/meminfo, che su Windows e macOS non
- * esiste: 0 -> budget 1 GB -> uno slot per layer, in silenzio. */
- return compat_mem_available_gb();
-}
-
static void expert_cache_init(GModel *m) {
const Cfg *c = &m->c;
const char *setting = getenv("GLM53_EXPERT_GB");
@@ -1293,19 +1284,39 @@ static void expert_cache_init(GModel *m) {
* versi -- su una macchina piccola va in OOM, su una grande lascia RAM
* inutilizzata mentre il disco fa tutto il lavoro, che e' esattamente
* quello che e' successo alla prima esecuzione vera. */
+ const int from = c->first_dense > m->layer_begin ? c->first_dense : m->layer_begin;
+ int sparse = m->layer_end - from;
+ if (sparse < 0) sparse = 0;
double budget;
if (setting) budget = atof(setting);
else {
- const double free_now = memory_available_gb();
- budget = free_now - 3.0;
+ /* MemAvailable counts reclaimable page cache as free. Sizing this LRU
+ * from it means allocating, as anonymous memory, the very pages the
+ * next slot miss would have been served from: both caches hold the
+ * same bytes, the model is paid for twice, and the cheap copy is the
+ * one that loses. So leave the model room to stay in page cache and
+ * take only what is left over. */
+ double total = 0.0, free_now = 0.0;
+ compat_meminfo_gb(&total, &free_now); /* one pass, both fields */
+ /* the routed experts dominate; the rest of the weights are ~7% */
+ const double model_gb = (double)sparse * c->n_experts * (double)m->e_slot / 1e9 * 1.07;
+ double margin = total * 0.08;
+ if (margin < 4.0) margin = 4.0;
+ if (total <= 0.0 || model_gb >= total - margin) {
+ /* The model does not fit in RAM anyway, so page cache cannot help:
+ * keep as many slots as possible, exactly as before. `total <= 0`
+ * is "could not measure" and takes the same safe path. */
+ budget = free_now - 3.0;
+ } else {
+ budget = total - model_gb - margin;
+ if (budget > free_now - 3.0) budget = free_now - 3.0;
+ }
if (budget < 1.0) budget = 1.0;
if (getenv("GLM53_VERBOSE"))
- fprintf(stderr, "expert budget: %.1f GB (%.1f available, 3 GB reserved)\n",
- budget, free_now);
+ fprintf(stderr, "expert budget: %.1f GB (%.1f total, %.1f model, "
+ "%.1f margin, %.1f available)\n",
+ budget, total, model_gb, margin, free_now);
}
- const int from = c->first_dense > m->layer_begin ? c->first_dense : m->layer_begin;
- int sparse = m->layer_end - from;
- if (sparse < 0) sparse = 0;
int cap = (int)((budget * 1e9) / ((double)m->e_slot * (sparse > 0 ? sparse : 1)));
if (g_cap_override > 0) cap = g_cap_override; /* scelta esplicita: vince */
if (cap < 1) cap = 1;
diff --git a/c/tests/test_mem_available.c b/c/tests/test_mem_available.c
index 8b05f7cb..a98b415f 100644
--- a/c/tests/test_mem_available.c
+++ b/c/tests/test_mem_available.c
@@ -18,21 +18,23 @@
static int fails;
static void check(int ok, const char *what){ if(!ok){ printf(" FAIL: %s\n", what); fails++; } }
+/* Il totale arriva ora dalla stessa funzione condivisa della disponibile
+ * (compat_meminfo_gb), su tutte e tre le piattaforme. Prima questo test
+ * tornava 0 su macOS -- "nessun totale a portata senza sysctl" -- e il
+ * vincolo "disponibile <= RAM fisica" saltava proprio sulla piattaforma dove
+ * la disponibile e' una stima. Ora non salta piu' da nessuna parte. */
static double total_gb(void){
-#ifdef _WIN32
- double t, a; compat_meminfo(&t, &a); return t;
-#elif defined(__APPLE__)
- return 0; /* nessun totale a portata senza sysctl: il vincolo superiore salta */
-#else
- FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;
- char ln[256]; double kb = 0;
- while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemTotal: %lf", &kb) == 1) break;
- fclose(f); return kb / 1e6;
-#endif
+ double t = 0, a = 0;
+ compat_meminfo_gb(&t, &a);
+ return t;
}
int main(void){
double avail = compat_mem_available_gb();
+ /* compat_mem_available_gb() e' ora un wrapper su compat_meminfo_gb(): le
+ * due devono dare lo stesso numero, o il wrapper ha perso qualcosa. */
+ { double t2 = 0, a2 = 0; compat_meminfo_gb(&t2, &a2);
+ check(a2 == avail, "compat_meminfo_gb e compat_mem_available_gb non concordano"); }
printf(" disponibile: %.2f GB\n", avail);
check(avail > 0.0, "la misura vale 0: la piattaforma non e' coperta (era il bug di Windows)");
double total = total_gb();