Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(memory): measure available RAM on every platform, and mind the commit limit (#1375)
JustVuggec6656ede06d
diff --git a/c/Makefile b/c/Makefileindex c3a22582..17f88907 100644--- a/c/Makefile+++ b/c/Makefile@@ -1709,6 +1709,11 @@ tests/test_serve_poll$(EXE): tests/test_serve_poll.c serve_poll.htests/test_rss_anon$(EXE): tests/test_rss_anon.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# #1375: la memoria disponibile e' misurata in compat.h per Linux, macOS e+# Windows; questo test gira nei tre job e fallisce dove la misura vale 0.+tests/test_mem_available$(EXE): tests/test_mem_available.c compat.h+ $(CC) $(CFLAGS) tests/test_mem_available.c -o $@ $(LDFLAGS)+tests/test_798_guards$(EXE): tests/test_798_guards.c st.h json.h compat.h route_trace.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/autotune.py b/c/autotune.pyindex d99c0e82..954289fa 100644--- a/c/autotune.py+++ b/c/autotune.py@@ -478,7 +478,7 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,# instead of repeatedly teaching the cache one prompt.for repeat in range(repeats):active_prompt = serving_prompts[repeat % len(serving_prompts)]- progress(f"{name} ({repeat + 1}/{repeats}, prompt "+ progress(f"{name} cap={cap} ({repeat + 1}/{repeats}, prompt "f"{repeat % len(serving_prompts) + 1}/"f"{len(serving_prompts)})")pieces = []@@ -529,7 +529,7 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,def measure(name, overlay, launch_cap):samples = []for repeat in range(repeats):- progress(f"{name} ({repeat + 1}/{repeats})")+ progress(f"{name} cap={cap} ({repeat + 1}/{repeats})")sample = run_once(name, overlay, launch_cap)sample["ttft_s"] = Nonesamples.append(sample)diff --git a/c/colibri.c b/c/colibri.cindex 6958dd8e..28cc9ea5 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -9972,22 +9972,10 @@ static double g_mem_avail_boot=0; /* MemAvailable all'avvio, prima di caricare* (stessa semantica: recuperabili senza swap). Senza questo ramo il fallback* "assumo 8 GB" castrava la cache expert proprio sulle macchine con piu' RAM. */static double mem_available_gb(void){-#ifdef __APPLE__- mach_msg_type_number_t cnt=HOST_VM_INFO64_COUNT;- vm_statistics64_data_t vm;- if(host_statistics64(mach_host_self(),HOST_VM_INFO64,(host_info64_t)&vm,&cnt)!=KERN_SUCCESS) return 0;- return ((double)vm.free_count+(double)vm.inactive_count+(double)vm.purgeable_count)- * (double)sysconf(_SC_PAGESIZE) / 1e9;-#elif defined(_WIN32)- double total, avail;- compat_meminfo(&total, &avail);- return avail;-#else- FILE *f=fopen("/proc/meminfo","r"); if(!f) return 0;- char ln[256]; double kb=0;- while(fgets(ln,sizeof(ln),f)) if(sscanf(ln,"MemAvailable: %lf",&kb)==1) break;- fclose(f); return kb/1e6;-#endif+ /* Era la sola copia giusta di questa misura; glm53.c ne aveva una che+ * leggeva /proc ovunque (#1375). Ora vive in compat.h e la chiamano+ * entrambi: su Windows tiene anche conto del commit disponibile. */+ return compat_mem_available_gb();}static int kv_slot_count(void){diff --git a/c/compat.h b/c/compat.hindex 688b560b..0d2dde3e 100644--- a/c/compat.h+++ b/c/compat.h@@ -600,4 +600,43 @@ static inline void coli_print_launcher_help(const char *engine)coli_hold_console();}+/* --- RAM disponibile ADESSO, in GB, per tutte le piattaforme ---------------+ * "Disponibile" = recuperabile senza swap: MemAvailable su Linux; free ++ * inactive + purgeable su macOS; su Windows ullAvailPhys MA limitata da+ * ullAvailPageFile, il commit ancora concedibile: e' quello che decide se+ * il prossimo malloc riesce, e su una macchina con pagefile piccolo puo'+ * essere molto meno della RAM fisica libera.+ *+ * #1375: glm53.c leggeva /proc/meminfo ovunque, e su Windows quel file non+ * esiste: la funzione tornava 0, il budget della cache esperti si clampava a+ * 1 GB, e Flash su Windows girava con uno slot per layer. colibri.c aveva la+ * versione giusta (macOS + Windows) da mesi, come funzione sua. Due copie di+ * cui una sbagliata: ora e' una, qui, e i motori la chiamano.+ * 0 = non misurabile; e' il chiamante a decidere il fallback. */+#ifdef __APPLE__+#include <mach/mach.h>+#endif+#include <unistd.h>+static inline double compat_mem_available_gb(void){+#ifdef __APPLE__+ mach_msg_type_number_t cnt = HOST_VM_INFO64_COUNT;+ vm_statistics64_data_t vm;+ if(host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vm, &cnt) != KERN_SUCCESS) return 0;+ return ((double)vm.free_count + (double)vm.inactive_count + (double)vm.purgeable_count)+ * (double)sysconf(_SC_PAGESIZE) / 1e9;+#elif defined(_WIN32)+ MEMORYSTATUSEX msx = {0};+ msx.dwLength = sizeof(msx);+ if(!GlobalMemoryStatusEx(&msx)) return 0;+ double phys = (double)msx.ullAvailPhys / 1e9;+ double commit = (double)msx.ullAvailPageFile / 1e9;+ return commit > 0 && commit < phys ? commit : phys;+#else+ FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;+ char ln[256]; double kb = 0;+ while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemAvailable: %lf", &kb) == 1) break;+ fclose(f); return kb / 1e6;+#endif+}+#endif /* COMPAT_H */diff --git a/c/glm53.c b/c/glm53.cindex 13dc21cd..1ce9317b 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -1215,16 +1215,9 @@ static void expert_table_init(GModel *m) {* la seconda ignora la page cache riutilizzabile e farebbe stimare molto meno* di quello che c'e'. */static double memory_available_gb(void) {- FILE *f = fopen("/proc/meminfo", "r");- if (!f) return 0.0;- char line[256];- double gb = 0.0;- while (fgets(line, sizeof(line), f)) {- long kb;- if (sscanf(line, "MemAvailable: %ld kB", &kb) == 1) { gb = kb / 1048576.0; break; }- }- fclose(f);- return gb;+ /* #1375: era una lettura di /proc/meminfo, che su Windows e macOS non+ * esiste: 0 -> budget 1 GB -> uno slot per layer, in silenzio. */+ return compat_mem_available_gb();}static void expert_cache_init(GModel *m) {@@ -1293,7 +1286,12 @@ static void expert_read(GModel *m, int layer, int eid, Slot *slot) {* di sola lettura che questo slot poteva star usando prima. */if (!slot->own) {slot->own = malloc((size_t)m->e_slot);- if (!slot->own) { fprintf(stderr, "OOM su uno slot esperto\n"); exit(1); }+ if (!slot->own) {+ fprintf(stderr, "OOM su uno slot esperto (%.1f MB): la cache esperti non ci sta "+ "in memoria; riduci con --ram N o GLM53_EXPERT_GB=N (#1375)\n",+ m->e_slot / 1e6);+ exit(1);+ }}for (int p = 0; p < GLM53_EXPERT_PIECES; p++) slot->piece[p] = slot->own + m->e_at[p];if (ref->contig) {diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 3050d7ec..7057b7fb 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -239,6 +239,31 @@ def analyze_model(model):return result+#: MEMORYSTATUSEX as Windows defines it, in order. Kept as data so a test can+#: pin the order without a Windows machine.+WINDOWS_MEMORYSTATUSEX_FIELDS = (+ ("dwLength", "c_ulong"), ("dwMemoryLoad", "c_ulong"),+ ("ullTotalPhys", "c_ulonglong"), ("ullAvailPhys", "c_ulonglong"),+ ("ullTotalPageFile", "c_ulonglong"), ("ullAvailPageFile", "c_ulonglong"),+ ("ullTotalVirtual", "c_ulonglong"), ("ullAvailVirtual", "c_ulonglong"),+ ("ullAvailExtendedVirtual", "c_ulonglong"),+)+++def windows_available_bytes(avail_phys, avail_pagefile):+ """What a Windows process can still get from malloc: the smaller of free+ physical memory and the commit still grantable (RAM + page file, minus+ what every process has already committed).++ #1375: the planner budgeted 88 % of ullAvailPhys on a 128 GB machine and+ handed the resulting cap to the engine; the engine filled it slot by slot+ until Windows refused the next 14 MB. Physical memory was there. Commit+ was not, and nothing had looked at it."""+ if avail_pagefile and 0 < avail_pagefile < avail_phys:+ return avail_pagefile+ return avail_phys++def memory_available():# Linux (and MSYS2/Git-Bash CPython where /proc exists): MemAvailable.try:@@ -254,20 +279,19 @@ def memory_available():import ctypesclass MEMORYSTATUSEX(ctypes.Structure):- _fields_ = [("dwLength", ctypes.c_ulong),- ("dwMemoryLoad", ctypes.c_ulong),- ("ullTotalPhys", ctypes.c_ulonglong),- ("ullAvailPhys", ctypes.c_ulonglong),- ("ullTotalVirtual", ctypes.c_ulonglong),- ("ullAvailVirtual", ctypes.c_ulonglong),- ("ullAvailExtendedVirtual", ctypes.c_ulonglong)]+ # The real layout. The previous copy skipped the two PageFile+ # fields, which put ullTotalVirtual/ullAvailVirtual at the+ # wrong offsets; ullAvailPhys happened to be right, so nobody+ # noticed. The PageFile pair is the point now (#1375).+ _fields_ = [(name, getattr(ctypes, kind))+ for name, kind in WINDOWS_MEMORYSTATUSEX_FIELDS]stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))kernel32 = ctypes.windll.kernel32kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.c_void_p]kernel32.GlobalMemoryStatusEx.restype = ctypes.c_intif kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) and stat.ullAvailPhys:- return stat.ullAvailPhys+ return windows_available_bytes(stat.ullAvailPhys, stat.ullAvailPageFile)# Fallback (e.g. sandboxed callers where GlobalMemoryStatusEx reports# nothing): total installed RAM in KB. Less precise than ullAvailPhys# — it ignores standby/reclaimable pages — but never returns 0 on adiff --git a/c/tests/test_mem_available.c b/c/tests/test_mem_available.cnew file mode 100644index 00000000..8b05f7cb--- /dev/null+++ b/c/tests/test_mem_available.c@@ -0,0 +1,54 @@+/* La memoria disponibile deve essere MISURATA su ogni piattaforma che+ * spediamo, non letta da un file che esiste solo su Linux.+ *+ * #1375: glm53.c leggeva /proc/meminfo ovunque. Su Windows il file non c'e',+ * la funzione tornava 0, il budget della cache esperti si clampava a 1 GB e+ * GLM-5.3-Flash girava con uno slot per layer -- in silenzio, per mesi,+ * perche' nessun test chiedeva alla misura di essere maggiore di zero.+ *+ * Questo test gira nei job Linux, macOS e Windows della CI e pretende tre+ * cose dalla funzione condivisa in compat.h: che misuri qualcosa, che il+ * numero stia dentro la RAM fisica, e -- dove il sistema espone il proprio+ * MemAvailable -- che coincida con quello. Il terzo vincolo e' quello che+ * impedisce di "passare" restituendo una costante. */+#include <stdio.h>+#include <stdlib.h>+#include "../compat.h"++static int fails;+static void check(int ok, const char *what){ if(!ok){ printf(" FAIL: %s\n", what); fails++; } }++static double total_gb(void){+#ifdef _WIN32+ double t, a; compat_meminfo(&t, &a); return t;+#elif defined(__APPLE__)+ return 0; /* nessun totale a portata senza sysctl: il vincolo superiore salta */+#else+ FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;+ char ln[256]; double kb = 0;+ while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemTotal: %lf", &kb) == 1) break;+ fclose(f); return kb / 1e6;+#endif+}++int main(void){+ double avail = compat_mem_available_gb();+ printf(" disponibile: %.2f GB\n", avail);+ check(avail > 0.0, "la misura vale 0: la piattaforma non e' coperta (era il bug di Windows)");+ double total = total_gb();+ if(total > 0){+ printf(" totale: %.2f GB\n", total);+ check(avail <= total * 1.001, "disponibile > RAM fisica: la misura non e' una misura");+ }+#if !defined(_WIN32) && !defined(__APPLE__)+ /* Linux: deve essere MemAvailable, non un'altra riga di meminfo. */+ FILE *f = fopen("/proc/meminfo", "r"); double kb = 0;+ if(f){ char ln[256]; while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemAvailable: %lf", &kb) == 1) break; fclose(f); }+ if(kb > 0){+ double ratio = avail / (kb / 1e6);+ check(ratio > 0.9 && ratio < 1.1, "non coincide con MemAvailable di /proc/meminfo");+ }+#endif+ if(fails){ printf("test_mem_available: %d fallimenti\n", fails); return 1; }+ printf("test_mem_available: ok\n"); return 0;+}diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex a3f99a12..240ff446 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -16,6 +16,8 @@ from resource_plan import (environment_for_plan,format_plan,memory_available,+ windows_available_bytes,+ WINDOWS_MEMORYSTATUSEX_FIELDS,parse_ssd_cache,physical_cpu_count,read_ssd_probe,@@ -982,3 +984,23 @@ class PhysicalCpuCountTest(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class WindowsCommitLimitTest(unittest.TestCase):+ """#1375: a 14 MB malloc failing on a 128 GB machine. Free physical memory+ is not what decides whether malloc succeeds on Windows; grantable commit+ is, and it can be far lower with a small page file."""++ def test_commit_caps_the_budget_when_lower_than_physical(self):+ self.assertEqual(windows_available_bytes(110 << 30, 40 << 30), 40 << 30)++ def test_physical_is_used_when_commit_is_larger_or_unknown(self):+ self.assertEqual(windows_available_bytes(110 << 30, 200 << 30), 110 << 30)+ self.assertEqual(windows_available_bytes(110 << 30, 0), 110 << 30)++ def test_memorystatusex_layout_is_the_documented_one(self):+ # Order matters for ctypes: a skipped field shifts every later one.+ self.assertEqual([n for n, _ in WINDOWS_MEMORYSTATUSEX_FIELDS], [+ "dwLength", "dwMemoryLoad", "ullTotalPhys", "ullAvailPhys",+ "ullTotalPageFile", "ullAvailPageFile", "ullTotalVirtual",+ "ullAvailVirtual", "ullAvailExtendedVirtual"])