Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(memory): measure available RAM on every platform, and mind the commit limit (#1375)

JustVuggec6656ede06d
diff --git a/c/Makefile b/c/Makefile
index c3a22582..17f88907 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1709,6 +1709,11 @@ tests/test_serve_poll$(EXE): tests/test_serve_poll.c serve_poll.h
tests/test_rss_anon$(EXE): tests/test_rss_anon.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+# #1375: la memoria disponibile e' misurata in compat.h per Linux, macOS e
+# Windows; questo test gira nei tre job e fallisce dove la misura vale 0.
+tests/test_mem_available$(EXE): tests/test_mem_available.c compat.h
+ $(CC) $(CFLAGS) tests/test_mem_available.c -o $@ $(LDFLAGS)
+
tests/test_798_guards$(EXE): tests/test_798_guards.c st.h json.h compat.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/autotune.py b/c/autotune.py
index d99c0e82..954289fa 100644
--- a/c/autotune.py
+++ b/c/autotune.py
@@ -478,7 +478,7 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,
# instead of repeatedly teaching the cache one prompt.
for repeat in range(repeats):
active_prompt = serving_prompts[repeat % len(serving_prompts)]
- progress(f"{name} ({repeat + 1}/{repeats}, prompt "
+ progress(f"{name} cap={cap} ({repeat + 1}/{repeats}, prompt "
f"{repeat % len(serving_prompts) + 1}/"
f"{len(serving_prompts)})")
pieces = []
@@ -529,7 +529,7 @@ def run_tune(engine: str, cap: int, base_env: dict, plan: dict, model: str,
def measure(name, overlay, launch_cap):
samples = []
for repeat in range(repeats):
- progress(f"{name} ({repeat + 1}/{repeats})")
+ progress(f"{name} cap={cap} ({repeat + 1}/{repeats})")
sample = run_once(name, overlay, launch_cap)
sample["ttft_s"] = None
samples.append(sample)
diff --git a/c/colibri.c b/c/colibri.c
index 6958dd8e..28cc9ea5 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -9972,22 +9972,10 @@ static double g_mem_avail_boot=0; /* MemAvailable all'avvio, prima di caricare
* (stessa semantica: recuperabili senza swap). Senza questo ramo il fallback
* "assumo 8 GB" castrava la cache expert proprio sulle macchine con piu' RAM. */
static double mem_available_gb(void){
-#ifdef __APPLE__
- mach_msg_type_number_t cnt=HOST_VM_INFO64_COUNT;
- vm_statistics64_data_t vm;
- if(host_statistics64(mach_host_self(),HOST_VM_INFO64,(host_info64_t)&vm,&cnt)!=KERN_SUCCESS) return 0;
- return ((double)vm.free_count+(double)vm.inactive_count+(double)vm.purgeable_count)
- * (double)sysconf(_SC_PAGESIZE) / 1e9;
-#elif defined(_WIN32)
- double total, avail;
- compat_meminfo(&total, &avail);
- return avail;
-#else
- FILE *f=fopen("/proc/meminfo","r"); if(!f) return 0;
- char ln[256]; double kb=0;
- while(fgets(ln,sizeof(ln),f)) if(sscanf(ln,"MemAvailable: %lf",&kb)==1) break;
- fclose(f); return kb/1e6;
-#endif
+ /* Era la sola copia giusta di questa misura; glm53.c ne aveva una che
+ * leggeva /proc ovunque (#1375). Ora vive in compat.h e la chiamano
+ * entrambi: su Windows tiene anche conto del commit disponibile. */
+ return compat_mem_available_gb();
}
static int kv_slot_count(void){
diff --git a/c/compat.h b/c/compat.h
index 688b560b..0d2dde3e 100644
--- a/c/compat.h
+++ b/c/compat.h
@@ -600,4 +600,43 @@ static inline void coli_print_launcher_help(const char *engine)
coli_hold_console();
}
+/* --- RAM disponibile ADESSO, in GB, per tutte le piattaforme ---------------
+ * "Disponibile" = recuperabile senza swap: MemAvailable su Linux; free +
+ * inactive + purgeable su macOS; su Windows ullAvailPhys MA limitata da
+ * ullAvailPageFile, il commit ancora concedibile: e' quello che decide se
+ * il prossimo malloc riesce, e su una macchina con pagefile piccolo puo'
+ * essere molto meno della RAM fisica libera.
+ *
+ * #1375: glm53.c leggeva /proc/meminfo ovunque, e su Windows quel file non
+ * esiste: la funzione tornava 0, il budget della cache esperti si clampava a
+ * 1 GB, e Flash su Windows girava con uno slot per layer. colibri.c aveva la
+ * versione giusta (macOS + Windows) da mesi, come funzione sua. Due copie di
+ * cui una sbagliata: ora e' una, qui, e i motori la chiamano.
+ * 0 = non misurabile; e' il chiamante a decidere il fallback. */
+#ifdef __APPLE__
+#include <mach/mach.h>
+#endif
+#include <unistd.h>
+static inline double compat_mem_available_gb(void){
+#ifdef __APPLE__
+ mach_msg_type_number_t cnt = HOST_VM_INFO64_COUNT;
+ vm_statistics64_data_t vm;
+ if(host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vm, &cnt) != KERN_SUCCESS) return 0;
+ return ((double)vm.free_count + (double)vm.inactive_count + (double)vm.purgeable_count)
+ * (double)sysconf(_SC_PAGESIZE) / 1e9;
+#elif defined(_WIN32)
+ MEMORYSTATUSEX msx = {0};
+ msx.dwLength = sizeof(msx);
+ if(!GlobalMemoryStatusEx(&msx)) return 0;
+ double phys = (double)msx.ullAvailPhys / 1e9;
+ double commit = (double)msx.ullAvailPageFile / 1e9;
+ return commit > 0 && commit < phys ? commit : phys;
+#else
+ FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;
+ char ln[256]; double kb = 0;
+ while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemAvailable: %lf", &kb) == 1) break;
+ fclose(f); return kb / 1e6;
+#endif
+}
+
#endif /* COMPAT_H */
diff --git a/c/glm53.c b/c/glm53.c
index 13dc21cd..1ce9317b 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -1215,16 +1215,9 @@ static void expert_table_init(GModel *m) {
* la seconda ignora la page cache riutilizzabile e farebbe stimare molto meno
* di quello che c'e'. */
static double memory_available_gb(void) {
- FILE *f = fopen("/proc/meminfo", "r");
- if (!f) return 0.0;
- char line[256];
- double gb = 0.0;
- while (fgets(line, sizeof(line), f)) {
- long kb;
- if (sscanf(line, "MemAvailable: %ld kB", &kb) == 1) { gb = kb / 1048576.0; break; }
- }
- fclose(f);
- return gb;
+ /* #1375: era una lettura di /proc/meminfo, che su Windows e macOS non
+ * esiste: 0 -> budget 1 GB -> uno slot per layer, in silenzio. */
+ return compat_mem_available_gb();
}
static void expert_cache_init(GModel *m) {
@@ -1293,7 +1286,12 @@ static void expert_read(GModel *m, int layer, int eid, Slot *slot) {
* di sola lettura che questo slot poteva star usando prima. */
if (!slot->own) {
slot->own = malloc((size_t)m->e_slot);
- if (!slot->own) { fprintf(stderr, "OOM su uno slot esperto\n"); exit(1); }
+ if (!slot->own) {
+ fprintf(stderr, "OOM su uno slot esperto (%.1f MB): la cache esperti non ci sta "
+ "in memoria; riduci con --ram N o GLM53_EXPERT_GB=N (#1375)\n",
+ m->e_slot / 1e6);
+ exit(1);
+ }
}
for (int p = 0; p < GLM53_EXPERT_PIECES; p++) slot->piece[p] = slot->own + m->e_at[p];
if (ref->contig) {
diff --git a/c/resource_plan.py b/c/resource_plan.py
index 3050d7ec..7057b7fb 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -239,6 +239,31 @@ def analyze_model(model):
return result
+#: MEMORYSTATUSEX as Windows defines it, in order. Kept as data so a test can
+#: pin the order without a Windows machine.
+WINDOWS_MEMORYSTATUSEX_FIELDS = (
+ ("dwLength", "c_ulong"), ("dwMemoryLoad", "c_ulong"),
+ ("ullTotalPhys", "c_ulonglong"), ("ullAvailPhys", "c_ulonglong"),
+ ("ullTotalPageFile", "c_ulonglong"), ("ullAvailPageFile", "c_ulonglong"),
+ ("ullTotalVirtual", "c_ulonglong"), ("ullAvailVirtual", "c_ulonglong"),
+ ("ullAvailExtendedVirtual", "c_ulonglong"),
+)
+
+
+def windows_available_bytes(avail_phys, avail_pagefile):
+ """What a Windows process can still get from malloc: the smaller of free
+ physical memory and the commit still grantable (RAM + page file, minus
+ what every process has already committed).
+
+ #1375: the planner budgeted 88 % of ullAvailPhys on a 128 GB machine and
+ handed the resulting cap to the engine; the engine filled it slot by slot
+ until Windows refused the next 14 MB. Physical memory was there. Commit
+ was not, and nothing had looked at it."""
+ if avail_pagefile and 0 < avail_pagefile < avail_phys:
+ return avail_pagefile
+ return avail_phys
+
+
def memory_available():
# Linux (and MSYS2/Git-Bash CPython where /proc exists): MemAvailable.
try:
@@ -254,20 +279,19 @@ def memory_available():
import ctypes
class MEMORYSTATUSEX(ctypes.Structure):
- _fields_ = [("dwLength", ctypes.c_ulong),
- ("dwMemoryLoad", ctypes.c_ulong),
- ("ullTotalPhys", ctypes.c_ulonglong),
- ("ullAvailPhys", ctypes.c_ulonglong),
- ("ullTotalVirtual", ctypes.c_ulonglong),
- ("ullAvailVirtual", ctypes.c_ulonglong),
- ("ullAvailExtendedVirtual", ctypes.c_ulonglong)]
+ # The real layout. The previous copy skipped the two PageFile
+ # fields, which put ullTotalVirtual/ullAvailVirtual at the
+ # wrong offsets; ullAvailPhys happened to be right, so nobody
+ # noticed. The PageFile pair is the point now (#1375).
+ _fields_ = [(name, getattr(ctypes, kind))
+ for name, kind in WINDOWS_MEMORYSTATUSEX_FIELDS]
stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
kernel32 = ctypes.windll.kernel32
kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.c_void_p]
kernel32.GlobalMemoryStatusEx.restype = ctypes.c_int
if kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) and stat.ullAvailPhys:
- return stat.ullAvailPhys
+ return windows_available_bytes(stat.ullAvailPhys, stat.ullAvailPageFile)
# Fallback (e.g. sandboxed callers where GlobalMemoryStatusEx reports
# nothing): total installed RAM in KB. Less precise than ullAvailPhys
# — it ignores standby/reclaimable pages — but never returns 0 on a
diff --git a/c/tests/test_mem_available.c b/c/tests/test_mem_available.c
new file mode 100644
index 00000000..8b05f7cb
--- /dev/null
+++ b/c/tests/test_mem_available.c
@@ -0,0 +1,54 @@
+/* La memoria disponibile deve essere MISURATA su ogni piattaforma che
+ * spediamo, non letta da un file che esiste solo su Linux.
+ *
+ * #1375: glm53.c leggeva /proc/meminfo ovunque. Su Windows il file non c'e',
+ * la funzione tornava 0, il budget della cache esperti si clampava a 1 GB e
+ * GLM-5.3-Flash girava con uno slot per layer -- in silenzio, per mesi,
+ * perche' nessun test chiedeva alla misura di essere maggiore di zero.
+ *
+ * Questo test gira nei job Linux, macOS e Windows della CI e pretende tre
+ * cose dalla funzione condivisa in compat.h: che misuri qualcosa, che il
+ * numero stia dentro la RAM fisica, e -- dove il sistema espone il proprio
+ * MemAvailable -- che coincida con quello. Il terzo vincolo e' quello che
+ * impedisce di "passare" restituendo una costante. */
+#include <stdio.h>
+#include <stdlib.h>
+#include "../compat.h"
+
+static int fails;
+static void check(int ok, const char *what){ if(!ok){ printf(" FAIL: %s\n", what); fails++; } }
+
+static double total_gb(void){
+#ifdef _WIN32
+ double t, a; compat_meminfo(&t, &a); return t;
+#elif defined(__APPLE__)
+ return 0; /* nessun totale a portata senza sysctl: il vincolo superiore salta */
+#else
+ FILE *f = fopen("/proc/meminfo", "r"); if(!f) return 0;
+ char ln[256]; double kb = 0;
+ while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemTotal: %lf", &kb) == 1) break;
+ fclose(f); return kb / 1e6;
+#endif
+}
+
+int main(void){
+ double avail = compat_mem_available_gb();
+ printf(" disponibile: %.2f GB\n", avail);
+ check(avail > 0.0, "la misura vale 0: la piattaforma non e' coperta (era il bug di Windows)");
+ double total = total_gb();
+ if(total > 0){
+ printf(" totale: %.2f GB\n", total);
+ check(avail <= total * 1.001, "disponibile > RAM fisica: la misura non e' una misura");
+ }
+#if !defined(_WIN32) && !defined(__APPLE__)
+ /* Linux: deve essere MemAvailable, non un'altra riga di meminfo. */
+ FILE *f = fopen("/proc/meminfo", "r"); double kb = 0;
+ if(f){ char ln[256]; while(fgets(ln, sizeof ln, f)) if(sscanf(ln, "MemAvailable: %lf", &kb) == 1) break; fclose(f); }
+ if(kb > 0){
+ double ratio = avail / (kb / 1e6);
+ check(ratio > 0.9 && ratio < 1.1, "non coincide con MemAvailable di /proc/meminfo");
+ }
+#endif
+ if(fails){ printf("test_mem_available: %d fallimenti\n", fails); return 1; }
+ printf("test_mem_available: ok\n"); return 0;
+}
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index a3f99a12..240ff446 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -16,6 +16,8 @@ from resource_plan import (
environment_for_plan,
format_plan,
memory_available,
+ windows_available_bytes,
+ WINDOWS_MEMORYSTATUSEX_FIELDS,
parse_ssd_cache,
physical_cpu_count,
read_ssd_probe,
@@ -982,3 +984,23 @@ class PhysicalCpuCountTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class WindowsCommitLimitTest(unittest.TestCase):
+ """#1375: a 14 MB malloc failing on a 128 GB machine. Free physical memory
+ is not what decides whether malloc succeeds on Windows; grantable commit
+ is, and it can be far lower with a small page file."""
+
+ def test_commit_caps_the_budget_when_lower_than_physical(self):
+ self.assertEqual(windows_available_bytes(110 << 30, 40 << 30), 40 << 30)
+
+ def test_physical_is_used_when_commit_is_larger_or_unknown(self):
+ self.assertEqual(windows_available_bytes(110 << 30, 200 << 30), 110 << 30)
+ self.assertEqual(windows_available_bytes(110 << 30, 0), 110 << 30)
+
+ def test_memorystatusex_layout_is_the_documented_one(self):
+ # Order matters for ctypes: a skipped field shifts every later one.
+ self.assertEqual([n for n, _ in WINDOWS_MEMORYSTATUSEX_FIELDS], [
+ "dwLength", "dwMemoryLoad", "ullTotalPhys", "ullAvailPhys",
+ "ullTotalPageFile", "ullAvailPageFile", "ullTotalVirtual",
+ "ullAvailVirtual", "ullAvailExtendedVirtual"])