Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
colibri.c/coli/openai_server.py: platform-aware cache defaults + precedence (#379, S2/S3)
monotophic2e3cf86270eb
diff --git a/.gitignore b/.gitignoreindex f98571ed..e5772037 100644--- a/.gitignore+++ b/.gitignore@@ -34,6 +34,8 @@ c/tests/test_*!c/tests/test_*.mm!c/tests/test_*.pyresult+c/tests/test_cap_precedence+c/tests/test_cap_precedence.exe# oracoli tiny generati (make_glm_oracle.py) e dati benchmark scaricatic/glm_tiny/diff --git a/c/Makefile b/c/Makefileindex 10b1fa46..67ca106b 100644--- a/c/Makefile+++ b/c/Makefile@@ -214,7 +214,7 @@ elsePYTHON ?= python3endifCUDA_OBJ =-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE) tests/test_pilot_ring$(EXE)+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_router_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE) tests/test_rans$(EXE) tests/test_corpus_draft$(EXE) tests/test_pilot_ring$(EXE) tests/test_cap_precedence$(EXE)ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endif@@ -560,6 +560,9 @@ tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h jsontests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_cap_precedence$(EXE): tests/test_cap_precedence.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_selecttests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.hdiff --git a/c/coli b/c/coliindex d31f98ac..c9e297dc 100755--- a/c/coli+++ b/c/coli@@ -1050,7 +1050,11 @@ def main():default=os.environ.get("COLI_POLICY","quality"),help="resource policy (explicit --topk/--topp overrides warn and proceed)")common.add_argument("--repin", type=int, default=0, help="adapt RAM/VRAM experts every N tokens")- common.add_argument("--cap", type=int, default=8); common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop token+ # 0 = not explicitly set: the engine picks (8 historically; 1 on Metal+darwin+fast+ # SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in <model>/.coli_ssd).+ # An explicit --cap always wins; see colibri.c's coli_resolve_cap().+ common.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")+ common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop tokencommon.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95)ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibri β run GLM-5.2 locally")diff --git a/c/colibri.c b/c/colibri.cindex 4a81a3b7..7c5f669c 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -88,6 +88,12 @@ static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pr#include <mach/mach.h> /* host_statistics64: MemAvailable di macOS */#endif+/* #379: set in main() when the F_NOCACHE storage probe (coli_ssd_probe_cached,+ * defined below) measured the model volume as fast (Metal + darwin only).+ * Stays 0 on every other platform/build, so every read of it below is a no-op+ * there -- see coli_resolve_cap() and cap_for_ram()'s CAP_RAISE default. */+static int g_ssd_fast;+typedef struct {int hidden, n_layers, n_heads, n_experts, topk, moe_inter, dense_inter;int first_dense, q_lora, kv_lora, qk_nope, qk_rope, qk_head, v_head, n_shared, vocab;@@ -7495,8 +7501,14 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){* Senza questo, una macchina da 128 GB girava con la LRU di una da 16* (cap=8 di default in coli): hit 23-28% con decine di GB inutilizzati.* Tetto a n_experts: oltre, ogni layer avrebbe slot che non puo' riempire.- * CAP_RAISE=0 ripristina il comportamento fisso. */- int raise_on = getenv("CAP_RAISE")?atoi(getenv("CAP_RAISE")):1;+ * CAP_RAISE=0 ripristina il comportamento fisso.+ * #379: on Metal + darwin + fast SSD, raising the cache back up is the+ * same anti-pattern the platform cap default exists to avoid, so the+ * default flips to off there. An explicit CAP_RAISE always wins either+ * way -- this only changes what happens when nobody set it.+ * CURRENT-STATE CALIBRATION (#379): same revisit trigger and+ * measurement stamp as the cap-1 default in coli_resolve_cap(). */+ int raise_on = getenv("CAP_RAISE")?atoi(getenv("CAP_RAISE")):(g_ssd_fast?0:1);int newcap = capmax>c->n_experts ? c->n_experts : capmax;if(raise_on && newcap>m->ecap){for(int i=0;i<=c->n_layers;i++) if(m->ecache[i]){@@ -7569,10 +7581,10 @@ static void prof_config(Model *m, double ram_env, int est_ctx){* number was 14 GB/s), so this probes the actual model volume the same way* compat_open_direct()/iobench.c's __APPLE__ branch do: open + F_NOCACHE,* random 16K-aligned pread, no readahead. Cheap and darwin-only: never touches- * non-Metal or non-darwin builds. Not called yet from main() in this commit --- * see the platform-default wiring that follows in the next one (S2/S3).+ * non-Metal or non-darwin builds.* Gated on __APPLE__ alone (not COLI_METAL) so these three functions build and- * are unit-testable on any macOS binary, Metal or not. */+ * are unit-testable on any macOS binary, Metal or not; the call site in main()+ * is the one that additionally requires g_metal_enabled, i.e. COLI_METAL. */#ifdef __APPLE__#define COLI_SSD_PROBE_BLK (4u*1024*1024) /* 4 MB reads, 16K-aligned offsets */#define COLI_SSD_PROBE_MAXB (300*1024*1024) /* "a few hundred MB", capped on fast media */@@ -7652,6 +7664,30 @@ static double coli_ssd_probe_cached(const char *snap_dir){}#endif /* __APPLE__ */+/* Expert-cache-cap precedence (#379, S2): explicit CLI positional > explicit+ * CAP env > platform default (Metal + darwin + fast SSD) > historic default.+ * `cli_given` distinguishes a bare invocation (no positional at all -> the+ * engine's historic 64, byte-identical to the old argc<=1 fallback) from the+ * coli wrapper's "0 = auto" sentinel (-> historic 8, what coli always forced+ * before). cap=0 as an explicit request for zero cache slots was undocumented+ * and unused repo-wide, so repurposing it as the sentinel is safe. Pure+ * function (no I/O, no globals) so precedence is unit-testable in isolation+ * from the probe and from argv/getenv. */+static int coli_resolve_cap(int cli_given, int cli, int env, int platform_fast, int *explicit_out){+ int is_explicit = (cli_given && cli!=0) || (env!=0);+ if(explicit_out) *explicit_out=is_explicit;+ if(cli_given && cli) return cli;+ if(env) return env;+ /* CURRENT-STATE CALIBRATION (#379; measured 2026-07, macOS 26.5, M5 Max,+ * engine base caa49f7): cap 1 encodes that stack's per-buffer residency+ * cost. Revisit when residency handling changes (the heap/residency-set+ * work in #379, or MTLTensor-based expert storage): with heap-backed+ * slabs cap 16 already beats cap 1. The machinery above and below is+ * durable; only this constant changes. */+ if(platform_fast) return 1;+ return cli_given ? 8 : 64; /* sentinel 0 -> coli's historic 8; bare ./glm -> historic 64 */+}+int main(int argc, char **argv){/* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are* tiny and back-to-back; with the default passive wait policy libgomp parks@@ -7877,7 +7913,11 @@ int main(int argc, char **argv){if(getenv("SEED")) g_rng = (uint64_t)atoll(getenv("SEED"))*0x9E3779B97F4A7C15ULL+1;else { struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts); g_rng ^= (uint64_t)ts.tv_nsec<<20 ^ (uint64_t)getpid(); }if(g_draft>63) g_draft=63; /* -1 = auto, risolto dopo model_init */- int cap = argc>1?atoi(argv[1]):64;+ /* cap itself is resolved below, once g_metal_enabled and the SSD probe (both+ * needed for the platform default) are known -- see coli_resolve_cap(). */+ int cap_given = argc>1;+ int cap_arg = cap_given?atoi(argv[1]):0;+ int cap_env = getenv("CAP")?atoi(getenv("CAP")):0;int ebits= argc>2?atoi(argv[2]):8;int dbits= argc>3?atoi(argv[3]):ebits;int kv_limit=(getenv("SERVE_BATCH")&&atoi(getenv("SERVE_BATCH")))?512:16;@@ -7993,6 +8033,23 @@ int main(int argc, char **argv){return 2;}#endif+ /* #379 (S2/S3): probe the model volume once Metal's on/off state is known,+ * cache the result in the model dir, and let it set the cap/CAP_RAISE+ * defaults -- never an explicit --cap/CAP/CAP_RAISE, per coli_resolve_cap()+ * above. Silent when the probe ran but storage was slow (defaults unchanged);+ * one stderr line when the platform default actually engages. */+ double coli_ssd_gbs = -1;+#if defined(COLI_METAL) && defined(__APPLE__)+ if(g_metal_enabled){+ coli_ssd_gbs = coli_ssd_probe_cached(snap);+ double ssd_fast_gbs = getenv("COLI_SSD_FAST_GBS")?atof(getenv("COLI_SSD_FAST_GBS")):4.0;+ g_ssd_fast = (coli_ssd_gbs>=ssd_fast_gbs);+ }+#endif+ int cap_explicit=0;+ int cap = coli_resolve_cap(cap_given, cap_arg, cap_env, g_ssd_fast, &cap_explicit);+ if(g_ssd_fast && !cap_explicit)+ fprintf(stderr,"METAL: fast SSD (%.1f GB/s) β page cache favored, expert cache minimal (cap 1); override with --cap\n", coli_ssd_gbs);printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits);g_mem_avail_boot = mem_available_gb();Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits);diff --git a/c/openai_server.py b/c/openai_server.pyindex 0d5d69d6..2e1ac420 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1152,7 +1152,11 @@ def read_engine_turn(stream, sentinel, on_bytes):class Engine:- def __init__(self, executable, model, cap=8, max_tokens=1024, env=None, kv_slots=1):+ # cap=0 = "not explicitly set": the engine resolves it (8 historically, 1 on+ # Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379). Same sentinel as+ # the --cap flags in coli and main() below, so programmatic callers that+ # never pass cap get the same auto behavior as the CLI.+ def __init__(self, executable, model, cap=0, max_tokens=1024, env=None, kv_slots=1):child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))self.process = subprocess.Popen(@@ -2290,7 +2294,7 @@ class APIHandler(BaseHTTPRequestHandler):def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_key=None,- cap=8, max_tokens=1024, engine=None, env=None, cors_origins=None,+ cap=0, max_tokens=1024, engine=None, env=None, cors_origins=None,max_queue=8, queue_timeout=300, kv_slots=1, allowed_hosts=()):if engine is None:engine = default_engine()@@ -2349,7 +2353,9 @@ def main():parser.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))parser.add_argument("--cors-origin", action="append", default=None,help="allowed browser origin; repeat as needed (use '*' for any origin)")- parser.add_argument("--cap", type=int, default=8)+ # 0 = not explicitly set: mirrors coli's --cap (0=auto; see colibri.c coli_resolve_cap+ # and issue #379 -- Metal+darwin+fast SSD defaults to 1, everything else to 8).+ parser.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")parser.add_argument("--max-tokens", type=int, default=1024)parser.add_argument("--max-queue", type=int, default=int(os.environ.get("COLI_MAX_QUEUE", "8")))parser.add_argument("--queue-timeout", type=float,diff --git a/c/tests/test_cap_precedence.c b/c/tests/test_cap_precedence.cnew file mode 100644index 00000000..eb7c45d2--- /dev/null+++ b/c/tests/test_cap_precedence.c@@ -0,0 +1,61 @@+/* test_cap_precedence.c -- #379 expert-cache-cap precedence contract.+ * coli_resolve_cap() (colibri.c) must implement exactly: explicit CLI positional >+ * explicit CAP env > platform default (Metal+darwin+fast SSD, cap=1) >+ * historic default. The historic default depends on how the engine was+ * invoked: a bare `./glm` (no positional at all) keeps the engine's old 64,+ * the coli wrapper's "0 = auto" sentinel keeps the 8 coli always forced.+ * It is a pure function (no I/O, no globals), so this exercises it directly+ * with fabricated inputs -- no model, no probe, no Metal hardware needed.+ * Portable: builds and runs on every platform, same as test_idot.c. */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++static int failures = 0;++static void check(const char *label, int cli_given, int cli, int env, int platform_fast,+ int want_cap, int want_explicit){+ int got_explicit = -1;+ int got_cap = coli_resolve_cap(cli_given, cli, env, platform_fast, &got_explicit);+ if(got_cap != want_cap || got_explicit != want_explicit){+ fprintf(stderr, "FAIL %-36s given=%d cli=%d env=%d fast=%d -> cap=%d explicit=%d "+ "(want cap=%d explicit=%d)\n",+ label, cli_given, cli, env, platform_fast, got_cap, got_explicit,+ want_cap, want_explicit);+ failures++;+ }+}++int main(void){+ /* bare invocation (argc<=1, no positional): byte-identical to the old+ * argc>1?atoi(argv[1]):64 fallback on every non-qualifying path. */+ check("bare, slow/no-metal", 0, 0, 0, 0, 64, 0);+ check("bare, fast SSD", 0, 0, 0, 1, 1, 0);+ check("bare + CAP env, slow", 0, 0, 4, 0, 4, 1);+ check("bare + CAP env, fast", 0, 0, 4, 1, 4, 1);+ /* wrapper sentinel (coli passes 0 = "auto"): coli's historic 8, or the+ * platform default when it engages. */+ check("sentinel, slow/no-metal", 1, 0, 0, 0, 8, 0);+ check("sentinel, fast SSD", 1, 0, 0, 1, 1, 0);+ check("sentinel + CAP env, slow", 1, 0, 6, 0, 6, 1);+ check("sentinel + CAP env, fast", 1, 0, 6, 1, 6, 1);+ /* explicit CLI always wins, whatever the platform says. */+ check("CLI wins over fast platform", 1, 16, 0, 1, 16, 1);+ check("CLI wins over slow platform", 1, 16, 0, 0, 16, 1);+ /* explicit CAP env loses to CLI. */+ check("CLI wins over env", 1, 16, 4, 1, 16, 1);+ /* cap=0 from either source is "not given", never a literal request for+ * zero cache slots (undocumented and unused repo-wide before #379). */+ check("sentinel does not read as explicit",1, 0, 0, 1, 1, 0);+ /* negative isn't 0, so it counts as explicit even though it is a+ * nonsensical cap -- the engine has never validated cap's range; this+ * test pins the precedence contract, not value sanity. */+ check("negative CLI still explicit", 1, -1, 0, 1, -1, 1);++ if(failures){+ fprintf(stderr, "cap precedence tests: %d FAILURE(S)\n", failures);+ return 1;+ }+ puts("cap precedence tests: ok");+ return 0;+}diff --git a/c/tools/eval_glm.py b/c/tools/eval_glm.pyindex 55e12d6d..32278c23 100644--- a/c/tools/eval_glm.py+++ b/c/tools/eval_glm.py@@ -109,7 +109,7 @@ def main():ap.add_argument("--tasks", default="smoke")ap.add_argument("--limit", type=int, default=40)ap.add_argument("--ram", type=int, default=0)- ap.add_argument("--cap", type=int, default=64)+ ap.add_argument("--cap", type=int, default=64) # pinned deliberately: benchmarks need a fixed cache size for reproducibility, not the platform-aware auto (#379)ap.add_argument("--bits", default="")ap.add_argument("--seed", type=int, default=1234)ap.add_argument("--dry", action="store_true", help="build requests and stop without running the engine")diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 49c87ca8..1a228c96 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -45,7 +45,9 @@ Format: `VAR` β default β effect.| `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. || `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7β40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. || `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. |-| `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. |+| `CAP` | unset | Expert-cache cap (slots/layer) when no CLI positional was given. Precedence: explicit `--cap`/positional > `CAP` > platform default > historic default (#379). Mainly for direct `./glm` use β `coli` users should prefer `--cap`. |+| `CAP_RAISE` | `1` (on); `0` on Metal + macOS + fast model volume (#379) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. When the platform-aware Metal cache default engages (F_NOCACHE probe measured the model volume fast), the *default* flips to `0` β auto-raise re-creates the Metal residency churn the minimal cache avoids. An explicit `CAP_RAISE` always wins. |+| `COLI_SSD_FAST_GBS` | `4.0` | Threshold (GB/s, measured F_NOCACHE, cached in `<model>/.coli_ssd`) at or above which the model volume counts as "fast" for the platform-aware Metal cache defaults (#379). || `PREFETCH` | `0` | Prefetch depth for streamed experts. || `COLI_MMAP` | `0` | `mmap` the weights instead of read()-ing into slabs. || `PIN` | unset | Path to a `.coli_usage`/stats file; pins the hottest experts into a resident "hot store" at startup. **`PIN=auto`** seeds from the model dir's live `.coli_usage` (appended after every turn, so each restart's pin placement follows the accumulated real workload) with `stats.txt` as the fallback for a virgin model dir; neither present β no pin this run. |diff --git a/docs/SETTINGS.md b/docs/SETTINGS.mdindex 89b836b2..7135206e 100644--- a/docs/SETTINGS.md+++ b/docs/SETTINGS.md@@ -35,7 +35,7 @@ Flags may also be given **after** the subcommand. Most flags map onto an engine| `--model` | `$COLI_MODEL` or built-in path | `SNAP` | Model snapshot directory. || `--ram` | `0` (auto β 88% free) | `RAM_GB` | RAM budget in GB for the expert working set. || `--ctx` | `0` (auto) | `CTX` | Context length. |-| `--cap` | `8` | `<cap>` argv | Expert-cache cap (starting point; see `CAP_RAISE`). |+| `--cap` | `0` (auto) | `<cap>` argv | Expert-cache cap (starting point; see `CAP_RAISE`). `0` lets the engine pick: `8` historically, `1` on Metal + macOS when the model volume measures fast (F_NOCACHE probe β₯ `COLI_SSD_FAST_GBS`, cached in `<model>/.coli_ssd` β #379). An explicit value always wins. || `--ngen` | `1024` | `NGEN` | Max tokens to generate. || `--temp` | none (`0`=greedy; engine default 1.0) | `TEMP` | Sampling temperature. || `--topp` | `0` | `TOPP` | Top-p filter. |@@ -96,7 +96,7 @@ Run directly (or via `coli serve`). OpenAI-compatible `/v1/chat/completions`.| `--api-key` | `$COLI_API_KEY` | Required bearer token. || `--cors-origin` | none (repeatable) | Allowed CORS origin(s). || `--allowed-host` | `$COLI_ALLOWED_HOSTS` or none (repeatable) | Additional Host header accepted by the DNS-rebinding guard. |-| `--cap` | `8` | Expert-cache cap. |+| `--cap` | `0` (auto) | Expert-cache cap; `0` = engine default (`8`, or `1` on Metal + macOS + fast model volume β #379). || `--max-tokens` | `1024` | Default max completion tokens. || `--max-queue` | `$COLI_MAX_QUEUE` or `8` | Max queued requests. || `--queue-timeout` | `$COLI_QUEUE_TIMEOUT` or `300` | Request queue timeout (s). |