Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
st.h: shared per-shard mapping primitive, opt-in via COLI_MAP_EXPERTS=1
Ronald Klompeabeb9a1a213
diff --git a/c/glm53.c b/c/glm53.cindex 7de8c553..13dc21cd 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -1145,7 +1145,9 @@ typedef struct ERef {int contig; /* i sei pezzi sono consecutivi in un file */} ERef;-typedef struct { int eid; uint8_t *base; uint64_t used; } Slot;+/* I sei pezzi non sono adiacenti nel file, ma non devono esserlo nemmeno in+ * memoria: expert_mats ci costruisce sopra solo tre viste in sola lettura. */+typedef struct { int eid; uint8_t *piece[GLM53_EXPERT_PIECES]; uint8_t *own; uint64_t used; } Slot;typedef struct LCache { Slot *s; int n, cap; } LCache;/* Lunghezze e posizioni dei sei pezzi dentro allo slot. Gate e up sono@@ -1280,15 +1282,25 @@ static Slot *slot_find(GModel *m, int layer, int eid) {static void expert_read(GModel *m, int layer, int eid, Slot *slot) {const ERef *ref = &m->eref[(size_t)layer * m->c.n_experts + eid];- if (!slot->base) {- slot->base = malloc((size_t)m->e_slot);- if (!slot->base) { fprintf(stderr, "OOM su uno slot esperto\n"); exit(1); }- }+ int mapped_ok = 1;+ for (int p = 0; p < GLM53_EXPERT_PIECES && mapped_ok; p++) {+ const void *pr = st_map_shard_range(ref->fd[p], ref->off[p], m->e_len[p]);+ if (!pr) { mapped_ok = 0; break; }+ slot->piece[p] = (uint8_t *)pr;+ }+ if (mapped_ok) { slot->eid = eid; return; }+ /* Fallback: si torna a scrivere in memoria NOSTRA, non in una mappatura+ * di sola lettura che questo slot poteva star usando prima. */+ if (!slot->own) {+ slot->own = malloc((size_t)m->e_slot);+ if (!slot->own) { fprintf(stderr, "OOM su uno slot esperto\n"); exit(1); }+ }+ for (int p = 0; p < GLM53_EXPERT_PIECES; p++) slot->piece[p] = slot->own + m->e_at[p];if (ref->contig) {- st_pread_full(ref->fd[0], slot->base, m->e_slot, ref->off[0], "expert");+ st_pread_full(ref->fd[0], slot->own, m->e_slot, ref->off[0], "expert");} else {for (int p = 0; p < GLM53_EXPERT_PIECES; p++)- st_pread_full(ref->fd[p], slot->base + m->e_at[p], m->e_len[p],+ st_pread_full(ref->fd[p], slot->own + m->e_at[p], m->e_len[p],ref->off[p], "expert piece");}slot->eid = eid;@@ -1327,11 +1339,11 @@ static Slot *expert_slot(GModel *m, int layer, int eid) {static void expert_mats(const GModel *m, const Slot *slot, Mat *gate, Mat *up, Mat *down) {const int hidden = m->c.hidden, inter = m->c.moe_inter;const Mat shape[3] = {- { 4, NULL, NULL, slot->base + m->e_at[0], (const float *)(slot->base + m->e_at[1]),+ { 4, NULL, NULL, slot->piece[0], (const float *)slot->piece[1],inter, hidden, 64 },- { 4, NULL, NULL, slot->base + m->e_at[2], (const float *)(slot->base + m->e_at[3]),+ { 4, NULL, NULL, slot->piece[2], (const float *)slot->piece[3],inter, hidden, 64 },- { 4, NULL, NULL, slot->base + m->e_at[4], (const float *)(slot->base + m->e_at[5]),+ { 4, NULL, NULL, slot->piece[4], (const float *)slot->piece[5],hidden, inter, 64 },};*gate = shape[0]; *up = shape[1]; *down = shape[2];@@ -1925,7 +1937,7 @@ static void model_release(GModel *m) {if (m->ecache) {for (int i = 0; i < m->c.n_layers; i++) {LCache *cache = &m->ecache[i];- for (int j = 0; j < cache->cap; j++) free(cache->s[j].base);+ for (int j = 0; j < cache->cap; j++) free(cache->s[j].own);free(cache->s);}free(m->ecache);diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex 2e1ac904..4fbe7b2c 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -1094,6 +1094,21 @@ static void q38_load_native_fp8_ranges(Model *m,int layer,int expert,Slot *slot,Cfg *c=&m->c;Q38ExpertScaleCache *cache=&m->expert_scales[layer];float *scales=cache->values+(int64_t)expert*3*cache->scale_count;+ /* Se i tre intervalli sono mappati (COLI_MAP_EXPERTS=1), lo slot li PUNTA+ * invece di copiarli: niente slab, niente 14 MB per miss. */+ {+ const uint8_t *pg=(const uint8_t*)st_map_shard_range(weight[0]->fd,weight[0]->off,weight[0]->nbytes);+ const uint8_t *pu=(const uint8_t*)st_map_shard_range(weight[1]->fd,weight[1]->off,weight[1]->nbytes);+ const uint8_t *pd=(const uint8_t*)st_map_shard_range(weight[2]->fd,weight[2]->off,weight[2]->nbytes);+ if(pg&&pu&&pd){+ int sc=(int)cache->scale_count;+ q38_bind_borrowed_fp8(&slot->gate,(void*)pg,scales,c->inter,c->hidden);+ q38_bind_borrowed_fp8(&slot->up,(void*)pu,scales+sc,c->inter,c->hidden);+ q38_bind_borrowed_fp8(&slot->down,(void*)pd,scales+2*sc,c->hidden,c->inter);+ if(slot->fp8_slab){free(slot->fp8_slab);slot->fp8_slab=NULL;slot->fp8_slab_bytes=0;}+ return;+ }+ }q38_bind_fp8_slot(slot,scales,(int)cache->scale_count,c->hidden,c->inter);int64_t pair_bytes=weight[0]->nbytes+weight[1]->nbytes;unsigned char *raw=(unsigned char*)slot->fp8_slab;diff --git a/c/st.h b/c/st.hindex 9f6a55f3..9aa60390 100644--- a/c/st.h+++ b/c/st.h@@ -992,6 +992,50 @@ static void st_unmap_raw(st_mapped_raw *mapped) {mapped->nbytes = 0;}+/* ---- per-shard mapping cache (opt-in: COLI_MAP_EXPERTS=1) --------------+ * One mapping per FILE, kept for the process lifetime (shards stay open for+ * the whole run, so there is nothing shorter-lived to tie the mapping to).+ * A failed mapping is remembered per fd so it is not retried on every call;+ * callers must treat a NULL return as "use pread", not as an error. */+#define ST_MAX_MAPPED_FD 4096+static uint8_t *g_st_shard_base[ST_MAX_MAPPED_FD];+static int64_t g_st_shard_len[ST_MAX_MAPPED_FD];+static signed char g_st_shard_tried[ST_MAX_MAPPED_FD];++static int st_map_experts_enabled(void) {+ static int on = -1;+ if (on < 0) { const char *e = getenv("COLI_MAP_EXPERTS"); on = (e && atoi(e)) ? 1 : 0; }+ return on;+}++static const uint8_t *st_shard_mapped(int fd) {+ if (fd < 0 || fd >= ST_MAX_MAPPED_FD || !st_map_experts_enabled()) return NULL;+ if (g_st_shard_base[fd]) return g_st_shard_base[fd];+ if (g_st_shard_tried[fd]) return NULL;+ g_st_shard_tried[fd] = 1;+ int64_t len = (int64_t)lseek(fd, 0, SEEK_END);+ if (len <= 0) return NULL;+ compat_ro_map map; const void *data;+ if (compat_map_readonly(fd, 0, (size_t)len, &map, &data) != 0) return NULL;+ /* The compat_ro_map itself is intentionally not tracked for unmap: shard+ * mappings live exactly as long as the fd they wrap, i.e. the process. */+ g_st_shard_base[fd] = (uint8_t *)data;+ g_st_shard_len[fd] = len;+ return g_st_shard_base[fd];+}++/* Serves [off, off+nbytes) of file `fd` directly out of its persistent+ * mapping -- no allocation, no copy. Returns NULL if mapping is disabled,+ * unavailable for this fd, or the range doesn't fit; the caller's existing+ * pread path is the correct fallback in every NULL case. */+static const void *st_map_shard_range(int fd, int64_t off, int64_t nbytes) {+ if (off < 0 || nbytes <= 0) return NULL;+ const uint8_t *base = st_shard_mapped(fd);+ if (!base) return NULL;+ if (off > g_st_shard_len[fd] - nbytes) return NULL;+ return base + off;+}+/* Read an exact byte slice from a tensor into a caller-owned buffer. This is* the raw counterpart of st_read_slice_f32: byte_off/nbytes are relative to* the tensor (not the shard), and cap is the actual byte capacity of `out`.