Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge pull request #193 from bokiko/feature/disk-split

Vincenzoa8895f2d84d0
diff --cc c/glm.c
index 3e9d020e,4857946c..cea69b77
--- a/c/glm.c
+++ b/c/glm.c
@@@ -900,10 -837,15 +906,15 @@@ static int g_looka=0; /* LOOKA=1: mi
* [2] post-attention del layer L -> routing di L+1 (un residuo MoE e
* un'attention di anticipo: il punto dove il prefetch avrebbe
* un intero giro di disco per lavorare in ombra). */
-static int64_t la_hit[3], la_tot[3];
-static int la_pred[2][130][16]; static signed char la_val[2][130];
+static int64_t la_hit[4], la_tot[4]; /* [0]=prev, [1]=skip-attn, [2]=PILOT, [3]=two-step */
+static int la_pred[3][130][16]; static signed char la_val[3][130];
static int g_pilot=0; /* PILOT=1: prefetch pilotato dal router (vedi pilot_prefetch) */
static int g_pilot_k=8; /* PILOT_K=k: prefetcha solo le prime k predizioni per posizione */
+ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD (miss LRU) in
+ * draft MTP / absorb / verify-main e in layer MTP (int8) vs main
+ * (int4), con i byte letti. Default OFF: a flag spento gli atomic
+ * non vengono MAI toccati (zero overhead), le righe extra di stats
+ * non vengono stampate. Solo misura: nessun effetto sull'output. */
/* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the
* GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */
static void *qalloc(size_t n){
@@@ -3701,9 -2864,15 +3724,16 @@@ static void run_text(Model *m, const ch
printf("speculation: %.2f tokens/forward (%llu forwards per %llu tokens) | MTP acceptance %.0f%% (%llu/%llu)\n",
m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit,
m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop);
+ if(g_cp_enq) printf("couple: %ld cross-layer prefetch hints enqueued\n", g_cp_enq);
if(g_gr_prop) printf("grammar: %.0f%% acceptance (%llu/%llu forced drafts)\n",
100.0*g_gr_acc/g_gr_prop, (unsigned long long)g_gr_acc, (unsigned long long)g_gr_prop);
+ if(g_disk_split) printf("disk-load split: draft %llu + absorb %llu + verify/main %llu misses | "
+ "MTP-layer %llu loads %.2f GB | main-layers %llu loads %.2f GB (MTP %.1f%% of bytes)\n",
+ (unsigned long long)m->miss_draft, (unsigned long long)m->miss_absorb,
+ (unsigned long long)(m->miss - m->miss_draft - m->miss_absorb),
+ (unsigned long long)m->ld_mtp, m->bytes_mtp/1e9,
+ (unsigned long long)m->ld_main, m->bytes_main/1e9,
+ (m->bytes_mtp+m->bytes_main)?100.0*m->bytes_mtp/(m->bytes_mtp+m->bytes_main):0.0);
#ifdef COLI_CUDA
if(m->gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n",
m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls);