Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge pull request #193 from bokiko/feature/disk-split
Vincenzoa8895f2d84d0
diff --cc c/glm.cindex 3e9d020e,4857946c..cea69b77--- a/c/glm.c+++ b/c/glm.c@@@ -900,10 -837,15 +906,15 @@@ static int g_looka=0; /* LOOKA=1: mi* [2] post-attention del layer L -> routing di L+1 (un residuo MoE e* un'attention di anticipo: il punto dove il prefetch avrebbe* un intero giro di disco per lavorare in ombra). */-static int64_t la_hit[3], la_tot[3];-static int la_pred[2][130][16]; static signed char la_val[2][130];+static int64_t la_hit[4], la_tot[4]; /* [0]=prev, [1]=skip-attn, [2]=PILOT, [3]=two-step */+static int la_pred[3][130][16]; static signed char la_val[3][130];static int g_pilot=0; /* PILOT=1: prefetch pilotato dal router (vedi pilot_prefetch) */static int g_pilot_k=8; /* PILOT_K=k: prefetcha solo le prime k predizioni per posizione */+ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD (miss LRU) in+ * draft MTP / absorb / verify-main e in layer MTP (int8) vs main+ * (int4), con i byte letti. Default OFF: a flag spento gli atomic+ * non vengono MAI toccati (zero overhead), le righe extra di stats+ * non vengono stampate. Solo misura: nessun effetto sull'output. *//* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the* GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */static void *qalloc(size_t n){@@@ -3701,9 -2864,15 +3724,16 @@@ static void run_text(Model *m, const chprintf("speculation: %.2f tokens/forward (%llu forwards per %llu tokens) | MTP acceptance %.0f%% (%llu/%llu)\n",m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit,m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop);+ if(g_cp_enq) printf("couple: %ld cross-layer prefetch hints enqueued\n", g_cp_enq);if(g_gr_prop) printf("grammar: %.0f%% acceptance (%llu/%llu forced drafts)\n",100.0*g_gr_acc/g_gr_prop, (unsigned long long)g_gr_acc, (unsigned long long)g_gr_prop);+ if(g_disk_split) printf("disk-load split: draft %llu + absorb %llu + verify/main %llu misses | "+ "MTP-layer %llu loads %.2f GB | main-layers %llu loads %.2f GB (MTP %.1f%% of bytes)\n",+ (unsigned long long)m->miss_draft, (unsigned long long)m->miss_absorb,+ (unsigned long long)(m->miss - m->miss_draft - m->miss_absorb),+ (unsigned long long)m->ld_mtp, m->bytes_mtp/1e9,+ (unsigned long long)m->ld_main, m->bytes_main/1e9,+ (m->bytes_mtp+m->bytes_main)?100.0*m->bytes_mtp/(m->bytes_mtp+m->bytes_main):0.0);#ifdef COLI_CUDAif(m->gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n",m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls);