Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(inkling): expert cache served the wrong experts; int4 dense container; --chat
JustVugg5f75d6789761
diff --git a/README.md b/README.mdindex d820a6ea..713a8fe8 100644--- a/README.md+++ b/README.md@@ -325,6 +325,27 @@ the full 756 GB on disk at once:./coli convert --model /nvme/glm52_i4 # download+convert shard by shard (python, one-time)```+#### Other supported models++GLM-5.2 is the reference model, but the same streaming approach runs three more+families. Each is a **sibling engine** β one C file, its own architecture, the same+`coli chat` / `coli serve` / `coli web` front end (the launcher picks the binary from+the model's `config.json`):++| Family | Total / active | Weights | Build | Docs |+|---|---|---|---|---|+| **GLM-5.2** | 744B / 40B | [`mastouri/β¦-int4-g64-with-int8-mtp`](https://huggingface.co/mastouri/GLM-5.2-colibri-int4-g64-with-int8-mtp) (372 GB) | `make -C c glm` | this page |+| **Inkling** (Thinking Machines) | 975B / 41B | [`nbeerbower/Inkling-colibri-int4`](https://huggingface.co/nbeerbower/Inkling-colibri-int4) (469 GB) | `make -C c inkling` | [inkling.md](docs/inkling.md) |+| **Kimi K3** (Moonshot) | 2.8T / 104B | [`moonshotai/Kimi-K3`](https://huggingface.co/moonshotai/Kimi-K3) β original checkpoint, routed experts stay **native MXFP4** | `make -C c kimi_k3` | [kimi_k3.md](docs/kimi_k3.md) |+| **OLMoE** (AI2) | 7B / 1B | converted with `c/tools/convert_olmoe_merged.py` | `make -C c olmoe` | β |++Kimi K3 needs no conversion: its QAT-trained MXFP4 experts are streamed straight from+the original Hugging Face shards, and the bf16 dense set is quantized at load time.+Inkling ships int4 experts but **bf16 dense weights** (49.4 GB resident); on a host+that cannot hold those, [inkling.md](docs/inkling.md) has a one-pass tool that brings+the dense set to 15.3 GB and lets the 975B run on a 25 GB box β with the honest+trade-off written down.+### 3. Run it```bashdiff --git a/c/inkling.c b/c/inkling.cindex 390c217c..408aec73 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -68,7 +68,14 @@ typedef struct {* move to VRAM (dev set, host freed): decode reads ~35 GB of residents per* token, so this trades the DDR5 bandwidth wall for VRAM bandwidth AND* frees the same RAM for the expert cache. */-typedef struct { float *f; uint16_t *h; void *dev; } Wt;+/* q4/qs: densa pre-quantizzata int4 group-scaled (gs=64) da un container+ * separato, opzionale β vedi load_w. La densa bf16 di Inkling e' 49.4 GB e non+ * entra in 25 GB (il caricamento la espande pure a f32, ~99 GB al picco: e' li'+ * l'OOM). A int4-gs64 diventa ~15 GB. Nessun campo q4 = comportamento invariato. */+typedef struct { float *f; uint16_t *h; void *dev;+ uint8_t *q4; /* int4 nibble-packed (qbits=4) o int8 (qbits=8) */+ float *qs; /* scale: [rows*ng] se qbits=4, [rows] se qbits=8 */+ int gs, qbits; int64_t qn; } Wt; /* qn = byte in q4, per il guard OOB */typedef struct {float *in_ln, *post_ln;@@ -100,6 +107,9 @@ typedef struct { Slot *slots; int n, cap; } LCache;typedef struct {Cfg c;shards S;+ shards Sq; /* container densa int4-gs64 (opzionale) */+ int has_q, q_loaded; /* has_q: container presente; q_loaded: tensori presi da li */+ int64_t q_bytes;int quant_bits; /* 0 = f32 experts (oracle mode) */int xq; /* experts on disk are a colibri container (U8 + .qs) */Wt embed, lm_head;@@ -196,6 +206,50 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,}/* dispatch on where the weight lives */+/* y[S,O] = x[S,I] @ W^T con W int4 GROUP-scaled: nibble +8, low = colonna pari,+ * una scala f32 ogni `gs` elementi lungo I (ng = ceil(I/gs) scale per riga).+ * Differenza da matmul_q4 (per-riga): la scala cambia DENTRO la riga, quindi+ * l'accumulo va chiuso a ogni gruppo invece che una volta sola a fine riga. */+static void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *scale,+ int S, int I, int O, int gs) {+ int ng = (I + gs - 1) / gs;+ int64_t rb = (I + 1) / 2;+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const uint8_t *w = p + (int64_t)o * rb;+ const float *sc = scale + (int64_t)o * ng;+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s * I;+ float acc = 0.f;+ for (int g = 0; g < ng; g++) {+ int i0 = g * gs, i1 = i0 + gs; if (i1 > I) i1 = I;+ float part = 0.f;+ for (int i = i0; i < i1; i++) {+ uint8_t b = w[i >> 1];+ int q = (i & 1) ? (b >> 4) : (b & 0x0F);+ part += xs[i] * (float)(q - 8);+ }+ acc += part * sc[g]; /* scala chiusa per gruppo */+ }+ y[(int64_t)s * O + o] = acc;+ }+ }+}+/* int8 per-riga (embed/lm_head: sensibili, non vanno a 4 bit) */+static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *scale,+ int S, int I, int O) {+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const int8_t *w = q + (int64_t)o * I; float sc = scale[o];+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s * I;+ float acc = 0.f;+ for (int i = 0; i < I; i++) acc += xs[i] * (float)w[i];+ y[(int64_t)s * O + o] = acc * sc;+ }+ }+}+static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {#ifdef COLI_CUDAif (W.dev) {@@ -203,6 +257,18 @@ static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {fprintf(stderr, "cuda matmul failed and host copy was freed\n"); exit(1);}#endif+ if (W.q4) {+ /* guard: il container e' un file, non un invariante β se la geometria non+ * torna si esce invece di leggere fuori dal buffer. */+ int64_t need = W.qbits == 8 ? (int64_t)O * I : (int64_t)O * ((I + 1) / 2);+ if (need > W.qn) {+ fprintf(stderr, "dense q4: geometria incoerente (serve %lld B, ho %lld) I=%d O=%d\n",+ (long long)need, (long long)W.qn, I, O); exit(1);+ }+ if (W.qbits == 8) matmul_i8r(y, x, (const int8_t*)W.q4, W.qs, S, I, O);+ else matmul_i4g(y, x, W.q4, W.qs, S, I, O, W.gs);+ return;+ }if (W.f) matmul(y, x, W.f, S, I, O);else matmul_h(y, x, W.h, S, I, O);}@@ -478,10 +544,46 @@ static void pread_all(int fd, void *buf, int64_t nb, int64_t off) {* checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact).* gpu_ok: bf16 tensors move to VRAM while budget lasts (embed stays host β* it's a row lookup, not a matmul). */+/* Container densa pre-quantizzata (opzionale): <snap>/dense-int4g64/.+ * Se il tensore c'e' li' dentro lo carichiamo int4-gs64 (U8 + sidecar .qs, la+ * stessa convenzione che il container usa gia' per gli esperti); altrimenti si+ * prosegue col percorso bf16/f32 di sempre. Container assente = nessun cambio. */+static int load_w_quant(Model *m, const char *name, int64_t orig_numel, Wt *out) {+ if (!m->has_q) return 0;+ st_tensor *t = st_find(&m->Sq, name);+ if (!t || t->dtype != 3) return 0; /* 3 = U8/byte grezzi */+ char qn[352]; snprintf(qn, sizeof(qn), "%s.qs", name);+ st_tensor *s = st_find(&m->Sq, qn);+ if (!s) return 0; /* senza scale non si decodifica */+ /* Geometria dedotta dai CONTEGGI, non dalle shape (st_tensor non le porta):+ * int4-gs64 -> byte ~= numel/2 e scale ~= numel/64+ * int8 -> byte == numel e scale == righe (numel/I, I ignoto qui:+ * basta che le scale siano molte meno dei byte)+ * Il controllo forte sull'OOB lo fa matmul_w con l'I vero del chiamante. */+ Wt w = {0};+ if (t->nbytes == orig_numel && s->numel * 64 < (int64_t)t->nbytes) {+ w.qbits = 8; /* int8 per riga */+ } else if (t->nbytes * 2 >= orig_numel && t->nbytes * 2 <= orig_numel + 2 * s->numel) {+ w.qbits = 4; w.gs = 64; /* int4 group-scaled */+ } else {+ fprintf(stderr, "[dense] %s: geometria non riconosciuta nel container, uso il bf16\n", name);+ return 0; /* dubbio -> percorso originale */+ }+ w.q4 = malloc(t->nbytes); if (!w.q4) { fprintf(stderr, "OOM %s\n", name); exit(1); }+ pread_all(t->fd, w.q4, t->nbytes, t->off);+ w.qn = t->nbytes;+ w.qs = falloc(s->numel);+ st_read_f32(&m->Sq, qn, w.qs, 0);+ *out = w;+ m->q_loaded++; m->q_bytes += t->nbytes + (int64_t)s->numel * 4;+ return 1;+}+static Wt load_w(Model *m, const char *name, int gpu_ok) {Wt w = {0};st_tensor *t = st_find(&m->S, name);if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }+ if (load_w_quant(m, name, t->numel, &w)) return w;if (t->dtype == 0) {w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); }pread_all(t->fd, w.h, t->nbytes, t->off);@@ -500,12 +602,40 @@ static Wt load_w(Model *m, const char *name, int gpu_ok) {}return w;}-static Wt wt_off(Wt w, int64_t off) {- Wt r = { w.f ? w.f + off : NULL, w.h ? w.h + off : NULL,- w.dev ? (char*)w.dev + off*2 : NULL }; /* dev is always bf16 */+/* `off` conta ELEMENTI ed e' sempre un multiplo di I (si affetta per riga: un+ * esperto condiviso dal tensore fuso [E,R,I]), quindi I arriva dal chiamante β+ * st_tensor non porta le shape e non voglio indovinarle. */+static Wt wt_off_i(Wt w, int64_t off, int I) {+ Wt r = w;+ r.f = w.f ? w.f + off : NULL;+ r.h = w.h ? w.h + off : NULL;+ r.dev = w.dev ? (char*)w.dev + off*2 : NULL; /* dev is always bf16 */+ if (w.q4) {+ int64_t row = off / I;+ if (w.qbits == 8) { r.q4 = w.q4 + off; r.qs = w.qs + row; r.qn = w.qn - off; }+ else { r.q4 = w.q4 + off / 2; r.qs = w.qs + row * ((I + w.gs - 1) / w.gs);+ r.qn = w.qn - off / 2; }+ }return r;}+/* dequantizza UNA riga (int4-gs64: nibble +8, low = colonna pari, scala ogni gs) */+static void wt_deq_row(Wt w, int64_t row, float *out, int I) {+ if (w.qbits == 8) {+ const int8_t *q = (const int8_t*)w.q4 + row * I; float s = w.qs[row];+ for (int i = 0; i < I; i++) out[i] = (float)q[i] * s;+ return;+ }+ int ng = (I + w.gs - 1) / w.gs;+ const uint8_t *p = w.q4 + row * ((I + 1) / 2);+ const float *sc = w.qs + row * ng;+ for (int i = 0; i < I; i++) {+ uint8_t b = p[i >> 1];+ int q = (i & 1) ? (b >> 4) : (b & 0x0F);+ out[i] = (float)(q - 8) * sc[i / w.gs];+ }+}static void wt_row_f32(Wt w, int64_t off, float *out, int n) {+ if (w.q4) { wt_deq_row(w, off / n, out, n); return; } /* n = dim di contrazione */if (w.f) memcpy(out, w.f + off, n * sizeof(float));else for (int i = 0; i < n; i++) { union { uint32_t u; float f; } v = { (uint32_t)w.h[off + i] << 16 }; out[i] = v.f; }}@@ -557,6 +687,18 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {m->quant_bits = bits;load_cfg(&m->c, snap);st_init(&m->S, snap);+ /* densa pre-quantizzata, se il container c'e' (INK_DENSE_Q4=0 la ignora).+ * Sta in una SOTTOCARTELLA apposta: nella dir dello snapshot i nomi tensore+ * andrebbero in collisione con i bf16 originali, che restano intatti. */+ { char qd[2100]; snprintf(qd, sizeof(qd), "%s/dense-int4g64", snap);+ const char *off = getenv("INK_DENSE_Q4");+ struct stat qs;+ if (!(off && *off == '0') && stat(qd, &qs) == 0 && S_ISDIR(qs.st_mode)) {+ st_init(&m->Sq, qd);+ if (m->Sq.n > 0) { m->has_q = 1;+ fprintf(stderr, "[dense] container int4-gs64: %s (%d tensori)\n", qd, m->Sq.n); }+ }+ }Cfg *c = &m->c;int D = c->hidden, K = c->conv_k;double t0 = now_s();@@ -945,8 +1087,30 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {for (int kk = 0; kk < K; kk++) { w[kk] = sigmoidf(lg[si[kk]]); sum += w[kk]; }for (int j = 0; j < ns; j++) { w[K+j] = sigmoidf(lg[E+j]); sum += w[K+j]; }for (int kk = 0; kk < K+ns; kk++) w[kk] *= c->route_scale * l->rgs / sum;- for (int kk = 0; kk < K; kk++) {- int eid = si[kk];+ }+ /* Il ciclo sotto ACQUISISCE uno slot per ogni coppia (token, esperto) e ne tiene+ * il puntatore fino al calcolo. slot_acquire evince l'LRU quando la cache e'+ * piena β anche uno slot gia' consegnato in QUESTA chiamata: il puntatore resta+ * valido ma lo slot ora contiene un ALTRO esperto, e il modello calcola con i+ * pesi sbagliati. In silenzio: niente crash, solo output incoerente. Acquisire+ * tutte le S*K coppie in una volta pretendeva quindi una cache capace di tenere+ * tutti gli esperti distinti del batch (18 token x topk 6 = fino a 108 slot per+ * layer), e sotto quella soglia il modello sembrava rotto.+ * Fix: si lavora a GIRI di al piu' `cap` coppie β acquisisci, riempi, calcola e+ * accumula β cosi' nessuno slot puo' essere evinto mentre serve. L'output MoE e'+ * una somma pesata, quindi accumulare a giri da' lo stesso risultato di una+ * passata sola, e la cache puo' scendere fino a 1 slot per layer (piu' letture+ * da disco, ma memoria proporzionale a `cap` invece che al batch). */+ int cap = m->cache[layer].cap; if (cap < 1) cap = 1;+ float *g = falloc(2*I), *u = g + I, *hh = falloc(D);+ int q4 = m->xq && m->rb13*2 == D; /* packed int4 vs int8 container */+ int64_t npair = (int64_t)S*K;+ for (int64_t base = 0; base < npair; base += cap) {+ int64_t end = base + cap < npair ? base + cap : npair;+ nfill = 0;+ for (int64_t t = base; t < end; t++) { /* acquisizione del giro */+ int s = (int)(t / K), kk = (int)(t % K);+ int eid = idx[(int64_t)s*K + kk];if (m->eusage[layer]) m->eusage[layer][eid]++;Slot *e = slot_find(m, layer, eid);if (e) m->hits++;@@ -955,27 +1119,21 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {e = slot_acquire(m, layer, eid);fill[nfill] = e; fl[nfill] = layer; nfill++;}- use[(int64_t)s*K + kk] = e;+ use[t - base] = e;+ }+ if (nfill) { /* riempimento in parallelo */+ double tf = now_s();+ #pragma omp parallel for schedule(dynamic,1)+ for (int j = 0; j < nfill; j++) slot_fill(m, fl[j], fill[j]);+ m->t_fill += now_s() - tf;}- }- /* pass 2: one parallel burst for every miss in this layer call */- if (nfill) {- double tf = now_s();- #pragma omp parallel for schedule(dynamic,1)- for (int j = 0; j < nfill; j++) slot_fill(m, fl[j], fill[j]);- m->t_fill += now_s() - tf;- }- /* pass 3: compute. gate+up run as ONE matmul over the fused 2I rows β- * halves the number of (expensive to open) parallel regions per expert */- float *g = falloc(2*I), *u = g + I, *hh = falloc(D);- int q4 = m->xq && m->rb13*2 == D; /* packed int4 vs int8 container */- for (int s = 0; s < S; s++) {- const float *xs = x + (int64_t)s*D;- float *os = out + (int64_t)s*D;- float *w = wgt + (int64_t)s*(K+ns);double te = now_s();- for (int kk = 0; kk < K; kk++) {- Slot *e = use[(int64_t)s*K + kk];+ for (int64_t t = base; t < end; t++) { /* calcolo + accumulo */+ int s = (int)(t / K), kk = (int)(t % K);+ const float *xs = x + (int64_t)s*D;+ float *os = out + (int64_t)s*D;+ float *w = wgt + (int64_t)s*(K+ns);+ Slot *e = use[t - base];if (m->xq) {if (q4) {matmul_q4(g, xs, e->p13, e->s13, D, 2*I); /* gate rows then up rows */@@ -997,13 +1155,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {}for (int d = 0; d < D; d++) os[d] += w[kk] * hh[d];}- double ts = now_s(); m->t_expert += ts - te;- /* shared experts: gamma inside (before down_proj is linear, so applied at the end) */+ m->t_expert += now_s() - te;+ }+ /* shared experts: una volta per token, fuori dai giri (non usano la cache) */+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s*D;+ float *os = out + (int64_t)s*D;+ float *w = wgt + (int64_t)s*(K+ns);+ double ts = now_s();+ /* gamma inside (before down_proj is linear, so applied at the end) */for (int j = 0; j < ns; j++) {- matmul_w(g, xs, wt_off(l->sh_g, (int64_t)j*I*D), 1, D, I);- matmul_w(u, xs, wt_off(l->sh_u, (int64_t)j*I*D), 1, D, I);+ matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);+ matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];- matmul_w(hh, g, wt_off(l->sh_d, (int64_t)j*D*I), 1, I, D);+ matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];}m->t_shared += now_s() - ts;@@ -1400,15 +1565,34 @@ int main(int argc, char **argv) {if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }/* flags: -p "prompt" [-n N] -> generate mode; positional: [cap] [bits] [ref.json] */const char *prompt = NULL, *pfile = NULL, *refpath = "ref_inkling.json";- int cap = -1, bits = 0, n_new = 256, npos = 0;+ int cap = -1, bits = 0, n_new = 256, npos = 0, chat = 0;for (int i = 1; i < argc; i++) {if (!strcmp(argv[i], "-p") && i+1 < argc) prompt = argv[++i];else if (!strcmp(argv[i], "-f") && i+1 < argc) pfile = argv[++i];else if (!strcmp(argv[i], "-n") && i+1 < argc) n_new = atoi(argv[++i]);+ else if (!strcmp(argv[i], "--chat")) chat = 1;else if (npos == 0) { cap = atoi(argv[i]); npos++; }else if (npos == 1) { bits = atoi(argv[i]); npos++; }else refpath = argv[i];}+ /* --chat: avvolge il prompt nel template di Inkling (sottoinsieme testuale di+ * chat_template.jinja, lo stesso che openai_server.py rende in render_chat_inkling):+ * token di ruolo + <|content_text|>, <|end_message|> a chiudere, il livello di+ * thinking come messaggio di sistema, e <|message_model|> come prompt di+ * generazione. Senza template un modello instruct riceve testo fuori+ * distribuzione. THINK=<0..1> alza lo sforzo di ragionamento (default 0). */+ char *chat_buf = NULL;+ if (chat && prompt) {+ const char *eff = getenv("THINK") ? getenv("THINK") : "0";+ size_t need = strlen(prompt) + strlen(eff) + 256;+ chat_buf = malloc(need);+ if (!chat_buf) { fprintf(stderr, "OOM chat template\n"); return 1; }+ snprintf(chat_buf, need,+ "<|message_user|><|content_text|>%s<|end_message|>"+ "<|message_system|><|content_text|>Thinking effort level: %s<|end_message|>"+ "<|message_model|>", prompt, eff);+ prompt = chat_buf;+ }if (cap < 0) cap = (prompt || pfile) ? 0 : 16; /* generate mode defaults to RAM-sized auto cap */if (bits && (bits < 2 || bits > 8)) { fprintf(stderr, "quant_bits must be 0 (f32) or 2..8\n"); return 1; }diff --git a/c/tools/convert_inkling_dense_int4.py b/c/tools/convert_inkling_dense_int4.pynew file mode 100644index 00000000..4e6a27a5--- /dev/null+++ b/c/tools/convert_inkling_dense_int4.py@@ -0,0 +1,290 @@+#!/usr/bin/env python3+"""Inkling: quantizza la DENSA RESIDENTE (bf16 -> int4-gs64) in un container a parte.++Serve solo se la RAM non basta. Il container int4 di Inkling+(nbeerbower/Inkling-colibri-int4) ha gli esperti routed gia' a int4 ma tiene la+densa in **bf16**: 49.4 GB che devono stare in RAM per ogni token. Su un host con+meno di ~64 GB non ci stanno β e il caricamento li espande pure a f32 (~99 GB al+picco), quindi il processo muore prima di generare. A int4-gs64 la densa scende a+**15.3 GB** e il modello gira su una macchina da 25 GB.++Con RAM abbondante NON serve: `inkling.c` carica la densa bf16 come sempre.++Cosa fa:+ - legge SOLO gli header degli shard (pochi MB) e costruisce un indice+ - quantizza un tensore alla volta, a blocchi di righe (picco RAM ~1 GB)+ - scrive UN file nuovo; gli esperti routed (464 GB) non vengono toccati+ - gli originali sono aperti in sola lettura: se qualcosa va storto si rifa'+ - riporta l'errore di quantizzazione L2 per classe, misurato sui pesi veri++Formato di output (stessa convenzione .qs che inkling.c usa gia' per gli esperti):+ <nome> U8 [O, ceil(I/2)] nibble packed, low = colonna pari, offset +8+ <nome>.qs F32 [O, ceil(I/64)] una scala ogni 64 elementi lungo l'INPUT+ __metadata__ {"<nome>": "fmt=4;gs=64"} / "fmt=1" per int8 / "fmt=raw"++Politica di precisione (misurata: vedi docs/inkling.md):+ attention / shared_experts / mlp denso -> int4-gs64 ~11% errore L2+ embed_tokens / lm_head -> int8 per-row ~0.9% (entrano in OGNI token)+ norm / bias / A_log / conv1d / router -> passthrough (minuscoli e delicati)+ATTN_BITS=8 porta anche l'attention a int8 (+4 GB, errore 11% -> 1.1%): non serve+per la coerenza dell'output, ma e' li' se vuoi spendere RAM per la qualita'.++Uso:+ python3 convert_inkling_dense_int4.py --dir /path/al/modello --plan # stima+ python3 convert_inkling_dense_int4.py --dir /path/al/modello # converte+ mkdir -p /path/al/modello/dense-int4g64+ mv /path/al/modello/dense-int4g64.safetensors \\+ /path/al/modello/dense-int4g64/dense.safetensors+Il motore trova il container da solo (INK_DENSE_Q4=0 lo ignora).+"""+import argparse, json, os, re, struct, sys, time+import numpy as np++DIR = os.environ.get("INKLING_DIR", ".")+OUT = os.path.join(DIR, "dense-int4g64.safetensors")+GS = 64+ROWS_PER_CHUNK = 4096 # blocchi di righe: tiene il picco RAM ~<1 GB++# ---------- classificazione (per nome) ----------+RE_EXPERT = re.compile(r"\.experts\.") # routed: NON toccare (shared_experts escluso sotto)+RE_SHARED = re.compile(r"shared_experts")+RE_ATTN = re.compile(r"self_attn")+RE_MLPD = re.compile(r"\.mlp\.(gate|up|down)_proj")+RE_EMB = re.compile(r"(embed_tokens|lm_head)")+RE_SMALL = re.compile(r"(norm|bias|A_log|dt_bias|conv1d|\.gate\.|rotary|scale$)", re.I)++MIN_ELEMS = 1 << 20 # sotto ~1M elementi non vale il rischio: passthrough+ATTN_BITS = int(os.environ.get("ATTN_BITS", "8")) # attention: 8 = int8 (default), 4 = int4-gs64++def classify(name, shape, dtype):+ """-> 'skip' (expert routed) | 'gs4' | 'int8' | 'raw'++ Regola GENERALE, non una whitelist di nomi: qualunque peso float grande e+ 2-D/3-D con l'ultima dimensione (quella di contrazione) >= GS va a int4-gs64.+ Una whitelist mi aveva gia' fatto perdere shared_experts (3-D [2,3072,6144])+ e mtp.*.input_proj, 14.5 GB finiti in passthrough senza accorgersene."""+ if RE_EXPERT.search(name) and not RE_SHARED.search(name):+ return "skip" # esperti routed: restano negli shard+ if dtype not in ("BF16", "F16", "F32"):+ return "raw" # gia' quantizzato o non-float+ if RE_EMB.search(name):+ return "int8" # embed/lm_head: sensibili, ogni token+ if RE_SMALL.search(name):+ return "raw" # norm/bias/A_log/conv1d/router: delicati+ # ATTN_BITS=8 (default): l'attention a 4 bit rende il modello incoerente β+ # misurato: 11% di errore per matmul x ~5 matmul x 66 layer e l'output degenera+ # (il container e il kernel sono verificati corretti, e' la precisione a mancare).+ # ATTN_BITS=4 la rimette a int4-gs64 per confronto.+ if RE_ATTN.search(name) and ATTN_BITS == 8:+ return "int8"+ n = 1+ for d in shape:+ n *= d+ if len(shape) < 2 or shape[-1] < GS or n < MIN_ELEMS:+ return "raw" # 1-D, contrazione corta o troppo piccolo+ return "gs4"++# ---------- I/O safetensors ----------+def read_header(path):+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ hdr = json.loads(f.read(n))+ return hdr, 8 + n++def read_rows(path, base, off0, dtype, cols, r0, r1):+ """Legge le righe [r0,r1) di un tensore 2-D e le restituisce in f32."""+ itemsize = {"BF16": 2, "F16": 2, "F32": 4, "U8": 1, "I8": 1}[dtype]+ start = base + off0 + r0 * cols * itemsize+ nbytes = (r1 - r0) * cols * itemsize+ with open(path, "rb") as f:+ f.seek(start)+ raw = f.read(nbytes)+ if dtype == "BF16": # bf16 -> f32: shift a sinistra di 16 bit+ u = np.frombuffer(raw, np.uint16).astype(np.uint32) << 16+ return u.view(np.float32).reshape(r1 - r0, cols)+ if dtype == "F16":+ return np.frombuffer(raw, np.float16).astype(np.float32).reshape(r1 - r0, cols)+ if dtype == "F32":+ return np.frombuffer(raw, np.float32).reshape(r1 - r0, cols)+ raise ValueError(f"dtype non gestito per la quantizzazione: {dtype}")++# ---------- quantizzatori ----------+def quant_gs4(w):+ """w [R, I] f32 -> (packed U8 [R, ceil(I/2)], scales f32 [R, ng]).+ Gruppi lungo I (dimensione di contrazione), simmetrico, nibble +8."""+ R, I = w.shape+ ng = (I + GS - 1) // GS+ pad = ng * GS - I+ wp = np.concatenate([w, np.zeros((R, pad), np.float32)], 1) if pad else w+ g = wp.reshape(R, ng, GS)+ amax = np.abs(g).max(-1, keepdims=True)+ scale = (amax / 7.0).astype(np.float32)+ scale[scale == 0] = 1.0+ q = np.clip(np.rint(g / scale), -8, 7).astype(np.int8) + 8 # 0..15+ q = q.reshape(R, ng * GS)[:, :I].astype(np.uint8)+ if I % 2: # coda dispari+ q = np.concatenate([q, np.full((R, 1), 8, np.uint8)], 1)+ packed = (q[:, 0::2] | (q[:, 1::2] << 4)).astype(np.uint8)+ return packed, scale.reshape(R, ng).astype(np.float32)++def dequant_gs4(packed, scale, I):+ R = packed.shape[0]+ q = np.empty((R, packed.shape[1] * 2), np.float32)+ q[:, 0::2] = (packed & 15).astype(np.float32) - 8.0+ q[:, 1::2] = (packed >> 4).astype(np.float32) - 8.0+ q = q[:, :I]+ ng = scale.shape[1]+ s = np.repeat(scale, GS, axis=1)[:, :I]+ return q * s++def quant_i8(w):+ amax = np.abs(w).max(1, keepdims=True)+ scale = (amax / 127.0).astype(np.float32)+ scale[scale == 0] = 1.0+ q = np.clip(np.rint(w / scale), -127, 127).astype(np.int8)+ return q, scale[:, 0].astype(np.float32)++def dequant_i8(q, scale):+ return q.astype(np.float32) * scale[:, None]++# ---------- main ----------+def main():+ global DIR+ ap = argparse.ArgumentParser()+ ap.add_argument("--plan", action="store_true", help="stima e piano, non scrive nulla")+ ap.add_argument("--dir", default=DIR, help="dir del modello")+ ap.add_argument("--out", default=OUT)+ a = ap.parse_args()++ DIR = a.dir+ if a.out == OUT: a.out = os.path.join(DIR, "dense-int4g64.safetensors")+ shards = sorted(f for f in os.listdir(DIR) if f.endswith(".safetensors") and not f.startswith("dense-"))+ print(f"[1/3] indicizzo {len(shards)} shard (solo header)β¦", flush=True)+ index = {} # name -> (path, base, off0, off1, dtype, shape)+ for sh in shards:+ p = os.path.join(DIR, sh)+ hdr, base = read_header(p)+ for k, t in hdr.items():+ if k == "__metadata__":+ continue+ o = t["data_offsets"]+ index[k] = (p, base, o[0], o[1], t["dtype"], t["shape"])++ def flat2d(shape):+ """[E,R,I] o [R,I] -> (righe, I): i gruppi vanno sull'ULTIMA dim (contrazione)."""+ I = shape[-1]+ R = 1+ for d in shape[:-1]:+ R *= d+ return R, I++ plan, tot_in, tot_out = [], 0, 0+ for name, (p, base, o0, o1, dt, shape) in sorted(index.items()):+ kind = classify(name, shape, dt)+ if kind == "skip":+ continue+ nbytes_in = o1 - o0+ tot_in += nbytes_in+ if kind == "gs4":+ R, I = flat2d(shape)+ ng = (I + GS - 1) // GS+ out_b = R * ((I + 1) // 2) + R * ng * 4+ elif kind == "int8":+ R, I = flat2d(shape)+ out_b = R * I + R * 4+ else:+ out_b = nbytes_in+ tot_out += out_b+ plan.append((name, kind, p, base, o0, o1, dt, tuple(shape), out_b))++ by = {}+ for name, kind, *_rest in plan:+ by[kind] = by.get(kind, 0) + 1+ print(f" densa da convertire: {len(plan)} tensori {by}")+ print(f" input bf16/f32 : {tot_in/1e9:7.2f} GB")+ print(f" output stimato : {tot_out/1e9:7.2f} GB ({100*tot_out/max(tot_in,1):.0f}%)")+ free = os.statvfs(DIR).f_bavail * os.statvfs(DIR).f_frsize+ print(f" spazio libero : {free/1e9:7.2f} GB")+ if a.plan:+ print("\n[--plan] nessuna scrittura. Rilancia senza --plan per convertire.")+ return+ if free < tot_out * 1.15:+ sys.exit(f"[ERR] spazio insufficiente: servono ~{tot_out*1.15/1e9:.0f} GB")++ # ---- header di output: shape/offset noti in anticipo ----+ print(f"[2/3] costruisco l'header di outputβ¦", flush=True)+ hdr, off, meta = {}, 0, {}+ for name, kind, p, base, o0, o1, dt, shape, _ob in plan:+ if kind == "gs4":+ R, I = flat2d(shape)+ ng = (I + GS - 1) // GS+ nb = R * ((I + 1) // 2)+ hdr[name] = {"dtype": "U8", "shape": [R, (I + 1) // 2], "data_offsets": [off, off + nb]}; off += nb+ nb = R * ng * 4+ hdr[name + ".qs"] = {"dtype": "F32", "shape": [R, ng], "data_offsets": [off, off + nb]}; off += nb+ # orig= conserva la shape logica (3-D per gli shared_experts): il loader+ # ricostruisce [E,R,I] dai byte piatti senza doverla indovinare.+ meta[name] = f"fmt=4;gs={GS};I={I};orig={'x'.join(str(d) for d in shape)}"+ elif kind == "int8":+ R, I = flat2d(shape)+ nb = R * I+ hdr[name] = {"dtype": "I8", "shape": [R, I], "data_offsets": [off, off + nb]}; off += nb+ nb = R * 4+ hdr[name + ".qs"] = {"dtype": "F32", "shape": [R], "data_offsets": [off, off + nb]}; off += nb+ meta[name] = f"fmt=1;orig={'x'.join(str(d) for d in shape)}"+ else:+ nb = o1 - o0+ hdr[name] = {"dtype": dt, "shape": list(shape), "data_offsets": [off, off + nb]}; off += nb+ meta[name] = "fmt=raw"+ meta["colibri.dense"] = "inkling-int4g64-v1"+ hdr["__metadata__"] = meta+ hjson = json.dumps(hdr, separators=(",", ":")).encode()+ pad = (8 - (len(hjson) % 8)) % 8 # header allineato a 8 byte+ hjson += b" " * pad++ tmp = a.out + ".part"+ print(f"[3/3] scrivo {a.out} ({off/1e9:.2f} GB payload)β¦", flush=True)+ err_by_kind = {}+ t0 = time.time()+ with open(tmp, "wb") as w:+ w.write(struct.pack("<Q", len(hjson))); w.write(hjson)+ done = 0+ for name, kind, p, base, o0, o1, dt, shape, _ob in plan:+ if kind == "raw":+ with open(p, "rb") as f:+ f.seek(base + o0)+ left = o1 - o0+ while left > 0:+ chunk = f.read(min(left, 8 << 20)); w.write(chunk); left -= len(chunk)+ else:+ R, I = flat2d(shape)+ packs, scales, errs = [], [], []+ for r0 in range(0, R, ROWS_PER_CHUNK):+ r1 = min(r0 + ROWS_PER_CHUNK, R)+ blk = read_rows(p, base, o0, dt, I, r0, r1)+ if kind == "gs4":+ pk, sc = quant_gs4(blk); rec = dequant_gs4(pk, sc, I)+ else:+ pk, sc = quant_i8(blk); rec = dequant_i8(pk, sc)+ n = float(np.linalg.norm(blk))+ if n > 0: errs.append(float(np.linalg.norm(blk - rec)) / n)+ packs.append(pk); scales.append(sc)+ w.write(np.concatenate(packs, 0).tobytes())+ w.write(np.concatenate(scales, 0).astype(np.float32).tobytes())+ if errs:+ k = "attention" if RE_ATTN.search(name) else \+ "shared" if RE_SHARED.search(name) else \+ "embed/head"if RE_EMB.search(name) else "mlp denso"+ err_by_kind.setdefault(k, []).append(float(np.mean(errs)))+ done += 1+ if done % 100 == 0:+ print(f" {done}/{len(plan)} ({time.time()-t0:.0f}s)", flush=True)+ os.replace(tmp, a.out)+ sz = os.path.getsize(a.out)+ print(f"\nβ scritto {a.out} {sz/1e9:.2f} GB in {time.time()-t0:.0f}s")+ print(f"\n=== errore di quantizzazione (L2 relativo, sui pesi VERI) ===")+ for k, v in sorted(err_by_kind.items()):+ print(f" {k:12} {100*float(np.mean(v)):.3f}% (su {len(v)} tensori)")++if __name__ == "__main__":+ main()diff --git a/docs/inkling.md b/docs/inkling.mdindex f86028a6..8b211544 100644--- a/docs/inkling.md+++ b/docs/inkling.md@@ -33,14 +33,77 @@ Requirements: ~120 GB RAM (CPU build keeps ~86 GB of bf16 residents in RAM;the CUDA build moves them to VRAM and uses the freed RAM for a larger expertcache), NVMe storage for the snapshot.+## If you have less RAM than that++The pre-converted container has **int4 routed experts but bf16 dense weights**, and+the dense set is resident: every token needs all of it. Measured on the real+snapshot it is **49.4 GB**, and `load_w` expands bf16 to f32 while loading, so the+peak is ~99 GB. Below roughly 64 GB of RAM the process dies before it generates+anything.++Quantizing just the dense set to **int4-gs64** brings it to **15.3 GB**, which fits a+25 GB box. One pass over the shards, originals untouched, ~14 minutes:++```sh+python3 c/tools/convert_inkling_dense_int4.py --dir ~/Models/inkling_i4 --plan # estimate first+python3 c/tools/convert_inkling_dense_int4.py --dir ~/Models/inkling_i4++mkdir -p ~/Models/inkling_i4/dense-int4g64+mv ~/Models/inkling_i4/dense-int4g64.safetensors \+ ~/Models/inkling_i4/dense-int4g64/dense.safetensors+```++The engine picks the container up automatically (it prints `[dense] container+int4-gs64: β¦`); `INK_DENSE_Q4=0` ignores it and goes back to bf16. **With enough RAM+you do not need any of this** β leave the container out and nothing changes.++Quantization error, measured against the real bf16 weights:++| Tensors | Format | Rel. L2 error |+|---|---|---|+| attention, shared experts, dense MLP | int4-gs64 | ~11% |+| `embed_tokens`, `lm_head` | int8 per-row | ~0.9% |+| norms, biases, router, conv1d | untouched | 0 |++11% is what 4 bits costs on this weight distribution (quantization noise β 0.135Ο for+16 levels over Β±amax in groups of 64) β not a defect of the conversion. It is enough+for coherent output: the 975B answers correctly on a 25 GB host. `embed`/`lm_head` are+kept at int8 because they enter *every* token. `ATTN_BITS=8` moves attention to int8+too (error 11% β 1.1%, +4 GB) if you would rather spend the RAM.++**Speed, honestly:** with only ~8 GB left for the expert cache after the dense set,+residency is ~1.7% of the 464 GB expert bank, so decode is disk-bound β tens of+seconds per token on a single NVMe, not interactive. This path is what makes the model+*runnable and testable* on a small host, not fast. Pinning (`PIN`) and a larger cache+help in proportion to the RAM you can give them.+## Modes| Invocation | What it does ||---|---|| `-p "text" [-n N]` | streaming greedy generation (stops at eos or N tokens) |+| `--chat -p "text"` | wraps the prompt in Inkling's chat template (role tokens + `<|content_text|>`, `<|message_model|>` as the generation prompt). Instruct models fed raw text are out of distribution and answer badly. `THINK=<0..1>` raises the reasoning effort (default 0) || `-f prompts.txt [-n N]` | one prompt per line (`#` comments skipped), single model load, state reset between prompts β the cache-warming workflow below || `[cap] [bits] [ref.json]` | token-exact oracle harness against a `tools/make_tiny_inkling.py` fixture (CI-style validation) |+`coli chat` / `coli serve` / `coli web` render the same template through the+gateway, so there is nothing to pass there.++## Expert cache size++`cap` (first positional argument, or `--cap` through `coli`) is how many experts each+layer keeps in RAM. Each slot is ~28 MB on the 975B, so `cap Γ n_layers Γ 28 MB` has to+fit next to the resident dense set β with the int4 dense container on a 25 GB host that+means `cap` around 2.++Small values are **correct** but slow: the engine processes the routed experts in rounds+of `cap` and accumulates, so an expert evicted mid-token is re-read rather than silently+replaced by the wrong one. (It used to be silently replaced: acquiring all `SΓtopk`+slots up front meant a full cache evicted slots that were already handed out for the+current token, and the model computed with whatever landed in them. A prefill of 18+tokens at `topk=6` needs up to 108 distinct experts per layer, so anything below that+produced incoherent output with no error message.)+## Cache warming (same idea as glm's `.coli_usage`)Expert selections are counted per `(layer, expert)` and written to