Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(inkling): expert cache served the wrong experts; int4 dense container; --chat

JustVugg5f75d6789761
diff --git a/README.md b/README.md
index d820a6ea..713a8fe8 100644
--- a/README.md
+++ b/README.md
@@ -325,6 +325,27 @@ the full 756 GB on disk at once:
./coli convert --model /nvme/glm52_i4 # download+convert shard by shard (python, one-time)
```
+#### Other supported models
+
+GLM-5.2 is the reference model, but the same streaming approach runs three more
+families. Each is a **sibling engine** β€” one C file, its own architecture, the same
+`coli chat` / `coli serve` / `coli web` front end (the launcher picks the binary from
+the model's `config.json`):
+
+| Family | Total / active | Weights | Build | Docs |
+|---|---|---|---|---|
+| **GLM-5.2** | 744B / 40B | [`mastouri/…-int4-g64-with-int8-mtp`](https://huggingface.co/mastouri/GLM-5.2-colibri-int4-g64-with-int8-mtp) (372 GB) | `make -C c glm` | this page |
+| **Inkling** (Thinking Machines) | 975B / 41B | [`nbeerbower/Inkling-colibri-int4`](https://huggingface.co/nbeerbower/Inkling-colibri-int4) (469 GB) | `make -C c inkling` | [inkling.md](docs/inkling.md) |
+| **Kimi K3** (Moonshot) | 2.8T / 104B | [`moonshotai/Kimi-K3`](https://huggingface.co/moonshotai/Kimi-K3) β€” original checkpoint, routed experts stay **native MXFP4** | `make -C c kimi_k3` | [kimi_k3.md](docs/kimi_k3.md) |
+| **OLMoE** (AI2) | 7B / 1B | converted with `c/tools/convert_olmoe_merged.py` | `make -C c olmoe` | β€” |
+
+Kimi K3 needs no conversion: its QAT-trained MXFP4 experts are streamed straight from
+the original Hugging Face shards, and the bf16 dense set is quantized at load time.
+Inkling ships int4 experts but **bf16 dense weights** (49.4 GB resident); on a host
+that cannot hold those, [inkling.md](docs/inkling.md) has a one-pass tool that brings
+the dense set to 15.3 GB and lets the 975B run on a 25 GB box β€” with the honest
+trade-off written down.
+
### 3. Run it
```bash
diff --git a/c/inkling.c b/c/inkling.c
index 390c217c..408aec73 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -68,7 +68,14 @@ typedef struct {
* move to VRAM (dev set, host freed): decode reads ~35 GB of residents per
* token, so this trades the DDR5 bandwidth wall for VRAM bandwidth AND
* frees the same RAM for the expert cache. */
-typedef struct { float *f; uint16_t *h; void *dev; } Wt;
+/* q4/qs: densa pre-quantizzata int4 group-scaled (gs=64) da un container
+ * separato, opzionale β€” vedi load_w. La densa bf16 di Inkling e' 49.4 GB e non
+ * entra in 25 GB (il caricamento la espande pure a f32, ~99 GB al picco: e' li'
+ * l'OOM). A int4-gs64 diventa ~15 GB. Nessun campo q4 = comportamento invariato. */
+typedef struct { float *f; uint16_t *h; void *dev;
+ uint8_t *q4; /* int4 nibble-packed (qbits=4) o int8 (qbits=8) */
+ float *qs; /* scale: [rows*ng] se qbits=4, [rows] se qbits=8 */
+ int gs, qbits; int64_t qn; } Wt; /* qn = byte in q4, per il guard OOB */
typedef struct {
float *in_ln, *post_ln;
@@ -100,6 +107,9 @@ typedef struct { Slot *slots; int n, cap; } LCache;
typedef struct {
Cfg c;
shards S;
+ shards Sq; /* container densa int4-gs64 (opzionale) */
+ int has_q, q_loaded; /* has_q: container presente; q_loaded: tensori presi da li */
+ int64_t q_bytes;
int quant_bits; /* 0 = f32 experts (oracle mode) */
int xq; /* experts on disk are a colibri container (U8 + .qs) */
Wt embed, lm_head;
@@ -196,6 +206,50 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,
}
/* dispatch on where the weight lives */
+/* y[S,O] = x[S,I] @ W^T con W int4 GROUP-scaled: nibble +8, low = colonna pari,
+ * una scala f32 ogni `gs` elementi lungo I (ng = ceil(I/gs) scale per riga).
+ * Differenza da matmul_q4 (per-riga): la scala cambia DENTRO la riga, quindi
+ * l'accumulo va chiuso a ogni gruppo invece che una volta sola a fine riga. */
+static void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *scale,
+ int S, int I, int O, int gs) {
+ int ng = (I + gs - 1) / gs;
+ int64_t rb = (I + 1) / 2;
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const uint8_t *w = p + (int64_t)o * rb;
+ const float *sc = scale + (int64_t)o * ng;
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s * I;
+ float acc = 0.f;
+ for (int g = 0; g < ng; g++) {
+ int i0 = g * gs, i1 = i0 + gs; if (i1 > I) i1 = I;
+ float part = 0.f;
+ for (int i = i0; i < i1; i++) {
+ uint8_t b = w[i >> 1];
+ int q = (i & 1) ? (b >> 4) : (b & 0x0F);
+ part += xs[i] * (float)(q - 8);
+ }
+ acc += part * sc[g]; /* scala chiusa per gruppo */
+ }
+ y[(int64_t)s * O + o] = acc;
+ }
+ }
+}
+/* int8 per-riga (embed/lm_head: sensibili, non vanno a 4 bit) */
+static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *scale,
+ int S, int I, int O) {
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const int8_t *w = q + (int64_t)o * I; float sc = scale[o];
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s * I;
+ float acc = 0.f;
+ for (int i = 0; i < I; i++) acc += xs[i] * (float)w[i];
+ y[(int64_t)s * O + o] = acc * sc;
+ }
+ }
+}
+
static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {
#ifdef COLI_CUDA
if (W.dev) {
@@ -203,6 +257,18 @@ static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {
fprintf(stderr, "cuda matmul failed and host copy was freed\n"); exit(1);
}
#endif
+ if (W.q4) {
+ /* guard: il container e' un file, non un invariante β€” se la geometria non
+ * torna si esce invece di leggere fuori dal buffer. */
+ int64_t need = W.qbits == 8 ? (int64_t)O * I : (int64_t)O * ((I + 1) / 2);
+ if (need > W.qn) {
+ fprintf(stderr, "dense q4: geometria incoerente (serve %lld B, ho %lld) I=%d O=%d\n",
+ (long long)need, (long long)W.qn, I, O); exit(1);
+ }
+ if (W.qbits == 8) matmul_i8r(y, x, (const int8_t*)W.q4, W.qs, S, I, O);
+ else matmul_i4g(y, x, W.q4, W.qs, S, I, O, W.gs);
+ return;
+ }
if (W.f) matmul(y, x, W.f, S, I, O);
else matmul_h(y, x, W.h, S, I, O);
}
@@ -478,10 +544,46 @@ static void pread_all(int fd, void *buf, int64_t nb, int64_t off) {
* checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact).
* gpu_ok: bf16 tensors move to VRAM while budget lasts (embed stays host β€”
* it's a row lookup, not a matmul). */
+/* Container densa pre-quantizzata (opzionale): <snap>/dense-int4g64/.
+ * Se il tensore c'e' li' dentro lo carichiamo int4-gs64 (U8 + sidecar .qs, la
+ * stessa convenzione che il container usa gia' per gli esperti); altrimenti si
+ * prosegue col percorso bf16/f32 di sempre. Container assente = nessun cambio. */
+static int load_w_quant(Model *m, const char *name, int64_t orig_numel, Wt *out) {
+ if (!m->has_q) return 0;
+ st_tensor *t = st_find(&m->Sq, name);
+ if (!t || t->dtype != 3) return 0; /* 3 = U8/byte grezzi */
+ char qn[352]; snprintf(qn, sizeof(qn), "%s.qs", name);
+ st_tensor *s = st_find(&m->Sq, qn);
+ if (!s) return 0; /* senza scale non si decodifica */
+ /* Geometria dedotta dai CONTEGGI, non dalle shape (st_tensor non le porta):
+ * int4-gs64 -> byte ~= numel/2 e scale ~= numel/64
+ * int8 -> byte == numel e scale == righe (numel/I, I ignoto qui:
+ * basta che le scale siano molte meno dei byte)
+ * Il controllo forte sull'OOB lo fa matmul_w con l'I vero del chiamante. */
+ Wt w = {0};
+ if (t->nbytes == orig_numel && s->numel * 64 < (int64_t)t->nbytes) {
+ w.qbits = 8; /* int8 per riga */
+ } else if (t->nbytes * 2 >= orig_numel && t->nbytes * 2 <= orig_numel + 2 * s->numel) {
+ w.qbits = 4; w.gs = 64; /* int4 group-scaled */
+ } else {
+ fprintf(stderr, "[dense] %s: geometria non riconosciuta nel container, uso il bf16\n", name);
+ return 0; /* dubbio -> percorso originale */
+ }
+ w.q4 = malloc(t->nbytes); if (!w.q4) { fprintf(stderr, "OOM %s\n", name); exit(1); }
+ pread_all(t->fd, w.q4, t->nbytes, t->off);
+ w.qn = t->nbytes;
+ w.qs = falloc(s->numel);
+ st_read_f32(&m->Sq, qn, w.qs, 0);
+ *out = w;
+ m->q_loaded++; m->q_bytes += t->nbytes + (int64_t)s->numel * 4;
+ return 1;
+}
+
static Wt load_w(Model *m, const char *name, int gpu_ok) {
Wt w = {0};
st_tensor *t = st_find(&m->S, name);
if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }
+ if (load_w_quant(m, name, t->numel, &w)) return w;
if (t->dtype == 0) {
w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); }
pread_all(t->fd, w.h, t->nbytes, t->off);
@@ -500,12 +602,40 @@ static Wt load_w(Model *m, const char *name, int gpu_ok) {
}
return w;
}
-static Wt wt_off(Wt w, int64_t off) {
- Wt r = { w.f ? w.f + off : NULL, w.h ? w.h + off : NULL,
- w.dev ? (char*)w.dev + off*2 : NULL }; /* dev is always bf16 */
+/* `off` conta ELEMENTI ed e' sempre un multiplo di I (si affetta per riga: un
+ * esperto condiviso dal tensore fuso [E,R,I]), quindi I arriva dal chiamante β€”
+ * st_tensor non porta le shape e non voglio indovinarle. */
+static Wt wt_off_i(Wt w, int64_t off, int I) {
+ Wt r = w;
+ r.f = w.f ? w.f + off : NULL;
+ r.h = w.h ? w.h + off : NULL;
+ r.dev = w.dev ? (char*)w.dev + off*2 : NULL; /* dev is always bf16 */
+ if (w.q4) {
+ int64_t row = off / I;
+ if (w.qbits == 8) { r.q4 = w.q4 + off; r.qs = w.qs + row; r.qn = w.qn - off; }
+ else { r.q4 = w.q4 + off / 2; r.qs = w.qs + row * ((I + w.gs - 1) / w.gs);
+ r.qn = w.qn - off / 2; }
+ }
return r;
}
+/* dequantizza UNA riga (int4-gs64: nibble +8, low = colonna pari, scala ogni gs) */
+static void wt_deq_row(Wt w, int64_t row, float *out, int I) {
+ if (w.qbits == 8) {
+ const int8_t *q = (const int8_t*)w.q4 + row * I; float s = w.qs[row];
+ for (int i = 0; i < I; i++) out[i] = (float)q[i] * s;
+ return;
+ }
+ int ng = (I + w.gs - 1) / w.gs;
+ const uint8_t *p = w.q4 + row * ((I + 1) / 2);
+ const float *sc = w.qs + row * ng;
+ for (int i = 0; i < I; i++) {
+ uint8_t b = p[i >> 1];
+ int q = (i & 1) ? (b >> 4) : (b & 0x0F);
+ out[i] = (float)(q - 8) * sc[i / w.gs];
+ }
+}
static void wt_row_f32(Wt w, int64_t off, float *out, int n) {
+ if (w.q4) { wt_deq_row(w, off / n, out, n); return; } /* n = dim di contrazione */
if (w.f) memcpy(out, w.f + off, n * sizeof(float));
else for (int i = 0; i < n; i++) { union { uint32_t u; float f; } v = { (uint32_t)w.h[off + i] << 16 }; out[i] = v.f; }
}
@@ -557,6 +687,18 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {
m->quant_bits = bits;
load_cfg(&m->c, snap);
st_init(&m->S, snap);
+ /* densa pre-quantizzata, se il container c'e' (INK_DENSE_Q4=0 la ignora).
+ * Sta in una SOTTOCARTELLA apposta: nella dir dello snapshot i nomi tensore
+ * andrebbero in collisione con i bf16 originali, che restano intatti. */
+ { char qd[2100]; snprintf(qd, sizeof(qd), "%s/dense-int4g64", snap);
+ const char *off = getenv("INK_DENSE_Q4");
+ struct stat qs;
+ if (!(off && *off == '0') && stat(qd, &qs) == 0 && S_ISDIR(qs.st_mode)) {
+ st_init(&m->Sq, qd);
+ if (m->Sq.n > 0) { m->has_q = 1;
+ fprintf(stderr, "[dense] container int4-gs64: %s (%d tensori)\n", qd, m->Sq.n); }
+ }
+ }
Cfg *c = &m->c;
int D = c->hidden, K = c->conv_k;
double t0 = now_s();
@@ -945,8 +1087,30 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
for (int kk = 0; kk < K; kk++) { w[kk] = sigmoidf(lg[si[kk]]); sum += w[kk]; }
for (int j = 0; j < ns; j++) { w[K+j] = sigmoidf(lg[E+j]); sum += w[K+j]; }
for (int kk = 0; kk < K+ns; kk++) w[kk] *= c->route_scale * l->rgs / sum;
- for (int kk = 0; kk < K; kk++) {
- int eid = si[kk];
+ }
+ /* Il ciclo sotto ACQUISISCE uno slot per ogni coppia (token, esperto) e ne tiene
+ * il puntatore fino al calcolo. slot_acquire evince l'LRU quando la cache e'
+ * piena β€” anche uno slot gia' consegnato in QUESTA chiamata: il puntatore resta
+ * valido ma lo slot ora contiene un ALTRO esperto, e il modello calcola con i
+ * pesi sbagliati. In silenzio: niente crash, solo output incoerente. Acquisire
+ * tutte le S*K coppie in una volta pretendeva quindi una cache capace di tenere
+ * tutti gli esperti distinti del batch (18 token x topk 6 = fino a 108 slot per
+ * layer), e sotto quella soglia il modello sembrava rotto.
+ * Fix: si lavora a GIRI di al piu' `cap` coppie β€” acquisisci, riempi, calcola e
+ * accumula β€” cosi' nessuno slot puo' essere evinto mentre serve. L'output MoE e'
+ * una somma pesata, quindi accumulare a giri da' lo stesso risultato di una
+ * passata sola, e la cache puo' scendere fino a 1 slot per layer (piu' letture
+ * da disco, ma memoria proporzionale a `cap` invece che al batch). */
+ int cap = m->cache[layer].cap; if (cap < 1) cap = 1;
+ float *g = falloc(2*I), *u = g + I, *hh = falloc(D);
+ int q4 = m->xq && m->rb13*2 == D; /* packed int4 vs int8 container */
+ int64_t npair = (int64_t)S*K;
+ for (int64_t base = 0; base < npair; base += cap) {
+ int64_t end = base + cap < npair ? base + cap : npair;
+ nfill = 0;
+ for (int64_t t = base; t < end; t++) { /* acquisizione del giro */
+ int s = (int)(t / K), kk = (int)(t % K);
+ int eid = idx[(int64_t)s*K + kk];
if (m->eusage[layer]) m->eusage[layer][eid]++;
Slot *e = slot_find(m, layer, eid);
if (e) m->hits++;
@@ -955,27 +1119,21 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
e = slot_acquire(m, layer, eid);
fill[nfill] = e; fl[nfill] = layer; nfill++;
}
- use[(int64_t)s*K + kk] = e;
+ use[t - base] = e;
+ }
+ if (nfill) { /* riempimento in parallelo */
+ double tf = now_s();
+ #pragma omp parallel for schedule(dynamic,1)
+ for (int j = 0; j < nfill; j++) slot_fill(m, fl[j], fill[j]);
+ m->t_fill += now_s() - tf;
}
- }
- /* pass 2: one parallel burst for every miss in this layer call */
- if (nfill) {
- double tf = now_s();
- #pragma omp parallel for schedule(dynamic,1)
- for (int j = 0; j < nfill; j++) slot_fill(m, fl[j], fill[j]);
- m->t_fill += now_s() - tf;
- }
- /* pass 3: compute. gate+up run as ONE matmul over the fused 2I rows β€”
- * halves the number of (expensive to open) parallel regions per expert */
- float *g = falloc(2*I), *u = g + I, *hh = falloc(D);
- int q4 = m->xq && m->rb13*2 == D; /* packed int4 vs int8 container */
- for (int s = 0; s < S; s++) {
- const float *xs = x + (int64_t)s*D;
- float *os = out + (int64_t)s*D;
- float *w = wgt + (int64_t)s*(K+ns);
double te = now_s();
- for (int kk = 0; kk < K; kk++) {
- Slot *e = use[(int64_t)s*K + kk];
+ for (int64_t t = base; t < end; t++) { /* calcolo + accumulo */
+ int s = (int)(t / K), kk = (int)(t % K);
+ const float *xs = x + (int64_t)s*D;
+ float *os = out + (int64_t)s*D;
+ float *w = wgt + (int64_t)s*(K+ns);
+ Slot *e = use[t - base];
if (m->xq) {
if (q4) {
matmul_q4(g, xs, e->p13, e->s13, D, 2*I); /* gate rows then up rows */
@@ -997,13 +1155,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
}
for (int d = 0; d < D; d++) os[d] += w[kk] * hh[d];
}
- double ts = now_s(); m->t_expert += ts - te;
- /* shared experts: gamma inside (before down_proj is linear, so applied at the end) */
+ m->t_expert += now_s() - te;
+ }
+ /* shared experts: una volta per token, fuori dai giri (non usano la cache) */
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s*D;
+ float *os = out + (int64_t)s*D;
+ float *w = wgt + (int64_t)s*(K+ns);
+ double ts = now_s();
+ /* gamma inside (before down_proj is linear, so applied at the end) */
for (int j = 0; j < ns; j++) {
- matmul_w(g, xs, wt_off(l->sh_g, (int64_t)j*I*D), 1, D, I);
- matmul_w(u, xs, wt_off(l->sh_u, (int64_t)j*I*D), 1, D, I);
+ matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);
+ matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);
for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
- matmul_w(hh, g, wt_off(l->sh_d, (int64_t)j*D*I), 1, I, D);
+ matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);
for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];
}
m->t_shared += now_s() - ts;
@@ -1400,15 +1565,34 @@ int main(int argc, char **argv) {
if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }
/* flags: -p "prompt" [-n N] -> generate mode; positional: [cap] [bits] [ref.json] */
const char *prompt = NULL, *pfile = NULL, *refpath = "ref_inkling.json";
- int cap = -1, bits = 0, n_new = 256, npos = 0;
+ int cap = -1, bits = 0, n_new = 256, npos = 0, chat = 0;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-p") && i+1 < argc) prompt = argv[++i];
else if (!strcmp(argv[i], "-f") && i+1 < argc) pfile = argv[++i];
else if (!strcmp(argv[i], "-n") && i+1 < argc) n_new = atoi(argv[++i]);
+ else if (!strcmp(argv[i], "--chat")) chat = 1;
else if (npos == 0) { cap = atoi(argv[i]); npos++; }
else if (npos == 1) { bits = atoi(argv[i]); npos++; }
else refpath = argv[i];
}
+ /* --chat: avvolge il prompt nel template di Inkling (sottoinsieme testuale di
+ * chat_template.jinja, lo stesso che openai_server.py rende in render_chat_inkling):
+ * token di ruolo + <|content_text|>, <|end_message|> a chiudere, il livello di
+ * thinking come messaggio di sistema, e <|message_model|> come prompt di
+ * generazione. Senza template un modello instruct riceve testo fuori
+ * distribuzione. THINK=<0..1> alza lo sforzo di ragionamento (default 0). */
+ char *chat_buf = NULL;
+ if (chat && prompt) {
+ const char *eff = getenv("THINK") ? getenv("THINK") : "0";
+ size_t need = strlen(prompt) + strlen(eff) + 256;
+ chat_buf = malloc(need);
+ if (!chat_buf) { fprintf(stderr, "OOM chat template\n"); return 1; }
+ snprintf(chat_buf, need,
+ "<|message_user|><|content_text|>%s<|end_message|>"
+ "<|message_system|><|content_text|>Thinking effort level: %s<|end_message|>"
+ "<|message_model|>", prompt, eff);
+ prompt = chat_buf;
+ }
if (cap < 0) cap = (prompt || pfile) ? 0 : 16; /* generate mode defaults to RAM-sized auto cap */
if (bits && (bits < 2 || bits > 8)) { fprintf(stderr, "quant_bits must be 0 (f32) or 2..8\n"); return 1; }
diff --git a/c/tools/convert_inkling_dense_int4.py b/c/tools/convert_inkling_dense_int4.py
new file mode 100644
index 00000000..4e6a27a5
--- /dev/null
+++ b/c/tools/convert_inkling_dense_int4.py
@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""Inkling: quantizza la DENSA RESIDENTE (bf16 -> int4-gs64) in un container a parte.
+
+Serve solo se la RAM non basta. Il container int4 di Inkling
+(nbeerbower/Inkling-colibri-int4) ha gli esperti routed gia' a int4 ma tiene la
+densa in **bf16**: 49.4 GB che devono stare in RAM per ogni token. Su un host con
+meno di ~64 GB non ci stanno β€” e il caricamento li espande pure a f32 (~99 GB al
+picco), quindi il processo muore prima di generare. A int4-gs64 la densa scende a
+**15.3 GB** e il modello gira su una macchina da 25 GB.
+
+Con RAM abbondante NON serve: `inkling.c` carica la densa bf16 come sempre.
+
+Cosa fa:
+ - legge SOLO gli header degli shard (pochi MB) e costruisce un indice
+ - quantizza un tensore alla volta, a blocchi di righe (picco RAM ~1 GB)
+ - scrive UN file nuovo; gli esperti routed (464 GB) non vengono toccati
+ - gli originali sono aperti in sola lettura: se qualcosa va storto si rifa'
+ - riporta l'errore di quantizzazione L2 per classe, misurato sui pesi veri
+
+Formato di output (stessa convenzione .qs che inkling.c usa gia' per gli esperti):
+ <nome> U8 [O, ceil(I/2)] nibble packed, low = colonna pari, offset +8
+ <nome>.qs F32 [O, ceil(I/64)] una scala ogni 64 elementi lungo l'INPUT
+ __metadata__ {"<nome>": "fmt=4;gs=64"} / "fmt=1" per int8 / "fmt=raw"
+
+Politica di precisione (misurata: vedi docs/inkling.md):
+ attention / shared_experts / mlp denso -> int4-gs64 ~11% errore L2
+ embed_tokens / lm_head -> int8 per-row ~0.9% (entrano in OGNI token)
+ norm / bias / A_log / conv1d / router -> passthrough (minuscoli e delicati)
+ATTN_BITS=8 porta anche l'attention a int8 (+4 GB, errore 11% -> 1.1%): non serve
+per la coerenza dell'output, ma e' li' se vuoi spendere RAM per la qualita'.
+
+Uso:
+ python3 convert_inkling_dense_int4.py --dir /path/al/modello --plan # stima
+ python3 convert_inkling_dense_int4.py --dir /path/al/modello # converte
+ mkdir -p /path/al/modello/dense-int4g64
+ mv /path/al/modello/dense-int4g64.safetensors \\
+ /path/al/modello/dense-int4g64/dense.safetensors
+Il motore trova il container da solo (INK_DENSE_Q4=0 lo ignora).
+"""
+import argparse, json, os, re, struct, sys, time
+import numpy as np
+
+DIR = os.environ.get("INKLING_DIR", ".")
+OUT = os.path.join(DIR, "dense-int4g64.safetensors")
+GS = 64
+ROWS_PER_CHUNK = 4096 # blocchi di righe: tiene il picco RAM ~<1 GB
+
+# ---------- classificazione (per nome) ----------
+RE_EXPERT = re.compile(r"\.experts\.") # routed: NON toccare (shared_experts escluso sotto)
+RE_SHARED = re.compile(r"shared_experts")
+RE_ATTN = re.compile(r"self_attn")
+RE_MLPD = re.compile(r"\.mlp\.(gate|up|down)_proj")
+RE_EMB = re.compile(r"(embed_tokens|lm_head)")
+RE_SMALL = re.compile(r"(norm|bias|A_log|dt_bias|conv1d|\.gate\.|rotary|scale$)", re.I)
+
+MIN_ELEMS = 1 << 20 # sotto ~1M elementi non vale il rischio: passthrough
+ATTN_BITS = int(os.environ.get("ATTN_BITS", "8")) # attention: 8 = int8 (default), 4 = int4-gs64
+
+def classify(name, shape, dtype):
+ """-> 'skip' (expert routed) | 'gs4' | 'int8' | 'raw'
+
+ Regola GENERALE, non una whitelist di nomi: qualunque peso float grande e
+ 2-D/3-D con l'ultima dimensione (quella di contrazione) >= GS va a int4-gs64.
+ Una whitelist mi aveva gia' fatto perdere shared_experts (3-D [2,3072,6144])
+ e mtp.*.input_proj, 14.5 GB finiti in passthrough senza accorgersene."""
+ if RE_EXPERT.search(name) and not RE_SHARED.search(name):
+ return "skip" # esperti routed: restano negli shard
+ if dtype not in ("BF16", "F16", "F32"):
+ return "raw" # gia' quantizzato o non-float
+ if RE_EMB.search(name):
+ return "int8" # embed/lm_head: sensibili, ogni token
+ if RE_SMALL.search(name):
+ return "raw" # norm/bias/A_log/conv1d/router: delicati
+ # ATTN_BITS=8 (default): l'attention a 4 bit rende il modello incoerente β€”
+ # misurato: 11% di errore per matmul x ~5 matmul x 66 layer e l'output degenera
+ # (il container e il kernel sono verificati corretti, e' la precisione a mancare).
+ # ATTN_BITS=4 la rimette a int4-gs64 per confronto.
+ if RE_ATTN.search(name) and ATTN_BITS == 8:
+ return "int8"
+ n = 1
+ for d in shape:
+ n *= d
+ if len(shape) < 2 or shape[-1] < GS or n < MIN_ELEMS:
+ return "raw" # 1-D, contrazione corta o troppo piccolo
+ return "gs4"
+
+# ---------- I/O safetensors ----------
+def read_header(path):
+ with open(path, "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ hdr = json.loads(f.read(n))
+ return hdr, 8 + n
+
+def read_rows(path, base, off0, dtype, cols, r0, r1):
+ """Legge le righe [r0,r1) di un tensore 2-D e le restituisce in f32."""
+ itemsize = {"BF16": 2, "F16": 2, "F32": 4, "U8": 1, "I8": 1}[dtype]
+ start = base + off0 + r0 * cols * itemsize
+ nbytes = (r1 - r0) * cols * itemsize
+ with open(path, "rb") as f:
+ f.seek(start)
+ raw = f.read(nbytes)
+ if dtype == "BF16": # bf16 -> f32: shift a sinistra di 16 bit
+ u = np.frombuffer(raw, np.uint16).astype(np.uint32) << 16
+ return u.view(np.float32).reshape(r1 - r0, cols)
+ if dtype == "F16":
+ return np.frombuffer(raw, np.float16).astype(np.float32).reshape(r1 - r0, cols)
+ if dtype == "F32":
+ return np.frombuffer(raw, np.float32).reshape(r1 - r0, cols)
+ raise ValueError(f"dtype non gestito per la quantizzazione: {dtype}")
+
+# ---------- quantizzatori ----------
+def quant_gs4(w):
+ """w [R, I] f32 -> (packed U8 [R, ceil(I/2)], scales f32 [R, ng]).
+ Gruppi lungo I (dimensione di contrazione), simmetrico, nibble +8."""
+ R, I = w.shape
+ ng = (I + GS - 1) // GS
+ pad = ng * GS - I
+ wp = np.concatenate([w, np.zeros((R, pad), np.float32)], 1) if pad else w
+ g = wp.reshape(R, ng, GS)
+ amax = np.abs(g).max(-1, keepdims=True)
+ scale = (amax / 7.0).astype(np.float32)
+ scale[scale == 0] = 1.0
+ q = np.clip(np.rint(g / scale), -8, 7).astype(np.int8) + 8 # 0..15
+ q = q.reshape(R, ng * GS)[:, :I].astype(np.uint8)
+ if I % 2: # coda dispari
+ q = np.concatenate([q, np.full((R, 1), 8, np.uint8)], 1)
+ packed = (q[:, 0::2] | (q[:, 1::2] << 4)).astype(np.uint8)
+ return packed, scale.reshape(R, ng).astype(np.float32)
+
+def dequant_gs4(packed, scale, I):
+ R = packed.shape[0]
+ q = np.empty((R, packed.shape[1] * 2), np.float32)
+ q[:, 0::2] = (packed & 15).astype(np.float32) - 8.0
+ q[:, 1::2] = (packed >> 4).astype(np.float32) - 8.0
+ q = q[:, :I]
+ ng = scale.shape[1]
+ s = np.repeat(scale, GS, axis=1)[:, :I]
+ return q * s
+
+def quant_i8(w):
+ amax = np.abs(w).max(1, keepdims=True)
+ scale = (amax / 127.0).astype(np.float32)
+ scale[scale == 0] = 1.0
+ q = np.clip(np.rint(w / scale), -127, 127).astype(np.int8)
+ return q, scale[:, 0].astype(np.float32)
+
+def dequant_i8(q, scale):
+ return q.astype(np.float32) * scale[:, None]
+
+# ---------- main ----------
+def main():
+ global DIR
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--plan", action="store_true", help="stima e piano, non scrive nulla")
+ ap.add_argument("--dir", default=DIR, help="dir del modello")
+ ap.add_argument("--out", default=OUT)
+ a = ap.parse_args()
+
+ DIR = a.dir
+ if a.out == OUT: a.out = os.path.join(DIR, "dense-int4g64.safetensors")
+ shards = sorted(f for f in os.listdir(DIR) if f.endswith(".safetensors") and not f.startswith("dense-"))
+ print(f"[1/3] indicizzo {len(shards)} shard (solo header)…", flush=True)
+ index = {} # name -> (path, base, off0, off1, dtype, shape)
+ for sh in shards:
+ p = os.path.join(DIR, sh)
+ hdr, base = read_header(p)
+ for k, t in hdr.items():
+ if k == "__metadata__":
+ continue
+ o = t["data_offsets"]
+ index[k] = (p, base, o[0], o[1], t["dtype"], t["shape"])
+
+ def flat2d(shape):
+ """[E,R,I] o [R,I] -> (righe, I): i gruppi vanno sull'ULTIMA dim (contrazione)."""
+ I = shape[-1]
+ R = 1
+ for d in shape[:-1]:
+ R *= d
+ return R, I
+
+ plan, tot_in, tot_out = [], 0, 0
+ for name, (p, base, o0, o1, dt, shape) in sorted(index.items()):
+ kind = classify(name, shape, dt)
+ if kind == "skip":
+ continue
+ nbytes_in = o1 - o0
+ tot_in += nbytes_in
+ if kind == "gs4":
+ R, I = flat2d(shape)
+ ng = (I + GS - 1) // GS
+ out_b = R * ((I + 1) // 2) + R * ng * 4
+ elif kind == "int8":
+ R, I = flat2d(shape)
+ out_b = R * I + R * 4
+ else:
+ out_b = nbytes_in
+ tot_out += out_b
+ plan.append((name, kind, p, base, o0, o1, dt, tuple(shape), out_b))
+
+ by = {}
+ for name, kind, *_rest in plan:
+ by[kind] = by.get(kind, 0) + 1
+ print(f" densa da convertire: {len(plan)} tensori {by}")
+ print(f" input bf16/f32 : {tot_in/1e9:7.2f} GB")
+ print(f" output stimato : {tot_out/1e9:7.2f} GB ({100*tot_out/max(tot_in,1):.0f}%)")
+ free = os.statvfs(DIR).f_bavail * os.statvfs(DIR).f_frsize
+ print(f" spazio libero : {free/1e9:7.2f} GB")
+ if a.plan:
+ print("\n[--plan] nessuna scrittura. Rilancia senza --plan per convertire.")
+ return
+ if free < tot_out * 1.15:
+ sys.exit(f"[ERR] spazio insufficiente: servono ~{tot_out*1.15/1e9:.0f} GB")
+
+ # ---- header di output: shape/offset noti in anticipo ----
+ print(f"[2/3] costruisco l'header di output…", flush=True)
+ hdr, off, meta = {}, 0, {}
+ for name, kind, p, base, o0, o1, dt, shape, _ob in plan:
+ if kind == "gs4":
+ R, I = flat2d(shape)
+ ng = (I + GS - 1) // GS
+ nb = R * ((I + 1) // 2)
+ hdr[name] = {"dtype": "U8", "shape": [R, (I + 1) // 2], "data_offsets": [off, off + nb]}; off += nb
+ nb = R * ng * 4
+ hdr[name + ".qs"] = {"dtype": "F32", "shape": [R, ng], "data_offsets": [off, off + nb]}; off += nb
+ # orig= conserva la shape logica (3-D per gli shared_experts): il loader
+ # ricostruisce [E,R,I] dai byte piatti senza doverla indovinare.
+ meta[name] = f"fmt=4;gs={GS};I={I};orig={'x'.join(str(d) for d in shape)}"
+ elif kind == "int8":
+ R, I = flat2d(shape)
+ nb = R * I
+ hdr[name] = {"dtype": "I8", "shape": [R, I], "data_offsets": [off, off + nb]}; off += nb
+ nb = R * 4
+ hdr[name + ".qs"] = {"dtype": "F32", "shape": [R], "data_offsets": [off, off + nb]}; off += nb
+ meta[name] = f"fmt=1;orig={'x'.join(str(d) for d in shape)}"
+ else:
+ nb = o1 - o0
+ hdr[name] = {"dtype": dt, "shape": list(shape), "data_offsets": [off, off + nb]}; off += nb
+ meta[name] = "fmt=raw"
+ meta["colibri.dense"] = "inkling-int4g64-v1"
+ hdr["__metadata__"] = meta
+ hjson = json.dumps(hdr, separators=(",", ":")).encode()
+ pad = (8 - (len(hjson) % 8)) % 8 # header allineato a 8 byte
+ hjson += b" " * pad
+
+ tmp = a.out + ".part"
+ print(f"[3/3] scrivo {a.out} ({off/1e9:.2f} GB payload)…", flush=True)
+ err_by_kind = {}
+ t0 = time.time()
+ with open(tmp, "wb") as w:
+ w.write(struct.pack("<Q", len(hjson))); w.write(hjson)
+ done = 0
+ for name, kind, p, base, o0, o1, dt, shape, _ob in plan:
+ if kind == "raw":
+ with open(p, "rb") as f:
+ f.seek(base + o0)
+ left = o1 - o0
+ while left > 0:
+ chunk = f.read(min(left, 8 << 20)); w.write(chunk); left -= len(chunk)
+ else:
+ R, I = flat2d(shape)
+ packs, scales, errs = [], [], []
+ for r0 in range(0, R, ROWS_PER_CHUNK):
+ r1 = min(r0 + ROWS_PER_CHUNK, R)
+ blk = read_rows(p, base, o0, dt, I, r0, r1)
+ if kind == "gs4":
+ pk, sc = quant_gs4(blk); rec = dequant_gs4(pk, sc, I)
+ else:
+ pk, sc = quant_i8(blk); rec = dequant_i8(pk, sc)
+ n = float(np.linalg.norm(blk))
+ if n > 0: errs.append(float(np.linalg.norm(blk - rec)) / n)
+ packs.append(pk); scales.append(sc)
+ w.write(np.concatenate(packs, 0).tobytes())
+ w.write(np.concatenate(scales, 0).astype(np.float32).tobytes())
+ if errs:
+ k = "attention" if RE_ATTN.search(name) else \
+ "shared" if RE_SHARED.search(name) else \
+ "embed/head"if RE_EMB.search(name) else "mlp denso"
+ err_by_kind.setdefault(k, []).append(float(np.mean(errs)))
+ done += 1
+ if done % 100 == 0:
+ print(f" {done}/{len(plan)} ({time.time()-t0:.0f}s)", flush=True)
+ os.replace(tmp, a.out)
+ sz = os.path.getsize(a.out)
+ print(f"\nβœ… scritto {a.out} {sz/1e9:.2f} GB in {time.time()-t0:.0f}s")
+ print(f"\n=== errore di quantizzazione (L2 relativo, sui pesi VERI) ===")
+ for k, v in sorted(err_by_kind.items()):
+ print(f" {k:12} {100*float(np.mean(v)):.3f}% (su {len(v)} tensori)")
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/inkling.md b/docs/inkling.md
index f86028a6..8b211544 100644
--- a/docs/inkling.md
+++ b/docs/inkling.md
@@ -33,14 +33,77 @@ Requirements: ~120 GB RAM (CPU build keeps ~86 GB of bf16 residents in RAM;
the CUDA build moves them to VRAM and uses the freed RAM for a larger expert
cache), NVMe storage for the snapshot.
+## If you have less RAM than that
+
+The pre-converted container has **int4 routed experts but bf16 dense weights**, and
+the dense set is resident: every token needs all of it. Measured on the real
+snapshot it is **49.4 GB**, and `load_w` expands bf16 to f32 while loading, so the
+peak is ~99 GB. Below roughly 64 GB of RAM the process dies before it generates
+anything.
+
+Quantizing just the dense set to **int4-gs64** brings it to **15.3 GB**, which fits a
+25 GB box. One pass over the shards, originals untouched, ~14 minutes:
+
+```sh
+python3 c/tools/convert_inkling_dense_int4.py --dir ~/Models/inkling_i4 --plan # estimate first
+python3 c/tools/convert_inkling_dense_int4.py --dir ~/Models/inkling_i4
+
+mkdir -p ~/Models/inkling_i4/dense-int4g64
+mv ~/Models/inkling_i4/dense-int4g64.safetensors \
+ ~/Models/inkling_i4/dense-int4g64/dense.safetensors
+```
+
+The engine picks the container up automatically (it prints `[dense] container
+int4-gs64: …`); `INK_DENSE_Q4=0` ignores it and goes back to bf16. **With enough RAM
+you do not need any of this** β€” leave the container out and nothing changes.
+
+Quantization error, measured against the real bf16 weights:
+
+| Tensors | Format | Rel. L2 error |
+|---|---|---|
+| attention, shared experts, dense MLP | int4-gs64 | ~11% |
+| `embed_tokens`, `lm_head` | int8 per-row | ~0.9% |
+| norms, biases, router, conv1d | untouched | 0 |
+
+11% is what 4 bits costs on this weight distribution (quantization noise β‰ˆ 0.135Οƒ for
+16 levels over Β±amax in groups of 64) β€” not a defect of the conversion. It is enough
+for coherent output: the 975B answers correctly on a 25 GB host. `embed`/`lm_head` are
+kept at int8 because they enter *every* token. `ATTN_BITS=8` moves attention to int8
+too (error 11% β†’ 1.1%, +4 GB) if you would rather spend the RAM.
+
+**Speed, honestly:** with only ~8 GB left for the expert cache after the dense set,
+residency is ~1.7% of the 464 GB expert bank, so decode is disk-bound β€” tens of
+seconds per token on a single NVMe, not interactive. This path is what makes the model
+*runnable and testable* on a small host, not fast. Pinning (`PIN`) and a larger cache
+help in proportion to the RAM you can give them.
+
## Modes
| Invocation | What it does |
|---|---|
| `-p "text" [-n N]` | streaming greedy generation (stops at eos or N tokens) |
+| `--chat -p "text"` | wraps the prompt in Inkling's chat template (role tokens + `<|content_text|>`, `<|message_model|>` as the generation prompt). Instruct models fed raw text are out of distribution and answer badly. `THINK=<0..1>` raises the reasoning effort (default 0) |
| `-f prompts.txt [-n N]` | one prompt per line (`#` comments skipped), single model load, state reset between prompts β€” the cache-warming workflow below |
| `[cap] [bits] [ref.json]` | token-exact oracle harness against a `tools/make_tiny_inkling.py` fixture (CI-style validation) |
+`coli chat` / `coli serve` / `coli web` render the same template through the
+gateway, so there is nothing to pass there.
+
+## Expert cache size
+
+`cap` (first positional argument, or `--cap` through `coli`) is how many experts each
+layer keeps in RAM. Each slot is ~28 MB on the 975B, so `cap Γ— n_layers Γ— 28 MB` has to
+fit next to the resident dense set β€” with the int4 dense container on a 25 GB host that
+means `cap` around 2.
+
+Small values are **correct** but slow: the engine processes the routed experts in rounds
+of `cap` and accumulates, so an expert evicted mid-token is re-read rather than silently
+replaced by the wrong one. (It used to be silently replaced: acquiring all `SΓ—topk`
+slots up front meant a full cache evicted slots that were already handed out for the
+current token, and the model computed with whatever landed in them. A prefill of 18
+tokens at `topk=6` needs up to 108 distinct experts per layer, so anything below that
+produced incoherent output with no error message.)
+
## Cache warming (same idea as glm's `.coli_usage`)
Expert selections are counted per `(layer, expert)` and written to