Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
inkling: resident dense matrices on Vulkan (COLI_VULKAN=1)
JustVugg3ca0ae97ccad
diff --git a/c/inkling.c b/c/inkling.cindex cfb51933..97e8252d 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -68,6 +68,14 @@ static int g_cuda = 0;#include "backend_metal.h"static int g_metal = 0;#endif+#ifdef COLI_VULKAN+/* Vulkan (opt-in, VK=1 build + COLI_VULKAN=1): the RESIDENT dense matrices that+ * the dense-int4g64 container holds in a format the qmatmul shader reads as it is+ * (int8 per row -> fmt 1, int4-g64 -> fmt 4), uploaded on first use. Routed+ * experts, which arrive from disk, and bf16/f32 residents stay on the CPU. */+#include "backend_vulkan.h"+static int g_vk_ready = 0;+#endif#if defined(__APPLE__)#include <mach/mach.h>#include <sys/sysctl.h>@@ -111,7 +119,12 @@ typedef struct {typedef struct { float *f; uint16_t *h; void *dev;uint8_t *q4; /* int4 nibble-packed (qbits=4) o int8 (qbits=8) */float *qs; /* scale: [rows*ng] se qbits=4, [rows] se qbits=8 */- int gs, qbits; int64_t qn; } Wt; /* qn = byte in q4, per il guard OOB */+ int gs, qbits; int64_t qn; /* qn = byte in q4, per il guard OOB */+#ifdef COLI_VULKAN+ void *vk; /* InkVk of the loaded tensor, shared by its wt_off_i+ * views (NULL = CPU only) */+#endif+ } Wt;typedef struct {float *in_ln, *post_ln;@@ -401,6 +414,61 @@ static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *s}}+#ifdef COLI_VULKAN+/* Device copies of one loaded tensor. A Wt travels by value and the shared+ * experts are read through wt_off_i views of one fused [ns][R,I] tensor, so the+ * cache cannot live in the copy matmul_w receives: it lives in a table the+ * loaded Wt points to and every view copies, one entry per view (keyed by the+ * view's first byte). Released with the tensor it belongs to. */+typedef struct { const uint8_t *q; ColiVkTensor *t; int dead; } InkVkView;+typedef struct { int n; InkVkView e[]; } InkVk;++/* Attach a table to a loaded tensor the shader reads exactly as the CPU kernel+ * does: int8 per row (matmul_i8r) is fmt 1, int4 group-scaled with +8 nibbles+ * and the low nibble on the even column (matmul_i4g) is fmt 4. */+static int ink_vk_attach(Wt *w, int views) {+ if (!w->q4 || w->vk || views < 1) return 0;+ if (w->qbits != 8 && !(w->qbits == 4 && w->gs >= 8 && w->gs % 8 == 0)) return 0;+ InkVk *v = calloc(1, sizeof(*v) + (size_t)views * sizeof(v->e[0]));+ if (!v) return 0;+ v->n = views;+ w->vk = v;+ return 1;+}+static void ink_vk_release(Wt *w) {+ InkVk *v = w->vk;+ if (!v) return;+ for (int k = 0; k < v->n; k++) if (v->e[k].t) coli_vk_tensor_free(v->e[k].t);+ free(v);+ w->vk = NULL;+}+/* 1 when y was computed on the device. One command buffer in the backend:+ * main thread only, never from inside a parallel region. */+static int ink_vk_matmul(float *y, const float *x, const Wt *W, int S, int I, int O) {+ InkVk *v = W->vk;+ if (!g_vk_ready || !v) return 0;+#ifdef _OPENMP+ if (omp_in_parallel()) return 0;+#endif+ for (int k = 0; k < v->n; k++) {+ InkVkView *e = &v->e[k];+ if (e->q && e->q != W->q4) continue;+ if (e->dead) return 0;+ e->q = W->q4;+ if (coli_vk_matmul(&e->t, y, x, W->q4, W->qs, W->qbits == 8 ? 1 : 4,+ S, I, O, W->gs))+ return 1;+ if (!e->t) e->dead = 1; /* refused at upload: this view stays on the CPU */+ return 0;+ }+ return 0;+}+static void ink_vk_report(void) {+ if (g_vk_ready)+ fprintf(stderr, "[VK] inkling: %llu matmuls on the GPU\n", coli_vk_matmul_calls());+}+#endif+#ifdef COLI_INKLING_SHARED_BATCH_TESTstatic uint64_t g_matmul_w_calls;#endif@@ -422,6 +490,9 @@ static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {fprintf(stderr, "dense q4: geometria incoerente (serve %lld B, ho %lld) I=%d O=%d\n",(long long)need, (long long)W.qn, I, O); exit(1);}+#ifdef COLI_VULKAN+ if (W.vk && ink_vk_matmul(y, x, &W, S, I, O)) return;+#endifif (W.qbits == 8) matmul_i8r(y, x, (const int8_t*)W.q4, W.qs, S, I, O);else matmul_i4g(y, x, W.q4, W.qs, S, I, O, W.gs);return;@@ -895,6 +966,29 @@ static void unpack_rows(const uint8_t *raw, int8_t *q, int64_t rows, int64_t colstatic double mem_avail_bytes(void);+#ifdef COLI_VULKAN+/* After the weights: open the device (COLI_VULKAN=1) and mark the resident+ * matrices it can take. Without the dense-int4g64 container every resident is+ * bf16 or f32, which the shader does not read, and the line says so. */+static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {+ if (!g_vk_ready) g_vk_ready = coli_vk_init_env("inkling");+ if (!g_vk_ready) return;+ int n = ink_vk_attach(&m->lm_head, 1);+ for (int i = layer_begin; i < layer_end; i++) {+ Layer *l = &m->L[i];+ n += ink_vk_attach(&l->q, 1) + ink_vk_attach(&l->k, 1) + ink_vk_attach(&l->v, 1)+ + ink_vk_attach(&l->r, 1) + ink_vk_attach(&l->o, 1);+ n += ink_vk_attach(&l->dg, 1) + ink_vk_attach(&l->du, 1) + ink_vk_attach(&l->dd, 1);+ int ns = m->c.n_shared;+ n += ink_vk_attach(&l->sh_g, ns) + ink_vk_attach(&l->sh_u, ns) + ink_vk_attach(&l->sh_d, ns);+ }+ if (n) fprintf(stderr, "[VK] inkling: %d resident tensors (int8, int4-g64) on the GPU; "+ "bf16/f32 residents and routed experts stay on the CPU\n", n);+ else fprintf(stderr, "[VK] inkling: no resident tensor in a format the GPU reads "+ "(int8, int4-g64: the dense-int4g64 container), everything stays on the CPU\n");+}+#endif+static void model_init_range(Model *m, const char *snap, int cap, int bits,int layer_begin, int layer_end,int load_boundaries, int allocate_state,@@ -1114,6 +1208,9 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,m->eusage = rt_counts_all(); /* alias: the bump sites stay as they are */}m->dense_load_s = now_s() - t0;+#ifdef COLI_VULKAN+ ink_vk_init_model(m, layer_begin, layer_end);+#endif}static void model_init(Model *m, const char *snap, int cap, int bits) {@@ -2136,6 +2233,9 @@ static void generate_stream(Model *m, Tok *T, const char *prompt, int n_new,printf("[phases] fill %.1fs | expert-mm %.1fs | shared %.1fs | attn %.1fs | other %.1fs\n",m->t_fill, m->t_expert, m->t_shared, m->t_attn,wall - m->t_fill - m->t_expert - m->t_shared - m->t_attn);+#ifdef COLI_VULKAN+ fflush(stdout); ink_vk_report();+#endiffree(ids);}@@ -2466,6 +2566,9 @@ static int serve_one(Model *m, Tok *T, SReq *q) {printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt, np, gen,m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, forwards);fflush(stdout);+#ifdef COLI_VULKAN+ ink_vk_report();+#endifserve_hits(m);free(ids);return 0;@@ -2806,6 +2909,9 @@ int main(int argc, char **argv) {#endifprintf("PEAK RSS: %.2f GB | expert cache hit %.1f%% | %.2f tok/s\n",rss_gb(), tot?100.0*m.hits/tot:0.0, ngen/dt);+#ifdef COLI_VULKAN+ fflush(stdout); ink_vk_report();+#endiffree(buf); free(arena);return (match == ngen) ? 0 : 1;}@@ -2829,6 +2935,9 @@ typedef struct {static void inkling_segment_wt_destroy(Wt *weight) {if (!weight) return;+#ifdef COLI_VULKAN+ ink_vk_release(weight);+#endiffree(weight->f); free(weight->h);free(weight->q4); free(weight->qs);memset(weight, 0, sizeof(*weight));@@ -3211,6 +3320,9 @@ typedef struct {static void inkling_edge_wt_destroy(Wt *weight) {if (!weight) return;+#ifdef COLI_VULKAN+ ink_vk_release(weight);+#endiffree(weight->f); free(weight->h); free(weight->q4); free(weight->qs);memset(weight, 0, sizeof(*weight));}