Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: resident dense matrices on Vulkan (COLI_VULKAN=1)

JustVugg3ca0ae97ccad
diff --git a/c/inkling.c b/c/inkling.c
index cfb51933..97e8252d 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -68,6 +68,14 @@ static int g_cuda = 0;
#include "backend_metal.h"
static int g_metal = 0;
#endif
+#ifdef COLI_VULKAN
+/* Vulkan (opt-in, VK=1 build + COLI_VULKAN=1): the RESIDENT dense matrices that
+ * the dense-int4g64 container holds in a format the qmatmul shader reads as it is
+ * (int8 per row -> fmt 1, int4-g64 -> fmt 4), uploaded on first use. Routed
+ * experts, which arrive from disk, and bf16/f32 residents stay on the CPU. */
+#include "backend_vulkan.h"
+static int g_vk_ready = 0;
+#endif
#if defined(__APPLE__)
#include <mach/mach.h>
#include <sys/sysctl.h>
@@ -111,7 +119,12 @@ typedef struct {
typedef struct { float *f; uint16_t *h; void *dev;
uint8_t *q4; /* int4 nibble-packed (qbits=4) o int8 (qbits=8) */
float *qs; /* scale: [rows*ng] se qbits=4, [rows] se qbits=8 */
- int gs, qbits; int64_t qn; } Wt; /* qn = byte in q4, per il guard OOB */
+ int gs, qbits; int64_t qn; /* qn = byte in q4, per il guard OOB */
+#ifdef COLI_VULKAN
+ void *vk; /* InkVk of the loaded tensor, shared by its wt_off_i
+ * views (NULL = CPU only) */
+#endif
+ } Wt;
typedef struct {
float *in_ln, *post_ln;
@@ -401,6 +414,61 @@ static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *s
}
}
+#ifdef COLI_VULKAN
+/* Device copies of one loaded tensor. A Wt travels by value and the shared
+ * experts are read through wt_off_i views of one fused [ns][R,I] tensor, so the
+ * cache cannot live in the copy matmul_w receives: it lives in a table the
+ * loaded Wt points to and every view copies, one entry per view (keyed by the
+ * view's first byte). Released with the tensor it belongs to. */
+typedef struct { const uint8_t *q; ColiVkTensor *t; int dead; } InkVkView;
+typedef struct { int n; InkVkView e[]; } InkVk;
+
+/* Attach a table to a loaded tensor the shader reads exactly as the CPU kernel
+ * does: int8 per row (matmul_i8r) is fmt 1, int4 group-scaled with +8 nibbles
+ * and the low nibble on the even column (matmul_i4g) is fmt 4. */
+static int ink_vk_attach(Wt *w, int views) {
+ if (!w->q4 || w->vk || views < 1) return 0;
+ if (w->qbits != 8 && !(w->qbits == 4 && w->gs >= 8 && w->gs % 8 == 0)) return 0;
+ InkVk *v = calloc(1, sizeof(*v) + (size_t)views * sizeof(v->e[0]));
+ if (!v) return 0;
+ v->n = views;
+ w->vk = v;
+ return 1;
+}
+static void ink_vk_release(Wt *w) {
+ InkVk *v = w->vk;
+ if (!v) return;
+ for (int k = 0; k < v->n; k++) if (v->e[k].t) coli_vk_tensor_free(v->e[k].t);
+ free(v);
+ w->vk = NULL;
+}
+/* 1 when y was computed on the device. One command buffer in the backend:
+ * main thread only, never from inside a parallel region. */
+static int ink_vk_matmul(float *y, const float *x, const Wt *W, int S, int I, int O) {
+ InkVk *v = W->vk;
+ if (!g_vk_ready || !v) return 0;
+#ifdef _OPENMP
+ if (omp_in_parallel()) return 0;
+#endif
+ for (int k = 0; k < v->n; k++) {
+ InkVkView *e = &v->e[k];
+ if (e->q && e->q != W->q4) continue;
+ if (e->dead) return 0;
+ e->q = W->q4;
+ if (coli_vk_matmul(&e->t, y, x, W->q4, W->qs, W->qbits == 8 ? 1 : 4,
+ S, I, O, W->gs))
+ return 1;
+ if (!e->t) e->dead = 1; /* refused at upload: this view stays on the CPU */
+ return 0;
+ }
+ return 0;
+}
+static void ink_vk_report(void) {
+ if (g_vk_ready)
+ fprintf(stderr, "[VK] inkling: %llu matmuls on the GPU\n", coli_vk_matmul_calls());
+}
+#endif
+
#ifdef COLI_INKLING_SHARED_BATCH_TEST
static uint64_t g_matmul_w_calls;
#endif
@@ -422,6 +490,9 @@ static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {
fprintf(stderr, "dense q4: geometria incoerente (serve %lld B, ho %lld) I=%d O=%d\n",
(long long)need, (long long)W.qn, I, O); exit(1);
}
+#ifdef COLI_VULKAN
+ if (W.vk && ink_vk_matmul(y, x, &W, S, I, O)) return;
+#endif
if (W.qbits == 8) matmul_i8r(y, x, (const int8_t*)W.q4, W.qs, S, I, O);
else matmul_i4g(y, x, W.q4, W.qs, S, I, O, W.gs);
return;
@@ -895,6 +966,29 @@ static void unpack_rows(const uint8_t *raw, int8_t *q, int64_t rows, int64_t col
static double mem_avail_bytes(void);
+#ifdef COLI_VULKAN
+/* After the weights: open the device (COLI_VULKAN=1) and mark the resident
+ * matrices it can take. Without the dense-int4g64 container every resident is
+ * bf16 or f32, which the shader does not read, and the line says so. */
+static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {
+ if (!g_vk_ready) g_vk_ready = coli_vk_init_env("inkling");
+ if (!g_vk_ready) return;
+ int n = ink_vk_attach(&m->lm_head, 1);
+ for (int i = layer_begin; i < layer_end; i++) {
+ Layer *l = &m->L[i];
+ n += ink_vk_attach(&l->q, 1) + ink_vk_attach(&l->k, 1) + ink_vk_attach(&l->v, 1)
+ + ink_vk_attach(&l->r, 1) + ink_vk_attach(&l->o, 1);
+ n += ink_vk_attach(&l->dg, 1) + ink_vk_attach(&l->du, 1) + ink_vk_attach(&l->dd, 1);
+ int ns = m->c.n_shared;
+ n += ink_vk_attach(&l->sh_g, ns) + ink_vk_attach(&l->sh_u, ns) + ink_vk_attach(&l->sh_d, ns);
+ }
+ if (n) fprintf(stderr, "[VK] inkling: %d resident tensors (int8, int4-g64) on the GPU; "
+ "bf16/f32 residents and routed experts stay on the CPU\n", n);
+ else fprintf(stderr, "[VK] inkling: no resident tensor in a format the GPU reads "
+ "(int8, int4-g64: the dense-int4g64 container), everything stays on the CPU\n");
+}
+#endif
+
static void model_init_range(Model *m, const char *snap, int cap, int bits,
int layer_begin, int layer_end,
int load_boundaries, int allocate_state,
@@ -1114,6 +1208,9 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
m->eusage = rt_counts_all(); /* alias: the bump sites stay as they are */
}
m->dense_load_s = now_s() - t0;
+#ifdef COLI_VULKAN
+ ink_vk_init_model(m, layer_begin, layer_end);
+#endif
}
static void model_init(Model *m, const char *snap, int cap, int bits) {
@@ -2136,6 +2233,9 @@ static void generate_stream(Model *m, Tok *T, const char *prompt, int n_new,
printf("[phases] fill %.1fs | expert-mm %.1fs | shared %.1fs | attn %.1fs | other %.1fs\n",
m->t_fill, m->t_expert, m->t_shared, m->t_attn,
wall - m->t_fill - m->t_expert - m->t_shared - m->t_attn);
+#ifdef COLI_VULKAN
+ fflush(stdout); ink_vk_report();
+#endif
free(ids);
}
@@ -2466,6 +2566,9 @@ static int serve_one(Model *m, Tok *T, SReq *q) {
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt, np, gen,
m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, forwards);
fflush(stdout);
+#ifdef COLI_VULKAN
+ ink_vk_report();
+#endif
serve_hits(m);
free(ids);
return 0;
@@ -2806,6 +2909,9 @@ int main(int argc, char **argv) {
#endif
printf("PEAK RSS: %.2f GB | expert cache hit %.1f%% | %.2f tok/s\n",
rss_gb(), tot?100.0*m.hits/tot:0.0, ngen/dt);
+#ifdef COLI_VULKAN
+ fflush(stdout); ink_vk_report();
+#endif
free(buf); free(arena);
return (match == ngen) ? 0 : 1;
}
@@ -2829,6 +2935,9 @@ typedef struct {
static void inkling_segment_wt_destroy(Wt *weight) {
if (!weight) return;
+#ifdef COLI_VULKAN
+ ink_vk_release(weight);
+#endif
free(weight->f); free(weight->h);
free(weight->q4); free(weight->qs);
memset(weight, 0, sizeof(*weight));
@@ -3211,6 +3320,9 @@ typedef struct {
static void inkling_edge_wt_destroy(Wt *weight) {
if (!weight) return;
+#ifdef COLI_VULKAN
+ ink_vk_release(weight);
+#endif
free(weight->f); free(weight->h); free(weight->q4); free(weight->qs);
memset(weight, 0, sizeof(*weight));
}