Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen36: the resident dense trunk on Vulkan (COLI_VULKAN=1)
JustVugg0be4ef0ab576
diff --git a/c/qwen36.c b/c/qwen36.cindex d4fb5dff..e8deded5 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -74,6 +74,10 @@ static int qwen36_max_ctx(void) {#ifdef COLI_METAL#include "backend_metal.h" /* coli_metal_init: affine pipelines for the store */#endif+#ifdef COLI_VULKAN+#include "backend_vulkan.h" /* COLI_VULKAN=1: the resident dense trunk on a Vulkan device */+static int g_vk_ready = 0;+#endif#ifdef COLI_SEGMENT_ADAPTER#include "segment_runtime.h"#include "segment_adapters.h"@@ -727,9 +731,17 @@ typedef struct {/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM* tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one* scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel- * reads; ng = I/64 groups per row. */-typedef struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } QW;+ * reads; ng = I/64 groups per row.+ * vk/vk_off: the Vulkan device copy (COLI_VULKAN=1), uploaded at the first+ * matmul_d and kept; vk_off = the upload failed once, this matrix stays on the+ * CPU. Both stay zero in a build without VK=1. */+typedef struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng;+ void *vk; int vk_off; } QW;static void qw_free(QW *w) {+#ifdef COLI_VULKAN+ if (w->vk) coli_vk_tensor_free((ColiVkTensor *)w->vk);+ w->vk = NULL; w->vk_off = 0;+#endiffree((void*)w->w); free(w->q); free(w->sc); free(w->q4); free(w->sg);w->w = NULL; w->q = NULL; w->sc = NULL; w->q4 = NULL; w->sg = NULL; w->ng = 0;}@@ -1294,9 +1306,71 @@ static void qw_quantize(const float *W, int I, int O, const char *tag, QW *out)} else { free(q4); free(sg); }}}+#ifdef COLI_VULKAN+/* COLI_VULKAN=1: a resident dense matrix answers from the Vulkan device, with+ * the weights the CPU would read. The int4 copy when there is one (the CPU+ * takes it first too), else the int8 rows (fmt 1, one scale per row). The+ * planar int4 blocks are not the shader's layout (fmt 4: low nibble = even+ * column), so they are repacked once at upload, nibble for nibble: the same+ * codes v+8 and the same per-64 scales, nothing requantized. Only the+ * activation differs from the CPU's default: it stays f32 on the device,+ * where the integer dot rounds it to int8 (COLI_DENSE_IDOT=0 is the CPU arm+ * with the same arithmetic). The device copy lives in w->vk from the first+ * call; a failed upload sets vk_off and the matrix stays on the CPU. The f32+ * reference (COLI_DENSE_I8=0) has no device format and never comes here.+ * The backend has one command buffer: never from a parallel region. */+static void vk_q4_planar_to_fmt4(const QW *w, uint8_t *dst) {+ const int I = w->I, rb = I / 2, ng = I / 64;+ for (int o = 0; o < w->O; o++) {+ const uint8_t *src = w->q4 + (size_t)o * rb;+ uint8_t *row = dst + (size_t)o * rb;+ for (int g = 0; g < ng; g++)+ for (int p = 0; p < 32; p++) {+ int lo = 2 * p, hi = 2 * p + 1; /* columns g*64+lo, g*64+hi */+ uint8_t a = lo < 32 ? (src[g * 32 + lo] & 15) : (src[g * 32 + lo - 32] >> 4);+ uint8_t b = hi < 32 ? (src[g * 32 + hi] & 15) : (src[g * 32 + hi - 32] >> 4);+ row[g * 32 + p] = (uint8_t)(a | (b << 4));+ }+ }+}+static int vk_dense_matmul(float *y, const float *x, const QW *w, int S, int I, int O) {+#ifdef _OPENMP+ if (omp_in_parallel()) return 0;+#endif+ if (w->vk_off || w->I != I || w->O != O || S < 1 || S > 65535) return 0;+ QW *mw = (QW *)w; /* vk is a cache in a matrix the forward pass treats as read-only */+ ColiVkTensor **t = (ColiVkTensor **)&mw->vk;+ int fmt = w->q4 ? 4 : 1, gs = w->q4 ? 64 : 0;+ const void *wq = w->q4 ? (const void *)w->q4 : (const void *)w->q;+ const float *sc = w->q4 ? w->sg : w->sc;+ if (!*t) {+ int ok;+ if (w->q4) {+ uint8_t *packed = malloc((size_t)O * (I / 2));+ if (packed) vk_q4_planar_to_fmt4(w, packed);+ ok = packed && coli_vk_tensor_ensure(t, packed, sc, fmt, I, O, gs);+ free(packed);+ } else ok = coli_vk_tensor_ensure(t, wq, sc, fmt, I, O, gs);+ if (!ok) { mw->vk_off = 1; return 0; }+ }+ return coli_vk_matmul(t, y, x, wq, sc, fmt, S, I, O, gs);+}+/* One line at the end of a run or a serve turn: how many matmuls the device+ * really answered, so a test can tell a used path from an initialised one. */+static void vk_report(void) {+ if (!g_vk_ready) return;+ size_t bytes = 0, tensors = 0;+ coli_vk_mem_info(&bytes, &tensors);+ fprintf(stderr, "[VK] qwen36: %llu matmuls on the GPU (%zu matrices resident, %.1f MiB)\n",+ coli_vk_matmul_calls(), tensors, bytes / 1048576.0);+}+#endifstatic void matmul_d(float *y, const float *x, const QW *w, int S, int I, int O){#ifdef COLI_QWEN_BATCH_TESTg_qwen_matmul_d_calls++;+#endif+#ifdef COLI_VULKAN+ if (g_vk_ready && (w->q || w->q4) && vk_dense_matmul(y, x, w, S, I, O)) return;#endifif (w->q || w->q4) {if (w->q4 || dense_idot_on()) {@@ -1733,6 +1807,7 @@ static void load_tq(Model *m, const char *name, int I, int O, int quantize, consfloat *p = load_t_n(m, name, (int64_t)I * O);out->w = p; out->q = NULL; out->sc = NULL; out->I = I; out->O = O;out->q4 = NULL; out->sg = NULL; out->ng = 0;+ out->vk = NULL; out->vk_off = 0;if (!quantize || !dense_i8_on()) return;qw_quantize(p, I, O, tag, out);if (getenv("COLI_KEEP_F32")) out->w = p; else { free(p); out->w = NULL; }@@ -4174,6 +4249,9 @@ static void serve_one(Model *m, ServeReq *q){printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n",q->id,gen,dt>0?gen/dt:0.0,0.0,rss_gb(),np,limited);fflush(stdout);+#ifdef COLI_VULKAN+ vk_report(); /* stderr: the wire protocol on stdout is untouched */+#endif}static void serve_loop(Model *m){@@ -4374,6 +4452,11 @@ int main(int argc, char **argv) {g_expert_gs = m.c.expert_gs;if (g_expert_gs) fprintf(stderr, "[qwen36] group-scaled experts: gs=%d\n", g_expert_gs);fprintf(stderr, "resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());+#ifdef COLI_VULKAN+ /* After the weights: the dense matrices upload at their first matmul_d.+ * No device (or COLI_VULKAN unset) leaves g_vk_ready 0, the CPU path. */+ g_vk_ready = coli_vk_init_env("qwen36");+#endifif (ref_image && ref_image->t == J_OBJ) {jval *gh = json_get(ref_image, "grid_h"), *gw = json_get(ref_image, "grid_w");jval *pv = json_get(ref_image, "patches");@@ -4574,6 +4657,9 @@ int main(int argc, char **argv) {(unsigned long long)m.hits, (unsigned long long)m.miss);route_footer(stdout, &m);printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb());+#ifdef COLI_VULKAN+ vk_report();+#endiffree(buf); free(arena); return 0;}@@ -4656,6 +4742,9 @@ int main(int argc, char **argv) {(unsigned long long)m.hits, (unsigned long long)m.miss);route_footer(stderr, &m);fprintf(stderr, "Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new);+#ifdef COLI_VULKAN+ vk_report();+#endiffree(buf); free(arena);/* Oracle mode is a gate, not a report: a mismatch must fail the caller.* inkling.c does the same (`return (match == ngen) ? 0 : 1;`) and its CI