Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
inkling: CUDA tier β bf16 residents in VRAM (A6000), +28% decode, best prefill
Nicholas Beerbowera34edb24eb2d
diff --git a/c/Makefile b/c/Makefileindex fbb68a1d..c7175681 100644--- a/c/Makefile+++ b/c/Makefile@@ -227,6 +227,7 @@ endifCFLAGS += -DCOLI_CUDALDFLAGS += -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcudart -lstdc++CUDA_OBJ = backend_cuda.o+INK_CUDA_OBJ = backend_cuda_ink.oendififeq ($(HIP),1)@@ -300,6 +301,10 @@ cuda-dll: backend_cuda.cu backend_cuda.h-L"$(CUDA_HOME)/lib/x64" -lcudart \backend_cuda.cu -o coli_cuda.dll+backend_cuda_ink.o: backend_cuda_ink.cu backend_cuda_ink.h .build-config+ @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }+ "$(NVCC)" $(NVCCFLAGS) -c backend_cuda_ink.cu -o $@+backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-config@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }"$(GPUCC)" $(GPUFLAGS) -c backend_cuda.cu -o $@@@ -339,8 +344,8 @@ cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensolmoe$(EXE): olmoe.c st.h json.h compat.h$(CC) $(CFLAGS) olmoe.c -o olmoe$(EXE) $(LDFLAGS)-inkling$(EXE): inkling.c st.h json.h compat.h- $(CC) $(CFLAGS) inkling.c -o inkling$(EXE) $(LDFLAGS)+inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ)+ $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) -o inkling$(EXE) $(LDFLAGS)# Use a baseline that matches the compiler target. macOS already targets a# portable baseline when ARCH is empty; forcing the x86 value there breaksdiff --git a/c/backend_cuda_ink.cu b/c/backend_cuda_ink.cunew file mode 100644index 00000000..6839f5fe--- /dev/null+++ b/c/backend_cuda_ink.cu@@ -0,0 +1,87 @@+/* CUDA backend for inkling.c β see backend_cuda_ink.h for scope.+ * One warp per output row, both operands rounded to bf16, f32 accumulate:+ * the same numeric contract as the CPU vdpbf16ps path (matmul_h), so GPU+ * and CPU runs stay closely comparable. */+#include <cuda_runtime.h>+#include <cuda_bf16.h>+#include <stdio.h>+#include "backend_cuda_ink.h"++static cudaStream_t g_st;+static float *g_dx = nullptr, *g_dy = nullptr; /* device activation buffers */+static float *g_hx = nullptr, *g_hy = nullptr; /* PINNED host staging: pageable+ * memcpyAsync degrades to a+ * synchronous bounce copy */+static size_t g_xcap = 0, g_ycap = 0;+static int g_ok = 0;++extern "C" int ink_cuda_init(int dev) {+ if (cudaSetDevice(dev) != cudaSuccess) return -1;+ if (cudaStreamCreate(&g_st) != cudaSuccess) return -1;+ /* fail fast on a broken runtime instead of at the first matmul */+ void *probe = nullptr;+ if (cudaMalloc(&probe, 1 << 20) != cudaSuccess) return -1;+ cudaFree(probe);+ g_ok = 1;+ return 0;+}++extern "C" size_t ink_cuda_free_bytes(void) {+ size_t fr = 0, to = 0;+ if (cudaMemGetInfo(&fr, &to) != cudaSuccess) return 0;+ return fr;+}++extern "C" void *ink_cuda_upload(const void *h, size_t n) {+ void *d = nullptr;+ if (cudaMalloc(&d, n) != cudaSuccess) return nullptr;+ if (cudaMemcpy(d, h, n, cudaMemcpyHostToDevice) != cudaSuccess) { cudaFree(d); return nullptr; }+ return d;+}++/* 4 warps per block, one output row per warp; lanes stride the contraction+ * dim two bf16 at a time (4-byte coalesced loads of the weight row). */+__global__ void mm_bf16_kernel(const __nv_bfloat16 * __restrict__ W,+ const float * __restrict__ x,+ float * __restrict__ y, int I, int O) {+ int o = blockIdx.x * (blockDim.x >> 5) + (threadIdx.x >> 5);+ int lane = threadIdx.x & 31;+ int s = blockIdx.y;+ if (o >= O) return;+ const __nv_bfloat16 *w = W + (size_t)o * I;+ const float *xs = x + (size_t)s * I;+ float acc = 0.f;+ for (int i = lane * 2; i < I; i += 64) {+ __nv_bfloat162 wv = *(const __nv_bfloat162 *)(w + i);+ float xa = __bfloat162float(__float2bfloat16(xs[i]));+ float xb = __bfloat162float(__float2bfloat16(xs[i + 1]));+ acc += __bfloat162float(wv.x) * xa + __bfloat162float(wv.y) * xb;+ }+ for (int off = 16; off; off >>= 1) acc += __shfl_down_sync(0xffffffffu, acc, off);+ if (!lane) y[(size_t)s * O + o] = acc;+}++extern "C" int ink_cuda_matmul_bf16(float *y, const float *x, const void *W, int S, int I, int O) {+ if (!g_ok || (I & 1)) return -1;+ size_t xn = (size_t)S * I * 4, yn = (size_t)S * O * 4;+ if (xn > g_xcap) {+ cudaFree(g_dx); cudaFreeHost(g_hx);+ if (cudaMalloc(&g_dx, xn * 2) != cudaSuccess ||+ cudaMallocHost(&g_hx, xn * 2) != cudaSuccess) { g_dx = g_hx = nullptr; g_xcap = 0; return -1; }+ g_xcap = xn * 2;+ }+ if (yn > g_ycap) {+ cudaFree(g_dy); cudaFreeHost(g_hy);+ if (cudaMalloc(&g_dy, yn * 2) != cudaSuccess ||+ cudaMallocHost(&g_hy, yn * 2) != cudaSuccess) { g_dy = g_hy = nullptr; g_ycap = 0; return -1; }+ g_ycap = yn * 2;+ }+ memcpy(g_hx, x, xn);+ cudaMemcpyAsync(g_dx, g_hx, xn, cudaMemcpyHostToDevice, g_st);+ dim3 grid((unsigned)((O + 3) / 4), (unsigned)S);+ mm_bf16_kernel<<<grid, 128, 0, g_st>>>((const __nv_bfloat16 *)W, g_dx, g_dy, I, O);+ cudaMemcpyAsync(g_hy, g_dy, yn, cudaMemcpyDeviceToHost, g_st);+ if (cudaStreamSynchronize(g_st) != cudaSuccess) return -1;+ memcpy(y, g_hy, yn);+ return 0;+}diff --git a/c/backend_cuda_ink.h b/c/backend_cuda_ink.hnew file mode 100644index 00000000..9383d2a5--- /dev/null+++ b/c/backend_cuda_ink.h@@ -0,0 +1,23 @@+/* Minimal CUDA backend for inkling.c: bf16 resident weights live in VRAM,+ * matmuls run on-device. Motivation is bandwidth, not FLOPs: decode reads+ * ~35 GB of bf16 residents per token, which caps CPU decode at the DDR5+ * bandwidth (~1.2 s/token); the GPU's VRAM feeds the same reads at ~12x.+ * Deliberately tiny API β expert streaming, attention and everything else+ * stay on the CPU in this phase. */+#ifndef BACKEND_CUDA_INK_H+#define BACKEND_CUDA_INK_H+#include <stddef.h>+#ifdef __cplusplus+extern "C" {+#endif++int ink_cuda_init(int dev); /* 0 = ok */+size_t ink_cuda_free_bytes(void);+void *ink_cuda_upload(const void *h, size_t n); /* NULL = OOM/error */+/* y[S,O] = x[S,I] @ W^T, W = device bf16 [O,I]; x,y host f32. 0 = ok */+int ink_cuda_matmul_bf16(float *y, const float *x, const void *W, int S, int I, int O);++#ifdef __cplusplus+}+#endif+#endifdiff --git a/c/inkling.c b/c/inkling.cindex 85fda0ef..205f0ccc 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -32,6 +32,10 @@#endif#include "st.h"#include "tok.h"+#ifdef COLI_CUDA+#include "backend_cuda_ink.h"+static int g_cuda = 0;+#endif#define MAXL 256@@ -59,8 +63,11 @@ typedef struct {/* ---------- resident weights ----------* Large matmul weights keep their on-disk dtype in RAM: bf16 for the real* 975B checkpoint (f32 residents would need ~172 GB, over sabre's 187),- * f32 for the tiny oracle (bit-exact validation). Exactly one pointer set. */-typedef struct { float *f; uint16_t *h; } Wt;+ * f32 for the tiny oracle (bit-exact validation). Under CUDA, bf16 tensors+ * move to VRAM (dev set, host freed): decode reads ~35 GB of residents per+ * token, so this trades the DDR5 bandwidth wall for VRAM bandwidth AND+ * frees the same RAM for the expert cache. */+typedef struct { float *f; uint16_t *h; void *dev; } Wt;typedef struct {float *in_ln, *post_ln;@@ -186,8 +193,14 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,}}-/* dispatch on the weight's resident dtype */+/* dispatch on where the weight lives */static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {+#ifdef COLI_CUDA+ if (W.dev) {+ if (ink_cuda_matmul_bf16(y, x, W.dev, S, I, O) == 0) return;+ fprintf(stderr, "cuda matmul failed and host copy was freed\n"); exit(1);+ }+#endifif (W.f) matmul(y, x, W.f, S, I, O);else matmul_h(y, x, W.h, S, I, O);}@@ -460,14 +473,25 @@ static void pread_all(int fd, void *buf, int64_t nb, int64_t off) {}/* big matmul weights keep their on-disk dtype resident: BF16 raw (real- * checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact) */-static Wt load_w(Model *m, const char *name) {+ * checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact).+ * gpu_ok: bf16 tensors move to VRAM while budget lasts (embed stays host β+ * it's a row lookup, not a matmul). */+static Wt load_w(Model *m, const char *name, int gpu_ok) {Wt w = {0};st_tensor *t = st_find(&m->S, name);if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }if (t->dtype == 0) {w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); }pread_all(t->fd, w.h, t->nbytes, t->off);+#ifdef COLI_CUDA+ /* keep 3 GB VRAM headroom for the activation buffers + future tiers */+ if (g_cuda && gpu_ok && ink_cuda_free_bytes() > (size_t)t->nbytes + (3ULL<<30)) {+ w.dev = ink_cuda_upload(w.h, t->nbytes);+ if (w.dev) { free(w.h); w.h = NULL; }+ }+#else+ (void)gpu_ok;+#endif} else {w.f = falloc(t->numel);st_read_f32(&m->S, name, w.f, 0);@@ -475,7 +499,8 @@ static Wt load_w(Model *m, const char *name) {return w;}static Wt wt_off(Wt w, int64_t off) {- Wt r = { w.f ? w.f + off : NULL, w.h ? w.h + off : NULL };+ Wt r = { w.f ? w.f + off : NULL, w.h ? w.h + off : NULL,+ w.dev ? (char*)w.dev + off*2 : NULL }; /* dev is always bf16 */return r;}static void wt_row_f32(Wt w, int64_t off, float *out, int n) {@@ -533,16 +558,26 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {Cfg *c = &m->c;int D = c->hidden, K = c->conv_k;double t0 = now_s();- m->embed = load_w(m, "model.embed_tokens.weight");+#ifdef COLI_CUDA+ if (!getenv("NOGPU")) {+ int dev = getenv("GPU_DEV") ? atoi(getenv("GPU_DEV")) : 0;+ if (ink_cuda_init(dev) == 0) {+ g_cuda = 1;+ fprintf(stderr, "[cuda] device %d ready, %.1f GB free β bf16 residents to VRAM\n",+ dev, ink_cuda_free_bytes()/1e9);+ } else fprintf(stderr, "[cuda] init failed, running on CPU\n");+ }+#endif+ m->embed = load_w(m, "model.embed_tokens.weight", 0);m->embed_norm = st_has(&m->S,"model.embed_norm.weight") ? load_t(m,"model.embed_norm.weight") : NULL;m->final_norm = load_t(m, "model.norm.weight");- m->lm_head = load_w(m, "lm_head.weight");+ m->lm_head = load_w(m, "lm_head.weight", 1);m->L = calloc(c->n_layers, sizeof(Layer));char nm[320];for (int i = 0; i < c->n_layers; i++) {Layer *l = &m->L[i];#define LD(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm)- #define LDW(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_w(m,nm)+ #define LDW(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_w(m,nm,1)LD(in_ln, "input_layernorm.weight");LD(post_ln,"post_attention_layernorm.weight");LDW(q, "self_attn.q_proj.weight"); LDW(k, "self_attn.k_proj.weight");@@ -1081,7 +1116,10 @@ int main(int argc, char **argv) {* per-expert matmul regions are tiny and back-to-back; the default passive* wait policy parks the team between regions and re-wake latency dominates.* libgomp reads OMP_/GOMP_ vars before main(), so seed them and re-exec- * once (COLI_OMP_TUNED guards the exec; COLI_NO_OMP_TUNE=1 disables). */+ * once (COLI_OMP_TUNED guards the exec; COLI_NO_OMP_TUNE=1 disables).+ * NOT under CUDA β same exception glm.c makes: a spinning 24-thread team+ * starves the CUDA driver during every stream sync. */+#ifndef COLI_CUDAif (!getenv("COLI_OMP_TUNED") && !getenv("COLI_NO_OMP_TUNE")) {setenv("OMP_WAIT_POLICY","active",0);setenv("GOMP_SPINCOUNT","200000",0);@@ -1093,6 +1131,7 @@ int main(int argc, char **argv) {perror("[OMP] execv self-reexec failed, running untuned");#endif}+#endif /* !COLI_CUDA */const char *snap = getenv("SNAP");if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }/* flags: -p "prompt" [-n N] -> generate mode; positional: [cap] [bits] [ref.json] */