Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: CUDA tier β€” bf16 residents in VRAM (A6000), +28% decode, best prefill

Nicholas Beerbowera34edb24eb2d
diff --git a/c/Makefile b/c/Makefile
index fbb68a1d..c7175681 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -227,6 +227,7 @@ endif
CFLAGS += -DCOLI_CUDA
LDFLAGS += -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcudart -lstdc++
CUDA_OBJ = backend_cuda.o
+INK_CUDA_OBJ = backend_cuda_ink.o
endif
ifeq ($(HIP),1)
@@ -300,6 +301,10 @@ cuda-dll: backend_cuda.cu backend_cuda.h
-L"$(CUDA_HOME)/lib/x64" -lcudart \
backend_cuda.cu -o coli_cuda.dll
+backend_cuda_ink.o: backend_cuda_ink.cu backend_cuda_ink.h .build-config
+ @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
+ "$(NVCC)" $(NVCCFLAGS) -c backend_cuda_ink.cu -o $@
+
backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-config
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) -c backend_cuda.cu -o $@
@@ -339,8 +344,8 @@ cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tens
olmoe$(EXE): olmoe.c st.h json.h compat.h
$(CC) $(CFLAGS) olmoe.c -o olmoe$(EXE) $(LDFLAGS)
-inkling$(EXE): inkling.c st.h json.h compat.h
- $(CC) $(CFLAGS) inkling.c -o inkling$(EXE) $(LDFLAGS)
+inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ)
+ $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) -o inkling$(EXE) $(LDFLAGS)
# Use a baseline that matches the compiler target. macOS already targets a
# portable baseline when ARCH is empty; forcing the x86 value there breaks
diff --git a/c/backend_cuda_ink.cu b/c/backend_cuda_ink.cu
new file mode 100644
index 00000000..6839f5fe
--- /dev/null
+++ b/c/backend_cuda_ink.cu
@@ -0,0 +1,87 @@
+/* CUDA backend for inkling.c β€” see backend_cuda_ink.h for scope.
+ * One warp per output row, both operands rounded to bf16, f32 accumulate:
+ * the same numeric contract as the CPU vdpbf16ps path (matmul_h), so GPU
+ * and CPU runs stay closely comparable. */
+#include <cuda_runtime.h>
+#include <cuda_bf16.h>
+#include <stdio.h>
+#include "backend_cuda_ink.h"
+
+static cudaStream_t g_st;
+static float *g_dx = nullptr, *g_dy = nullptr; /* device activation buffers */
+static float *g_hx = nullptr, *g_hy = nullptr; /* PINNED host staging: pageable
+ * memcpyAsync degrades to a
+ * synchronous bounce copy */
+static size_t g_xcap = 0, g_ycap = 0;
+static int g_ok = 0;
+
+extern "C" int ink_cuda_init(int dev) {
+ if (cudaSetDevice(dev) != cudaSuccess) return -1;
+ if (cudaStreamCreate(&g_st) != cudaSuccess) return -1;
+ /* fail fast on a broken runtime instead of at the first matmul */
+ void *probe = nullptr;
+ if (cudaMalloc(&probe, 1 << 20) != cudaSuccess) return -1;
+ cudaFree(probe);
+ g_ok = 1;
+ return 0;
+}
+
+extern "C" size_t ink_cuda_free_bytes(void) {
+ size_t fr = 0, to = 0;
+ if (cudaMemGetInfo(&fr, &to) != cudaSuccess) return 0;
+ return fr;
+}
+
+extern "C" void *ink_cuda_upload(const void *h, size_t n) {
+ void *d = nullptr;
+ if (cudaMalloc(&d, n) != cudaSuccess) return nullptr;
+ if (cudaMemcpy(d, h, n, cudaMemcpyHostToDevice) != cudaSuccess) { cudaFree(d); return nullptr; }
+ return d;
+}
+
+/* 4 warps per block, one output row per warp; lanes stride the contraction
+ * dim two bf16 at a time (4-byte coalesced loads of the weight row). */
+__global__ void mm_bf16_kernel(const __nv_bfloat16 * __restrict__ W,
+ const float * __restrict__ x,
+ float * __restrict__ y, int I, int O) {
+ int o = blockIdx.x * (blockDim.x >> 5) + (threadIdx.x >> 5);
+ int lane = threadIdx.x & 31;
+ int s = blockIdx.y;
+ if (o >= O) return;
+ const __nv_bfloat16 *w = W + (size_t)o * I;
+ const float *xs = x + (size_t)s * I;
+ float acc = 0.f;
+ for (int i = lane * 2; i < I; i += 64) {
+ __nv_bfloat162 wv = *(const __nv_bfloat162 *)(w + i);
+ float xa = __bfloat162float(__float2bfloat16(xs[i]));
+ float xb = __bfloat162float(__float2bfloat16(xs[i + 1]));
+ acc += __bfloat162float(wv.x) * xa + __bfloat162float(wv.y) * xb;
+ }
+ for (int off = 16; off; off >>= 1) acc += __shfl_down_sync(0xffffffffu, acc, off);
+ if (!lane) y[(size_t)s * O + o] = acc;
+}
+
+extern "C" int ink_cuda_matmul_bf16(float *y, const float *x, const void *W, int S, int I, int O) {
+ if (!g_ok || (I & 1)) return -1;
+ size_t xn = (size_t)S * I * 4, yn = (size_t)S * O * 4;
+ if (xn > g_xcap) {
+ cudaFree(g_dx); cudaFreeHost(g_hx);
+ if (cudaMalloc(&g_dx, xn * 2) != cudaSuccess ||
+ cudaMallocHost(&g_hx, xn * 2) != cudaSuccess) { g_dx = g_hx = nullptr; g_xcap = 0; return -1; }
+ g_xcap = xn * 2;
+ }
+ if (yn > g_ycap) {
+ cudaFree(g_dy); cudaFreeHost(g_hy);
+ if (cudaMalloc(&g_dy, yn * 2) != cudaSuccess ||
+ cudaMallocHost(&g_hy, yn * 2) != cudaSuccess) { g_dy = g_hy = nullptr; g_ycap = 0; return -1; }
+ g_ycap = yn * 2;
+ }
+ memcpy(g_hx, x, xn);
+ cudaMemcpyAsync(g_dx, g_hx, xn, cudaMemcpyHostToDevice, g_st);
+ dim3 grid((unsigned)((O + 3) / 4), (unsigned)S);
+ mm_bf16_kernel<<<grid, 128, 0, g_st>>>((const __nv_bfloat16 *)W, g_dx, g_dy, I, O);
+ cudaMemcpyAsync(g_hy, g_dy, yn, cudaMemcpyDeviceToHost, g_st);
+ if (cudaStreamSynchronize(g_st) != cudaSuccess) return -1;
+ memcpy(y, g_hy, yn);
+ return 0;
+}
diff --git a/c/backend_cuda_ink.h b/c/backend_cuda_ink.h
new file mode 100644
index 00000000..9383d2a5
--- /dev/null
+++ b/c/backend_cuda_ink.h
@@ -0,0 +1,23 @@
+/* Minimal CUDA backend for inkling.c: bf16 resident weights live in VRAM,
+ * matmuls run on-device. Motivation is bandwidth, not FLOPs: decode reads
+ * ~35 GB of bf16 residents per token, which caps CPU decode at the DDR5
+ * bandwidth (~1.2 s/token); the GPU's VRAM feeds the same reads at ~12x.
+ * Deliberately tiny API β€” expert streaming, attention and everything else
+ * stay on the CPU in this phase. */
+#ifndef BACKEND_CUDA_INK_H
+#define BACKEND_CUDA_INK_H
+#include <stddef.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int ink_cuda_init(int dev); /* 0 = ok */
+size_t ink_cuda_free_bytes(void);
+void *ink_cuda_upload(const void *h, size_t n); /* NULL = OOM/error */
+/* y[S,O] = x[S,I] @ W^T, W = device bf16 [O,I]; x,y host f32. 0 = ok */
+int ink_cuda_matmul_bf16(float *y, const float *x, const void *W, int S, int I, int O);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/c/inkling.c b/c/inkling.c
index 85fda0ef..205f0ccc 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -32,6 +32,10 @@
#endif
#include "st.h"
#include "tok.h"
+#ifdef COLI_CUDA
+#include "backend_cuda_ink.h"
+static int g_cuda = 0;
+#endif
#define MAXL 256
@@ -59,8 +63,11 @@ typedef struct {
/* ---------- resident weights ----------
* Large matmul weights keep their on-disk dtype in RAM: bf16 for the real
* 975B checkpoint (f32 residents would need ~172 GB, over sabre's 187),
- * f32 for the tiny oracle (bit-exact validation). Exactly one pointer set. */
-typedef struct { float *f; uint16_t *h; } Wt;
+ * f32 for the tiny oracle (bit-exact validation). Under CUDA, bf16 tensors
+ * move to VRAM (dev set, host freed): decode reads ~35 GB of residents per
+ * token, so this trades the DDR5 bandwidth wall for VRAM bandwidth AND
+ * frees the same RAM for the expert cache. */
+typedef struct { float *f; uint16_t *h; void *dev; } Wt;
typedef struct {
float *in_ln, *post_ln;
@@ -186,8 +193,14 @@ static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I,
}
}
-/* dispatch on the weight's resident dtype */
+/* dispatch on where the weight lives */
static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {
+#ifdef COLI_CUDA
+ if (W.dev) {
+ if (ink_cuda_matmul_bf16(y, x, W.dev, S, I, O) == 0) return;
+ fprintf(stderr, "cuda matmul failed and host copy was freed\n"); exit(1);
+ }
+#endif
if (W.f) matmul(y, x, W.f, S, I, O);
else matmul_h(y, x, W.h, S, I, O);
}
@@ -460,14 +473,25 @@ static void pread_all(int fd, void *buf, int64_t nb, int64_t off) {
}
/* big matmul weights keep their on-disk dtype resident: BF16 raw (real
- * checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact) */
-static Wt load_w(Model *m, const char *name) {
+ * checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact).
+ * gpu_ok: bf16 tensors move to VRAM while budget lasts (embed stays host β€”
+ * it's a row lookup, not a matmul). */
+static Wt load_w(Model *m, const char *name, int gpu_ok) {
Wt w = {0};
st_tensor *t = st_find(&m->S, name);
if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }
if (t->dtype == 0) {
w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); }
pread_all(t->fd, w.h, t->nbytes, t->off);
+#ifdef COLI_CUDA
+ /* keep 3 GB VRAM headroom for the activation buffers + future tiers */
+ if (g_cuda && gpu_ok && ink_cuda_free_bytes() > (size_t)t->nbytes + (3ULL<<30)) {
+ w.dev = ink_cuda_upload(w.h, t->nbytes);
+ if (w.dev) { free(w.h); w.h = NULL; }
+ }
+#else
+ (void)gpu_ok;
+#endif
} else {
w.f = falloc(t->numel);
st_read_f32(&m->S, name, w.f, 0);
@@ -475,7 +499,8 @@ static Wt load_w(Model *m, const char *name) {
return w;
}
static Wt wt_off(Wt w, int64_t off) {
- Wt r = { w.f ? w.f + off : NULL, w.h ? w.h + off : NULL };
+ Wt r = { w.f ? w.f + off : NULL, w.h ? w.h + off : NULL,
+ w.dev ? (char*)w.dev + off*2 : NULL }; /* dev is always bf16 */
return r;
}
static void wt_row_f32(Wt w, int64_t off, float *out, int n) {
@@ -533,16 +558,26 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {
Cfg *c = &m->c;
int D = c->hidden, K = c->conv_k;
double t0 = now_s();
- m->embed = load_w(m, "model.embed_tokens.weight");
+#ifdef COLI_CUDA
+ if (!getenv("NOGPU")) {
+ int dev = getenv("GPU_DEV") ? atoi(getenv("GPU_DEV")) : 0;
+ if (ink_cuda_init(dev) == 0) {
+ g_cuda = 1;
+ fprintf(stderr, "[cuda] device %d ready, %.1f GB free β€” bf16 residents to VRAM\n",
+ dev, ink_cuda_free_bytes()/1e9);
+ } else fprintf(stderr, "[cuda] init failed, running on CPU\n");
+ }
+#endif
+ m->embed = load_w(m, "model.embed_tokens.weight", 0);
m->embed_norm = st_has(&m->S,"model.embed_norm.weight") ? load_t(m,"model.embed_norm.weight") : NULL;
m->final_norm = load_t(m, "model.norm.weight");
- m->lm_head = load_w(m, "lm_head.weight");
+ m->lm_head = load_w(m, "lm_head.weight", 1);
m->L = calloc(c->n_layers, sizeof(Layer));
char nm[320];
for (int i = 0; i < c->n_layers; i++) {
Layer *l = &m->L[i];
#define LD(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm)
- #define LDW(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_w(m,nm)
+ #define LDW(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_w(m,nm,1)
LD(in_ln, "input_layernorm.weight");
LD(post_ln,"post_attention_layernorm.weight");
LDW(q, "self_attn.q_proj.weight"); LDW(k, "self_attn.k_proj.weight");
@@ -1081,7 +1116,10 @@ int main(int argc, char **argv) {
* per-expert matmul regions are tiny and back-to-back; the default passive
* wait policy parks the team between regions and re-wake latency dominates.
* libgomp reads OMP_/GOMP_ vars before main(), so seed them and re-exec
- * once (COLI_OMP_TUNED guards the exec; COLI_NO_OMP_TUNE=1 disables). */
+ * once (COLI_OMP_TUNED guards the exec; COLI_NO_OMP_TUNE=1 disables).
+ * NOT under CUDA β€” same exception glm.c makes: a spinning 24-thread team
+ * starves the CUDA driver during every stream sync. */
+#ifndef COLI_CUDA
if (!getenv("COLI_OMP_TUNED") && !getenv("COLI_NO_OMP_TUNE")) {
setenv("OMP_WAIT_POLICY","active",0);
setenv("GOMP_SPINCOUNT","200000",0);
@@ -1093,6 +1131,7 @@ int main(int argc, char **argv) {
perror("[OMP] execv self-reexec failed, running untuned");
#endif
}
+#endif /* !COLI_CUDA */
const char *snap = getenv("SNAP");
if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }
/* flags: -p "prompt" [-n N] -> generate mode; positional: [cap] [bits] [ref.json] */