Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda(win): export + load the #111 pipeline ABI — 24 missing DLLEXPORT decorations + loader wrappers, unbreaks CUDA_DLL=1 (#201)
Tom Olorin21cbf14d591e
diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex c84a265a..acbe4bc8 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -60,7 +60,7 @@ COLI_CUDA_DLLEXPORT int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTenso* memory, activations are converted to FP16 per tile, and Tensor Cores* accumulate into FP32. Unlike COLI_CUDA_TC_INT4 this does not quantize the* activation to INT4. */-int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,+COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,ColiCudaTensor *down, float *y,const float *x, int S);@@ -97,46 +97,46 @@ COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);/* Replace a resident tensor's contents without reallocating its device slot. */-int coli_cuda_tensor_update(ColiCudaTensor *tensor,+COLI_CUDA_DLLEXPORT int coli_cuda_tensor_update(ColiCudaTensor *tensor,const void *weights, const float *scales);/* ---- resident-pipeline primitives (Inc.0): device-pointer entry points ---- */-float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);-void *coli_cuda_pipe_alloc(int device,size_t bytes);-void coli_cuda_pipe_free(int device,void *p);-int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);-int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);-int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,+COLI_CUDA_DLLEXPORT float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);+COLI_CUDA_DLLEXPORT void *coli_cuda_pipe_alloc(int device,size_t bytes);+COLI_CUDA_DLLEXPORT void coli_cuda_pipe_free(int device,void *p);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,const float *w_dev,int S,int D,float eps);-int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,int stride,int offset,int R,int heads,float theta);-int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);-int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);-int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,const int *rows_dev,int nrows,int D);-int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);-int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,const float *w_dev,int S,int D,float eps,int xstride,int ystride);-int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,int stride,int offset,int R,int heads,float theta);-int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,int spitch,int width,int height);-int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,+COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,float *out,const float *q_dev,const float *latent_dev,const float *rope_dev,int S,int H,int Q,int R,int V,int K,int T,float scale);-int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,+COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,const float *q_dev,const float *latent_dev,const float *rope_dev,int S,int H,int Q,int R,int V,int K,int T,float scale);-int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,+COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T,float scale);-int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,const float *src,size_t bytes);-int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,+COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev,int S,int H,int Q,int R,int V,int K,int T,float scale);-int coli_cuda_pipe_sync(int device);+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);#ifdef __cplusplus}diff --git a/c/backend_loader.c b/c/backend_loader.cindex 6fea70c1..0635e081 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -52,6 +52,35 @@ typedef void (*fn_tensor_free)(ColiCudaTensor *tensor);typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor);typedef int (*fn_tensor_device)(const ColiCudaTensor *tensor);+/* --- #111 GPU resident pipeline additions (matched to backend_cuda.h) --- */+++/* --- #111 GPU resident pipeline additions (matched to backend_cuda.h) --- */+typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent,const float *rope,int S, int H,int Q,int R,int V,int K,int T, float attention_scale);+typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);+typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale);+typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale);+typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);+typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);+typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n);+typedef void * (*fn_pipe_alloc)(int device,size_t bytes);+typedef int (*fn_pipe_copy2d)(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height);+typedef int (*fn_pipe_download)(int device,const void *src,void *dst,size_t bytes);+typedef void (*fn_pipe_free)(int device,void *p);+typedef int (*fn_pipe_gemm)(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);+typedef int (*fn_pipe_peer_copy)(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes);+typedef int (*fn_pipe_rmsnorm)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps);+typedef int (*fn_pipe_rmsnorm_s)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride);+typedef int (*fn_pipe_rope)(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta);+typedef int (*fn_pipe_rope_base)(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta);+typedef int (*fn_pipe_rows_add)(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D);+typedef float * (*fn_pipe_scratch)(int device,int slot,size_t bytes);+typedef int (*fn_pipe_silu_mul)(int device,float *gate_dev,const float *up_dev,size_t n);+typedef int (*fn_pipe_sync)(int device);+typedef int (*fn_pipe_upload)(int device,void *dst,const void *src,size_t bytes);+typedef int (*fn_shared_mlp_w4a16)(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S);+typedef int (*fn_tensor_update)(ColiCudaTensor *tensor, const void *weights, const float *scales);+/* Resolved pointers, plus a flag so we attempt the load at most once. */static struct {int loaded; /* 1 = load attempted (success or fail), 0 = not yet */@@ -72,6 +101,31 @@ static struct {fn_tensor_free tensor_free;fn_tensor_bytes tensor_bytes;fn_tensor_device tensor_device;++ fn_attention_absorb_batch attention_absorb_batch;+ fn_attention_absorb_batch_dev attention_absorb_batch_dev;+ fn_attention_absorb_kvdev attention_absorb_kvdev;+ fn_attention_project_batch attention_project_batch;+ fn_attention_project_batch_dev attention_project_batch_dev;+ fn_attention_project_batch_dev_out attention_project_batch_dev_out;+ fn_pipe_add pipe_add;+ fn_pipe_alloc pipe_alloc;+ fn_pipe_copy2d pipe_copy2d;+ fn_pipe_download pipe_download;+ fn_pipe_free pipe_free;+ fn_pipe_gemm pipe_gemm;+ fn_pipe_peer_copy pipe_peer_copy;+ fn_pipe_rmsnorm pipe_rmsnorm;+ fn_pipe_rmsnorm_s pipe_rmsnorm_s;+ fn_pipe_rope pipe_rope;+ fn_pipe_rope_base pipe_rope_base;+ fn_pipe_rows_add pipe_rows_add;+ fn_pipe_scratch pipe_scratch;+ fn_pipe_silu_mul pipe_silu_mul;+ fn_pipe_sync pipe_sync;+ fn_pipe_upload pipe_upload;+ fn_shared_mlp_w4a16 shared_mlp_w4a16;+ fn_tensor_update tensor_update;} g_cuda;/* Resolve the DLL and all 11 symbols. Returns 1 on success, 0 otherwise.@@ -119,6 +173,31 @@ static int coli_cuda_load(void){RESOLVE(tensor_free, fn_tensor_free)RESOLVE(tensor_bytes, fn_tensor_bytes)RESOLVE(tensor_device, fn_tensor_device)++ RESOLVE(attention_absorb_batch, fn_attention_absorb_batch)+ RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev)+ RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev)+ RESOLVE(attention_project_batch, fn_attention_project_batch)+ RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev)+ RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out)+ RESOLVE(pipe_add, fn_pipe_add)+ RESOLVE(pipe_alloc, fn_pipe_alloc)+ RESOLVE(pipe_copy2d, fn_pipe_copy2d)+ RESOLVE(pipe_download, fn_pipe_download)+ RESOLVE(pipe_free, fn_pipe_free)+ RESOLVE(pipe_gemm, fn_pipe_gemm)+ RESOLVE(pipe_peer_copy, fn_pipe_peer_copy)+ RESOLVE(pipe_rmsnorm, fn_pipe_rmsnorm)+ RESOLVE(pipe_rmsnorm_s, fn_pipe_rmsnorm_s)+ RESOLVE(pipe_rope, fn_pipe_rope)+ RESOLVE(pipe_rope_base, fn_pipe_rope_base)+ RESOLVE(pipe_rows_add, fn_pipe_rows_add)+ RESOLVE(pipe_scratch, fn_pipe_scratch)+ RESOLVE(pipe_silu_mul, fn_pipe_silu_mul)+ RESOLVE(pipe_sync, fn_pipe_sync)+ RESOLVE(pipe_upload, fn_pipe_upload)+ RESOLVE(shared_mlp_w4a16, fn_shared_mlp_w4a16)+ RESOLVE(tensor_update, fn_tensor_update)#undef RESOLVEg_cuda.available = 1;@@ -216,4 +295,129 @@ int coli_cuda_tensor_device(const ColiCudaTensor *tensor){return g_cuda.tensor_device(tensor);}+/* ---- #111 pipeline wrappers ---- */+++/* ---- #111 pipeline wrappers (see header for semantics) ---- */++int coli_cuda_attention_absorb_batch(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent,const float *rope,int S, int H,int Q,int R,int V,int K,int T, float attention_scale){+ if(!g_cuda.available){ return 0; }+ return g_cuda.attention_absorb_batch(kv_b, ctx, q, latent, rope, S, H, Q, R, V, K, T, attention_scale);+}++int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){+ if(!g_cuda.available){ return 0; }+ return g_cuda.attention_absorb_batch_dev(kv_b_shard, ctx_dev, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);+}++int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale){+ if(!g_cuda.available){ return 0; }+ return g_cuda.attention_absorb_kvdev(kv_b, ctx, q, latent_dev, rope_dev, H, Q, R, V, K, T, scale);+}++int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale){+ if(!g_cuda.available){ return 0; }+ return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale);+}++int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){+ if(!g_cuda.available){ return 0; }+ return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);+}++int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){+ if(!g_cuda.available){ return 0; }+ return g_cuda.attention_project_batch_dev_out(kv_b, o_proj, out_dev, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);+}++int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_add(device, x_dev, t_dev, n);+}++void * coli_cuda_pipe_alloc(int device,size_t bytes){+ if(!g_cuda.available){ return NULL; }+ return g_cuda.pipe_alloc(device, bytes);+}++int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_copy2d(device, dst, dpitch, src, spitch, width, height);+}++int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_download(device, src, dst, bytes);+}++void coli_cuda_pipe_free(int device,void *p){+ if(!g_cuda.available){ return; }+ g_cuda.pipe_free(device, p);+}++int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_gemm(t, y_dev, x_dev, S);+}++int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_peer_copy(dst_dev, dst, src_dev, src, bytes);+}++int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_rmsnorm(device, y_dev, x_dev, w_dev, S, D, eps);+}++int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_rmsnorm_s(device, y_dev, x_dev, w_dev, S, D, eps, xstride, ystride);+}++int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_rope(device, v_dev, pos_dev, rows, stride, offset, R, heads, theta);+}++int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_rope_base(device, v_dev, pos_base, rows, stride, offset, R, heads, theta);+}++int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_rows_add(device, x_dev, partial_dev, rows_dev, nrows, D);+}++float * coli_cuda_pipe_scratch(int device,int slot,size_t bytes){+ if(!g_cuda.available){ return NULL; }+ return g_cuda.pipe_scratch(device, slot, bytes);+}++int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_silu_mul(device, gate_dev, up_dev, n);+}++int coli_cuda_pipe_sync(int device){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_sync(device);+}++int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes){+ if(!g_cuda.available){ return 0; }+ return g_cuda.pipe_upload(device, dst, src, bytes);+}++int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S){+ if(!g_cuda.available){ return 0; }+ return g_cuda.shared_mlp_w4a16(gate, up, down, y, x, S);+}++int coli_cuda_tensor_update(ColiCudaTensor *tensor, const void *weights, const float *scales){+ if(!g_cuda.available){ return 0; }+ return g_cuda.tensor_update(tensor, weights, scales);+}+#endif /* _WIN32 */