Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda(win): export + load the #111 pipeline ABI — 24 missing DLLEXPORT decorations + loader wrappers, unbreaks CUDA_DLL=1 (#201)

Tom Olorin21cbf14d591e
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index c84a265a..acbe4bc8 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -60,7 +60,7 @@ COLI_CUDA_DLLEXPORT int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTenso
* memory, activations are converted to FP16 per tile, and Tensor Cores
* accumulate into FP32. Unlike COLI_CUDA_TC_INT4 this does not quantize the
* activation to INT4. */
-int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,
+COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y,
const float *x, int S);
@@ -97,46 +97,46 @@ COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
/* Replace a resident tensor's contents without reallocating its device slot. */
-int coli_cuda_tensor_update(ColiCudaTensor *tensor,
+COLI_CUDA_DLLEXPORT int coli_cuda_tensor_update(ColiCudaTensor *tensor,
const void *weights, const float *scales);
/* ---- resident-pipeline primitives (Inc.0): device-pointer entry points ---- */
-float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);
-void *coli_cuda_pipe_alloc(int device,size_t bytes);
-void coli_cuda_pipe_free(int device,void *p);
-int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);
-int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);
-int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,
+COLI_CUDA_DLLEXPORT float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);
+COLI_CUDA_DLLEXPORT void *coli_cuda_pipe_alloc(int device,size_t bytes);
+COLI_CUDA_DLLEXPORT void coli_cuda_pipe_free(int device,void *p);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,
const float *w_dev,int S,int D,float eps);
-int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,
int stride,int offset,int R,int heads,float theta);
-int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);
-int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);
-int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,
const int *rows_dev,int nrows,int D);
-int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);
-int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,
const float *w_dev,int S,int D,float eps,
int xstride,int ystride);
-int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,
int stride,int offset,int R,int heads,float theta);
-int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,
int spitch,int width,int height);
-int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
+COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q_dev,const float *latent_dev,const float *rope_dev,
int S,int H,int Q,int R,int V,int K,int T,float scale);
-int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,
+COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,
const float *q_dev,const float *latent_dev,const float *rope_dev,
int S,int H,int Q,int R,int V,int K,int T,float scale);
-int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,
+COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,
const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T,
float scale);
-int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,
const float *src,size_t bytes);
-int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
+COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev,
int S,int H,int Q,int R,int V,int K,int T,float scale);
-int coli_cuda_pipe_sync(int device);
+COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);
#ifdef __cplusplus
}
diff --git a/c/backend_loader.c b/c/backend_loader.c
index 6fea70c1..0635e081 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -52,6 +52,35 @@ typedef void (*fn_tensor_free)(ColiCudaTensor *tensor);
typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor);
typedef int (*fn_tensor_device)(const ColiCudaTensor *tensor);
+/* --- #111 GPU resident pipeline additions (matched to backend_cuda.h) --- */
+
+
+/* --- #111 GPU resident pipeline additions (matched to backend_cuda.h) --- */
+typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent,const float *rope,int S, int H,int Q,int R,int V,int K,int T, float attention_scale);
+typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
+typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale);
+typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale);
+typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
+typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
+typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n);
+typedef void * (*fn_pipe_alloc)(int device,size_t bytes);
+typedef int (*fn_pipe_copy2d)(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height);
+typedef int (*fn_pipe_download)(int device,const void *src,void *dst,size_t bytes);
+typedef void (*fn_pipe_free)(int device,void *p);
+typedef int (*fn_pipe_gemm)(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);
+typedef int (*fn_pipe_peer_copy)(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes);
+typedef int (*fn_pipe_rmsnorm)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps);
+typedef int (*fn_pipe_rmsnorm_s)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride);
+typedef int (*fn_pipe_rope)(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta);
+typedef int (*fn_pipe_rope_base)(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta);
+typedef int (*fn_pipe_rows_add)(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D);
+typedef float * (*fn_pipe_scratch)(int device,int slot,size_t bytes);
+typedef int (*fn_pipe_silu_mul)(int device,float *gate_dev,const float *up_dev,size_t n);
+typedef int (*fn_pipe_sync)(int device);
+typedef int (*fn_pipe_upload)(int device,void *dst,const void *src,size_t bytes);
+typedef int (*fn_shared_mlp_w4a16)(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S);
+typedef int (*fn_tensor_update)(ColiCudaTensor *tensor, const void *weights, const float *scales);
+
/* Resolved pointers, plus a flag so we attempt the load at most once. */
static struct {
int loaded; /* 1 = load attempted (success or fail), 0 = not yet */
@@ -72,6 +101,31 @@ static struct {
fn_tensor_free tensor_free;
fn_tensor_bytes tensor_bytes;
fn_tensor_device tensor_device;
+
+ fn_attention_absorb_batch attention_absorb_batch;
+ fn_attention_absorb_batch_dev attention_absorb_batch_dev;
+ fn_attention_absorb_kvdev attention_absorb_kvdev;
+ fn_attention_project_batch attention_project_batch;
+ fn_attention_project_batch_dev attention_project_batch_dev;
+ fn_attention_project_batch_dev_out attention_project_batch_dev_out;
+ fn_pipe_add pipe_add;
+ fn_pipe_alloc pipe_alloc;
+ fn_pipe_copy2d pipe_copy2d;
+ fn_pipe_download pipe_download;
+ fn_pipe_free pipe_free;
+ fn_pipe_gemm pipe_gemm;
+ fn_pipe_peer_copy pipe_peer_copy;
+ fn_pipe_rmsnorm pipe_rmsnorm;
+ fn_pipe_rmsnorm_s pipe_rmsnorm_s;
+ fn_pipe_rope pipe_rope;
+ fn_pipe_rope_base pipe_rope_base;
+ fn_pipe_rows_add pipe_rows_add;
+ fn_pipe_scratch pipe_scratch;
+ fn_pipe_silu_mul pipe_silu_mul;
+ fn_pipe_sync pipe_sync;
+ fn_pipe_upload pipe_upload;
+ fn_shared_mlp_w4a16 shared_mlp_w4a16;
+ fn_tensor_update tensor_update;
} g_cuda;
/* Resolve the DLL and all 11 symbols. Returns 1 on success, 0 otherwise.
@@ -119,6 +173,31 @@ static int coli_cuda_load(void){
RESOLVE(tensor_free, fn_tensor_free)
RESOLVE(tensor_bytes, fn_tensor_bytes)
RESOLVE(tensor_device, fn_tensor_device)
+
+ RESOLVE(attention_absorb_batch, fn_attention_absorb_batch)
+ RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev)
+ RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev)
+ RESOLVE(attention_project_batch, fn_attention_project_batch)
+ RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev)
+ RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out)
+ RESOLVE(pipe_add, fn_pipe_add)
+ RESOLVE(pipe_alloc, fn_pipe_alloc)
+ RESOLVE(pipe_copy2d, fn_pipe_copy2d)
+ RESOLVE(pipe_download, fn_pipe_download)
+ RESOLVE(pipe_free, fn_pipe_free)
+ RESOLVE(pipe_gemm, fn_pipe_gemm)
+ RESOLVE(pipe_peer_copy, fn_pipe_peer_copy)
+ RESOLVE(pipe_rmsnorm, fn_pipe_rmsnorm)
+ RESOLVE(pipe_rmsnorm_s, fn_pipe_rmsnorm_s)
+ RESOLVE(pipe_rope, fn_pipe_rope)
+ RESOLVE(pipe_rope_base, fn_pipe_rope_base)
+ RESOLVE(pipe_rows_add, fn_pipe_rows_add)
+ RESOLVE(pipe_scratch, fn_pipe_scratch)
+ RESOLVE(pipe_silu_mul, fn_pipe_silu_mul)
+ RESOLVE(pipe_sync, fn_pipe_sync)
+ RESOLVE(pipe_upload, fn_pipe_upload)
+ RESOLVE(shared_mlp_w4a16, fn_shared_mlp_w4a16)
+ RESOLVE(tensor_update, fn_tensor_update)
#undef RESOLVE
g_cuda.available = 1;
@@ -216,4 +295,129 @@ int coli_cuda_tensor_device(const ColiCudaTensor *tensor){
return g_cuda.tensor_device(tensor);
}
+/* ---- #111 pipeline wrappers ---- */
+
+
+/* ---- #111 pipeline wrappers (see header for semantics) ---- */
+
+int coli_cuda_attention_absorb_batch(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent,const float *rope,int S, int H,int Q,int R,int V,int K,int T, float attention_scale){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.attention_absorb_batch(kv_b, ctx, q, latent, rope, S, H, Q, R, V, K, T, attention_scale);
+}
+
+int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.attention_absorb_batch_dev(kv_b_shard, ctx_dev, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);
+}
+
+int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.attention_absorb_kvdev(kv_b, ctx, q, latent_dev, rope_dev, H, Q, R, V, K, T, scale);
+}
+
+int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale);
+}
+
+int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);
+}
+
+int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.attention_project_batch_dev_out(kv_b, o_proj, out_dev, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);
+}
+
+int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_add(device, x_dev, t_dev, n);
+}
+
+void * coli_cuda_pipe_alloc(int device,size_t bytes){
+ if(!g_cuda.available){ return NULL; }
+ return g_cuda.pipe_alloc(device, bytes);
+}
+
+int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_copy2d(device, dst, dpitch, src, spitch, width, height);
+}
+
+int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_download(device, src, dst, bytes);
+}
+
+void coli_cuda_pipe_free(int device,void *p){
+ if(!g_cuda.available){ return; }
+ g_cuda.pipe_free(device, p);
+}
+
+int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_gemm(t, y_dev, x_dev, S);
+}
+
+int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_peer_copy(dst_dev, dst, src_dev, src, bytes);
+}
+
+int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_rmsnorm(device, y_dev, x_dev, w_dev, S, D, eps);
+}
+
+int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_rmsnorm_s(device, y_dev, x_dev, w_dev, S, D, eps, xstride, ystride);
+}
+
+int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_rope(device, v_dev, pos_dev, rows, stride, offset, R, heads, theta);
+}
+
+int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_rope_base(device, v_dev, pos_base, rows, stride, offset, R, heads, theta);
+}
+
+int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_rows_add(device, x_dev, partial_dev, rows_dev, nrows, D);
+}
+
+float * coli_cuda_pipe_scratch(int device,int slot,size_t bytes){
+ if(!g_cuda.available){ return NULL; }
+ return g_cuda.pipe_scratch(device, slot, bytes);
+}
+
+int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_silu_mul(device, gate_dev, up_dev, n);
+}
+
+int coli_cuda_pipe_sync(int device){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_sync(device);
+}
+
+int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.pipe_upload(device, dst, src, bytes);
+}
+
+int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.shared_mlp_w4a16(gate, up, down, y, x, S);
+}
+
+int coli_cuda_tensor_update(ColiCudaTensor *tensor, const void *weights, const float *scales){
+ if(!g_cuda.available){ return 0; }
+ return g_cuda.tensor_update(tensor, weights, scales);
+}
+
#endif /* _WIN32 */