Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
backend: coli_cuda_tensor_overwrite, new bytes into an uploaded tensor's own buffers
kreuzzelgc01b71351027
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex d1bd3efd..aa13675c 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -1556,6 +1556,26 @@ extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,return r;}+extern "C" int coli_cuda_tensor_overwrite(ColiCudaTensor *t, const void *weights, const float *scales) {+ if (!t || !weights || !t->weights || !t->weights_owned) return 0;+#ifdef COLI_ANS+ if (t->compressed) return 0; /* archive bytes, not a weight buffer of this size */+#endif+ if (t->fmt && t->fmt != 6 && (!scales || !t->scales)) return 0;+ DeviceContext *ctx = find_ctx(t->device);+ if (!select_ctx(ctx)) return 0;+ /* Same steps as the upload, into the buffers already there. A failure+ * half-way leaves the tensor's content undefined: the caller frees it. */+ if (!cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor overwrite")) return 0;+ if (t->fmt==2||t->fmt==4) {+ offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);+ if (!cuda_ok(cudaGetLastError(),"int4 weight conversion")) return 0;+ }+ if (t->fmt && t->fmt != 6 &&+ !cuda_ok(cudaMemcpy(t->scales, scales, tensor_scale_bytes(t), cudaMemcpyHostToDevice), "scale overwrite")) return 0;+ return 1;+}+#ifdef COLI_ANSstruct AnsSidecarHeader {uint32_t magic,raw_bytes,archive_bytes,fmt,I,O;diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex 9ecf6976..e4aca5d3 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -161,6 +161,13 @@ COLI_CUDA_DLLEXPORT int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ringCOLI_CUDA_DLLEXPORT int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec);COLI_CUDA_DLLEXPORT int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,const float *x, float *out, const float *egh, const float *beta);+/* Overwrite an uploaded tensor in place with new bytes of the same shape and+ * format. The expert tier swaps residents, and every expert of a model has the+ * same geometry, so the victim's device buffers can take the newcomer without+ * the cudaFree + cudaMalloc pair, which synchronises the device under the+ * async expert groups. Optional in the DLL loader: 0 from a backend that+ * predates it (or on any failure), and the caller frees and uploads as before. */+COLI_CUDA_DLLEXPORT int coli_cuda_tensor_overwrite(ColiCudaTensor *tensor, const void *weights, const float *scales);#ifdef COLI_ANS/* Experimental Linux-only GPU-resident entropy tier. The archive remains in* VRAM and is decoded into per-device scratch immediately before a groupeddiff --git a/c/backend_loader.c b/c/backend_loader.cindex f990135d..e45979dc 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -100,6 +100,7 @@ typedef void (*fn_dn_free)(ColiCudaDn *d);typedef int (*fn_dn_set_state)(ColiCudaDn *d, const float *ring, const float *rec);typedef int (*fn_dn_get_state)(ColiCudaDn *d, float *ring, float *rec);typedef int (*fn_dn_step)(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta);+typedef int (*fn_tensor_overwrite)(ColiCudaTensor *tensor, const void *weights, const float *scales);typedef int (*fn_e8_set_grid)(const void *grid);typedef int (*fn_fp8_set_lut)(const float *lut);typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,@@ -186,6 +187,7 @@ static struct {fn_tensor_upload tensor_upload;fn_tensor_upload_g tensor_upload_g;fn_dn_create dn_create; fn_dn_free dn_free; fn_dn_set_state dn_set_state; fn_dn_get_state dn_get_state; fn_dn_step dn_step; /* optional: gated delta layer on the device */+ fn_tensor_overwrite tensor_overwrite; /* optional: DLLs before it leave it NULL */fn_e8_set_grid e8_set_grid;fn_fp8_set_lut fp8_set_lut;fn_matmul matmul;@@ -1443,6 +1445,7 @@ static int coli_cuda_load(void){RESOLVE_OPT(dn_set_state, fn_dn_set_state)RESOLVE_OPT(dn_get_state, fn_dn_get_state)RESOLVE_OPT(dn_step, fn_dn_step)+ RESOLVE_OPT(tensor_overwrite, fn_tensor_overwrite)RESOLVE_OPT(e8_set_grid, fn_e8_set_grid)RESOLVE_OPT(fp8_set_lut, fn_fp8_set_lut)RESOLVE(matmul, fn_matmul)@@ -1664,6 +1667,11 @@ int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,return d && g_cuda.dn_step ? g_cuda.dn_step(d, proj, outp, x, out, egh, beta) : 0;}+int coli_cuda_tensor_overwrite(ColiCudaTensor *tensor, const void *weights, const float *scales){+ if(!g_cuda.available || !g_cuda.tensor_overwrite) return 0; /* older DLL: the tier frees and uploads instead */+ return g_cuda.tensor_overwrite(tensor, weights, scales);+}+int coli_cuda_e8_set_grid(const void *grid){if(!g_cuda.available || !g_cuda.e8_set_grid) return 0; /* fmt=6 stays CPU-side */return g_cuda.e8_set_grid(grid);diff --git a/c/tests/qwen36_fake_cuda.h b/c/tests/qwen36_fake_cuda.hindex c1c5b9dd..b67705d9 100644--- a/c/tests/qwen36_fake_cuda.h+++ b/c/tests/qwen36_fake_cuda.h@@ -32,6 +32,8 @@struct ColiCudaTensor { int fmt, I, O, device, gs; const void *w; const float *sc; };static int fake_uploads;+static int fake_overwrites; /* coli_cuda_tensor_overwrite calls (in-place swaps) */+static int fake_overwrite_fail; /* 1: refuse overwrites, as a backend without the symbol */static int last_fmt = -1;static size_t last_bytes;static unsigned char captured[4096];@@ -79,6 +81,15 @@ void coli_cuda_tensor_free(ColiCudaTensor *t) {if (t && fake_dense_compute && t->fmt == 1) { free((void *)t->w); free((void *)t->sc); }free(t);}+int coli_cuda_tensor_overwrite(ColiCudaTensor *t, const void *w, const float *sc) {+ if (!t || !w || fake_overwrite_fail) return 0;+ if (fake_upload_hook) fake_upload_hook(t->fmt);+ if (fake_dense_compute && t->fmt == 1 && t->w && t->sc) {+ memcpy((void *)t->w, w, (size_t)t->I * t->O); memcpy((void *)t->sc, sc, (size_t)t->O * sizeof(float));+ } else { t->w = w; t->sc = sc; }+ fake_overwrites++;+ return 1;+}int coli_cuda_available_device_count(void) { return fake_ndev; }int coli_cuda_device_count(void) { return fake_ndev; }int coli_cuda_init(const int *d, int n) { (void)d; (void)n; return 1; }