Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

backend: coli_cuda_tensor_overwrite, new bytes into an uploaded tensor's own buffers

kreuzzelgc01b71351027
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index d1bd3efd..aa13675c 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -1556,6 +1556,26 @@ extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
return r;
}
+extern "C" int coli_cuda_tensor_overwrite(ColiCudaTensor *t, const void *weights, const float *scales) {
+ if (!t || !weights || !t->weights || !t->weights_owned) return 0;
+#ifdef COLI_ANS
+ if (t->compressed) return 0; /* archive bytes, not a weight buffer of this size */
+#endif
+ if (t->fmt && t->fmt != 6 && (!scales || !t->scales)) return 0;
+ DeviceContext *ctx = find_ctx(t->device);
+ if (!select_ctx(ctx)) return 0;
+ /* Same steps as the upload, into the buffers already there. A failure
+ * half-way leaves the tensor's content undefined: the caller frees it. */
+ if (!cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor overwrite")) return 0;
+ if (t->fmt==2||t->fmt==4) {
+ offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);
+ if (!cuda_ok(cudaGetLastError(),"int4 weight conversion")) return 0;
+ }
+ if (t->fmt && t->fmt != 6 &&
+ !cuda_ok(cudaMemcpy(t->scales, scales, tensor_scale_bytes(t), cudaMemcpyHostToDevice), "scale overwrite")) return 0;
+ return 1;
+}
+
#ifdef COLI_ANS
struct AnsSidecarHeader {
uint32_t magic,raw_bytes,archive_bytes,fmt,I,O;
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index 9ecf6976..e4aca5d3 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -161,6 +161,13 @@ COLI_CUDA_DLLEXPORT int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring
COLI_CUDA_DLLEXPORT int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec);
COLI_CUDA_DLLEXPORT int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,
const float *x, float *out, const float *egh, const float *beta);
+/* Overwrite an uploaded tensor in place with new bytes of the same shape and
+ * format. The expert tier swaps residents, and every expert of a model has the
+ * same geometry, so the victim's device buffers can take the newcomer without
+ * the cudaFree + cudaMalloc pair, which synchronises the device under the
+ * async expert groups. Optional in the DLL loader: 0 from a backend that
+ * predates it (or on any failure), and the caller frees and uploads as before. */
+COLI_CUDA_DLLEXPORT int coli_cuda_tensor_overwrite(ColiCudaTensor *tensor, const void *weights, const float *scales);
#ifdef COLI_ANS
/* Experimental Linux-only GPU-resident entropy tier. The archive remains in
* VRAM and is decoded into per-device scratch immediately before a grouped
diff --git a/c/backend_loader.c b/c/backend_loader.c
index f990135d..e45979dc 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -100,6 +100,7 @@ typedef void (*fn_dn_free)(ColiCudaDn *d);
typedef int (*fn_dn_set_state)(ColiCudaDn *d, const float *ring, const float *rec);
typedef int (*fn_dn_get_state)(ColiCudaDn *d, float *ring, float *rec);
typedef int (*fn_dn_step)(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta);
+typedef int (*fn_tensor_overwrite)(ColiCudaTensor *tensor, const void *weights, const float *scales);
typedef int (*fn_e8_set_grid)(const void *grid);
typedef int (*fn_fp8_set_lut)(const float *lut);
typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,
@@ -186,6 +187,7 @@ static struct {
fn_tensor_upload tensor_upload;
fn_tensor_upload_g tensor_upload_g;
fn_dn_create dn_create; fn_dn_free dn_free; fn_dn_set_state dn_set_state; fn_dn_get_state dn_get_state; fn_dn_step dn_step; /* optional: gated delta layer on the device */
+ fn_tensor_overwrite tensor_overwrite; /* optional: DLLs before it leave it NULL */
fn_e8_set_grid e8_set_grid;
fn_fp8_set_lut fp8_set_lut;
fn_matmul matmul;
@@ -1443,6 +1445,7 @@ static int coli_cuda_load(void){
RESOLVE_OPT(dn_set_state, fn_dn_set_state)
RESOLVE_OPT(dn_get_state, fn_dn_get_state)
RESOLVE_OPT(dn_step, fn_dn_step)
+ RESOLVE_OPT(tensor_overwrite, fn_tensor_overwrite)
RESOLVE_OPT(e8_set_grid, fn_e8_set_grid)
RESOLVE_OPT(fp8_set_lut, fn_fp8_set_lut)
RESOLVE(matmul, fn_matmul)
@@ -1664,6 +1667,11 @@ int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,
return d && g_cuda.dn_step ? g_cuda.dn_step(d, proj, outp, x, out, egh, beta) : 0;
}
+int coli_cuda_tensor_overwrite(ColiCudaTensor *tensor, const void *weights, const float *scales){
+ if(!g_cuda.available || !g_cuda.tensor_overwrite) return 0; /* older DLL: the tier frees and uploads instead */
+ return g_cuda.tensor_overwrite(tensor, weights, scales);
+}
+
int coli_cuda_e8_set_grid(const void *grid){
if(!g_cuda.available || !g_cuda.e8_set_grid) return 0; /* fmt=6 stays CPU-side */
return g_cuda.e8_set_grid(grid);
diff --git a/c/tests/qwen36_fake_cuda.h b/c/tests/qwen36_fake_cuda.h
index c1c5b9dd..b67705d9 100644
--- a/c/tests/qwen36_fake_cuda.h
+++ b/c/tests/qwen36_fake_cuda.h
@@ -32,6 +32,8 @@
struct ColiCudaTensor { int fmt, I, O, device, gs; const void *w; const float *sc; };
static int fake_uploads;
+static int fake_overwrites; /* coli_cuda_tensor_overwrite calls (in-place swaps) */
+static int fake_overwrite_fail; /* 1: refuse overwrites, as a backend without the symbol */
static int last_fmt = -1;
static size_t last_bytes;
static unsigned char captured[4096];
@@ -79,6 +81,15 @@ void coli_cuda_tensor_free(ColiCudaTensor *t) {
if (t && fake_dense_compute && t->fmt == 1) { free((void *)t->w); free((void *)t->sc); }
free(t);
}
+int coli_cuda_tensor_overwrite(ColiCudaTensor *t, const void *w, const float *sc) {
+ if (!t || !w || fake_overwrite_fail) return 0;
+ if (fake_upload_hook) fake_upload_hook(t->fmt);
+ if (fake_dense_compute && t->fmt == 1 && t->w && t->sc) {
+ memcpy((void *)t->w, w, (size_t)t->I * t->O); memcpy((void *)t->sc, sc, (size_t)t->O * sizeof(float));
+ } else { t->w = w; t->sc = sc; }
+ fake_overwrites++;
+ return 1;
+}
int coli_cuda_available_device_count(void) { return fake_ndev; }
int coli_cuda_device_count(void) { return fake_ndev; }
int coli_cuda_init(const int *d, int n) { (void)d; (void)n; return 1; }