Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#ifndef COLIBRI_BACKEND_CUDA_H
#define COLIBRI_BACKEND_CUDA_H
#include <stddef.h>
#include <stdint.h>
/* COLI_CUDA_DLLEXPORT marks functions exported from coli_cuda.dll on Windows.
* Define COLI_CUDA_BUILDING_DLL when compiling the .cu into the DLL (so the
* functions are __declspec(dllexport)); the host loader does NOT include this
* header's declarations — it resolves symbols at runtime via GetProcAddress. */
#if defined(_WIN32) && defined(COLI_CUDA_BUILDING_DLL)
#define COLI_CUDA_DLLEXPORT __declspec(dllexport)
#else
#define COLI_CUDA_DLLEXPORT
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define COLI_CUDA_MAX_DEVICES 16
/* Weight formats the generic per-element device decoder (weight_at,
* backend_cuda.cu) can actually decode: f32, int8-row, int4 nibbles (fmt=2 and
* the grouped fmt=4, same packing), int2, and fmt=8 (fp8-e4m3 raw bytes,
* decoded through the c_e4m3 LUT -- absorb-path support; absorb_scale supplies
* its per-128x128-block scale). Nothing else.
*
* WHY THIS IS A PREDICATE AND NOT A COMMENT. weight_at used to END in the int2
* decode as an unguarded fall-through, so ANY other format handed to it -- a
* format with a different element width, a different scale geometry, or no
* device decoder at all -- was silently read as 2-bit values and returned
* plausible-looking numbers. The CPU twins refuse the same input loudly:
* qt_addrow and qt_matvec_rows (colibri.c) both exit(1) naming the function and
* the fmt. Two backends given identical unsupported input, one refusing and one
* fabricating, is the defect -- not the missing decoder.
*
* Defined here, in the host header, rather than inside the .cu: the host gates
* that keep unsupported formats off the device (absorb_fmt_ok and friends) and
* the device-side backstop must agree by construction rather than by two people
* writing the same list twice, and a CPU test build can then unit-test the truth
* table without a GPU or a CUDA toolchain (tests/test_cuda_fmt_guard.c) -- the
* same arrangement colibri.c uses for metal_fused_fmt_ok.
*
* NOT a statement about which formats the CUDA BACKEND supports: quant_matmul
* has its own explicit branches for fmt=6 (E8/IQ3) and fmt=7 (MXFP4) that
* never route through weight_at (and its own fmt=8 branch for the dense path
* -- weight_at's fmt=8 branch serves the absorb kernels, which share the same
* c_e4m3 LUT). This predicate is scoped to weight_at's own dispatch, which is
* what the absorb and grouped-expert kernels decode through.
*
* fmt=8 CAVEAT, stated because the truth table alone cannot carry it: a fmt=8
* decode additionally requires the e4m3 LUT to have been published to the
* configured devices (coli_cuda_fp8_set_lut). The exact mechanism, so the
* claim cannot outrun it: coli_cuda_fp8_set_lut copies the table into every
* context live AT CALL TIME and sets a process-wide flag; the flag gates
* fmt=8 uploads (coli_cuda_tensor_upload refuses until it is set).
* coli_cuda_shutdown clears the flag; coli_cuda_init never writes it. That is
* enough because init will not rebuild contexts underneath a live set: a
* re-init naming the same device set returns success and leaves the contexts,
* and the table published to them, untouched, while one naming a different
* device set is refused before any context is rebuilt. So the device set
* cannot widen past what the last publish covered without going through
* shutdown, and no fmt=8 ColiCudaTensor can reach a kernel whose device has
* an unwritten table. This predicate
* deliberately does not restate that gate: it answers "does weight_at have a
* decode branch for this fmt", which is the question the launch-site gates
* and the device-side __trap() backstop share. */
static inline int coli_cuda_weight_at_supported(int fmt) {
return fmt == 0 || fmt == 1 || fmt == 2 || fmt == 3 || fmt == 4 || fmt == 8;
}
/* The two decisions the fmt=8 LUT gate rests on, as pure predicates. They live
* here rather than inline in backend_cuda.cu so a host-side test can pin them
* with no CUDA toolchain and no GPU (tests/test_cuda_lut_gate.c). backend_cuda.cu
* calls BOTH at the real decision sites, so the test pins the engine's own
* logic rather than a second copy that could drift from it -- which is the
* failure this factoring exists to prevent, the gate having no CI reach
* otherwise. */
/* Does the upload gate admit this tensor? Only fmt=8 needs the published
* table; every other format decodes without one. */
static inline int coli_cuda_fp8_gate_admits(int fmt, int lut_ready) {
return fmt != 8 || lut_ready != 0;
}
/* What coli_cuda_init must do with a request while a device set may be live.
* BUILD: nothing is live, build the contexts. ACCEPT: the same set is already
* live -- return success and touch nothing, so the table published to those
* contexts stays valid. REFUSE: a different set is live -- refuse before
* rebuilding anything, so the set cannot widen past the last publish. */
enum { COLI_CUDA_INIT_BUILD = 0, COLI_CUDA_INIT_ACCEPT = 1, COLI_CUDA_INIT_REFUSE = -1 };
static inline int coli_cuda_init_disposition(int nctx, int count,
const int *want, const int *live) {
int i;
if (nctx <= 0) return COLI_CUDA_INIT_BUILD;
if (count != nctx) return COLI_CUDA_INIT_REFUSE;
for (i = 0; i < count; i++) if (want[i] != live[i]) return COLI_CUDA_INIT_REFUSE;
return COLI_CUDA_INIT_ACCEPT;
}
/* Opaque, persistent device copy of one resident quantized tensor. */
typedef struct ColiCudaTensor ColiCudaTensor;
/* Devices are CUDA ordinals, not positions in the input list.
* Repeating the same ordered list preserves active contexts. Changing an
* active list returns 0 without replacing it; release tensors and shut down
* before selecting a different list. Init/shutdown require caller serialization. */
COLI_CUDA_DLLEXPORT int coli_cuda_init(const int *devices, int count);
COLI_CUDA_DLLEXPORT void coli_cuda_shutdown(void);
/* Number of CUDA devices visible to this process, before a device list is
* selected. Returns 0 when the runtime cannot discover any device. */
COLI_CUDA_DLLEXPORT int coli_cuda_available_device_count(void);
COLI_CUDA_DLLEXPORT int coli_cuda_device_count(void);
COLI_CUDA_DLLEXPORT int coli_cuda_device_at(int index);
COLI_CUDA_DLLEXPORT int coli_cuda_mem_info(int device, size_t *free_bytes, size_t *total_bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_device_integrated(int device);
/* device < 0 returns aggregate statistics for all configured devices. */
COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
double *h2d_ms, double *kernel_ms, double *d2h_ms);
/* Per-device form of coli_cuda_group_stats; unknown devices return zeros. */
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats_device(
int device, uint64_t *calls, uint64_t *experts, uint64_t *rows,
double *h2d_ms, double *kernel_ms, double *d2h_ms);
/* Publish the E8 codebook (quant.h's e8_grid, 256x4 bytes) to every configured
* device. Must be called after coli_cuda_init and before any fmt=6 upload; the
* backend keeps no copy of the table so it cannot drift from the CPU decoder. */
COLI_CUDA_DLLEXPORT int coli_cuda_e8_set_grid(const void *grid);
/* Publish the fmt=8 e4m3 decode table (quant.h's E4M3_LUT, 256 f32) the same
* way. Must be called after coli_cuda_init; fmt=8 uploads are refused until it
* succeeds, because kernels would decode against a zero-initialized table. */
COLI_CUDA_DLLEXPORT int coli_cuda_fp8_set_lut(const float *lut);
/* Upload without executing, so capacity failures happen during model startup. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
const void *weights, const float *scales,
int fmt, int I, int O, int device, int gs);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
const void *weights, const float *scales,
int fmt, int I, int O, int device);
/* Gated delta layer on the device (the linear-attention layers of Qwen3.6/3.8,
* Kimi K3's KDA, GLM-5.3-Flash). One object per layer holds the short-conv ring
* [conv_dim x (convk-1)], the recurrent state [vh x kdim x vdim], the conv and
* norm weights and the scratch for one token. coli_cuda_dn_step runs a whole
* decode step on the card: x (host) -> in_proj GEMV -> causal conv + SiLU ->
* per head L2 norm, decay, delta rule, gated RMSNorm -> out_proj GEMV -> out
* (host). Only x and out cross the bus; the state never does. Gates (egh =
* exp(g) per value head, beta) arrive by value. All optional in the DLL
* loader: a backend without them leaves the layer on the CPU. */
typedef struct ColiCudaDn ColiCudaDn;
COLI_CUDA_DLLEXPORT ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim,
int conv_dim, int convk, int hidden,
const float *conv_w, const float *norm_w, float eps, int gate_sigmoid);
COLI_CUDA_DLLEXPORT void coli_cuda_dn_free(ColiCudaDn *d);
COLI_CUDA_DLLEXPORT int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec); /* NULL = zero */
COLI_CUDA_DLLEXPORT int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec);
/* proj: the in_proj rows. Fused qkv ++ z in one matrix (projz NULL, qwen36's
* dnproj) or qkv alone with z in projz (qwen38's dnqkv / dnz). gate_sigmoid at
* create: the gated RMSNorm multiplies by silu(z) (0, Qwen3.6) or sigmoid(z)
* (1, Qwen3.8). */
COLI_CUDA_DLLEXPORT int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp,
const float *x, float *out, const float *egh, const float *beta);
/* Overwrite an uploaded tensor in place with new bytes of the same shape and
* format. The expert tier swaps residents, and every expert of a model has the
* same geometry, so the victim's device buffers can take the newcomer without
* the cudaFree + cudaMalloc pair, which synchronises the device under the
* async expert groups. Optional in the DLL loader: 0 from a backend that
* predates it (or on any failure), and the caller frees and uploads as before. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_overwrite(ColiCudaTensor *tensor, const void *weights, const float *scales);
#ifdef COLI_ANS
/* Experimental Linux-only GPU-resident entropy tier. The archive remains in
* VRAM and is decoded into per-device scratch immediately before a grouped
* expert launch. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tensor,
const void *weights, const float *scales,
int fmt, int I, int O, int device);
#endif
/*
* y[S,O] = x[S,I] @ W[O,I]^T.
* fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2, 4=grouped int4.
* gs is the group size for fmt=4 (0 for all other formats).
* The first successful call uploads W and its scales; later calls reuse it.
* Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
*/
/* y[S,O] = x[S,I] @ dequant_mxfp4(W[O,I])^T, fmt=7 (OCP microscaling FP4).
* q4 is [O, ceil(I/2)] e2m1 nibbles, e8s is [O, ceil(I/32)] ue8m0 exponents --
* BYTES, not floats, which is why this is not folded into coli_cuda_matmul.
* Stateless: weights are uploaded per call, matching the streaming expert tier
* Kimi K3 uses. Returns 1 on success, 0 (y untouched) to fall back to CPU. */
COLI_CUDA_DLLEXPORT int coli_cuda_matmul_mxfp4(float *y, const float *x,
const unsigned char *q4,
const unsigned char *e8s,
int S, int I, int O);
/* Streaming Kimi expert: down(SiTU(gate(x), up(x))). Weights are MXFP4
* host buffers; intermediate activations remain on device. No weight cache.
* Returns 0 on failure; callers must accumulate y only after success. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_mxfp4(float *y, const float *x,
const unsigned char *gate_w, const unsigned char *gate_s,
const unsigned char *up_w, const unsigned char *up_s,
const unsigned char *down_w, const unsigned char *down_s,
int S, int D, int I, float b1, float b2);
COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,
float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int device, int gs);
/* Fused expert pipeline: y = down(silu(gate(x)) * up(x)). All three tensors
* must already be resident on one device. Activations cross PCIe once in
* each direction instead of once per matrix. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y, const float *x, int S);
/* Prefill-oriented shared expert path. INT4 weights stay packed in global
* memory, activations are converted to FP16 per tile, and Tensor Cores
* accumulate into FP32. Unlike COLI_CUDA_TC_INT4 this does not quantize the
* activation to INT4. */
COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y,
const float *x, int S);
/* Packed group of same-shaped experts. Inputs and outputs contain sum(rows)
* consecutive [D] rows in call order. */
/* Async issue/take split of the group call below (Inc.4): issue launches on the
* device stream and returns; take syncs and returns the pinned result rows (valid
* until the next issue on that device). Small totals only (<=8 rows); one
* outstanding issue per device. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
ColiCudaTensor *const *ups,
ColiCudaTensor *const *downs,
const int *rows, int count, const float *x);
COLI_CUDA_DLLEXPORT const float *coli_cuda_expert_group_take(int device);
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group(ColiCudaTensor *const *gates,
ColiCudaTensor *const *ups,
ColiCudaTensor *const *downs,
const int *rows, int count,
float *y, const float *x);
/* Same operation, but force the small-batch grouped kernel family when
* pin_small_batch is nonzero. Speculative verification uses this to keep
* CUDA on the same numeric family as S=1 regardless of accepted draft depth. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_pinned(ColiCudaTensor *const *gates,
ColiCudaTensor *const *ups,
ColiCudaTensor *const *downs,
const int *rows, int count,
float *y, const float *x, int pin_small_batch);
/* Decode-only MLA weight-absorption core for one token. kv_b is [H*(Q+V),K]. */
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb(ColiCudaTensor *kv_b,float *ctx,const float *q,
const float *latent,const float *rope,int H,int Q,
int R,int V,int K,int T,float attention_scale);
/* Causal MLA absorption for S contiguous rows from one sequence. The KV
* arrays contain T rows ending at the final query; query s attends T-S+s+1
* rows. One transfer and one launch replace S host round-trips. */
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch(ColiCudaTensor *kv_b,float *ctx,const float *q,
const float *latent,const float *rope,int S,
int H,int Q,int R,int V,int K,int T,
float attention_scale);
/* Same attention batch followed immediately by resident o_proj on the same
* device. Only the final [S,D] tensor crosses back to the host. */
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q,const float *latent,
const float *rope,int S,int H,int Q,int R,
int V,int K,int T,float attention_scale);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q,const void *const *keys,
const float *const *latent,const float *const *rope,
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);
COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT size_t coli_cuda_alloc_footprint(size_t bytes);
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
/* Replace a resident tensor's contents without reallocating its device slot. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_update(ColiCudaTensor *tensor,
const void *weights, const float *scales);
/* ---- resident-pipeline primitives (Inc.0): device-pointer entry points ---- */
COLI_CUDA_DLLEXPORT float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);
COLI_CUDA_DLLEXPORT void *coli_cuda_pipe_alloc(int device,size_t bytes);
COLI_CUDA_DLLEXPORT void coli_cuda_pipe_free(int device,void *p);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,
const float *w_dev,int S,int D,float eps);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,
int stride,int offset,int R,int heads,float theta);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,
const int *rows_dev,int nrows,int D);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,
const float *w_dev,int S,int D,float eps,
int xstride,int ystride);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,
int stride,int offset,int R,int heads,float theta);
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,
ColiCudaTensor *const *ups, ColiCudaTensor *const *downs,
const float *weights, int count,
int home_device, const float *x_src_dev, float *partial_slot_dev);
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_take(int home_device,const int *devices,
int n_issued,float *slots_dev,float *acc_dev,int D);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_router(int device,const float *x_dev,
const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,
float topp,int norm_topk,float routed_scale,
int *idx_host,float *w_host,int *keff_host);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,
int spitch,int width,int height);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out,const float *q_dev,const float *latent_dev,const float *rope_dev,
int S,int H,int Q,int R,int V,int K,int T,float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,
const float *q_dev,const float *latent_dev,const float *rope_dev,
int S,int H,int Q,int R,int V,int K,int T,float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,
const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T,
float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,
const float *src,size_t bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev,
int S,int H,int Q,int R,int V,int K,int T,float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);
#ifdef __cplusplus
}
#endif
#endif