Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
20.7 KB
Raw Blame
#ifndef COLIBRI_BACKEND_CUDA_H
#define COLIBRI_BACKEND_CUDA_H

#include <stddef.h>
#include <stdint.h>

/* COLI_CUDA_DLLEXPORT marks functions exported from coli_cuda.dll on Windows.
 * Define COLI_CUDA_BUILDING_DLL when compiling the .cu into the DLL (so the
 * functions are __declspec(dllexport)); the host loader does NOT include this
 * header's declarations — it resolves symbols at runtime via GetProcAddress. */
#if defined(_WIN32) && defined(COLI_CUDA_BUILDING_DLL)
#define COLI_CUDA_DLLEXPORT __declspec(dllexport)
#else
#define COLI_CUDA_DLLEXPORT
#endif


#ifdef __cplusplus
extern "C" {
#endif

#define COLI_CUDA_MAX_DEVICES 16

/* Weight formats the generic per-element device decoder (weight_at,
 * backend_cuda.cu) can actually decode: f32, int8-row, int4 nibbles (fmt=2 and
 * the grouped fmt=4, same packing), int2, and fmt=8 (fp8-e4m3 raw bytes,
 * decoded through the c_e4m3 LUT -- absorb-path support; absorb_scale supplies
 * its per-128x128-block scale). Nothing else.
 *
 * WHY THIS IS A PREDICATE AND NOT A COMMENT. weight_at used to END in the int2
 * decode as an unguarded fall-through, so ANY other format handed to it -- a
 * format with a different element width, a different scale geometry, or no
 * device decoder at all -- was silently read as 2-bit values and returned
 * plausible-looking numbers. The CPU twins refuse the same input loudly:
 * qt_addrow and qt_matvec_rows (colibri.c) both exit(1) naming the function and
 * the fmt. Two backends given identical unsupported input, one refusing and one
 * fabricating, is the defect -- not the missing decoder.
 *
 * Defined here, in the host header, rather than inside the .cu: the host gates
 * that keep unsupported formats off the device (absorb_fmt_ok and friends) and
 * the device-side backstop must agree by construction rather than by two people
 * writing the same list twice, and a CPU test build can then unit-test the truth
 * table without a GPU or a CUDA toolchain (tests/test_cuda_fmt_guard.c) -- the
 * same arrangement colibri.c uses for metal_fused_fmt_ok.
 *
 * NOT a statement about which formats the CUDA BACKEND supports: quant_matmul
 * has its own explicit branches for fmt=6 (E8/IQ3) and fmt=7 (MXFP4) that
 * never route through weight_at (and its own fmt=8 branch for the dense path
 * -- weight_at's fmt=8 branch serves the absorb kernels, which share the same
 * c_e4m3 LUT). This predicate is scoped to weight_at's own dispatch, which is
 * what the absorb and grouped-expert kernels decode through.
 *
 * fmt=8 CAVEAT, stated because the truth table alone cannot carry it: a fmt=8
 * decode additionally requires the e4m3 LUT to have been published to the
 * configured devices (coli_cuda_fp8_set_lut). The exact mechanism, so the
 * claim cannot outrun it: coli_cuda_fp8_set_lut copies the table into every
 * context live AT CALL TIME and sets a process-wide flag; the flag gates
 * fmt=8 uploads (coli_cuda_tensor_upload refuses until it is set).
 * coli_cuda_shutdown clears the flag; coli_cuda_init never writes it. That is
 * enough because init will not rebuild contexts underneath a live set: a
 * re-init naming the same device set returns success and leaves the contexts,
 * and the table published to them, untouched, while one naming a different
 * device set is refused before any context is rebuilt. So the device set
 * cannot widen past what the last publish covered without going through
 * shutdown, and no fmt=8 ColiCudaTensor can reach a kernel whose device has
 * an unwritten table. This predicate
 * deliberately does not restate that gate: it answers "does weight_at have a
 * decode branch for this fmt", which is the question the launch-site gates
 * and the device-side __trap() backstop share. */
static inline int coli_cuda_weight_at_supported(int fmt) {
    return fmt == 0 || fmt == 1 || fmt == 2 || fmt == 3 || fmt == 4 || fmt == 8;
}

/* The two decisions the fmt=8 LUT gate rests on, as pure predicates. They live
 * here rather than inline in backend_cuda.cu so a host-side test can pin them
 * with no CUDA toolchain and no GPU (tests/test_cuda_lut_gate.c). backend_cuda.cu
 * calls BOTH at the real decision sites, so the test pins the engine's own
 * logic rather than a second copy that could drift from it -- which is the
 * failure this factoring exists to prevent, the gate having no CI reach
 * otherwise. */

/* Does the upload gate admit this tensor? Only fmt=8 needs the published
 * table; every other format decodes without one. */
static inline int coli_cuda_fp8_gate_admits(int fmt, int lut_ready) {
    return fmt != 8 || lut_ready != 0;
}

/* What coli_cuda_init must do with a request while a device set may be live.
 * BUILD: nothing is live, build the contexts. ACCEPT: the same set is already
 * live -- return success and touch nothing, so the table published to those
 * contexts stays valid. REFUSE: a different set is live -- refuse before
 * rebuilding anything, so the set cannot widen past the last publish. */
enum { COLI_CUDA_INIT_BUILD = 0, COLI_CUDA_INIT_ACCEPT = 1, COLI_CUDA_INIT_REFUSE = -1 };
static inline int coli_cuda_init_disposition(int nctx, int count,
                                             const int *want, const int *live) {
    int i;
    if (nctx <= 0) return COLI_CUDA_INIT_BUILD;
    if (count != nctx) return COLI_CUDA_INIT_REFUSE;
    for (i = 0; i < count; i++) if (want[i] != live[i]) return COLI_CUDA_INIT_REFUSE;
    return COLI_CUDA_INIT_ACCEPT;
}

/* Opaque, persistent device copy of one resident quantized tensor. */
typedef struct ColiCudaTensor ColiCudaTensor;

/* Devices are CUDA ordinals, not positions in the input list.
 * Repeating the same ordered list preserves active contexts. Changing an
 * active list returns 0 without replacing it; release tensors and shut down
 * before selecting a different list. Init/shutdown require caller serialization. */
COLI_CUDA_DLLEXPORT int coli_cuda_init(const int *devices, int count);
COLI_CUDA_DLLEXPORT void coli_cuda_shutdown(void);
/* Number of CUDA devices visible to this process, before a device list is
 * selected. Returns 0 when the runtime cannot discover any device. */
COLI_CUDA_DLLEXPORT int coli_cuda_available_device_count(void);
COLI_CUDA_DLLEXPORT int coli_cuda_device_count(void);
COLI_CUDA_DLLEXPORT int coli_cuda_device_at(int index);
COLI_CUDA_DLLEXPORT int coli_cuda_mem_info(int device, size_t *free_bytes, size_t *total_bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_device_integrated(int device);
/* device < 0 returns aggregate statistics for all configured devices. */
COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
                           double *h2d_ms, double *kernel_ms, double *d2h_ms);
/* Per-device form of coli_cuda_group_stats; unknown devices return zeros. */
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats_device(
    int device, uint64_t *calls, uint64_t *experts, uint64_t *rows,
    double *h2d_ms, double *kernel_ms, double *d2h_ms);

/* Publish the E8 codebook (quant.h's e8_grid, 256x4 bytes) to every configured
 * device. Must be called after coli_cuda_init and before any fmt=6 upload; the
 * backend keeps no copy of the table so it cannot drift from the CPU decoder. */
COLI_CUDA_DLLEXPORT int coli_cuda_e8_set_grid(const void *grid);

/* Publish the fmt=8 e4m3 decode table (quant.h's E4M3_LUT, 256 f32) the same
 * way. Must be called after coli_cuda_init; fmt=8 uploads are refused until it
 * succeeds, because kernels would decode against a zero-initialized table. */
COLI_CUDA_DLLEXPORT int coli_cuda_fp8_set_lut(const float *lut);

/* Upload without executing, so capacity failures happen during model startup. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
        const void *weights, const float *scales,
        int fmt, int I, int O, int device, int gs);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
                            const void *weights, const float *scales,
                            int fmt, int I, int O, int device);

/* Gated delta layer on the device (the linear-attention layers of Qwen3.6/3.8,
 * Kimi K3's KDA, GLM-5.3-Flash). One object per layer holds the short-conv ring
 * [conv_dim x (convk-1)], the recurrent state [vh x kdim x vdim], the conv and
 * norm weights and the scratch for one token. coli_cuda_dn_step runs a whole
 * decode step on the card: x (host) -> in_proj GEMV -> causal conv + SiLU ->
 * per head L2 norm, decay, delta rule, gated RMSNorm -> out_proj GEMV -> out
 * (host). Only x and out cross the bus; the state never does. Gates (egh =
 * exp(g) per value head, beta) arrive by value. All optional in the DLL
 * loader: a backend without them leaves the layer on the CPU. */
typedef struct ColiCudaDn ColiCudaDn;
COLI_CUDA_DLLEXPORT ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim,
                            int conv_dim, int convk, int hidden,
                            const float *conv_w, const float *norm_w, float eps, int gate_sigmoid);
COLI_CUDA_DLLEXPORT void coli_cuda_dn_free(ColiCudaDn *d);
COLI_CUDA_DLLEXPORT int  coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec);  /* NULL = zero */
COLI_CUDA_DLLEXPORT int  coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec);
/* proj: the in_proj rows. Fused qkv ++ z in one matrix (projz NULL, qwen36's
 * dnproj) or qkv alone with z in projz (qwen38's dnqkv / dnz). gate_sigmoid at
 * create: the gated RMSNorm multiplies by silu(z) (0, Qwen3.6) or sigmoid(z)
 * (1, Qwen3.8). */
COLI_CUDA_DLLEXPORT int  coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp,
                            const float *x, float *out, const float *egh, const float *beta);
/* Overwrite an uploaded tensor in place with new bytes of the same shape and
 * format. The expert tier swaps residents, and every expert of a model has the
 * same geometry, so the victim's device buffers can take the newcomer without
 * the cudaFree + cudaMalloc pair, which synchronises the device under the
 * async expert groups. Optional in the DLL loader: 0 from a backend that
 * predates it (or on any failure), and the caller frees and uploads as before. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_overwrite(ColiCudaTensor *tensor, const void *weights, const float *scales);
#ifdef COLI_ANS
/* Experimental Linux-only GPU-resident entropy tier. The archive remains in
 * VRAM and is decoded into per-device scratch immediately before a grouped
 * expert launch. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tensor,
                            const void *weights, const float *scales,
                            int fmt, int I, int O, int device);
#endif

/*
 * y[S,O] = x[S,I] @ W[O,I]^T.
 * fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2, 4=grouped int4.
 * gs is the group size for fmt=4 (0 for all other formats).
 * The first successful call uploads W and its scales; later calls reuse it.
 * Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
 */
/* y[S,O] = x[S,I] @ dequant_mxfp4(W[O,I])^T, fmt=7 (OCP microscaling FP4).
 * q4 is [O, ceil(I/2)] e2m1 nibbles, e8s is [O, ceil(I/32)] ue8m0 exponents --
 * BYTES, not floats, which is why this is not folded into coli_cuda_matmul.
 * Stateless: weights are uploaded per call, matching the streaming expert tier
 * Kimi K3 uses. Returns 1 on success, 0 (y untouched) to fall back to CPU. */
COLI_CUDA_DLLEXPORT int coli_cuda_matmul_mxfp4(float *y, const float *x,
                                               const unsigned char *q4,
                                               const unsigned char *e8s,
                                               int S, int I, int O);

/* Streaming Kimi expert: down(SiTU(gate(x), up(x))). Weights are MXFP4
 * host buffers; intermediate activations remain on device. No weight cache.
 * Returns 0 on failure; callers must accumulate y only after success. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_mxfp4(float *y, const float *x,
        const unsigned char *gate_w, const unsigned char *gate_s,
        const unsigned char *up_w, const unsigned char *up_s,
        const unsigned char *down_w, const unsigned char *down_s,
        int S, int D, int I, float b1, float b2);

COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,
                     float *y, const float *x,
                     const void *weights, const float *scales,
                     int fmt, int S, int I, int O, int device, int gs);

/* Fused expert pipeline: y = down(silu(gate(x)) * up(x)).  All three tensors
 * must already be resident on one device.  Activations cross PCIe once in
 * each direction instead of once per matrix. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
                         ColiCudaTensor *down, float *y, const float *x, int S);

/* Prefill-oriented shared expert path.  INT4 weights stay packed in global
 * memory, activations are converted to FP16 per tile, and Tensor Cores
 * accumulate into FP32.  Unlike COLI_CUDA_TC_INT4 this does not quantize the
 * activation to INT4. */
COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,
                               ColiCudaTensor *down, float *y,
                               const float *x, int S);

/* Packed group of same-shaped experts. Inputs and outputs contain sum(rows)
 * consecutive [D] rows in call order. */
/* Async issue/take split of the group call below (Inc.4): issue launches on the
 * device stream and returns; take syncs and returns the pinned result rows (valid
 * until the next issue on that device). Small totals only (<=8 rows); one
 * outstanding issue per device. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
                               ColiCudaTensor *const *ups,
                               ColiCudaTensor *const *downs,
                               const int *rows, int count, const float *x);
COLI_CUDA_DLLEXPORT const float *coli_cuda_expert_group_take(int device);

COLI_CUDA_DLLEXPORT int coli_cuda_expert_group(ColiCudaTensor *const *gates,
                           ColiCudaTensor *const *ups,
                           ColiCudaTensor *const *downs,
                           const int *rows, int count,
                           float *y, const float *x);
/* Same operation, but force the small-batch grouped kernel family when
 * pin_small_batch is nonzero.  Speculative verification uses this to keep
 * CUDA on the same numeric family as S=1 regardless of accepted draft depth. */
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_pinned(ColiCudaTensor *const *gates,
                           ColiCudaTensor *const *ups,
                           ColiCudaTensor *const *downs,
                           const int *rows, int count,
                           float *y, const float *x, int pin_small_batch);

/* Decode-only MLA weight-absorption core for one token. kv_b is [H*(Q+V),K]. */
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb(ColiCudaTensor *kv_b,float *ctx,const float *q,
                               const float *latent,const float *rope,int H,int Q,
                               int R,int V,int K,int T,float attention_scale);

/* Causal MLA absorption for S contiguous rows from one sequence.  The KV
 * arrays contain T rows ending at the final query; query s attends T-S+s+1
 * rows.  One transfer and one launch replace S host round-trips. */
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch(ColiCudaTensor *kv_b,float *ctx,const float *q,
                                     const float *latent,const float *rope,int S,
                                     int H,int Q,int R,int V,int K,int T,
                                     float attention_scale);

/* Same attention batch followed immediately by resident o_proj on the same
 * device.  Only the final [S,D] tensor crosses back to the host. */
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
                                      float *out,const float *q,const float *latent,
                                      const float *rope,int S,int H,int Q,int R,
                                      int V,int K,int T,float attention_scale);

COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
        float *out,const float *q,const void *const *keys,
        const float *const *latent,const float *const *rope,
        const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);

COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT size_t coli_cuda_alloc_footprint(size_t bytes);
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_vram(const ColiCudaTensor *tensor);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);

/* Replace a resident tensor's contents without reallocating its device slot. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_update(ColiCudaTensor *tensor,
                            const void *weights, const float *scales);

/* ---- resident-pipeline primitives (Inc.0): device-pointer entry points ---- */
COLI_CUDA_DLLEXPORT float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);
COLI_CUDA_DLLEXPORT void *coli_cuda_pipe_alloc(int device,size_t bytes);
COLI_CUDA_DLLEXPORT void coli_cuda_pipe_free(int device,void *p);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,
                           const float *w_dev,int S,int D,float eps);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,
                        int stride,int offset,int R,int heads,float theta);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,
                            const int *rows_dev,int nrows,int D);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,
                             const float *w_dev,int S,int D,float eps,
                             int xstride,int ystride);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,
                             int stride,int offset,int R,int heads,float theta);
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,
        ColiCudaTensor *const *ups, ColiCudaTensor *const *downs,
        const float *weights, int count,
        int home_device, const float *x_src_dev, float *partial_slot_dev);
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_take(int home_device,const int *devices,
        int n_issued,float *slots_dev,float *acc_dev,int D);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_router(int device,const float *x_dev,
        const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,
        float topp,int norm_topk,float routed_scale,
        int *idx_host,float *w_host,int *keff_host);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,
                          int spitch,int width,int height);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
        float *out,const float *q_dev,const float *latent_dev,const float *rope_dev,
        int S,int H,int Q,int R,int V,int K,int T,float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,
        const float *q_dev,const float *latent_dev,const float *rope_dev,
        int S,int H,int Q,int R,int V,int K,int T,float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,
        const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T,
        float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,
                             const float *src,size_t bytes);
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
        float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev,
        int S,int H,int Q,int R,int V,int K,int T,float scale);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);

#ifdef __cplusplus
}
#endif

#endif