Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge remote-tracking branch 'upstream/dev' into fix/dense-fmt4-scales

ZacharyZcRd8277d6c6bbf
diff --cc c/backend_cuda.cu
index 298fcee5,327915e4..9154c6d2
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@@ -636,15 -667,11 +667,16 @@@ static int fault_injected(void)
extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor,
float *y, const float *x,
const void *weights, const float *scales,
- int fmt, int S, int I, int O, int device) {
+ int fmt, int S, int I, int O, int device, int gs) {
if (fault_injected()) return 0;
- /* fmt=4 carries [O, ceil(I/gs)] scales and quant_matmul reads them per row:
- * the plain upload would also truncate the buffer to O floats. #298 fixed the
- * dense-resident path's callers; this closes the entry itself so no future
- * caller can reach the kernel with grouped scales. Grouped tensors take the
- * expert-group path (#451) or stay on the CPU. */
- if (fmt == 4) return 0;
- if (S < 1 || !coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device)) return 0;
++ /* fmt=4 carries [O, ceil(I/gs)] scales: without the group size the plain
++ * upload truncates the buffer to O floats and quant_matmul divides by
++ * gs==0. Callers must come through the gs>0 path (upload_g) or stay on
++ * the CPU (#298, #334). */
++ if (fmt == 4 && gs <= 0) return 0;
+ if (S < 1) return 0;
+ if (gs > 0) { if (!coli_cuda_tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs)) return 0; }
+ else { if (!coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device)) return 0; }
ColiCudaTensor *t = *tensor;
DeviceContext *ctx = find_ctx(t->device);
if (!select_ctx(ctx)) return 0;
@@@ -663,8 -690,6 +695,11 @@@ extern "C" int coli_cuda_expert_mlp(Col
ColiCudaTensor *down, float *y,
const float *x, int S) {
if (fault_injected()) return 0;
- /* same reason as coli_cuda_matmul: this path's kernels are per-row only. */
- if (gate && (gate->fmt == 4 || (up && up->fmt == 4) || (down && down->fmt == 4))) return 0;
++ /* same reason as coli_cuda_matmul: fmt=4 without recorded group info would
++ * misread the scales (and divide by gs==0 in the kernel). */
++ if (gate && ((gate->fmt == 4 && gate->gs <= 0) ||
++ (up && up->fmt == 4 && up->gs <= 0) ||
++ (down && down->fmt == 4 && down->gs <= 0))) return 0;
if (!gate || !up || !down || !x || !y || S < 1 ||
gate->device != up->device || gate->device != down->device ||
gate->I != up->I || gate->O != up->O ||