Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
metal: fix fmt=0 (raw f32) NULL-scale segfault in coli_metal_matmul
njloof90ec4f677e18
diff --git a/c/backend_metal.mm b/c/backend_metal.mmindex f4cb807e..b7cdf70e 100644--- a/c/backend_metal.mm+++ b/c/backend_metal.mm@@ -150,8 +150,10 @@ kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight by}acc = simd_sum(acc);// fmt==4 (per-group) and fmt==8 (per-block) already folded their scale into acc- // above -- do not scale again.- if (slane == 0) y[row] = (fmt == 4 || fmt == 8) ? acc : acc * scale[o];+ // above -- do not scale again. fmt==0 is raw f32 and has NO scale array (see+ // fmt_scale_bytes): its `scale` binding is a nil buffer, so reading scale[o] here+ // yielded 0 and zeroed the whole result. Only fmt 1/2/3 carry a per-row scale.+ if (slane == 0) y[row] = (fmt == 0 || fmt == 4 || fmt == 8) ? acc : acc * scale[o];}// Batched bindless expert GEMV: each row gr belongs to expert erow[gr], whose weight and@@ -764,10 +766,18 @@ static size_t fmt_bytes(int fmt, int I, int O) {// UE8M0 encoding exists at all: qt_resolve_fmt refuses it on the CPU read path before any// tensor in that encoding could ever reach this Metal-side sizing helper.static size_t fmt_scale_bytes(int fmt, int I, int O, int gs) {+ // fmt=0 is RAW f32: the weights are already in their final units, so there is no+ // scale array at all and callers legitimately pass scales == NULL (kimi_k3.c's+ // k3_matmul_f32 does, for the KDA fa/fb/bp projections). Returning the per-row+ // size below for fmt=0 made coli_metal_matmul wrap a NULL pointer, which segfaulted+ // inside newBufferWithBytes' memmove whenever O*4 was not a page multiple, and+ // silently produced a nil buffer (-> all-zero output) when it was. Must stay 0, and+ // mm_gemv must correspondingly not apply a scale for fmt=0.+ if (fmt == 0) return 0;if (fmt == 4) return (size_t)O * ((I + gs - 1) / gs) * sizeof(float);if (fmt == 6) return (size_t)O * ((I + gs - 1) / gs) * 2; // e8: 2-byte group scalesif (fmt == 8) return (size_t)((O + 127) / 128) * (size_t)((I + 127) / 128) * sizeof(float); // fp8 block scales- return (size_t)O * sizeof(float); // per-row scale, fmt 0/1/2/3 (dev catch-all; #790 rebase must keep this)+ return (size_t)O * sizeof(float); // per-row scale, fmt 1/2/3 (dev catch-all; #790 rebase must keep this)}// Wrap host memory zero-copy if page-aligned, else copy into a shared buffer.@@ -958,6 +968,17 @@ extern "C" int coli_metal_matmul(ColiMetalTensor **tp, float *y, const float *x,* contiguous range check below: it is not adjacent to it. */if (!g_dev || fmt < 0 || (fmt > 4 && fmt != 8)) return 0;if(!weights||!x||!y) return 0;+ /* Fail CLOSED to the caller's CPU path if a format that NEEDS a scale array was+ * handed a NULL one, rather than letting wrap() memmove from address 0. Sized off+ * fmt_scale_bytes so the two can't drift: any format it reports bytes for must have+ * a real pointer behind them. (fmt=0 reports 0 bytes and is exempt by construction.) */+ if (!scales && fmt_scale_bytes(fmt, I, O, gs) != 0) {+ static bool warned = false;+ if (!warned) { warned = true;+ fprintf(stderr, "[metal] matmul: fmt=%d needs a scale array but scales==NULL; "+ "falling back to CPU\n", fmt); }+ return 0;+ }@autoreleasepool {ColiMetalTensor *t = *tp;if (!t) {diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mmindex 1c66137d..54c07586 100644--- a/c/tests/test_backend_metal.mm+++ b/c/tests/test_backend_metal.mm@@ -56,6 +56,52 @@ static int run(int fmt, int O, int I, int S, const char *name) {return ok?0:1;}+// ---- fmt=0 (raw f32) with scales == NULL: own harness -------------------------------+// run() above CANNOT cover this case, and that gap is why the K3 Metal segfault shipped.+// run() always hands coli_metal_matmul a non-NULL [O] scale array, filling it with 1.0f+// for F32 -- so it exercised neither of the two things real fmt=0 callers do:+// (a) pass scales == NULL, because raw f32 weights are already in final units, and+// (b) expect NO scale multiply to be applied to the result.+// The 1.0f fill made (b) invisible (1.0 is the identity), and the non-NULL pointer made+// (a) invisible. kimi_k3.c's k3_matmul_f32 does both, for the KDA fa/fb/bp projections.+//+// This harness therefore passes scales == NULL and uses a reference with no scale term.+// The contract it pins down: for fmt=0, fmt_scale_bytes() must report 0 bytes (so no+// wrap of the NULL pointer is attempted) and mm_gemv must not apply scale[o].+//+// WARNING to whoever reads a failing run: on a build where that contract is broken,+// the O=12288 case below reports a clean MISMATCH (its O*4 scale size is an exact+// multiple of the 16384-byte page, so wrap() takes newBufferWithBytesNoCopy and yields+// a nil buffer -> shader reads scale as 0 -> all-zero output), but the O=128 and O=96+// cases SEGFAULT the whole test binary rather than returning (their O*4 is not a page+// multiple, so wrap() falls back to newBufferWithBytes, which memmoves from address 0).+// A hard crash here is the reproduction, not a broken test -- the printf is flushed+// before the call so the last line printed names the offending case.+static int run_f32_noscale(int O, int I, int S, const char *name) {+ std::vector<float> W((size_t)O*I), x((size_t)S*I), yr((size_t)S*O), yg((size_t)S*O);+ srand(4242);+ for(auto&v:W) v=((rand()%2000)-1000)/1000.f;+ for(auto&v:x) v=((rand()%2000)-1000)/1000.f;+ // reference: plain y[S,O] = x[S,I] @ W[O,I]^T, NO per-row scale anywhere+ for(int o=0;o<O;o++) for(int si=0;si<S;si++){+ const float *xr=&x[(size_t)si*I]; double acc=0;+ for(int i=0;i<I;i++) acc += (double)W[(size_t)o*I+i]*xr[i];+ yr[(size_t)si*O+o]=(float)acc;+ }+ printf(" %-46s ", name); fflush(stdout); // flush BEFORE: a segfault must name the case+ ColiMetalTensor *t=nullptr;+ if (!coli_metal_matmul(&t, yg.data(), x.data(), W.data(), /*scales=*/nullptr,+ F32, S, I, O, 0)) {+ printf("FAIL (matmul returned 0 -- fmt=0 must not be declined)\n"); return 1; }+ double maxabs=0, ymax=0;+ for(size_t i=0;i<(size_t)S*O;i++){ maxabs=fmax(maxabs,fabs(yg[i]-yr[i])); ymax=fmax(ymax,fabs(yr[i])); }+ double nerr=maxabs/(ymax+1e-9);+ int ok = nerr < 1e-4;+ printf("nerr=%.2e %s\n", nerr, ok?"ok":"*** MISMATCH");+ coli_metal_tensor_free(t);+ return ok?0:1;+}+// ---- fmt=4 (grouped int4): own CPU reference + harness -----------------------------// cpu_ref's [O] per-row scale layout can't express fmt=4's [O,ceil(I/gs)] per-group// scale, so this is a separate reference rather than a branch of cpu_ref. Mirrors@@ -813,6 +859,18 @@ int main(void) {fail |= run(I8, 2048,6144,4, "int8 gate/up S=4");fail |= run(I4, 2048,6144,7, "int4 gate/up S=7 (odd)");fail |= run(I4, 2050,6146,3, "int4 non-mult-4 dims");+ // fmt=0 with scales == NULL -- the real kimi_k3.c k3_matmul_f32 contract. Shapes are+ // Kimi K3's three KDA projections verbatim (hidden=7168, kda_hd=128, kda_heads=96,+ // kda_proj=96*128=12288), because the page-alignment of O*sizeof(float) is what selects+ // between the two failure modes and these are the sizes that occur in practice.+ // fb (O=12288) is ordered FIRST deliberately: it is the one case that reports a clean+ // MISMATCH instead of crashing, so on a regressed build the log shows a real diagnostic+ // before the fa case takes the process down. See run_f32_noscale's header comment.+ printf("Metal fmt=0 raw-f32 NULL-scale tests (Kimi K3 KDA fa/fb/bp projections):\n");+ fail |= run_f32_noscale(12288, 128, 1, "f32 no-scale fb O=12288 I=128 (page-mult scale sz)");+ fail |= run_f32_noscale(128, 7168, 1, "f32 no-scale fa O=128 I=7168 (non-page-mult)");+ fail |= run_f32_noscale(96, 7168, 1, "f32 no-scale bp O=96 I=7168 (non-page-mult)");+ fail |= run_f32_noscale(128, 7168, 4, "f32 no-scale fa O=128 I=7168 S=4 (prefill)");printf("Metal fmt=4 grouped-int4 tests (coli_metal_matmul vs matmul_i4_grouped semantics):\n");// I multiple of gs=64, S=1 and S>1 (real g64-checkpoint shapes: gate/up I=6144, down I=2048)fail |= run_grouped(2048,6144,64,1,0, "grouped gate/up I=6144(mult64) S=1");