Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(metal): whole-slot buffers with offset-addressed affine dispatch
Léon Avic Simmons1517776df93c
diff --git a/c/backend_metal.h b/c/backend_metal.hindex a9dfe881..3dbdd911 100644--- a/c/backend_metal.h+++ b/c/backend_metal.h@@ -123,6 +123,42 @@ int coli_metal_matmul_affine(ColiMetalTensor **tensor,float *y, const float *x, int batch,const ColiAffineQuantizedView *view);+/* Whole-slot buffers for bounded, refillable qpack expert slots. A slot+ * buffer wraps one page-aligned host slot in an MTLBuffer exactly ONCE+ * (zero-copy); the pool refills the same memory with different experts, and+ * every dispatch addresses the projection weight, scale, and bias sections+ * by BYTE OFFSET inside that one registered buffer. Nothing is keyed on the+ * refillable host pointers and no bytes are copied at registration, so a+ * refill cannot leave a stale GPU-side snapshot behind -- the exact hazard+ * that makes the resident-handle path above (which caches by stable host+ * pointer) unusable for reused memory.+ *+ * Registration requires base aligned to 16384 (the Apple page) and len a+ * multiple of it and REFUSES anything else (returns NULL): the copying wrap+ * fallback used elsewhere would silently detach the GPU from later refills.+ * qpack expert strides already satisfy this (COLI_QPACK_PAGE_ALIGNMENT).+ * Slot buffers count in coli_metal_stats() like resident tensor handles.+ *+ * coli_metal_matmul_affine_slot runs the same Q4/Q8 affine kernels as+ * coli_metal_matmul_affine but takes the registered slot plus section byte+ * offsets instead of a cached per-projection handle. The checked view must+ * describe the slot's CURRENT fill: every section must lie inside the+ * registered range, the weights offset must be 4-byte aligned (the kernel+ * reads uint32 words), and the view's host pointers must equal base+offset.+ * A descriptor from a previous fill or the wrong slot is refused (returns 0,+ * CPU fallback), never dispatched. Synchronous like the rest of this API:+ * when it returns, the GPU is done reading the slot and the caller may+ * refill it. */+typedef struct ColiMetalSlotBuffer ColiMetalSlotBuffer;+ColiMetalSlotBuffer *coli_metal_slot_register(void *base, size_t len);+void coli_metal_slot_unregister(ColiMetalSlotBuffer *slot);+int coli_metal_matmul_affine_slot(ColiMetalSlotBuffer *slot,+ size_t weights_offset,+ size_t scales_offset,+ size_t biases_offset,+ float *y, const float *x, int batch,+ const ColiAffineQuantizedView *view);+void coli_metal_tensor_free(ColiMetalTensor *tensor);/* Weight payload bytes only, preserving the established QT API contract. */size_t coli_metal_tensor_bytes(const ColiMetalTensor *tensor);diff --git a/c/backend_metal.mm b/c/backend_metal.mmindex 22671d51..a6fb9dcd 100644--- a/c/backend_metal.mm+++ b/c/backend_metal.mm@@ -1176,6 +1176,48 @@ struct AffineGemvParamsHost {static_assert(sizeof(AffineGemvParamsHost) == 5*sizeof(uint32_t),"Metal affine parameter layout drift");+/* Shared encode/run body for the two affine entry points: the buffers are+ * either a resident handle's per-section wraps (offsets 0) or one whole-slot+ * buffer addressed by section byte offsets. Synchronous; returns 0 on+ * any allocation, encoder, or command-buffer failure. */+static int affine_dispatch_run(id<MTLBuffer> wbuf, size_t woff,+ id<MTLBuffer> sbuf, size_t soff,+ id<MTLBuffer> bbuf, size_t boff,+ float *y, const float *x, int batch,+ const ColiAffineQuantizedView *view) {+ const size_t input_bytes=(size_t)batch*view->input_dim*sizeof(float);+ const size_t output_bytes=(size_t)batch*view->output_dim*sizeof(float);+ id<MTLBuffer> bx=[g_dev newBufferWithBytes:x length:input_bytes+ options:MTLResourceStorageModeShared];+ id<MTLBuffer> by=[g_dev newBufferWithLength:output_bytes+ options:MTLResourceStorageModeShared];+ if (!bx || !by) return 0;+ AffineGemvParamsHost p = {+ (uint32_t)view->output_dim, (uint32_t)view->input_dim,+ (uint32_t)view->group_size, (uint32_t)view->scalar_format,+ (uint32_t)batch+ };+ id<MTLCommandBuffer> cb=[g_queue commandBuffer];+ id<MTLComputeCommandEncoder> e=[cb computeCommandEncoder];+ if (!cb || !e) return 0;+ [e setComputePipelineState:(view->format == COLI_AFFINE_MLX_Q4 ?+ g_affine_q4 : g_affine_q8)];+ [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:wbuf offset:woff atIndex:1];+ [e setBuffer:sbuf offset:soff atIndex:2]; [e setBuffer:bbuf offset:boff atIndex:3];+ [e setBuffer:by offset:0 atIndex:4]; [e setBytes:&p length:sizeof(p) atIndex:5];+ [e dispatchThreads:MTLSizeMake(32,(NSUInteger)batch*view->output_dim,1)+ threadsPerThreadgroup:MTLSizeMake(32,1,1)];+ [e endEncoding]; [cb commit]; [cb waitUntilCompleted];+ if (cb.status != MTLCommandBufferStatusCompleted) {+ fprintf(stderr,"[metal] affine GEMV command buffer failed (status=%ld): %s\n",+ (long)cb.status,+ cb.error ? [[cb.error localizedDescription] UTF8String] : "?");+ return 0;+ }+ memcpy(y,[by contents],output_bytes);+ return 1;+}+extern "C" int coli_metal_matmul_affine(ColiMetalTensor **tp,float *y, const float *x, int batch,const ColiAffineQuantizedView *view) {@@ -1214,43 +1256,10 @@ extern "C" int coli_metal_matmul_affine(ColiMetalTensor **tp,t->host_w=view->weights; t->host_s=view->scales; t->host_b=view->biases;}- const size_t input_bytes=(size_t)batch*view->input_dim*sizeof(float);- const size_t output_bytes=(size_t)batch*view->output_dim*sizeof(float);- id<MTLBuffer> bx=[g_dev newBufferWithBytes:x length:input_bytes- options:MTLResourceStorageModeShared];- id<MTLBuffer> by=[g_dev newBufferWithLength:output_bytes- options:MTLResourceStorageModeShared];- if (!bx || !by) {+ if (!affine_dispatch_run(t->w, 0, t->s, 0, t->b, 0, y, x, batch, view)) {if (fresh) tensor_destroy(t);return 0;}- AffineGemvParamsHost p = {- (uint32_t)view->output_dim, (uint32_t)view->input_dim,- (uint32_t)view->group_size, (uint32_t)view->scalar_format,- (uint32_t)batch- };- id<MTLCommandBuffer> cb=[g_queue commandBuffer];- id<MTLComputeCommandEncoder> e=[cb computeCommandEncoder];- if (!cb || !e) {- if (fresh) tensor_destroy(t);- return 0;- }- [e setComputePipelineState:(view->format == COLI_AFFINE_MLX_Q4 ?- g_affine_q4 : g_affine_q8)];- [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:t->w offset:0 atIndex:1];- [e setBuffer:t->s offset:0 atIndex:2]; [e setBuffer:t->b offset:0 atIndex:3];- [e setBuffer:by offset:0 atIndex:4]; [e setBytes:&p length:sizeof(p) atIndex:5];- [e dispatchThreads:MTLSizeMake(32,(NSUInteger)batch*view->output_dim,1)- threadsPerThreadgroup:MTLSizeMake(32,1,1)];- [e endEncoding]; [cb commit]; [cb waitUntilCompleted];- if (cb.status != MTLCommandBufferStatusCompleted) {- fprintf(stderr,"[metal] affine GEMV command buffer failed (status=%ld): %s\n",- (long)cb.status,- cb.error ? [[cb.error localizedDescription] UTF8String] : "?");- if (fresh) tensor_destroy(t);- return 0;- }- memcpy(y,[by contents],output_bytes);if (fresh) {*tp=t;g_tensor_count++;@@ -1260,6 +1269,80 @@ extern "C" int coli_metal_matmul_affine(ColiMetalTensor **tp,return 1;}+// ---- Whole-slot buffers for bounded, refillable qpack expert slots ----+// One MTLBuffer per slot, wrapped zero-copy EXACTLY once; the pool refills the+// same host memory with different experts, and dispatches address the weight,+// scale, and bias sections by byte offset inside the registered buffer. The+// zero-copy wrap is mandatory, not best-effort: the copying fallback in wrap()+// would snapshot the first fill and silently detach the GPU from every later+// refill, which is exactly the stale-projection hazard the whole-slot+// buffers exist to remove.+struct ColiMetalSlotBuffer {+ id<MTLBuffer> buf;+ void *base;+ size_t len;+};++extern "C" ColiMetalSlotBuffer *coli_metal_slot_register(void *base, size_t len) {+ const size_t pg = 16384; // Apple page: newBufferWithBytesNoCopy contract+ if (!g_dev || !base || !len) return NULL;+ if (((uintptr_t)base % pg) != 0 || (len % pg) != 0) return NULL;+ id<MTLBuffer> b = [g_dev newBufferWithBytesNoCopy:base length:len+ options:MTLResourceStorageModeShared+ deallocator:nil];+ if (!b) return NULL;+ ColiMetalSlotBuffer *s = new ColiMetalSlotBuffer();+ s->buf = b; s->base = base; s->len = len;+ g_tensor_count++; g_tensor_bytes += len;+ return s;+}++extern "C" void coli_metal_slot_unregister(ColiMetalSlotBuffer *slot) {+ if (!slot) return;+ g_tensor_count--; g_tensor_bytes -= slot->len;+ slot->buf = nil;+ delete slot;+}++extern "C" int coli_metal_matmul_affine_slot(ColiMetalSlotBuffer *slot,+ size_t weights_offset,+ size_t scales_offset,+ size_t biases_offset,+ float *y, const float *x, int batch,+ const ColiAffineQuantizedView *view) {+ if (!slot || !slot->buf || !y || !x || !coli_metal_affine_available() ||+ !coli_metal_affine_dispatch_supported(view,batch))+ return 0;+ @autoreleasepool {+ const unsigned bits=coli_affine_bits(view->format), per_word=32u/bits;+ const size_t groups=view->input_dim/view->group_size;+ const size_t wbytes=view->output_dim*(view->input_dim/per_word)*sizeof(uint32_t);+ const size_t scalar_bytes=view->output_dim*groups*+ coli_affine_scalar_size(view->scalar_format);+ /* Every section must lie inside the registered slot, and the weight words+ * must land 4-byte aligned (base is page-aligned; the kernel casts each+ * row to device const uint*). */+ if (weights_offset % 4 != 0) return 0;+ if (weights_offset > slot->len || wbytes > slot->len - weights_offset)+ return 0;+ if (scales_offset > slot->len || scalar_bytes > slot->len - scales_offset)+ return 0;+ if (biases_offset > slot->len || scalar_bytes > slot->len - biases_offset)+ return 0;+ /* Tripwire: the descriptor must describe THIS slot's current fill. A+ * view carried across a refill (or aimed at another slot) disagrees with+ * base+offset and is refused instead of dispatched. */+ const unsigned char *base=(const unsigned char *)slot->base;+ if (view->weights != base + weights_offset ||+ view->scales != base + scales_offset ||+ view->biases != base + biases_offset)+ return 0;+ return affine_dispatch_run(slot->buf, weights_offset,+ slot->buf, scales_offset,+ slot->buf, biases_offset, y, x, batch, view);+ }+}+// ---- fused decode attention scratch (GLM-5.2 dims) ----enum { AH=6144, AHEADS=64, AQLORA=2048, AKVL=512, AROPE=64, AVH=256, AQH=256, ANOPE=192, AROWSH=448, AHQH=AHEADS*AQH, AHVH=AHEADS*AVH, AMAXS=4 };static id<MTLBuffer> ax_,aqr_,aqf_,acomp_,aqabs_,ascore_,aclat_,actx_,aout_,aqaln_,akvaln_; static size_t ascore_cap;diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mmindex 165e673e..9416e719 100644--- a/c/tests/test_backend_metal.mm+++ b/c/tests/test_backend_metal.mm@@ -200,6 +200,115 @@ static int run_affine(ColiAffineFormat format, ColiAffineScalarFormat scalar_forreturn ok?0:1;}+// ---- Slot buffers: registered once, offset-addressed, refilled in place ----+// Three claims: (1) offset addressing inside one whole-slot buffer matches the+// CPU reference; (2) refilling the SAME registered memory is seen by the next+// dispatch (write-through -- the property the resident-handle cache cannot give+// reused memory); (3) the refusal arms hold: misaligned registration, sections+// outside the registered range, and a view that does not describe the slot's+// current fill are refused, never dispatched.+static int run_affine_slot(ColiAffineFormat format, ColiAffineScalarFormat scalar_format,+ int S, const char *name) {+ enum { O=48, I=2048, GROUP=32, HEADER=256 }; // HEADER: prove nonzero offsets+ const size_t pg=16384;+ unsigned bits=coli_affine_bits(format), per_word=32u/bits;+ size_t packed_words=(size_t)O*(I/per_word);+ size_t wbytes=packed_words*sizeof(uint32_t);+ size_t scalar_count=(size_t)O*(I/GROUP);+ size_t sbytes=scalar_count*coli_affine_scalar_size(scalar_format);+ size_t woff=HEADER, soff=woff+wbytes, boff=soff+sbytes;+ size_t len=((boff+sbytes)+pg-1)&~(pg-1);+ uint8_t *slab=nullptr;+ if (posix_memalign((void**)&slab,pg,len)) {+ printf(" %-38s FAIL (posix_memalign)\n",name); return 1; }+ memset(slab,0,len);++ auto fill=[&](uint32_t seed){+ uint32_t state=seed;+ uint32_t *w=(uint32_t*)(void*)(slab+woff);+ for(size_t i=0;i<packed_words;i++){ state=state*1664525u+1013904223u; w[i]=state; }+ std::vector<uint8_t> sc(sbytes), bi(sbytes);+ for(size_t i=0;i<scalar_count;i++){+ state=state*1664525u+1013904223u;+ float scale=(float)((int)(state%7u)-3)*0.03125f;+ float bias=(float)((int)((state>>8)%5u)-2)*0.015625f;+ if(scale==0.0f) scale=0.046875f;+ affine_store_scalar(sc,i,scalar_format,scale);+ affine_store_scalar(bi,i,scalar_format,bias);+ }+ memcpy(slab+soff,sc.data(),sbytes); memcpy(slab+boff,bi.data(),sbytes);+ };++ ColiAffineQuantizedView view = {+ slab+woff, slab+soff, slab+boff, wbytes, sbytes, sbytes,+ (size_t)O, (size_t)I, (size_t)GROUP, format, scalar_format+ };+ std::vector<float> x((size_t)S*I), expected((size_t)S*O), actual((size_t)S*O);+ for(size_t i=0;i<x.size();i++) x[i]=(float)((int)((i*13+5)%29)-14)*0.0625f;+ auto relative_error=[&](){+ double maxabs=0.0, ymax=0.0;+ for(size_t i=0;i<actual.size();i++){+ if(!std::isfinite(actual[i]) || !std::isfinite(expected[i]))+ return (double)INFINITY;+ maxabs=fmax(maxabs,fabs((double)actual[i]-expected[i]));+ ymax=fmax(ymax,fabs((double)expected[i]));+ }+ return maxabs/(ymax+1e-9);+ };++ size_t count_before=0, bytes_before=0;+ coli_metal_stats(&count_before,&bytes_before);+ // A misaligned base or a non-page-multiple length must not register at all:+ // those layouts would take wrap()'s copying fallback and go stale on refill.+ int misaligned_refused = coli_metal_slot_register(slab+4,len)==NULL &&+ coli_metal_slot_register(slab,len-4)==NULL;+ ColiMetalSlotBuffer *slot=coli_metal_slot_register(slab,len);+ size_t count_reg=0, bytes_reg=0;+ coli_metal_stats(&count_reg,&bytes_reg);+ int registered = slot!=NULL && count_reg==count_before+1 &&+ bytes_reg==bytes_before+len;++ fill(0xC4C40001u ^ (uint32_t)format ^ ((uint32_t)scalar_format<<4));+ int ref_ok=coli_affine_matmul_ref(expected.data(),x.data(),(size_t)S,&view)==COLI_AFFINE_OK;+ std::fill(actual.begin(),actual.end(),NAN);+ int first = registered && ref_ok &&+ coli_metal_matmul_affine_slot(slot,woff,soff,boff,actual.data(),x.data(),S,&view);+ double first_nerr = first ? relative_error() : INFINITY;++ // Refill the same registered memory with a different expert; the next+ // dispatch must read the NEW bytes through the same registration.+ fill(0x5EED0002u ^ (uint32_t)format ^ ((uint32_t)scalar_format<<4));+ int ref2_ok=coli_affine_matmul_ref(expected.data(),x.data(),(size_t)S,&view)==COLI_AFFINE_OK;+ std::fill(actual.begin(),actual.end(),NAN);+ int refilled = first && ref2_ok &&+ coli_metal_matmul_affine_slot(slot,woff,soff,boff,actual.data(),x.data(),S,&view);+ double refill_nerr = refilled ? relative_error() : INFINITY;++ // Sections outside the registered range are refused.+ int oob_refused = refilled &&+ !coli_metal_matmul_affine_slot(slot,len-4,soff,boff,actual.data(),x.data(),S,&view) &&+ !coli_metal_matmul_affine_slot(slot,woff,len,boff,actual.data(),x.data(),S,&view) &&+ !coli_metal_matmul_affine_slot(slot,woff,soff,len,actual.data(),x.data(),S,&view);+ // A view that does not describe the claimed offsets (a stale or misdirected+ // descriptor) is refused, not dispatched.+ int mismatch_refused = refilled &&+ !coli_metal_matmul_affine_slot(slot,woff+4,soff,boff,actual.data(),x.data(),S,&view) &&+ !coli_metal_matmul_affine_slot(slot,woff,boff,soff,actual.data(),x.data(),S,&view) &&+ !coli_metal_matmul_affine_slot(NULL,woff,soff,boff,actual.data(),x.data(),S,&view);++ coli_metal_slot_unregister(slot);+ size_t count_after=0, bytes_after=0;+ coli_metal_stats(&count_after,&bytes_after);+ int restored = count_after==count_before && bytes_after==bytes_before;+ free(slab);+ int ok = misaligned_refused && registered && first && first_nerr<1e-4 &&+ refilled && refill_nerr<1e-4 && oob_refused && mismatch_refused &&+ restored;+ printf(" %-38s nerr=%.2e/%.2e (fill/refill) S=%d %s\n",+ name,first_nerr,refill_nerr,S,ok?"ok":"*** MISMATCH");+ return ok?0:1;+}+// ---- fmt=4 (grouped int4): own CPU reference + harness -----------------------------// cpu_ref's [O] per-row scale layout can't express fmt=4's [O,ceil(I/gs)] per-group// scale, so this is a separate reference rather than a branch of cpu_ref. Mirrors@@ -992,6 +1101,10 @@ int main(void) {(int)coli_affine_bits((ColiAffineFormat)format),scalar_name[scalar]);fail |= run_affine((ColiAffineFormat)format,(ColiAffineScalarFormat)scalar,3,name);}+ printf("Metal MLX affine slot-buffer tests (register once, offset-address, refill):\n");+ fail |= run_affine_slot(COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F32, 3, "affine slot Q4/f32 offset+refill");+ fail |= run_affine_slot(COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F16, 2, "affine slot Q4/f16 offset+refill");+ fail |= run_affine_slot(COLI_AFFINE_MLX_Q8, COLI_AFFINE_SCALAR_BF16, 3, "affine slot Q8/bf16 offset+refill");} else if (affine_capability==COLI_METAL_AFFINE_CAP_SIMD_WIDTH_UNSUPPORTED) {printf("Metal MLX affine packed GEMV tests: SKIPPED (requires 32-lane simdgroups)\n");} else {