Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(metal): whole-slot buffers with offset-addressed affine dispatch

Léon Avic Simmons1517776df93c
diff --git a/c/backend_metal.h b/c/backend_metal.h
index a9dfe881..3dbdd911 100644
--- a/c/backend_metal.h
+++ b/c/backend_metal.h
@@ -123,6 +123,42 @@ int coli_metal_matmul_affine(ColiMetalTensor **tensor,
float *y, const float *x, int batch,
const ColiAffineQuantizedView *view);
+/* Whole-slot buffers for bounded, refillable qpack expert slots. A slot
+ * buffer wraps one page-aligned host slot in an MTLBuffer exactly ONCE
+ * (zero-copy); the pool refills the same memory with different experts, and
+ * every dispatch addresses the projection weight, scale, and bias sections
+ * by BYTE OFFSET inside that one registered buffer. Nothing is keyed on the
+ * refillable host pointers and no bytes are copied at registration, so a
+ * refill cannot leave a stale GPU-side snapshot behind -- the exact hazard
+ * that makes the resident-handle path above (which caches by stable host
+ * pointer) unusable for reused memory.
+ *
+ * Registration requires base aligned to 16384 (the Apple page) and len a
+ * multiple of it and REFUSES anything else (returns NULL): the copying wrap
+ * fallback used elsewhere would silently detach the GPU from later refills.
+ * qpack expert strides already satisfy this (COLI_QPACK_PAGE_ALIGNMENT).
+ * Slot buffers count in coli_metal_stats() like resident tensor handles.
+ *
+ * coli_metal_matmul_affine_slot runs the same Q4/Q8 affine kernels as
+ * coli_metal_matmul_affine but takes the registered slot plus section byte
+ * offsets instead of a cached per-projection handle. The checked view must
+ * describe the slot's CURRENT fill: every section must lie inside the
+ * registered range, the weights offset must be 4-byte aligned (the kernel
+ * reads uint32 words), and the view's host pointers must equal base+offset.
+ * A descriptor from a previous fill or the wrong slot is refused (returns 0,
+ * CPU fallback), never dispatched. Synchronous like the rest of this API:
+ * when it returns, the GPU is done reading the slot and the caller may
+ * refill it. */
+typedef struct ColiMetalSlotBuffer ColiMetalSlotBuffer;
+ColiMetalSlotBuffer *coli_metal_slot_register(void *base, size_t len);
+void coli_metal_slot_unregister(ColiMetalSlotBuffer *slot);
+int coli_metal_matmul_affine_slot(ColiMetalSlotBuffer *slot,
+ size_t weights_offset,
+ size_t scales_offset,
+ size_t biases_offset,
+ float *y, const float *x, int batch,
+ const ColiAffineQuantizedView *view);
+
void coli_metal_tensor_free(ColiMetalTensor *tensor);
/* Weight payload bytes only, preserving the established QT API contract. */
size_t coli_metal_tensor_bytes(const ColiMetalTensor *tensor);
diff --git a/c/backend_metal.mm b/c/backend_metal.mm
index 22671d51..a6fb9dcd 100644
--- a/c/backend_metal.mm
+++ b/c/backend_metal.mm
@@ -1176,6 +1176,48 @@ struct AffineGemvParamsHost {
static_assert(sizeof(AffineGemvParamsHost) == 5*sizeof(uint32_t),
"Metal affine parameter layout drift");
+/* Shared encode/run body for the two affine entry points: the buffers are
+ * either a resident handle's per-section wraps (offsets 0) or one whole-slot
+ * buffer addressed by section byte offsets. Synchronous; returns 0 on
+ * any allocation, encoder, or command-buffer failure. */
+static int affine_dispatch_run(id<MTLBuffer> wbuf, size_t woff,
+ id<MTLBuffer> sbuf, size_t soff,
+ id<MTLBuffer> bbuf, size_t boff,
+ float *y, const float *x, int batch,
+ const ColiAffineQuantizedView *view) {
+ const size_t input_bytes=(size_t)batch*view->input_dim*sizeof(float);
+ const size_t output_bytes=(size_t)batch*view->output_dim*sizeof(float);
+ id<MTLBuffer> bx=[g_dev newBufferWithBytes:x length:input_bytes
+ options:MTLResourceStorageModeShared];
+ id<MTLBuffer> by=[g_dev newBufferWithLength:output_bytes
+ options:MTLResourceStorageModeShared];
+ if (!bx || !by) return 0;
+ AffineGemvParamsHost p = {
+ (uint32_t)view->output_dim, (uint32_t)view->input_dim,
+ (uint32_t)view->group_size, (uint32_t)view->scalar_format,
+ (uint32_t)batch
+ };
+ id<MTLCommandBuffer> cb=[g_queue commandBuffer];
+ id<MTLComputeCommandEncoder> e=[cb computeCommandEncoder];
+ if (!cb || !e) return 0;
+ [e setComputePipelineState:(view->format == COLI_AFFINE_MLX_Q4 ?
+ g_affine_q4 : g_affine_q8)];
+ [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:wbuf offset:woff atIndex:1];
+ [e setBuffer:sbuf offset:soff atIndex:2]; [e setBuffer:bbuf offset:boff atIndex:3];
+ [e setBuffer:by offset:0 atIndex:4]; [e setBytes:&p length:sizeof(p) atIndex:5];
+ [e dispatchThreads:MTLSizeMake(32,(NSUInteger)batch*view->output_dim,1)
+ threadsPerThreadgroup:MTLSizeMake(32,1,1)];
+ [e endEncoding]; [cb commit]; [cb waitUntilCompleted];
+ if (cb.status != MTLCommandBufferStatusCompleted) {
+ fprintf(stderr,"[metal] affine GEMV command buffer failed (status=%ld): %s\n",
+ (long)cb.status,
+ cb.error ? [[cb.error localizedDescription] UTF8String] : "?");
+ return 0;
+ }
+ memcpy(y,[by contents],output_bytes);
+ return 1;
+}
+
extern "C" int coli_metal_matmul_affine(ColiMetalTensor **tp,
float *y, const float *x, int batch,
const ColiAffineQuantizedView *view) {
@@ -1214,43 +1256,10 @@ extern "C" int coli_metal_matmul_affine(ColiMetalTensor **tp,
t->host_w=view->weights; t->host_s=view->scales; t->host_b=view->biases;
}
- const size_t input_bytes=(size_t)batch*view->input_dim*sizeof(float);
- const size_t output_bytes=(size_t)batch*view->output_dim*sizeof(float);
- id<MTLBuffer> bx=[g_dev newBufferWithBytes:x length:input_bytes
- options:MTLResourceStorageModeShared];
- id<MTLBuffer> by=[g_dev newBufferWithLength:output_bytes
- options:MTLResourceStorageModeShared];
- if (!bx || !by) {
+ if (!affine_dispatch_run(t->w, 0, t->s, 0, t->b, 0, y, x, batch, view)) {
if (fresh) tensor_destroy(t);
return 0;
}
- AffineGemvParamsHost p = {
- (uint32_t)view->output_dim, (uint32_t)view->input_dim,
- (uint32_t)view->group_size, (uint32_t)view->scalar_format,
- (uint32_t)batch
- };
- id<MTLCommandBuffer> cb=[g_queue commandBuffer];
- id<MTLComputeCommandEncoder> e=[cb computeCommandEncoder];
- if (!cb || !e) {
- if (fresh) tensor_destroy(t);
- return 0;
- }
- [e setComputePipelineState:(view->format == COLI_AFFINE_MLX_Q4 ?
- g_affine_q4 : g_affine_q8)];
- [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:t->w offset:0 atIndex:1];
- [e setBuffer:t->s offset:0 atIndex:2]; [e setBuffer:t->b offset:0 atIndex:3];
- [e setBuffer:by offset:0 atIndex:4]; [e setBytes:&p length:sizeof(p) atIndex:5];
- [e dispatchThreads:MTLSizeMake(32,(NSUInteger)batch*view->output_dim,1)
- threadsPerThreadgroup:MTLSizeMake(32,1,1)];
- [e endEncoding]; [cb commit]; [cb waitUntilCompleted];
- if (cb.status != MTLCommandBufferStatusCompleted) {
- fprintf(stderr,"[metal] affine GEMV command buffer failed (status=%ld): %s\n",
- (long)cb.status,
- cb.error ? [[cb.error localizedDescription] UTF8String] : "?");
- if (fresh) tensor_destroy(t);
- return 0;
- }
- memcpy(y,[by contents],output_bytes);
if (fresh) {
*tp=t;
g_tensor_count++;
@@ -1260,6 +1269,80 @@ extern "C" int coli_metal_matmul_affine(ColiMetalTensor **tp,
return 1;
}
+// ---- Whole-slot buffers for bounded, refillable qpack expert slots ----
+// One MTLBuffer per slot, wrapped zero-copy EXACTLY once; the pool refills the
+// same host memory with different experts, and dispatches address the weight,
+// scale, and bias sections by byte offset inside the registered buffer. The
+// zero-copy wrap is mandatory, not best-effort: the copying fallback in wrap()
+// would snapshot the first fill and silently detach the GPU from every later
+// refill, which is exactly the stale-projection hazard the whole-slot
+// buffers exist to remove.
+struct ColiMetalSlotBuffer {
+ id<MTLBuffer> buf;
+ void *base;
+ size_t len;
+};
+
+extern "C" ColiMetalSlotBuffer *coli_metal_slot_register(void *base, size_t len) {
+ const size_t pg = 16384; // Apple page: newBufferWithBytesNoCopy contract
+ if (!g_dev || !base || !len) return NULL;
+ if (((uintptr_t)base % pg) != 0 || (len % pg) != 0) return NULL;
+ id<MTLBuffer> b = [g_dev newBufferWithBytesNoCopy:base length:len
+ options:MTLResourceStorageModeShared
+ deallocator:nil];
+ if (!b) return NULL;
+ ColiMetalSlotBuffer *s = new ColiMetalSlotBuffer();
+ s->buf = b; s->base = base; s->len = len;
+ g_tensor_count++; g_tensor_bytes += len;
+ return s;
+}
+
+extern "C" void coli_metal_slot_unregister(ColiMetalSlotBuffer *slot) {
+ if (!slot) return;
+ g_tensor_count--; g_tensor_bytes -= slot->len;
+ slot->buf = nil;
+ delete slot;
+}
+
+extern "C" int coli_metal_matmul_affine_slot(ColiMetalSlotBuffer *slot,
+ size_t weights_offset,
+ size_t scales_offset,
+ size_t biases_offset,
+ float *y, const float *x, int batch,
+ const ColiAffineQuantizedView *view) {
+ if (!slot || !slot->buf || !y || !x || !coli_metal_affine_available() ||
+ !coli_metal_affine_dispatch_supported(view,batch))
+ return 0;
+ @autoreleasepool {
+ const unsigned bits=coli_affine_bits(view->format), per_word=32u/bits;
+ const size_t groups=view->input_dim/view->group_size;
+ const size_t wbytes=view->output_dim*(view->input_dim/per_word)*sizeof(uint32_t);
+ const size_t scalar_bytes=view->output_dim*groups*
+ coli_affine_scalar_size(view->scalar_format);
+ /* Every section must lie inside the registered slot, and the weight words
+ * must land 4-byte aligned (base is page-aligned; the kernel casts each
+ * row to device const uint*). */
+ if (weights_offset % 4 != 0) return 0;
+ if (weights_offset > slot->len || wbytes > slot->len - weights_offset)
+ return 0;
+ if (scales_offset > slot->len || scalar_bytes > slot->len - scales_offset)
+ return 0;
+ if (biases_offset > slot->len || scalar_bytes > slot->len - biases_offset)
+ return 0;
+ /* Tripwire: the descriptor must describe THIS slot's current fill. A
+ * view carried across a refill (or aimed at another slot) disagrees with
+ * base+offset and is refused instead of dispatched. */
+ const unsigned char *base=(const unsigned char *)slot->base;
+ if (view->weights != base + weights_offset ||
+ view->scales != base + scales_offset ||
+ view->biases != base + biases_offset)
+ return 0;
+ return affine_dispatch_run(slot->buf, weights_offset,
+ slot->buf, scales_offset,
+ slot->buf, biases_offset, y, x, batch, view);
+ }
+}
+
// ---- fused decode attention scratch (GLM-5.2 dims) ----
enum { AH=6144, AHEADS=64, AQLORA=2048, AKVL=512, AROPE=64, AVH=256, AQH=256, ANOPE=192, AROWSH=448, AHQH=AHEADS*AQH, AHVH=AHEADS*AVH, AMAXS=4 };
static id<MTLBuffer> ax_,aqr_,aqf_,acomp_,aqabs_,ascore_,aclat_,actx_,aout_,aqaln_,akvaln_; static size_t ascore_cap;
diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mm
index 165e673e..9416e719 100644
--- a/c/tests/test_backend_metal.mm
+++ b/c/tests/test_backend_metal.mm
@@ -200,6 +200,115 @@ static int run_affine(ColiAffineFormat format, ColiAffineScalarFormat scalar_for
return ok?0:1;
}
+// ---- Slot buffers: registered once, offset-addressed, refilled in place ----
+// Three claims: (1) offset addressing inside one whole-slot buffer matches the
+// CPU reference; (2) refilling the SAME registered memory is seen by the next
+// dispatch (write-through -- the property the resident-handle cache cannot give
+// reused memory); (3) the refusal arms hold: misaligned registration, sections
+// outside the registered range, and a view that does not describe the slot's
+// current fill are refused, never dispatched.
+static int run_affine_slot(ColiAffineFormat format, ColiAffineScalarFormat scalar_format,
+ int S, const char *name) {
+ enum { O=48, I=2048, GROUP=32, HEADER=256 }; // HEADER: prove nonzero offsets
+ const size_t pg=16384;
+ unsigned bits=coli_affine_bits(format), per_word=32u/bits;
+ size_t packed_words=(size_t)O*(I/per_word);
+ size_t wbytes=packed_words*sizeof(uint32_t);
+ size_t scalar_count=(size_t)O*(I/GROUP);
+ size_t sbytes=scalar_count*coli_affine_scalar_size(scalar_format);
+ size_t woff=HEADER, soff=woff+wbytes, boff=soff+sbytes;
+ size_t len=((boff+sbytes)+pg-1)&~(pg-1);
+ uint8_t *slab=nullptr;
+ if (posix_memalign((void**)&slab,pg,len)) {
+ printf(" %-38s FAIL (posix_memalign)\n",name); return 1; }
+ memset(slab,0,len);
+
+ auto fill=[&](uint32_t seed){
+ uint32_t state=seed;
+ uint32_t *w=(uint32_t*)(void*)(slab+woff);
+ for(size_t i=0;i<packed_words;i++){ state=state*1664525u+1013904223u; w[i]=state; }
+ std::vector<uint8_t> sc(sbytes), bi(sbytes);
+ for(size_t i=0;i<scalar_count;i++){
+ state=state*1664525u+1013904223u;
+ float scale=(float)((int)(state%7u)-3)*0.03125f;
+ float bias=(float)((int)((state>>8)%5u)-2)*0.015625f;
+ if(scale==0.0f) scale=0.046875f;
+ affine_store_scalar(sc,i,scalar_format,scale);
+ affine_store_scalar(bi,i,scalar_format,bias);
+ }
+ memcpy(slab+soff,sc.data(),sbytes); memcpy(slab+boff,bi.data(),sbytes);
+ };
+
+ ColiAffineQuantizedView view = {
+ slab+woff, slab+soff, slab+boff, wbytes, sbytes, sbytes,
+ (size_t)O, (size_t)I, (size_t)GROUP, format, scalar_format
+ };
+ std::vector<float> x((size_t)S*I), expected((size_t)S*O), actual((size_t)S*O);
+ for(size_t i=0;i<x.size();i++) x[i]=(float)((int)((i*13+5)%29)-14)*0.0625f;
+ auto relative_error=[&](){
+ double maxabs=0.0, ymax=0.0;
+ for(size_t i=0;i<actual.size();i++){
+ if(!std::isfinite(actual[i]) || !std::isfinite(expected[i]))
+ return (double)INFINITY;
+ maxabs=fmax(maxabs,fabs((double)actual[i]-expected[i]));
+ ymax=fmax(ymax,fabs((double)expected[i]));
+ }
+ return maxabs/(ymax+1e-9);
+ };
+
+ size_t count_before=0, bytes_before=0;
+ coli_metal_stats(&count_before,&bytes_before);
+ // A misaligned base or a non-page-multiple length must not register at all:
+ // those layouts would take wrap()'s copying fallback and go stale on refill.
+ int misaligned_refused = coli_metal_slot_register(slab+4,len)==NULL &&
+ coli_metal_slot_register(slab,len-4)==NULL;
+ ColiMetalSlotBuffer *slot=coli_metal_slot_register(slab,len);
+ size_t count_reg=0, bytes_reg=0;
+ coli_metal_stats(&count_reg,&bytes_reg);
+ int registered = slot!=NULL && count_reg==count_before+1 &&
+ bytes_reg==bytes_before+len;
+
+ fill(0xC4C40001u ^ (uint32_t)format ^ ((uint32_t)scalar_format<<4));
+ int ref_ok=coli_affine_matmul_ref(expected.data(),x.data(),(size_t)S,&view)==COLI_AFFINE_OK;
+ std::fill(actual.begin(),actual.end(),NAN);
+ int first = registered && ref_ok &&
+ coli_metal_matmul_affine_slot(slot,woff,soff,boff,actual.data(),x.data(),S,&view);
+ double first_nerr = first ? relative_error() : INFINITY;
+
+ // Refill the same registered memory with a different expert; the next
+ // dispatch must read the NEW bytes through the same registration.
+ fill(0x5EED0002u ^ (uint32_t)format ^ ((uint32_t)scalar_format<<4));
+ int ref2_ok=coli_affine_matmul_ref(expected.data(),x.data(),(size_t)S,&view)==COLI_AFFINE_OK;
+ std::fill(actual.begin(),actual.end(),NAN);
+ int refilled = first && ref2_ok &&
+ coli_metal_matmul_affine_slot(slot,woff,soff,boff,actual.data(),x.data(),S,&view);
+ double refill_nerr = refilled ? relative_error() : INFINITY;
+
+ // Sections outside the registered range are refused.
+ int oob_refused = refilled &&
+ !coli_metal_matmul_affine_slot(slot,len-4,soff,boff,actual.data(),x.data(),S,&view) &&
+ !coli_metal_matmul_affine_slot(slot,woff,len,boff,actual.data(),x.data(),S,&view) &&
+ !coli_metal_matmul_affine_slot(slot,woff,soff,len,actual.data(),x.data(),S,&view);
+ // A view that does not describe the claimed offsets (a stale or misdirected
+ // descriptor) is refused, not dispatched.
+ int mismatch_refused = refilled &&
+ !coli_metal_matmul_affine_slot(slot,woff+4,soff,boff,actual.data(),x.data(),S,&view) &&
+ !coli_metal_matmul_affine_slot(slot,woff,boff,soff,actual.data(),x.data(),S,&view) &&
+ !coli_metal_matmul_affine_slot(NULL,woff,soff,boff,actual.data(),x.data(),S,&view);
+
+ coli_metal_slot_unregister(slot);
+ size_t count_after=0, bytes_after=0;
+ coli_metal_stats(&count_after,&bytes_after);
+ int restored = count_after==count_before && bytes_after==bytes_before;
+ free(slab);
+ int ok = misaligned_refused && registered && first && first_nerr<1e-4 &&
+ refilled && refill_nerr<1e-4 && oob_refused && mismatch_refused &&
+ restored;
+ printf(" %-38s nerr=%.2e/%.2e (fill/refill) S=%d %s\n",
+ name,first_nerr,refill_nerr,S,ok?"ok":"*** MISMATCH");
+ return ok?0:1;
+}
+
// ---- fmt=4 (grouped int4): own CPU reference + harness -----------------------------
// cpu_ref's [O] per-row scale layout can't express fmt=4's [O,ceil(I/gs)] per-group
// scale, so this is a separate reference rather than a branch of cpu_ref. Mirrors
@@ -992,6 +1101,10 @@ int main(void) {
(int)coli_affine_bits((ColiAffineFormat)format),scalar_name[scalar]);
fail |= run_affine((ColiAffineFormat)format,(ColiAffineScalarFormat)scalar,3,name);
}
+ printf("Metal MLX affine slot-buffer tests (register once, offset-address, refill):\n");
+ fail |= run_affine_slot(COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F32, 3, "affine slot Q4/f32 offset+refill");
+ fail |= run_affine_slot(COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F16, 2, "affine slot Q4/f16 offset+refill");
+ fail |= run_affine_slot(COLI_AFFINE_MLX_Q8, COLI_AFFINE_SCALAR_BF16, 3, "affine slot Q8/bf16 offset+refill");
} else if (affine_capability==COLI_METAL_AFFINE_CAP_SIMD_WIDTH_UNSUPPORTED) {
printf("Metal MLX affine packed GEMV tests: SKIPPED (requires 32-lane simdgroups)\n");
} else {