Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Metal DeepSeek: stop leaking Metal buffers and command buffers (16 -> 3 tok/s after a few requests)

Niko1221061c54201016
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 05c397ab0..78188d8aa 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -1267,6 +1267,11 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
dsp.tg[0] = 256;
dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
+ // the stand-ins for absent inputs are fresh Metal buffers (16 KB of address space each): give them back
+ if (hw == nullptr) g_cx->free(bw);
+ if (hc == nullptr) g_cx->free(bc);
+ if (hs == nullptr) g_cx->free(bs);
+ if (hk == nullptr) g_cx->free(bk);
if (use_mx && attn_mode == 2) { // debugging: the scalar kernel on the same inputs, then the difference
mtl::Pipeline ps;
try { ps = g_cx->pipeline(g_lib, "ds4_sparse_attn_g8"); } catch (...) { return true; }
@@ -1347,6 +1352,7 @@ bool route(const float* logits, const float* bias, int n, int k, int T, float ro
dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
g_cx->free(bbv);
+ if (hv == nullptr) g_cx->free(bv);
return true;
}
@@ -1439,6 +1445,8 @@ bool combine(const float* ed, const float* cpu, const float* sh, const void* c,
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
+ if (hc == nullptr) g_cx->free(bc);
+ if (hs == nullptr) g_cx->free(bs);
return true;
}
@@ -2702,6 +2710,9 @@ extern "C" void* ds4_ctx_mark() {
if (!g_cx) return nullptr;
return g_cx->mark();
}
+extern "C" void ds4_ctx_drop(void* token) {
+ if (g_cx && token) g_cx->drop_mark(token);
+}
extern "C" void ds4_ctx_wait(void* token) {
static const bool no_wait_split = std::getenv("DS4_NO_WAIT_SPLIT") != nullptr;
if (g_cx && !no_wait_split) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index f0547effa..43e2d30ed 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -175,8 +175,22 @@ const Alloc* find(const void* p, size_t* off) {
return nullptr;
}
+#if defined(STRATA_DS4_RUNTIME)
+// The engine's host threads (the prompt path, the loader, the expert cache) have no autorelease pool, and metal-cpp's
+// commandBuffer() / computeCommandEncoder() / blitCommandEncoder() hand back autoreleased objects: with no pool that last reference was
+// never given back, so every command buffer and encoder stayed alive with the buffers it had named (~800 command buffers a
+// long prompt). A pool around each creation (and, for the blits, around the whole encode) gives it back.
+struct ScopedPool {
+ NS::AutoreleasePool* p;
+ ScopedPool() : p(NS::AutoreleasePool::alloc()->init()) {}
+ ~ScopedPool() { p->release(); }
+};
+#endif
MTL::CommandBuffer* open_cb(StreamImpl* s) {
if (!s->cb) {
+#if defined(STRATA_DS4_RUNTIME)
+ ScopedPool pool;
+#endif
s->cb = s->q->commandBufferWithUnretainedReferences();
s->cb->retain();
++rt().stats.command_buffers;
@@ -192,7 +206,7 @@ static std::recursive_mutex g_stream_mu; // encode_copy/flush nest
void end_encoder(StreamImpl* s) {
#if defined(STRATA_DS4_RUNTIME)
if (s->enc) { if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] close compute enc %p\n", (void*) s->enc);
- s->enc->endEncoding(); s->enc = nullptr; }
+ s->enc->endEncoding(); s->enc->release(); s->enc = nullptr; }
#else
if (s->enc) { s->enc->endEncoding(); s->enc = nullptr; }
#endif
@@ -200,7 +214,9 @@ void end_encoder(StreamImpl* s) {
MTL::ComputeCommandEncoder* encoder(StreamImpl* s) {
#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
- if (!s->enc) { s->enc = open_cb(s)->computeCommandEncoder();
+ if (!s->enc) {
+ MTL::CommandBuffer* cb = open_cb(s);
+ { ScopedPool pool; s->enc = cb->computeCommandEncoder(); s->enc->retain(); } // (released in end_encoder)
if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] open compute enc %p (cb %p)\n", (void*) s->enc, (void*) s->cb); } // serial dispatch: each dispatch sees the last
#else
if (!s->enc) s->enc = open_cb(s)->computeCommandEncoder(); // serial dispatch: each dispatch sees the last
@@ -229,6 +245,7 @@ extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) __attribute__
extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) __attribute__((weak_import));
extern "C" void* ds4_ctx_mark() __attribute__((weak_import));
extern "C" void ds4_ctx_wait(void* token) __attribute__((weak_import));
+extern "C" void ds4_ctx_drop(void* token) __attribute__((weak_import));
static bool ctx_queue_ops() { // DS4_STREAM_COPIES=1: the old way (a copy on a stream's own queue, a full barrier each)
static const bool on = ds4_ctx_copy != nullptr && std::getenv("DS4_STREAM_COPIES") == nullptr;
return on;
@@ -262,6 +279,9 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
check_purged(as, "copy");
if (ad && as) { // buffer to buffer: a blit, in stream order
end_encoder(s);
+#if defined(STRATA_DS4_RUNTIME)
+ ScopedPool pool;
+#endif
MTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();
#if defined(STRATA_DS4_RUNTIME)
if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] blit copy %zu\n", n);
@@ -290,6 +310,9 @@ Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {
if (!a) return fail(kErrorInvalidValue);
check_purged(a, "fill");
end_encoder(s);
+#if defined(STRATA_DS4_RUNTIME)
+ ScopedPool pool;
+#endif
MTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();
b->fillBuffer(a->buf, NS::Range::Make(off, n), (uint8_t) value);
b->endEncoding();
@@ -659,10 +682,17 @@ Error launch_host_func(Stream st, HostFn fn, void* arg) {
}
Error event_create(Event* e, unsigned) { *e = new EventImpl; return kSuccess; }
-Error event_destroy(Event e) { delete e; return kSuccess; }
+Error event_destroy(Event e) {
+#if defined(STRATA_DS4_RUNTIME)
+ if (e && e->token && ds4_ctx_drop) { ds4_ctx_drop(e->token); e->token = nullptr; } // (a mark nobody waited for holds its command buffer)
+#endif
+ delete e;
+ return kSuccess;
+}
Error event_record(Event e, Stream st) {
#if defined(STRATA_DS4_RUNTIME)
if (ctx_queue_ops() && !S(st)->rec) { // the work the event follows is in the context's command buffer: end it, wait later
+ if (e->token && ds4_ctx_drop) ds4_ctx_drop(e->token); // (recorded again before it was waited for: the earlier mark would leak its command buffer)
e->token = ds4_ctx_mark();
e->ev = nullptr;
e->value = 0;
diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.h
index 1fd137109..fb1e6b631 100644
--- a/metal/src/mtl_rt.h
+++ b/metal/src/mtl_rt.h
@@ -113,6 +113,7 @@ public:
// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.
void* mark(); // split_runs() that returns a token for the buffer just committed (nullptr: nothing was queued)
void wait_mark(void* token); // block until that buffer has completed, then drop the token
+ void drop_mark(void* token); // drop a token without waiting (an event recorded again, or destroyed, before it was waited on)
void split_runs(); // commit the batched dispatches as their own command buffer, without waiting
void flush_runs(); // commit and wait for the dispatches batched since the last flush
double run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 2b3786648..104434a85 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -549,6 +549,26 @@ void require_fit(const Pipeline& p, uint32_t threads, const char* where) {
}
Buffer Context::alloc(size_t bytes) {
+#if defined(STRATA_DS4_RUNTIME)
+ // Metal returns the new buffer with one more reference parked in the calling thread's autorelease pool. The engine's host
+ // threads have no pool (only Context::run opens one), so that reference was never given back: every buffer made here lived for
+ // the rest of the process, free() or not (a 16 KB job array per expert matvec, ~90K a request: +1.3 GB and a slower queue every
+ // call, 16 -> 3 tok/s after a few requests on a big Mac).
+ Buffer r;
+ @autoreleasepool {
+ id<MTLBuffer> b = [impl_->dev newBufferWithLength:bytes options:MTLResourceStorageModeShared];
+ if (!b) {
+ std::fprintf(stderr, "mtl: alloc %zu failed\n", bytes);
+ std::exit(4);
+ }
+ std::memset([b contents], 0, bytes);
+ r.handle = (__bridge_retained void*)b;
+ r.ptr = [b contents];
+ }
+ allocated_ += bytes;
+ r.bytes = bytes;
+ return r;
+#else
id<MTLBuffer> b = [impl_->dev newBufferWithLength:bytes options:MTLResourceStorageModeShared];
if (!b) {
std::fprintf(stderr, "mtl: alloc %zu failed\n", bytes);
@@ -561,6 +581,7 @@ Buffer Context::alloc(size_t bytes) {
r.ptr = [b contents];
r.bytes = bytes;
return r;
+#endif
}
Buffer Context::wrap(void* p, size_t bytes, size_t max_span) {
@@ -621,6 +642,21 @@ Buffer Context::wrap(void* p, size_t bytes) {
if (std::getenv("DS4_WRAP_LOG")) // debugging: a wrap that is not inside one runtime allocation gets a second GPU mapping of the pages
std::fprintf(stderr, "mtl: wrap FALLBACK p=%p bytes=%zu alloc_remaining=%zu\n", p, bytes, ds4_gpu_alloc_remaining ? ds4_gpu_alloc_remaining(p) : (size_t) 0);
#endif
+#if defined(STRATA_DS4_RUNTIME)
+ Buffer r;
+ @autoreleasepool { // (see alloc: the engine's host threads have no pool of their own)
+ id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];
+ if (!b) {
+ std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);
+ std::exit(4);
+ }
+ r.handle = (__bridge_retained void*)b;
+ }
+ allocated_ += bytes;
+ r.ptr = p;
+ r.bytes = bytes;
+ return r;
+#else
id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];
if (!b) {
std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);
@@ -632,6 +668,7 @@ Buffer Context::wrap(void* p, size_t bytes) {
r.ptr = p;
r.bytes = bytes;
return r;
+#endif
}
void Context::free(Buffer& b) {
@@ -803,6 +840,11 @@ void* Context::mark() {
return (__bridge_retained void*) impl_->inflight;
}
+void Context::drop_mark(void* token) {
+ // The token holds a +1 on a committed command buffer; one nobody waits for is never freed.
+ if (token) { id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token; cb = nil; }
+}
+
void Context::wait_mark(void* token) {
id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;
const auto tw0 = std::chrono::steady_clock::now();