Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Metal DeepSeek: stop leaking Metal buffers and command buffers (16 -> 3 tok/s after a few requests)
Niko1221061c54201016
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 05c397ab0..78188d8aa 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -1267,6 +1267,11 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intdsp.tg[0] = 256;dsp.tg[1] = dsp.tg[2] = 1;g_cx->run({dsp});+ // the stand-ins for absent inputs are fresh Metal buffers (16 KB of address space each): give them back+ if (hw == nullptr) g_cx->free(bw);+ if (hc == nullptr) g_cx->free(bc);+ if (hs == nullptr) g_cx->free(bs);+ if (hk == nullptr) g_cx->free(bk);if (use_mx && attn_mode == 2) { // debugging: the scalar kernel on the same inputs, then the differencemtl::Pipeline ps;try { ps = g_cx->pipeline(g_lib, "ds4_sparse_attn_g8"); } catch (...) { return true; }@@ -1347,6 +1352,7 @@ bool route(const float* logits, const float* bias, int n, int k, int T, float rodsp.tg[1] = dsp.tg[2] = 1;g_cx->run({dsp});g_cx->free(bbv);+ if (hv == nullptr) g_cx->free(bv);return true;}@@ -1439,6 +1445,8 @@ bool combine(const float* ed, const float* cpu, const float* sh, const void* c,dsp.grid[2] = 1;dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;g_cx->run({dsp});+ if (hc == nullptr) g_cx->free(bc);+ if (hs == nullptr) g_cx->free(bs);return true;}@@ -2702,6 +2710,9 @@ extern "C" void* ds4_ctx_mark() {if (!g_cx) return nullptr;return g_cx->mark();}+extern "C" void ds4_ctx_drop(void* token) {+ if (g_cx && token) g_cx->drop_mark(token);+}extern "C" void ds4_ctx_wait(void* token) {static const bool no_wait_split = std::getenv("DS4_NO_WAIT_SPLIT") != nullptr;if (g_cx && !no_wait_split) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex f0547effa..43e2d30ed 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -175,8 +175,22 @@ const Alloc* find(const void* p, size_t* off) {return nullptr;}+#if defined(STRATA_DS4_RUNTIME)+// The engine's host threads (the prompt path, the loader, the expert cache) have no autorelease pool, and metal-cpp's+// commandBuffer() / computeCommandEncoder() / blitCommandEncoder() hand back autoreleased objects: with no pool that last reference was+// never given back, so every command buffer and encoder stayed alive with the buffers it had named (~800 command buffers a+// long prompt). A pool around each creation (and, for the blits, around the whole encode) gives it back.+struct ScopedPool {+ NS::AutoreleasePool* p;+ ScopedPool() : p(NS::AutoreleasePool::alloc()->init()) {}+ ~ScopedPool() { p->release(); }+};+#endifMTL::CommandBuffer* open_cb(StreamImpl* s) {if (!s->cb) {+#if defined(STRATA_DS4_RUNTIME)+ ScopedPool pool;+#endifs->cb = s->q->commandBufferWithUnretainedReferences();s->cb->retain();++rt().stats.command_buffers;@@ -192,7 +206,7 @@ static std::recursive_mutex g_stream_mu; // encode_copy/flush nestvoid end_encoder(StreamImpl* s) {#if defined(STRATA_DS4_RUNTIME)if (s->enc) { if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] close compute enc %p\n", (void*) s->enc);- s->enc->endEncoding(); s->enc = nullptr; }+ s->enc->endEncoding(); s->enc->release(); s->enc = nullptr; }#elseif (s->enc) { s->enc->endEncoding(); s->enc = nullptr; }#endif@@ -200,7 +214,9 @@ void end_encoder(StreamImpl* s) {MTL::ComputeCommandEncoder* encoder(StreamImpl* s) {#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk(g_stream_mu);- if (!s->enc) { s->enc = open_cb(s)->computeCommandEncoder();+ if (!s->enc) {+ MTL::CommandBuffer* cb = open_cb(s);+ { ScopedPool pool; s->enc = cb->computeCommandEncoder(); s->enc->retain(); } // (released in end_encoder)if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] open compute enc %p (cb %p)\n", (void*) s->enc, (void*) s->cb); } // serial dispatch: each dispatch sees the last#elseif (!s->enc) s->enc = open_cb(s)->computeCommandEncoder(); // serial dispatch: each dispatch sees the last@@ -229,6 +245,7 @@ extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) __attribute__extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) __attribute__((weak_import));extern "C" void* ds4_ctx_mark() __attribute__((weak_import));extern "C" void ds4_ctx_wait(void* token) __attribute__((weak_import));+extern "C" void ds4_ctx_drop(void* token) __attribute__((weak_import));static bool ctx_queue_ops() { // DS4_STREAM_COPIES=1: the old way (a copy on a stream's own queue, a full barrier each)static const bool on = ds4_ctx_copy != nullptr && std::getenv("DS4_STREAM_COPIES") == nullptr;return on;@@ -262,6 +279,9 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {check_purged(as, "copy");if (ad && as) { // buffer to buffer: a blit, in stream orderend_encoder(s);+#if defined(STRATA_DS4_RUNTIME)+ ScopedPool pool;+#endifMTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();#if defined(STRATA_DS4_RUNTIME)if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] blit copy %zu\n", n);@@ -290,6 +310,9 @@ Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {if (!a) return fail(kErrorInvalidValue);check_purged(a, "fill");end_encoder(s);+#if defined(STRATA_DS4_RUNTIME)+ ScopedPool pool;+#endifMTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();b->fillBuffer(a->buf, NS::Range::Make(off, n), (uint8_t) value);b->endEncoding();@@ -659,10 +682,17 @@ Error launch_host_func(Stream st, HostFn fn, void* arg) {}Error event_create(Event* e, unsigned) { *e = new EventImpl; return kSuccess; }-Error event_destroy(Event e) { delete e; return kSuccess; }+Error event_destroy(Event e) {+#if defined(STRATA_DS4_RUNTIME)+ if (e && e->token && ds4_ctx_drop) { ds4_ctx_drop(e->token); e->token = nullptr; } // (a mark nobody waited for holds its command buffer)+#endif+ delete e;+ return kSuccess;+}Error event_record(Event e, Stream st) {#if defined(STRATA_DS4_RUNTIME)if (ctx_queue_ops() && !S(st)->rec) { // the work the event follows is in the context's command buffer: end it, wait later+ if (e->token && ds4_ctx_drop) ds4_ctx_drop(e->token); // (recorded again before it was waited for: the earlier mark would leak its command buffer)e->token = ds4_ctx_mark();e->ev = nullptr;e->value = 0;diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex 1fd137109..fb1e6b631 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -113,6 +113,7 @@ public:// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.void* mark(); // split_runs() that returns a token for the buffer just committed (nullptr: nothing was queued)void wait_mark(void* token); // block until that buffer has completed, then drop the token+ void drop_mark(void* token); // drop a token without waiting (an event recorded again, or destroyed, before it was waited on)void split_runs(); // commit the batched dispatches as their own command buffer, without waitingvoid flush_runs(); // commit and wait for the dispatches batched since the last flushdouble run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 2b3786648..104434a85 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -549,6 +549,26 @@ void require_fit(const Pipeline& p, uint32_t threads, const char* where) {}Buffer Context::alloc(size_t bytes) {+#if defined(STRATA_DS4_RUNTIME)+ // Metal returns the new buffer with one more reference parked in the calling thread's autorelease pool. The engine's host+ // threads have no pool (only Context::run opens one), so that reference was never given back: every buffer made here lived for+ // the rest of the process, free() or not (a 16 KB job array per expert matvec, ~90K a request: +1.3 GB and a slower queue every+ // call, 16 -> 3 tok/s after a few requests on a big Mac).+ Buffer r;+ @autoreleasepool {+ id<MTLBuffer> b = [impl_->dev newBufferWithLength:bytes options:MTLResourceStorageModeShared];+ if (!b) {+ std::fprintf(stderr, "mtl: alloc %zu failed\n", bytes);+ std::exit(4);+ }+ std::memset([b contents], 0, bytes);+ r.handle = (__bridge_retained void*)b;+ r.ptr = [b contents];+ }+ allocated_ += bytes;+ r.bytes = bytes;+ return r;+#elseid<MTLBuffer> b = [impl_->dev newBufferWithLength:bytes options:MTLResourceStorageModeShared];if (!b) {std::fprintf(stderr, "mtl: alloc %zu failed\n", bytes);@@ -561,6 +581,7 @@ Buffer Context::alloc(size_t bytes) {r.ptr = [b contents];r.bytes = bytes;return r;+#endif}Buffer Context::wrap(void* p, size_t bytes, size_t max_span) {@@ -621,6 +642,21 @@ Buffer Context::wrap(void* p, size_t bytes) {if (std::getenv("DS4_WRAP_LOG")) // debugging: a wrap that is not inside one runtime allocation gets a second GPU mapping of the pagesstd::fprintf(stderr, "mtl: wrap FALLBACK p=%p bytes=%zu alloc_remaining=%zu\n", p, bytes, ds4_gpu_alloc_remaining ? ds4_gpu_alloc_remaining(p) : (size_t) 0);#endif+#if defined(STRATA_DS4_RUNTIME)+ Buffer r;+ @autoreleasepool { // (see alloc: the engine's host threads have no pool of their own)+ id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];+ if (!b) {+ std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);+ std::exit(4);+ }+ r.handle = (__bridge_retained void*)b;+ }+ allocated_ += bytes;+ r.ptr = p;+ r.bytes = bytes;+ return r;+#elseid<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];if (!b) {std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);@@ -632,6 +668,7 @@ Buffer Context::wrap(void* p, size_t bytes) {r.ptr = p;r.bytes = bytes;return r;+#endif}void Context::free(Buffer& b) {@@ -803,6 +840,11 @@ void* Context::mark() {return (__bridge_retained void*) impl_->inflight;}+void Context::drop_mark(void* token) {+ // The token holds a +1 on a committed command buffer; one nobody waits for is never freed.+ if (token) { id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token; cb = nil; }+}+void Context::wait_mark(void* token) {id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;const auto tw0 = std::chrono::steady_clock::now();