Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: dispatches BATCH into one command buffer - decode 0.16 -> 48.7 tok/s (300x)
Niko1221611530d9929b
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 8dabb42cd..e309b0b63 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -14,6 +14,11 @@#include <cstdint>#include <cstring>+// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read+extern "C" void ds4_metal_flush();++#define DSV41_PROBE(fmt, ...) do { static bool once_ = false; if (!once_) { once_ = true; std::fprintf(stderr, "[probe] " fmt, __VA_ARGS__); } } while (0)+namespace strata::dsv41::metal_host {// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Finish (Metal has no 'long long': 64-bit is 'long')@@ -1409,4 +1414,9 @@ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, voireturn true;}-} // namespace strata::dsv41::metal_host++extern "C" void ds4_metal_flush() {+ if (g_cx) g_cx->flush_runs();+}++} // namespace strata::dsv41::metal_host\ No newline at end of filediff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex fb01c0e4a..a4510c0c8 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -189,6 +189,8 @@ void flush(StreamImpl* s) {s->cb = nullptr;s->pending = 0;}+extern "C" void ds4_metal_flush();+Error wait_value(MTL::SharedEvent* ev, uint64_t v) {while (ev->signaledValue() < v) std::this_thread::yield();++rt().stats.host_syncs;@@ -373,6 +375,7 @@ Error get_device(int* dev) { *dev = 0; return kSuccess; }Error set_device(int dev) { return dev == 0 ? kSuccess : fail(kErrorInvalidValue); }Error device_count(int* n) { *n = rt().dev ? 1 : 0; return kSuccess; }Error device_synchronize() {+ ds4_metal_flush();Runtime& r = rt();StreamImpl* s = S(nullptr);flush(s);diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex 6bd3bfacf..136dab06e 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -100,6 +100,7 @@ public:// Encodes `ds` into ONE compute encoder (serial or concurrent dispatch type) and ONE command buffer, `reps`// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.+ void flush_runs(); // commit and wait for the dispatches batched since the last flushdouble run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);// A dependency graph in ONE concurrent encoder: a memory barrier only before dispatches with `barrier` set, so// the dispatches between two barriers may overlap. Returns the GPU time in seconds.diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 0fb436222..4f94480af 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -29,6 +29,9 @@ std::string read_file(const std::string& path) {}struct Context::Impl {+ id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate in+ id<MTLComputeCommandEncoder> enc_def = nil;+ int def_n = 0;id<MTLDevice> dev;id<MTLCommandQueue> q;std::vector<id<MTLLibrary>> libs;@@ -272,8 +275,31 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];}+void Context::flush_runs() {+ if (impl_->cb_def == nil) return;+ [impl_->enc_def endEncoding];+ [impl_->cb_def commit];+ [impl_->cb_def waitUntilCompleted];+ if (impl_->cb_def.status != MTLCommandBufferStatusCompleted)+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->cb_def.error localizedDescription] UTF8String]);+ impl_->cb_def = nil;+ impl_->enc_def = nil;+ impl_->def_n = 0;+}+double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {@autoreleasepool {+ if (ds.size() == 1 && reps == 1 && !barrier_between) {+ if (impl_->cb_def == nil || impl_->def_n >= 20000) {+ flush_runs();+ impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];+ impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];+ }+ encode(impl_->enc_def, ds[0]);+ ++impl_->def_n;+ return 0.0;+ }+ flush_runs();id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];id<MTLComputeCommandEncoder> enc =[cb computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];