Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: keep the GPU busy (flag-released spin kernel) while the CPU lane runs: decode +25-35%
Niko12218d3c976166b9
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 12816ab0d..8a0ba7978 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -81,6 +81,7 @@ struct Timeline {std::vector<std::string> names; // per encoderuint64_t cpu0 = 0, gpu0 = 0;bool reg = false;+ bool gate = true; // ds4_timeline_gate: record only while openbool att = false; // the encoder just made carries a sample slot} g_tl;@@ -119,8 +120,9 @@ id<MTLComputeCommandEncoder> tl_make_encoder(id<MTLDevice> dev, id<MTLCommandBufg_tl.gpu0 = g0;g_tl.names.assign(g_tl.cap, std::string());g_tl.after = std::getenv("DS4_TL_AFTER") ? std::atol(std::getenv("DS4_TL_AFTER")) : 0;+ if (std::getenv("DS4_TL_DRAFT")) g_tl.gate = false;}- if (g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];+ if (!g_tl.gate || g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];const size_t bi = g_tl.n / g_tl.per, si = g_tl.n % g_tl.per;if (bi >= g_tl.bufs.size()) {MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];@@ -203,6 +205,71 @@ void tl_report(id<MTLDevice> dev) {}} // namespace+extern "C" void ds4_timeline_gate(int open) { g_tl.gate = open != 0; }+++// ---- ds4_busy_begin / ds4_busy_end: a one-threadgroup kernel on a second queue that spins on a shared flag while the host works+// (the CPU lane), so the GPU is not idle between two bursts (idle -> the next kernels run 2-3x slower); the host sets the flag the+// moment it is done and the kernel leaves within microseconds. DS4_BUSY=0 turns it off.+namespace {+struct Busy {+ bool init = false, ok = false;+ id<MTLCommandQueue> q = nil;+ id<MTLComputePipelineState> ps = nil;+ id<MTLBuffer> flag = nil;+ id<MTLCommandBuffer> last = nil;+};+Busy g_busy;+} // namespace++extern "C" void ds4_busy_begin() {+ static const bool off = std::getenv("DS4_BUSY") && std::string(std::getenv("DS4_BUSY")) == "0";+ if (off) return;+ @autoreleasepool {+ if (!g_busy.init) {+ g_busy.init = true;+ id<MTLDevice> dev = MTLCreateSystemDefaultDevice();+ static const char* kSrc = R"MSL(+#include <metal_stdlib>+using namespace metal;+kernel void busy_spin(device volatile atomic_uint* flag [[buffer(0)]], constant uint& cap [[buffer(1)]], uint t [[thread_position_in_grid]]) {+ uint x = t;+ for (uint i = 0; i < cap; ++i) {+ for (uint j = 0; j < 64; ++j) x = x * 1664525u + 1013904223u;+ if (atomic_load_explicit(flag, memory_order_relaxed) != 0u) break;+ }+ if (x == 0xdeadbeefu) atomic_store_explicit(flag, 2u, memory_order_relaxed);+}+)MSL";+ NSError* err = nil;+ id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSrc] options:nil error:&err];+ if (lib) g_busy.ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"busy_spin"] error:&err];+ if (g_busy.ps) {+ g_busy.q = [dev newCommandQueue];+ g_busy.flag = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];+ g_busy.ok = true;+ }+ }+ if (!g_busy.ok) return;+ if (g_busy.last != nil) [g_busy.last waitUntilCompleted]; // (the previous spin has left)+ *(volatile uint32_t*) [g_busy.flag contents] = 0;+ uint32_t cap = std::getenv("DS4_BUSY_CAP") ? (uint32_t) std::atoi(std::getenv("DS4_BUSY_CAP")) : 20000u; // ~ tens of ms at most+ id<MTLCommandBuffer> cb = [g_busy.q commandBuffer];+ id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];+ [enc setComputePipelineState:g_busy.ps];+ [enc setBuffer:g_busy.flag offset:0 atIndex:0];+ [enc setBytes:&cap length:4 atIndex:1];+ [enc dispatchThreads:MTLSizeMake(256, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];+ [enc endEncoding];+ [cb commit];+ g_busy.last = cb;+ }+}+extern "C" void ds4_busy_end() {+ if (!g_busy.ok || g_busy.last == nil) return;+ *(volatile uint32_t*) [g_busy.flag contents] = 1;+}+Context::Context() : impl_(new Impl) {impl_->dev = MTLCreateSystemDefaultDevice();if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 3e36b2fcc..1ecf4fa2a 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -160,6 +160,15 @@ struct RoundTimes { // DS4_ROUND_TIMES: where a decode round's time goes (ms,#if defined(STRATA_METAL)extern "C" unsigned long long ds4_dispatch_seq();#endif+#if defined(STRATA_METAL)+extern "C" void ds4_timeline_gate(int);+#endif+static void tl_gate(bool open) {+#if defined(STRATA_METAL)+ static const bool only_draft = std::getenv("DS4_TL_DRAFT") != nullptr;+ if (only_draft) ds4_timeline_gate(open ? 1 : 0);+#endif+}static unsigned long long rt_seq() {#if defined(STRATA_METAL)return ds4_dispatch_seq();@@ -177,10 +186,14 @@ bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min,std::vector<int> drafts;std::vector<float> conf;++g_rt.rounds;+ static const bool tl_init = (tl_gate(false), true);+ (void) tl_init;+ tl_gate(true);const double rt0 = rt_now();const unsigned long long rs0 = rt_seq();if (!m.draft(tok, drafts, conf, err, temperature, top_p, top_k, min_p, seed, counter)) return false;g_rt.draft += rt_now() - rt0;+ tl_gate(false);g_rt.draft_d += rt_seq() - rs0;g_round_conf = conf;g_round_nd = (int) drafts.size();diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 98d008f8b..60f56c5ba 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -32,6 +32,8 @@#include <cuda_runtime.h>#if defined(STRATA_METAL)+extern "C" void ds4_busy_begin();+extern "C" void ds4_busy_end();#include <sys/sysctl.h>#include <mach/mach.h>#endif@@ -8388,10 +8390,14 @@ struct Model::Impl {const double tc = now_ms();#if defined(STRATA_METAL)ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)+ ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)#endifstatic const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)if (!skip_cpu) cpu_experts(l, h_x, ents);else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)+#if defined(STRATA_METAL)+ ds4_busy_end();+#endifstd::fill(h_part, h_part + (size_t) T * D, 0.0f);for (int sl = 0; sl < nslots; ++sl) {float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;