Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: commit the queued GPU work before the CPU lane runs (they overlap)
Niko12217c77f388dd8f
diff --git a/metal/include/metal_compat/cuda_runtime.h b/metal/include/metal_compat/cuda_runtime.hindex 62cb432ff..29aee70ab 100644--- a/metal/include/metal_compat/cuda_runtime.h+++ b/metal/include/metal_compat/cuda_runtime.h@@ -138,6 +138,7 @@ inline cudaError_t cudaDeviceDisablePeerAccess(int) { return cudaSuccess; } //// the deferred command buffer mtl_rt batches the port's dispatches into: a copy (or any host read of a device result)// has to commit it first, or it reads memory the GPU has not written yet. Defined in ds4_finish.cpp.extern "C" void ds4_metal_flush();+extern "C" void ds4_metal_kick();template <class T> inline cudaError_t cudaMalloc(T** p, size_t n) { return strata::gpu::malloc_device(p, n); }inline cudaError_t cudaFree(void* p) { return strata::gpu::free_device(p); }diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex af2eba254..f747aedbb 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -2101,8 +2101,13 @@ extern "C" void* ds4_ctx_mark() {return g_cx->mark();}extern "C" void ds4_ctx_wait(void* token) {+ if (g_cx) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)if (g_cx && token) g_cx->wait_mark(token);}+// commit the batched dispatches without waiting: the GPU starts on them while the host goes on (the CPU lane)+extern "C" void ds4_metal_kick() {+ if (g_cx) g_cx->split_runs();+}extern "C" void ds4_metal_flush() {// The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's defaultdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex a64ed75f6..282ffe9d4 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -8285,6 +8285,9 @@ struct Model::Impl {}if (!ents.empty()) {const double tc = now_ms();+#if defined(STRATA_METAL)+ ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)+#endifcpu_experts(l, h_x, ents);std::fill(h_part, h_part + (size_t) T * D, 0.0f);for (int sl = 0; sl < nslots; ++sl) {