Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: commit the queued GPU work before the CPU lane runs (they overlap)

Niko12217c77f388dd8f
diff --git a/metal/include/metal_compat/cuda_runtime.h b/metal/include/metal_compat/cuda_runtime.h
index 62cb432ff..29aee70ab 100644
--- a/metal/include/metal_compat/cuda_runtime.h
+++ b/metal/include/metal_compat/cuda_runtime.h
@@ -138,6 +138,7 @@ inline cudaError_t cudaDeviceDisablePeerAccess(int) { return cudaSuccess; } //
// the deferred command buffer mtl_rt batches the port's dispatches into: a copy (or any host read of a device result)
// has to commit it first, or it reads memory the GPU has not written yet. Defined in ds4_finish.cpp.
extern "C" void ds4_metal_flush();
+extern "C" void ds4_metal_kick();
template <class T> inline cudaError_t cudaMalloc(T** p, size_t n) { return strata::gpu::malloc_device(p, n); }
inline cudaError_t cudaFree(void* p) { return strata::gpu::free_device(p); }
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index af2eba254..f747aedbb 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -2101,8 +2101,13 @@ extern "C" void* ds4_ctx_mark() {
return g_cx->mark();
}
extern "C" void ds4_ctx_wait(void* token) {
+ if (g_cx) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)
if (g_cx && token) g_cx->wait_mark(token);
}
+// commit the batched dispatches without waiting: the GPU starts on them while the host goes on (the CPU lane)
+extern "C" void ds4_metal_kick() {
+ if (g_cx) g_cx->split_runs();
+}
extern "C" void ds4_metal_flush() {
// The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index a64ed75f6..282ffe9d4 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -8285,6 +8285,9 @@ struct Model::Impl {
}
if (!ents.empty()) {
const double tc = now_ms();
+#if defined(STRATA_METAL)
+ ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)
+#endif
cpu_experts(l, h_x, ents);
std::fill(h_part, h_part + (size_t) T * D, 0.0f);
for (int sl = 0; sl < nslots; ++sl) {