Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: the engine runs on Metal and names its kernel queue; two kernels ported

Niko122145cbb451e70f
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txt
index f585cb4a7..3c08027dc 100644
--- a/metal/CMakeLists.txt
+++ b/metal/CMakeLists.txt
@@ -144,6 +144,6 @@ add_executable(strata_ds4 ${STRATA_DS4_SOURCES})
target_include_directories(strata_ds4 PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/src
${STRATA_ROOT}/third_party/ggml ${STRATA_GGML_DIR}/ggml/include ${STRATA_GGML_DIR}/ggml/src ${STRATA_GGML_DIR}/ggml/src/ggml-cpu)
-target_compile_definitions(strata_ds4 PRIVATE STRATA_METAL=1)
+target_compile_definitions(strata_ds4 PRIVATE STRATA_METAL=1) # the real host CPU lane (ds4_gg_cpu.cpp)
target_compile_options(strata_ds4 PRIVATE -Wno-unused-parameter -Wno-deprecated-declarations -Wno-implicit-function-declaration)
target_link_libraries(strata_ds4 PRIVATE strata_metal_kernels strata_metal_host)
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
new file mode 100644
index 000000000..7c2919d9f
--- /dev/null
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -0,0 +1,24 @@
+// metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch, as the engine calls them.
+//
+// A launch site in the shared dsv41 sources becomes, under STRATA_METAL, a call to one of these instead of the generic
+// strata_ds4_launch() stub. Each implementation wraps the engine's own device buffers into the Metal runtime and dispatches
+// the MSL kernel of the same name; metal/src/engine/ds4_host.cpp holds them.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace strata::gpu {
+struct StreamImpl;
+using Stream = StreamImpl*;
+} // namespace strata::gpu
+
+namespace strata::dsv41::metal_host {
+
+// ds4_kernels.cu:1477 init_pre_kernel: pre[i] = (i % 8) == 0 ? 1 : 0, i < T*8
+bool init_pre(float* pre, int T, void* stream);
+// ds4_kernels.cu:130 embed_kernel: h[(t*hc + s)*dim + e] = bf16(rows[t*dim + e])
+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void* stream);
+
+} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cpp
index 35687b2bf..0d5cc31c5 100644
--- a/metal/src/engine/ds4_host.cpp
+++ b/metal/src/engine/ds4_host.cpp
@@ -12,6 +12,8 @@
#include <cstdlib>
#include <cstring>
#include <string>
+
+#include <strata/gpu/runtime.hpp>
#include <vector>
namespace strata::dsv41::metal_host {
@@ -26,7 +28,7 @@ bool ensure(mtl::Context& cx) {
if (g_tried && g_lib < 0) return false;
g_tried = true;
// the kernel lives in its own file and is self-contained (metal_stdlib + ggml_grids.h)
- g_lib = cx.compile({"src/engine/msl/ds4_quant.metal"}, "", false);
+ g_lib = cx.compile({"../src/engine/msl/ds4_quant.metal"}, "", false);
if (g_lib < 0) {
std::fprintf(stderr, "dsv41 metal: compiling ds4_quant.metal failed\n");
return false;
@@ -62,3 +64,79 @@ bool q8_quant(mtl::Context& cx, const float* x, int ldx, int n, void* q, void* d
}
} // namespace strata::dsv41::metal_host
+
+namespace strata::dsv41::metal_host {
+namespace {
+mtl::Context* g_init_cx = nullptr;
+int g_lib_init = -1;
+
+mtl::Context& init_cx() {
+ if (!g_init_cx) {
+ g_init_cx = new mtl::Context();
+ g_lib_init = g_init_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
+ if (g_lib_init < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
+ }
+ return *g_init_cx;
+}
+} // namespace
+
+// ds4_kernels.cu:1477 - pre[i] = (i % 8) == 0 ? 1 : 0 for i < T*8
+bool init_pre(float* pre, int T, void*) {
+ init_cx();
+ if (g_lib_init < 0) return false;
+ void* host = strata::gpu::host_view(pre);
+ if (!host) { std::fprintf(stderr, "dsv41 metal: init_pre has no host view\n"); return false; }
+ const size_t bytes = (size_t) T * 8 * sizeof(float);
+ const size_t page = 16384;
+ mtl::Buffer bp = init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)),
+ ((bytes + page - 1) / page) * page);
+ if (!bp.handle) { std::fprintf(stderr, "dsv41 metal: init_pre wrap failed\n"); return false; }
+ const uintptr_t delta = (uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1));
+ mtl::Pipeline pl;
+ try {
+ pl = init_cx().pipeline(g_lib_init, "ds4_init_pre");
+ } catch (...) {
+ return false;
+ }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bp, delta), mtl::Arg::V((unsigned) T)};
+ d.grid[0] = (uint32_t) ((T * 8 + 127) / 128);
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 128;
+ d.tg[1] = d.tg[2] = 1;
+ init_cx().run({d});
+ return true;
+}
+
+
+// ds4_kernels.cu:130 - the BF16 embedding gather, hc copies a token
+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
+ init_cx();
+ if (g_lib_init < 0) { std::fprintf(stderr, "dsv41 metal: embed: no library\n"); return false; }
+ void* hrows = strata::gpu::host_view((void*) rows);
+ void* hh = strata::gpu::host_view(h);
+ if (!hrows || !hh) { std::fprintf(stderr, "dsv41 metal: embed has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ return init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ };
+ mtl::Buffer br = wrap(hrows, (size_t) T * dim * 2);
+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4);
+ if (!br.handle || !bh.handle) { std::fprintf(stderr, "dsv41 metal: embed wrap failed\n"); return false; }
+ const size_t dro = (size_t) (((uintptr_t) hrows) - (((uintptr_t) hrows) & ~(uintptr_t) (page - 1)));
+ const size_t dho = (size_t) (((uintptr_t) hh) - (((uintptr_t) hh) & ~(uintptr_t) (page - 1)));
+ mtl::Pipeline pl;
+ try { pl = init_cx().pipeline(g_lib_init, "ds4_embed"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(br, dro), mtl::Arg::B(bh, dho), mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
+ d.grid[0] = (uint32_t) ((dim + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ init_cx().run({d});
+ return true;
+}
+
+} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
new file mode 100644
index 000000000..d51f61898
--- /dev/null
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -0,0 +1,27 @@
+// metal/src/engine/msl/ds4_init.metal - the first kernels the engine asks for, ported from ds4_kernels.cu.
+//
+// ds4_kernels.cu:1477 __global__ void init_pre_kernel(float* pre, int T) {
+// const int i = blockIdx.x * blockDim.x + threadIdx.x;
+// if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;
+// }
+#include <metal_stdlib>
+
+using namespace metal;
+
+// the grid is threadgroups of 128; the flat id is what CUDA's blockIdx*blockDim+threadIdx was
+kernel void ds4_init_pre(device float* pre [[buffer(0)]], constant uint& T [[buffer(1)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (gid < T * 8u) pre[gid] = (gid % 8u) == 0u ? 1.0f : 0.0f;
+}
+
+// ds4_kernels.cu:130 embed_kernel: v = bf16(rows[t*dim + e]); h[(t*hc + s)*dim + e] = v for each of hc copies
+kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [[buffer(1)]],
+ constant uint& dim [[buffer(2)]], constant uint& hc [[buffer(3)]],
+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + uint(lane);
+ if (e >= dim) return;
+ const uint t = tgp.y;
+ const uint bits = uint(rows[(size_t) t * dim + e]) << 16; // BF16 -> F32 is the top half
+ const float v = as_type<float>(bits);
+ for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;
+}
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index edc311554..fb01c0e4a 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -24,10 +24,11 @@
#include <atomic>
#include <chrono>
#include <cstdio>
+#include <cstdlib>
#include <cstring>
#include <iterator>
#include <map>
-#include <mutex>
+#include <mutex> // recursive_mutex
#include <thread>
#include <vector>
@@ -162,16 +163,26 @@ MTL::CommandBuffer* open_cb(StreamImpl* s) {
}
return s->cb;
}
+// StreamImpl's encoder/command-buffer state is touched from the loader, the prefetcher and the compute path.
+// Without this lock two encoders can be open on one command buffer at once, which Metal asserts on.
+static std::recursive_mutex g_stream_mu; // encode_copy/flush nest
+
void end_encoder(StreamImpl* s) {
- if (s->enc) { s->enc->endEncoding(); s->enc = nullptr; }
+ if (s->enc) { if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] close compute enc %p\n", (void*) s->enc);
+ s->enc->endEncoding(); s->enc = nullptr; }
}
MTL::ComputeCommandEncoder* encoder(StreamImpl* s) {
- if (!s->enc) s->enc = open_cb(s)->computeCommandEncoder(); // serial dispatch: each dispatch sees the last
+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
+ if (!s->enc) { s->enc = open_cb(s)->computeCommandEncoder();
+ if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] open compute enc %p (cb %p)\n", (void*) s->enc, (void*) s->cb); } // serial dispatch: each dispatch sees the last
return s->enc;
}
void flush(StreamImpl* s) {
+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
if (!s->cb) return;
end_encoder(s);
+ if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] signal %llu on cb %p (enc %p)\n",
+ (unsigned long long) (s->submitted + 1), (void*) s->cb, (void*) s->enc);
s->cb->encodeSignalEvent(s->tl, ++s->submitted);
s->cb->commit();
s->cb->release();
@@ -195,6 +206,7 @@ void check_purged(const Alloc* a, const char* what) {
}
Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
size_t od = 0, os = 0;
const Alloc* ad = find(dst, &od);
const Alloc* as = find(src, &os);
@@ -203,6 +215,7 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
if (ad && as) { // buffer to buffer: a blit, in stream order
end_encoder(s);
MTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();
+ if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] blit copy %zu\n", n);
b->copyFromBuffer(as->buf, os, ad->buf, od, n);
b->endEncoding();
++rt().stats.blits;
@@ -219,6 +232,7 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
}
Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {
+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
size_t off = 0;
const Alloc* a = find(p, &off);
if (!a) return fail(kErrorInvalidValue);
@@ -232,6 +246,7 @@ Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {
}
Error encode_dispatch(StreamImpl* s, const Op& o) {
+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
MTL::ComputeCommandEncoder* e = encoder(s);
e->setComputePipelineState(o.pipe);
if (o.big_args.empty()) e->setBytes(o.args, o.arg_bytes, 0);
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 956ccb501..1967d3a62 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -3,6 +3,9 @@
// src/ds4/ds4_kernels.cu - DeepSeek-V4.1's device kernels (see ds4_kernels.cuh). No fast math here: the rotary
// angles, the softmaxes and the roundings follow the reference's F32 arithmetic. A window's token is blockIdx.y.
#include "ds4_kernels.cuh"
+#if defined(STRATA_METAL)
+#include <ds4_metal_host.h>
+#endif
#include "ds4_numerics.cuh"
#include "ds4_q6k.cuh"
#include "ds4_attn_split.cuh"
@@ -1859,9 +1862,17 @@ void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long
strata_ds4_launch(zc_fix_kernel, 1, 32, 0, s, plan, idx, zc_tab, k, stage, stride);
}
-void init_pre(float* pre, int T, cudaStream_t s) { strata_ds4_launch(init_pre_kernel, cdiv(T * 8, 128), 128, 0, s, pre, T); }
+void init_pre(float* pre, int T, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::init_pre(pre, T, s)) return; // the ported MSL kernel
+#endif
+ strata_ds4_launch(init_pre_kernel, cdiv(T * 8, 128), 128, 0, s, pre, T);
+}
void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::embed(rows, T, h, dim, hc, s)) return;
+#endif
strata_ds4_launch(embed_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, rows, h, dim, hc);
}