Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the engine runs on Metal and names its kernel queue; two kernels ported
Niko122145cbb451e70f
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txtindex f585cb4a7..3c08027dc 100644--- a/metal/CMakeLists.txt+++ b/metal/CMakeLists.txt@@ -144,6 +144,6 @@ add_executable(strata_ds4 ${STRATA_DS4_SOURCES})target_include_directories(strata_ds4 PRIVATE${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/src${STRATA_ROOT}/third_party/ggml ${STRATA_GGML_DIR}/ggml/include ${STRATA_GGML_DIR}/ggml/src ${STRATA_GGML_DIR}/ggml/src/ggml-cpu)-target_compile_definitions(strata_ds4 PRIVATE STRATA_METAL=1)+target_compile_definitions(strata_ds4 PRIVATE STRATA_METAL=1) # the real host CPU lane (ds4_gg_cpu.cpp)target_compile_options(strata_ds4 PRIVATE -Wno-unused-parameter -Wno-deprecated-declarations -Wno-implicit-function-declaration)target_link_libraries(strata_ds4 PRIVATE strata_metal_kernels strata_metal_host)diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hnew file mode 100644index 000000000..7c2919d9f--- /dev/null+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -0,0 +1,24 @@+// metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch, as the engine calls them.+//+// A launch site in the shared dsv41 sources becomes, under STRATA_METAL, a call to one of these instead of the generic+// strata_ds4_launch() stub. Each implementation wraps the engine's own device buffers into the Metal runtime and dispatches+// the MSL kernel of the same name; metal/src/engine/ds4_host.cpp holds them.+#pragma once++#include <cstddef>+#include <cstdint>+#include <vector>++namespace strata::gpu {+struct StreamImpl;+using Stream = StreamImpl*;+} // namespace strata::gpu++namespace strata::dsv41::metal_host {++// ds4_kernels.cu:1477 init_pre_kernel: pre[i] = (i % 8) == 0 ? 1 : 0, i < T*8+bool init_pre(float* pre, int T, void* stream);+// ds4_kernels.cu:130 embed_kernel: h[(t*hc + s)*dim + e] = bf16(rows[t*dim + e])+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void* stream);++} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cppindex 35687b2bf..0d5cc31c5 100644--- a/metal/src/engine/ds4_host.cpp+++ b/metal/src/engine/ds4_host.cpp@@ -12,6 +12,8 @@#include <cstdlib>#include <cstring>#include <string>++#include <strata/gpu/runtime.hpp>#include <vector>namespace strata::dsv41::metal_host {@@ -26,7 +28,7 @@ bool ensure(mtl::Context& cx) {if (g_tried && g_lib < 0) return false;g_tried = true;// the kernel lives in its own file and is self-contained (metal_stdlib + ggml_grids.h)- g_lib = cx.compile({"src/engine/msl/ds4_quant.metal"}, "", false);+ g_lib = cx.compile({"../src/engine/msl/ds4_quant.metal"}, "", false);if (g_lib < 0) {std::fprintf(stderr, "dsv41 metal: compiling ds4_quant.metal failed\n");return false;@@ -62,3 +64,79 @@ bool q8_quant(mtl::Context& cx, const float* x, int ldx, int n, void* q, void* d}} // namespace strata::dsv41::metal_host++namespace strata::dsv41::metal_host {+namespace {+mtl::Context* g_init_cx = nullptr;+int g_lib_init = -1;++mtl::Context& init_cx() {+ if (!g_init_cx) {+ g_init_cx = new mtl::Context();+ g_lib_init = g_init_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);+ if (g_lib_init < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");+ }+ return *g_init_cx;+}+} // namespace++// ds4_kernels.cu:1477 - pre[i] = (i % 8) == 0 ? 1 : 0 for i < T*8+bool init_pre(float* pre, int T, void*) {+ init_cx();+ if (g_lib_init < 0) return false;+ void* host = strata::gpu::host_view(pre);+ if (!host) { std::fprintf(stderr, "dsv41 metal: init_pre has no host view\n"); return false; }+ const size_t bytes = (size_t) T * 8 * sizeof(float);+ const size_t page = 16384;+ mtl::Buffer bp = init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)),+ ((bytes + page - 1) / page) * page);+ if (!bp.handle) { std::fprintf(stderr, "dsv41 metal: init_pre wrap failed\n"); return false; }+ const uintptr_t delta = (uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1));+ mtl::Pipeline pl;+ try {+ pl = init_cx().pipeline(g_lib_init, "ds4_init_pre");+ } catch (...) {+ return false;+ }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bp, delta), mtl::Arg::V((unsigned) T)};+ d.grid[0] = (uint32_t) ((T * 8 + 127) / 128);+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 128;+ d.tg[1] = d.tg[2] = 1;+ init_cx().run({d});+ return true;+}+++// ds4_kernels.cu:130 - the BF16 embedding gather, hc copies a token+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {+ init_cx();+ if (g_lib_init < 0) { std::fprintf(stderr, "dsv41 metal: embed: no library\n"); return false; }+ void* hrows = strata::gpu::host_view((void*) rows);+ void* hh = strata::gpu::host_view(h);+ if (!hrows || !hh) { std::fprintf(stderr, "dsv41 metal: embed has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ return init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ };+ mtl::Buffer br = wrap(hrows, (size_t) T * dim * 2);+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4);+ if (!br.handle || !bh.handle) { std::fprintf(stderr, "dsv41 metal: embed wrap failed\n"); return false; }+ const size_t dro = (size_t) (((uintptr_t) hrows) - (((uintptr_t) hrows) & ~(uintptr_t) (page - 1)));+ const size_t dho = (size_t) (((uintptr_t) hh) - (((uintptr_t) hh) & ~(uintptr_t) (page - 1)));+ mtl::Pipeline pl;+ try { pl = init_cx().pipeline(g_lib_init, "ds4_embed"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(br, dro), mtl::Arg::B(bh, dho), mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};+ d.grid[0] = (uint32_t) ((dim + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ init_cx().run({d});+ return true;+}++} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalnew file mode 100644index 000000000..d51f61898--- /dev/null+++ b/metal/src/engine/msl/ds4_init.metal@@ -0,0 +1,27 @@+// metal/src/engine/msl/ds4_init.metal - the first kernels the engine asks for, ported from ds4_kernels.cu.+//+// ds4_kernels.cu:1477 __global__ void init_pre_kernel(float* pre, int T) {+// const int i = blockIdx.x * blockDim.x + threadIdx.x;+// if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;+// }+#include <metal_stdlib>++using namespace metal;++// the grid is threadgroups of 128; the flat id is what CUDA's blockIdx*blockDim+threadIdx was+kernel void ds4_init_pre(device float* pre [[buffer(0)]], constant uint& T [[buffer(1)]],+ uint gid [[thread_position_in_grid]]) {+ if (gid < T * 8u) pre[gid] = (gid % 8u) == 0u ? 1.0f : 0.0f;+}++// ds4_kernels.cu:130 embed_kernel: v = bf16(rows[t*dim + e]); h[(t*hc + s)*dim + e] = v for each of hc copies+kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [[buffer(1)]],+ constant uint& dim [[buffer(2)]], constant uint& hc [[buffer(3)]],+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + uint(lane);+ if (e >= dim) return;+ const uint t = tgp.y;+ const uint bits = uint(rows[(size_t) t * dim + e]) << 16; // BF16 -> F32 is the top half+ const float v = as_type<float>(bits);+ for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;+}diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex edc311554..fb01c0e4a 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -24,10 +24,11 @@#include <atomic>#include <chrono>#include <cstdio>+#include <cstdlib>#include <cstring>#include <iterator>#include <map>-#include <mutex>+#include <mutex> // recursive_mutex#include <thread>#include <vector>@@ -162,16 +163,26 @@ MTL::CommandBuffer* open_cb(StreamImpl* s) {}return s->cb;}+// StreamImpl's encoder/command-buffer state is touched from the loader, the prefetcher and the compute path.+// Without this lock two encoders can be open on one command buffer at once, which Metal asserts on.+static std::recursive_mutex g_stream_mu; // encode_copy/flush nest+void end_encoder(StreamImpl* s) {- if (s->enc) { s->enc->endEncoding(); s->enc = nullptr; }+ if (s->enc) { if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] close compute enc %p\n", (void*) s->enc);+ s->enc->endEncoding(); s->enc = nullptr; }}MTL::ComputeCommandEncoder* encoder(StreamImpl* s) {- if (!s->enc) s->enc = open_cb(s)->computeCommandEncoder(); // serial dispatch: each dispatch sees the last+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);+ if (!s->enc) { s->enc = open_cb(s)->computeCommandEncoder();+ if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] open compute enc %p (cb %p)\n", (void*) s->enc, (void*) s->cb); } // serial dispatch: each dispatch sees the lastreturn s->enc;}void flush(StreamImpl* s) {+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);if (!s->cb) return;end_encoder(s);+ if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] signal %llu on cb %p (enc %p)\n",+ (unsigned long long) (s->submitted + 1), (void*) s->cb, (void*) s->enc);s->cb->encodeSignalEvent(s->tl, ++s->submitted);s->cb->commit();s->cb->release();@@ -195,6 +206,7 @@ void check_purged(const Alloc* a, const char* what) {}Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);size_t od = 0, os = 0;const Alloc* ad = find(dst, &od);const Alloc* as = find(src, &os);@@ -203,6 +215,7 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {if (ad && as) { // buffer to buffer: a blit, in stream orderend_encoder(s);MTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();+ if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] blit copy %zu\n", n);b->copyFromBuffer(as->buf, os, ad->buf, od, n);b->endEncoding();++rt().stats.blits;@@ -219,6 +232,7 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {}Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);size_t off = 0;const Alloc* a = find(p, &off);if (!a) return fail(kErrorInvalidValue);@@ -232,6 +246,7 @@ Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {}Error encode_dispatch(StreamImpl* s, const Op& o) {+ std::lock_guard<std::recursive_mutex> lk(g_stream_mu);MTL::ComputeCommandEncoder* e = encoder(s);e->setComputePipelineState(o.pipe);if (o.big_args.empty()) e->setBytes(o.args, o.arg_bytes, 0);diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 956ccb501..1967d3a62 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -3,6 +3,9 @@// src/ds4/ds4_kernels.cu - DeepSeek-V4.1's device kernels (see ds4_kernels.cuh). No fast math here: the rotary// angles, the softmaxes and the roundings follow the reference's F32 arithmetic. A window's token is blockIdx.y.#include "ds4_kernels.cuh"+#if defined(STRATA_METAL)+#include <ds4_metal_host.h>+#endif#include "ds4_numerics.cuh"#include "ds4_q6k.cuh"#include "ds4_attn_split.cuh"@@ -1859,9 +1862,17 @@ void zc_stage(unsigned long long* plan, const int* idx, const unsigned long longstrata_ds4_launch(zc_fix_kernel, 1, 32, 0, s, plan, idx, zc_tab, k, stage, stride);}-void init_pre(float* pre, int T, cudaStream_t s) { strata_ds4_launch(init_pre_kernel, cdiv(T * 8, 128), 128, 0, s, pre, T); }+void init_pre(float* pre, int T, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::init_pre(pre, T, s)) return; // the ported MSL kernel+#endif+ strata_ds4_launch(init_pre_kernel, cdiv(T * 8, 128), 128, 0, s, pre, T);+}void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::embed(rows, T, h, dim, hc, s)) return;+#endifstrata_ds4_launch(embed_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, rows, h, dim, hc);}