Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the mHC dense product wired - cublasSgemm dispatches an MSL kernel instead of aborting
Niko1221b5b4a97c0a9b
diff --git a/metal/include/metal_compat/cublas_v2.h b/metal/include/metal_compat/cublas_v2.hindex 8df805130..4c9c42c58 100644--- a/metal/include/metal_compat/cublas_v2.h+++ b/metal/include/metal_compat/cublas_v2.h@@ -12,6 +12,7 @@#include <cstdlib>#include "cuda_runtime.h"+#include "ds4_metal_host.h" // the kernels that have a Metal dispatchtypedef void* cublasHandle_t;@@ -40,8 +41,14 @@ static inline cublasStatus_t cublasSetWorkspace(cublasHandle_t, void*, size_t) {std::abort();}-static inline cublasStatus_t cublasSgemm(cublasHandle_t, cublasOperation_t, cublasOperation_t, int, int, int,- const float*, const float*, int, const float*, int, const float*, float*, int) {+static inline cublasStatus_t cublasSgemm(cublasHandle_t, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k,+ const float* alpha, const float* A, int lda, const float* B, int ldb,+ const float* beta, float* C, int ldc) {+ // the mHC mixes (ds4_model.cu:2737): C = A^T B, F32, alpha 1, beta 0+ if (opA == CUBLAS_OP_T && opB == CUBLAS_OP_N && alpha && beta && *alpha == 1.0f && *beta == 0.0f &&+ strata::dsv41::metal_host::sgemm_tn(A, B, C, m, n, k, lda, ldb, ldc)) {+ return CUBLAS_STATUS_SUCCESS;+ }strata_metal_dense_gemm_missing("cublasSgemm");}diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex 7c2919d9f..00f9e1ec3 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -19,6 +19,9 @@ namespace strata::dsv41::metal_host {// ds4_kernels.cu:1477 init_pre_kernel: pre[i] = (i % 8) == 0 ? 1 : 0, i < T*8bool init_pre(float* pre, int T, void* stream);// ds4_kernels.cu:130 embed_kernel: h[(t*hc + s)*dim + e] = bf16(rows[t*dim + e])+// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, ...): C[t][m] = sum_k A[m][k] * B[t][k], F32, alpha 1, beta 0+bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void* stream);} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cppindex 0d5cc31c5..b3dbdcb0c 100644--- a/metal/src/engine/ds4_host.cpp+++ b/metal/src/engine/ds4_host.cpp@@ -139,4 +139,37 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {return true;}++// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0+bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {+ init_cx();+ if (g_lib_init < 0) return false;+ void* ha = strata::gpu::host_view((void*) A);+ void* hb = strata::gpu::host_view((void*) B);+ void* hc = strata::gpu::host_view((void*) C);+ if (!ha || !hb || !hc) { std::fprintf(stderr, "dsv41 metal: sgemm_tn has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ return init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer ba = wrap(ha, (size_t) m * lda * 4);+ mtl::Buffer bb = wrap(hb, (size_t) nt * ldb * 4);+ mtl::Buffer bc = wrap(hc, (size_t) nt * ldc * 4);+ if (!ba.handle || !bb.handle || !bc.handle) return false;+ mtl::Pipeline pl;+ try { pl = init_cx().pipeline(g_lib_init, "ds4_sgemm_tn"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bc, off(hc)),+ mtl::Arg::V((unsigned) m), mtl::Arg::V((unsigned) nt), mtl::Arg::V((unsigned) k),+ mtl::Arg::V((unsigned) lda), mtl::Arg::V((unsigned) ldb), mtl::Arg::V((unsigned) ldc)};+ const uint64_t total = (uint64_t) m * (uint64_t) nt;+ d.grid[0] = (uint32_t) ((total + 127) / 128);+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;+ init_cx().run({d});+ return true;+}+} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex d51f61898..1b4e8c6fa 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -25,3 +25,19 @@ kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [const float v = as_type<float>(bits);for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;}++// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, n_mix, T, hcd, ...) - the mHC mixes: mixes[t][m] = h[t] . fn[m]+// one thread an output element; both operands F32, alpha = 1, beta = 0 (every call site seen)+kernel void ds4_sgemm_tn(device const float* A [[buffer(0)]], device const float* B [[buffer(1)]],+ device float* C [[buffer(2)]], constant uint& m [[buffer(3)]], constant uint& nt [[buffer(4)]],+ constant uint& k [[buffer(5)]], constant uint& lda [[buffer(6)]], constant uint& ldb [[buffer(7)]],+ constant uint& ldc [[buffer(8)]], uint gid [[thread_position_in_grid]]) {+ const uint total = m * nt;+ if (gid >= total) return;+ const uint t = gid / m, mm = gid % m;+ const device float* a = A + (size_t) mm * lda; // row mm of A+ const device float* b = B + (size_t) t * ldb; // row t of B+ float acc = 0.0f;+ for (uint i = 0; i < k; ++i) acc += a[i] * b[i];+ C[(size_t) t * ldc + mm] = acc;+}diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 1967d3a62..8db876184 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1882,6 +1882,8 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudreturn;}if (n_vec <= 0) return;+ std::fprintf(stderr, "ds4 finish: variant %s quant %d len %d dst %p pdst %p w %p rd %d\n",+ f.pdst ? "pack" : "plain", f.quant, len, (void*) f.dst, (void*) f.pdst, (const void*) f.w, f.rd);if (f.pdst != nullptr) {if (f.quant < kKvFp8 || f.quant > kKvFp4p || len % 32 != 0) {std::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);