Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: the mHC dense product wired - cublasSgemm dispatches an MSL kernel instead of aborting

Niko1221b5b4a97c0a9b
diff --git a/metal/include/metal_compat/cublas_v2.h b/metal/include/metal_compat/cublas_v2.h
index 8df805130..4c9c42c58 100644
--- a/metal/include/metal_compat/cublas_v2.h
+++ b/metal/include/metal_compat/cublas_v2.h
@@ -12,6 +12,7 @@
#include <cstdlib>
#include "cuda_runtime.h"
+#include "ds4_metal_host.h" // the kernels that have a Metal dispatch
typedef void* cublasHandle_t;
@@ -40,8 +41,14 @@ static inline cublasStatus_t cublasSetWorkspace(cublasHandle_t, void*, size_t) {
std::abort();
}
-static inline cublasStatus_t cublasSgemm(cublasHandle_t, cublasOperation_t, cublasOperation_t, int, int, int,
- const float*, const float*, int, const float*, int, const float*, float*, int) {
+static inline cublasStatus_t cublasSgemm(cublasHandle_t, cublasOperation_t opA, cublasOperation_t opB, int m, int n, int k,
+ const float* alpha, const float* A, int lda, const float* B, int ldb,
+ const float* beta, float* C, int ldc) {
+ // the mHC mixes (ds4_model.cu:2737): C = A^T B, F32, alpha 1, beta 0
+ if (opA == CUBLAS_OP_T && opB == CUBLAS_OP_N && alpha && beta && *alpha == 1.0f && *beta == 0.0f &&
+ strata::dsv41::metal_host::sgemm_tn(A, B, C, m, n, k, lda, ldb, ldc)) {
+ return CUBLAS_STATUS_SUCCESS;
+ }
strata_metal_dense_gemm_missing("cublasSgemm");
}
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index 7c2919d9f..00f9e1ec3 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -19,6 +19,9 @@ namespace strata::dsv41::metal_host {
// ds4_kernels.cu:1477 init_pre_kernel: pre[i] = (i % 8) == 0 ? 1 : 0, i < T*8
bool init_pre(float* pre, int T, void* stream);
// ds4_kernels.cu:130 embed_kernel: h[(t*hc + s)*dim + e] = bf16(rows[t*dim + e])
+// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, ...): C[t][m] = sum_k A[m][k] * B[t][k], F32, alpha 1, beta 0
+bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void* stream);
} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cpp
index 0d5cc31c5..b3dbdcb0c 100644
--- a/metal/src/engine/ds4_host.cpp
+++ b/metal/src/engine/ds4_host.cpp
@@ -139,4 +139,37 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
return true;
}
+
+// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0
+bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {
+ init_cx();
+ if (g_lib_init < 0) return false;
+ void* ha = strata::gpu::host_view((void*) A);
+ void* hb = strata::gpu::host_view((void*) B);
+ void* hc = strata::gpu::host_view((void*) C);
+ if (!ha || !hb || !hc) { std::fprintf(stderr, "dsv41 metal: sgemm_tn has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ return init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer ba = wrap(ha, (size_t) m * lda * 4);
+ mtl::Buffer bb = wrap(hb, (size_t) nt * ldb * 4);
+ mtl::Buffer bc = wrap(hc, (size_t) nt * ldc * 4);
+ if (!ba.handle || !bb.handle || !bc.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = init_cx().pipeline(g_lib_init, "ds4_sgemm_tn"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bc, off(hc)),
+ mtl::Arg::V((unsigned) m), mtl::Arg::V((unsigned) nt), mtl::Arg::V((unsigned) k),
+ mtl::Arg::V((unsigned) lda), mtl::Arg::V((unsigned) ldb), mtl::Arg::V((unsigned) ldc)};
+ const uint64_t total = (uint64_t) m * (uint64_t) nt;
+ d.grid[0] = (uint32_t) ((total + 127) / 128);
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
+ init_cx().run({d});
+ return true;
+}
+
} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index d51f61898..1b4e8c6fa 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -25,3 +25,19 @@ kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [
const float v = as_type<float>(bits);
for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;
}
+
+// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, n_mix, T, hcd, ...) - the mHC mixes: mixes[t][m] = h[t] . fn[m]
+// one thread an output element; both operands F32, alpha = 1, beta = 0 (every call site seen)
+kernel void ds4_sgemm_tn(device const float* A [[buffer(0)]], device const float* B [[buffer(1)]],
+ device float* C [[buffer(2)]], constant uint& m [[buffer(3)]], constant uint& nt [[buffer(4)]],
+ constant uint& k [[buffer(5)]], constant uint& lda [[buffer(6)]], constant uint& ldb [[buffer(7)]],
+ constant uint& ldc [[buffer(8)]], uint gid [[thread_position_in_grid]]) {
+ const uint total = m * nt;
+ if (gid >= total) return;
+ const uint t = gid / m, mm = gid % m;
+ const device float* a = A + (size_t) mm * lda; // row mm of A
+ const device float* b = B + (size_t) t * ldb; // row t of B
+ float acc = 0.0f;
+ for (uint i = 0; i < k; ++i) acc += a[i] * b[i];
+ C[(size_t) t * ldc + mm] = acc;
+}
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 1967d3a62..8db876184 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1882,6 +1882,8 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cud
return;
}
if (n_vec <= 0) return;
+ std::fprintf(stderr, "ds4 finish: variant %s quant %d len %d dst %p pdst %p w %p rd %d\n",
+ f.pdst ? "pack" : "plain", f.quant, len, (void*) f.dst, (void*) f.pdst, (const void*) f.w, f.rd);
if (f.pdst != nullptr) {
if (f.quant < kKvFp8 || f.quant > kKvFp4p || len % 32 != 0) {
std::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);