Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek Metal: chunked prompt path (MPP expert tile GEMMs, dense gemm/dq, rows_f16), NC-column matvecs, dangling buffer and null-arg offset fixes (WIP, chunk output not yet matching window path)

Niko122158e5d7851e0e
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hpp
index b080a5970..9ab2482fd 100644
--- a/include/strata/dsv41/exl3.hpp
+++ b/include/strata/dsv41/exl3.hpp
@@ -152,6 +152,8 @@ struct SetArgs {
const int* bounds = nullptr;
int row0 = 0;
int max_rows = 0; // the most rows one expert of the set has
+ const int* hbounds = nullptr; // (Metal) the same bounds on the host: [n_exp + 1], absolute rows
+ int total_rows = 0; // (Metal) the set's rows in all
int n_embd = 4096, n_ff = 2048;
float limit = 0.0f;
};
diff --git a/metal/include/metal_compat/cublas_v2.h b/metal/include/metal_compat/cublas_v2.h
index 4c9c42c58..c5ebf75e8 100644
--- a/metal/include/metal_compat/cublas_v2.h
+++ b/metal/include/metal_compat/cublas_v2.h
@@ -54,7 +54,19 @@ static inline cublasStatus_t cublasSgemm(cublasHandle_t, cublasOperation_t opA,
// the mixed-precision form: any argument list, because the engine passes cuBLAS's full 19-argument spelling; the body
// names kernels/dense_mm.metal, which is where the real implementation goes.
+// the chunk path's call (F16 weights [m][k] x F16 rows [n][k] -> F32 rows [n][m]): the MPP tile GEMM
+static inline cublasStatus_t cublasGemmEx(cublasHandle_t, cublasOperation_t ta, cublasOperation_t tb, int m, int n, int k,
+ const void* alpha, const void* A, cudaDataType_t at, int lda, const void* B,
+ cudaDataType_t bt, int ldb, const void* beta, void* C, cudaDataType_t ct, int ldc,
+ cublasComputeType_t, cublasGemmAlgo_t) {
+ if (ta == CUBLAS_OP_T && tb == CUBLAS_OP_N && at == CUDA_R_16F && bt == CUDA_R_16F && ct == CUDA_R_32F && lda == k &&
+ strata::dsv41::metal_host::gemm_f16_nt((const uint16_t*) A, (const uint16_t*) B, ldb, (float*) C, ldc, n, m, k, *(const float*) alpha,
+ *(const float*) beta))
+ return CUBLAS_STATUS_SUCCESS;
+ strata_metal_dense_gemm_missing("cublasGemmEx (this operand combination)");
+}
+
template <typename... A>
-static inline cublasStatus_t cublasGemmEx(A...) {
+static inline cublasStatus_t cublasGemmEx_unsupported(A...) {
strata_metal_dense_gemm_missing("cublasGemmEx");
}
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index d380dfd9c..8d7205341 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -49,6 +49,11 @@ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float
bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out);
+bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td);
+bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td);
+bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out);
+bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha, float beta);
bool swiglu(const float* g, const float* u, int n, float lim, float* h);
bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index f747aedbb..966eb491f 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -15,11 +15,13 @@
#include <cstdint>
#include <cstring>
#include <algorithm>
+#include <map>
#include <vector>
// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read
extern "C" void ds4_metal_flush();
extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);
+extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off);
extern "C" void ds4_gpu_sync_all_streams();
extern "C" unsigned long long ds4_dispatch_seq();
static unsigned long long g_q8_deduped = 0;
@@ -191,17 +193,18 @@ bool hc_post(const float* y, float* h, const float* post, const float* comb, int
mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
if (!by.handle || !bh.handle) return false;
- // post is [T][8], comb is [T][64], straight from the caller's host/shared memory
- mtl::Buffer bp = g_cx->alloc((size_t) T * 8 * 4);
- mtl::Buffer bc = g_cx->alloc((size_t) T * 64 * 4);
- if (!bp.ptr || !bc.ptr) return false;
- std::memcpy(bp.ptr, post, (size_t) T * 8 * 4);
- std::memcpy(bc.ptr, comb, (size_t) T * 64 * 4);
+ // post is [T][8], comb is [T][64]: device operands like the others (reached through their host views)
+ void* hpost = strata::gpu::host_view((void*) post);
+ void* hcomb = strata::gpu::host_view((void*) comb);
+ if (!hpost || !hcomb) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view for post / comb\n"); return false; }
+ mtl::Buffer bp = wrap(hpost, (size_t) T * 8 * 4);
+ mtl::Buffer bc = wrap(hcomb, (size_t) T * 64 * 4);
+ if (!bp.handle || !bc.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_post"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
- d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp), mtl::Arg::B(bc),
+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp, off(hpost)), mtl::Arg::B(bc, off(hcomb)),
mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
d.grid[0] = (uint32_t) ((dim + 255) / 256);
d.grid[1] = (uint32_t) T;
@@ -584,16 +587,26 @@ bool iq_mv(const void* jobs, int nj) {
if (nj <= 0) return true;
if (!ensure()) return false;
const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;
- std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1, exp_x2, exp_x3, exp_x4;
- exp_iq.reserve((size_t) nj);
- exp_k.reserve((size_t) nj);
+ // Each job goes to the kernel of its type and column count: NC = 1, 2, 4 or 8 activations decode the row's weights once
+ // (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.
+ std::map<std::string, std::vector<Ds4IqMvJob>> groups;
int max_n = 0;
+ static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;
for (int i = 0; i < nj; ++i) {
const Ds4IqMvJob& q = src[i];
if (q.n > max_n) max_n = q.n;
const int ncol = q.ncol > 0 ? q.ncol : 1;
- static const int maxstep = std::getenv("DS4_IQ_STEP") ? std::atoi(std::getenv("DS4_IQ_STEP")) : 16;
- const int step = ncol > maxstep ? maxstep : ncol; // both kernels keep 16 accumulators
+ const char* tn = nullptr;
+ switch (q.type) {
+ case 13: tn = "ds4_q5k_mvn"; break;
+ case 16: tn = "ds4_iq2xxs_mvn"; break;
+ case 17: tn = "ds4_iq2xs_mvn"; break;
+ case 18: tn = "ds4_iq3xxs_mvn"; break;
+ case 22: tn = "ds4_iq2s_mvn"; break;
+ default: break;
+ }
+ const bool grid = tn != nullptr && !old_only;
+ const int step = grid ? 8 : (ncol > 16 ? 16 : ncol);
for (int c0 = 0; c0 < ncol; c0 += step) {
Ds4IqMvJob b = q;
b.ncol = (c0 + step < ncol) ? step : ncol - c0;
@@ -602,19 +615,20 @@ bool iq_mv(const void* jobs, int nj) {
b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
b.y = q.y + (size_t) c0 * (size_t) q.ycs;
}
- static const bool fast1 = std::getenv("DS4_NO_MV1") == nullptr;
- if (fast1 && b.ncol == 1 && b.type == 13) exp_q5.push_back(b); // the decode step's one-activation kernels
- else if (fast1 && b.ncol == 1 && b.type == 16) exp_x1.push_back(b);
- else if (fast1 && b.ncol == 1 && b.type == 17) exp_x2.push_back(b);
- else if (fast1 && b.ncol == 1 && b.type == 22) exp_x3.push_back(b);
- else if (fast1 && b.ncol == 1 && b.type == 18) exp_x4.push_back(b);
- else (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);
+ std::string fn;
+ if (grid) {
+ const int nc = b.ncol <= 1 ? 1 : (b.ncol <= 2 ? 2 : (b.ncol <= 4 ? 4 : 8));
+ fn = std::string(tn) + std::to_string(nc);
+ } else {
+ fn = (b.type == 13 || b.type == 14) ? "ds4_iq_mv_k" : "ds4_iq_mv";
+ }
+ groups[fn].push_back(b);
}
}
if (max_n <= 0) return true;
const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group
- auto run_group = [&](std::vector<Ds4IqMvJob>& v, const char* fn) -> bool {
- if (v.empty()) return true;
+ for (auto& kv : groups) {
+ std::vector<Ds4IqMvJob>& v = kv.second;
mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");
@@ -623,7 +637,7 @@ bool iq_mv(const void* jobs, int nj) {
std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));
mtl::Pipeline pl;
try {
- pl = g_cx->pipeline(g_lib_iq, fn);
+ pl = g_cx->pipeline(g_lib_iq, kv.first);
} catch (...) {
g_cx->free(bj);
return false;
@@ -642,15 +656,7 @@ bool iq_mv(const void* jobs, int nj) {
}
g_cx->run(ds);
g_cx->free(bj);
- return true;
- };
- if (!run_group(exp_q5, "ds4_q5k_mv1")) return false;
- if (!run_group(exp_x1, "ds4_iq2xxs_mv1")) return false;
- if (!run_group(exp_x2, "ds4_iq2xs_mv1")) return false;
- if (!run_group(exp_x3, "ds4_iq2s_mv1")) return false;
- if (!run_group(exp_x4, "ds4_iq3xxs_mv1")) return false;
- if (!run_group(exp_k, "ds4_iq_mv_k")) return false;
- if (!run_group(exp_iq, "ds4_iq_mv")) return false;
+ }
return true;
}
@@ -1075,11 +1081,11 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
}
mtl::Dispatch dsp;
dsp.p = &pl;
- dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, off(hw != nullptr ? hw : bw.ptr)),
- mtl::Arg::B(bc, off(hc != nullptr ? hc : bc.ptr)),
- mtl::Arg::B(bs, off(hs != nullptr ? hs : bs.ptr)),
+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, (hw != nullptr ? off(hw) : (size_t) 0)),
+ mtl::Arg::B(bc, (hc != nullptr ? off(hc) : (size_t) 0)),
+ mtl::Arg::B(bs, (hs != nullptr ? off(hs) : (size_t) 0)),
mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),
- mtl::Arg::B(bk, off(hk != nullptr ? hk : bk.ptr)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),
+ mtl::Arg::B(bk, (hk != nullptr ? off(hk) : (size_t) 0)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),
mtl::Arg::B(bo, off(ho))};
dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);
dsp.grid[1] = (uint32_t) T;
@@ -2043,6 +2049,349 @@ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, voi
+
+// ---- the prompt chunk's dense GEMM: float rows -> half, a quantized matrix -> half rows, MPP matmul2d (kernels/dense_mm.metal)
+namespace {
+mtl::Buffer g_gm_xs, g_gm_yt; // grow-only scratch: packed activations, the contiguous result
+int g_lib_mm = -2;
+mtl::Buffer scratch(mtl::Buffer& b, size_t bytes) {
+ if (!b.handle || b.bytes < bytes) {
+ if (b.handle) g_cx->free(b);
+ b = g_cx->alloc(bytes + (bytes >> 2));
+ }
+ return b;
+}
+} // namespace
+
+bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out) {
+ if (nrows <= 0 || ncols <= 0) return true;
+ if (!ensure()) return false;
+ void* hx = strata::gpu::host_view((void*) x);
+ void* ho = strata::gpu::host_view(out);
+ void* hi = idx ? strata::gpu::host_view((void*) idx) : nullptr;
+ if (!hx || !ho) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bx = wrap(hx, ((size_t) (nrows - 1) * (size_t) ldx + (size_t) ncols) * 4);
+ mtl::Buffer bo = wrap(ho, (size_t) nrows * (size_t) ncols * 2);
+ mtl::Buffer bi = hi ? wrap(hi, (size_t) nrows * 4) : bx;
+ if (!bx.handle || !bo.handle || !bi.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_rows_f16"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bi, hi ? off(hi) : off(hx)),
+ mtl::Arg::V((unsigned) (hi ? 1 : 0)), mtl::Arg::V((unsigned) ncols), mtl::Arg::B(bo, off(ho))};
+ d.grid[0] = (uint32_t) ((ncols + 255) / 256);
+ d.grid[1] = (uint32_t) nrows;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+// kind: 0 BF16 values, 8 Q8_0, 13 Q5_K. n rows of k values, ld the row stride (bytes; values for BF16), out half rows.
+bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out) {
+ if (n <= 0 || k <= 0) return true;
+ if (!ensure()) return false;
+ const char* fn = kind == 13 ? "ds4_dq_q5k" : kind == 8 ? "ds4_dq_q8" : kind == 0 ? "ds4_dq_bf16" : nullptr;
+ if (!fn) return false;
+ if ((kind == 13 && k % 256 != 0) || (kind == 8 && k % 32 != 0)) return false;
+ void* hw = strata::gpu::host_view((void*) raw);
+ void* ho = strata::gpu::host_view(out);
+ if (!hw || !ho) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t wbytes = (size_t) (n - 1) * ld * (kind == 0 ? 2 : 1) + (kind == 0 ? (size_t) k * 2 : (kind == 13 ? (size_t) (k / 256) * 176 : (size_t) (k / 32) * 34));
+ if (!std::getenv("DS4_NO_TOUCH")) { // (the matrix may be file-backed pages not read yet: the GPU does not page them in, it stalls)
+ volatile unsigned char sink = 0;
+ for (size_t o = 0; o < wbytes; o += 4096) sink = sink + ((const volatile unsigned char*) hw)[o];
+ (void) sink;
+ }
+ mtl::Buffer bw = wrap(hw, wbytes);
+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) k * 2);
+ if (!bw.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }
+ size_t items = kind == 13 ? (size_t) n * (k / 256) * 8 : kind == 8 ? (size_t) n * (k / 32) : (size_t) n * k;
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::V((unsigned long) ld), mtl::Arg::V((unsigned) k), mtl::Arg::V((unsigned) n),
+ mtl::Arg::B(bo, off(ho))};
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ d.grid[0] = (uint32_t) ((items + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+// Y[r][c] = alpha * (X16[r] . W16[c]) + beta * Y[r][c]: X16 rows x K (stride ldx halves), W16 N x K contiguous, Y rows x N float
+// (stride ldy). The cuBLAS call's operands in row-major terms; the MPP tile kernel wants contiguous operands and result.
+bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha,
+ float beta) {
+ if (rows <= 0 || N <= 0) return true;
+ if (!ensure()) return false;
+ if (g_lib_mm == -2) {
+ g_lib_mm = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "dense_mm.metal"});
+ if (g_lib_mm < 0) std::fprintf(stderr, "dsv41 metal: compiling dense_mm.metal failed\n");
+ }
+ if (g_lib_mm < 0) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ void* hw = strata::gpu::host_view((void*) W16);
+ void* hx = strata::gpu::host_view((void*) X16);
+ void* hy = strata::gpu::host_view(Y);
+ if (!hw || !hx || !hy) return false;
+ mtl::Buffer bw = wrap(hw, (size_t) N * K * 2);
+ mtl::Buffer bx = wrap(hx, ((size_t) (rows - 1) * (size_t) ldx + (size_t) K) * 2);
+ mtl::Buffer by = wrap(hy, ((size_t) (rows - 1) * (size_t) ldy + (size_t) N) * 4);
+ if (!bw.handle || !bx.handle || !by.handle) return false;
+ // activations contiguous
+ mtl::Buffer ax = bx;
+ size_t ax_off = off(hx);
+ if (ldx != K) {
+ ax = scratch(g_gm_xs, (size_t) rows * K * 2);
+ ax_off = 0;
+ mtl::Pipeline pp;
+ try { pp = g_cx->pipeline(g_lib, "ds4_pack_h16"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pp;
+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) K), mtl::Arg::B(ax)};
+ d.grid[0] = (uint32_t) ((K + 255) / 256);
+ d.grid[1] = (uint32_t) rows;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ }
+ const bool direct = ldy == N && alpha == 1.0f && beta == 0.0f;
+ mtl::Buffer ty = by;
+ size_t ty_off = off(hy);
+ if (!direct) {
+ ty = scratch(g_gm_yt, (size_t) rows * N * 4);
+ ty_off = 0;
+ }
+ const int tm = 64, tn = N <= 64 ? 32 : 64;
+ const std::string fn = "gemm_f16_" + std::to_string(tm) + "_" + std::to_string(tn) + "_4";
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_mm, fn); } catch (...) { return false; }
+ struct GemmArgs { int M, N, K, pad; } ga{rows, N, K, 0};
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(ax, ax_off), mtl::Arg::B(bw, off(hw)), mtl::Arg::B(ty, ty_off), mtl::Arg::V(ga)};
+ d.grid[0] = (uint32_t) ((N + tn - 1) / tn);
+ d.grid[1] = (uint32_t) ((rows + tm - 1) / tm);
+ d.grid[2] = 1;
+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ if (!direct) {
+ mtl::Pipeline pa;
+ try { pa = g_cx->pipeline(g_lib, "ds4_axpby_rows"); } catch (...) { return false; }
+ mtl::Dispatch e;
+ e.p = &pa;
+ e.args = {mtl::Arg::B(ty), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned long) ldy), mtl::Arg::V((unsigned) N),
+ mtl::Arg::V(alpha), mtl::Arg::V(beta)};
+ e.grid[0] = (uint32_t) ((N + 255) / 256);
+ e.grid[1] = (uint32_t) rows;
+ e.grid[2] = 1;
+ e.tg[0] = 256; e.tg[1] = e.tg[2] = 1;
+ g_cx->run({e});
+ }
+ return true;
+}
+
+
+// ---- the prompt chunk's expert GEMMs (ds4_mm.metal): a set = n_exp consecutive expert slots, their rows sorted by expert ----
+namespace {
+int g_lib_moe = -2;
+mtl::Buffer g_mm_xs, g_mm_hs, g_mm_ys; // grow-only scratch: the sorted rows' half activations; the gate/up's swiglu rows (the down's input)
+bool g_mm_hs_ok = false;
+size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : 0; }
+struct DsMmArgsH {
+ unsigned long gate_off, up_off, down_off, stride;
+ unsigned gu_row, d_row, n_embd, n_ff;
+ float limit;
+ unsigned pad0;
+};
+bool mm_prepare() {
+ if (!ensure()) return false;
+ if (g_lib_moe == -2) {
+ g_lib_moe = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "moe_mm.metal", "../src/engine/msl/ds4_mm.metal"});
+ if (g_lib_moe < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_mm.metal failed\n");
+ }
+ return g_lib_moe >= 0;
+}
+} // namespace
+
+static bool mm_set(bool gate_up, const uint8_t* wbase, size_t stride, const size_t* tr, const int* hb, int n_exp, int total_rows,
+ int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, const uint16_t* hin, uint16_t* hout,
+ int ldh, float* yout, int ldy, int tg, int td) {
+ const int type = gate_up ? tg : td;
+ if (n_exp <= 0 || total_rows <= 0) return true;
+ if (iq_bs(tg) == 0 || iq_bs(td) == 0 || n_embd % 256 != 0 || n_ff % 256 != 0) return false;
+ if (ldh != n_ff || (!gate_up && ldy != n_embd)) return false;
+ if (gate_up ? (n_ff % 128 != 0) : (n_embd % 128 != 0)) return false;
+ if (!mm_prepare()) return false;
+ constexpr int TM = 16;
+ // the tile list: (slot, first row, valid rows) a TM-row piece of one expert
+ std::vector<int> tiles;
+ for (int e = 0; e < n_exp; ++e) {
+ const int lo = hb[e] - hb[0], hi = hb[e + 1] - hb[0];
+ for (int r = lo; r < hi; r += TM) { tiles.push_back(e); tiles.push_back(r); tiles.push_back(std::min(TM, hi - r)); tiles.push_back(0); }
+ }
+ const int ntiles = (int) (tiles.size() / 4);
+ if (ntiles == 0) return true;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ const size_t len = ((end - base + page - 1) / page) * page;
+ if (std::getenv("DS4_MM_DBG")) { size_t o2 = 0; std::fprintf(stderr, "mm wrap %p len %zu own=%d rem=%zu\n", (void*) base, len, ds4_gpu_buffer_for_cpu((void*) base, len, &o2) != nullptr, ds4_gpu_alloc_remaining((void*) base)); }
+ return g_cx->wrap((void*) base, len);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ void* hw = strata::gpu::host_view((void*) wbase);
+ if (!hw) return false;
+ mtl::Buffer bw = wrap(hw, (size_t) n_exp * stride);
+ mtl::Buffer bt = g_cx->alloc(tiles.size() * 4);
+ std::memcpy(bt.ptr, tiles.data(), tiles.size() * 4);
+ DsMmArgsH a{tr[0], tr[1], tr[2], (unsigned long) stride, (unsigned) ((size_t) n_embd / 256 * iq_bs(tg)), (unsigned) ((size_t) n_ff / 256 * iq_bs(td)),
+ (unsigned) n_embd, (unsigned) n_ff, limit, std::getenv("DS4_MM_ABL") ? (unsigned) std::atoi(std::getenv("DS4_MM_ABL")) : 0u};
+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "mm %s embd %d ff %d tg %d td %d lim %g stride %zu tr %zu %zu %zu gu_row %u d_row %u\n", gate_up ? "gu" : "dn", n_embd, n_ff, tg, td, limit, stride, tr[0], tr[1], tr[2], a.gu_row, a.d_row);
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_moe, std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type)); } catch (...) { g_cx->free(bt); return false; }
+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "pipe %s max_threads %d tgmem %d width %d\n", pl.name.c_str(), pl.max_threads, pl.static_tg_mem, pl.simd_width);
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
+ d.grid[2] = 1;
+ d.grid[1] = (uint32_t) ntiles;
+ if (gate_up) {
+ // the sorted rows' activations, half
+ const size_t xs_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 2;
+ if (!g_mm_xs.handle || g_mm_xs.bytes < xs_bytes) {
+ if (g_mm_xs.handle) g_cx->free(g_mm_xs);
+ g_mm_xs = g_cx->alloc(xs_bytes + (xs_bytes >> 2));
+ }
+ void* hx = strata::gpu::host_view((void*) xf);
+ void* hi = strata::gpu::host_view((void*) idx);
+ void* ho = strata::gpu::host_view(hout);
+ if (!hx || !hi || !ho) { g_cx->free(bt); return false; }
+ // (the float rows: the whole token slab the engine's c.x holds; its extent is the allocation's)
+ const size_t xrem = ds4_gpu_alloc_remaining(hx);
+ mtl::Buffer bxf = wrap(hx, xrem > 0 ? xrem : (size_t) ldx * 4);
+ mtl::Buffer bix = wrap(hi, (size_t) total_rows * 4);
+ mtl::Pipeline pg;
+ try { pg = g_cx->pipeline(g_lib_moe, "ds4_mm_gather"); } catch (...) { g_cx->free(bt); return false; }
+ mtl::Dispatch dg;
+ dg.p = &pg;
+ dg.args = {mtl::Arg::B(bxf, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bix, off(hi)), mtl::Arg::V((unsigned) n_embd), mtl::Arg::B(g_mm_xs)};
+ dg.grid[0] = (uint32_t) ((n_embd + 255) / 256);
+ dg.grid[1] = (uint32_t) total_rows;
+ dg.grid[2] = 1;
+ dg.tg[0] = 256; dg.tg[1] = dg.tg[2] = 1;
+ g_cx->run({dg});
+ const size_t hs_bytes = ((size_t) total_rows + TM) * (size_t) n_ff * 2;
+ if (!g_mm_hs.handle || g_mm_hs.bytes < hs_bytes) {
+ if (g_mm_hs.handle) g_cx->free(g_mm_hs);
+ g_mm_hs = g_cx->alloc(std::max<size_t>(hs_bytes + (hs_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
+ }
+ g_mm_hs_ok = true;
+ { // (the down's result rows are made here too)
+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;
+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {
+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);
+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
+ }
+ }
+ (void) ho;
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_xs), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};
+ d.grid[0] = (uint32_t) (n_ff / 128);
+ } else {
+ void* hh = strata::gpu::host_view((void*) hin);
+ void* hy = strata::gpu::host_view(yout);
+ void* hy_real = nullptr;
+ if (!hh || !hy) { g_cx->free(bt); return false; }
+ const size_t hrem = ds4_gpu_alloc_remaining(hh);
+ if (!g_mm_hs_ok) { g_cx->free(bt); return false; }
+ g_mm_hs_ok = false;
+ (void) hrem;
+ mtl::Buffer& bh = g_mm_hs;
+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;
+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {
+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);
+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
+ }
+ mtl::Buffer by = g_mm_ys;
+ hy_real = hy;
+ hy = by.ptr;
+ if (std::getenv("DS4_MM_YALT")) { hy = hh; by = wrap(hh, (size_t) total_rows * (size_t) n_ff * 2); }
+ if (std::getenv("DS4_MM_EXP1")) { by = g_cx->alloc((size_t) total_rows * (size_t) n_embd * 4 + 4096); hy = by.ptr; }
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_ys), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};
+ d.grid[0] = (uint32_t) (n_embd / 128);
+ }
+ if (!gate_up && std::getenv("DS4_MM_ASGU")) { // test: the gate/up kernel on the down's buffers
+ static mtl::Pipeline pgu;
+ pgu = g_cx->pipeline(g_lib_moe, "ds4_mm_gu_" + std::to_string(tg));
+ d.p = &pgu;
+ d.args[1] = mtl::Arg::B(g_mm_xs);
+ d.args[2] = mtl::Arg::B(g_mm_ys);
+ if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / 128);
+ }
+ if (!gate_up && std::getenv("DS4_MM_DROP")) {
+ const char* dk = std::getenv("DS4_MM_DROP");
+ static mtl::Buffer dummy;
+ if (!dummy.handle) dummy = g_cx->alloc(1 << 26);
+ static mtl::Buffer dummy2;
+ if (!dummy2.handle) dummy2 = g_cx->alloc(1 << 26);
+ for (int k = 0; k < 4; ++k) if (std::strchr(dk, '0' + k)) d.args[k] = mtl::Arg::B(std::getenv("DS4_MM_D2") && k == std::atoi(std::getenv("DS4_MM_D2")) ? dummy2 : dummy);
+ }
+ if (!gate_up && std::getenv("DS4_MM_ONE")) { d.grid[1] = 1; if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "dn tile0 %d %d %d ntiles %d x %u rows %d\n", tiles[0], tiles[1], tiles[2], ntiles, d.grid[0], total_rows); }
+ if (std::getenv("DS4_MM_DBG")) for (size_t q = 0; q < d.args.size(); ++q) if (d.args[q].buf) std::fprintf(stderr, "arg %s %zu bytes %zu bias %zu off %zu ptr %p\n", gate_up ? "gu" : "dn", q, d.args[q].buf->bytes, (size_t) d.args[q].buf->bias, (size_t) d.args[q].off, d.args[q].buf->ptr);
+ if (std::getenv(gate_up ? "DS4_MM_NOGU" : "DS4_MM_NODN")) { g_cx->free(bt); return true; }
+ g_cx->run({d});
+ if (!std::getenv("DS4_MM_LEAK")) g_cx->free(bt);
+ if (!gate_up) { // the result into the engine's rows
+ void* hyr = strata::gpu::host_view(yout);
+ const size_t n4 = (size_t) total_rows * (size_t) n_embd;
+ const size_t base = (size_t) ((uintptr_t) hyr) & ~(size_t) (page - 1);
+ mtl::Buffer bd = g_cx->wrap((void*) base, ((((size_t) (uintptr_t) hyr + n4 * 4 - base) + page - 1) / page) * page);
+ mtl::Pipeline pc;
+ try { pc = g_cx->pipeline(g_lib, "ds4_copy4"); } catch (...) { return false; }
+ mtl::Dispatch c;
+ c.p = &pc;
+ c.args = {mtl::Arg::B(g_mm_ys), mtl::Arg::B(bd, (size_t) ((uintptr_t) hyr - base)), mtl::Arg::V((unsigned long) n4)};
+ c.tg[0] = 256; c.tg[1] = c.tg[2] = 1;
+ c.grid[0] = (uint32_t) ((n4 + 255) / 256);
+ c.grid[1] = c.grid[2] = 1;
+ g_cx->run({c});
+ }
+ return true;
+}
+
+bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,
+ float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td) {
+ return mm_set(true, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, xf, ldx, idx, nullptr, h, ldh, nullptr, 0, tg, td);
+}
+bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,
+ float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td) {
+ return mm_set(false, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, nullptr, 0, nullptr, h16, nullptr, ldh, y, ldy, tg, td);
+}
+
// ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------
// strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched
// command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index db5af13ee..c9d49e89e 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -2074,3 +2074,90 @@ kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffe
uint gid [[thread_position_in_grid]]) {
if (ulong(gid) < n) dst[gid] = uchar(v);
}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// The prompt chunk's dense GEMM operands (see ds4_gemm_f16_nt on the host): float rows -> half, a quantized matrix -> half rows,
+// strided half rows packed, and the float result scattered into the engine's strided output.
+
+kernel void ds4_rows_f16(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],
+ device const int* idx [[buffer(2)]], constant uint& has_idx [[buffer(3)]],
+ constant uint& ncols [[buffer(4)]], device half* out [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint r = tgp.y, c = tgp.x * 256u + tid;
+ if (c >= ncols) return;
+ const ulong row = has_idx != 0u ? ulong(idx[r]) : ulong(r);
+ out[(ulong) r * ncols + c] = half(x[row * ldx + c]);
+}
+
+static inline void ds4_kscale2(const device uchar* q, int j, thread int& d, thread int& m) {
+ if (j < 4) {
+ d = int(q[j]) & 63;
+ m = int(q[j + 4]) & 63;
+ } else {
+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);
+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);
+ }
+}
+
+// a thread a 32-value sub-block of a Q5_K matrix (n rows of k values, ld bytes a row) -> out[n][k] half
+kernel void ds4_dq_q5k(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ const uint nb = k / 256u;
+ const ulong total = (ulong) n * nb * 8ul;
+ if (ulong(gid) >= total) return;
+ const uint ib = gid & 7u;
+ const uint blk = (gid >> 3) % nb;
+ const uint row = uint(ulong(gid) / (8ul * nb));
+ const device uchar* p = W + (ulong) row * ld + (ulong) blk * 176ul;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ int sc, mm;
+ ds4_kscale2(p + 4, int(ib), sc, mm);
+ const device uchar* qh = p + 16;
+ const device uchar* qs = p + 48 + 32u * (ib >> 1);
+ const uint sh = 4u * (ib & 1u);
+ device half* o = out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u;
+ for (uint l = 0; l < 32u; ++l) {
+ const uint q = ((uint(qs[l]) >> sh) & 15u) | (((uint(qh[l]) >> ib) & 1u) << 4);
+ o[l] = half(d * float(sc) * float(q) - dmin * float(mm));
+ }
+}
+
+// Q8_0: 34-byte blocks (fp16 d, 32 int8)
+kernel void ds4_dq_q8(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ const uint ng = k / 32u;
+ const ulong total = (ulong) n * ng;
+ if (ulong(gid) >= total) return;
+ const uint g = gid % ng, row = gid / ng;
+ const device uchar* p = W + (ulong) row * ld + (ulong) g * 34ul;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ device half* o = out + (ulong) row * k + (ulong) g * 32ul;
+ for (uint l = 0; l < 32u; ++l) o[l] = half(d * float(char(p[2 + l])));
+}
+
+// BF16 values (ld values a row) -> half
+kernel void ds4_dq_bf16(device const ushort* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ const ulong total = (ulong) n * k;
+ if (ulong(gid) >= total) return;
+ const ulong r = ulong(gid) / k, c = ulong(gid) - r * k;
+ out[gid] = half(as_type<float>(uint(W[r * ld + c]) << 16));
+}
+
+// half rows with a row stride -> contiguous rows
+kernel void ds4_pack_h16(device const ushort* x [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ device ushort* out [[buffer(3)]], uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint c = tgp.x * 256u + tid;
+ if (c < k) out[(ulong) tgp.y * k + c] = x[(ulong) tgp.y * ld + c];
+}
+
+// y[r][c] = alpha * t[r][c] + beta * y[r][c]
+kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* y [[buffer(1)]], constant ulong& ldy [[buffer(2)]],
+ constant uint& n [[buffer(3)]], constant float& alpha [[buffer(4)]], constant float& beta [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint c = tgp.x * 256u + tid;
+ if (c >= n) return;
+ const ulong yi = (ulong) tgp.y * ldy + c;
+ y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);
+}
diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metal
index 10d7500b4..15f60df47 100644
--- a/metal/src/engine/msl/ds4_iq.metal
+++ b/metal/src/engine/msl/ds4_iq.metal
@@ -874,3 +874,187 @@ kernel void ds4_iq3xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[t
const float v = simd_sum(acc);
if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// Matvecs for NC activations at once (1, 2, 4, 8): the prompt windows' and the drafter's verify windows' columns. A lane takes a
+// 32-value sub-block (a 64-value pair for Q5_K) of the row, decodes its weights ONCE and dots them with each of the NC columns'
+// int8 activations, so the weight bytes (the cost) are read once for the window. ncol <= NC (the rest of the accumulators idle).
+
+template <uint NC>
+static inline void ds4_mvn_store(const device Ds4MvJob& b, uint row, ushort lane, thread float (&acc)[NC]) {
+ for (uint c = 0; c < NC; ++c) {
+ const float v = simd_sum(acc[c]);
+ if (lane == 0u && int(c) < b.ncol) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+template <uint NC>
+kernel void ds4_q5k_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint j = lane & 3u;
+ const uint bsel = lane >> 2;
+ float acc[NC];
+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 8u) {
+ const device uchar* p = w + (size_t) blk * 176u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ int sc0, mm0, sc1, mm1;
+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);
+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);
+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);
+ const device uint4* qh4 = (const device uint4*) (p + 16);
+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];
+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};
+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};
+ uint q0w[8], q1w[8];
+ for (uint t = 0; t < 8u; ++t) {
+ q0w[t] = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);
+ q1w[t] = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);
+ }
+ for (uint c = 0; c < NC; ++c) {
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 64u * j);
+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3];
+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};
+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;
+ for (uint t = 0; t < 8u; ++t) {
+ for (uint i = 0; i < 4u; ++i) {
+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));
+ sq0 += int((q0w[t] >> (8u * i)) & 0xffu) * x0;
+ sq1 += int((q1w[t] >> (8u * i)) & 0xffu) * x1;
+ sx0 += x0;
+ sx1 += x1;
+ }
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc[c]);
+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc[c]);
+ }
+ }
+ ds4_mvn_store<NC>(b, row, lane, acc);
+}
+
+// the grid-coded types: decode the sub-block's 32 weights (signs applied) into w[32] ...
+static inline void ds4_w32_iq2xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib);
+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);
+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);
+ db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];
+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ for (uint i = 0; i < 8u; ++i) {
+ const int code = int((g >> (8u * i)) & 0xffu);
+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;
+ }
+ }
+}
+static inline void ds4_w32_iq2xs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);
+ const uint sc = uint(p[66u + ib]);
+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[l]);
+ const ulong g = iq2xs_grid[q & 511u];
+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);
+ for (uint i = 0; i < 8u; ++i) {
+ const int code = int((g >> (8u * i)) & 0xffu);
+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;
+ }
+ }
+}
+static inline void ds4_w32_iq2s(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u + 4u * ib;
+ const device uchar* sgb = p + 34u + 4u * ib;
+ const uint qh = uint(p[66u + ib]);
+ const uint sc = uint(p[74u + ib]);
+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);
+ const ulong g = iq2s_grid[idx];
+ const uint sg = uint(sgb[l]);
+ for (uint i = 0; i < 8u; ++i) {
+ const int code = int((g >> (8u * i)) & 0xffu);
+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;
+ }
+ }
+}
+static inline void ds4_w32_iq3xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u;
+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);
+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);
+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);
+ db = d * (0.5f + float(aux >> 28)) * 0.5f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];
+ for (uint i = 0; i < 4u; ++i) {
+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);
+ if ((sgn & (1u << i)) != 0u) v0 = -v0;
+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;
+ w[l * 8u + i] = v0;
+ w[l * 8u + 4u + i] = v1;
+ }
+ }
+}
+
+// ... and dot them with each column (the sub-block's four 8-value groups: l < 2 use db0, l >= 2 db1)
+template <uint NC, uint TYPE>
+kernel void ds4_iqg_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;
+ constexpr uint bs = TYPE == 16 ? 66u : (TYPE == 17 ? 74u : (TYPE == 18 ? 98u : 82u));
+ const device uchar* w0p = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc[NC];
+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w0p + (size_t) blk * bs;
+ int w[32];
+ float db0, db1;
+ if (TYPE == 16) { ds4_w32_iq2xxs(p, ib, w, db0); db1 = db0; }
+ else if (TYPE == 17) ds4_w32_iq2xs(p, ib, w, db0, db1);
+ else if (TYPE == 22) ds4_w32_iq2s(p, ib, w, db0, db1);
+ else { ds4_w32_iq3xxs(p, ib, w, db0); db1 = db0; }
+ for (uint c = 0; c < NC; ++c) {
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int s0 = 0, s1 = 0;
+ for (uint t = 0; t < 8u; ++t) {
+ int part = 0;
+ for (uint i = 0; i < 4u; ++i) part += w[t * 4u + i] * int(char(xw[t] >> (8u * i)));
+ if (t < 4u) s0 += part; else s1 += part;
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc[c]);
+ }
+ }
+ ds4_mvn_store<NC>(b, row, lane, acc);
+}
+
+#define DS4_MVN_INST(NC) \
+ template [[host_name("ds4_q5k_mvn" #NC)]] kernel void ds4_q5k_mvn<NC>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq2xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 16>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq2xs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 17>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq2s_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 22>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq3xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 18>(device const uchar*, uint2, uint, ushort);
+DS4_MVN_INST(1)
+DS4_MVN_INST(2)
+DS4_MVN_INST(4)
+DS4_MVN_INST(8)
diff --git a/metal/src/engine/msl/ds4_mm.metal b/metal/src/engine/msl/ds4_mm.metal
new file mode 100644
index 000000000..61ebb303d
--- /dev/null
+++ b/metal/src/engine/msl/ds4_mm.metal
@@ -0,0 +1,137 @@
+// metal/src/engine/msl/ds4_mm.metal - the DeepSeek prompt chunk's expert GEMMs: moe_mm.metal's mm_gu3 / mm_down3 (matmul2d on dequantized
+// weight tiles) over the engine's own expert slots and its UNPADDED sort order. Compiled after quant/units/mma/moe_mm.metal.
+// tiles[i] = (slot, first row, valid rows): a tile is TM rows of one expert; rows past `valid` (the next expert's) are computed and
+// dropped (the operand rows exist: the scratch carries TM rows of slack), the swiglu is DeepSeek's clamped one.
+
+struct DsMmArgs {
+ ulong gate_off;
+ ulong up_off;
+ ulong down_off;
+ ulong stride; // bytes between slots
+ uint gu_row; // bytes per gate/up row (n_embd values)
+ uint d_row; // bytes per down row (n_ff values)
+ uint n_embd;
+ uint n_ff;
+ float limit;
+ uint pad0;
+};
+
+// xs[r][c] = half(x[idx[r]][c]) for r < rows
+kernel void ds4_mm_gather(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]], device const int* idx [[buffer(2)]],
+ constant uint& n_embd [[buffer(3)]], device half* xs [[buffer(4)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint c = tgp.x * 256u + tid;
+ if (c >= n_embd) return;
+ xs[(ulong) tgp.y * n_embd + c] = half(x[(ulong) idx[tgp.y] * ldx + c]);
+}
+
+template <int F, int TM, int TN, int TK, int SG>
+kernel void ds4_mm_gu(device const uchar* W [[buffer(0)]], device const half* XS [[buffer(1)]], device half* HS [[buffer(2)]],
+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {
+ if (tg.y >= n_tiles) return;
+ threadgroup float ot[SG][TM * TN];
+ threadgroup uint tab[fmt<F>::TAB];
+ load_tab<F>(tab, tid, SG * 32);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int4 tl = tiles[tg.y];
+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;
+ device const uchar* E = W + ulong(e) * a.stride;
+ device const uchar* g0 = E + a.gate_off + ulong(n0) * a.gu_row;
+ device const uchar* u0 = E + a.up_off + ulong(n0) * a.gu_row;
+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);
+ matmul2d<desc, execution_simdgroup> op;
+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tX((device half*)XS, dextents<int32_t, 2>(a.n_embd, 1 << 30));
+ auto rg = op.template get_right_input_cooperative_tensor<half, half, float>();
+ auto ru = op.template get_right_input_cooperative_tensor<half, half, float>();
+ auto tA = tX.template slice<TK, TM>(0, row0);
+ auto accg = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();
+ auto accu = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();
+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)
+ if (accg.is_valid_element(i)) { accg[i] = 0.0f; accu[i] = 0.0f; }
+ for (int k0 = 0; k0 < int(a.n_embd); k0 += TK) {
+ const uint16_t cap = rg.get_capacity(), half_cap = cap / 2;
+#pragma clang loop unroll(full)
+ for (uint16_t i = 0; i < half_cap; i += 4) {
+ const auto ix = rg.get_multidimensional_index(i);
+ const int k = k0 + ix[0], n = ix[1];
+ device const uchar* rgw = g0 + ulong(n) * a.gu_row;
+ device const uchar* ruw = u0 + ulong(n) * a.gu_row;
+ half4 wg, wu, wg2, wu2;
+ W8<F>::run(rgw, k, tab, wg, wg2);
+ W8<F>::run(ruw, k, tab, wu, wu2);
+ rg[i] = wg.x; rg[i + 1] = wg.y; rg[i + 2] = wg.z; rg[i + 3] = wg.w;
+ ru[i] = wu.x; ru[i + 1] = wu.y; ru[i + 2] = wu.z; ru[i + 3] = wu.w;
+ const uint16_t j = i + half_cap;
+ rg[j] = wg2.x; rg[j + 1] = wg2.y; rg[j + 2] = wg2.z; rg[j + 3] = wg2.w;
+ ru[j] = wu2.x; ru[j + 1] = wu2.y; ru[j + 2] = wu2.z; ru[j + 3] = wu2.w;
+ }
+ tA = tX.template slice<TK, TM>(k0, row0);
+ op.run(tA, rg, accg);
+ op.run(tA, ru, accu);
+ }
+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)
+ if (accg.is_valid_element(i)) {
+ const float g = fmin(accg[i], a.limit), u = fmin(fmax(accu[i], -a.limit), a.limit);
+ accg[i] = g / (1.0f + exp(-g)) * u;
+ }
+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));
+ accg.store(tO);
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ for (int it = lane; it < TM * TN; it += 32) {
+ const int m = it / TN, f = it - m * TN;
+ if (m < valid) HS[ulong(row0 + m) * a.n_ff + n0 + f] = half(ot[sg][it]);
+ }
+}
+
+template <int F, int TM, int TN, int TK, int SG>
+kernel void ds4_mm_dn(device const uchar* W [[buffer(0)]], device half* YEH [[buffer(1)]], device const half* HS [[buffer(2)]],
+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {
+ if (tg.y >= n_tiles || a.pad0 == 3u) return;
+ threadgroup float ot[SG][TM * TN];
+ threadgroup uint tab[fmt<F>::TAB];
+ load_tab<F>(tab, tid, SG * 32);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int4 tl = tiles[tg.y];
+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;
+ device const uchar* w0 = W + ulong(e) * a.stride + a.down_off + ulong(n0) * a.d_row;
+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);
+ matmul2d<desc, execution_simdgroup> op;
+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tH((device half*)HS, dextents<int32_t, 2>(a.n_ff, 1 << 30));
+ auto rw = op.template get_right_input_cooperative_tensor<half, half, float>();
+ auto tA = tH.template slice<TK, TM>(0, row0);
+ auto acc = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rw)>, float>();
+ for (uint16_t i = 0; i < acc.get_capacity(); ++i)
+ if (acc.is_valid_element(i)) acc[i] = 0.0f;
+ for (int k0 = 0; k0 < int(a.n_ff); k0 += TK) {
+ const uint16_t half_cap = rw.get_capacity() / 2;
+#pragma clang loop unroll(full)
+ for (uint16_t i = 0; i < half_cap; i += 4) {
+ const auto ix = rw.get_multidimensional_index(i);
+ half4 w, w2;
+ if (a.pad0 == 1u) { w = half4(0); w2 = half4(0); } else if (a.pad0 == 2u) { w = half4(1); w2 = half4(1); W8<F>::run(w0, 0, tab, w, w2); w = half4(0); } else
+ W8<F>::run(w0 + ulong(ix[1]) * a.d_row, k0 + ix[0], tab, w, w2);
+ rw[i] = w.x; rw[i + 1] = w.y; rw[i + 2] = w.z; rw[i + 3] = w.w;
+ const uint16_t j = i + half_cap;
+ rw[j] = w2.x; rw[j + 1] = w2.y; rw[j + 2] = w2.z; rw[j + 3] = w2.w;
+ }
+ tA = tH.template slice<TK, TM>(k0, row0);
+ op.run(tA, rw, acc);
+ }
+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));
+ acc.store(tO);
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ for (int it = lane; it < TM * TN; it += 32) {
+ const int m = it / TN, f = it - m * TN;
+ if (m < valid) ((device float*) YEH)[ulong(row0 + m) * a.n_embd + n0 + f] = ot[sg][it];
+ }
+}
+
+#define DS4_MM_INST(F, T) template [[host_name("ds4_mm_gu_" #T)]] kernel decltype(ds4_mm_gu<F, 16, 32, 32, 4>) ds4_mm_gu<F, 16, 32, 32, 4>; template [[host_name("ds4_mm_dn_" #T)]] kernel decltype(ds4_mm_dn<F, 16, 32, 32, 4>) ds4_mm_dn<F, 16, 32, 32, 4>;
+DS4_MM_INST(T_IQ2_XXS, 16)
+DS4_MM_INST(T_IQ2_XS, 17)
+DS4_MM_INST(T_IQ2_S, 22)
+DS4_MM_INST(T_IQ3_XXS, 18)
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 69ac72851..79ed8fe37 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -12,6 +12,7 @@
#include <mutex>
#include <unordered_map>
#include <string>
+#include <deque>
#include <cstdio>
#include <cstdlib>
#include <fstream>
@@ -39,6 +40,7 @@ struct Context::Impl {
id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate in
id<MTLComputeCommandEncoder> enc_def = nil;
int def_n = 0;
+ std::string def_names, inflight_names; // (DS4_ERRINFO) the kernels of the open / the committed command buffer
id<MTLSharedEvent> qev = nil; // consecutive deferred buffers are chained through it (untracked buffers: the queue alone does not order them)
uint64_t qev_v = 0;
id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)
@@ -320,6 +322,8 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];
}
+static std::deque<std::string> g_recent; // the last dispatches encoded (printed with a command buffer failure)
+static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }
static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed
static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatches
static std::mutex g_prof_mu;
@@ -368,6 +372,7 @@ void Context::split_runs() {
[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
[impl_->cb_def commit];
impl_->inflight = impl_->cb_def;
+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
impl_->cb_def = nil;
impl_->enc_def = nil;
impl_->def_n = 0;
@@ -388,6 +393,7 @@ void Context::wait_mark(void* token) {
++g_wait_n;
if (cb.status != MTLCommandBufferStatusCompleted)
std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
+ if (cb.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: recent: %s\n", recent_str().c_str()), std::fprintf(stderr, "mtl: info: %s\n", [[[cb.error userInfo] description] UTF8String]);
}
void Context::flush_runs() {
@@ -401,6 +407,7 @@ void Context::flush_runs() {
[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
[impl_->cb_def commit];
impl_->inflight = impl_->cb_def;
+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
impl_->cb_def = nil;
impl_->enc_def = nil;
impl_->def_n = 0;
@@ -415,6 +422,7 @@ void Context::flush_runs() {
if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }
if (impl_->inflight.status != MTLCommandBufferStatusCompleted)
std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);
+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: failed cb held: %s\n", impl_->inflight_names.c_str());
impl_->inflight = nil;
}
for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
@@ -427,12 +435,20 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
if (ds.size() == 1 && reps == 1 && !barrier_between) {
if (impl_->cb_def == nil || impl_->def_n >= 2000000) {
if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)
+ if (std::getenv("DS4_ERRINFO")) {
+ MTLCommandBufferDescriptor* cd = [MTLCommandBufferDescriptor new];
+ cd.retainedReferences = NO;
+ cd.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;
+ impl_->cb_def = [impl_->q commandBufferWithDescriptor:cd];
+ } else
impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];
if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];
impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
}
const auto te0 = std::chrono::steady_clock::now();
encode(impl_->enc_def, ds[0]);
+ g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();
+ if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";
g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();
static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;
if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 6595a041f..297025f51 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1256,6 +1256,12 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
}
void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (!bf16 && m.raw != nullptr) { // the prompt chunk's dense matrices: F16 rows for the MPP GEMM
+ const int kind = m.fmt == kFmtGG ? m.K : m.fmt == kFmtQ8_0 ? 8 : m.fmt == kFmtBF16 ? 0 : -1;
+ if (kind >= 0 && ::strata::dsv41::metal_host::dequant_rows(kind, m.raw, row_stride(m), m.k, m.n, dst)) return;
+ }
+#endif
if (m.fmt == kFmtGG && m.raw != nullptr) { // a K-quant / i-quant dense matrix (the IST-DAS sets' Q5_K): its own dequantizer
if (!iq::dequant_rows(m.K, m.raw, row_stride(m), m.k, m.n, dst, bf16, s)) fail("rows: a low-bit type not covered");
return;
@@ -1486,6 +1492,8 @@ iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (th
q.n_embd = sa.n_embd;
q.n_ff = sa.n_ff;
q.limit = sa.limit;
+ q.hbounds = sa.hbounds;
+ q.total_rows = sa.total_rows;
return q;
}
} // namespace
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index dfe7503ad..9d4965c44 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -1225,6 +1225,10 @@ bool light_down(const SetArgs& a, cudaStream_t s) {
}
bool set_gate_up(const SetArgs& a, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_gu(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xf, a.ldx, a.idx, a.h, a.ldo, a.tg, a.td))
+ return true;
+#endif
if (is_tq(a.tg)) return tq::set_gate_up(a, s);
if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;
SetArgs t = a;
@@ -1240,6 +1244,10 @@ bool set_gate_up(const SetArgs& a, cudaStream_t s) {
}
bool set_down(const SetArgs& a, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_dn(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xh, a.ldx, a.y, a.ldo, a.tg, a.td))
+ return true;
+#endif
if (is_tq(a.td)) return tq::set_down(a, s);
if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;
SetArgs t = a;
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 5fc415499..60b434cb0 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -91,6 +91,8 @@ struct SetArgs {
float* y = nullptr;
int ldo = 0;
int tg = 0, td = 0;
+ const int* hbounds = nullptr; // (Metal) the bounds on the host: [n_exp + 1], absolute rows
+ int total_rows = 0; // (Metal) the set's rows in all
int skip_le = 0; // (the tiles) the experts of at most this many rows are left out: the light kernels took them
};
bool set_gate_up(const SetArgs& a, cudaStream_t s);
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index dca4e18c9..b07b1b9fe 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -12,6 +12,11 @@
// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to
// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures
// (the file: see read_pics); the state is shared only where the same picture sits at the same place.
+#if defined(__APPLE__)
+#include <execinfo.h>
+#include <csignal>
+#include <unistd.h>
+#endif
#include "ds4_model.hpp"
#include "ds4_packinfo.hpp"
@@ -734,6 +739,14 @@ int spec_eval(strata::ds4::Model& m, const std::string& ids_path, int n, int win
} // namespace
int main(int argc, char** argv) {
+#if defined(__APPLE__)
+ if (std::getenv("DS4_BT")) { // a crash prints where it was (no debugger on the Mac box)
+ signal(SIGSEGV, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(139); });
+ signal(SIGBUS, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(138); });
+ signal(SIGABRT, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(134); });
+ }
+#endif
+
strata::ds4::Options o;
std::string ids_s, dump, eos = "1";
int n = 0, window = 16;
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 282ffe9d4..6bd5d6bdf 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1888,6 +1888,9 @@ struct Model::Impl {
// to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16
par = o.threads || tp > 1;
prompt_par = par || o.tp > 1;
+#if defined(STRATA_METAL)
+ if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path
+#endif
const bool tp_all = tp > 1 && tp == (int) devs.size();
for (size_t i = 0; i < devs.size();) {
Group G;
@@ -2726,7 +2729,7 @@ struct Model::Impl {
dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&
dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&
dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&
- dalloc(d, c.h16, C * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
+ dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
halloc(c.bounds_pin, 2048, err);
if (!ok) return false;
if (has_first && !dalloc(d, c.emb, C * D, err)) return false;
@@ -3373,6 +3376,8 @@ struct Model::Impl {
sa.bounds = d_bounds + i0;
sa.row0 = a;
sa.max_rows = max_rows;
+ sa.hbounds = b + i0;
+ sa.total_rows = nrows;
sa.n_embd = D;
sa.n_ff = FF;
sa.limit = g.lim;
diff --git a/src/dsv41/vendor/exl3.cu b/src/dsv41/vendor/exl3.cu
index 4f8ecd26b..01bf0cd44 100644
--- a/src/dsv41/vendor/exl3.cu
+++ b/src/dsv41/vendor/exl3.cu
@@ -19,6 +19,9 @@
// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy
// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). The
// prompt path below is this file's on every GPU.
+#if defined(STRATA_METAL)
+#include <ds4_metal_host.h>
+#endif
#include "strata/dsv41/exl3.hpp"
#include <cuda_fp16.h>
@@ -1728,6 +1731,9 @@ void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh,
void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s) {
if (nrows <= 0) return;
+#if defined(STRATA_METAL)
+ if (::strata::dsv41::metal_host::rows_f16(x, ldx, idx, nrows, ncols, out)) return;
+#endif
strata_ds4_launch(rows_f16_kernel, dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s, x, ldx, idx,
ncols, out);
launch_check("rows_f16");