Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: chunked prompt path (MPP expert tile GEMMs, dense gemm/dq, rows_f16), NC-column matvecs, dangling buffer and null-arg offset fixes (WIP, chunk output not yet matching window path)
Niko122158e5d7851e0e
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hppindex b080a5970..9ab2482fd 100644--- a/include/strata/dsv41/exl3.hpp+++ b/include/strata/dsv41/exl3.hpp@@ -152,6 +152,8 @@ struct SetArgs {const int* bounds = nullptr;int row0 = 0;int max_rows = 0; // the most rows one expert of the set has+ const int* hbounds = nullptr; // (Metal) the same bounds on the host: [n_exp + 1], absolute rows+ int total_rows = 0; // (Metal) the set's rows in allint n_embd = 4096, n_ff = 2048;float limit = 0.0f;};diff --git a/metal/include/metal_compat/cublas_v2.h b/metal/include/metal_compat/cublas_v2.hindex 4c9c42c58..c5ebf75e8 100644--- a/metal/include/metal_compat/cublas_v2.h+++ b/metal/include/metal_compat/cublas_v2.h@@ -54,7 +54,19 @@ static inline cublasStatus_t cublasSgemm(cublasHandle_t, cublasOperation_t opA,// the mixed-precision form: any argument list, because the engine passes cuBLAS's full 19-argument spelling; the body// names kernels/dense_mm.metal, which is where the real implementation goes.+// the chunk path's call (F16 weights [m][k] x F16 rows [n][k] -> F32 rows [n][m]): the MPP tile GEMM+static inline cublasStatus_t cublasGemmEx(cublasHandle_t, cublasOperation_t ta, cublasOperation_t tb, int m, int n, int k,+ const void* alpha, const void* A, cudaDataType_t at, int lda, const void* B,+ cudaDataType_t bt, int ldb, const void* beta, void* C, cudaDataType_t ct, int ldc,+ cublasComputeType_t, cublasGemmAlgo_t) {+ if (ta == CUBLAS_OP_T && tb == CUBLAS_OP_N && at == CUDA_R_16F && bt == CUDA_R_16F && ct == CUDA_R_32F && lda == k &&+ strata::dsv41::metal_host::gemm_f16_nt((const uint16_t*) A, (const uint16_t*) B, ldb, (float*) C, ldc, n, m, k, *(const float*) alpha,+ *(const float*) beta))+ return CUBLAS_STATUS_SUCCESS;+ strata_metal_dense_gemm_missing("cublasGemmEx (this operand combination)");+}+template <typename... A>-static inline cublasStatus_t cublasGemmEx(A...) {+static inline cublasStatus_t cublasGemmEx_unsupported(A...) {strata_metal_dense_gemm_missing("cublasGemmEx");}diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex d380dfd9c..8d7205341 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -49,6 +49,11 @@ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, floatbool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out);+bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td);+bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td);+bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out);+bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha, float beta);bool swiglu(const float* g, const float* u, int n, float lim, float* h);bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex f747aedbb..966eb491f 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -15,11 +15,13 @@#include <cstdint>#include <cstring>#include <algorithm>+#include <map>#include <vector>// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host readextern "C" void ds4_metal_flush();extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);+extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off);extern "C" void ds4_gpu_sync_all_streams();extern "C" unsigned long long ds4_dispatch_seq();static unsigned long long g_q8_deduped = 0;@@ -191,17 +193,18 @@ bool hc_post(const float* y, float* h, const float* post, const float* comb, intmtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );if (!by.handle || !bh.handle) return false;- // post is [T][8], comb is [T][64], straight from the caller's host/shared memory- mtl::Buffer bp = g_cx->alloc((size_t) T * 8 * 4);- mtl::Buffer bc = g_cx->alloc((size_t) T * 64 * 4);- if (!bp.ptr || !bc.ptr) return false;- std::memcpy(bp.ptr, post, (size_t) T * 8 * 4);- std::memcpy(bc.ptr, comb, (size_t) T * 64 * 4);+ // post is [T][8], comb is [T][64]: device operands like the others (reached through their host views)+ void* hpost = strata::gpu::host_view((void*) post);+ void* hcomb = strata::gpu::host_view((void*) comb);+ if (!hpost || !hcomb) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view for post / comb\n"); return false; }+ mtl::Buffer bp = wrap(hpost, (size_t) T * 8 * 4);+ mtl::Buffer bc = wrap(hcomb, (size_t) T * 64 * 4);+ if (!bp.handle || !bc.handle) return false;mtl::Pipeline pl;try { pl = g_cx->pipeline(g_lib, "ds4_hc_post"); } catch (...) { return false; }mtl::Dispatch d;d.p = &pl;- d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp), mtl::Arg::B(bc),+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp, off(hpost)), mtl::Arg::B(bc, off(hcomb)),mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};d.grid[0] = (uint32_t) ((dim + 255) / 256);d.grid[1] = (uint32_t) T;@@ -584,16 +587,26 @@ bool iq_mv(const void* jobs, int nj) {if (nj <= 0) return true;if (!ensure()) return false;const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;- std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1, exp_x2, exp_x3, exp_x4;- exp_iq.reserve((size_t) nj);- exp_k.reserve((size_t) nj);+ // Each job goes to the kernel of its type and column count: NC = 1, 2, 4 or 8 activations decode the row's weights once+ // (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.+ std::map<std::string, std::vector<Ds4IqMvJob>> groups;int max_n = 0;+ static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;for (int i = 0; i < nj; ++i) {const Ds4IqMvJob& q = src[i];if (q.n > max_n) max_n = q.n;const int ncol = q.ncol > 0 ? q.ncol : 1;- static const int maxstep = std::getenv("DS4_IQ_STEP") ? std::atoi(std::getenv("DS4_IQ_STEP")) : 16;- const int step = ncol > maxstep ? maxstep : ncol; // both kernels keep 16 accumulators+ const char* tn = nullptr;+ switch (q.type) {+ case 13: tn = "ds4_q5k_mvn"; break;+ case 16: tn = "ds4_iq2xxs_mvn"; break;+ case 17: tn = "ds4_iq2xs_mvn"; break;+ case 18: tn = "ds4_iq3xxs_mvn"; break;+ case 22: tn = "ds4_iq2s_mvn"; break;+ default: break;+ }+ const bool grid = tn != nullptr && !old_only;+ const int step = grid ? 8 : (ncol > 16 ? 16 : ncol);for (int c0 = 0; c0 < ncol; c0 += step) {Ds4IqMvJob b = q;b.ncol = (c0 + step < ncol) ? step : ncol - c0;@@ -602,19 +615,20 @@ bool iq_mv(const void* jobs, int nj) {b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);b.y = q.y + (size_t) c0 * (size_t) q.ycs;}- static const bool fast1 = std::getenv("DS4_NO_MV1") == nullptr;- if (fast1 && b.ncol == 1 && b.type == 13) exp_q5.push_back(b); // the decode step's one-activation kernels- else if (fast1 && b.ncol == 1 && b.type == 16) exp_x1.push_back(b);- else if (fast1 && b.ncol == 1 && b.type == 17) exp_x2.push_back(b);- else if (fast1 && b.ncol == 1 && b.type == 22) exp_x3.push_back(b);- else if (fast1 && b.ncol == 1 && b.type == 18) exp_x4.push_back(b);- else (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);+ std::string fn;+ if (grid) {+ const int nc = b.ncol <= 1 ? 1 : (b.ncol <= 2 ? 2 : (b.ncol <= 4 ? 4 : 8));+ fn = std::string(tn) + std::to_string(nc);+ } else {+ fn = (b.type == 13 || b.type == 14) ? "ds4_iq_mv_k" : "ds4_iq_mv";+ }+ groups[fn].push_back(b);}}if (max_n <= 0) return true;const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group- auto run_group = [&](std::vector<Ds4IqMvJob>& v, const char* fn) -> bool {- if (v.empty()) return true;+ for (auto& kv : groups) {+ std::vector<Ds4IqMvJob>& v = kv.second;mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));if (!bj.ptr) {std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");@@ -623,7 +637,7 @@ bool iq_mv(const void* jobs, int nj) {std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));mtl::Pipeline pl;try {- pl = g_cx->pipeline(g_lib_iq, fn);+ pl = g_cx->pipeline(g_lib_iq, kv.first);} catch (...) {g_cx->free(bj);return false;@@ -642,15 +656,7 @@ bool iq_mv(const void* jobs, int nj) {}g_cx->run(ds);g_cx->free(bj);- return true;- };- if (!run_group(exp_q5, "ds4_q5k_mv1")) return false;- if (!run_group(exp_x1, "ds4_iq2xxs_mv1")) return false;- if (!run_group(exp_x2, "ds4_iq2xs_mv1")) return false;- if (!run_group(exp_x3, "ds4_iq2s_mv1")) return false;- if (!run_group(exp_x4, "ds4_iq3xxs_mv1")) return false;- if (!run_group(exp_k, "ds4_iq_mv_k")) return false;- if (!run_group(exp_iq, "ds4_iq_mv")) return false;+ }return true;}@@ -1075,11 +1081,11 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int}mtl::Dispatch dsp;dsp.p = &pl;- dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, off(hw != nullptr ? hw : bw.ptr)),- mtl::Arg::B(bc, off(hc != nullptr ? hc : bc.ptr)),- mtl::Arg::B(bs, off(hs != nullptr ? hs : bs.ptr)),+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, (hw != nullptr ? off(hw) : (size_t) 0)),+ mtl::Arg::B(bc, (hc != nullptr ? off(hc) : (size_t) 0)),+ mtl::Arg::B(bs, (hs != nullptr ? off(hs) : (size_t) 0)),mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),- mtl::Arg::B(bk, off(hk != nullptr ? hk : bk.ptr)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),+ mtl::Arg::B(bk, (hk != nullptr ? off(hk) : (size_t) 0)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),mtl::Arg::B(bo, off(ho))};dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);dsp.grid[1] = (uint32_t) T;@@ -2043,6 +2049,349 @@ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, voi++// ---- the prompt chunk's dense GEMM: float rows -> half, a quantized matrix -> half rows, MPP matmul2d (kernels/dense_mm.metal)+namespace {+mtl::Buffer g_gm_xs, g_gm_yt; // grow-only scratch: packed activations, the contiguous result+int g_lib_mm = -2;+mtl::Buffer scratch(mtl::Buffer& b, size_t bytes) {+ if (!b.handle || b.bytes < bytes) {+ if (b.handle) g_cx->free(b);+ b = g_cx->alloc(bytes + (bytes >> 2));+ }+ return b;+}+} // namespace++bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out) {+ if (nrows <= 0 || ncols <= 0) return true;+ if (!ensure()) return false;+ void* hx = strata::gpu::host_view((void*) x);+ void* ho = strata::gpu::host_view(out);+ void* hi = idx ? strata::gpu::host_view((void*) idx) : nullptr;+ if (!hx || !ho) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bx = wrap(hx, ((size_t) (nrows - 1) * (size_t) ldx + (size_t) ncols) * 4);+ mtl::Buffer bo = wrap(ho, (size_t) nrows * (size_t) ncols * 2);+ mtl::Buffer bi = hi ? wrap(hi, (size_t) nrows * 4) : bx;+ if (!bx.handle || !bo.handle || !bi.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_rows_f16"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bi, hi ? off(hi) : off(hx)),+ mtl::Arg::V((unsigned) (hi ? 1 : 0)), mtl::Arg::V((unsigned) ncols), mtl::Arg::B(bo, off(ho))};+ d.grid[0] = (uint32_t) ((ncols + 255) / 256);+ d.grid[1] = (uint32_t) nrows;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}++// kind: 0 BF16 values, 8 Q8_0, 13 Q5_K. n rows of k values, ld the row stride (bytes; values for BF16), out half rows.+bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out) {+ if (n <= 0 || k <= 0) return true;+ if (!ensure()) return false;+ const char* fn = kind == 13 ? "ds4_dq_q5k" : kind == 8 ? "ds4_dq_q8" : kind == 0 ? "ds4_dq_bf16" : nullptr;+ if (!fn) return false;+ if ((kind == 13 && k % 256 != 0) || (kind == 8 && k % 32 != 0)) return false;+ void* hw = strata::gpu::host_view((void*) raw);+ void* ho = strata::gpu::host_view(out);+ if (!hw || !ho) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t wbytes = (size_t) (n - 1) * ld * (kind == 0 ? 2 : 1) + (kind == 0 ? (size_t) k * 2 : (kind == 13 ? (size_t) (k / 256) * 176 : (size_t) (k / 32) * 34));+ if (!std::getenv("DS4_NO_TOUCH")) { // (the matrix may be file-backed pages not read yet: the GPU does not page them in, it stalls)+ volatile unsigned char sink = 0;+ for (size_t o = 0; o < wbytes; o += 4096) sink = sink + ((const volatile unsigned char*) hw)[o];+ (void) sink;+ }+ mtl::Buffer bw = wrap(hw, wbytes);+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) k * 2);+ if (!bw.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }+ size_t items = kind == 13 ? (size_t) n * (k / 256) * 8 : kind == 8 ? (size_t) n * (k / 32) : (size_t) n * k;+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::V((unsigned long) ld), mtl::Arg::V((unsigned) k), mtl::Arg::V((unsigned) n),+ mtl::Arg::B(bo, off(ho))};+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ d.grid[0] = (uint32_t) ((items + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ g_cx->run({d});+ return true;+}++// Y[r][c] = alpha * (X16[r] . W16[c]) + beta * Y[r][c]: X16 rows x K (stride ldx halves), W16 N x K contiguous, Y rows x N float+// (stride ldy). The cuBLAS call's operands in row-major terms; the MPP tile kernel wants contiguous operands and result.+bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha,+ float beta) {+ if (rows <= 0 || N <= 0) return true;+ if (!ensure()) return false;+ if (g_lib_mm == -2) {+ g_lib_mm = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "dense_mm.metal"});+ if (g_lib_mm < 0) std::fprintf(stderr, "dsv41 metal: compiling dense_mm.metal failed\n");+ }+ if (g_lib_mm < 0) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ void* hw = strata::gpu::host_view((void*) W16);+ void* hx = strata::gpu::host_view((void*) X16);+ void* hy = strata::gpu::host_view(Y);+ if (!hw || !hx || !hy) return false;+ mtl::Buffer bw = wrap(hw, (size_t) N * K * 2);+ mtl::Buffer bx = wrap(hx, ((size_t) (rows - 1) * (size_t) ldx + (size_t) K) * 2);+ mtl::Buffer by = wrap(hy, ((size_t) (rows - 1) * (size_t) ldy + (size_t) N) * 4);+ if (!bw.handle || !bx.handle || !by.handle) return false;+ // activations contiguous+ mtl::Buffer ax = bx;+ size_t ax_off = off(hx);+ if (ldx != K) {+ ax = scratch(g_gm_xs, (size_t) rows * K * 2);+ ax_off = 0;+ mtl::Pipeline pp;+ try { pp = g_cx->pipeline(g_lib, "ds4_pack_h16"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pp;+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) K), mtl::Arg::B(ax)};+ d.grid[0] = (uint32_t) ((K + 255) / 256);+ d.grid[1] = (uint32_t) rows;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ }+ const bool direct = ldy == N && alpha == 1.0f && beta == 0.0f;+ mtl::Buffer ty = by;+ size_t ty_off = off(hy);+ if (!direct) {+ ty = scratch(g_gm_yt, (size_t) rows * N * 4);+ ty_off = 0;+ }+ const int tm = 64, tn = N <= 64 ? 32 : 64;+ const std::string fn = "gemm_f16_" + std::to_string(tm) + "_" + std::to_string(tn) + "_4";+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_mm, fn); } catch (...) { return false; }+ struct GemmArgs { int M, N, K, pad; } ga{rows, N, K, 0};+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(ax, ax_off), mtl::Arg::B(bw, off(hw)), mtl::Arg::B(ty, ty_off), mtl::Arg::V(ga)};+ d.grid[0] = (uint32_t) ((N + tn - 1) / tn);+ d.grid[1] = (uint32_t) ((rows + tm - 1) / tm);+ d.grid[2] = 1;+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ if (!direct) {+ mtl::Pipeline pa;+ try { pa = g_cx->pipeline(g_lib, "ds4_axpby_rows"); } catch (...) { return false; }+ mtl::Dispatch e;+ e.p = &pa;+ e.args = {mtl::Arg::B(ty), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned long) ldy), mtl::Arg::V((unsigned) N),+ mtl::Arg::V(alpha), mtl::Arg::V(beta)};+ e.grid[0] = (uint32_t) ((N + 255) / 256);+ e.grid[1] = (uint32_t) rows;+ e.grid[2] = 1;+ e.tg[0] = 256; e.tg[1] = e.tg[2] = 1;+ g_cx->run({e});+ }+ return true;+}+++// ---- the prompt chunk's expert GEMMs (ds4_mm.metal): a set = n_exp consecutive expert slots, their rows sorted by expert ----+namespace {+int g_lib_moe = -2;+mtl::Buffer g_mm_xs, g_mm_hs, g_mm_ys; // grow-only scratch: the sorted rows' half activations; the gate/up's swiglu rows (the down's input)+bool g_mm_hs_ok = false;+size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : 0; }+struct DsMmArgsH {+ unsigned long gate_off, up_off, down_off, stride;+ unsigned gu_row, d_row, n_embd, n_ff;+ float limit;+ unsigned pad0;+};+bool mm_prepare() {+ if (!ensure()) return false;+ if (g_lib_moe == -2) {+ g_lib_moe = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "moe_mm.metal", "../src/engine/msl/ds4_mm.metal"});+ if (g_lib_moe < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_mm.metal failed\n");+ }+ return g_lib_moe >= 0;+}+} // namespace++static bool mm_set(bool gate_up, const uint8_t* wbase, size_t stride, const size_t* tr, const int* hb, int n_exp, int total_rows,+ int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, const uint16_t* hin, uint16_t* hout,+ int ldh, float* yout, int ldy, int tg, int td) {+ const int type = gate_up ? tg : td;+ if (n_exp <= 0 || total_rows <= 0) return true;+ if (iq_bs(tg) == 0 || iq_bs(td) == 0 || n_embd % 256 != 0 || n_ff % 256 != 0) return false;+ if (ldh != n_ff || (!gate_up && ldy != n_embd)) return false;+ if (gate_up ? (n_ff % 128 != 0) : (n_embd % 128 != 0)) return false;+ if (!mm_prepare()) return false;+ constexpr int TM = 16;+ // the tile list: (slot, first row, valid rows) a TM-row piece of one expert+ std::vector<int> tiles;+ for (int e = 0; e < n_exp; ++e) {+ const int lo = hb[e] - hb[0], hi = hb[e + 1] - hb[0];+ for (int r = lo; r < hi; r += TM) { tiles.push_back(e); tiles.push_back(r); tiles.push_back(std::min(TM, hi - r)); tiles.push_back(0); }+ }+ const int ntiles = (int) (tiles.size() / 4);+ if (ntiles == 0) return true;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ const size_t len = ((end - base + page - 1) / page) * page;+ if (std::getenv("DS4_MM_DBG")) { size_t o2 = 0; std::fprintf(stderr, "mm wrap %p len %zu own=%d rem=%zu\n", (void*) base, len, ds4_gpu_buffer_for_cpu((void*) base, len, &o2) != nullptr, ds4_gpu_alloc_remaining((void*) base)); }+ return g_cx->wrap((void*) base, len);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ void* hw = strata::gpu::host_view((void*) wbase);+ if (!hw) return false;+ mtl::Buffer bw = wrap(hw, (size_t) n_exp * stride);+ mtl::Buffer bt = g_cx->alloc(tiles.size() * 4);+ std::memcpy(bt.ptr, tiles.data(), tiles.size() * 4);+ DsMmArgsH a{tr[0], tr[1], tr[2], (unsigned long) stride, (unsigned) ((size_t) n_embd / 256 * iq_bs(tg)), (unsigned) ((size_t) n_ff / 256 * iq_bs(td)),+ (unsigned) n_embd, (unsigned) n_ff, limit, std::getenv("DS4_MM_ABL") ? (unsigned) std::atoi(std::getenv("DS4_MM_ABL")) : 0u};+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "mm %s embd %d ff %d tg %d td %d lim %g stride %zu tr %zu %zu %zu gu_row %u d_row %u\n", gate_up ? "gu" : "dn", n_embd, n_ff, tg, td, limit, stride, tr[0], tr[1], tr[2], a.gu_row, a.d_row);+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_moe, std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type)); } catch (...) { g_cx->free(bt); return false; }+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "pipe %s max_threads %d tgmem %d width %d\n", pl.name.c_str(), pl.max_threads, pl.static_tg_mem, pl.simd_width);+ mtl::Dispatch d;+ d.p = &pl;+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;+ d.grid[2] = 1;+ d.grid[1] = (uint32_t) ntiles;+ if (gate_up) {+ // the sorted rows' activations, half+ const size_t xs_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 2;+ if (!g_mm_xs.handle || g_mm_xs.bytes < xs_bytes) {+ if (g_mm_xs.handle) g_cx->free(g_mm_xs);+ g_mm_xs = g_cx->alloc(xs_bytes + (xs_bytes >> 2));+ }+ void* hx = strata::gpu::host_view((void*) xf);+ void* hi = strata::gpu::host_view((void*) idx);+ void* ho = strata::gpu::host_view(hout);+ if (!hx || !hi || !ho) { g_cx->free(bt); return false; }+ // (the float rows: the whole token slab the engine's c.x holds; its extent is the allocation's)+ const size_t xrem = ds4_gpu_alloc_remaining(hx);+ mtl::Buffer bxf = wrap(hx, xrem > 0 ? xrem : (size_t) ldx * 4);+ mtl::Buffer bix = wrap(hi, (size_t) total_rows * 4);+ mtl::Pipeline pg;+ try { pg = g_cx->pipeline(g_lib_moe, "ds4_mm_gather"); } catch (...) { g_cx->free(bt); return false; }+ mtl::Dispatch dg;+ dg.p = &pg;+ dg.args = {mtl::Arg::B(bxf, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bix, off(hi)), mtl::Arg::V((unsigned) n_embd), mtl::Arg::B(g_mm_xs)};+ dg.grid[0] = (uint32_t) ((n_embd + 255) / 256);+ dg.grid[1] = (uint32_t) total_rows;+ dg.grid[2] = 1;+ dg.tg[0] = 256; dg.tg[1] = dg.tg[2] = 1;+ g_cx->run({dg});+ const size_t hs_bytes = ((size_t) total_rows + TM) * (size_t) n_ff * 2;+ if (!g_mm_hs.handle || g_mm_hs.bytes < hs_bytes) {+ if (g_mm_hs.handle) g_cx->free(g_mm_hs);+ g_mm_hs = g_cx->alloc(std::max<size_t>(hs_bytes + (hs_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));+ }+ g_mm_hs_ok = true;+ { // (the down's result rows are made here too)+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));+ }+ }+ (void) ho;+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_xs), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};+ d.grid[0] = (uint32_t) (n_ff / 128);+ } else {+ void* hh = strata::gpu::host_view((void*) hin);+ void* hy = strata::gpu::host_view(yout);+ void* hy_real = nullptr;+ if (!hh || !hy) { g_cx->free(bt); return false; }+ const size_t hrem = ds4_gpu_alloc_remaining(hh);+ if (!g_mm_hs_ok) { g_cx->free(bt); return false; }+ g_mm_hs_ok = false;+ (void) hrem;+ mtl::Buffer& bh = g_mm_hs;+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));+ }+ mtl::Buffer by = g_mm_ys;+ hy_real = hy;+ hy = by.ptr;+ if (std::getenv("DS4_MM_YALT")) { hy = hh; by = wrap(hh, (size_t) total_rows * (size_t) n_ff * 2); }+ if (std::getenv("DS4_MM_EXP1")) { by = g_cx->alloc((size_t) total_rows * (size_t) n_embd * 4 + 4096); hy = by.ptr; }+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_ys), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};+ d.grid[0] = (uint32_t) (n_embd / 128);+ }+ if (!gate_up && std::getenv("DS4_MM_ASGU")) { // test: the gate/up kernel on the down's buffers+ static mtl::Pipeline pgu;+ pgu = g_cx->pipeline(g_lib_moe, "ds4_mm_gu_" + std::to_string(tg));+ d.p = &pgu;+ d.args[1] = mtl::Arg::B(g_mm_xs);+ d.args[2] = mtl::Arg::B(g_mm_ys);+ if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / 128);+ }+ if (!gate_up && std::getenv("DS4_MM_DROP")) {+ const char* dk = std::getenv("DS4_MM_DROP");+ static mtl::Buffer dummy;+ if (!dummy.handle) dummy = g_cx->alloc(1 << 26);+ static mtl::Buffer dummy2;+ if (!dummy2.handle) dummy2 = g_cx->alloc(1 << 26);+ for (int k = 0; k < 4; ++k) if (std::strchr(dk, '0' + k)) d.args[k] = mtl::Arg::B(std::getenv("DS4_MM_D2") && k == std::atoi(std::getenv("DS4_MM_D2")) ? dummy2 : dummy);+ }+ if (!gate_up && std::getenv("DS4_MM_ONE")) { d.grid[1] = 1; if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "dn tile0 %d %d %d ntiles %d x %u rows %d\n", tiles[0], tiles[1], tiles[2], ntiles, d.grid[0], total_rows); }+ if (std::getenv("DS4_MM_DBG")) for (size_t q = 0; q < d.args.size(); ++q) if (d.args[q].buf) std::fprintf(stderr, "arg %s %zu bytes %zu bias %zu off %zu ptr %p\n", gate_up ? "gu" : "dn", q, d.args[q].buf->bytes, (size_t) d.args[q].buf->bias, (size_t) d.args[q].off, d.args[q].buf->ptr);+ if (std::getenv(gate_up ? "DS4_MM_NOGU" : "DS4_MM_NODN")) { g_cx->free(bt); return true; }+ g_cx->run({d});+ if (!std::getenv("DS4_MM_LEAK")) g_cx->free(bt);+ if (!gate_up) { // the result into the engine's rows+ void* hyr = strata::gpu::host_view(yout);+ const size_t n4 = (size_t) total_rows * (size_t) n_embd;+ const size_t base = (size_t) ((uintptr_t) hyr) & ~(size_t) (page - 1);+ mtl::Buffer bd = g_cx->wrap((void*) base, ((((size_t) (uintptr_t) hyr + n4 * 4 - base) + page - 1) / page) * page);+ mtl::Pipeline pc;+ try { pc = g_cx->pipeline(g_lib, "ds4_copy4"); } catch (...) { return false; }+ mtl::Dispatch c;+ c.p = &pc;+ c.args = {mtl::Arg::B(g_mm_ys), mtl::Arg::B(bd, (size_t) ((uintptr_t) hyr - base)), mtl::Arg::V((unsigned long) n4)};+ c.tg[0] = 256; c.tg[1] = c.tg[2] = 1;+ c.grid[0] = (uint32_t) ((n4 + 255) / 256);+ c.grid[1] = c.grid[2] = 1;+ g_cx->run({c});+ }+ return true;+}++bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,+ float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td) {+ return mm_set(true, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, xf, ldx, idx, nullptr, h, ldh, nullptr, 0, tg, td);+}+bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,+ float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td) {+ return mm_set(false, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, nullptr, 0, nullptr, h16, nullptr, ldh, y, ldy, tg, td);+}+// ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------// strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched// command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex db5af13ee..c9d49e89e 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -2074,3 +2074,90 @@ kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffeuint gid [[thread_position_in_grid]]) {if (ulong(gid) < n) dst[gid] = uchar(v);}++// ---------------------------------------------------------------------------------------------------------------------+// The prompt chunk's dense GEMM operands (see ds4_gemm_f16_nt on the host): float rows -> half, a quantized matrix -> half rows,+// strided half rows packed, and the float result scattered into the engine's strided output.++kernel void ds4_rows_f16(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],+ device const int* idx [[buffer(2)]], constant uint& has_idx [[buffer(3)]],+ constant uint& ncols [[buffer(4)]], device half* out [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint r = tgp.y, c = tgp.x * 256u + tid;+ if (c >= ncols) return;+ const ulong row = has_idx != 0u ? ulong(idx[r]) : ulong(r);+ out[(ulong) r * ncols + c] = half(x[row * ldx + c]);+}++static inline void ds4_kscale2(const device uchar* q, int j, thread int& d, thread int& m) {+ if (j < 4) {+ d = int(q[j]) & 63;+ m = int(q[j + 4]) & 63;+ } else {+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);+ }+}++// a thread a 32-value sub-block of a Q5_K matrix (n rows of k values, ld bytes a row) -> out[n][k] half+kernel void ds4_dq_q5k(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ const uint nb = k / 256u;+ const ulong total = (ulong) n * nb * 8ul;+ if (ulong(gid) >= total) return;+ const uint ib = gid & 7u;+ const uint blk = (gid >> 3) % nb;+ const uint row = uint(ulong(gid) / (8ul * nb));+ const device uchar* p = W + (ulong) row * ld + (ulong) blk * 176ul;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ int sc, mm;+ ds4_kscale2(p + 4, int(ib), sc, mm);+ const device uchar* qh = p + 16;+ const device uchar* qs = p + 48 + 32u * (ib >> 1);+ const uint sh = 4u * (ib & 1u);+ device half* o = out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u;+ for (uint l = 0; l < 32u; ++l) {+ const uint q = ((uint(qs[l]) >> sh) & 15u) | (((uint(qh[l]) >> ib) & 1u) << 4);+ o[l] = half(d * float(sc) * float(q) - dmin * float(mm));+ }+}++// Q8_0: 34-byte blocks (fp16 d, 32 int8)+kernel void ds4_dq_q8(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ const uint ng = k / 32u;+ const ulong total = (ulong) n * ng;+ if (ulong(gid) >= total) return;+ const uint g = gid % ng, row = gid / ng;+ const device uchar* p = W + (ulong) row * ld + (ulong) g * 34ul;+ const float d = float(as_type<half>(*(const device ushort*) p));+ device half* o = out + (ulong) row * k + (ulong) g * 32ul;+ for (uint l = 0; l < 32u; ++l) o[l] = half(d * float(char(p[2 + l])));+}++// BF16 values (ld values a row) -> half+kernel void ds4_dq_bf16(device const ushort* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ const ulong total = (ulong) n * k;+ if (ulong(gid) >= total) return;+ const ulong r = ulong(gid) / k, c = ulong(gid) - r * k;+ out[gid] = half(as_type<float>(uint(W[r * ld + c]) << 16));+}++// half rows with a row stride -> contiguous rows+kernel void ds4_pack_h16(device const ushort* x [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ device ushort* out [[buffer(3)]], uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint c = tgp.x * 256u + tid;+ if (c < k) out[(ulong) tgp.y * k + c] = x[(ulong) tgp.y * ld + c];+}++// y[r][c] = alpha * t[r][c] + beta * y[r][c]+kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* y [[buffer(1)]], constant ulong& ldy [[buffer(2)]],+ constant uint& n [[buffer(3)]], constant float& alpha [[buffer(4)]], constant float& beta [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint c = tgp.x * 256u + tid;+ if (c >= n) return;+ const ulong yi = (ulong) tgp.y * ldy + c;+ y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);+}diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metalindex 10d7500b4..15f60df47 100644--- a/metal/src/engine/msl/ds4_iq.metal+++ b/metal/src/engine/msl/ds4_iq.metal@@ -874,3 +874,187 @@ kernel void ds4_iq3xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[tconst float v = simd_sum(acc);if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);}++// ---------------------------------------------------------------------------------------------------------------------+// Matvecs for NC activations at once (1, 2, 4, 8): the prompt windows' and the drafter's verify windows' columns. A lane takes a+// 32-value sub-block (a 64-value pair for Q5_K) of the row, decodes its weights ONCE and dots them with each of the NC columns'+// int8 activations, so the weight bytes (the cost) are read once for the window. ncol <= NC (the rest of the accumulators idle).++template <uint NC>+static inline void ds4_mvn_store(const device Ds4MvJob& b, uint row, ushort lane, thread float (&acc)[NC]) {+ for (uint c = 0; c < NC; ++c) {+ const float v = simd_sum(acc[c]);+ if (lane == 0u && int(c) < b.ncol) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++template <uint NC>+kernel void ds4_q5k_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint j = lane & 3u;+ const uint bsel = lane >> 2;+ float acc[NC];+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 8u) {+ const device uchar* p = w + (size_t) blk * 176u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ int sc0, mm0, sc1, mm1;+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);+ const device uint4* qh4 = (const device uint4*) (p + 16);+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};+ uint q0w[8], q1w[8];+ for (uint t = 0; t < 8u; ++t) {+ q0w[t] = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);+ q1w[t] = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);+ }+ for (uint c = 0; c < NC; ++c) {+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 64u * j);+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3];+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;+ for (uint t = 0; t < 8u; ++t) {+ for (uint i = 0; i < 4u; ++i) {+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));+ sq0 += int((q0w[t] >> (8u * i)) & 0xffu) * x0;+ sq1 += int((q1w[t] >> (8u * i)) & 0xffu) * x1;+ sx0 += x0;+ sx1 += x1;+ }+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc[c]);+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc[c]);+ }+ }+ ds4_mvn_store<NC>(b, row, lane, acc);+}++// the grid-coded types: decode the sub-block's 32 weights (signs applied) into w[32] ...+static inline void ds4_w32_iq2xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib);+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);+ db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ for (uint i = 0; i < 8u; ++i) {+ const int code = int((g >> (8u * i)) & 0xffu);+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;+ }+ }+}+static inline void ds4_w32_iq2xs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);+ const uint sc = uint(p[66u + ib]);+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[l]);+ const ulong g = iq2xs_grid[q & 511u];+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);+ for (uint i = 0; i < 8u; ++i) {+ const int code = int((g >> (8u * i)) & 0xffu);+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;+ }+ }+}+static inline void ds4_w32_iq2s(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u + 4u * ib;+ const device uchar* sgb = p + 34u + 4u * ib;+ const uint qh = uint(p[66u + ib]);+ const uint sc = uint(p[74u + ib]);+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);+ const ulong g = iq2s_grid[idx];+ const uint sg = uint(sgb[l]);+ for (uint i = 0; i < 8u; ++i) {+ const int code = int((g >> (8u * i)) & 0xffu);+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;+ }+ }+}+static inline void ds4_w32_iq3xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u;+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);+ db = d * (0.5f + float(aux >> 28)) * 0.5f;+ for (uint l = 0; l < 4u; ++l) {+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];+ for (uint i = 0; i < 4u; ++i) {+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);+ if ((sgn & (1u << i)) != 0u) v0 = -v0;+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;+ w[l * 8u + i] = v0;+ w[l * 8u + 4u + i] = v1;+ }+ }+}++// ... and dot them with each column (the sub-block's four 8-value groups: l < 2 use db0, l >= 2 db1)+template <uint NC, uint TYPE>+kernel void ds4_iqg_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;+ constexpr uint bs = TYPE == 16 ? 66u : (TYPE == 17 ? 74u : (TYPE == 18 ? 98u : 82u));+ const device uchar* w0p = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc[NC];+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w0p + (size_t) blk * bs;+ int w[32];+ float db0, db1;+ if (TYPE == 16) { ds4_w32_iq2xxs(p, ib, w, db0); db1 = db0; }+ else if (TYPE == 17) ds4_w32_iq2xs(p, ib, w, db0, db1);+ else if (TYPE == 22) ds4_w32_iq2s(p, ib, w, db0, db1);+ else { ds4_w32_iq3xxs(p, ib, w, db0); db1 = db0; }+ for (uint c = 0; c < NC; ++c) {+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int s0 = 0, s1 = 0;+ for (uint t = 0; t < 8u; ++t) {+ int part = 0;+ for (uint i = 0; i < 4u; ++i) part += w[t * 4u + i] * int(char(xw[t] >> (8u * i)));+ if (t < 4u) s0 += part; else s1 += part;+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc[c]);+ }+ }+ ds4_mvn_store<NC>(b, row, lane, acc);+}++#define DS4_MVN_INST(NC) \+ template [[host_name("ds4_q5k_mvn" #NC)]] kernel void ds4_q5k_mvn<NC>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq2xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 16>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq2xs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 17>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq2s_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 22>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq3xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 18>(device const uchar*, uint2, uint, ushort);+DS4_MVN_INST(1)+DS4_MVN_INST(2)+DS4_MVN_INST(4)+DS4_MVN_INST(8)diff --git a/metal/src/engine/msl/ds4_mm.metal b/metal/src/engine/msl/ds4_mm.metalnew file mode 100644index 000000000..61ebb303d--- /dev/null+++ b/metal/src/engine/msl/ds4_mm.metal@@ -0,0 +1,137 @@+// metal/src/engine/msl/ds4_mm.metal - the DeepSeek prompt chunk's expert GEMMs: moe_mm.metal's mm_gu3 / mm_down3 (matmul2d on dequantized+// weight tiles) over the engine's own expert slots and its UNPADDED sort order. Compiled after quant/units/mma/moe_mm.metal.+// tiles[i] = (slot, first row, valid rows): a tile is TM rows of one expert; rows past `valid` (the next expert's) are computed and+// dropped (the operand rows exist: the scratch carries TM rows of slack), the swiglu is DeepSeek's clamped one.++struct DsMmArgs {+ ulong gate_off;+ ulong up_off;+ ulong down_off;+ ulong stride; // bytes between slots+ uint gu_row; // bytes per gate/up row (n_embd values)+ uint d_row; // bytes per down row (n_ff values)+ uint n_embd;+ uint n_ff;+ float limit;+ uint pad0;+};++// xs[r][c] = half(x[idx[r]][c]) for r < rows+kernel void ds4_mm_gather(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]], device const int* idx [[buffer(2)]],+ constant uint& n_embd [[buffer(3)]], device half* xs [[buffer(4)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint c = tgp.x * 256u + tid;+ if (c >= n_embd) return;+ xs[(ulong) tgp.y * n_embd + c] = half(x[(ulong) idx[tgp.y] * ldx + c]);+}++template <int F, int TM, int TN, int TK, int SG>+kernel void ds4_mm_gu(device const uchar* W [[buffer(0)]], device const half* XS [[buffer(1)]], device half* HS [[buffer(2)]],+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {+ if (tg.y >= n_tiles) return;+ threadgroup float ot[SG][TM * TN];+ threadgroup uint tab[fmt<F>::TAB];+ load_tab<F>(tab, tid, SG * 32);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int4 tl = tiles[tg.y];+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;+ device const uchar* E = W + ulong(e) * a.stride;+ device const uchar* g0 = E + a.gate_off + ulong(n0) * a.gu_row;+ device const uchar* u0 = E + a.up_off + ulong(n0) * a.gu_row;+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);+ matmul2d<desc, execution_simdgroup> op;+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tX((device half*)XS, dextents<int32_t, 2>(a.n_embd, 1 << 30));+ auto rg = op.template get_right_input_cooperative_tensor<half, half, float>();+ auto ru = op.template get_right_input_cooperative_tensor<half, half, float>();+ auto tA = tX.template slice<TK, TM>(0, row0);+ auto accg = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();+ auto accu = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)+ if (accg.is_valid_element(i)) { accg[i] = 0.0f; accu[i] = 0.0f; }+ for (int k0 = 0; k0 < int(a.n_embd); k0 += TK) {+ const uint16_t cap = rg.get_capacity(), half_cap = cap / 2;+#pragma clang loop unroll(full)+ for (uint16_t i = 0; i < half_cap; i += 4) {+ const auto ix = rg.get_multidimensional_index(i);+ const int k = k0 + ix[0], n = ix[1];+ device const uchar* rgw = g0 + ulong(n) * a.gu_row;+ device const uchar* ruw = u0 + ulong(n) * a.gu_row;+ half4 wg, wu, wg2, wu2;+ W8<F>::run(rgw, k, tab, wg, wg2);+ W8<F>::run(ruw, k, tab, wu, wu2);+ rg[i] = wg.x; rg[i + 1] = wg.y; rg[i + 2] = wg.z; rg[i + 3] = wg.w;+ ru[i] = wu.x; ru[i + 1] = wu.y; ru[i + 2] = wu.z; ru[i + 3] = wu.w;+ const uint16_t j = i + half_cap;+ rg[j] = wg2.x; rg[j + 1] = wg2.y; rg[j + 2] = wg2.z; rg[j + 3] = wg2.w;+ ru[j] = wu2.x; ru[j + 1] = wu2.y; ru[j + 2] = wu2.z; ru[j + 3] = wu2.w;+ }+ tA = tX.template slice<TK, TM>(k0, row0);+ op.run(tA, rg, accg);+ op.run(tA, ru, accu);+ }+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)+ if (accg.is_valid_element(i)) {+ const float g = fmin(accg[i], a.limit), u = fmin(fmax(accu[i], -a.limit), a.limit);+ accg[i] = g / (1.0f + exp(-g)) * u;+ }+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));+ accg.store(tO);+ simdgroup_barrier(mem_flags::mem_threadgroup);+ for (int it = lane; it < TM * TN; it += 32) {+ const int m = it / TN, f = it - m * TN;+ if (m < valid) HS[ulong(row0 + m) * a.n_ff + n0 + f] = half(ot[sg][it]);+ }+}++template <int F, int TM, int TN, int TK, int SG>+kernel void ds4_mm_dn(device const uchar* W [[buffer(0)]], device half* YEH [[buffer(1)]], device const half* HS [[buffer(2)]],+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {+ if (tg.y >= n_tiles || a.pad0 == 3u) return;+ threadgroup float ot[SG][TM * TN];+ threadgroup uint tab[fmt<F>::TAB];+ load_tab<F>(tab, tid, SG * 32);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int4 tl = tiles[tg.y];+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;+ device const uchar* w0 = W + ulong(e) * a.stride + a.down_off + ulong(n0) * a.d_row;+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);+ matmul2d<desc, execution_simdgroup> op;+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tH((device half*)HS, dextents<int32_t, 2>(a.n_ff, 1 << 30));+ auto rw = op.template get_right_input_cooperative_tensor<half, half, float>();+ auto tA = tH.template slice<TK, TM>(0, row0);+ auto acc = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rw)>, float>();+ for (uint16_t i = 0; i < acc.get_capacity(); ++i)+ if (acc.is_valid_element(i)) acc[i] = 0.0f;+ for (int k0 = 0; k0 < int(a.n_ff); k0 += TK) {+ const uint16_t half_cap = rw.get_capacity() / 2;+#pragma clang loop unroll(full)+ for (uint16_t i = 0; i < half_cap; i += 4) {+ const auto ix = rw.get_multidimensional_index(i);+ half4 w, w2;+ if (a.pad0 == 1u) { w = half4(0); w2 = half4(0); } else if (a.pad0 == 2u) { w = half4(1); w2 = half4(1); W8<F>::run(w0, 0, tab, w, w2); w = half4(0); } else+ W8<F>::run(w0 + ulong(ix[1]) * a.d_row, k0 + ix[0], tab, w, w2);+ rw[i] = w.x; rw[i + 1] = w.y; rw[i + 2] = w.z; rw[i + 3] = w.w;+ const uint16_t j = i + half_cap;+ rw[j] = w2.x; rw[j + 1] = w2.y; rw[j + 2] = w2.z; rw[j + 3] = w2.w;+ }+ tA = tH.template slice<TK, TM>(k0, row0);+ op.run(tA, rw, acc);+ }+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));+ acc.store(tO);+ simdgroup_barrier(mem_flags::mem_threadgroup);+ for (int it = lane; it < TM * TN; it += 32) {+ const int m = it / TN, f = it - m * TN;+ if (m < valid) ((device float*) YEH)[ulong(row0 + m) * a.n_embd + n0 + f] = ot[sg][it];+ }+}++#define DS4_MM_INST(F, T) template [[host_name("ds4_mm_gu_" #T)]] kernel decltype(ds4_mm_gu<F, 16, 32, 32, 4>) ds4_mm_gu<F, 16, 32, 32, 4>; template [[host_name("ds4_mm_dn_" #T)]] kernel decltype(ds4_mm_dn<F, 16, 32, 32, 4>) ds4_mm_dn<F, 16, 32, 32, 4>;+DS4_MM_INST(T_IQ2_XXS, 16)+DS4_MM_INST(T_IQ2_XS, 17)+DS4_MM_INST(T_IQ2_S, 22)+DS4_MM_INST(T_IQ3_XXS, 18)diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 69ac72851..79ed8fe37 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -12,6 +12,7 @@#include <mutex>#include <unordered_map>#include <string>+#include <deque>#include <cstdio>#include <cstdlib>#include <fstream>@@ -39,6 +40,7 @@ struct Context::Impl {id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate inid<MTLComputeCommandEncoder> enc_def = nil;int def_n = 0;+ std::string def_names, inflight_names; // (DS4_ERRINFO) the kernels of the open / the committed command bufferid<MTLSharedEvent> qev = nil; // consecutive deferred buffers are chained through it (untracked buffers: the queue alone does not order them)uint64_t qev_v = 0;id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)@@ -320,6 +322,8 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];}+static std::deque<std::string> g_recent; // the last dispatches encoded (printed with a command buffer failure)+static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summedstatic std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatchesstatic std::mutex g_prof_mu;@@ -368,6 +372,7 @@ void Context::split_runs() {[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];[impl_->cb_def commit];impl_->inflight = impl_->cb_def;+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();impl_->cb_def = nil;impl_->enc_def = nil;impl_->def_n = 0;@@ -388,6 +393,7 @@ void Context::wait_mark(void* token) {++g_wait_n;if (cb.status != MTLCommandBufferStatusCompleted)std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);+ if (cb.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: recent: %s\n", recent_str().c_str()), std::fprintf(stderr, "mtl: info: %s\n", [[[cb.error userInfo] description] UTF8String]);}void Context::flush_runs() {@@ -401,6 +407,7 @@ void Context::flush_runs() {[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];[impl_->cb_def commit];impl_->inflight = impl_->cb_def;+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();impl_->cb_def = nil;impl_->enc_def = nil;impl_->def_n = 0;@@ -415,6 +422,7 @@ void Context::flush_runs() {if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }if (impl_->inflight.status != MTLCommandBufferStatusCompleted)std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: failed cb held: %s\n", impl_->inflight_names.c_str());impl_->inflight = nil;}for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }@@ -427,12 +435,20 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,if (ds.size() == 1 && reps == 1 && !barrier_between) {if (impl_->cb_def == nil || impl_->def_n >= 2000000) {if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)+ if (std::getenv("DS4_ERRINFO")) {+ MTLCommandBufferDescriptor* cd = [MTLCommandBufferDescriptor new];+ cd.retainedReferences = NO;+ cd.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;+ impl_->cb_def = [impl_->q commandBufferWithDescriptor:cd];+ } elseimpl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];}const auto te0 = std::chrono::steady_clock::now();encode(impl_->enc_def, ds[0]);+ g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();+ if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 6595a041f..297025f51 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1256,6 +1256,12 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {}void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (!bf16 && m.raw != nullptr) { // the prompt chunk's dense matrices: F16 rows for the MPP GEMM+ const int kind = m.fmt == kFmtGG ? m.K : m.fmt == kFmtQ8_0 ? 8 : m.fmt == kFmtBF16 ? 0 : -1;+ if (kind >= 0 && ::strata::dsv41::metal_host::dequant_rows(kind, m.raw, row_stride(m), m.k, m.n, dst)) return;+ }+#endifif (m.fmt == kFmtGG && m.raw != nullptr) { // a K-quant / i-quant dense matrix (the IST-DAS sets' Q5_K): its own dequantizerif (!iq::dequant_rows(m.K, m.raw, row_stride(m), m.k, m.n, dst, bf16, s)) fail("rows: a low-bit type not covered");return;@@ -1486,6 +1492,8 @@ iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (thq.n_embd = sa.n_embd;q.n_ff = sa.n_ff;q.limit = sa.limit;+ q.hbounds = sa.hbounds;+ q.total_rows = sa.total_rows;return q;}} // namespacediff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex dfe7503ad..9d4965c44 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -1225,6 +1225,10 @@ bool light_down(const SetArgs& a, cudaStream_t s) {}bool set_gate_up(const SetArgs& a, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_gu(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xf, a.ldx, a.idx, a.h, a.ldo, a.tg, a.td))+ return true;+#endifif (is_tq(a.tg)) return tq::set_gate_up(a, s);if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;SetArgs t = a;@@ -1240,6 +1244,10 @@ bool set_gate_up(const SetArgs& a, cudaStream_t s) {}bool set_down(const SetArgs& a, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_dn(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xh, a.ldx, a.y, a.ldo, a.tg, a.td))+ return true;+#endifif (is_tq(a.td)) return tq::set_down(a, s);if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;SetArgs t = a;diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 5fc415499..60b434cb0 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -91,6 +91,8 @@ struct SetArgs {float* y = nullptr;int ldo = 0;int tg = 0, td = 0;+ const int* hbounds = nullptr; // (Metal) the bounds on the host: [n_exp + 1], absolute rows+ int total_rows = 0; // (Metal) the set's rows in allint skip_le = 0; // (the tiles) the experts of at most this many rows are left out: the light kernels took them};bool set_gate_up(const SetArgs& a, cudaStream_t s);diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex dca4e18c9..b07b1b9fe 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -12,6 +12,11 @@// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures// (the file: see read_pics); the state is shared only where the same picture sits at the same place.+#if defined(__APPLE__)+#include <execinfo.h>+#include <csignal>+#include <unistd.h>+#endif#include "ds4_model.hpp"#include "ds4_packinfo.hpp"@@ -734,6 +739,14 @@ int spec_eval(strata::ds4::Model& m, const std::string& ids_path, int n, int win} // namespaceint main(int argc, char** argv) {+#if defined(__APPLE__)+ if (std::getenv("DS4_BT")) { // a crash prints where it was (no debugger on the Mac box)+ signal(SIGSEGV, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(139); });+ signal(SIGBUS, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(138); });+ signal(SIGABRT, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(134); });+ }+#endif+strata::ds4::Options o;std::string ids_s, dump, eos = "1";int n = 0, window = 16;diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 282ffe9d4..6bd5d6bdf 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1888,6 +1888,9 @@ struct Model::Impl {// to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16par = o.threads || tp > 1;prompt_par = par || o.tp > 1;+#if defined(STRATA_METAL)+ if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path+#endifconst bool tp_all = tp > 1 && tp == (int) devs.size();for (size_t i = 0; i < devs.size();) {Group G;@@ -2726,7 +2729,7 @@ struct Model::Impl {dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&- dalloc(d, c.h16, C * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&+ dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&halloc(c.bounds_pin, 2048, err);if (!ok) return false;if (has_first && !dalloc(d, c.emb, C * D, err)) return false;@@ -3373,6 +3376,8 @@ struct Model::Impl {sa.bounds = d_bounds + i0;sa.row0 = a;sa.max_rows = max_rows;+ sa.hbounds = b + i0;+ sa.total_rows = nrows;sa.n_embd = D;sa.n_ff = FF;sa.limit = g.lim;diff --git a/src/dsv41/vendor/exl3.cu b/src/dsv41/vendor/exl3.cuindex 4f8ecd26b..01bf0cd44 100644--- a/src/dsv41/vendor/exl3.cu+++ b/src/dsv41/vendor/exl3.cu@@ -19,6 +19,9 @@// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). The// prompt path below is this file's on every GPU.+#if defined(STRATA_METAL)+#include <ds4_metal_host.h>+#endif#include "strata/dsv41/exl3.hpp"#include <cuda_fp16.h>@@ -1728,6 +1731,9 @@ void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh,void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s) {if (nrows <= 0) return;+#if defined(STRATA_METAL)+ if (::strata::dsv41::metal_host::rows_f16(x, ldx, idx, nrows, ncols, out)) return;+#endifstrata_ds4_launch(rows_f16_kernel, dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s, x, ldx, idx,ncols, out);launch_check("rows_f16");