Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: sparse attention + prompt matvec columns + exact wrap extents - prefill measured at 556 tok/s
Niko1221c486f17251e5
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex 8a7c247cb..8c42502c2 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -54,6 +54,7 @@ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long l// the entry points, as metal/src/engine/ds4_finish.cpp defines thembool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);+bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);@@ -61,6 +62,7 @@ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd,bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);+bool gather_rows(const float* x, const void* g, int n, int cols, float* out);bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy);bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride);@@ -83,9 +85,11 @@ bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, consbool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);+bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w, const float* bias_vl, const void* vl);bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out);bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows, const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt);bool swiglu(const float* g, const float* u, int n, float lim, float* h);bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out, int out_stride);bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex e309b0b63..93b1be059 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -13,6 +13,8 @@#include <cstdio>#include <cstdint>#include <cstring>+#include <algorithm>+#include <vector>// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host readextern "C" void ds4_metal_flush();@@ -236,8 +238,8 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,void* hb = strata::gpu::host_view((void*) base);if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }const size_t nbase = (size_t) (2 * hc + hc * hc);- mtl::Buffer bs = wrap(hs, 256);- mtl::Buffer bb = wrap(hb, (nbase * 4 + 65535) / 65536 * 65536 );+ mtl::Buffer bs = wrap(hs, 3 * sizeof(float));+ mtl::Buffer bb = wrap(hb, nbase * sizeof(float));if (!bs.handle || !bb.handle) return false;mtl::Pipeline pl;try { pl = g_cx->pipeline(g_lib, "ds4_hc_split"); } catch (...) { return false; }@@ -271,7 +273,7 @@ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* boundsreturn g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- const size_t t_bytes = (size_t) T * k * 4 + 65536;+ const size_t t_bytes = (size_t) T * (size_t) k * 4;mtl::Buffer bi = wrap(hi, t_bytes);mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );@@ -455,32 +457,82 @@ bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt) {// ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they// travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.+static inline void* cpu_addr(void* p);+namespace {+// mirrors ds4_iq.cuh's MvJob (and ds4_iq.metal's Ds4MvJob): 80 bytes on arm64+struct Ds4IqMvJob {+ const uint8_t* w;+ uint64_t ld;+ int type, k, n, ncol;+ const int8_t* xq;+ const float* xd;+ float* y;+ uint64_t ycs;+ const float* bias;+ float alpha;+};+static_assert(sizeof(Ds4IqMvJob) == 80, "Ds4IqMvJob layout must match ds4_iq.cuh's MvJob");+} // namespace+bool iq_mv(const void* jobs, int nj) {if (nj <= 0) return true;if (!ensure()) return false;- // max rows across the jobs: the grid's x extent- const unsigned char* jb = (const unsigned char*) jobs;+ const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;+ std::vector<Ds4IqMvJob> exp_iq, exp_k;+ exp_iq.reserve((size_t) nj);+ exp_k.reserve((size_t) nj);int max_n = 0;for (int i = 0; i < nj; ++i) {- int n = 0;- std::memcpy(&n, jb + (size_t) i * 80 + 24, sizeof(int)); // MvJob::n- if (n > max_n) max_n = n;+ const Ds4IqMvJob& q = src[i];+ if (q.n > max_n) max_n = q.n;+ const int ncol = q.ncol > 0 ? q.ncol : 1;+ const int step = ncol > 16 ? 16 : ncol; // both kernels keep 16 accumulators+ for (int c0 = 0; c0 < ncol; c0 += step) {+ Ds4IqMvJob b = q;+ b.ncol = (c0 + step < ncol) ? step : ncol - c0;+ if (c0 != 0) {+ b.xq = q.xq + (size_t) c0 * (size_t) q.k;+ b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;+ }+ (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);+ }}if (max_n <= 0) return true;- mtl::Buffer bj = g_cx->alloc((size_t) nj * 80);- if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n"); return false; }- std::memcpy(bj.ptr, jobs, (size_t) nj * 80);- mtl::Pipeline pl;- try { pl = g_cx->pipeline(g_lib_iq, "ds4_iq_mv"); } catch (...) { return false; }- mtl::Dispatch d;- d.p = &pl;- d.args = {mtl::Arg::B(bj)};- d.grid[0] = (uint32_t) ((max_n + 7) / 8);- d.grid[1] = (uint32_t) nj;- d.grid[2] = 1;- d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;- g_cx->run({d});- g_cx->free(bj);+ const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group+ auto run_group = [&](std::vector<Ds4IqMvJob>& v, const char* fn) -> bool {+ if (v.empty()) return true;+ mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");+ return false;+ }+ std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));+ mtl::Pipeline pl;+ try {+ pl = g_cx->pipeline(g_lib_iq, fn);+ } catch (...) {+ g_cx->free(bj);+ return false;+ }+ std::vector<mtl::Dispatch> ds;+ for (size_t base = 0; base < v.size(); base += kBatch) {+ const size_t cnt = std::min(kBatch, v.size() - base);+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj, base * sizeof(Ds4IqMvJob))};+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);+ d.grid[1] = (uint32_t) cnt;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ ds.push_back(d);+ }+ g_cx->run(ds);+ g_cx->free(bj);+ return true;+ };+ if (!run_group(exp_k, "ds4_iq_mv_k")) return false;+ if (!run_group(exp_iq, "ds4_iq_mv")) return false;return true;}@@ -710,6 +762,268 @@ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d,// ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)+namespace {+// mirrors metal/src/engine/msl/ds4_init.metal's Ds4AttnRows (ds4_kernels.cuh:149 AttnRows, kWin = 16)+struct Ds4AttnRowsJob {+ int win0[16], nwin[16], nsel[16];+ int gen, t0, win, R, kcap;+ int64_t p0;+};+static_assert(sizeof(Ds4AttnRowsJob) == 224, "Ds4AttnRowsJob layout must match the MSL struct");+} // namespace++// ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false> - the prompt path's sparse attention.+// The packed kernel stages rows once for G heads of a query; the BF16 kernel handles rows already in BF16 (a DSpark block).+bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows,+ const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt) {+ if (T <= 0 || heads <= 0) return true;+ if (d != 512 || rows == nullptr || (ckv_fmt != 0 && ckv_fmt != 2)) { std::fprintf(stderr, "dsv41 metal: sparse_attn unsupported\n"); return false; }+ Ds4AttnRowsJob ar{};+ std::memcpy(&ar, rows, sizeof(ar));+ if (q == nullptr || out == nullptr) return false;+ if (ar.gen == 0 && T > 16) { std::fprintf(stderr, "dsv41 metal: sparse_attn gen0 T>16\n"); return false; }+ if (!ensure()) return false;++ void* hq = cpu_addr((void*) q);+ void* hw = cpu_addr((void*) win);+ void* hc = cpu_addr((void*) ckv);+ void* hs = cpu_addr((void*) sel);+ void* hk = cpu_addr((void*) sink);+ void* ho = cpu_addr((void*) out);+ if (!hq || !ho) {+ std::fprintf(stderr, "dsv41 metal: sparse_attn has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };++ int wmax = 0;+ if (ar.gen != 0) {+ wmax = ar.win + T + 1;+ } else {+ for (int t = 0; t < T; ++t) wmax = ar.win0[t] + ar.nwin[t] > wmax ? ar.win0[t] + ar.nwin[t] : wmax;+ wmax += 1;+ }+ if (wmax > 0 && hw == nullptr) { std::fprintf(stderr, "dsv41 metal: sparse_attn win null (wmax %d)\n", wmax); return false; }+ const size_t wrows = (size_t) (wmax > 0 ? wmax : 1);+ size_t crows = 0;+ if (ckv_fmt == 2) {+ if (ar.gen != 0 && ar.R > 0) {+ crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position+ } else {+ int maxrow = -1; // the selected rows, not the cache's whole cap+ if (hs != nullptr && sel_stride > 0)+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < ar.nsel[t]; ++i) {+ const int r = ((const int*) hs)[(size_t) t * (size_t) sel_stride + (size_t) i];+ if (r > maxrow) maxrow = r;+ }+ crows = (size_t) (maxrow + 1);+ }+ if (crows < 1) crows = 1;+ } else {+ crows = (size_t) (T > 16 ? T : 16);+ }+ const size_t qbytes = (size_t) T * (size_t) heads * 512 * 4;+ const size_t wbytes = wrows * 512 * 2;+ const size_t cbytes = crows * (ckv_fmt == 2 ? 288 : 1024);+ const size_t obytes = (size_t) T * (size_t) heads * 512 * 4;+ const size_t sbytes = (size_t) T * (size_t) (sel_stride > 0 ? sel_stride : 1) * 4;+ mtl::Buffer bq = wrap(hq, qbytes);+ mtl::Buffer bw = hw != nullptr ? wrap(hw, wbytes) : g_cx->alloc(64);+ mtl::Buffer bc = hc != nullptr ? wrap(hc, cbytes) : g_cx->alloc(64);+ mtl::Buffer bs = hs != nullptr ? wrap(hs, sbytes) : g_cx->alloc(4);+ mtl::Buffer bk = hk != nullptr ? wrap(hk, (size_t) heads * 4) : g_cx->alloc((size_t) heads * 4);+ mtl::Buffer bo = wrap(ho, obytes);+ if (!bq.handle || !bw.handle || !bc.handle || !bs.handle || !bk.handle || !bo.handle) {+ std::fprintf(stderr, "dsv41 metal: sparse_attn wrap failed\n");+ return false;+ }++ int G = 8;+ if (heads % 8 != 0) G = heads % 4 == 0 ? 4 : (heads % 2 == 0 ? 2 : 1);+ while (G > 1 && (heads / G) * T < 64) G /= 2; // keep enough threadgroups on the GPU (CUDA's 2 * SMs, in effect)++ const char* fn = ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"+ : G == 2 ? "ds4_sparse_attn_g2" : "ds4_sparse_attn_g1")+ : "ds4_sparse_attn_bf16";+ mtl::Pipeline pl;+ try {+ pl = g_cx->pipeline(g_lib, fn);+ } catch (...) {+ std::fprintf(stderr, "dsv41 metal: sparse_attn pipeline %s unavailable\n", fn);+ return false;+ }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, off(hw != nullptr ? hw : bw.ptr)),+ mtl::Arg::B(bc, off(hc != nullptr ? hc : bc.ptr)),+ mtl::Arg::B(bs, off(hs != nullptr ? hs : bs.ptr)),+ mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),+ mtl::Arg::B(bk, off(hk != nullptr ? hk : bk.ptr)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),+ mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256;+ dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}+++// ds4_kernels.cu:461 route_kernel - sqrt(softplus(logits)) top k with bias, then the unbiased weights. vl is a VlBits+// (1024 bytes) or null; bias_vl applies to the rows its bits select.+bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,+ const float* bias_vl, const void* vl) {+ if (T <= 0) return true;+ if (logits == nullptr || bias == nullptr || idx == nullptr || w == nullptr || n <= 0 || n > 1024 || k <= 0 || k > 8)+ return false;+ if (!ensure()) return false;+ void* hl = cpu_addr((void*) logits);+ void* hb = cpu_addr((void*) bias);+ void* hi = cpu_addr((void*) idx);+ void* hw = cpu_addr((void*) w);+ void* hv = bias_vl != nullptr ? cpu_addr((void*) bias_vl) : nullptr;+ if (!hl || !hb || !hi || !hw) {+ std::fprintf(stderr, "dsv41 metal: route has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4);+ mtl::Buffer bb = wrap(hb, (size_t) n * 4);+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4);+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4);+ mtl::Buffer bv = hv != nullptr ? wrap(hv, (size_t) n * 4) : g_cx->alloc(4);+ mtl::Buffer bbv = g_cx->alloc(1024);+ if (!bl.handle || !bb.handle || !bi.handle || !bw.handle || !bv.handle || !bbv.ptr) return false;+ const bool has_vl = vl != nullptr;+ if (has_vl) std::memcpy(bbv.ptr, vl, 1024);+ mtl::Pipeline pl;+ try {+ pl = g_cx->pipeline(g_lib, "ds4_route");+ } catch (...) {+ std::fprintf(stderr, "dsv41 metal: route pipeline unavailable\n");+ return false;+ }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),+ mtl::Arg::V(route_scale), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::B(bv, hv != nullptr ? off(hv) : 0), mtl::Arg::V(has_vl ? 1u : 0u), mtl::Arg::B(bbv)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 512;+ dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(bbv);+ return true;+}+++namespace {+struct Ds4GatherJob {+ int src[16 * 8];+};+struct Ds4CombineJob {+ int n[16];+ int row[16][8];+ float w[16][8];+ int cpu[16];+};+static_assert(sizeof(Ds4GatherJob) == 512, "Ds4GatherJob layout must match the MSL struct");+static_assert(sizeof(Ds4CombineJob) == 1152, "Ds4CombineJob layout must match the MSL struct");+} // namespace++// ds4_kernels.cu:1336 gather_kernel - out[r] = x[g.src[r]] for n rows of cols floats+bool gather_rows(const float* x, const void* g, int n, int cols, float* out) {+ if (n <= 0 || cols <= 0) return true;+ if (x == nullptr || g == nullptr || out == nullptr || n > 16 * 8) return false;+ if (!ensure()) return false;+ void* hx = cpu_addr((void*) x);+ void* ho = cpu_addr((void*) out);+ if (!hx || !ho) return false;+ Ds4GatherJob gj{};+ std::memcpy(&gj, g, sizeof(gj));+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ int max_src = 0;+ for (int i = 0; i < n; ++i) max_src = gj.src[i] > max_src ? gj.src[i] : max_src;+ mtl::Buffer bx = wrap(hx, (size_t) (max_src + 1) * cols * 4); // src rows may be beyond the n output rows+ mtl::Buffer bo = wrap(ho, (size_t) n * cols * 4);+ if (!bx.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_gather"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V(gj), mtl::Arg::V((unsigned) cols), mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) ((cols + 255) / 256);+ dsp.grid[1] = (uint32_t) n;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}++// ds4_kernels.cu:1342 combine_kernel - out[t] = sum_j w[t][j] ed[row[t][j]] + cpu[t] (when set) + sh[t]+bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out) {+ if (T <= 0 || dim <= 0) return true;+ if (ed == nullptr || c == nullptr || out == nullptr || T > 16) return false;+ if (!ensure()) return false;+ void* he = cpu_addr((void*) ed);+ void* hc = cpu != nullptr ? cpu_addr((void*) cpu) : nullptr;+ void* hs = sh != nullptr ? cpu_addr((void*) sh) : nullptr;+ void* ho = cpu_addr((void*) out);+ if (!he || !ho) return false;+ Ds4CombineJob cj{};+ std::memcpy(&cj, c, sizeof(cj));+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ int max_row = 0;+ for (int t = 0; t < T; ++t)+ for (int j = 0; j < cj.n[t]; ++j) max_row = cj.row[t][j] > max_row ? cj.row[t][j] : max_row;+ mtl::Buffer be = wrap(he, (size_t) (max_row + 1) * dim * 4); // rows may be beyond T * kPicks+ mtl::Buffer bc = hc != nullptr ? wrap(hc, (size_t) T * dim * 4) : g_cx->alloc(4);+ mtl::Buffer bs = hs != nullptr ? wrap(hs, (size_t) T * dim * 4) : g_cx->alloc(4);+ mtl::Buffer bo = wrap(ho, (size_t) T * dim * 4);+ if (!be.handle || !bc.handle || !bs.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_combine"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(bc, hc != nullptr ? off(hc) : 0),+ mtl::Arg::B(bs, hs != nullptr ? off(hs) : 0), mtl::Arg::V(cj), mtl::Arg::V((unsigned) dim),+ mtl::Arg::V(hc != nullptr ? 1u : 0u), mtl::Arg::V(hs != nullptr ? 1u : 0u), mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}++// On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about// (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.static inline void* cpu_addr(void* p) {@@ -734,7 +1048,8 @@ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, freturn g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);+ const size_t per = (size_t) (hd / 32) * 18; // fmt 2: IQ4_NL blocks, 18 bytes a 32 values+ mtl::Buffer br = wrap(hr, (size_t) n * per);mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);if (!br.handle || !bo.handle) return false;mtl::Pipeline pl;@@ -1155,7 +1470,17 @@ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long lreturn g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- mtl::Buffer bc = wrap(hc, (size_t) 4096 * row_bytes );+ int maxrow = -1;+ if (gen != 0 && R > 0) {+ maxrow = (int) ((p0 + t0 + T) / R) + 1;+ } else if (gen == 0 && nsel != nullptr) {+ for (int t = 0; t < T && t < nsel_count; ++t)+ for (int i = 0; i < nsel[t]; ++i) {+ const int r = sel[(size_t) t * (size_t) sel_stride + (size_t) i];+ if (r > maxrow) maxrow = r;+ }+ }+ mtl::Buffer bc = wrap(hc, (size_t) (maxrow + 1 > 0 ? maxrow + 1 : 1) * row_bytes);mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );if (!bc.handle || !bsl.handle || !bo.handle) return false;@@ -1349,8 +1674,9 @@ bool swiglu(const float* g, const float* u, int n, float lim, float* h) {if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }const size_t page = 16384;auto wrap = [&](void* host) {- const size_t bytes = (size_t) n * 4 + 65536;- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + (size_t) n * 4;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 668d3ee42..c12ea9108 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -1519,3 +1519,416 @@ kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constanif (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)}}++// ---------------------------------------------------------------- sparse attention (the prompt path)+//+// ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false>. The packed kernel stages sixteen+// rows of a tile in threadgroup memory (the window rows as they are, the compressed rows decoded to BF16) once for all+// G heads of a threadgroup, then does each head's dots, softmax and weighted sum. The BF16 kernel is the same arithmetic+// for row sets already stored as BF16 (a DSpark block); one head and query a threadgroup.+struct Ds4AttnRows { // ds4_kernels.cuh:149 AttnRows (kWin = 16)+ int win0[16], nwin[16], nsel[16];+ int gen, t0, win, R, kcap;+ long p0;+};++static inline void ds4_attn_counts(constant Ds4AttnRows& ar, int t, thread int& n_win, thread int& n_sel, thread int& w0) {+ if (ar.gen != 0) {+ const long ta = (long) ar.t0 + (long) t;+ const long pa = ar.p0 + ta;+ const long a = (long) ar.win - ar.p0, b = ta + 1;+ const long first = a > b ? a : b; // max(win - p0, ta + 1)+ w0 = int(first);+ n_win = int((long) ar.win + ta - first + 1);+ n_sel = ar.R > 0 ? int(((pa + 1) / (long) ar.R) < (long) ar.kcap ? (pa + 1) / (long) ar.R : (long) ar.kcap) : 0;+ } else {+ w0 = ar.win0[t];+ n_win = ar.nwin[t];+ n_sel = ar.nsel[t];+ }+ if (n_win < 0) n_win = 0;+ if (n_sel < 0) n_sel = 0;+}++static inline float ds4_bf16_u32(uint v) { return as_type<float>(v << 16); }++// 32 values of a packed compressed row (kKvFp4, 512 values) as BF16 in four uint4s+static inline void ds4_attn_unpack32(const device uchar* rr, int part, threadgroup uint4* dst) {+ const uint4 cw = *(const device uint4*) (rr + (size_t) part * 16u);+ const float k0 = ds4_e4m3(uint(rr[256u + uint(part) * 2u])) * 16384.0f;+ const float k1 = ds4_e4m3(uint(rr[256u + uint(part) * 2u + 1u])) * 16384.0f;+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};+ uint o[16];+ for (uint k = 0; k < 16u; ++k) {+ const uint b = wv[k >> 2] >> (8u * (k & 3u));+ const float ks = k < 8u ? k0 : k1;+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);+ }+ dst[0] = uint4(o[0], o[1], o[2], o[3]);+ dst[1] = uint4(o[4], o[5], o[6], o[7]);+ dst[2] = uint4(o[8], o[9], o[10], o[11]);+ dst[3] = uint4(o[12], o[13], o[14], o[15]);+}++// rows [s0, s0 + n) of the tile at t0 into tile4; a thread stages one row's 32 values+static inline void ds4_attn_stage(device const ushort* wbase, device const uchar* ckv, device const int* sl,+ int n_win, int t0, int s0, int n, int sr, int part, threadgroup uint4* tile4) {+ if (sr >= n) return;+ const int j = s0 + sr;+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) part * 4u;+ const int jj = t0 + j;+ if (jj < n_win) {+ const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) part * 32u);+ dst[0] = src[0];+ dst[1] = src[1];+ dst[2] = src[2];+ dst[3] = src[3];+ return;+ }+ const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;+ ds4_attn_unpack32(rr, part, dst);+}++static inline float ds4_attn_warp_max(float v) {+ v = fmax(v, simd_shuffle_xor(v, 16));+ v = fmax(v, simd_shuffle_xor(v, 8));+ v = fmax(v, simd_shuffle_xor(v, 4));+ v = fmax(v, simd_shuffle_xor(v, 2));+ v = fmax(v, simd_shuffle_xor(v, 1));+ return v;+}++template <int G>+static inline void ds4_sparse_packed_run(device const float* q, device const ushort* win, device const uchar* ckv,+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,+ device const float* sink, uint heads, float scale, device float* out,+ threadgroup uint4* tile4, threadgroup float* sc, threadgroup float* m_run_s,+ threadgroup float* tscale_s, threadgroup float* den_s, uint2 tgp, uint tid,+ ushort lane_in, ushort wp_in) {+ constexpr int W = 8 / G; // warps a head+ const int hb = int(tgp.x) * G, t = int(tgp.y);+ int n_win, n_sel, w0;+ ds4_attn_counts(ar, t, n_win, n_sel, w0);+ const device ushort* wbase = win + (size_t) w0 * 512u;+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;+ const uint lane = uint(lane_in), wp = uint(wp_in);+ const uint gw = wp / uint(W), wr = wp % uint(W);+ float qr[16];+ for (int x = 0; x < 16; ++x)+ qr[x] = q[((size_t) t * heads + uint(hb) + gw) * 512u + lane * 16u + uint(x)];+ if (tid < uint(G)) {+ m_run_s[tid] = -1e30f;+ den_s[tid] = 0.0f;+ }+ float o0[G], o1[G];+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;+ const int c = int(tid) * 2;+ const int sr = int(tid) >> 4, part = int(tid) & 15;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int total = n_win + n_sel;+ for (int t0 = 0; t0 < total; t0 += 64) {+ const int nt = min(64, total - t0);+ for (int s0 = 0; s0 < nt; s0 += 16) { // the dots+ const int n = min(16, nt - s0);+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int r = int(wr); r < n; r += W) {+ const threadgroup uint4* rp = tile4 + (size_t) r * 64u + lane * 2u;+ const uint4 a = rp[0], b = rp[1];+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};+ float s = 0.0f;+ for (int x = 0; x < 4; ++x) {+ s += qr[2 * x] * ds4_bf16_u32(av[x] & 0xffffu);+ s += qr[2 * x + 1] * ds4_bf16_u32(av[x] >> 16);+ }+ for (int x = 0; x < 4; ++x) {+ s += qr[8 + 2 * x] * ds4_bf16_u32(bv[x] & 0xffffu);+ s += qr[8 + 2 * x + 1] * ds4_bf16_u32(bv[x] >> 16);+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) sc[uint(gw) * 64u + uint(s0 + r)] = s * scale;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (wp < uint(G)) { // head wp's softmax over the tile+ threadgroup float* scg = sc + wp * 64u;+ float m = -INFINITY;+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, scg[j]);+ m = ds4_attn_warp_max(m);+ const float m_old = m_run_s[wp];+ const float m_new = fmax(m_old, m);+ float s = 0.0f;+ for (int j = int(lane); j < nt; j += 32) {+ const float p = exp(scg[j] - m_new);+ scg[j] = p;+ s += p;+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) {+ tscale_s[wp] = exp(m_old - m_new);+ den_s[wp] = den_s[wp] * tscale_s[wp] + s;+ m_run_s[wp] = m_new;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int g = 0; g < G; ++g) {+ const float cs = tscale_s[g];+ o0[g] *= cs;+ o1[g] *= cs;+ }+ const threadgroup uint* tu = (const threadgroup uint*) tile4;+ for (int s0 = 0; s0 < nt; s0 += 16) { // the weighted sums+ const int n = min(16, nt - s0);+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int r = 0; r < n; ++r) {+ const uint pr = tu[(size_t) r * 256u + uint(tid)];+ const float v0 = ds4_bf16_u32(pr & 0xffffu), v1 = ds4_bf16_u32(pr >> 16);+ for (int g = 0; g < G; ++g) {+ const float p = sc[uint(g) * 64u + uint(s0 + r)];+ o0[g] = fma(p, v0, o0[g]);+ o1[g] = fma(p, v1, o1[g]);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }+ for (int g = 0; g < G; ++g) {+ const float den = den_s[g] + exp(sink[hb + g] - m_run_s[g]);+ device float* ov = out + ((size_t) t * heads + uint(hb + g)) * 512u;+ ov[c] = o0[g] / den;+ ov[c + 1] = o1[g] / den;+ }+}++#define DS4_SPARSE_PACKED_SIG(NAME) \+ kernel void NAME(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]], \+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]], \+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]], \+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]], \+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]], \+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], \+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]])++#define DS4_SPARSE_PACKED_BODY(G) \+ { \+ threadgroup uint4 tile4[16u * 64u]; \+ threadgroup float sc[8u * 64u]; \+ threadgroup float m_run_s[8u], tscale_s[8u], den_s[8u]; \+ ds4_sparse_packed_run<G>(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, tile4, sc, m_run_s, \+ tscale_s, den_s, tgp, tid, lane, wp); \+ }++DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g8) DS4_SPARSE_PACKED_BODY(8)+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g4) DS4_SPARSE_PACKED_BODY(4)+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g2) DS4_SPARSE_PACKED_BODY(2)+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g1) DS4_SPARSE_PACKED_BODY(1)++#undef DS4_SPARSE_PACKED_SIG+#undef DS4_SPARSE_PACKED_BODY++// kContig == false (the row set is BF16 already): one head and query a threadgroup, a warp a row's dot and warp 0 the+// softmax, every thread two output values.+static inline void ds4_sparse_bf16_run(device const float* q, device const ushort* win, device const ushort* ckv,+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,+ device const float* sink, uint heads, float scale, device float* out,+ threadgroup float* qs, threadgroup float* sc, threadgroup int* rowix,+ threadgroup float* m_run_s, threadgroup float* tscale_s,+ threadgroup float* den_s, uint2 tgp, uint tid, ushort lane_in,+ ushort wp_in) {+ const int h = int(tgp.x), t = int(tgp.y);+ int n_win, n_sel, w0;+ ds4_attn_counts(ar, t, n_win, n_sel, w0);+ const device ushort* wbase = win + (size_t) w0 * 512u;+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;+ const uint lane = uint(lane_in), wp = uint(wp_in);+ for (uint i = tid; i < 512u; i += 256u) qs[i] = q[((size_t) t * heads + uint(h)) * 512u + i];+ if (tid == 0u) {+ m_run_s[0] = -1e30f;+ den_s[0] = 0.0f;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float o0 = 0.0f, o1 = 0.0f;+ const int c = int(tid) * 2;+ const int total = n_win + n_sel;+ for (int t0 = 0; t0 < total; t0 += 64) {+ const int nt = min(64, total - t0);+ if (tid < uint(nt)) {+ const int j = t0 + int(tid);+ rowix[tid] = j < n_win ? -1 - j : sl[j - n_win];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int jj = int(wp); jj < nt; jj += 8) {+ const int code = rowix[jj];+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u+ : ckv + (size_t) code * 512u;+ const device uint4* rp = (const device uint4*) (row + lane * 16u);+ const uint4 a = rp[0], b = rp[1];+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};+ float s = 0.0f;+ for (int x = 0; x < 4; ++x) {+ s += qs[lane * 16u + uint(2 * x)] * ds4_bf16_u32(av[x] & 0xffffu);+ s += qs[lane * 16u + uint(2 * x + 1)] * ds4_bf16_u32(av[x] >> 16);+ }+ for (int x = 0; x < 4; ++x) {+ s += qs[lane * 16u + uint(8 + 2 * x)] * ds4_bf16_u32(bv[x] & 0xffffu);+ s += qs[lane * 16u + uint(9 + 2 * x)] * ds4_bf16_u32(bv[x] >> 16);+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) sc[jj] = s * scale;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wp == 0u) {+ float m = -INFINITY;+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, sc[j]);+ m = ds4_attn_warp_max(m);+ const float m_old = m_run_s[0];+ const float m_new = fmax(m_old, m);+ float s = 0.0f;+ for (int j = int(lane); j < nt; j += 32) {+ const float p = exp(sc[j] - m_new);+ sc[j] = p;+ s += p;+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) {+ tscale_s[0] = exp(m_old - m_new);+ den_s[0] = den_s[0] * tscale_s[0] + s;+ m_run_s[0] = m_new;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const float cs = tscale_s[0];+ o0 *= cs;+ o1 *= cs;+ for (int j = 0; j < nt; ++j) {+ const int code = rowix[j];+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u+ : ckv + (size_t) code * 512u;+ const uint pr = *(const device uint*) (row + c);+ o0 = fma(sc[j], ds4_bf16_u32(pr & 0xffffu), o0);+ o1 = fma(sc[j], ds4_bf16_u32(pr >> 16), o1);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const float den = den_s[0] + exp(sink[h] - m_run_s[0]);+ device float* ov = out + ((size_t) t * heads + uint(h)) * 512u;+ ov[c] = o0 / den;+ ov[c + 1] = o1 / den;+}++kernel void ds4_sparse_attn_bf16(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],+ device const ushort* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {+ threadgroup float qs[512];+ threadgroup float sc[64];+ threadgroup int rowix[64];+ threadgroup float m_run_s, tscale_s, den_s;+ ds4_sparse_bf16_run(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, qs, sc, rowix, &m_run_s,+ &tscale_s, &den_s, tgp, tid, lane, wp);+}++// ds4_kernels.cu:461 route_kernel - softplus scores, their top k with bias, weights from the unbiased scores. One+// threadgroup (512) a token, as the CUDA kernel; rows 1024 values apart, the picks kPicks = 8 apart.+kernel void ds4_route(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant uint& k [[buffer(3)]], constant float& route_scale [[buffer(4)]],+ device int* idx [[buffer(5)]], device float* w [[buffer(6)]], device const float* bias_vl [[buffer(7)]],+ constant uint& has_vl [[buffer(8)]], device const uint* vlw [[buffer(9)]],+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {+ threadgroup float sc[1024], bs[1024];+ threadgroup float bv[32];+ threadgroup int bi[32];+ threadgroup int pick[16];+ const uint t = tg;+ device const float* l = logits + (size_t) t * 1024u;+ device const float* bp = bias;+ if (has_vl != 0u && ((vlw[t >> 5u] >> (t & 31u)) & 1u) != 0u) bp = bias_vl;+ for (uint i = tid; i < n; i += 512u) {+ const float v = l[i];+ const float sp = v > 20.0f ? v : log(1.0f + exp(v));+ sc[i] = sqrt(sp);+ bs[i] = sc[i] + bp[i];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint r = 0; r < k; ++r) {+ float best = -INFINITY;+ int bix = 0x7fffffff;+ for (uint i = tid; i < n; i += 512u)+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) {+ best = bs[i];+ bix = int(i);+ }+ for (uint o = 16u; o > 0u; o >>= 1u) {+ const float ov = simd_shuffle_xor(best, o);+ const int oi = simd_shuffle_xor(bix, o);+ if (ov > best || (ov == best && oi < bix)) {+ best = ov;+ bix = oi;+ }+ }+ if (lane == 0u) {+ bv[wp] = best;+ bi[wp] = bix;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) {+ float b = -INFINITY;+ int ix = 0x7fffffff;+ for (uint q = 0; q < 16u; ++q)+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) {+ b = bv[q];+ ix = bi[q];+ }+ pick[r] = ix;+ bs[ix] = -INFINITY;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (tid == 0u) {+ float s = 0.0f;+ for (uint r = 0; r < k; ++r) s += sc[pick[r]];+ for (uint r = 0; r < k; ++r) {+ idx[(size_t) t * 8u + r] = pick[r];+ w[(size_t) t * 8u + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;+ }+ }+}++// ds4_kernels.cu:1336 gather_kernel and :1342 combine_kernel - a token's gathered expert-output rows and the MoE's sum.+struct Ds4Gather {+ int src[16 * 8];+};+struct Ds4Combine {+ int n[16];+ int row[16][8];+ float w[16][8];+ int cpu[16];+};++kernel void ds4_gather(device const float* x [[buffer(0)]], constant Ds4Gather& g [[buffer(1)]],+ constant uint& cols [[buffer(2)]], device float* out [[buffer(3)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint r = tgp.y;+ const uint c = tgp.x * 256u + tid;+ if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];+}++kernel void ds4_combine(device const float* ed [[buffer(0)]], device const float* cpu [[buffer(1)]],+ device const float* sh [[buffer(2)]], constant Ds4Combine& cm [[buffer(3)]],+ constant uint& dim [[buffer(4)]], constant uint& has_cpu [[buffer(5)]],+ constant uint& has_sh [[buffer(6)]], device float* out [[buffer(7)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + tid, t = tgp.y;+ if (e >= dim) return;+ float a = 0.0f;+ for (int j = 0; j < cm.n[t]; ++j) a = fma(cm.w[t][j], ed[(size_t) cm.row[t][j] * dim + e], a);+ if (has_cpu != 0u && cm.cpu[t] != 0) a += cpu[(size_t) t * dim + e];+ if (has_sh != 0u) a += sh[(size_t) t * dim + e];+ out[(size_t) t * dim + e] = a;+}diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metalindex 7e9fb5aa5..c89e9a8e6 100644--- a/metal/src/engine/msl/ds4_iq.metal+++ b/metal/src/engine/msl/ds4_iq.metal@@ -1,7 +1,7 @@#include <metal_stdlib>using namespace metal;// ds4_iq.cu:809 mv_kernel - the I-quant expert matvec. Same frame as the MXFP4/Q8_0 ones (a warp a row, a lane strides the-// 256-value blocks, eight accumulator columns for the token window), with the block decoder chosen per job by b.type:+// 256-value blocks, sixteen accumulator columns for the token window), with the block decoder chosen per job by b.type:// 16 IQ2_XXS (66 B), 17 IQ2_XS (74 B), 22 IQ2_S (82 B). The activations are the quantiser's int8 (xq) with their scales (xd),// so each sub-group's dot is integer and the two scales fold in at the end - the same arithmetic as the float probes, at a// quarter of the traffic.@@ -26,8 +26,8 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadconst uint nblk = uint(b.k) / 256u;const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : 82u);const device uchar* w = b.w + (size_t) row * b.ld;- float acc[8];- for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;+ float acc[16];+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;for (uint blk = lane; blk < nblk; blk += 32u) {const device uchar* p = w + (size_t) blk * bs;const float d = float(as_type<half>(*(const device ushort*) p));@@ -75,7 +75,7 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadelse g = iq2s_grid[idx[l] & 1023u];const float db = (l < 2u) ? db0 : db1;const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;- for (uint c = 0; c < 8u; ++c) {+ for (uint c = 0; c < 16u; ++c) {if (int(c) >= b.ncol) break;const device char* xq = b.xq + (size_t) c * b.k + off;int sum = 0;@@ -90,7 +90,107 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[thread}}}- for (uint c = 0; c < 8u; ++c) {+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// Q5_K / Q6_K (the dense matrices' K-quants; ggml blocks of 176 / 210 bytes): the same warp a row frame, a lane a+// 256-value block, each value's code times its activation's int8. Q5_K's weight is d*sc*q - dmin*m (both parts of the+// 32-value group's scales), Q6_K's d*sc*(q - 32).+static inline void ds4_k_scale(const device uchar* q, int j, thread int& d, thread int& m) {+ if (j < 4) {+ d = int(q[j]) & 63;+ m = int(q[j + 4]) & 63;+ } else {+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);+ }+}++kernel void ds4_iq_mv_k(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nsb = uint(b.k) / 32u;+ const uint nblk = uint(b.k) / 256u;+ const uint bs = b.type == 13 ? 176u : 210u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ float acc[16];+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;+ for (uint blk = lane; blk < nblk; blk += 32u) {+ const device uchar* p = w + (size_t) blk * bs;+ if (b.type == 13) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ const device uchar* scales = p + 4;+ const device uchar* qh = p + 16;+ const device uchar* qs = p + 48;+ for (uint ib = 0; ib < 8u; ++ib) {+ int sc, mm;+ ds4_k_scale(scales, int(ib), sc, mm);+ const device uchar* qsp = qs + 32u * (ib >> 1);+ const uint sh = 4u * (ib & 1u);+ for (uint t = 0; t < 8u; ++t) {+ const uint q4 = (*(const device uint*) (qsp + 4u * t) >> sh) & 0x0f0f0f0fu;+ const uint h4 = (*(const device uint*) (qh + 4u * t) >> ib) & 0x01010101u;+ const uint q = q4 | (h4 << 4);+ const uint off = blk * 256u + ib * 32u + t * 4u;+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sumq = 0, sumx = 0;+ for (uint j = 0; j < 4u; ++j) {+ const int code = int((q >> (8u * j)) & 0xffu);+ const int xv = int(xq[j]);+ sumq += code * xv;+ sumx += xv;+ }+ const float dot = d * float(sc) * float(sumq) - dmin * float(mm) * float(sumx);+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)], dot, acc[c]);+ }+ }+ }+ } else {+ const float d = float(as_type<half>(*(const device ushort*) (p + 208)));+ const device uchar* ql = p;+ const device uchar* qh = p + 128;+ const device uchar* scl = p + 192;+ for (uint ib = 0; ib < 8u; ++ib) {+ const uint h = ib >> 2, quarter = ib & 3u;+ const device uchar* qlp = ql + 64u * h + ((quarter & 1u) ? 32u : 0u);+ const device uchar* qhp = qh + 32u * h;+ const uint sh = 4u * (quarter >> 1);+ const int s0 = int(((const device char*) scl)[8u * h + 2u * quarter]);+ const int s1 = int(((const device char*) scl)[8u * h + 2u * quarter + 1u]);+ for (uint t = 0; t < 8u; ++t) {+ const uint x = ((*(const device uint*) (qlp + 4u * t) >> sh) & 0x0f0f0f0fu) |+ (((*(const device uint*) (qhp + 4u * t) >> (2u * quarter)) & 0x03030303u) << 4);+ const uint off = blk * 256u + ib * 32u + t * 4u;+ const float s = float(t < 4u ? s0 : s1);+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sumq = 0;+ for (uint j = 0; j < 4u; ++j) {+ const int code = int((x >> (8u * j)) & 0xffu) - 32;+ sumq += code * int(xq[j]);+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * d * s, float(sumq), acc[c]);+ }+ }+ }+ }+ }+ for (uint c = 0; c < 16u; ++c) {if (int(c) >= b.ncol) break;float v = acc[c];v += simd_shuffle_xor(v, 16);diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 02fadda68..8f3a141cb 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1103,7 +1103,10 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {if (m.fmt == kFmtGG) { // the activations rounded to int8 (job i's rows of the scratch), then ds4_iq's GEMVScratch* sc = scratch(s);if (sc == nullptr || m.raw == nullptr || q.xf == nullptr || q.y == nullptr || q.ncol < 1 ||- q.ncol > x3::kMaxMvCols || m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {+#if !defined(STRATA_METAL)+ q.ncol > x3::kMaxMvCols ||+#endif+ m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (type %s k %d n %d)\n", i, type_name(m.K), m.k, m.n);g_errors.fetch_add(1, std::memory_order_relaxed);return false;diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex c89c9507c..c2a86dbb8 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -2012,6 +2012,10 @@ void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, intvoid route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,cudaStream_t s, const float* bias_vl, const VlBits* vl) {static const VlBits none = {};+#if defined(STRATA_METAL)+ // route_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::route(logits, bias, n, k, T, route_scale, idx, w, bias_vl, vl)) return;+#endifstrata_ds4_launch(route_kernel, T, 512, 0, s, logits, bias, n, k, route_scale, idx, w, vl != nullptr ? bias_vl : nullptr,vl != nullptr ? *vl : none);}@@ -2109,6 +2113,11 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intstd::fprintf(stderr, "ds4 sparse_attn: head dim %d (rows %d) not covered\n", d, ckv_fmt);return;}+#if defined(STRATA_METAL)+ // sparse_attn_packed_kernel<G> / sparse_attn_kernel<false> on the Metal dispatch; the launches below are the fallback+ if (strata::dsv41::metal_host::sparse_attn(q, win, ckv, sel, sel_stride, &a, sink, heads, d, T, scale, out, ckv_fmt))+ return;+#endifif (ckv_fmt == kKvBf16) {strata_ds4_launch(sparse_attn_kernel<false>, dim3(heads, T), kThreads, 0, s, q, win, (const uint16_t*) ckv, sel, sel_stride, a,sink, heads, d, scale, out);@@ -2238,11 +2247,19 @@ void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int}void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s) {+#if defined(STRATA_METAL)+ // gather_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::gather_rows(x, &g, n, cols, out)) return;+#endifif (n > 0) strata_ds4_launch(gather_kernel, dim3(cdiv(cols, kThreads), n), kThreads, 0, s, x, g, cols, out);}void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,cudaStream_t s) {+#if defined(STRATA_METAL)+ // combine_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::combine(ed, cpu, sh, &c, T, dim, out)) return;+#endifstrata_ds4_launch(combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, ed, cpu, sh, c, dim, out);}