Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: sparse attention + prompt matvec columns + exact wrap extents - prefill measured at 556 tok/s

Niko1221c486f17251e5
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index 8a7c247cb..8c42502c2 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -54,6 +54,7 @@ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long l
// the entry points, as metal/src/engine/ds4_finish.cpp defines them
bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);
+bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);
bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);
@@ -61,6 +62,7 @@ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd,
bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);
bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);
bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);
+bool gather_rows(const float* x, const void* g, int n, int cols, float* out);
bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy);
bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);
bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride);
@@ -83,9 +85,11 @@ bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, cons
bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);
bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);
bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
+bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w, const float* bias_vl, const void* vl);
bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out);
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows, const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt);
bool swiglu(const float* g, const float* u, int n, float lim, float* h);
bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out, int out_stride);
bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index e309b0b63..93b1be059 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -13,6 +13,8 @@
#include <cstdio>
#include <cstdint>
#include <cstring>
+#include <algorithm>
+#include <vector>
// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read
extern "C" void ds4_metal_flush();
@@ -236,8 +238,8 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,
void* hb = strata::gpu::host_view((void*) base);
if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }
const size_t nbase = (size_t) (2 * hc + hc * hc);
- mtl::Buffer bs = wrap(hs, 256);
- mtl::Buffer bb = wrap(hb, (nbase * 4 + 65535) / 65536 * 65536 );
+ mtl::Buffer bs = wrap(hs, 3 * sizeof(float));
+ mtl::Buffer bb = wrap(hb, nbase * sizeof(float));
if (!bs.handle || !bb.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_split"); } catch (...) { return false; }
@@ -271,7 +273,7 @@ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds
return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- const size_t t_bytes = (size_t) T * k * 4 + 65536;
+ const size_t t_bytes = (size_t) T * (size_t) k * 4;
mtl::Buffer bi = wrap(hi, t_bytes);
mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );
mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );
@@ -455,32 +457,82 @@ bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt) {
// ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they
// travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.
+static inline void* cpu_addr(void* p);
+namespace {
+// mirrors ds4_iq.cuh's MvJob (and ds4_iq.metal's Ds4MvJob): 80 bytes on arm64
+struct Ds4IqMvJob {
+ const uint8_t* w;
+ uint64_t ld;
+ int type, k, n, ncol;
+ const int8_t* xq;
+ const float* xd;
+ float* y;
+ uint64_t ycs;
+ const float* bias;
+ float alpha;
+};
+static_assert(sizeof(Ds4IqMvJob) == 80, "Ds4IqMvJob layout must match ds4_iq.cuh's MvJob");
+} // namespace
+
bool iq_mv(const void* jobs, int nj) {
if (nj <= 0) return true;
if (!ensure()) return false;
- // max rows across the jobs: the grid's x extent
- const unsigned char* jb = (const unsigned char*) jobs;
+ const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;
+ std::vector<Ds4IqMvJob> exp_iq, exp_k;
+ exp_iq.reserve((size_t) nj);
+ exp_k.reserve((size_t) nj);
int max_n = 0;
for (int i = 0; i < nj; ++i) {
- int n = 0;
- std::memcpy(&n, jb + (size_t) i * 80 + 24, sizeof(int)); // MvJob::n
- if (n > max_n) max_n = n;
+ const Ds4IqMvJob& q = src[i];
+ if (q.n > max_n) max_n = q.n;
+ const int ncol = q.ncol > 0 ? q.ncol : 1;
+ const int step = ncol > 16 ? 16 : ncol; // both kernels keep 16 accumulators
+ for (int c0 = 0; c0 < ncol; c0 += step) {
+ Ds4IqMvJob b = q;
+ b.ncol = (c0 + step < ncol) ? step : ncol - c0;
+ if (c0 != 0) {
+ b.xq = q.xq + (size_t) c0 * (size_t) q.k;
+ b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;
+ }
+ (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);
+ }
}
if (max_n <= 0) return true;
- mtl::Buffer bj = g_cx->alloc((size_t) nj * 80);
- if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n"); return false; }
- std::memcpy(bj.ptr, jobs, (size_t) nj * 80);
- mtl::Pipeline pl;
- try { pl = g_cx->pipeline(g_lib_iq, "ds4_iq_mv"); } catch (...) { return false; }
- mtl::Dispatch d;
- d.p = &pl;
- d.args = {mtl::Arg::B(bj)};
- d.grid[0] = (uint32_t) ((max_n + 7) / 8);
- d.grid[1] = (uint32_t) nj;
- d.grid[2] = 1;
- d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
- g_cx->run({d});
- g_cx->free(bj);
+ const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group
+ auto run_group = [&](std::vector<Ds4IqMvJob>& v, const char* fn) -> bool {
+ if (v.empty()) return true;
+ mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));
+ mtl::Pipeline pl;
+ try {
+ pl = g_cx->pipeline(g_lib_iq, fn);
+ } catch (...) {
+ g_cx->free(bj);
+ return false;
+ }
+ std::vector<mtl::Dispatch> ds;
+ for (size_t base = 0; base < v.size(); base += kBatch) {
+ const size_t cnt = std::min(kBatch, v.size() - base);
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj, base * sizeof(Ds4IqMvJob))};
+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);
+ d.grid[1] = (uint32_t) cnt;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ ds.push_back(d);
+ }
+ g_cx->run(ds);
+ g_cx->free(bj);
+ return true;
+ };
+ if (!run_group(exp_k, "ds4_iq_mv_k")) return false;
+ if (!run_group(exp_iq, "ds4_iq_mv")) return false;
return true;
}
@@ -710,6 +762,268 @@ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d,
// ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)
+namespace {
+// mirrors metal/src/engine/msl/ds4_init.metal's Ds4AttnRows (ds4_kernels.cuh:149 AttnRows, kWin = 16)
+struct Ds4AttnRowsJob {
+ int win0[16], nwin[16], nsel[16];
+ int gen, t0, win, R, kcap;
+ int64_t p0;
+};
+static_assert(sizeof(Ds4AttnRowsJob) == 224, "Ds4AttnRowsJob layout must match the MSL struct");
+} // namespace
+
+// ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false> - the prompt path's sparse attention.
+// The packed kernel stages rows once for G heads of a query; the BF16 kernel handles rows already in BF16 (a DSpark block).
+bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows,
+ const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt) {
+ if (T <= 0 || heads <= 0) return true;
+ if (d != 512 || rows == nullptr || (ckv_fmt != 0 && ckv_fmt != 2)) { std::fprintf(stderr, "dsv41 metal: sparse_attn unsupported\n"); return false; }
+ Ds4AttnRowsJob ar{};
+ std::memcpy(&ar, rows, sizeof(ar));
+ if (q == nullptr || out == nullptr) return false;
+ if (ar.gen == 0 && T > 16) { std::fprintf(stderr, "dsv41 metal: sparse_attn gen0 T>16\n"); return false; }
+ if (!ensure()) return false;
+
+ void* hq = cpu_addr((void*) q);
+ void* hw = cpu_addr((void*) win);
+ void* hc = cpu_addr((void*) ckv);
+ void* hs = cpu_addr((void*) sel);
+ void* hk = cpu_addr((void*) sink);
+ void* ho = cpu_addr((void*) out);
+ if (!hq || !ho) {
+ std::fprintf(stderr, "dsv41 metal: sparse_attn has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+
+ int wmax = 0;
+ if (ar.gen != 0) {
+ wmax = ar.win + T + 1;
+ } else {
+ for (int t = 0; t < T; ++t) wmax = ar.win0[t] + ar.nwin[t] > wmax ? ar.win0[t] + ar.nwin[t] : wmax;
+ wmax += 1;
+ }
+ if (wmax > 0 && hw == nullptr) { std::fprintf(stderr, "dsv41 metal: sparse_attn win null (wmax %d)\n", wmax); return false; }
+ const size_t wrows = (size_t) (wmax > 0 ? wmax : 1);
+ size_t crows = 0;
+ if (ckv_fmt == 2) {
+ if (ar.gen != 0 && ar.R > 0) {
+ crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position
+ } else {
+ int maxrow = -1; // the selected rows, not the cache's whole cap
+ if (hs != nullptr && sel_stride > 0)
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < ar.nsel[t]; ++i) {
+ const int r = ((const int*) hs)[(size_t) t * (size_t) sel_stride + (size_t) i];
+ if (r > maxrow) maxrow = r;
+ }
+ crows = (size_t) (maxrow + 1);
+ }
+ if (crows < 1) crows = 1;
+ } else {
+ crows = (size_t) (T > 16 ? T : 16);
+ }
+ const size_t qbytes = (size_t) T * (size_t) heads * 512 * 4;
+ const size_t wbytes = wrows * 512 * 2;
+ const size_t cbytes = crows * (ckv_fmt == 2 ? 288 : 1024);
+ const size_t obytes = (size_t) T * (size_t) heads * 512 * 4;
+ const size_t sbytes = (size_t) T * (size_t) (sel_stride > 0 ? sel_stride : 1) * 4;
+ mtl::Buffer bq = wrap(hq, qbytes);
+ mtl::Buffer bw = hw != nullptr ? wrap(hw, wbytes) : g_cx->alloc(64);
+ mtl::Buffer bc = hc != nullptr ? wrap(hc, cbytes) : g_cx->alloc(64);
+ mtl::Buffer bs = hs != nullptr ? wrap(hs, sbytes) : g_cx->alloc(4);
+ mtl::Buffer bk = hk != nullptr ? wrap(hk, (size_t) heads * 4) : g_cx->alloc((size_t) heads * 4);
+ mtl::Buffer bo = wrap(ho, obytes);
+ if (!bq.handle || !bw.handle || !bc.handle || !bs.handle || !bk.handle || !bo.handle) {
+ std::fprintf(stderr, "dsv41 metal: sparse_attn wrap failed\n");
+ return false;
+ }
+
+ int G = 8;
+ if (heads % 8 != 0) G = heads % 4 == 0 ? 4 : (heads % 2 == 0 ? 2 : 1);
+ while (G > 1 && (heads / G) * T < 64) G /= 2; // keep enough threadgroups on the GPU (CUDA's 2 * SMs, in effect)
+
+ const char* fn = ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"
+ : G == 2 ? "ds4_sparse_attn_g2" : "ds4_sparse_attn_g1")
+ : "ds4_sparse_attn_bf16";
+ mtl::Pipeline pl;
+ try {
+ pl = g_cx->pipeline(g_lib, fn);
+ } catch (...) {
+ std::fprintf(stderr, "dsv41 metal: sparse_attn pipeline %s unavailable\n", fn);
+ return false;
+ }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, off(hw != nullptr ? hw : bw.ptr)),
+ mtl::Arg::B(bc, off(hc != nullptr ? hc : bc.ptr)),
+ mtl::Arg::B(bs, off(hs != nullptr ? hs : bs.ptr)),
+ mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),
+ mtl::Arg::B(bk, off(hk != nullptr ? hk : bk.ptr)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),
+ mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256;
+ dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+
+// ds4_kernels.cu:461 route_kernel - sqrt(softplus(logits)) top k with bias, then the unbiased weights. vl is a VlBits
+// (1024 bytes) or null; bias_vl applies to the rows its bits select.
+bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
+ const float* bias_vl, const void* vl) {
+ if (T <= 0) return true;
+ if (logits == nullptr || bias == nullptr || idx == nullptr || w == nullptr || n <= 0 || n > 1024 || k <= 0 || k > 8)
+ return false;
+ if (!ensure()) return false;
+ void* hl = cpu_addr((void*) logits);
+ void* hb = cpu_addr((void*) bias);
+ void* hi = cpu_addr((void*) idx);
+ void* hw = cpu_addr((void*) w);
+ void* hv = bias_vl != nullptr ? cpu_addr((void*) bias_vl) : nullptr;
+ if (!hl || !hb || !hi || !hw) {
+ std::fprintf(stderr, "dsv41 metal: route has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4);
+ mtl::Buffer bb = wrap(hb, (size_t) n * 4);
+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4);
+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4);
+ mtl::Buffer bv = hv != nullptr ? wrap(hv, (size_t) n * 4) : g_cx->alloc(4);
+ mtl::Buffer bbv = g_cx->alloc(1024);
+ if (!bl.handle || !bb.handle || !bi.handle || !bw.handle || !bv.handle || !bbv.ptr) return false;
+ const bool has_vl = vl != nullptr;
+ if (has_vl) std::memcpy(bbv.ptr, vl, 1024);
+ mtl::Pipeline pl;
+ try {
+ pl = g_cx->pipeline(g_lib, "ds4_route");
+ } catch (...) {
+ std::fprintf(stderr, "dsv41 metal: route pipeline unavailable\n");
+ return false;
+ }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),
+ mtl::Arg::V(route_scale), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::B(bv, hv != nullptr ? off(hv) : 0), mtl::Arg::V(has_vl ? 1u : 0u), mtl::Arg::B(bbv)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 512;
+ dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(bbv);
+ return true;
+}
+
+
+namespace {
+struct Ds4GatherJob {
+ int src[16 * 8];
+};
+struct Ds4CombineJob {
+ int n[16];
+ int row[16][8];
+ float w[16][8];
+ int cpu[16];
+};
+static_assert(sizeof(Ds4GatherJob) == 512, "Ds4GatherJob layout must match the MSL struct");
+static_assert(sizeof(Ds4CombineJob) == 1152, "Ds4CombineJob layout must match the MSL struct");
+} // namespace
+
+// ds4_kernels.cu:1336 gather_kernel - out[r] = x[g.src[r]] for n rows of cols floats
+bool gather_rows(const float* x, const void* g, int n, int cols, float* out) {
+ if (n <= 0 || cols <= 0) return true;
+ if (x == nullptr || g == nullptr || out == nullptr || n > 16 * 8) return false;
+ if (!ensure()) return false;
+ void* hx = cpu_addr((void*) x);
+ void* ho = cpu_addr((void*) out);
+ if (!hx || !ho) return false;
+ Ds4GatherJob gj{};
+ std::memcpy(&gj, g, sizeof(gj));
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ int max_src = 0;
+ for (int i = 0; i < n; ++i) max_src = gj.src[i] > max_src ? gj.src[i] : max_src;
+ mtl::Buffer bx = wrap(hx, (size_t) (max_src + 1) * cols * 4); // src rows may be beyond the n output rows
+ mtl::Buffer bo = wrap(ho, (size_t) n * cols * 4);
+ if (!bx.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_gather"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V(gj), mtl::Arg::V((unsigned) cols), mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) ((cols + 255) / 256);
+ dsp.grid[1] = (uint32_t) n;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+// ds4_kernels.cu:1342 combine_kernel - out[t] = sum_j w[t][j] ed[row[t][j]] + cpu[t] (when set) + sh[t]
+bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out) {
+ if (T <= 0 || dim <= 0) return true;
+ if (ed == nullptr || c == nullptr || out == nullptr || T > 16) return false;
+ if (!ensure()) return false;
+ void* he = cpu_addr((void*) ed);
+ void* hc = cpu != nullptr ? cpu_addr((void*) cpu) : nullptr;
+ void* hs = sh != nullptr ? cpu_addr((void*) sh) : nullptr;
+ void* ho = cpu_addr((void*) out);
+ if (!he || !ho) return false;
+ Ds4CombineJob cj{};
+ std::memcpy(&cj, c, sizeof(cj));
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ int max_row = 0;
+ for (int t = 0; t < T; ++t)
+ for (int j = 0; j < cj.n[t]; ++j) max_row = cj.row[t][j] > max_row ? cj.row[t][j] : max_row;
+ mtl::Buffer be = wrap(he, (size_t) (max_row + 1) * dim * 4); // rows may be beyond T * kPicks
+ mtl::Buffer bc = hc != nullptr ? wrap(hc, (size_t) T * dim * 4) : g_cx->alloc(4);
+ mtl::Buffer bs = hs != nullptr ? wrap(hs, (size_t) T * dim * 4) : g_cx->alloc(4);
+ mtl::Buffer bo = wrap(ho, (size_t) T * dim * 4);
+ if (!be.handle || !bc.handle || !bs.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_combine"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(bc, hc != nullptr ? off(hc) : 0),
+ mtl::Arg::B(bs, hs != nullptr ? off(hs) : 0), mtl::Arg::V(cj), mtl::Arg::V((unsigned) dim),
+ mtl::Arg::V(hc != nullptr ? 1u : 0u), mtl::Arg::V(hs != nullptr ? 1u : 0u), mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+
// On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about
// (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.
static inline void* cpu_addr(void* p) {
@@ -734,7 +1048,8 @@ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, f
return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);
+ const size_t per = (size_t) (hd / 32) * 18; // fmt 2: IQ4_NL blocks, 18 bytes a 32 values
+ mtl::Buffer br = wrap(hr, (size_t) n * per);
mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);
if (!br.handle || !bo.handle) return false;
mtl::Pipeline pl;
@@ -1155,7 +1470,17 @@ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long l
return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- mtl::Buffer bc = wrap(hc, (size_t) 4096 * row_bytes );
+ int maxrow = -1;
+ if (gen != 0 && R > 0) {
+ maxrow = (int) ((p0 + t0 + T) / R) + 1;
+ } else if (gen == 0 && nsel != nullptr) {
+ for (int t = 0; t < T && t < nsel_count; ++t)
+ for (int i = 0; i < nsel[t]; ++i) {
+ const int r = sel[(size_t) t * (size_t) sel_stride + (size_t) i];
+ if (r > maxrow) maxrow = r;
+ }
+ }
+ mtl::Buffer bc = wrap(hc, (size_t) (maxrow + 1 > 0 ? maxrow + 1 : 1) * row_bytes);
mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );
mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );
if (!bc.handle || !bsl.handle || !bo.handle) return false;
@@ -1349,8 +1674,9 @@ bool swiglu(const float* g, const float* u, int n, float lim, float* h) {
if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host) {
- const size_t bytes = (size_t) n * 4 + 65536;
- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + (size_t) n * 4;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index 668d3ee42..c12ea9108 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -1519,3 +1519,416 @@ kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constan
if (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)
}
}
+
+// ---------------------------------------------------------------- sparse attention (the prompt path)
+//
+// ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false>. The packed kernel stages sixteen
+// rows of a tile in threadgroup memory (the window rows as they are, the compressed rows decoded to BF16) once for all
+// G heads of a threadgroup, then does each head's dots, softmax and weighted sum. The BF16 kernel is the same arithmetic
+// for row sets already stored as BF16 (a DSpark block); one head and query a threadgroup.
+struct Ds4AttnRows { // ds4_kernels.cuh:149 AttnRows (kWin = 16)
+ int win0[16], nwin[16], nsel[16];
+ int gen, t0, win, R, kcap;
+ long p0;
+};
+
+static inline void ds4_attn_counts(constant Ds4AttnRows& ar, int t, thread int& n_win, thread int& n_sel, thread int& w0) {
+ if (ar.gen != 0) {
+ const long ta = (long) ar.t0 + (long) t;
+ const long pa = ar.p0 + ta;
+ const long a = (long) ar.win - ar.p0, b = ta + 1;
+ const long first = a > b ? a : b; // max(win - p0, ta + 1)
+ w0 = int(first);
+ n_win = int((long) ar.win + ta - first + 1);
+ n_sel = ar.R > 0 ? int(((pa + 1) / (long) ar.R) < (long) ar.kcap ? (pa + 1) / (long) ar.R : (long) ar.kcap) : 0;
+ } else {
+ w0 = ar.win0[t];
+ n_win = ar.nwin[t];
+ n_sel = ar.nsel[t];
+ }
+ if (n_win < 0) n_win = 0;
+ if (n_sel < 0) n_sel = 0;
+}
+
+static inline float ds4_bf16_u32(uint v) { return as_type<float>(v << 16); }
+
+// 32 values of a packed compressed row (kKvFp4, 512 values) as BF16 in four uint4s
+static inline void ds4_attn_unpack32(const device uchar* rr, int part, threadgroup uint4* dst) {
+ const uint4 cw = *(const device uint4*) (rr + (size_t) part * 16u);
+ const float k0 = ds4_e4m3(uint(rr[256u + uint(part) * 2u])) * 16384.0f;
+ const float k1 = ds4_e4m3(uint(rr[256u + uint(part) * 2u + 1u])) * 16384.0f;
+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};
+ uint o[16];
+ for (uint k = 0; k < 16u; ++k) {
+ const uint b = wv[k >> 2] >> (8u * (k & 3u));
+ const float ks = k < 8u ? k0 : k1;
+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
+ }
+ dst[0] = uint4(o[0], o[1], o[2], o[3]);
+ dst[1] = uint4(o[4], o[5], o[6], o[7]);
+ dst[2] = uint4(o[8], o[9], o[10], o[11]);
+ dst[3] = uint4(o[12], o[13], o[14], o[15]);
+}
+
+// rows [s0, s0 + n) of the tile at t0 into tile4; a thread stages one row's 32 values
+static inline void ds4_attn_stage(device const ushort* wbase, device const uchar* ckv, device const int* sl,
+ int n_win, int t0, int s0, int n, int sr, int part, threadgroup uint4* tile4) {
+ if (sr >= n) return;
+ const int j = s0 + sr;
+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) part * 4u;
+ const int jj = t0 + j;
+ if (jj < n_win) {
+ const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) part * 32u);
+ dst[0] = src[0];
+ dst[1] = src[1];
+ dst[2] = src[2];
+ dst[3] = src[3];
+ return;
+ }
+ const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;
+ ds4_attn_unpack32(rr, part, dst);
+}
+
+static inline float ds4_attn_warp_max(float v) {
+ v = fmax(v, simd_shuffle_xor(v, 16));
+ v = fmax(v, simd_shuffle_xor(v, 8));
+ v = fmax(v, simd_shuffle_xor(v, 4));
+ v = fmax(v, simd_shuffle_xor(v, 2));
+ v = fmax(v, simd_shuffle_xor(v, 1));
+ return v;
+}
+
+template <int G>
+static inline void ds4_sparse_packed_run(device const float* q, device const ushort* win, device const uchar* ckv,
+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,
+ device const float* sink, uint heads, float scale, device float* out,
+ threadgroup uint4* tile4, threadgroup float* sc, threadgroup float* m_run_s,
+ threadgroup float* tscale_s, threadgroup float* den_s, uint2 tgp, uint tid,
+ ushort lane_in, ushort wp_in) {
+ constexpr int W = 8 / G; // warps a head
+ const int hb = int(tgp.x) * G, t = int(tgp.y);
+ int n_win, n_sel, w0;
+ ds4_attn_counts(ar, t, n_win, n_sel, w0);
+ const device ushort* wbase = win + (size_t) w0 * 512u;
+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;
+ const uint lane = uint(lane_in), wp = uint(wp_in);
+ const uint gw = wp / uint(W), wr = wp % uint(W);
+ float qr[16];
+ for (int x = 0; x < 16; ++x)
+ qr[x] = q[((size_t) t * heads + uint(hb) + gw) * 512u + lane * 16u + uint(x)];
+ if (tid < uint(G)) {
+ m_run_s[tid] = -1e30f;
+ den_s[tid] = 0.0f;
+ }
+ float o0[G], o1[G];
+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;
+ const int c = int(tid) * 2;
+ const int sr = int(tid) >> 4, part = int(tid) & 15;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += 64) {
+ const int nt = min(64, total - t0);
+ for (int s0 = 0; s0 < nt; s0 += 16) { // the dots
+ const int n = min(16, nt - s0);
+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int r = int(wr); r < n; r += W) {
+ const threadgroup uint4* rp = tile4 + (size_t) r * 64u + lane * 2u;
+ const uint4 a = rp[0], b = rp[1];
+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};
+ float s = 0.0f;
+ for (int x = 0; x < 4; ++x) {
+ s += qr[2 * x] * ds4_bf16_u32(av[x] & 0xffffu);
+ s += qr[2 * x + 1] * ds4_bf16_u32(av[x] >> 16);
+ }
+ for (int x = 0; x < 4; ++x) {
+ s += qr[8 + 2 * x] * ds4_bf16_u32(bv[x] & 0xffffu);
+ s += qr[8 + 2 * x + 1] * ds4_bf16_u32(bv[x] >> 16);
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) sc[uint(gw) * 64u + uint(s0 + r)] = s * scale;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (wp < uint(G)) { // head wp's softmax over the tile
+ threadgroup float* scg = sc + wp * 64u;
+ float m = -INFINITY;
+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, scg[j]);
+ m = ds4_attn_warp_max(m);
+ const float m_old = m_run_s[wp];
+ const float m_new = fmax(m_old, m);
+ float s = 0.0f;
+ for (int j = int(lane); j < nt; j += 32) {
+ const float p = exp(scg[j] - m_new);
+ scg[j] = p;
+ s += p;
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) {
+ tscale_s[wp] = exp(m_old - m_new);
+ den_s[wp] = den_s[wp] * tscale_s[wp] + s;
+ m_run_s[wp] = m_new;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int g = 0; g < G; ++g) {
+ const float cs = tscale_s[g];
+ o0[g] *= cs;
+ o1[g] *= cs;
+ }
+ const threadgroup uint* tu = (const threadgroup uint*) tile4;
+ for (int s0 = 0; s0 < nt; s0 += 16) { // the weighted sums
+ const int n = min(16, nt - s0);
+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int r = 0; r < n; ++r) {
+ const uint pr = tu[(size_t) r * 256u + uint(tid)];
+ const float v0 = ds4_bf16_u32(pr & 0xffffu), v1 = ds4_bf16_u32(pr >> 16);
+ for (int g = 0; g < G; ++g) {
+ const float p = sc[uint(g) * 64u + uint(s0 + r)];
+ o0[g] = fma(p, v0, o0[g]);
+ o1[g] = fma(p, v1, o1[g]);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+ for (int g = 0; g < G; ++g) {
+ const float den = den_s[g] + exp(sink[hb + g] - m_run_s[g]);
+ device float* ov = out + ((size_t) t * heads + uint(hb + g)) * 512u;
+ ov[c] = o0[g] / den;
+ ov[c + 1] = o1[g] / den;
+ }
+}
+
+#define DS4_SPARSE_PACKED_SIG(NAME) \
+ kernel void NAME(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]], \
+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]], \
+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]], \
+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]], \
+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]], \
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], \
+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]])
+
+#define DS4_SPARSE_PACKED_BODY(G) \
+ { \
+ threadgroup uint4 tile4[16u * 64u]; \
+ threadgroup float sc[8u * 64u]; \
+ threadgroup float m_run_s[8u], tscale_s[8u], den_s[8u]; \
+ ds4_sparse_packed_run<G>(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, tile4, sc, m_run_s, \
+ tscale_s, den_s, tgp, tid, lane, wp); \
+ }
+
+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g8) DS4_SPARSE_PACKED_BODY(8)
+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g4) DS4_SPARSE_PACKED_BODY(4)
+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g2) DS4_SPARSE_PACKED_BODY(2)
+DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g1) DS4_SPARSE_PACKED_BODY(1)
+
+#undef DS4_SPARSE_PACKED_SIG
+#undef DS4_SPARSE_PACKED_BODY
+
+// kContig == false (the row set is BF16 already): one head and query a threadgroup, a warp a row's dot and warp 0 the
+// softmax, every thread two output values.
+static inline void ds4_sparse_bf16_run(device const float* q, device const ushort* win, device const ushort* ckv,
+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,
+ device const float* sink, uint heads, float scale, device float* out,
+ threadgroup float* qs, threadgroup float* sc, threadgroup int* rowix,
+ threadgroup float* m_run_s, threadgroup float* tscale_s,
+ threadgroup float* den_s, uint2 tgp, uint tid, ushort lane_in,
+ ushort wp_in) {
+ const int h = int(tgp.x), t = int(tgp.y);
+ int n_win, n_sel, w0;
+ ds4_attn_counts(ar, t, n_win, n_sel, w0);
+ const device ushort* wbase = win + (size_t) w0 * 512u;
+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;
+ const uint lane = uint(lane_in), wp = uint(wp_in);
+ for (uint i = tid; i < 512u; i += 256u) qs[i] = q[((size_t) t * heads + uint(h)) * 512u + i];
+ if (tid == 0u) {
+ m_run_s[0] = -1e30f;
+ den_s[0] = 0.0f;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float o0 = 0.0f, o1 = 0.0f;
+ const int c = int(tid) * 2;
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += 64) {
+ const int nt = min(64, total - t0);
+ if (tid < uint(nt)) {
+ const int j = t0 + int(tid);
+ rowix[tid] = j < n_win ? -1 - j : sl[j - n_win];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int jj = int(wp); jj < nt; jj += 8) {
+ const int code = rowix[jj];
+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u
+ : ckv + (size_t) code * 512u;
+ const device uint4* rp = (const device uint4*) (row + lane * 16u);
+ const uint4 a = rp[0], b = rp[1];
+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};
+ float s = 0.0f;
+ for (int x = 0; x < 4; ++x) {
+ s += qs[lane * 16u + uint(2 * x)] * ds4_bf16_u32(av[x] & 0xffffu);
+ s += qs[lane * 16u + uint(2 * x + 1)] * ds4_bf16_u32(av[x] >> 16);
+ }
+ for (int x = 0; x < 4; ++x) {
+ s += qs[lane * 16u + uint(8 + 2 * x)] * ds4_bf16_u32(bv[x] & 0xffffu);
+ s += qs[lane * 16u + uint(9 + 2 * x)] * ds4_bf16_u32(bv[x] >> 16);
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) sc[jj] = s * scale;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wp == 0u) {
+ float m = -INFINITY;
+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, sc[j]);
+ m = ds4_attn_warp_max(m);
+ const float m_old = m_run_s[0];
+ const float m_new = fmax(m_old, m);
+ float s = 0.0f;
+ for (int j = int(lane); j < nt; j += 32) {
+ const float p = exp(sc[j] - m_new);
+ sc[j] = p;
+ s += p;
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) {
+ tscale_s[0] = exp(m_old - m_new);
+ den_s[0] = den_s[0] * tscale_s[0] + s;
+ m_run_s[0] = m_new;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const float cs = tscale_s[0];
+ o0 *= cs;
+ o1 *= cs;
+ for (int j = 0; j < nt; ++j) {
+ const int code = rowix[j];
+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u
+ : ckv + (size_t) code * 512u;
+ const uint pr = *(const device uint*) (row + c);
+ o0 = fma(sc[j], ds4_bf16_u32(pr & 0xffffu), o0);
+ o1 = fma(sc[j], ds4_bf16_u32(pr >> 16), o1);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const float den = den_s[0] + exp(sink[h] - m_run_s[0]);
+ device float* ov = out + ((size_t) t * heads + uint(h)) * 512u;
+ ov[c] = o0 / den;
+ ov[c + 1] = o1 / den;
+}
+
+kernel void ds4_sparse_attn_bf16(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
+ device const ushort* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float qs[512];
+ threadgroup float sc[64];
+ threadgroup int rowix[64];
+ threadgroup float m_run_s, tscale_s, den_s;
+ ds4_sparse_bf16_run(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, qs, sc, rowix, &m_run_s,
+ &tscale_s, &den_s, tgp, tid, lane, wp);
+}
+
+// ds4_kernels.cu:461 route_kernel - softplus scores, their top k with bias, weights from the unbiased scores. One
+// threadgroup (512) a token, as the CUDA kernel; rows 1024 values apart, the picks kPicks = 8 apart.
+kernel void ds4_route(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant uint& k [[buffer(3)]], constant float& route_scale [[buffer(4)]],
+ device int* idx [[buffer(5)]], device float* w [[buffer(6)]], device const float* bias_vl [[buffer(7)]],
+ constant uint& has_vl [[buffer(8)]], device const uint* vlw [[buffer(9)]],
+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float sc[1024], bs[1024];
+ threadgroup float bv[32];
+ threadgroup int bi[32];
+ threadgroup int pick[16];
+ const uint t = tg;
+ device const float* l = logits + (size_t) t * 1024u;
+ device const float* bp = bias;
+ if (has_vl != 0u && ((vlw[t >> 5u] >> (t & 31u)) & 1u) != 0u) bp = bias_vl;
+ for (uint i = tid; i < n; i += 512u) {
+ const float v = l[i];
+ const float sp = v > 20.0f ? v : log(1.0f + exp(v));
+ sc[i] = sqrt(sp);
+ bs[i] = sc[i] + bp[i];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint r = 0; r < k; ++r) {
+ float best = -INFINITY;
+ int bix = 0x7fffffff;
+ for (uint i = tid; i < n; i += 512u)
+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) {
+ best = bs[i];
+ bix = int(i);
+ }
+ for (uint o = 16u; o > 0u; o >>= 1u) {
+ const float ov = simd_shuffle_xor(best, o);
+ const int oi = simd_shuffle_xor(bix, o);
+ if (ov > best || (ov == best && oi < bix)) {
+ best = ov;
+ bix = oi;
+ }
+ }
+ if (lane == 0u) {
+ bv[wp] = best;
+ bi[wp] = bix;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) {
+ float b = -INFINITY;
+ int ix = 0x7fffffff;
+ for (uint q = 0; q < 16u; ++q)
+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) {
+ b = bv[q];
+ ix = bi[q];
+ }
+ pick[r] = ix;
+ bs[ix] = -INFINITY;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (tid == 0u) {
+ float s = 0.0f;
+ for (uint r = 0; r < k; ++r) s += sc[pick[r]];
+ for (uint r = 0; r < k; ++r) {
+ idx[(size_t) t * 8u + r] = pick[r];
+ w[(size_t) t * 8u + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;
+ }
+ }
+}
+
+// ds4_kernels.cu:1336 gather_kernel and :1342 combine_kernel - a token's gathered expert-output rows and the MoE's sum.
+struct Ds4Gather {
+ int src[16 * 8];
+};
+struct Ds4Combine {
+ int n[16];
+ int row[16][8];
+ float w[16][8];
+ int cpu[16];
+};
+
+kernel void ds4_gather(device const float* x [[buffer(0)]], constant Ds4Gather& g [[buffer(1)]],
+ constant uint& cols [[buffer(2)]], device float* out [[buffer(3)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint r = tgp.y;
+ const uint c = tgp.x * 256u + tid;
+ if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];
+}
+
+kernel void ds4_combine(device const float* ed [[buffer(0)]], device const float* cpu [[buffer(1)]],
+ device const float* sh [[buffer(2)]], constant Ds4Combine& cm [[buffer(3)]],
+ constant uint& dim [[buffer(4)]], constant uint& has_cpu [[buffer(5)]],
+ constant uint& has_sh [[buffer(6)]], device float* out [[buffer(7)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + tid, t = tgp.y;
+ if (e >= dim) return;
+ float a = 0.0f;
+ for (int j = 0; j < cm.n[t]; ++j) a = fma(cm.w[t][j], ed[(size_t) cm.row[t][j] * dim + e], a);
+ if (has_cpu != 0u && cm.cpu[t] != 0) a += cpu[(size_t) t * dim + e];
+ if (has_sh != 0u) a += sh[(size_t) t * dim + e];
+ out[(size_t) t * dim + e] = a;
+}
diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metal
index 7e9fb5aa5..c89e9a8e6 100644
--- a/metal/src/engine/msl/ds4_iq.metal
+++ b/metal/src/engine/msl/ds4_iq.metal
@@ -1,7 +1,7 @@
#include <metal_stdlib>
using namespace metal;
// ds4_iq.cu:809 mv_kernel - the I-quant expert matvec. Same frame as the MXFP4/Q8_0 ones (a warp a row, a lane strides the
-// 256-value blocks, eight accumulator columns for the token window), with the block decoder chosen per job by b.type:
+// 256-value blocks, sixteen accumulator columns for the token window), with the block decoder chosen per job by b.type:
// 16 IQ2_XXS (66 B), 17 IQ2_XS (74 B), 22 IQ2_S (82 B). The activations are the quantiser's int8 (xq) with their scales (xd),
// so each sub-group's dot is integer and the two scales fold in at the end - the same arithmetic as the float probes, at a
// quarter of the traffic.
@@ -26,8 +26,8 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[thread
const uint nblk = uint(b.k) / 256u;
const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : 82u);
const device uchar* w = b.w + (size_t) row * b.ld;
- float acc[8];
- for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
+ float acc[16];
+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
for (uint blk = lane; blk < nblk; blk += 32u) {
const device uchar* p = w + (size_t) blk * bs;
const float d = float(as_type<half>(*(const device ushort*) p));
@@ -75,7 +75,7 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[thread
else g = iq2s_grid[idx[l] & 1023u];
const float db = (l < 2u) ? db0 : db1;
const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;
- for (uint c = 0; c < 8u; ++c) {
+ for (uint c = 0; c < 16u; ++c) {
if (int(c) >= b.ncol) break;
const device char* xq = b.xq + (size_t) c * b.k + off;
int sum = 0;
@@ -90,7 +90,107 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[thread
}
}
}
- for (uint c = 0; c < 8u; ++c) {
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// Q5_K / Q6_K (the dense matrices' K-quants; ggml blocks of 176 / 210 bytes): the same warp a row frame, a lane a
+// 256-value block, each value's code times its activation's int8. Q5_K's weight is d*sc*q - dmin*m (both parts of the
+// 32-value group's scales), Q6_K's d*sc*(q - 32).
+static inline void ds4_k_scale(const device uchar* q, int j, thread int& d, thread int& m) {
+ if (j < 4) {
+ d = int(q[j]) & 63;
+ m = int(q[j + 4]) & 63;
+ } else {
+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);
+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);
+ }
+}
+
+kernel void ds4_iq_mv_k(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nsb = uint(b.k) / 32u;
+ const uint nblk = uint(b.k) / 256u;
+ const uint bs = b.type == 13 ? 176u : 210u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ float acc[16];
+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
+ for (uint blk = lane; blk < nblk; blk += 32u) {
+ const device uchar* p = w + (size_t) blk * bs;
+ if (b.type == 13) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ const device uchar* scales = p + 4;
+ const device uchar* qh = p + 16;
+ const device uchar* qs = p + 48;
+ for (uint ib = 0; ib < 8u; ++ib) {
+ int sc, mm;
+ ds4_k_scale(scales, int(ib), sc, mm);
+ const device uchar* qsp = qs + 32u * (ib >> 1);
+ const uint sh = 4u * (ib & 1u);
+ for (uint t = 0; t < 8u; ++t) {
+ const uint q4 = (*(const device uint*) (qsp + 4u * t) >> sh) & 0x0f0f0f0fu;
+ const uint h4 = (*(const device uint*) (qh + 4u * t) >> ib) & 0x01010101u;
+ const uint q = q4 | (h4 << 4);
+ const uint off = blk * 256u + ib * 32u + t * 4u;
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sumq = 0, sumx = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ const int code = int((q >> (8u * j)) & 0xffu);
+ const int xv = int(xq[j]);
+ sumq += code * xv;
+ sumx += xv;
+ }
+ const float dot = d * float(sc) * float(sumq) - dmin * float(mm) * float(sumx);
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)], dot, acc[c]);
+ }
+ }
+ }
+ } else {
+ const float d = float(as_type<half>(*(const device ushort*) (p + 208)));
+ const device uchar* ql = p;
+ const device uchar* qh = p + 128;
+ const device uchar* scl = p + 192;
+ for (uint ib = 0; ib < 8u; ++ib) {
+ const uint h = ib >> 2, quarter = ib & 3u;
+ const device uchar* qlp = ql + 64u * h + ((quarter & 1u) ? 32u : 0u);
+ const device uchar* qhp = qh + 32u * h;
+ const uint sh = 4u * (quarter >> 1);
+ const int s0 = int(((const device char*) scl)[8u * h + 2u * quarter]);
+ const int s1 = int(((const device char*) scl)[8u * h + 2u * quarter + 1u]);
+ for (uint t = 0; t < 8u; ++t) {
+ const uint x = ((*(const device uint*) (qlp + 4u * t) >> sh) & 0x0f0f0f0fu) |
+ (((*(const device uint*) (qhp + 4u * t) >> (2u * quarter)) & 0x03030303u) << 4);
+ const uint off = blk * 256u + ib * 32u + t * 4u;
+ const float s = float(t < 4u ? s0 : s1);
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sumq = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ const int code = int((x >> (8u * j)) & 0xffu) - 32;
+ sumq += code * int(xq[j]);
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * d * s, float(sumq), acc[c]);
+ }
+ }
+ }
+ }
+ }
+ for (uint c = 0; c < 16u; ++c) {
if (int(c) >= b.ncol) break;
float v = acc[c];
v += simd_shuffle_xor(v, 16);
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 02fadda68..8f3a141cb 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1103,7 +1103,10 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
if (m.fmt == kFmtGG) { // the activations rounded to int8 (job i's rows of the scratch), then ds4_iq's GEMV
Scratch* sc = scratch(s);
if (sc == nullptr || m.raw == nullptr || q.xf == nullptr || q.y == nullptr || q.ncol < 1 ||
- q.ncol > x3::kMaxMvCols || m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {
+#if !defined(STRATA_METAL)
+ q.ncol > x3::kMaxMvCols ||
+#endif
+ m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {
std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (type %s k %d n %d)\n", i, type_name(m.K), m.k, m.n);
g_errors.fetch_add(1, std::memory_order_relaxed);
return false;
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index c89c9507c..c2a86dbb8 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -2012,6 +2012,10 @@ void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int
void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
cudaStream_t s, const float* bias_vl, const VlBits* vl) {
static const VlBits none = {};
+#if defined(STRATA_METAL)
+ // route_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::route(logits, bias, n, k, T, route_scale, idx, w, bias_vl, vl)) return;
+#endif
strata_ds4_launch(route_kernel, T, 512, 0, s, logits, bias, n, k, route_scale, idx, w, vl != nullptr ? bias_vl : nullptr,
vl != nullptr ? *vl : none);
}
@@ -2109,6 +2113,11 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
std::fprintf(stderr, "ds4 sparse_attn: head dim %d (rows %d) not covered\n", d, ckv_fmt);
return;
}
+#if defined(STRATA_METAL)
+ // sparse_attn_packed_kernel<G> / sparse_attn_kernel<false> on the Metal dispatch; the launches below are the fallback
+ if (strata::dsv41::metal_host::sparse_attn(q, win, ckv, sel, sel_stride, &a, sink, heads, d, T, scale, out, ckv_fmt))
+ return;
+#endif
if (ckv_fmt == kKvBf16) {
strata_ds4_launch(sparse_attn_kernel<false>, dim3(heads, T), kThreads, 0, s, q, win, (const uint16_t*) ckv, sel, sel_stride, a,
sink, heads, d, scale, out);
@@ -2238,11 +2247,19 @@ void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int
}
void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // gather_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::gather_rows(x, &g, n, cols, out)) return;
+#endif
if (n > 0) strata_ds4_launch(gather_kernel, dim3(cdiv(cols, kThreads), n), kThreads, 0, s, x, g, cols, out);
}
void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,
cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // combine_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::combine(ed, cpu, sh, &c, T, dim, out)) return;
+#endif
strata_ds4_launch(combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, ed, cpu, sh, c, dim, out);
}