Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: THE ENGINE GENERATES TOKENS ON THE MAC - 17 more kernels, the whole path runs

Niko122133bf441cd4dc
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index d26707802..8a7c247cb 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -1,8 +1,7 @@
-// metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch, as the engine calls them.
+// metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch.
//
-// A launch site in the shared dsv41 sources becomes, under STRATA_METAL, a call to one of these instead of the generic
-// strata_ds4_launch() stub. Each implementation wraps the engine's own device buffers into the Metal runtime and dispatches
-// the MSL kernel of the same name; metal/src/engine/ds4_host.cpp holds them.
+// GENERATED from metal/src/engine/ds4_host.cpp and ds4_finish.cpp, which are the source of truth: do not hand-edit the
+// declaration block, regenerate it (see RIGS/m1002.md).
#pragma once
#include <cstddef>
@@ -16,11 +15,7 @@ using Stream = StreamImpl*;
namespace strata::dsv41::metal_host {
-// ds4_kernels.cu:1477 init_pre_kernel: pre[i] = (i % 8) == 0 ? 1 : 0, i < T*8
-bool init_pre(float* pre, int T, void* stream);
-// ds4_kernels.cu:130 embed_kernel: h[(t*hc + s)*dim + e] = bf16(rows[t*dim + e])
-// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, ...): C[t][m] = sum_k A[m][k] * B[t][k], F32, alpha 1, beta 0
-// ds4_kernels.cu:1894 finish_kernel<false>, the engine's Finish as ds4_kernels.cuh declares it
+// ds4_kernels.cu:1894 finish_kernel<false> - the engine's Finish, as ds4_kernels.cuh declares it
struct FinishIn {
const float* w;
float eps;
@@ -32,26 +27,67 @@ struct FinishIn {
uint8_t* pdst;
long long pdst_stride;
};
-// ds4_kernels.cu:1938 moe_sort: counts, prefix, fill
-bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
-
-// ds4_kernels.cu:1908 hc_split (the sinkhorn normalisation)
-bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
- float* pre, float* post, float* comb);
-// ds4_kernels.cu:1901 hc_mix
+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
+bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);
+bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);
+bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);
+bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);
bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);
-
-// ds4_kernels.cu:1692 hc_mix_scale
bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);
-
-// ds4_kernels.cu:1677 hc_post_inplace
bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);
+bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);
+bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);
+bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);
+bool init_pre(float* pre, int T, void*);
+bool iq_mv(const void* jobs, int nj);
+bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);
+bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
+bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);
+bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);
+bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);
+bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
+bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
+bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+bool swiglu(const float* g, const float* u, int n, float lim, float* h);
+bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
+// the entry points, as metal/src/engine/ds4_finish.cpp defines them
+bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);
+bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);
+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
+bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);
+bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);
+bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);
+bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);
bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);
-
+bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy);
+bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);
+bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride);
+bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);
+bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);
+bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);
+bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);
+bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);
+bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);
+bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand, int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt);
+bool init_pre(float* pre, int T, void*);
+bool iota_rows(int* out, int T, int n, int stride);
+bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int type, int groups);
+bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int type, int groups);
+bool iq_mv(const void* jobs, int nj);
+bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);
+bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
+bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);
+bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride, const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd);
+bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);
+bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);
+bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
+bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
+bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out);
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+bool swiglu(const float* g, const float* u, int n, float lim, float* h);
+bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out, int out_stride);
+bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
-bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void* stream);
-
-} // namespace strata::dsv41::metal_host
+} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index e0a01b7a0..8dabb42cd 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -32,12 +32,15 @@ struct GpuFinish {
namespace {
mtl::Context* g_cx = nullptr;
int g_lib = -1;
+int g_lib_iq = -1;
bool ensure() {
if (g_cx) return g_lib >= 0;
g_cx = new mtl::Context();
g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
+ g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);
+ if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");
return g_lib >= 0;
}
} // namespace
@@ -57,12 +60,9 @@ struct FinishIn {
bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {
if (n_vec <= 0 || len <= 0) return true;
- if (f.rd > 0 || f.quant != 0 || f.pdst != nullptr) {
- std::fprintf(stderr,
- "dsv41 metal: finish needs the unwritten branches (rd %d quant %d pdst %p) - not ported yet\n",
- f.rd, f.quant, (const void*) f.pdst);
- return false;
- }
+ // finish_kernel<true> (the KV pack: the row quantised to codes + group scales) or <false>; the <false> rounding
+ // branch (quant without pdst) is the one still unwritten
+ const bool pack = f.pdst != nullptr;
if (!ensure()) return false;
void* hx = strata::gpu::host_view(x);
if (!hx) { std::fprintf(stderr, "dsv41 metal: finish has no host view\n"); return false; }
@@ -76,17 +76,17 @@ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {
GpuFinish gf{};
gf.w = (uint64_t) (uintptr_t) f.w;
gf.eps = f.eps;
- gf.rd = 0; gf.pos = 0; gf.inverse = 0; gf.vpp = 1; gf.pos_step = 1;
+ gf.rd = f.rd; gf.pos = f.pos; gf.inverse = f.inverse; gf.vpp = f.vpp; gf.pos_step = f.pos_step;
gf.freqs = (uint64_t) (uintptr_t) f.freqs;
- gf.quant = 0;
+ gf.quant = f.quant;
gf.dst = (uint64_t) (uintptr_t) f.dst;
gf.dst_stride = f.dst_stride;
- gf.pdst = 0;
- gf.pdst_stride = 0;
+ gf.pdst = (uint64_t) (uintptr_t) f.pdst;
+ gf.pdst_stride = f.pdst_stride;
mtl::Pipeline pl;
try {
- pl = g_cx->pipeline(g_lib, "ds4_finish");
+ pl = g_cx->pipeline(g_lib, pack ? "ds4_finish_pack" : "ds4_finish");
} catch (...) {
return false;
}
@@ -110,12 +110,14 @@ bool hc_post(const float* y, float* h, const float* post, const float* comb, int
void* hh = strata::gpu::host_view(h);
if (!hy || !hh) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }
const size_t page = 16384;
- auto wrap = [&](void* host, size_t bytes) {
- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 + 65536);
- mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 + 65536);
+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
if (!by.handle || !bh.handle) return false;
// post is [T][8], comb is [T][64], straight from the caller's host/shared memory
mtl::Buffer bp = g_cx->alloc((size_t) T * 8 * 4);
@@ -147,12 +149,14 @@ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float*
void* hm = strata::gpu::host_view((void*) mixes);
if (!hh || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix_scale has no host view\n"); return false; }
const size_t page = 16384;
- auto wrap = [&](void* host, size_t bytes) {
- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 + 65536);
- mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);
+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );
+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
if (!bh.handle || !bm.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_scale"); } catch (...) { return false; }
@@ -176,13 +180,15 @@ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float
void* hm = strata::gpu::host_view((void*) mixes);
if (!hh || !hf || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix has no host view\n"); return false; }
const size_t page = 16384;
- auto wrap = [&](void* host, size_t bytes) {
- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 + 65536);
- mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 + 65536);
- mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);
+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );
+ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 );
+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
if (!bh.handle || !bf.handle || !bm.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix"); } catch (...) { return false; }
@@ -209,14 +215,16 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,
void* hc_ = strata::gpu::host_view((void*) comb);
if (!hm || !hp || !ho || !hc_) { std::fprintf(stderr, "dsv41 metal: hc_split has no host view\n"); return false; }
const size_t page = 16384;
- auto wrap = [&](void* host, size_t bytes) {
- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);
- mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 + 65536);
- mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 + 65536);
- mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 + 65536);
+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
+ mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 );
+ mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 );
if (!bm.handle || !bp.handle || !bo.handle || !bc.handle) return false;
// scale and base come from the engine like every other operand: reach them through host_view, not as raw host pointers
void* hs = strata::gpu::host_view((void*) scale);
@@ -224,7 +232,7 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,
if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }
const size_t nbase = (size_t) (2 * hc + hc * hc);
mtl::Buffer bs = wrap(hs, 256);
- mtl::Buffer bb = wrap(hb, (nbase * 4 + 65535) / 65536 * 65536 + 65536);
+ mtl::Buffer bb = wrap(hb, (nbase * 4 + 65535) / 65536 * 65536 );
if (!bs.handle || !bb.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_split"); } catch (...) { return false; }
@@ -252,14 +260,16 @@ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds
void* hp = strata::gpu::host_view((void*) pos);
if (!hi || !hv || !hb || !hr || !hp) { std::fprintf(stderr, "dsv41 metal: moe_sort has no host view\n"); return false; }
const size_t page = 16384;
- auto wrap = [&](void* host, size_t bytes) {
- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
const size_t t_bytes = (size_t) T * k * 4 + 65536;
mtl::Buffer bi = wrap(hi, t_bytes);
- mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 + 65536);
- mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 + 65536);
+ mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );
+ mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );
mtl::Buffer br = wrap(hr, t_bytes);
mtl::Buffer bp = wrap(hp, t_bytes);
if (!bi.handle || !bv.handle || !bb.handle || !br.handle || !bp.handle) return false;
@@ -298,4 +308,1105 @@ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds
return true;
}
+
+// ds4_kernels.cu:1690 - the Engram row decoder (fmt 2 = IQ4_NL, this model's tables)
+bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt) {
+ if (T <= 0) return true;
+ if (!ensure()) return false;
+ void* hs = strata::gpu::host_view((void*) src);
+ void* ho = strata::gpu::host_view((void*) out);
+ if (!hs || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant_tok has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t rows = (size_t) T * (size_t) cols;
+ mtl::Buffer bs = wrap(hs, (size_t) T * per );
+ mtl::Buffer bo = wrap(ho, rows * (size_t) hd * 4 );
+ if (!bs.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((unsigned long) per), mtl::Arg::V((unsigned) cols),
+ mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((int) fmt)};
+ d.grid[0] = (uint32_t) rows;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// ds4_kernels.cu:341 - hc_pre_norm: a = sum_s mix[s]*h[s][e]; x = a * rsqrt(mean(a^2) + eps) (x w when given)
+bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x) {
+ if (!ensure()) return false;
+ void* hh = strata::gpu::host_view((void*) h);
+ void* hm = strata::gpu::host_view((void*) mix);
+ void* hx = strata::gpu::host_view((void*) x);
+ if (!hh || !hm || !hx) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
+ mtl::Buffer bm = wrap(hm, (size_t) T * 8 * 4 );
+ mtl::Buffer bx = wrap(hx, (size_t) T * dim * 4 );
+ if (!bh.handle || !bm.handle || !bx.handle) return false;
+ mtl::Buffer bw;
+ size_t w_off = 0;
+ if (w) {
+ void* hw = strata::gpu::host_view((void*) w);
+ if (!hw) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm weight has no host view\n"); return false; }
+ bw = wrap(hw, (size_t) dim * 4 );
+ w_off = off(hw);
+ if (!bw.handle) return false;
+ } else {
+ bw = bx; // never read when has_w == 0
+ }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_pre_norm"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bw, w_off),
+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc), mtl::Arg::V(eps), mtl::Arg::B(bx, off(hx)),
+ mtl::Arg::V((unsigned) (w ? 1 : 0))};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// ds4_gguf.cu:186 - the activation quantiser the expert matmuls consume (Q8_0: 32 values, one scale)
+bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {
+ if (rows <= 0 || k <= 0) return true;
+ if (!ensure()) return false;
+ void* hx = strata::gpu::host_view((void*) x);
+ void* hq = strata::gpu::host_view((void*) q);
+ void* hd = strata::gpu::host_view((void*) d);
+ if (!hx || !hq || !hd) { std::fprintf(stderr, "dsv41 metal: q8_quant has no host view\n"); return false; }
+ const size_t page = 16384;
+ // The pages the buffer itself spans: [base, base + off + bytes). A fixed slack (the +65536 the call sites used to
+ // carry) walks off the end of the mapping when the buffer is small and sits at an arena chunk's edge, and Metal then
+ // refuses the no-copy wrap ("mtl: wrap 81920 failed" - the hd scales here, a few pages).
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bx = wrap(hx, (size_t) rows * ldx * 4);
+ mtl::Buffer bq = wrap(hq, (size_t) rows * k);
+ mtl::Buffer bd = wrap(hd, (size_t) rows * (k / 32) * 4);
+ if (!bx.handle || !bq.handle || !bd.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) k),
+ mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bd, off(hd))};
+ const uint groups = (uint32_t) ((k / 32 + 7) / 8);
+ dsp.grid[0] = groups;
+ dsp.grid[1] = (uint32_t) rows;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+
+// ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so
+// the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.
+bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt) {
+ if (nj <= 0 || max_n <= 0) return true;
+ if (!ensure()) return false;
+ mtl::Buffer bj = g_cx->alloc((size_t) nj * job_bytes);
+ if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: mx_mv job buffer failed\n"); return false; }
+ std::memcpy(bj.ptr, jobs, (size_t) nj * job_bytes);
+ mtl::Pipeline pl;
+ // fmt 1 = MXFP4, 2 = Q8_0, 0 = BF16 (a dense matrix: the engram's wkv, ds4_gguf.cu:1064)
+ const char* fn = fmt == 2 ? "ds4_q8_mv" : (fmt == 1 ? "ds4_mx_mv" : "ds4_bf16_mv");
+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);
+ d.grid[1] = (uint32_t) nj;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+
+
+// ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they
+// travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.
+bool iq_mv(const void* jobs, int nj) {
+ if (nj <= 0) return true;
+ if (!ensure()) return false;
+ // max rows across the jobs: the grid's x extent
+ const unsigned char* jb = (const unsigned char*) jobs;
+ int max_n = 0;
+ for (int i = 0; i < nj; ++i) {
+ int n = 0;
+ std::memcpy(&n, jb + (size_t) i * 80 + 24, sizeof(int)); // MvJob::n
+ if (n > max_n) max_n = n;
+ }
+ if (max_n <= 0) return true;
+ mtl::Buffer bj = g_cx->alloc((size_t) nj * 80);
+ if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n"); return false; }
+ std::memcpy(bj.ptr, jobs, (size_t) nj * 80);
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_iq, "ds4_iq_mv"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);
+ d.grid[1] = (uint32_t) nj;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+
+
+// ds4_iq.cu:536 gate_up_kernel<T, NE> / :588 down_kernel<T, NE> - the prompt / window path's expert GEMM (the launches at
+// :958 launch_gu / :960 launch_dn). A threadgroup of 256 is 8 warps: a warp RW rows (gate / up 1, down 2), a lane the
+// row's sub-blocks, the group's entries four at a pass. The group's blob and its entries are read the way the CUDA kernel
+// reads them, so the plan / window pointer arrays travel as they are: every pointer here is a CPU address (unified memory)
+// and a kernel may take raw gpuAddresses (as ds4_finish's f.w does).
+namespace {
+// (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4IqJob (Metal has no 'long long': 64-bit is 'long')
+struct IqJob {
+ const void* plan_ptr;
+ const void* grp_ptr;
+ const void* grp_start;
+ const void* grp_ent;
+ const void* n_groups;
+ const void* xq;
+ const void* xd;
+ const void* hq;
+ const void* hd;
+ void* h;
+ void* part;
+ int64_t tr0, tr1, tr2;
+ int n_embd, n_ff, type;
+ float limit;
+};
+
+static bool iq_gemm(const IqJob& j, const char* fn, int grid_x, int groups) {
+ if (!ensure()) return false;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) grid_x;
+ d.grid[1] = (uint32_t) groups;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+} // namespace
+
+// gate / up of the group's entries: h = the clamped swiglu (ds4_iq.cuh's Args: tr[0] gate, tr[1] up, tr[2] down)
+bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,
+ const int8_t* xq, const float* xd, float* h, int type, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 256 != 0 || n_ff % 8 != 0) return false;
+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ IqJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.xq = xq;
+ j.xd = xd;
+ j.h = h;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ j.type = type;
+ j.limit = limit;
+ return iq_gemm(j, "ds4_iq_gu", n_ff / 8, groups);
+}
+
+// down of the group's entries against their int8 h: part (a row of n_embd an entry)
+bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,
+ const float* hd, float* part, int type, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 256 != 0 || n_embd % 16 != 0) return false;
+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ IqJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.hq = hq;
+ j.hd = hd;
+ j.part = part;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ j.type = type;
+ j.limit = 0.0f;
+ return iq_gemm(j, "ds4_iq_dn", n_embd / 16, groups);
+}
+
+
+// ds4_gguf.cu:577 mx_combine_kernel - a token's expert outputs summed in plan order (the experts off the device add 0),
+// the CPU lane's part when its flag is set, then the shared expert's: ds4_gguf.cu:169's combine(), 128 threads a block
+namespace {
+// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Comb (a 'long' there is int64_t here)
+struct MxCombJob {
+ const void* plan_ptr;
+ const void* plan_w;
+ const void* part;
+ const void* cpu_part;
+ const int* cpu_flag;
+ const void* sh_out;
+ void* out;
+ int64_t cpu_stride, out_stride;
+ int k, n_embd;
+};
+} // namespace
+
+bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride,
+ const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd) {
+ if (T <= 0 || n_embd <= 0) return true;
+ if (k <= 0 || plan_ptr == nullptr || plan_w == nullptr || part == nullptr || out == nullptr) return false;
+ if (!ensure()) return false;
+ MxCombJob j{};
+ j.plan_ptr = plan_ptr;
+ j.plan_w = plan_w;
+ j.part = part;
+ j.cpu_part = cpu_part;
+ j.cpu_flag = cpu_flag;
+ j.sh_out = sh_out;
+ j.out = out;
+ j.cpu_stride = (int64_t) cpu_stride;
+ j.out_stride = (int64_t) out_stride;
+ j.k = k;
+ j.n_embd = n_embd;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: mx_combine job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_mx_combine"); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((n_embd + 127) / 128);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+
+// ds4_kernels.cu:1092 - the KV ring row to BF16 (E4M3 code x E8M0 group scale)
+bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes) {
+ if (n <= 0 || d <= 0) return true;
+ if (!ensure()) return false;
+ void* hr = strata::gpu::host_view((void*) ring);
+ void* he = strata::gpu::host_view(ext);
+ if (!hr || !he) { std::fprintf(stderr, "dsv41 metal: ring_to_ext has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );
+ mtl::Buffer be = wrap(he, (size_t) n * d * 2 );
+ if (!br.handle || !be.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_ring_to_ext"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::B(be, off(he)), mtl::Arg::V((long) p0),
+ mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};
+ dsp.grid[0] = (uint32_t) n;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+
+// ds4_kernels.cu:1104 - the window back into the ring (E4M3 codes + E8M0 group scales, optional backup)
+bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup) {
+ if (T <= 0 || n <= 0 || d <= 0) return true;
+ if (!ensure()) return false;
+ void* he = strata::gpu::host_view((void*) ext);
+ void* hr = strata::gpu::host_view(ring);
+ if (!he || !hr) { std::fprintf(stderr, "dsv41 metal: ext_to_ring has no host view\n"); return false; }
+ void* hb = backup ? strata::gpu::host_view(backup) : nullptr;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer be = wrap(he, (size_t) (n + T) * d * 2 );
+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );
+ if (!be.handle || !br.handle) return false;
+ mtl::Buffer bb = br; // never written when backup is null
+ size_t b_off = 0;
+ if (hb) { bb = wrap(hb, (size_t) T * row_bytes ); b_off = off(hb); if (!bb.handle) return false; }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_ext_to_ring"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),
+ mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),
+ mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+
+// ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)
+
+// On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about
+// (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.
+static inline void* cpu_addr(void* p) {
+ void* h = strata::gpu::host_view(p);
+ return h ? h : p;
+}
+
+// ds4_kernels.cu:1286 engram_dequant_kernel - the Engram rows (fmt 2 = IQ4_NL, the tables this model's pack carries; the
+// other formats are left to the CUDA launch, which reports the line)
+bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt) {
+ (void) scales; // (fmt 2 keeps every block's scale in its own 18 bytes)
+ if (n <= 0 || hd <= 0) return true;
+ if (fmt != 2 || rows == nullptr || out == nullptr) return false;
+ if (!ensure()) return false;
+ void* hr = cpu_addr((void*) rows);
+ void* ho = cpu_addr((void*) out);
+ if (!hr || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);
+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);
+ if (!br.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant_rows"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(fmt)};
+ d.grid[0] = (uint32_t) n;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+// ds4_kernels.cu:1311 engram_gate_kernel - the Engram gate over a (token, copy) row: h += sigmoid(normalised dot) * val
+bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps) {
+ if (T <= 0 || hc <= 0 || dim <= 0) return true;
+ if (h == nullptr || kv == nullptr || w == nullptr) return false;
+ if (!ensure()) return false;
+ void* hh = cpu_addr((void*) h);
+ void* hk = cpu_addr((void*) kv);
+ void* hw = cpu_addr((void*) w);
+ if (!hh || !hk || !hw) { std::fprintf(stderr, "dsv41 metal: engram_gate has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);
+ mtl::Buffer bk = wrap(hk, (size_t) T * (size_t) (hc + 1) * (size_t) dim * 4);
+ mtl::Buffer bw = wrap(hw, (size_t) hc * (size_t) dim * 4);
+ if (!bh.handle || !bk.handle || !bw.handle) {
+ std::fprintf(stderr, "dsv41 metal: engram_gate wrap failed (h %p kv %p w %p; %p %p %p)\n", hh, hk, hw,
+ bh.handle, bk.handle, bw.handle);
+ return false;
+ }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_gate"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bk, off(hk)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::V((unsigned) hc), mtl::Arg::V((unsigned) dim), mtl::Arg::V(eps)};
+ d.grid[0] = (uint32_t) hc;
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+namespace {
+// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Rows (ds4_kernels.cuh:115 Rows, kWin = 16)
+struct Ds4RowsJob {
+ int n[16];
+ int k[16];
+ int gen, R, kcap, ccap, blk;
+ int64_t p0;
+};
+} // namespace
+
+// ds4_kernels.cu:524 compress_kernel - the window compressor (ratio > 1): each token's kv / score row into its slot, and
+// every ratio'th token pooled by a per-value softmax over the slots, scaled by the row RMS and the norm weights
+bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
+ int d, const float* w, float eps, float* latent) {
+ if (T <= 0 || d <= 0 || ratio <= 0) return true;
+ if (d > 2 * 512) return false; // (the kernel holds d <= 2 * blockDim)
+ if (kv == nullptr || score == nullptr || kv_state == nullptr || score_state == nullptr || w == nullptr ||
+ latent == nullptr)
+ return false;
+ if (!ensure()) return false;
+ void* hkv = cpu_addr((void*) kv);
+ void* hsc = cpu_addr((void*) score);
+ void* hks = cpu_addr((void*) kv_state);
+ void* hss = cpu_addr((void*) score_state);
+ void* hw = cpu_addr((void*) w);
+ void* hla = cpu_addr((void*) latent);
+ if (!hkv || !hsc || !hks || !hss || !hw || !hla) {
+ std::fprintf(stderr, "dsv41 metal: compress_win has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bkv = wrap(hkv, (size_t) T * (size_t) d * 4);
+ mtl::Buffer bsc = wrap(hsc, (size_t) T * (size_t) d * 4);
+ mtl::Buffer bks = wrap(hks, (size_t) ratio * (size_t) d * 4);
+ mtl::Buffer bss = wrap(hss, (size_t) ratio * (size_t) d * 4);
+ mtl::Buffer bw = wrap(hw, (size_t) d * 4);
+ mtl::Buffer bla = wrap(hla, (size_t) T * (size_t) d * 4);
+ if (!bkv.handle || !bsc.handle || !bks.handle || !bss.handle || !bw.handle || !bla.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_compress"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bkv, off(hkv)), mtl::Arg::B(bsc, off(hsc)), mtl::Arg::B(bks, off(hks)),
+ mtl::Arg::B(bss, off(hss)), mtl::Arg::V(ratio), mtl::Arg::V(p0), mtl::Arg::V(T),
+ mtl::Arg::V((unsigned) d), mtl::Arg::B(bw, off(hw)), mtl::Arg::V(eps), mtl::Arg::B(bla, off(hla))};
+ dsp.grid[0] = 1;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 512; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+// ds4_kernels.cu:582 index_score_kernel<128, kfmt> - the indexer's scores: a position's key row against every head's
+// query, relu, the heads' weighted sum (rows = ds4_kernels.cuh's Rows, by value there and copied here)
+bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand,
+ int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt) {
+ if (n <= 0 || T <= 0) return true;
+ if (hd != 128 || heads <= 0 || heads > 32 || (kfmt != 3 && kfmt != 0)) return false;
+ if (q == nullptr || wq == nullptr || kc == nullptr || out == nullptr || rows == nullptr) return false;
+ if (!ensure()) return false;
+ void* hq = cpu_addr((void*) q);
+ void* hwq = cpu_addr((void*) wq);
+ void* hkc = cpu_addr((void*) kc);
+ void* hout = cpu_addr((void*) out);
+ if (!hq || !hwq || !hkc || !hout) {
+ std::fprintf(stderr, "dsv41 metal: index_score has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t kb = kfmt == 3 ? (size_t) ((hd / 2 + hd / 32 + 15) / 16 * 16) : (size_t) hd * 2;
+ mtl::Buffer bq = wrap(hq, (size_t) T * (size_t) heads * (size_t) hd * 4);
+ mtl::Buffer bwq = wrap(hwq, (size_t) T * 64 * 4);
+ mtl::Buffer bkc = wrap(hkc, (size_t) n * kb);
+ mtl::Buffer bo = wrap(hout, ((size_t) (T - 1) * (size_t) out_stride + (size_t) n) * 4);
+ if (!bq.handle || !bwq.handle || !bkc.handle || !bo.handle) return false;
+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;
+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);
+ if (!bc.handle) return false;
+ // Rows is a host struct: it travels as a copy (no gpu address)
+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: index_score rows buffer failed\n"); return false; }
+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_index_score"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),
+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::B(bo, off(hout)),
+ mtl::Arg::B(br), mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V(cand_stride), mtl::Arg::V(blk),
+ mtl::Arg::V((long) out_stride), mtl::Arg::V(kfmt), mtl::Arg::V(cand != nullptr ? 1 : 0)};
+ dsp.grid[0] = (uint32_t) ((n + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(br);
+ if (cand == nullptr) g_cx->free(bc);
+ return true;
+}
+
+// ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations, a warp a row of the matrix and 2 tokens
+// a block (the indexer's wproj: ds4_model.cu:7743)
+bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy) {
+ if (T <= 0 || n <= 0 || k <= 0) return true;
+ if (k % 8 != 0 || W == nullptr || x == nullptr || y == nullptr) return false;
+ if (!ensure()) return false;
+ void* hw = cpu_addr((void*) W);
+ void* hx = cpu_addr((void*) x);
+ void* hy = cpu_addr((void*) y);
+ if (!hw || !hx || !hy) {
+ std::fprintf(stderr, "dsv41 metal: gemv_f16_rows has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bw = wrap(hw, (size_t) n * (size_t) k * 2);
+ mtl::Buffer bx = wrap(hx, ((size_t) (T - 1) * ldx + (size_t) k) * 4);
+ mtl::Buffer by = wrap(hy, ((size_t) (T - 1) * ldy + (size_t) n) * 4);
+ if (!bw.handle || !bx.handle || !by.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f16_rows"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V(n),
+ mtl::Arg::V(k), mtl::Arg::V(T), mtl::Arg::V(alpha), mtl::Arg::B(by, off(hy)),
+ mtl::Arg::V((unsigned long) ldy)};
+ dsp.grid[0] = (uint32_t) ((n + 7) / 8);
+ dsp.grid[1] = (uint32_t) ((T + 1) / 2);
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+// ds4_kernels.cu:650 topk_kernel - a token's k largest indexer scores (four 8-bit radix passes, then the picks in index
+// order); Rows by value there and copied here
+bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out,
+ int out_stride) {
+ if (T <= 0) return true;
+ if (vals == nullptr || out == nullptr || rows == nullptr) return false;
+ if (!ensure()) return false;
+ void* hv = cpu_addr((void*) vals);
+ void* ho = cpu_addr((void*) out);
+ if (!hv || !ho) {
+ std::fprintf(stderr, "dsv41 metal: topk has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bv = wrap(hv, ((size_t) (T - 1) * (size_t) stride + 1) * 4);
+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + 1) * 4);
+ if (!bv.handle || !bo.handle) return false;
+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;
+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);
+ if (!bc.handle) return false;
+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: topk rows buffer failed\n"); return false; }
+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_topk"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bv, off(hv)), mtl::Arg::V((long) stride), mtl::Arg::B(br),
+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::V(cand_stride),
+ mtl::Arg::V(blk), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(out_stride),
+ mtl::Arg::V(cand != nullptr ? 1 : 0)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(br);
+ if (cand == nullptr) g_cx->free(bc);
+ return true;
+}
+
+// ds4_kernels.cu:751 block_max_kernel - a block of the indexer's positions and its largest score (the newest pinned at +inf)
+bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride,
+ int nb_max) {
+ if (T <= 0 || nb_max <= 0 || blk <= 0) return true;
+ if (score == nullptr || bm == nullptr || rows == nullptr) return false;
+ if (!ensure()) return false;
+ void* hs = cpu_addr((void*) score);
+ void* hb = cpu_addr((void*) bm);
+ if (!hs || !hb) {
+ std::fprintf(stderr, "dsv41 metal: block_max has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bs = wrap(hs, ((size_t) (T - 1) * (size_t) stride + (size_t) nb_max * (size_t) blk) * 4);
+ mtl::Buffer bb = wrap(hb, ((size_t) (T - 1) * (size_t) bm_stride + (size_t) nb_max) * 4);
+ if (!bs.handle || !bb.handle) return false;
+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: block_max rows buffer failed\n"); return false; }
+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_block_max"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((long) stride), mtl::Arg::B(br), mtl::Arg::V(blk),
+ mtl::Arg::B(bb, off(hb)), mtl::Arg::V((long) bm_stride)};
+ dsp.grid[0] = (uint32_t) ((nb_max + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(br);
+ return true;
+}
+
+// ds4_kernels.cu:1141 hc_mean_kernel - a token's mean over its hc streams (grid (ceil(dim / 256), T), 256 threads)
+bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride) {
+ if (T <= 0 || hc <= 0 || dim <= 0) return true;
+ if (h == nullptr || out == nullptr) return false;
+ if (!ensure()) return false;
+ void* hh = cpu_addr((void*) h);
+ void* ho = cpu_addr((void*) out);
+ if (!hh || !ho) {
+ std::fprintf(stderr, "dsv41 metal: hc_mean has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);
+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + (size_t) dim) * 4);
+ if (!bh.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mean"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::V(hc), mtl::Arg::V(dim), mtl::Arg::B(bo, off(ho)),
+ mtl::Arg::V((long) out_stride)};
+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+// ds4_kernels.cu:1150 iota_rows_kernel - out[t][i] = i
+bool iota_rows(int* out, int T, int n, int stride) {
+ if (T <= 0 || n <= 0) return true;
+ if (out == nullptr) return false;
+ if (!ensure()) return false;
+ void* ho = cpu_addr((void*) out);
+ if (!ho) {
+ std::fprintf(stderr, "dsv41 metal: iota_rows has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) stride + (size_t) n) * 4);
+ if (!bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_iota_rows"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bo, off(ho)), mtl::Arg::V(n), mtl::Arg::V(stride)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 64; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+}
+
+// vendor/sampler.cu:93 sampler_greedy_kernel - the plain argmax over the vocabulary, one block (1024) a token, the
+// repetition penalties from the history window's bitmap (the caller takes this path when greedy or temperature <= 0)
+bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n,
+ float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out) {
+ if (n_tokens <= 0 || n_vocab <= 0) return true;
+ if (logits == nullptr || out == nullptr) return false;
+ if ((n_vocab + 31) / 32 > 4096) return false; // (the kernel's threadgroup bitmap)
+ if (!ensure()) return false;
+ void* hl = cpu_addr((void*) logits);
+ void* ho = cpu_addr((void*) out);
+ if (!hl || !ho) {
+ std::fprintf(stderr, "dsv41 metal: sample_greedy has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bl = wrap(hl, (size_t) n_tokens * (size_t) n_vocab * 4);
+ mtl::Buffer bo = wrap(ho, (size_t) n_tokens * 4);
+ if (!bl.handle || !bo.handle) return false;
+ const size_t hbytes = (size_t) n_tokens * (size_t) (history_len > 0 ? history_len : 1) * 4;
+ mtl::Buffer bh = history != nullptr ? wrap(cpu_addr((void*) history), hbytes) : g_cx->alloc(4);
+ mtl::Buffer bb = logit_bias != nullptr ? wrap(cpu_addr((void*) logit_bias), (size_t) n_vocab * 4) : g_cx->alloc(4);
+ if (!bh.handle || !bb.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_sampler_greedy"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::V(n_vocab),
+ mtl::Arg::B(bh, history != nullptr ? off(cpu_addr((void*) history)) : 0),
+ mtl::Arg::V(history_len), mtl::Arg::V(penalty_last_n), mtl::Arg::V(penalty_repeat),
+ mtl::Arg::V(penalty_freq), mtl::Arg::V(penalty_present),
+ mtl::Arg::B(bb, logit_bias != nullptr ? off(cpu_addr((void*) logit_bias)) : 0),
+ mtl::Arg::V(logit_bias != nullptr ? 1 : 0), mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) n_tokens;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ if (history == nullptr) g_cx->free(bh);
+ if (logit_bias == nullptr) g_cx->free(bb);
+ return true;
+}
+
+bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap,
+ const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes) {
+ if (T <= 0 || sel_stride <= 0 || nsel_count <= 0) return true;
+ // nothing selected yet (a prompt at position 0): the scratch stays untouched, which is what the CUDA launch would do with
+ // no rows to write
+ if (ckv == nullptr || sel == nullptr || out == nullptr) return true;
+ if (!ensure()) return false;
+ void* hc = cpu_addr((void*) ckv);
+ void* hs = cpu_addr((void*) sel);
+ void* ho = cpu_addr((void*) out);
+ if (!hc || !hs || !ho) { std::fprintf(stderr, "dsv41 metal: unpack_sel has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bc = wrap(hc, (size_t) 4096 * row_bytes );
+ mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );
+ mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );
+ if (!bc.handle || !bsl.handle || !bo.handle) return false;
+ mtl::Buffer bn = g_cx->alloc((size_t) nsel_count * 4);
+ if (!bn.ptr) return false;
+ std::memcpy(bn.ptr, nsel, (size_t) nsel_count * 4);
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_unpack_sel"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bc, off(hc)), mtl::Arg::B(bsl, off(hs)), mtl::Arg::V((unsigned) sel_stride),
+ mtl::Arg::V((int) gen), mtl::Arg::V((long) p0), mtl::Arg::V((int) t0), mtl::Arg::V((int) R),
+ mtl::Arg::V((int) kcap), mtl::Arg::B(bn), mtl::Arg::V((unsigned) cap), mtl::Arg::B(bo, off(ho)),
+ mtl::Arg::V((unsigned long) row_bytes)};
+ d.grid[0] = (uint32_t) ((sel_stride + 15) / 16);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bn);
+ return true;
+}
+
+
+// ds4_kernels.cu:1772 - the expert outputs back into the token rows
+bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim) {
+ if (T <= 0 || dim <= 0) return true;
+ if (!ensure()) return false;
+ void* ha = cpu_addr(acc);
+ void* hy = cpu_addr((void*) y);
+ void* hp = cpu_addr((void*) pos);
+ void* hw = cpu_addr((void*) w_);
+ if (!ha || !hy || !hp || !hw) { std::fprintf(stderr, "dsv41 metal: moe_combine has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer ba = wrap(ha, (size_t) T * dim * 4 );
+ mtl::Buffer by = wrap(hy, (size_t) (r1 > r0 ? r1 - r0 : 1) * dim * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );
+ if (!ba.handle || !by.handle || !bp.handle || !bw.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_moe_combine"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(by, off(hy)), mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::V((int) k), mtl::Arg::V((int) r0), mtl::Arg::V((int) r1), mtl::Arg::V((unsigned) dim)};
+ d.grid[0] = (uint32_t) ((dim + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// ds4_kernels.cu:362 - hc_post: out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]
+bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out) {
+ if (T <= 0 || dim <= 0) return true;
+ if (!ensure()) return false;
+ void* hy = cpu_addr((void*) y);
+ void* hr = cpu_addr((void*) res);
+ void* hp = cpu_addr((void*) post);
+ void* hco = cpu_addr((void*) comb);
+ void* ho = cpu_addr((void*) out);
+ if (!hy || !hr || !hp || !hco || !ho) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
+ mtl::Buffer br = wrap(hr, (size_t) T * hc * dim * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
+ mtl::Buffer bc = wrap(hco, (size_t) T * 64 * 4 );
+ mtl::Buffer bo = wrap(ho, (size_t) T * hc * dim * 4 );
+ if (!by.handle || !br.handle || !bp.handle || !bc.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post_v"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp)),
+ mtl::Arg::B(bc, off(hco)), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((unsigned) dim),
+ mtl::Arg::V((unsigned) hc)};
+ d.grid[0] = (uint32_t) ((dim * hc + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// ds4_kernels.cu:441 - the F32 GEMV (residual / dense projections)
+bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy) {
+ if (T <= 0 || n <= 0 || k <= 0) return true;
+ if (!ensure()) return false;
+ void* hw = cpu_addr((void*) W);
+ void* hx = cpu_addr((void*) x);
+ void* hy = cpu_addr((void*) y);
+ if (!hw || !hx || !hy) { std::fprintf(stderr, "dsv41 metal: gemv_f32 has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bw = wrap(hw, (size_t) n * k * 4 );
+ mtl::Buffer bx = wrap(hx, (size_t) T * k * 4 );
+ mtl::Buffer by = wrap(hy, (size_t) T * (ldy > 0 ? ldy : n) * 4 );
+ if (!bw.handle || !bx.handle || !by.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f32"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned) n),
+ mtl::Arg::V((unsigned) k), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned) ldy)};
+ d.grid[0] = (uint32_t) ((n + 7) / 8);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// ds4_kernels.cu:1353 - the router's selection (idx, w, plan_ptr). The prefetch requests stay unpublished for now.
+bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram,
+ int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl) {
+ if (T <= 0 || n <= 0) return true;
+ if (!ensure()) return false;
+ void* hl = cpu_addr((void*) logits);
+ void* hb = cpu_addr((void*) bias);
+ void* hv = cpu_addr((void*) vram);
+ void* hi = cpu_addr((void*) idx);
+ void* hw = cpu_addr((void*) w);
+ void* hp = cpu_addr((void*) plan_ptr);
+ if (!hl || !hb || !hv || !hi || !hw || !hp) { std::fprintf(stderr, "dsv41 metal: route_publish has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4 );
+ mtl::Buffer bb = wrap(hb, (size_t) n * 4 );
+ mtl::Buffer bv = wrap(hv, (size_t) n * 8 );
+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4 );
+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 8 );
+ if (!bl.handle || !bb.handle || !bv.handle || !bi.handle || !bw.handle || !bp.handle) return false;
+ mtl::Buffer bvl = bb;
+ size_t vl_off = 0;
+ if (bias_vl) {
+ void* hvl = cpu_addr((void*) bias_vl);
+ if (!hvl) return false;
+ bvl = wrap(hvl, (size_t) n * 4 );
+ vl_off = off(hvl);
+ }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_route_publish"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),
+ mtl::Arg::V(route_scale), mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bvl, vl_off), mtl::Arg::V((unsigned) vl),
+ mtl::Arg::V((unsigned) (bias_vl ? 1 : 0))};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 512; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// ds4_kernels.cu swiglu: h = silu(g) * u (clamped when lim > 0)
+bool swiglu(const float* g, const float* u, int n, float lim, float* h) {
+ if (n <= 0) return true;
+ if (!ensure()) return false;
+ void* hg = cpu_addr((void*) g);
+ void* hu = cpu_addr((void*) u);
+ void* hh = cpu_addr((void*) h);
+ if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host) {
+ const size_t bytes = (size_t) n * 4 + 65536;
+ return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);
+ if (!bg.handle || !bu.handle || !bh.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_swiglu"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bg, off(hg)), mtl::Arg::B(bu, off(hu)), mtl::Arg::V((unsigned) n), mtl::Arg::V(lim),
+ mtl::Arg::B(bh, off(hh))};
+ d.grid[0] = (uint32_t) ((n + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+
+
+// vendor/moe_win.cu:49 moe_win_group - the expert -> group index for a window. One thread in CUDA; on the Mac every device
+// pointer is a CPU address, so the same 25-line loop runs on the host, which keeps the first-appearance order the kernel has and
+// avoids mirroring the 17-field MoeWin struct through a constant buffer.
+bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k) {
+ if (T <= 0 || k <= 0) return true;
+ if (T * 8 > 4096) { std::fprintf(stderr, "dsv41 metal: moe_win_group window of %d too wide\n", T); return false; }
+ const unsigned long long* plan = (const unsigned long long*) cpu_addr((void*) plan_ptr);
+ unsigned long long* gptr = (unsigned long long*) cpu_addr(grp_ptr);
+ int* gstart = (int*) cpu_addr(grp_start);
+ int* gent = (int*) cpu_addr(grp_ent);
+ int* ng_out = (int*) cpu_addr(n_groups);
+ if (!plan || !gptr || !gstart || !gent || !ng_out) return false;
+ static unsigned long long gp[4096];
+ static int cnt[4096], fill[4097], gof[4096];
+ int ng = 0;
+ for (int t = 0; t < T; ++t) {
+ for (int i = 0; i < 8; ++i) {
+ const int e = t * 8 + i;
+ gof[e] = -1;
+ const unsigned long long p = i < k ? plan[e] : 0ull;
+ if (p == 0ull) continue;
+ int g = 0;
+ while (g < ng && gp[g] != p) ++g;
+ if (g == ng) {
+ gp[ng] = p;
+ cnt[ng++] = 0;
+ }
+ ++cnt[g];
+ gof[e] = g;
+ }
+ }
+ fill[0] = 0;
+ for (int g = 0; g < ng; ++g) {
+ gptr[g] = gp[g];
+ gstart[g] = fill[g];
+ fill[g + 1] = fill[g] + cnt[g];
+ }
+ gstart[ng] = fill[ng];
+ for (int e = 0; e < T * 8; ++e) {
+ if (gof[e] >= 0) gent[fill[gof[e]]++] = e;
+ }
+ *ng_out = ng;
+ return true;
+}
+
} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index c2550903d..668d3ee42 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -77,6 +77,42 @@ static inline float ds4_block_sum(float v, threadgroup float* red, ushort lane,
return red[0];
}
+// ds4_kernels.cu:65 pow2_ceil / :88 fp8_rt / :90 e2m1_rt - the KV quantiser's scale and roundings, shared by finish_pack
+// (the packed rows) and ds4_finish's rounding branch
+static inline float ds4_pow2_ceil(float x) {
+ const uint b = as_type<uint>(x);
+ const int e = int((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);
+ return as_type<float>(uint(e + 127) << 23);
+}
+
+static inline float ds4_e4m3_round(float v) { // E4M3, nearest even (the input is clamped)
+ const float a = fabs(v);
+ if (!(a == a)) return v;
+ float r;
+ if (a >= 448.0f) r = 448.0f;
+ else {
+ const int e = int((as_type<uint>(a) >> 23) & 0xffu) - 127;
+ const float step = e < -6 ? 0.001953125f : as_type<float>(uint(e - 3 + 127) << 23);
+ r = a == 0.0f ? 0.0f : rint(a / step) * step;
+ r = fmin(r, 448.0f);
+ }
+ return copysign(r, v);
+}
+
+static inline float ds4_e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even
+ const float a = fabs(v);
+ float r;
+ if (a <= 0.25f) r = 0.0f;
+ else if (a < 0.75f) r = 0.5f;
+ else if (a <= 1.25f) r = 1.0f;
+ else if (a < 1.75f) r = 1.5f;
+ else if (a <= 2.5f) r = 2.0f;
+ else if (a < 3.5f) r = 3.0f;
+ else if (a <= 5.0f) r = 4.0f;
+ else r = 6.0f;
+ return copysign(r, v);
+}
+
kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],
constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],
uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
@@ -101,6 +137,44 @@ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[bu
if (e < len) r[j] = r[j] * inv * f.w[e];
}
}
+ if (f.rd > 0) {
+ // mrope: the last rd values move in (re, im) pairs a lane apart (ds4_kernels.cu:161): the partner comes from a
+ // lane xor 1, the angle from the token's position and the frequency of the pair
+ const uint r0 = len - uint(f.rd);
+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ const float other = simd_shuffle_xor(r[j], 1);
+ if (e >= r0 && e < len) {
+ const uint i = (e - r0) >> 1;
+ const float ang = float(pos) * f.freqs[i];
+ const float c = cos(ang);
+ const float sn = f.inverse ? -sin(ang) : sin(ang);
+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;
+ }
+ }
+ }
+ if (f.quant != 0) { // the KV rounding without the pack: the group's scale, then the value rounded to the format
+ const uint g = f.quant == 2 ? 16u : 32u;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ float a = e < len ? fabs(r[j]) : 0.0f;
+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));
+ if (e >= len) continue;
+ if (f.quant == 1) {
+ const float s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));
+ r[j] = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f)) * s;
+ } else if (f.quant == 2) {
+ const float s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);
+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;
+ } else {
+ const float s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;
+ }
+ }
+ }
for (uint j = 0; j < 32u; ++j) {
if (j >= nj) break;
const uint e = tid + j * 256u;
@@ -309,3 +383,1139 @@ kernel void ds4_moe_sort_fill(device const int* idx [[buffer(0)]], constant uint
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
+
+// ds4_kernels.cu:1690 engram_dequant_tok_kernel - one hash-table row a threadgroup: IQ4_NL blocks to F32.
+// fmt 2 (this model's Engram): an 18-byte block is fp16 d then 16 packed nibbles; the low nibble of q[j] is value j and the
+// high nibble value j + 16, each an index into ggml-common.h's kvalues_iq4nl (ds4_kernels.cu:1280).
+constant int ds4_kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+
+kernel void ds4_engram_dequant(device const uchar* src [[buffer(0)]], constant ulong& per [[buffer(1)]],
+ constant uint& cols [[buffer(2)]], constant uint& hd [[buffer(3)]],
+ device float* out [[buffer(4)]], constant int& fmt [[buffer(5)]],
+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint t = r / cols, c = r - t * cols;
+ const device uchar* rows = src + (size_t) t * per + (size_t) c * hd;
+ if (fmt == 2) {
+ for (uint e = tid; e < hd; e += 256u) {
+ const device uchar* b = rows + (size_t) (e / 32u) * 18u;
+ const uint q = b[2u + (e & 15u)];
+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);
+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);
+ out[(size_t) r * hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);
+ }
+ return;
+ }
+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: write NaN, so a wrong path shows as NaN in the output rather than
+ // as a silent zero.
+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * hd + e] = NAN;
+}
+
+// ds4_kernels.cu:341 hc_pre_norm_kernel - per token: a = sum_s mix[s]*h[token][s][e], one RMS over dim, then x = a*inv
+// (times w when given). One threadgroup a token, 1024 threads; a thread holds at most 8 values (dim <= 8*1024 = 8192).
+kernel void ds4_hc_pre_norm(device const float* h [[buffer(0)]], device const float* mix [[buffer(1)]],
+ device const float* w [[buffer(2)]], constant uint& dim [[buffer(3)]],
+ constant uint& hc [[buffer(4)]], constant float& eps [[buffer(5)]],
+ device float* x [[buffer(6)]], constant uint& has_w [[buffer(7)]],
+ uint b [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const device float* hh = h + (size_t) b * hc * dim;
+ const device float* mm = mix + (size_t) b * 8u;
+ device float* xx = x + (size_t) b * dim;
+ float r[8];
+ float ss = 0.0f;
+ uint n = 0;
+ for (uint e = tid; e < dim; e += 1024u, ++n) {
+ float a = 0.0f;
+ for (uint s = 0; s < 8u; ++s) if (s < hc) a += mm[s] * hh[(size_t) s * dim + e];
+ if (n < 8u) r[n] = a;
+ ss += a * a;
+ }
+ // a 1024-thread reduction: 32 simdgroups, each into red[], then the first simdgroup sums those
+ float v = ds4_warp_sum(ss);
+ if (lane == 0u) red[wg] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wg == 0u) {
+ float t = lane < 32u ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t);
+ if (lane == 0u) red[0] = t;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const float inv = rsqrt(red[0] / float(dim) + eps);
+ n = 0;
+ for (uint e = tid; e < dim; e += 1024u, ++n) {
+ if (n >= 8u) break;
+ xx[e] = has_w != 0u ? r[n] * inv * w[e] : r[n];
+ }
+}
+
+// ds4_gguf.cu:186 q8_quant_kernel: a warp a 32-group - amax across the warp, sc = amax/127, q = rint(v/sc), d = sc.
+// grid = (ceil((k/32) / 8 warps), rows), threadgroup = 256 (8 warps of 32).
+kernel void ds4_q8_quant(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],
+ constant uint& k [[buffer(2)]], device char* q [[buffer(3)]],
+ device float* d [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const uint warp = tid / 32u, lane = tid % 32u;
+ const uint b = tgp.x * 8u + warp; // the 32-group index
+ const uint r = tgp.y;
+ if ((size_t) b * 32u >= (size_t) k) return;
+ const device float* xr = x + (size_t) r * ldx;
+ const float v = xr[(size_t) b * 32u + lane];
+ const float am = simd_max(fabs(v));
+ const float sc = am / 127.0f;
+ q[(size_t) r * k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);
+ if (lane == 0u) d[(size_t) r * (k / 32u) + b] = sc;
+}
+
+// ds4_gguf.cu:371 mx_mv_kernel - the MXFP4 expert matrix-vector.
+// grid = (ceil(n / 8 warps), jobs), threadgroup = 256. A warp a row; a lane strides the 32-value blocks; NC columns of
+// activations share each decoded block (the T=8 window), then a warp sum and y[c][row] = alpha*v + bias[row].
+// Block layout (kMxBytes = 17 a 32 values): the E8M0 scale byte, then 16 bytes whose low nibble is value j and high nibble
+// value j+16, each an index into the 16-entry codebook {0,1,2,3,4,6,8,12} and its negatives.
+struct Ds4MxJob {
+ device const void* w;
+ ulong ld; // bytes a row
+ device const float* x;
+ ulong xcs;
+ device float* y;
+ ulong ycs;
+ device const float* bias;
+ float alpha;
+ int k, n, ncol;
+ device const char* xq;
+ device const float* xd;
+};
+
+static inline float ds4_e8m0(uint e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF
+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));
+}
+
+static inline void ds4_mx_block(device const uchar* p, threadgroup const float* lut, thread float* w) {
+ const float d = ds4_e8m0(uint(p[0]));
+ for (uint j = 0; j < 16u; ++j) {
+ const uint q = uint(p[1u + j]);
+ w[j] = lut[q & 15u] * d;
+ w[j + 16u] = lut[q >> 4] * d;
+ }
+}
+
+kernel void ds4_mx_mv(device const uchar* jobs [[buffer(0)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]]) {
+ threadgroup float lut[16];
+ if (tid < 16u) {
+ const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};
+ lut[tid] = tid < 8u ? v[tid] : -v[tid - 8u];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;
+ thread float acc[8];
+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
+ const uint nblk = uint(b.k) / 32u;
+ for (uint kb = lane; kb < nblk; kb += 32u) {
+ float wv[32];
+ ds4_mx_block(w + (size_t) kb * 17u, lut, wv);
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device float* x = b.x + c * b.xcs + (size_t) kb * 32u;
+ float a = acc[c];
+ for (uint j = 0; j < 32u; ++j) a = fma(wv[j], x[j], a);
+ acc[c] = a;
+ }
+ }
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// ds4_gguf.cu:320 q8_mv_kernel - the Q8_0 matvec against quantised activations.
+// grid = (ceil(n / 8), jobs), threadgroup 256: a warp a row, a lane strides the 34-byte blocks (fp16 d + 32 int8), the dot is
+// integer (CUDA's __dp4a in four-value groups; here the same sum written out), then acc += (dw * xd[c]) * sum.
+// The staged-prefetch (ST) and vector variants of the CUDA kernel are prefetch strategies only - this reads the blocks directly.
+kernel void ds4_q8_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;
+ const uint nb = uint(b.k) / 32u;
+ float acc[8];
+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
+ for (uint kb = lane; kb < nb; kb += 32u) {
+ const device uchar* p = w + (size_t) kb * 34u;
+ const ushort dh = ushort(p[0]) | (ushort(p[1]) << 8);
+ const float dw = float(as_type<half>(dh));
+ int wv[32];
+ for (uint j = 0; j < 32u; ++j) wv[j] = int((char) p[2u + j]);
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + (size_t) kb * 32u;
+ int sum = 0;
+ for (uint j = 0; j < 32u; ++j) sum += wv[j] * int(xq[j]);
+ acc[c] = fma(dw * b.xd[(size_t) c * nb + kb], float(sum), acc[c]);
+ }
+ }
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// ds4_kernels.cu:1092 ring_to_ext_kernel - one ring row of the KV cache to BF16: each value an E4M3 code times its group's
+// E8M0 power of two (row[e] and row[d + e/32]); rows before the window start write zero.
+static inline float ds4_e4m3(uint b) { // E4M3 (bias 7, no infinities, max 448): sign, 4-bit exp, 3-bit mantissa
+ const float s = (b & 0x80u) ? -1.0f : 1.0f;
+ const uint ex = (b >> 3) & 0xfu, m = b & 7u;
+ if (ex == 0u) return s * float(m) * (1.0f / 8.0f) * (1.0f / 64.0f);
+ if (ex == 15u && m == 7u) return NAN;
+ return s * exp2(float(ex) - 7.0f) * (1.0f + float(m) * 0.125f);
+}
+static inline float ds4_ue8m0(uint b) { return as_type<float>(b << 23); } // 2^(b - 127)
+static inline ushort ds4_f2bf16(float f) {
+ const uint u = as_type<uint>(f);
+ return ushort((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);
+}
+
+
+// ds4_kernels.cu:1104 ext_to_ring_kernel - the window's BF16 rows packed back into the E4M3 ring, each 32-value group at the
+// smallest power-of-two scale it fits under (k), so v / 2^k is an E4M3 value times a power of two.
+static inline uint ds4_e4m3_code(float v, int k) {
+ const uint sign = (v < 0.0f) ? 0x80u : 0u;
+ float a = fabs(v) * exp2(-float(k));
+ if (a == 0.0f) return sign;
+ int e = int(floor(log2(a)));
+ int mant = int(round((a / exp2(float(e)) - 1.0f) * 8.0f));
+ int ex = e + 7;
+ if (mant >= 8) { mant = 0; ++ex; }
+ if (ex <= 0) { mant = 0; ex = 0; }
+ if (ex > 15) { mant = 7; ex = 15; }
+ return sign | (uint(ex) << 3) | uint(mant & 7);
+}
+
+kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],
+ device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],
+ constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],
+ constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]],
+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ if (T - t > n) return; // overwritten by a later row of the window anyway
+ device uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;
+ if (backup != nullptr) {
+ for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const device ushort* src = ext + (size_t) (n + t) * d;
+ for (uint e0 = 0; e0 < d; e0 += 256u) {
+ const uint e = e0 + tid;
+ const float v = e < d ? float(as_type<half>(src[e])) : 0.0f;
+ uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bits
+ for (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));
+ const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);
+ if (e >= d) continue;
+ row[e] = uchar(ds4_e4m3_code(v, k));
+ if ((e & 31u) == 0u) row[d + e / 32u] = uchar(k + 127);
+ }
+}
+
+
+// ds4_kernels.cu:916 unpack32 - 16 threads expand one MXFP4 KV row (288 B at d=512) into 32 BF16 values a thread:
+// 16 code bytes at part*16, two E4M3 scale bytes at 256 + part*2, and the 4-bit code is itself an FP4 bit pattern
+// (e2m1_val14: sign and exponent/mantissa bits spliced into a half).
+static inline float ds4_e2m1_val14(uint c) {
+ const ushort h = (ushort) (((c & 8u) << 12) | ((c & 7u) << 9));
+ return float(as_type<half>(h));
+}
+
+// ds4_kernels.cu:937 unpack_sel_kernel - the selected rows of the compressed cache into the attention scratch, one row a
+// group of 16 threads. n_sel comes either from the generated schedule (gen: min(kcap, (pos+1)/R)) or from nsel[t].
+kernel void ds4_unpack_sel(device const uchar* ckv [[buffer(0)]], device const int* sel [[buffer(1)]],
+ constant uint& sel_stride [[buffer(2)]], constant int& gen [[buffer(3)]],
+ constant long& p0 [[buffer(4)]], constant int& t0 [[buffer(5)]],
+ constant int& R [[buffer(6)]], constant int& kcap [[buffer(7)]],
+ device const int* nsel [[buffer(8)]], constant uint& cap [[buffer(9)]],
+ device ushort* out [[buffer(10)]], constant ulong& rb [[buffer(11)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint t = tgp.y;
+ int n_sel;
+ if (gen != 0) {
+ const long pa = p0 + long(t0) + long(t);
+ n_sel = R > 0 ? int(min(long(kcap), (pa + 1) / long(R))) : 0;
+ } else {
+ n_sel = nsel[t];
+ }
+ const uint i = tgp.x * 16u + (tid >> 4u), part = tid & 15u;
+ if (int(i) >= n_sel) return;
+ const size_t r = (size_t) t * cap + i;
+ const device uchar* rr = ckv + (size_t) sel[(size_t) t * sel_stride + i] * rb;
+ const uint4 cw = *(const device uint4*) (rr + part * 16u);
+ const float k0 = ds4_e4m3(uint(rr[256u + part * 2u])) * 16384.0f;
+ const float k1 = ds4_e4m3(uint(rr[256u + part * 2u + 1u])) * 16384.0f;
+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};
+ uint o[16];
+ for (uint k = 0; k < 16u; ++k) {
+ const uint b = wv[k >> 2] >> (8u * (k & 3u));
+ const float ks = k < 8u ? k0 : k1;
+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
+ }
+ device uint* dst = (device uint*) (out + r * 512u + part * 32u);
+ for (uint q = 0; q < 4u; ++q) {
+ dst[q * 4u + 0u] = o[4u * q + 0u];
+ dst[q * 4u + 1u] = o[4u * q + 1u];
+ dst[q * 4u + 2u] = o[4u * q + 2u];
+ dst[q * 4u + 3u] = o[4u * q + 3u];
+ }
+}
+
+
+// ds4_kernels.cu:1772 moe_combine_kernel - a token's expert outputs back into its row: for each pick whose routed row is in
+// [r0, r1), acc += w[t][i] * y[r - r0][e]; untouched when no pick falls in range.
+kernel void ds4_moe_combine(device float* acc [[buffer(0)]], device const float* y [[buffer(1)]],
+ device const int* pos [[buffer(2)]], device const float* w [[buffer(3)]],
+ constant int& kk [[buffer(4)]], constant int& r0 [[buffer(5)]],
+ constant int& r1 [[buffer(6)]], constant uint& dim [[buffer(7)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + tid;
+ if (e >= dim) return;
+ const uint t = tgp.y;
+ float a = acc[(size_t) t * dim + e];
+ bool any = false;
+ for (int i = 0; i < 8; ++i) {
+ if (i >= kk) break;
+ const int r = pos[(size_t) t * 8u + i];
+ if (r < r0 || r >= r1) continue;
+ a = fma(w[(size_t) t * 8u + i], y[(size_t) (r - r0) * dim + e], a);
+ any = true;
+ }
+ if (any) acc[(size_t) t * dim + e] = a;
+}
+
+
+// ds4_kernels.cu:362 hc_post_kernel - like hc_post_inplace, but reads a separate residual and writes a separate output:
+// out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]
+kernel void ds4_hc_post_v(device const float* y [[buffer(0)]], device const float* res [[buffer(1)]],
+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],
+ device float* out [[buffer(4)]], constant uint& dim [[buffer(5)]],
+ constant uint& hc [[buffer(6)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const uint i = tgp.x * 256u + tid;
+ if (i >= dim * hc) return;
+ const uint t = tgp.y, k = i / dim, e = i - k * dim;
+ const device float* r = res + (size_t) t * hc * dim;
+ float a = post[(size_t) t * 8u + k] * y[(size_t) t * dim + e];
+ for (uint j = 0; j < 8u; ++j) {
+ if (j >= hc) break;
+ a += comb[(size_t) t * 64u + j * hc + k] * r[(size_t) j * dim + e];
+ }
+ out[(size_t) t * hc * dim + i] = a;
+}
+
+
+// ds4_kernels.cu:441 gemv_f32_kernel - a warp a row of F32 weights: float4 loads of W[row] and x[t], four FMAs a step,
+// lane-strided over k/4, then the warp sum into y[t][row].
+kernel void ds4_gemv_f32(device const float* W [[buffer(0)]], device const float* x [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],
+ device float* y [[buffer(4)]], constant uint& ldy [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]]) {
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (row >= n) return;
+ const uint t = tgp.y;
+ const device float4* w = (const device float4*) (W + (size_t) row * kk);
+ const device float4* xv = (const device float4*) (x + (size_t) t * kk);
+ float acc = 0.0f;
+ for (uint c = lane; c < kk / 4u; c += 32u) {
+ const float4 a = w[c], b = xv[c];
+ acc = fma(a.x, b.x, acc);
+ acc = fma(a.y, b.y, acc);
+ acc = fma(a.z, b.z, acc);
+ acc = fma(a.w, b.w, acc);
+ }
+ acc += simd_shuffle_xor(acc, 16);
+ acc += simd_shuffle_xor(acc, 8);
+ acc += simd_shuffle_xor(acc, 4);
+ acc += simd_shuffle_xor(acc, 2);
+ acc += simd_shuffle_xor(acc, 1);
+ if (lane == 0u) y[(size_t) t * ldy + row] = acc;
+}
+
+
+// ds4_kernels.cu:1353 route_publish_kernel - the MoE router: sc = sqrt(softplus(logit)), bs = sc + bias, then k rounds of a
+// block-wide argmax (a strided max, a warp reduction, a 16-warp pass), the picks' sc summed into the weights, and each pick's
+// device address published to plan_ptr. The prefetch requests (req) are NOT published here yet: that is the schedule, not the
+// answer, and it can be turned on separately.
+kernel void ds4_route_publish(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],
+ constant float& route_scale [[buffer(4)]], device const ulong* vram [[buffer(5)]],
+ device int* idx [[buffer(6)]], device float* w [[buffer(7)]],
+ device ulong* plan_ptr [[buffer(8)]], device const float* bias_vl [[buffer(9)]],
+ constant uint& vl [[buffer(10)]], constant uint& has_vl [[buffer(11)]],
+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup float sc[1024];
+ threadgroup float bs[1024];
+ threadgroup float bv[16];
+ threadgroup int bi[16];
+ threadgroup int pick[8];
+ const device float* lg = logits + (size_t) t * 1024u;
+ const device float* b = (has_vl != 0u && ((vl >> t) & 1u) != 0u) ? bias_vl : bias;
+ for (uint i = tid; i < n; i += 512u) {
+ const float l = lg[i];
+ const float sp = l > 20.0f ? l : log(1.0f + exp(l));
+ sc[i] = sqrt(sp);
+ bs[i] = sc[i] + b[i];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint lane = tid & 31u, wp = tid >> 5u;
+ for (uint r = 0; r < 8u; ++r) {
+ if (r >= kk) break;
+ float best = -3.402823466e+38f;
+ int bix = 0x7fffffff;
+ for (uint i = tid; i < n; i += 512u) {
+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) { best = bs[i]; bix = int(i); }
+ }
+ for (uint o = 16u; o > 0u; o >>= 1u) {
+ const float ov = simd_shuffle_xor(best, (ushort) o);
+ const int oi = simd_shuffle_xor(bix, (ushort) o);
+ if (ov > best || (ov == best && oi < bix)) { best = ov; bix = oi; }
+ }
+ if (lane == 0u && wp < 16u) { bv[wp] = best; bi[wp] = bix; }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) {
+ float bb = -3.402823466e+38f;
+ int ix = 0x7fffffff;
+ for (uint q = 0; q < 16u; ++q) {
+ if (bv[q] > bb || (bv[q] == bb && bi[q] < ix)) { bb = bv[q]; ix = bi[q]; }
+ }
+ pick[r] = ix;
+ bs[ix] = -3.402823466e+38f;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (tid == 0u) {
+ float sum = 0.0f;
+ for (uint r = 0; r < 8u; ++r) { if (r >= kk) break; sum += sc[pick[r]]; }
+ for (uint r = 0; r < 8u; ++r) {
+ if (r >= kk) break;
+ const uint p_ = uint(pick[r]);
+ const float wv = sc[p_] / (sum + 1e-20f) * route_scale;
+ idx[(size_t) t * 8u + r] = int(p_);
+ w[(size_t) t * 8u + r] = wv;
+ plan_ptr[(size_t) t * 8u + r] = vram[p_];
+ }
+ }
+}
+
+
+// ds4_kernels.cu swiglu_kernel - the expert MLP's activation: h = silu(g) * u, with the clamp when lim > 0
+kernel void ds4_swiglu(device const float* g [[buffer(0)]], device const float* u [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant float& lim [[buffer(3)]],
+ device float* h [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ if (gid >= n) return;
+ float gv = g[gid], uv = u[gid];
+ if (lim > 0.0f) { // both inputs clamped, then the gate itself
+ gv = fmin(gv, lim);
+ uv = fmin(fmax(uv, -lim), lim);
+ }
+ h[gid] = (gv / (1.0f + exp(-gv))) * uv; // silu(g) * u
+}
+
+kernel void ds4_ring_to_ext(device const uchar* ring [[buffer(0)]], device ushort* ext [[buffer(1)]],
+ constant long& p0 [[buffer(2)]], constant uint& n [[buffer(3)]],
+ constant uint& d [[buffer(4)]], constant ulong& rb [[buffer(5)]],
+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const long p = p0 - long(n) + long(r);
+ const device uchar* row = ring + (size_t) (p >= 0 ? (p % long(n)) : 0) * rb;
+ for (uint e = tid; e < d; e += 256u) {
+ ext[(size_t) r * d + e] = p >= 0 ? ds4_f2bf16(ds4_e4m3(uint(row[e])) * ds4_ue8m0(uint(row[d + e / 32u]))) : ushort(0);
+ }
+}
+
+// ---------------------------------------------------------------- ds4_gguf.cu:577 mx_combine_kernel
+//
+// A token's expert outputs summed in plan order (an expert off the device adds 0), the CPU lane's part added when its
+// flag is set, then the shared expert's - ds4_gguf.cu:169's combine(). Grid (n_embd / 128, T), 128 threads.
+struct Ds4Comb {
+ device const ulong* plan_ptr; // [T][8]: nonzero when the device holds that expert
+ device const float* plan_w; // [T][8]
+ device const float* part; // [entries][n_embd]
+ device const float* cpu_part; // [T][cpu_stride], or null
+ device const int* cpu_flag; // [T], or null
+ device const float* sh_out; // [T][n_embd], or null
+ device float* out;
+ long cpu_stride, out_stride;
+ int k, n_embd;
+};
+
+kernel void ds4_mx_combine(constant Ds4Comb& a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const uint r = tgp.x * 128u + tid;
+ if (int(r) >= a.n_embd) return;
+ const int t = int(tgp.y);
+ float m = 0.0f;
+ for (int i = 0; i < a.k; ++i) { // kPicks <= 8
+ const int e = t * 8 + i;
+ const float pi = a.plan_ptr[e] != 0ul ? a.part[(size_t) e * (size_t) a.n_embd + r] : 0.0f;
+ m = fma(a.plan_w[t * 8 + i], pi, m);
+ }
+ if (a.cpu_flag != nullptr && a.cpu_flag[t] != 0) m += a.cpu_part[(size_t) t * (size_t) a.cpu_stride + r];
+ m += a.sh_out != nullptr ? a.sh_out[(size_t) t * (size_t) a.n_embd + r] : 0.0f;
+ a.out[(size_t) t * (size_t) a.out_stride + r] = m;
+}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:1286 engram_dequant_kernel / :1311 engram_gate_kernel
+//
+// The Engram tables' row decoder and gate (this model's tables are fmt 2: IQ4_NL, an 18-byte block of 32 values whose
+// nibbles index ds4_kIq4nl above).
+kernel void ds4_engram_dequant_rows(device const uchar* rows [[buffer(0)]], constant uint& hd [[buffer(1)]],
+ device float* out [[buffer(2)]], constant int& fmt [[buffer(3)]],
+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ if (fmt == 2) {
+ const device uchar* row = rows + (size_t) r * (size_t) hd;
+ for (uint e = tid; e < hd; e += 256u) {
+ const device uchar* b = row + (size_t) (e / 32u) * 18u;
+ const uint q = b[2u + (e & 15u)];
+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);
+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);
+ out[(size_t) r * (size_t) hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);
+ }
+ return;
+ }
+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: NaN rather than a silent zero (as ds4_engram_dequant does)
+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * (size_t) hd + e] = NAN;
+}
+
+// ds4_kernels.cu's block_sum for a 1024-thread threadgroup: a warp sum a warp, then every warp sums the 32 partials (the
+// same result in all of them, so no warp diverges at a barrier)
+static inline float ds4_block_sum32(float v, threadgroup float* red, ushort lane, ushort w) {
+ v = ds4_warp_sum(v);
+ if (lane == 0) red[w] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float t = lane < 32u ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return t;
+}
+
+kernel void ds4_engram_gate(device float* h [[buffer(0)]], device const float* kv [[buffer(1)]],
+ device const float* w [[buffer(2)]], constant uint& hc [[buffer(3)]],
+ constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const uint k = tgp.x, t = tgp.y;
+ device float* hk = h + ((size_t) t * hc + (size_t) k) * (size_t) dim;
+ const device float* kvt = kv + (size_t) t * (size_t) (hc + 1u) * (size_t) dim;
+ const device float* key = kvt + (size_t) k * (size_t) dim;
+ const device float* val = kvt + (size_t) hc * (size_t) dim;
+ const device float* wk = w + (size_t) k * (size_t) dim;
+ float sh = 0.0f, sk = 0.0f, sd = 0.0f;
+ for (uint e = tid; e < dim; e += 1024u) {
+ const float hv = hk[e], kve = key[e];
+ sh += hv * hv;
+ sk += kve * kve;
+ sd += hv * wk[e] * kve;
+ }
+ sh = ds4_block_sum32(sh, red, lane, wg);
+ sk = ds4_block_sum32(sk, red, lane, wg);
+ sd = ds4_block_sum32(sd, red, lane, wg);
+ const float rstd = rsqrt(sh / float(dim) + eps) * rsqrt(sk / float(dim) + eps);
+ const float dot = sd * rstd * rsqrt(float(dim));
+ const float sq = copysign(sqrt(fmax(fabs(dot), 1e-6f)), dot);
+ const float gate = 1.0f / (1.0f + exp(-sq));
+ for (uint e = tid; e < dim; e += 1024u) hk[e] = hk[e] + gate * val[e];
+}
+
+// ds4_gguf.cu:72 bf16_mv_kernel<NC, true> - a BF16 matrix's matvec against F32 activations (the engram WKV here, and the
+// BF16 dense matrices the jobs' f == 0 names). Same frame as ds4_mx_mv above: a warp a row, a lane strides the row; here a
+// lane 8 BF16 weights (16 bytes = one uint4) and 8 activations, so the row is walked in 256-value steps.
+// A pair's F32 is bit-exact - the low half zeroed - which is bf16_mv_kernel's __uint_as_float(w << 16).
+kernel void ds4_bf16_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const device ushort* w = (const device ushort*) b.w + (size_t) row * b.ld;
+ thread float acc[16]; // (the CUDA template's largest NC; ncol cuts the rest)
+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
+ for (uint i = uint(lane) * 8u; i < uint(b.k); i += 256u) {
+ const uint4 q = *(const device uint4*) (w + i);
+ const float wf[8] = {as_type<float>(q.x << 16), as_type<float>(q.x & 0xffff0000u),
+ as_type<float>(q.y << 16), as_type<float>(q.y & 0xffff0000u),
+ as_type<float>(q.z << 16), as_type<float>(q.z & 0xffff0000u),
+ as_type<float>(q.w << 16), as_type<float>(q.w & 0xffff0000u)};
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device float4* xp = (const device float4*) (b.x + c * b.xcs + (size_t) i);
+ const float4 x0 = xp[0], x1 = xp[1];
+ float a = acc[c];
+ a = fma(wf[0], x0.x, a);
+ a = fma(wf[1], x0.y, a);
+ a = fma(wf[2], x0.z, a);
+ a = fma(wf[3], x0.w, a);
+ a = fma(wf[4], x1.x, a);
+ a = fma(wf[5], x1.y, a);
+ a = fma(wf[6], x1.z, a);
+ a = fma(wf[7], x1.w, a);
+ acc[c] = a;
+ }
+ }
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:524 compress_kernel / :582 index_score_kernel
+//
+// The compressor (ratio > 1 layers): each token's kv / score row into its slot, and every ratio'th token pooled by a
+// per-value softmax over the slots, scaled by a row RMS and the norm weights. Grid (1), 512 threads (d <= 1024 values).
+static inline float ds4_block_sum_n(float v, threadgroup float* red, ushort lane, ushort w, uint nw) {
+ v = ds4_warp_sum(v);
+ if (lane == 0) red[w] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float t = lane < nw ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t); // (every warp sums the same partials: no divergence at a barrier)
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return t;
+}
+
+kernel void ds4_compress(device const float* kv [[buffer(0)]], device const float* score [[buffer(1)]],
+ device float* kv_state [[buffer(2)]], device float* score_state [[buffer(3)]],
+ constant int& ratio [[buffer(4)]], constant int& p0 [[buffer(5)]], constant int& T [[buffer(6)]],
+ constant uint& d [[buffer(7)]], device const float* wn [[buffer(8)]],
+ constant float& eps [[buffer(9)]], device float* latent [[buffer(10)]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort wg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const float kfmax = -3.4028234663852886e+38f; // -FLT_MAX, as the CUDA writes it
+ int g = 0;
+ for (int t = 0; t < T; ++t) {
+ const int slot = (p0 + t) % ratio;
+ const bool pool = (p0 + t + 1) % ratio == 0;
+ for (uint e = tid; e < d; e += 512u) {
+ kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];
+ score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (!pool) continue;
+ float val[2] = {0.0f, 0.0f}; // (d <= 2 * 512)
+ float ss = 0.0f;
+ uint n = 0;
+ for (uint e = tid; e < d; e += 512u, ++n) {
+ float m = kfmax;
+ for (int s = 0; s < ratio; ++s) m = fmax(m, score_state[(size_t) s * d + e]);
+ float den = 0.0f;
+ for (int s = 0; s < ratio; ++s) den += exp(score_state[(size_t) s * d + e] - m);
+ float acc = 0.0f;
+ for (int s = 0; s < ratio; ++s)
+ acc += kv_state[(size_t) s * d + e] * (exp(score_state[(size_t) s * d + e] - m) / den);
+ val[n] = acc;
+ ss += acc * acc;
+ }
+ const float inv = rsqrt(ds4_block_sum_n(ss, red, lane, wg, 16u) / float(d) + eps);
+ n = 0;
+ for (uint e = tid; e < d; e += 512u, ++n) latent[(size_t) g * d + e] = val[n] * inv * wn[e];
+ ++g;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:582 index_score_kernel<128, KF>
+//
+// A thread a position: its key row (kfmt 3: 64 bytes of E2M1 codes (value 2i in the low nibble) then 4 UE8M0 scales; kfmt
+// 0: BF16) against every head's query (a threadgroup tile), relu, then the heads' weighted sum. Grid (ceil(n / 256), T).
+struct Ds4Rows { // ds4_kernels.cuh:115 Rows (kWin = 16)
+ int n[16];
+ int k[16];
+ int gen, R, kcap, ccap, blk;
+ long p0;
+};
+
+static inline void ds4_rows_at(constant Ds4Rows& r, int t, thread int& n, thread int& k) {
+ if (r.gen == 0) {
+ n = r.n[t];
+ k = r.k[t];
+ return;
+ }
+ const int nn = int((r.p0 + t + 1) / r.R);
+ const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);
+ switch (r.gen) {
+ case 1: n = nn; k = min(r.kcap, nn); break;
+ case 2: n = nn; k = cb; break;
+ case 3: n = cb * r.blk; k = min(r.kcap, nn); break;
+ default: n = (nn + r.blk - 1) / r.blk; k = r.kcap; break;
+ }
+}
+
+kernel void ds4_index_score(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],
+ device const uchar* kc [[buffer(2)]], device const int* cand [[buffer(3)]],
+ device float* out [[buffer(4)]], constant Ds4Rows& lim [[buffer(5)]],
+ constant int& heads [[buffer(6)]], constant int& n [[buffer(7)]],
+ constant int& cand_stride [[buffer(8)]], constant int& blk [[buffer(9)]],
+ constant long& out_stride [[buffer(10)]], constant int& kfmt [[buffer(11)]],
+ constant int& has_cand [[buffer(12)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ constexpr int HD = 128;
+ threadgroup float qs[32 * HD + 32];
+ const int t = int(tgp.y);
+ for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qs[i] = q[(size_t) t * (size_t) heads * HD + i];
+ for (uint i = tid; i < uint(heads); i += 256u) qs[uint(heads) * HD + i] = wq[(size_t) t * 64 + i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int i = int(tgp.x) * 256 + int(tid);
+ if (i >= n) return;
+ device float* o = out + (long) t * out_stride;
+ int ln, lk;
+ ds4_rows_at(lim, t, ln, lk);
+ int p = i;
+ if (has_cand != 0) {
+ if (i >= lk * blk) {
+ o[i] = -INFINITY;
+ return;
+ }
+ p = cand[(size_t) t * (size_t) cand_stride + (size_t) (i / blk)] * blk + i % blk;
+ }
+ if (p >= ln) {
+ o[i] = -INFINITY;
+ return;
+ }
+ const size_t kb = kfmt == 3 ? (size_t) ((HD / 2 + HD / 32 + 15) / 16 * 16) : (size_t) HD * 2;
+ const device uchar* kr = kc + (size_t) p * kb;
+ float acc[32];
+ for (uint h = 0; h < 32u; ++h) acc[h] = 0.0f;
+ for (int c = 0; c < HD; c += 8) {
+ float kvv[8];
+ if (kfmt == 3) { // 8 codes in 4 bytes, the group's power-of-two scale (2^(b - 127))
+ const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |
+ (uint(kr[c / 2 + 3]) << 24);
+ const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);
+ for (uint x = 0; x < 8u; ++x) kvv[x] = ds4_e2m1_val14(cw >> (4 * x)) * 16384.0f * ks;
+ } else {
+ const device ushort* b = (const device ushort*) (kr + (size_t) c * 2);
+ for (uint x = 0; x < 8u; ++x) kvv[x] = as_type<float>(uint(b[x]) << 16);
+ }
+ for (uint h = 0; h < 32u; ++h) {
+ if (int(h) >= heads) break;
+ threadgroup const float* qh = qs + h * HD + c;
+ for (uint x = 0; x < 8u; ++x) acc[h] += qh[x] * kvv[x];
+ }
+ }
+ float s = 0.0f;
+ for (uint h = 0; h < 32u; ++h) {
+ if (int(h) >= heads) break;
+ s += fmax(acc[h], 0.0f) * qs[uint(heads) * HD + h];
+ }
+ o[i] = s;
+}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:142 finish_kernel<true> (the KV pack)
+//
+// The same load, RMS and mrope as ds4_finish above, then the row written as the cache's packed codes (kv_row_bytes): the
+// quantiser's scale byte a group (32 values, or 16 for the 4-bit MXFP4-style rows), the codes themselves from bit 0 up -
+// for the 4-bit formats two values a byte, the second from the next lane. Written from the CUDA of the same name.
+
+
+
+static inline uint ds4_e2m1_code(float q) { // (bit 3 the sign, -0 kept; the magnitudes as 0 .. 7)
+ const float a = fabs(q);
+ const uint m = a <= 1.5f ? uint(a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;
+ return m | ((as_type<uint>(q) >> 31) << 3);
+}
+
+kernel void ds4_finish_pack(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {
+ threadgroup float red[8];
+ device float* v = x + (long) b * stride;
+ const uint nj = (len + 255u) / 256u;
+ float r[32];
+ float ss = 0.0f;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ r[j] = e < len ? v[e] : 0.0f;
+ ss += r[j] * r[j];
+ }
+ if (f.w != nullptr) {
+ const float tot = ds4_block_sum(ss, red, lane, w);
+ const float inv = rsqrt(tot / float(len) + f.eps);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ if (e < len) r[j] = r[j] * inv * f.w[e];
+ }
+ }
+ if (f.rd > 0) {
+ const uint r0 = len - uint(f.rd);
+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ const float other = simd_shuffle_xor(r[j], 1);
+ if (e >= r0 && e < len) {
+ const uint i = (e - r0) >> 1;
+ const float ang = float(pos) * f.freqs[i];
+ const float c = cos(ang);
+ const float sn = f.inverse ? -sin(ang) : sin(ang);
+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;
+ }
+ }
+ }
+ // the packed row: a group's scale and the group's codes as the format's (quant 1: d E4M3 codes at pr[e] and d/32
+ // scale bytes at pr[len + e/32]; 2: E2M1 pairs at pr[e/2] with E4M3 scales at pr[len/2 + e/16]; 3: E2M1 pairs with
+ // UE8M0 scales at pr[len/2 + e/32])
+ const uint g = f.quant == 2 ? 16u : 32u;
+ device uchar* pr = f.pdst + (long) b * f.pdst_stride;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ float a = e < len ? fabs(r[j]) : 0.0f;
+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));
+ float q = 0.0f, s = 1.0f;
+ if (e < len) {
+ if (f.quant == 1) {
+ s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));
+ q = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f));
+ } else if (f.quant == 2) {
+ s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);
+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));
+ } else {
+ s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));
+ }
+ r[j] = q * s;
+ }
+ if (f.quant == 1) {
+ if (e < len) pr[e] = uchar(ds4_e4m3_code(q, 0));
+ } else {
+ const uint c = ds4_e2m1_code(q);
+ const uint hi = simd_shuffle_down(c, 1); // (value e + 1: the next lane's)
+ if (e < len && (e & 1u) == 0u) pr[e >> 1] = uchar(c | (hi << 4));
+ }
+ if (e < len && e % g == 0u)
+ pr[(f.quant == 1 ? len : len / 2u) + e / g] =
+ uchar(f.quant == 2 ? ds4_e4m3_code(s, 0) : (as_type<uint>(s) >> 23));
+ }
+ device ushort* d = f.dst != nullptr ? f.dst + (long) b * f.dst_stride : nullptr;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ if (e >= len) continue;
+ if (d != nullptr) {
+ const uint bits = as_type<uint>(r[j]);
+ d[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even
+ }
+ }
+}
+
+// ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations (the indexer's wproj here): a warp a row
+// of the matrix, a lane 8 halves (16 bytes, the uint4 above) and 8 activations a step, kRowsT = 2 tokens a block. Grid
+// (ceil(n / 8), ceil(T / 2)), threadgroup 256.
+kernel void ds4_gemv_f16_rows(device const ushort* W [[buffer(0)]], device const float* x [[buffer(1)]],
+ constant ulong& ldx [[buffer(2)]], constant int& n [[buffer(3)]],
+ constant int& k [[buffer(4)]], constant int& T [[buffer(5)]],
+ constant float& alpha [[buffer(6)]], device float* y [[buffer(7)]],
+ constant ulong& ldy [[buffer(8)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ constexpr int kRowsT = 2;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ const int t0 = int(tgp.y) * kRowsT, nt = min(kRowsT, T - t0);
+ if (row >= n) return;
+ const device ushort* w = W + (size_t) row * (size_t) k;
+ float acc[kRowsT];
+ for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;
+ for (int c = int(lane) * 8; c < k; c += 32 * 8) {
+ const uint4 q = *(const device uint4*) (w + c);
+ const float2 f0 = float2(as_type<half2>(q.x)), f1 = float2(as_type<half2>(q.y));
+ const float2 f2 = float2(as_type<half2>(q.z)), f3 = float2(as_type<half2>(q.w));
+ const float2 fs[4] = {f0, f1, f2, f3};
+ float xv[kRowsT][8];
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) {
+ const device float4* xp = (const device float4*) (x + (size_t) (t0 + t) * ldx + (size_t) c);
+ const float4 a4 = xp[0], b4 = xp[1];
+ xv[t][0] = a4.x;
+ xv[t][1] = a4.y;
+ xv[t][2] = a4.z;
+ xv[t][3] = a4.w;
+ xv[t][4] = b4.x;
+ xv[t][5] = b4.y;
+ xv[t][6] = b4.z;
+ xv[t][7] = b4.w;
+ }
+ for (int i = 0; i < 4; ++i) {
+ const float2 f = fs[i];
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];
+ }
+ }
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) {
+ float a = acc[t];
+ a += simd_shuffle_xor(a, 16);
+ a += simd_shuffle_xor(a, 8);
+ a += simd_shuffle_xor(a, 4);
+ a += simd_shuffle_xor(a, 2);
+ a += simd_shuffle_xor(a, 1);
+ if (lane == 0u) y[(size_t) (t0 + t) * ldy + row] = alpha * a;
+ }
+}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:650 topk_kernel
+//
+// A token's k largest scores: four 8-bit radix passes over the order keys of the row (a threadgroup of 1024, shared
+// histograms), then the picks written in index order by a scan a pass, so the attention sums them in the same order every
+// run and the ties kept are the lowest positions.
+static inline uint ds4_ord_key(float f) {
+ const uint b = as_type<uint>(f);
+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);
+}
+
+kernel void ds4_topk(device const float* vals [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant Ds4Rows& r [[buffer(2)]], device const int* cand [[buffer(3)]],
+ constant int& cand_stride [[buffer(4)]], constant int& blk [[buffer(5)]],
+ device int* out [[buffer(6)]], constant int& out_stride [[buffer(7)]],
+ constant int& has_cand [[buffer(8)]], uint tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort wid [[simdgroup_index_in_threadgroup]]) {
+ threadgroup uint hist[256];
+ threadgroup uint prefix_s, mask_s;
+ threadgroup int remaining_s, cnt_gt, cnt_eq;
+ threadgroup int wgt[32], weq[32];
+ const int t = int(tgp);
+ int n, k;
+ ds4_rows_at(r, t, n, k);
+ device const float* v = vals + (long) t * stride;
+ device int* o = out + (size_t) t * (size_t) out_stride;
+ device const int* cd = has_cand != 0 ? cand + (size_t) t * (size_t) cand_stride : nullptr;
+ if (n <= 0) return;
+ if (k >= n) {
+ for (uint i = tid; i < uint(n); i += 1024u)
+ o[i] = cd != nullptr ? cd[i / uint(blk)] * blk + int(i % uint(blk)) : int(i);
+ return;
+ }
+ if (tid == 0u) {
+ prefix_s = 0u;
+ mask_s = 0u;
+ remaining_s = k;
+ cnt_gt = 0;
+ cnt_eq = 0;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int shift = 24; shift >= 0; shift -= 8) {
+ for (uint i = tid; i < 256u; i += 1024u) hist[i] = 0u;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint prefix = prefix_s, mask = mask_s;
+ for (int i = int(tid); i < n; i += 1024) {
+ const uint u = ds4_ord_key(v[i]);
+ if ((u & mask) == prefix) {
+ threadgroup atomic_uint* hp = (threadgroup atomic_uint*) &hist[(u >> shift) & 0xffu];
+ atomic_fetch_add_explicit(hp, 1u, memory_order_relaxed);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) {
+ const int rem = remaining_s;
+ int cum = 0;
+ for (int d = 255; d >= 0; --d) {
+ const int h = int(hist[d]);
+ if (cum + h >= rem) {
+ prefix_s = prefix | (uint(d) << shift);
+ mask_s = mask | (0xffu << shift);
+ remaining_s = rem - cum;
+ break;
+ }
+ cum += h;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const uint thr = prefix_s;
+ const int remaining = remaining_s;
+ const int n_gt = k - remaining;
+ for (int i0 = 0; i0 < n; i0 += 1024) {
+ const int i = i0 + int(tid);
+ const uint u = i < n ? ds4_ord_key(v[i]) : 0u;
+ const bool gt = i < n && u > thr, eq = i < n && u == thr;
+ const auto bgv = simd_ballot(gt);
+ const auto bev = simd_ballot(eq);
+ const uint bg = uint((simd_vote::vote_t)bgv), be = uint((simd_vote::vote_t)bev);
+ if (lane == 0u) {
+ wgt[wid] = int(popcount(bg));
+ weq[wid] = int(popcount(be));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wid == 0u) {
+ const int g0 = wgt[lane], e0 = weq[lane];
+ int gs = g0, es = e0;
+ for (uint o2 = 1u; o2 < 32u; o2 <<= 1) {
+ const int a = simd_shuffle_up(gs, ushort(o2)), b = simd_shuffle_up(es, ushort(o2));
+ if (uint(lane) >= o2) {
+ gs += a;
+ es += b;
+ }
+ }
+ wgt[lane] = cnt_gt + gs - g0;
+ weq[lane] = cnt_eq + es - e0;
+ simdgroup_barrier(mem_flags::mem_none);
+ if (lane == 31u) {
+ cnt_gt += gs;
+ cnt_eq += es;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ int slot = -1;
+ const uint below = (1u << lane) - 1u;
+ if (gt) slot = wgt[wid] + int(popcount(bg & below));
+ else if (eq) {
+ const int e = weq[wid] + int(popcount(be & below));
+ if (e < remaining) slot = n_gt + e;
+ }
+ if (slot >= 0) o[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+}
+
+// ds4_kernels.cu:751 block_max_kernel - a block of `blk` positions' largest score (the indexer's candidate blocks), the
+// newest position's block pinned in at +inf. Grid (ceil(nb_max / 256), T), 256 threads.
+kernel void ds4_block_max(device const float* score [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant Ds4Rows& r [[buffer(2)]], constant int& blk [[buffer(3)]],
+ device float* bm [[buffer(4)]], constant long& bm_stride [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const int b = int(tgp.x) * 256 + int(tid), t = int(tgp.y);
+ int n, k_;
+ ds4_rows_at(r, t, n, k_);
+ const int nb = (n + blk - 1) / blk;
+ if (b >= nb) return;
+ float m = -INFINITY;
+ for (int j = 0; j < blk; ++j) {
+ const int p = b * blk + j;
+ if (p < n) m = fmax(m, score[(long) t * stride + p]);
+ }
+ if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in
+ bm[(long) t * bm_stride + b] = m;
+}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:1141 hc_mean_kernel / :1150 iota_rows_kernel
+//
+// The two small shapes the decode loop asks for: a token's mean over its hc streams, and a row's indices.
+kernel void ds4_hc_mean(device const float* h [[buffer(0)]], constant int& hc [[buffer(1)]],
+ constant int& dim [[buffer(2)]], device float* out [[buffer(3)]],
+ constant long& out_stride [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const int e = int(tgp.x) * 256 + int(tid), t = int(tgp.y);
+ if (e >= dim) return;
+ float a = 0.0f;
+ for (int s = 0; s < hc; ++s) a += h[((size_t) t * (size_t) hc + (size_t) s) * (size_t) dim + (size_t) e];
+ out[(long) t * out_stride + e] = a / float(hc);
+}
+
+kernel void ds4_iota_rows(device int* out [[buffer(0)]], constant int& n [[buffer(1)]], constant int& stride [[buffer(2)]],
+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint t = tg;
+ for (uint i = tid; i < uint(n); i += 64u) out[(size_t) t * (size_t) stride + i] = int(i);
+}
+
+// ---------------------------------------------------------------- vendor/sampler.cu:93 sampler_greedy_kernel
+//
+// The plain argmax over the vocabulary, ONE BLOCK (1024) a token: a thread's strided candidates with the repetition
+// penalties applied (a threadgroup bitmap of the history window says which tokens can be penalised at all), a warp
+// reduction, then warp 0's - a tie keeps the larger value and the SMALLER index, so this is the serial scan's answer.
+kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constant int& n_vocab [[buffer(1)]],
+ device const int* history [[buffer(2)]], constant int& history_len [[buffer(3)]],
+ constant int& plen [[buffer(4)]], constant float& pen_repeat [[buffer(5)]],
+ constant float& pen_freq [[buffer(6)]], constant float& pen_present [[buffer(7)]],
+ device const float* logit_bias [[buffer(8)]], constant int& has_bias [[buffer(9)]],
+ device int* out [[buffer(10)]], uint tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort wid [[simdgroup_index_in_threadgroup]]) {
+ threadgroup uint penal_bits[4096]; // (n_vocab + 31) / 32 words (the host checks it fits)
+ threadgroup float sv[32];
+ threadgroup int si[32];
+ const int t = int(tg);
+ device const float* l = logits + (size_t) t * (size_t) n_vocab;
+ device const int* hrow = history != nullptr ? history + (size_t) t * (size_t) history_len : nullptr;
+ int hlen = 0;
+ if (hrow != nullptr) {
+ hlen = min(plen, history_len);
+ if (hlen < 0) hlen = 0;
+ hrow += history_len - hlen; // the window is the tail
+ }
+ const int bits_words = (n_vocab + 31) / 32;
+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+ if (use_bits) {
+ for (int w = int(tid); w < bits_words; w += 1024) penal_bits[w] = 0u;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int i = int(tid); i < hlen; i += 1024)
+ if (hrow[i] >= 0 && hrow[i] < n_vocab) {
+ threadgroup atomic_uint* pb = (threadgroup atomic_uint*) &penal_bits[hrow[i] >> 5];
+ atomic_fetch_or_explicit(pb, 1u << (hrow[i] & 31), memory_order_relaxed);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ float bv = -INFINITY;
+ int best = n_vocab; // (the "no candidate" index: it loses every comparison)
+ for (int v = int(tid); v < n_vocab; v += 1024) {
+ float s = has_bias != 0 ? l[v] + logit_bias[v] : l[v];
+ int cnt = 0;
+ if (use_bits && (penal_bits[v >> 5] & (1u << (v & 31))) != 0u)
+ for (int i = 0; i < hlen; ++i)
+ if (hrow[i] == v) ++cnt;
+ if (cnt > 0) {
+ if (s <= 0.0f) s *= pen_repeat;
+ else s /= pen_repeat;
+ s -= float(cnt) * pen_freq + 1.0f * pen_present;
+ }
+ if (s > bv) {
+ bv = s;
+ best = v;
+ }
+ }
+ for (uint o = 16u; o > 0u; o >>= 1) {
+ const float ov = simd_shuffle_down(bv, ushort(o));
+ const int oi = simd_shuffle_down(best, ushort(o));
+ if (ov > bv || (ov == bv && oi < best)) {
+ bv = ov;
+ best = oi;
+ }
+ }
+ if (lane == 0u) {
+ sv[wid] = bv;
+ si[wid] = best;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wid == 0u) {
+ float wv = sv[lane];
+ int wi = si[lane];
+ for (uint o = 16u; o > 0u; o >>= 1) {
+ const float ov = simd_shuffle_down(wv, ushort(o));
+ const int oi = simd_shuffle_down(wi, ushort(o));
+ if (ov > wv || (ov == wv && oi < wi)) {
+ wv = ov;
+ wi = oi;
+ }
+ }
+ if (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)
+ }
+}
diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metal
new file mode 100644
index 000000000..7e9fb5aa5
--- /dev/null
+++ b/metal/src/engine/msl/ds4_iq.metal
@@ -0,0 +1,361 @@
+#include <metal_stdlib>
+using namespace metal;
+// ds4_iq.cu:809 mv_kernel - the I-quant expert matvec. Same frame as the MXFP4/Q8_0 ones (a warp a row, a lane strides the
+// 256-value blocks, eight accumulator columns for the token window), with the block decoder chosen per job by b.type:
+// 16 IQ2_XXS (66 B), 17 IQ2_XS (74 B), 22 IQ2_S (82 B). The activations are the quantiser's int8 (xq) with their scales (xd),
+// so each sub-group's dot is integer and the two scales fold in at the end - the same arithmetic as the float probes, at a
+// quarter of the traffic.
+struct Ds4MvJob {
+ device const uchar* w;
+ ulong ld;
+ int type, k, n, ncol;
+ device const char* xq;
+ device const float* xd;
+ device float* y;
+ ulong ycs;
+ device const float* bias;
+ float alpha;
+};
+
+kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nsb = uint(b.k) / 32u;
+ const uint nblk = uint(b.k) / 256u;
+ const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : 82u);
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ float acc[8];
+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
+ for (uint blk = lane; blk < nblk; blk += 32u) {
+ const device uchar* p = w + (size_t) blk * bs;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ for (uint ib = 0; ib < 8u; ++ib) {
+ float db0, db1;
+ uint idx[4];
+ uint sg[4];
+ if (b.type == 16) {
+ const uint aux0 = uint(p[2 + 4 * ib]) | (uint(p[3 + 4 * ib]) << 8) | (uint(p[4 + 4 * ib]) << 16) |
+ (uint(p[5 + 4 * ib]) << 24);
+ const uint aux1 = uint(p[6 + 4 * ib]) | (uint(p[7 + 4 * ib]) << 8) | (uint(p[8 + 4 * ib]) << 16) |
+ (uint(p[9 + 4 * ib]) << 24);
+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ db0 = db1 = db;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = (aux0 >> (8u * l)) & 0xffu;
+ sg[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ }
+ } else if (b.type == 17) {
+ const device ushort* qs = (const device ushort*) (p + 2);
+ const device uchar* sc = p + 66;
+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[4u * ib + l]);
+ idx[l] = q & 511u;
+ sg[l] = 0xffffffffu ^ ((q >> 9) * 0x00204081u); // the 7 sign bits, expanded (ggml's sign trick)
+ }
+ } else {
+ const device uchar* qs = p + 2;
+ const device uchar* sgb = qs + 32;
+ const device uchar* qh = p + 66;
+ const device uchar* sc = p + 74;
+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = uint(qs[4u * ib + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);
+ sg[l] = uint(sgb[4u * ib + l]);
+ }
+ }
+ for (uint l = 0; l < 4u; ++l) {
+ ulong g;
+ if (b.type == 16) g = iq2xxs_grid[idx[l]];
+ else if (b.type == 17) g = iq2xs_grid[idx[l]];
+ else g = iq2s_grid[idx[l] & 1023u];
+ const float db = (l < 2u) ? db0 : db1;
+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sum = 0;
+ for (uint j = 0; j < 8u; ++j) {
+ const int code = int((g >> (8u * j)) & 0xffu);
+ const bool neg = b.type == 16 ? ((sg[l] & uint(kmask_iq2xs[j])) != 0u)
+ : ((sg[l] & (1u << j)) != 0u);
+ sum += (neg ? -code : code) * int(xq[j]);
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db, float(sum), acc[c]);
+ }
+ }
+ }
+ }
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// ---------------------------------------------------------------- ds4_iq.cu:536 gate_up_kernel / :588 down_kernel
+//
+// The prompt / window path's expert GEMM (ds4_iq.cu's launch_gu / launch_dn): a threadgroup of 256 = 8 warps, a warp
+// RW rows of the gate and of the up (down: RW rows of the down), a lane 32 / RW sub-blocks of the row, and the group's
+// entries NE at a pass so that a sub-block the weights decode once serves them all. Grid (rows / (8 RW), groups).
+//
+// The group's blob and its entries are the CUDA kernel's `group` (ds4_iq.cu:512): a plan group (grp_ptr null) is one
+// expert whose entry is the group's own index and whose activation row is 0, a window group's entries come from
+// grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.
+struct Ds4IqJob {
+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)
+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob
+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)
+ device const int* grp_ent; // the entries (token * 8 + pick)
+ device const int* n_groups;
+ device const char* xq; // the int8 activations of a row (n_embd an entry)
+ device const float* xd; // their scales (n_embd / 32 a row)
+ device const char* hq; // down's int8 input (n_ff an entry)
+ device const float* hd;
+ device float* h; // [entries][n_ff]: the clamped swiglu
+ device float* part; // [entries][n_embd]: each entry's down output
+ long tr0, tr1, tr2; // the roles' offsets in a blob
+ int n_embd, n_ff;
+ int type; // 16 IQ2_XXS, 17 IQ2_XS, 22 IQ2_S (ds4_iq.cuh's kIQ2_*)
+ float limit; // swiglu_limit (0: none)
+};
+
+// a 32-value sub-block as 32 int8 codes (8 packed ints, values 4 w .. 4 w + 3 in word w) and the two halves' scales
+// (ggml's d (2 s + 1) / 8, the .cu's Sub.d * Sub.s0 in one F32: the codes of the second half are scaled by s1)
+struct Ds4IqSub {
+ int q[8];
+ float s0, s1;
+};
+
+// the block bytes of the types these kernels take: 16 IQ2_XXS, 17 IQ2_XS, 18 IQ3_XXS, 22 IQ2_S (ds4_iq.cu's Ty<T>::B)
+static inline uint ds4_iq_bs(int type) {
+ return type == 16 ? 66u : type == 17 ? 74u : type == 18 ? 98u : 82u;
+}
+
+// ds4_iq.cu:139 decode<T>: the block's 8 sub-blocks, kb's is the (kb & 7)'th; the grids are ggml-common.h's
+static inline Ds4IqSub ds4_iq_sub(device const uchar* row, uint bs, int kb, int type) {
+ Ds4IqSub s;
+ const int ib = kb & 7;
+ const device uchar* p = row + (size_t) (kb >> 3) * bs;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ uint idx[4], sm[4];
+ float db0, db1;
+ if (type == 18) { // IQ3_XXS: d, qs[64] grid bytes (4 a sub-block), scales[8] (the top nibble the group's s)
+ const device uchar* qs = p + 2;
+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) |
+ (uint(qs[8 * ib + 3]) << 24);
+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) |
+ (uint(qs[8 * ib + 7]) << 24);
+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) |
+ (uint(qs[67 + 4 * ib]) << 24);
+ for (uint l = 0; l < 4u; ++l) {
+ const uint w = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sg = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(w >> sh) & 255u], g1 = iq3xxs_grid[(w >> (sh + 8u)) & 255u];
+ int a0 = 0, a1 = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);
+ if ((sg & (1u << j)) != 0u) v0 = -v0;
+ if ((sg & (1u << (j + 4u))) != 0u) v1 = -v1;
+ a0 |= (v0 & 0xff) << int(8u * j);
+ a1 |= (v1 & 0xff) << int(8u * j);
+ }
+ s.q[2u * l] = a0;
+ s.q[2u * l + 1u] = a1;
+ }
+ const float ds = d * (0.5f + float(aux >> 28)) * 0.5f; // ggml's db: d (0.5 + s) 0.5
+ s.s0 = ds;
+ s.s1 = ds;
+ return s;
+ }
+ if (type == 16) { // d, qs[8] i16 (4 grid bytes, then 4 sign/scale bytes), scales in the top nibbles
+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |
+ (uint(p[5 + 8 * ib]) << 24);
+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |
+ (uint(p[9 + 8 * ib]) << 24);
+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ db0 = db1 = db;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = (aux0 >> (8u * l)) & 0xffu;
+ sm[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ }
+ } else if (type == 17) { // d, qs[16] i16 (9-bit grid, 7 sign bits), scales[8] (two 4-bit)
+ const device ushort* qs = (const device ushort*) (p + 2);
+ const device uchar* sc = p + 66;
+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[4u * uint(ib) + l]);
+ idx[l] = q & 511u;
+ sm[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]);
+ }
+ } else { // IQ2_S: d, qs[32] (8-bit grid), signs[32], qh[8] (2 bits more), scales[8]
+ const device uchar* qs = p + 2;
+ const device uchar* sgb = qs + 32;
+ const device uchar* qh = p + 66;
+ const device uchar* sc = p + 74;
+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = uint(qs[4u * uint(ib) + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);
+ sm[l] = uint(sgb[4u * uint(ib) + l]);
+ }
+ }
+ for (uint l = 0; l < 4u; ++l) {
+ const ulong g = type == 16 ? iq2xxs_grid[idx[l]] : (type == 17 ? iq2xs_grid[idx[l]] : iq2s_grid[idx[l] & 1023u]);
+ int w0 = 0, w1 = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ int v0 = int((g >> (8u * j)) & 0xffu), v1 = int((g >> (8u * (j + 4u))) & 0xffu);
+ if ((sm[l] & (1u << j)) != 0u) v0 = -v0;
+ if ((sm[l] & (1u << (j + 4u))) != 0u) v1 = -v1;
+ w0 |= (v0 & 0xff) << int(8u * j);
+ w1 |= (v1 & 0xff) << int(8u * j);
+ }
+ s.q[2u * l] = w0;
+ s.q[2u * l + 1u] = w1;
+ }
+ s.s0 = db0;
+ s.s1 = db1;
+ return s;
+}
+
+// 4 int8 codes against the 4 int8 activations packed in x (ds4_iq.cu:350's __dp4a: Metal has no such instruction)
+static inline int ds4_iq_dot4(int q, int x) {
+ const char4 a = as_type<char4>(q), b = as_type<char4>(x);
+ return int(a.x) * int(b.x) + int(a.y) * int(b.y) + int(a.z) * int(b.z) + int(a.w) * int(b.w);
+}
+
+// ds4_iq.cu:349 sub_dot: the sub-block against its 32 int8 activations (8 packed ints), the scales in F32
+static inline float ds4_iq_dot(Ds4IqSub s, device const int* xp) {
+ const int lo = ds4_iq_dot4(s.q[0], xp[0]) + ds4_iq_dot4(s.q[1], xp[1]) + ds4_iq_dot4(s.q[2], xp[2]) +
+ ds4_iq_dot4(s.q[3], xp[3]);
+ const int hi = ds4_iq_dot4(s.q[4], xp[4]) + ds4_iq_dot4(s.q[5], xp[5]) + ds4_iq_dot4(s.q[6], xp[6]) +
+ ds4_iq_dot4(s.q[7], xp[7]);
+ return s.s0 * float(lo) + s.s1 * float(hi);
+}
+
+// ds4_iq.cu:502 lanes_sum: the row's sum over the 32 / RW lanes that share it
+template <int RW>
+static inline float ds4_iq_lsum(float v) {
+ for (int o = 16 / RW; o > 0; o >>= 1) v += simd_shuffle_xor(v, ushort(o));
+ return v;
+}
+
+// ds4_iq.cu:497 swiglu: the limit's clamps, then g / (1 + e^-g) * u
+static inline float ds4_iq_swiglu(float gv, float uv, float limit) {
+ if (limit > 0.0f) {
+ gv = min(gv, limit);
+ uv = min(max(uv, -limit), limit);
+ }
+ return (gv / (1.0f + exp(-gv))) * uv;
+}
+
+// the group's blob and its entry range (ds4_iq.cu:512 group); false: the threadgroup has nothing to do
+static inline bool ds4_iq_group(constant Ds4IqJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {
+ if (j.grp_ptr == nullptr) {
+ const ulong p = j.plan_ptr[g];
+ if (p == 0ul) return false;
+ blob = p;
+ m0 = int(g);
+ m1 = m0 + 1;
+ return true;
+ }
+ if (int(g) >= *j.n_groups) return false;
+ blob = j.grp_ptr[g];
+ m0 = j.grp_start[g];
+ m1 = j.grp_start[g + 1];
+ return true;
+}
+
+// gate_up_kernel<T, NE> (kGuRW = 1: a warp a row, a lane a sub-block): h = the clamped swiglu of the gate and the up
+kernel void ds4_iq_gu(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ if (row >= j.n_ff) return;
+ const uint bs = ds4_iq_bs(j.type);
+ const uint nsb = uint(j.n_embd) / 32u;
+ const size_t rb = (size_t) (j.n_embd / 256) * (size_t) bs;
+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;
+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 4) {
+ const int ne = min(4, m1 - mm);
+ int es[4], xr[4];
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ es[i] = e;
+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);
+ }
+ }
+ float ag[4] = {0.0f, 0.0f, 0.0f, 0.0f}, au[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ for (uint kb = uint(lane); kb < nsb; kb += 32u) {
+ const Ds4IqSub sg = ds4_iq_sub(g, bs, int(kb), j.type);
+ const Ds4IqSub su = ds4_iq_sub(u, bs, int(kb), j.type);
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const device int* xp = (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd +
+ (size_t) kb * 32u);
+ const float dx = j.xd[(size_t) xr[i] * nsb + kb];
+ ag[i] += dx * ds4_iq_dot(sg, xp);
+ au[i] += dx * ds4_iq_dot(su, xp);
+ }
+ }
+ }
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);
+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);
+ }
+ }
+ }
+}
+
+// down_kernel<T, NE> (kDnRW = 2: a warp two rows, 16 lanes a row): part = the down against the int8 h
+kernel void ds4_iq_dn(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = (int(tgp.x) * 8 + int(tid >> 5)) * 2 + int(lane) / 16;
+ if (row >= j.n_embd) return;
+ const uint kl = uint(lane) % 16u;
+ const uint bs = ds4_iq_bs(j.type);
+ const uint nsb = uint(j.n_ff) / 32u;
+ const size_t rb = (size_t) (j.n_ff / 256) * (size_t) bs;
+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 4) {
+ const int ne = min(4, m1 - mm);
+ int es[4];
+ for (int i = 0; i < 4; ++i)
+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ float ad[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ for (uint kb = kl; kb < nsb; kb += 16u) {
+ const Ds4IqSub sd = ds4_iq_sub(w, bs, int(kb), j.type);
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const device int* hp = (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff +
+ (size_t) kb * 32u);
+ ad[i] += j.hd[(size_t) es[i] * nsb + kb] * ds4_iq_dot(sd, hp);
+ }
+ }
+ }
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const float v = ds4_iq_lsum<2>(ad[i]);
+ if (kl == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;
+ }
+ }
+ }
+}
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index a72bcf7c8..02fadda68 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -2,6 +2,9 @@
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).
#include "ds4_gguf.cuh"
+#if defined(STRATA_METAL)
+#include <ds4_metal_host.h>
+#endif
#include "ds4_iq.cuh"
#include "ds4_numerics.cuh"
@@ -198,6 +201,9 @@ __global__ void __launch_bounds__(kThreads) q8_quant_kernel(const float* __restr
}
void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {
const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);
+#if defined(STRATA_METAL)
+ if (::strata::dsv41::metal_host::q8_quant_rows(x, ldx, rows, k, (int8_t*) q, d)) return;
+#endif
strata_ds4_launch(q8_quant_kernel, grid, kThreads, 0, s, x, ldx, k, q, d);
}
@@ -1040,12 +1046,26 @@ template <int NC>
void mv_launch(int f, const BfJobs& J, int nj, int max_n, bool vec, cudaStream_t s) {
const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);
if (f == 1) {
+#if defined(STRATA_METAL)
+ // the MXFP4 matvec: the engine's jobs go across as they are (same layout on both sides)
+ if (max_n > 0 && nj > 0 &&
+ ::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 1)) return;
+#endif
if (vec) strata_ds4_launch(mx_mv_kernel<NC, true, false>, grid, kThreads, 0, s, J);
else strata_ds4_launch(mx_mv_kernel<NC, false, false>, grid, kThreads, 0, s, J);
} else if (f == 2) { // (vec here: every row 4-byte aligned, so staged)
+#if defined(STRATA_METAL)
+ if (max_n > 0 && nj > 0 &&
+ ::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 2)) return;
+#endif
if (vec) strata_ds4_launch(q8_mv_kernel<NC, true>, grid, kThreads, 0, s, J);
else strata_ds4_launch(q8_mv_kernel<NC, false>, grid, kThreads, 0, s, J);
} else {
+#if defined(STRATA_METAL)
+ // the BF16 matvec (vec: the rows and activations 16-byte aligned, so the 8-value step is safe)
+ if (vec && max_n > 0 && nj > 0 &&
+ ::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 0)) return;
+#endif
if (vec) strata_ds4_launch(bf16_mv_kernel<NC, true>, grid, kThreads, 0, s, J);
else strata_ds4_launch(bf16_mv_kernel<NC, false>, grid, kThreads, 0, s, J);
}
@@ -1265,6 +1285,10 @@ void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float*
c.out_stride = (size_t) a.n_embd;
c.k = a.k;
c.n_embd = a.n_embd;
+#if defined(STRATA_METAL)
+ if (!::strata::dsv41::metal_host::mx_combine(c.plan_ptr, c.plan_w, c.part, c.cpu_part, c.cpu_stride,
+ c.cpu_flag, c.sh_out, c.out, c.out_stride, 1, c.k, c.n_embd))
+#endif
strata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s, c);
check("moe_down");
}
@@ -1312,7 +1336,10 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {
q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)
if (mx) launch_groups<DownK>(m, a.n_embd, groups, a.T, s);
else iq::down(ia, groups, a.T, s);
+#if !defined(STRATA_METAL)
+ // (this port's dispatches are synchronous and nothing is published to wait for: ds4_kernels.cu's wait_seq)
if (a.wait_ptr != nullptr) strata_ds4_launch(mx_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);
+#endif
CombArgs c{};
c.plan_ptr = a.plan_ptr;
c.plan_w = a.plan_w;
@@ -1325,6 +1352,11 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {
c.out_stride = a.out_stride;
c.k = a.k;
c.n_embd = a.n_embd;
+#if defined(STRATA_METAL)
+ // mx_combine_kernel on the Metal dispatch; the launch below is the fallback
+ if (!::strata::dsv41::metal_host::mx_combine(c.plan_ptr, c.plan_w, c.part, c.cpu_part, c.cpu_stride,
+ c.cpu_flag, c.sh_out, c.out, c.out_stride, a.T, c.k, c.n_embd))
+#endif
strata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s, c);
check("win_experts");
return true;
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index 1baa84e5d..dfe7503ad 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -3,6 +3,9 @@
// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are
// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.
#include "ds4_iq.cuh"
+#if defined(STRATA_METAL)
+#include <ds4_metal_host.h>
+#endif
#include "ds4_numerics.cuh"
#include <cuda_fp16.h>
@@ -1179,6 +1182,12 @@ const char* name(int t) {
}
bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // gate_up_kernel on the Metal dispatch (the prompt / window path's expert GEMM); the launch below is the fallback
+ if (::strata::dsv41::metal_host::iq_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,
+ a.n_ff, a.limit, a.xq, a.xd, a.h, a.tg, groups))
+ return true;
+#endif
if (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);
if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;
const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);
@@ -1189,6 +1198,12 @@ bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
}
bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // down_kernel on the Metal dispatch; the launch below is the fallback
+ if (::strata::dsv41::metal_host::iq_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,
+ a.n_ff, a.hq, a.hd, a.part, a.td, groups))
+ return true;
+#endif
if (is_tq(a.td)) return tq::down(a, groups, ne_max, s);
if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;
const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);
@@ -1240,6 +1255,9 @@ bool set_down(const SetArgs& a, cudaStream_t s) {
}
bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (n > 0 && n <= 64 && ::strata::dsv41::metal_host::iq_mv((const void*) jobs, n)) return true;
+#endif
{ // the trellis jobs go to their kernels, the others on
MvJob tqj[64], oth[64];
int nt = 0, no = 0;
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 21343c8fd..c89c9507c 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1826,15 +1826,36 @@ void route_publish(const float* logits, const float* bias, int n, int k, int T,
const unsigned long long* vram, const float* x, int dim, int* idx, float* w,
unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl,
unsigned vl, int na0, int na1) {
+#if defined(STRATA_METAL)
+ // selection only: the prefetch schedule (req) is published by the CUDA kernel, and is turned on separately here
+ if (strata::dsv41::metal_host::route_publish(logits, bias, n, k, T, route_scale, vram, idx, w, plan_ptr,
+ vl != 0 ? bias_vl : nullptr, vl)) {
+ // no prefetcher on the Mac path, so publish_kernel has no consumer: the selection is the whole job here
+ (void) req; (void) seq; (void) na0; (void) na1; (void) x; (void) dim;
+ return;
+ }
+#endif
strata_ds4_launch(route_publish_kernel, T, 512, 0, s, logits, bias, n, k, route_scale, vram, x, dim, idx, w, plan_ptr, req,
vl != 0 ? bias_vl : nullptr, vl);
if (req != nullptr) strata_ds4_launch(publish_kernel, 1, 1, 0, s, req, T, seq, na0, na1);
}
void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // no prefetch schedule on the Mac path yet: the router publishes no requests, so there is nothing to look ahead for
+ (void) logits; (void) bias; (void) n; (void) m; (void) T; (void) req; (void) off; (void) s;
+ return;
+#endif
if (req != nullptr && m > 0 && T > 0 && n <= 1024 && off + m <= kAhead)
strata_ds4_launch(route_ahead_kernel, T, 512, 0, s, logits, bias, n, m, req, off);
}
-void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { strata_ds4_launch(wait_seq_kernel, 1, 32, 0, s, seq_ptr, seq); }
+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // nothing is published, so nothing is waited for: the CUDA path spins until the host's prefetcher acknowledges
+ (void) seq_ptr; (void) seq; (void) s;
+ return;
+#endif
+ strata_ds4_launch(wait_seq_kernel, 1, 32, 0, s, seq_ptr, seq);
+}
// the RAM store's experts for the device (a decode token, window of one): a pick the VRAM table left at 0 whose expert the store holds
// (zc_tab[expert] = the slot's device address over PCIe, 0: not) is copied to staging slot i with wide coalesced loads (the PCIe
@@ -1887,6 +1908,10 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cud
std::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);
return;
}
+#if defined(STRATA_METAL)
+ const auto& mp = *reinterpret_cast<const strata::dsv41::metal_host::FinishIn*>(&f);
+ if (strata::dsv41::metal_host::finish(x, n_vec, len, stride, mp)) return; // finish_kernel<true> (the KV pack)
+#endif
strata_ds4_launch(finish_kernel<true>, n_vec, kThreads, 0, s, x, len, stride, f);
} else {
#if defined(STRATA_METAL)
@@ -1912,10 +1937,16 @@ void hc_split(const float* mixes, const float* scale, const float* base, int hc,
}
void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,
cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::hc_pre_norm(h, mix, w, dim, hc, T, eps, x)) return;
+#endif
strata_ds4_launch(hc_pre_norm_kernel, T, 1024, 0, s, h, mix, w, dim, hc, eps, x);
}
void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,
float* out, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::hc_post_v(y, res, post, comb, dim, hc, T, out)) return;
+#endif
strata_ds4_launch(hc_post_kernel, dim3(cdiv(dim * hc, kThreads), T), kThreads, 0, s, y, res, post, comb, dim, hc, out);
}
@@ -1933,6 +1964,9 @@ void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float*
strata_ds4_launch(hc_mix_scale_kernel, T, kThreads, 0, s, h, hcdim, n_mix, eps, mixes);
}
void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int fmt) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::engram_dequant_tok(src, per, cols, T, hd, out, fmt)) return;
+#endif
if (T > 0) strata_ds4_launch(engram_dequant_tok_kernel, T * cols, kThreads, 0, s, src, per, cols, hd, out, fmt);
}
void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
@@ -1946,7 +1980,12 @@ void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds
}
void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,
cudaStream_t s) {
- if (r1 > r0 && T > 0) strata_ds4_launch(moe_combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, acc, y, pos, w, k, r0, r1, dim);
+ if (r1 > r0 && T > 0) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::moe_combine(acc, y, pos, w, T, k, r0, r1, dim)) return;
+#endif
+ strata_ds4_launch(moe_combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, acc, y, pos, w, k, r0, r1, dim);
+ }
}
void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s) {
@@ -1954,12 +1993,19 @@ void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alph
}
void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,
cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // gemv_f16_rows_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::gemv_f16_rows(W, x, ldx, n, k, T, alpha, y, ldy)) return;
+#endif
if (T > 0)
strata_ds4_launch(gemv_f16_rows_kernel, dim3(cdiv(n, kThreads / 32), cdiv(T, kRowsT)), kThreads, 0, s, W, x, ldx, n, k, T, alpha,
y, ldy);
}
void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (T > 0 && ::strata::dsv41::metal_host::gemv_f32(W, x, n, k, T, y, ldy)) return;
+#endif
if (T > 0) strata_ds4_launch(gemv_f32_kernel, dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s, W, x, n, k, y, ldy);
}
@@ -1971,11 +2017,20 @@ void route(const float* logits, const float* bias, int n, int k, int T, float ro
}
void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // declared here rather than in the header: the header carries the other entry points, this one is only used here
+ namespace mh = strata::dsv41::metal_host;
+ if (n > 0 && mh::swiglu(g, u, n, lim, h)) return;
+#endif
if (n > 0) strata_ds4_launch(swiglu_kernel, cdiv(n, kThreads), kThreads, 0, s, g, u, n, lim, h);
}
void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
int d, const float* w, float eps, float* latent, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // compress_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::compress_win(kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent)) return;
+#endif
strata_ds4_launch(compress_kernel, 1, 512, 0, s, kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent);
}
@@ -1987,6 +2042,12 @@ void index_score(const float* q, const float* w, const void* k, int heads, int h
return;
}
if (n <= 0) return;
+#if defined(STRATA_METAL)
+ // index_score_kernel<128, kfmt> on the Metal dispatch; the launches below are the fallback
+ if (strata::dsv41::metal_host::index_score(q, w, (const uint8_t*) k, heads, hd, T, n, cand, cand_stride, blk, &lim,
+ out, out_stride, kfmt))
+ return;
+#endif
const size_t smem = (size_t) (heads * hd + heads) * sizeof(float);
const dim3 grid(cdiv(n, kThreads), T);
if (kfmt == kKvFp4p)
@@ -1999,6 +2060,10 @@ void index_score(const float* q, const float* w, const void* k, int heads, int h
void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,
int* out, int out_stride, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // topk_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::topk(vals, stride, T, &r, cand, cand_stride, blk, out, out_stride)) return;
+#endif
strata_ds4_launch(topk_kernel, T, 1024, 0, s, vals, stride, r, cand, cand_stride, blk, out, out_stride);
}
@@ -2016,6 +2081,10 @@ void cand_blocks(const float* score, long long stride, int T, const Rows& r, int
rb.k[t] = r.k[t];
nb_max = rb.n[t] > nb_max ? rb.n[t] : nb_max;
}
+#if defined(STRATA_METAL)
+ // block_max_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::block_max(score, stride, T, &r, blk, bm, bm_stride, nb_max)) return;
+#endif
strata_ds4_launch(block_max_kernel, dim3(cdiv(nb_max, kThreads), T), kThreads, 0, s, score, stride, r, blk, bm, bm_stride);
topk(bm, bm_stride, T, rb, nullptr, 0, 1, out, out_stride, s);
}
@@ -2064,6 +2133,10 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
// the BF16 rows' kernel itself (heads_per_block -1: this way at any T)
const bool few = heads_per_block == -1 || (heads_per_block == 0 && T <= kAttnScratchT);
if (few && scr != nullptr && scr->rows != nullptr && T <= scr->max_t && sel_stride <= scr->cap) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::unpack_sel(ckv, sel, sel_stride, a.gen, a.p0, a.t0, a.R, a.kcap, a.nsel,
+ scr->cap, scr->rows, T, a.gen ? 1 : (T < kWin ? T : kWin), kv_row_bytes(kKvFp4, 512))) return;
+#endif
strata_ds4_launch(unpack_sel_kernel, dim3(cdiv(sel_stride, kStage), T), kThreads, 0, s, (const uint8_t*) ckv, sel, sel_stride, a,
scr->cap, scr->rows);
#if !defined(STRATA_USE_HIP)
@@ -2092,6 +2165,9 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
}
void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::ring_to_ext(ring, ext, (long long) p0, n, d, kv_row_bytes(kKvFp8, d))) return;
+#endif
strata_ds4_launch(ring_to_ext_kernel, n, kThreads, 0, s, ring, ext, (long long) p0, n, d);
}
void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s, uint8_t* backup) {
@@ -2099,6 +2175,9 @@ void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, i
std::fprintf(stderr, "ds4 ext_to_ring: row of %d not covered\n", d);
return;
}
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::ext_to_ring(ext, ring, (long long) p0, n, T, d, kv_row_bytes(kKvFp8, d), backup)) return;
+#endif
strata_ds4_launch(ext_to_ring_kernel, T, kThreads, 0, s, ext, ring, (long long) p0, n, T, d, backup);
}
void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {
@@ -2111,9 +2190,19 @@ void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T,
kv_row_bytes(kKvFp8, d));
}
void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // hc_mean_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::hc_mean(h, T, hc, dim, out, out_stride)) return;
+#endif
strata_ds4_launch(hc_mean_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, h, hc, dim, out, out_stride);
}
-void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride); }
+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // iota_rows_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::iota_rows(out, T, n, stride)) return;
+#endif
+ strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride);
+}
void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {
strata_ds4_launch(markov_kernel, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, H, tok, V, R);
}
@@ -2133,10 +2222,18 @@ void confidence(const float* x, const uint16_t* E, const int* tok, const float*
}
void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt) {
+#if defined(STRATA_METAL)
+ // engram_dequant_kernel on the Metal dispatch (fmt 2, this model's IQ4_NL tables); the launch below is the fallback
+ if (strata::dsv41::metal_host::engram_dequant(rows, scales, n, hd, out, fmt)) return;
+#endif
strata_ds4_launch(engram_dequant_kernel, n, kThreads, 0, s, rows, scales, hd, out, fmt);
}
void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // engram_gate_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::engram_gate(h, kv, w, hc, dim, T, eps)) return;
+#endif
strata_ds4_launch(engram_gate_kernel, dim3(hc, T), 1024, 0, s, h, kv, w, hc, dim, eps);
}
diff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cu
index 94944c6fb..996c02002 100644
--- a/src/dsv41/vendor/sampler.cu
+++ b/src/dsv41/vendor/sampler.cu
@@ -16,6 +16,9 @@
// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /
// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).
#include "strata/dsv41/sampler.hpp"
+#if defined(STRATA_METAL)
+#include <ds4_metal_host.h>
+#endif
#include <cuda_runtime.h>
@@ -525,6 +528,13 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hi
if (p.greedy || p.temperature <= 0.0f) {
// One block per token, 1,024 threads over the vocabulary. See `sampler_greedy_kernel`.
const int gthreads = 1024;
+#if defined(STRATA_METAL)
+ // sampler_greedy_kernel on the Metal dispatch; the launch below is the fallback
+ if (::strata::dsv41::metal_host::sample_greedy(logits, n_tokens, n_vocab, history, history_len,
+ p.penalty_last_n, p.penalty_repeat, p.penalty_freq,
+ p.penalty_present, p.logit_bias, out))
+ return;
+#endif
strata_ds4_launch(sampler_greedy_kernel, (unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream,
logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
} else {