Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: THE ENGINE GENERATES TOKENS ON THE MAC - 17 more kernels, the whole path runs
Niko122133bf441cd4dc
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex d26707802..8a7c247cb 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -1,8 +1,7 @@-// metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch, as the engine calls them.+// metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch.//-// A launch site in the shared dsv41 sources becomes, under STRATA_METAL, a call to one of these instead of the generic-// strata_ds4_launch() stub. Each implementation wraps the engine's own device buffers into the Metal runtime and dispatches-// the MSL kernel of the same name; metal/src/engine/ds4_host.cpp holds them.+// GENERATED from metal/src/engine/ds4_host.cpp and ds4_finish.cpp, which are the source of truth: do not hand-edit the+// declaration block, regenerate it (see RIGS/m1002.md).#pragma once#include <cstddef>@@ -16,11 +15,7 @@ using Stream = StreamImpl*;namespace strata::dsv41::metal_host {-// ds4_kernels.cu:1477 init_pre_kernel: pre[i] = (i % 8) == 0 ? 1 : 0, i < T*8-bool init_pre(float* pre, int T, void* stream);-// ds4_kernels.cu:130 embed_kernel: h[(t*hc + s)*dim + e] = bf16(rows[t*dim + e])-// ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, ...): C[t][m] = sum_k A[m][k] * B[t][k], F32, alpha 1, beta 0-// ds4_kernels.cu:1894 finish_kernel<false>, the engine's Finish as ds4_kernels.cuh declares it+// ds4_kernels.cu:1894 finish_kernel<false> - the engine's Finish, as ds4_kernels.cuh declares itstruct FinishIn {const float* w;float eps;@@ -32,26 +27,67 @@ struct FinishIn {uint8_t* pdst;long long pdst_stride;};-// ds4_kernels.cu:1938 moe_sort: counts, prefix, fill-bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);--// ds4_kernels.cu:1908 hc_split (the sinkhorn normalisation)-bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,- float* pre, float* post, float* comb);-// ds4_kernels.cu:1901 hc_mix+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);+bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);+bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);+bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);+bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);--// ds4_kernels.cu:1692 hc_mix_scalebool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);--// ds4_kernels.cu:1677 hc_post_inplacebool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);+bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);+bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);+bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);+bool init_pre(float* pre, int T, void*);+bool iq_mv(const void* jobs, int nj);+bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);+bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);+bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);+bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);+bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);+bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);+bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);+bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+bool swiglu(const float* g, const float* u, int n, float lim, float* h);+bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);+// the entry points, as metal/src/engine/ds4_finish.cpp defines them+bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);+bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);+bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);+bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);+bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);+bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);+bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);-+bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy);+bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);+bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride);+bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);+bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);+bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);+bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);+bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);+bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);+bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand, int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt);+bool init_pre(float* pre, int T, void*);+bool iota_rows(int* out, int T, int n, int stride);+bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int type, int groups);+bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int type, int groups);+bool iq_mv(const void* jobs, int nj);+bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);+bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);+bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);+bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride, const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd);+bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);+bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);+bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);+bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);+bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out);bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+bool swiglu(const float* g, const float* u, int n, float lim, float* h);+bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out, int out_stride);+bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);-bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void* stream);--} // namespace strata::dsv41::metal_host+} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex e0a01b7a0..8dabb42cd 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -32,12 +32,15 @@ struct GpuFinish {namespace {mtl::Context* g_cx = nullptr;int g_lib = -1;+int g_lib_iq = -1;bool ensure() {if (g_cx) return g_lib >= 0;g_cx = new mtl::Context();g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");+ g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);+ if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");return g_lib >= 0;}} // namespace@@ -57,12 +60,9 @@ struct FinishIn {bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {if (n_vec <= 0 || len <= 0) return true;- if (f.rd > 0 || f.quant != 0 || f.pdst != nullptr) {- std::fprintf(stderr,- "dsv41 metal: finish needs the unwritten branches (rd %d quant %d pdst %p) - not ported yet\n",- f.rd, f.quant, (const void*) f.pdst);- return false;- }+ // finish_kernel<true> (the KV pack: the row quantised to codes + group scales) or <false>; the <false> rounding+ // branch (quant without pdst) is the one still unwritten+ const bool pack = f.pdst != nullptr;if (!ensure()) return false;void* hx = strata::gpu::host_view(x);if (!hx) { std::fprintf(stderr, "dsv41 metal: finish has no host view\n"); return false; }@@ -76,17 +76,17 @@ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {GpuFinish gf{};gf.w = (uint64_t) (uintptr_t) f.w;gf.eps = f.eps;- gf.rd = 0; gf.pos = 0; gf.inverse = 0; gf.vpp = 1; gf.pos_step = 1;+ gf.rd = f.rd; gf.pos = f.pos; gf.inverse = f.inverse; gf.vpp = f.vpp; gf.pos_step = f.pos_step;gf.freqs = (uint64_t) (uintptr_t) f.freqs;- gf.quant = 0;+ gf.quant = f.quant;gf.dst = (uint64_t) (uintptr_t) f.dst;gf.dst_stride = f.dst_stride;- gf.pdst = 0;- gf.pdst_stride = 0;+ gf.pdst = (uint64_t) (uintptr_t) f.pdst;+ gf.pdst_stride = f.pdst_stride;mtl::Pipeline pl;try {- pl = g_cx->pipeline(g_lib, "ds4_finish");+ pl = g_cx->pipeline(g_lib, pack ? "ds4_finish_pack" : "ds4_finish");} catch (...) {return false;}@@ -110,12 +110,14 @@ bool hc_post(const float* y, float* h, const float* post, const float* comb, intvoid* hh = strata::gpu::host_view(h);if (!hy || !hh) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }const size_t page = 16384;- auto wrap = [&](void* host, size_t bytes) {- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 + 65536);- mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 + 65536);+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );if (!by.handle || !bh.handle) return false;// post is [T][8], comb is [T][64], straight from the caller's host/shared memorymtl::Buffer bp = g_cx->alloc((size_t) T * 8 * 4);@@ -147,12 +149,14 @@ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float*void* hm = strata::gpu::host_view((void*) mixes);if (!hh || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix_scale has no host view\n"); return false; }const size_t page = 16384;- auto wrap = [&](void* host, size_t bytes) {- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 + 65536);- mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );if (!bh.handle || !bm.handle) return false;mtl::Pipeline pl;try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_scale"); } catch (...) { return false; }@@ -176,13 +180,15 @@ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, floatvoid* hm = strata::gpu::host_view((void*) mixes);if (!hh || !hf || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix has no host view\n"); return false; }const size_t page = 16384;- auto wrap = [&](void* host, size_t bytes) {- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 + 65536);- mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 + 65536);- mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );+ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 );+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );if (!bh.handle || !bf.handle || !bm.handle) return false;mtl::Pipeline pl;try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix"); } catch (...) { return false; }@@ -209,14 +215,16 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,void* hc_ = strata::gpu::host_view((void*) comb);if (!hm || !hp || !ho || !hc_) { std::fprintf(stderr, "dsv41 metal: hc_split has no host view\n"); return false; }const size_t page = 16384;- auto wrap = [&](void* host, size_t bytes) {- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);- mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 + 65536);- mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 + 65536);- mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 + 65536);+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );+ mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 );+ mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 );if (!bm.handle || !bp.handle || !bo.handle || !bc.handle) return false;// scale and base come from the engine like every other operand: reach them through host_view, not as raw host pointersvoid* hs = strata::gpu::host_view((void*) scale);@@ -224,7 +232,7 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }const size_t nbase = (size_t) (2 * hc + hc * hc);mtl::Buffer bs = wrap(hs, 256);- mtl::Buffer bb = wrap(hb, (nbase * 4 + 65535) / 65536 * 65536 + 65536);+ mtl::Buffer bb = wrap(hb, (nbase * 4 + 65535) / 65536 * 65536 );if (!bs.handle || !bb.handle) return false;mtl::Pipeline pl;try { pl = g_cx->pipeline(g_lib, "ds4_hc_split"); } catch (...) { return false; }@@ -252,14 +260,16 @@ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* boundsvoid* hp = strata::gpu::host_view((void*) pos);if (!hi || !hv || !hb || !hr || !hp) { std::fprintf(stderr, "dsv41 metal: moe_sort has no host view\n"); return false; }const size_t page = 16384;- auto wrap = [&](void* host, size_t bytes) {- return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };const size_t t_bytes = (size_t) T * k * 4 + 65536;mtl::Buffer bi = wrap(hi, t_bytes);- mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 + 65536);- mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 + 65536);+ mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );+ mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );mtl::Buffer br = wrap(hr, t_bytes);mtl::Buffer bp = wrap(hp, t_bytes);if (!bi.handle || !bv.handle || !bb.handle || !br.handle || !bp.handle) return false;@@ -298,4 +308,1105 @@ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* boundsreturn true;}++// ds4_kernels.cu:1690 - the Engram row decoder (fmt 2 = IQ4_NL, this model's tables)+bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt) {+ if (T <= 0) return true;+ if (!ensure()) return false;+ void* hs = strata::gpu::host_view((void*) src);+ void* ho = strata::gpu::host_view((void*) out);+ if (!hs || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant_tok has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t rows = (size_t) T * (size_t) cols;+ mtl::Buffer bs = wrap(hs, (size_t) T * per );+ mtl::Buffer bo = wrap(ho, rows * (size_t) hd * 4 );+ if (!bs.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((unsigned long) per), mtl::Arg::V((unsigned) cols),+ mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((int) fmt)};+ d.grid[0] = (uint32_t) rows;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_kernels.cu:341 - hc_pre_norm: a = sum_s mix[s]*h[s][e]; x = a * rsqrt(mean(a^2) + eps) (x w when given)+bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x) {+ if (!ensure()) return false;+ void* hh = strata::gpu::host_view((void*) h);+ void* hm = strata::gpu::host_view((void*) mix);+ void* hx = strata::gpu::host_view((void*) x);+ if (!hh || !hm || !hx) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );+ mtl::Buffer bm = wrap(hm, (size_t) T * 8 * 4 );+ mtl::Buffer bx = wrap(hx, (size_t) T * dim * 4 );+ if (!bh.handle || !bm.handle || !bx.handle) return false;+ mtl::Buffer bw;+ size_t w_off = 0;+ if (w) {+ void* hw = strata::gpu::host_view((void*) w);+ if (!hw) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm weight has no host view\n"); return false; }+ bw = wrap(hw, (size_t) dim * 4 );+ w_off = off(hw);+ if (!bw.handle) return false;+ } else {+ bw = bx; // never read when has_w == 0+ }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_pre_norm"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bw, w_off),+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc), mtl::Arg::V(eps), mtl::Arg::B(bx, off(hx)),+ mtl::Arg::V((unsigned) (w ? 1 : 0))};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_gguf.cu:186 - the activation quantiser the expert matmuls consume (Q8_0: 32 values, one scale)+bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {+ if (rows <= 0 || k <= 0) return true;+ if (!ensure()) return false;+ void* hx = strata::gpu::host_view((void*) x);+ void* hq = strata::gpu::host_view((void*) q);+ void* hd = strata::gpu::host_view((void*) d);+ if (!hx || !hq || !hd) { std::fprintf(stderr, "dsv41 metal: q8_quant has no host view\n"); return false; }+ const size_t page = 16384;+ // The pages the buffer itself spans: [base, base + off + bytes). A fixed slack (the +65536 the call sites used to+ // carry) walks off the end of the mapping when the buffer is small and sits at an arena chunk's edge, and Metal then+ // refuses the no-copy wrap ("mtl: wrap 81920 failed" - the hd scales here, a few pages).+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bx = wrap(hx, (size_t) rows * ldx * 4);+ mtl::Buffer bq = wrap(hq, (size_t) rows * k);+ mtl::Buffer bd = wrap(hd, (size_t) rows * (k / 32) * 4);+ if (!bx.handle || !bq.handle || !bd.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) k),+ mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bd, off(hd))};+ const uint groups = (uint32_t) ((k / 32 + 7) / 8);+ dsp.grid[0] = groups;+ dsp.grid[1] = (uint32_t) rows;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}+++// ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so+// the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.+bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt) {+ if (nj <= 0 || max_n <= 0) return true;+ if (!ensure()) return false;+ mtl::Buffer bj = g_cx->alloc((size_t) nj * job_bytes);+ if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: mx_mv job buffer failed\n"); return false; }+ std::memcpy(bj.ptr, jobs, (size_t) nj * job_bytes);+ mtl::Pipeline pl;+ // fmt 1 = MXFP4, 2 = Q8_0, 0 = BF16 (a dense matrix: the engram's wkv, ds4_gguf.cu:1064)+ const char* fn = fmt == 2 ? "ds4_q8_mv" : (fmt == 1 ? "ds4_mx_mv" : "ds4_bf16_mv");+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);+ d.grid[1] = (uint32_t) nj;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+}+++// ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they+// travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.+bool iq_mv(const void* jobs, int nj) {+ if (nj <= 0) return true;+ if (!ensure()) return false;+ // max rows across the jobs: the grid's x extent+ const unsigned char* jb = (const unsigned char*) jobs;+ int max_n = 0;+ for (int i = 0; i < nj; ++i) {+ int n = 0;+ std::memcpy(&n, jb + (size_t) i * 80 + 24, sizeof(int)); // MvJob::n+ if (n > max_n) max_n = n;+ }+ if (max_n <= 0) return true;+ mtl::Buffer bj = g_cx->alloc((size_t) nj * 80);+ if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n"); return false; }+ std::memcpy(bj.ptr, jobs, (size_t) nj * 80);+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_iq, "ds4_iq_mv"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);+ d.grid[1] = (uint32_t) nj;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+}+++// ds4_iq.cu:536 gate_up_kernel<T, NE> / :588 down_kernel<T, NE> - the prompt / window path's expert GEMM (the launches at+// :958 launch_gu / :960 launch_dn). A threadgroup of 256 is 8 warps: a warp RW rows (gate / up 1, down 2), a lane the+// row's sub-blocks, the group's entries four at a pass. The group's blob and its entries are read the way the CUDA kernel+// reads them, so the plan / window pointer arrays travel as they are: every pointer here is a CPU address (unified memory)+// and a kernel may take raw gpuAddresses (as ds4_finish's f.w does).+namespace {+// (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4IqJob (Metal has no 'long long': 64-bit is 'long')+struct IqJob {+ const void* plan_ptr;+ const void* grp_ptr;+ const void* grp_start;+ const void* grp_ent;+ const void* n_groups;+ const void* xq;+ const void* xd;+ const void* hq;+ const void* hd;+ void* h;+ void* part;+ int64_t tr0, tr1, tr2;+ int n_embd, n_ff, type;+ float limit;+};++static bool iq_gemm(const IqJob& j, const char* fn, int grid_x, int groups) {+ if (!ensure()) return false;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) grid_x;+ d.grid[1] = (uint32_t) groups;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+}+} // namespace++// gate / up of the group's entries: h = the clamped swiglu (ds4_iq.cuh's Args: tr[0] gate, tr[1] up, tr[2] down)+bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,+ const int8_t* xq, const float* xd, float* h, int type, int groups) {+ if (groups <= 0) return true;+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 256 != 0 || n_ff % 8 != 0) return false;+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;+ IqJob j{};+ j.plan_ptr = plan_ptr;+ j.grp_ptr = grp_ptr;+ j.grp_start = grp_start;+ j.grp_ent = grp_ent;+ j.n_groups = n_groups;+ j.xq = xq;+ j.xd = xd;+ j.h = h;+ j.tr0 = (int64_t) tr[0];+ j.tr1 = (int64_t) tr[1];+ j.tr2 = (int64_t) tr[2];+ j.n_embd = n_embd;+ j.n_ff = n_ff;+ j.type = type;+ j.limit = limit;+ return iq_gemm(j, "ds4_iq_gu", n_ff / 8, groups);+}++// down of the group's entries against their int8 h: part (a row of n_embd an entry)+bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,+ const float* hd, float* part, int type, int groups) {+ if (groups <= 0) return true;+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 256 != 0 || n_embd % 16 != 0) return false;+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;+ IqJob j{};+ j.plan_ptr = plan_ptr;+ j.grp_ptr = grp_ptr;+ j.grp_start = grp_start;+ j.grp_ent = grp_ent;+ j.n_groups = n_groups;+ j.hq = hq;+ j.hd = hd;+ j.part = part;+ j.tr0 = (int64_t) tr[0];+ j.tr1 = (int64_t) tr[1];+ j.tr2 = (int64_t) tr[2];+ j.n_embd = n_embd;+ j.n_ff = n_ff;+ j.type = type;+ j.limit = 0.0f;+ return iq_gemm(j, "ds4_iq_dn", n_embd / 16, groups);+}+++// ds4_gguf.cu:577 mx_combine_kernel - a token's expert outputs summed in plan order (the experts off the device add 0),+// the CPU lane's part when its flag is set, then the shared expert's: ds4_gguf.cu:169's combine(), 128 threads a block+namespace {+// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Comb (a 'long' there is int64_t here)+struct MxCombJob {+ const void* plan_ptr;+ const void* plan_w;+ const void* part;+ const void* cpu_part;+ const int* cpu_flag;+ const void* sh_out;+ void* out;+ int64_t cpu_stride, out_stride;+ int k, n_embd;+};+} // namespace++bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride,+ const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd) {+ if (T <= 0 || n_embd <= 0) return true;+ if (k <= 0 || plan_ptr == nullptr || plan_w == nullptr || part == nullptr || out == nullptr) return false;+ if (!ensure()) return false;+ MxCombJob j{};+ j.plan_ptr = plan_ptr;+ j.plan_w = plan_w;+ j.part = part;+ j.cpu_part = cpu_part;+ j.cpu_flag = cpu_flag;+ j.sh_out = sh_out;+ j.out = out;+ j.cpu_stride = (int64_t) cpu_stride;+ j.out_stride = (int64_t) out_stride;+ j.k = k;+ j.n_embd = n_embd;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: mx_combine job buffer failed\n");+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_mx_combine"); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) ((n_embd + 127) / 128);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+}++// ds4_kernels.cu:1092 - the KV ring row to BF16 (E4M3 code x E8M0 group scale)+bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes) {+ if (n <= 0 || d <= 0) return true;+ if (!ensure()) return false;+ void* hr = strata::gpu::host_view((void*) ring);+ void* he = strata::gpu::host_view(ext);+ if (!hr || !he) { std::fprintf(stderr, "dsv41 metal: ring_to_ext has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );+ mtl::Buffer be = wrap(he, (size_t) n * d * 2 );+ if (!br.handle || !be.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_ring_to_ext"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::B(be, off(he)), mtl::Arg::V((long) p0),+ mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};+ dsp.grid[0] = (uint32_t) n;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}+++// ds4_kernels.cu:1104 - the window back into the ring (E4M3 codes + E8M0 group scales, optional backup)+bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup) {+ if (T <= 0 || n <= 0 || d <= 0) return true;+ if (!ensure()) return false;+ void* he = strata::gpu::host_view((void*) ext);+ void* hr = strata::gpu::host_view(ring);+ if (!he || !hr) { std::fprintf(stderr, "dsv41 metal: ext_to_ring has no host view\n"); return false; }+ void* hb = backup ? strata::gpu::host_view(backup) : nullptr;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer be = wrap(he, (size_t) (n + T) * d * 2 );+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );+ if (!be.handle || !br.handle) return false;+ mtl::Buffer bb = br; // never written when backup is null+ size_t b_off = 0;+ if (hb) { bb = wrap(hb, (size_t) T * row_bytes ); b_off = off(hb); if (!bb.handle) return false; }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_ext_to_ring"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),+ mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),+ mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}+++// ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)++// On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about+// (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.+static inline void* cpu_addr(void* p) {+ void* h = strata::gpu::host_view(p);+ return h ? h : p;+}++// ds4_kernels.cu:1286 engram_dequant_kernel - the Engram rows (fmt 2 = IQ4_NL, the tables this model's pack carries; the+// other formats are left to the CUDA launch, which reports the line)+bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt) {+ (void) scales; // (fmt 2 keeps every block's scale in its own 18 bytes)+ if (n <= 0 || hd <= 0) return true;+ if (fmt != 2 || rows == nullptr || out == nullptr) return false;+ if (!ensure()) return false;+ void* hr = cpu_addr((void*) rows);+ void* ho = cpu_addr((void*) out);+ if (!hr || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);+ if (!br.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant_rows"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(fmt)};+ d.grid[0] = (uint32_t) n;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}++// ds4_kernels.cu:1311 engram_gate_kernel - the Engram gate over a (token, copy) row: h += sigmoid(normalised dot) * val+bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps) {+ if (T <= 0 || hc <= 0 || dim <= 0) return true;+ if (h == nullptr || kv == nullptr || w == nullptr) return false;+ if (!ensure()) return false;+ void* hh = cpu_addr((void*) h);+ void* hk = cpu_addr((void*) kv);+ void* hw = cpu_addr((void*) w);+ if (!hh || !hk || !hw) { std::fprintf(stderr, "dsv41 metal: engram_gate has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);+ mtl::Buffer bk = wrap(hk, (size_t) T * (size_t) (hc + 1) * (size_t) dim * 4);+ mtl::Buffer bw = wrap(hw, (size_t) hc * (size_t) dim * 4);+ if (!bh.handle || !bk.handle || !bw.handle) {+ std::fprintf(stderr, "dsv41 metal: engram_gate wrap failed (h %p kv %p w %p; %p %p %p)\n", hh, hk, hw,+ bh.handle, bk.handle, bw.handle);+ return false;+ }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_gate"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bk, off(hk)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::V((unsigned) hc), mtl::Arg::V((unsigned) dim), mtl::Arg::V(eps)};+ d.grid[0] = (uint32_t) hc;+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}++namespace {+// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Rows (ds4_kernels.cuh:115 Rows, kWin = 16)+struct Ds4RowsJob {+ int n[16];+ int k[16];+ int gen, R, kcap, ccap, blk;+ int64_t p0;+};+} // namespace++// ds4_kernels.cu:524 compress_kernel - the window compressor (ratio > 1): each token's kv / score row into its slot, and+// every ratio'th token pooled by a per-value softmax over the slots, scaled by the row RMS and the norm weights+bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,+ int d, const float* w, float eps, float* latent) {+ if (T <= 0 || d <= 0 || ratio <= 0) return true;+ if (d > 2 * 512) return false; // (the kernel holds d <= 2 * blockDim)+ if (kv == nullptr || score == nullptr || kv_state == nullptr || score_state == nullptr || w == nullptr ||+ latent == nullptr)+ return false;+ if (!ensure()) return false;+ void* hkv = cpu_addr((void*) kv);+ void* hsc = cpu_addr((void*) score);+ void* hks = cpu_addr((void*) kv_state);+ void* hss = cpu_addr((void*) score_state);+ void* hw = cpu_addr((void*) w);+ void* hla = cpu_addr((void*) latent);+ if (!hkv || !hsc || !hks || !hss || !hw || !hla) {+ std::fprintf(stderr, "dsv41 metal: compress_win has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bkv = wrap(hkv, (size_t) T * (size_t) d * 4);+ mtl::Buffer bsc = wrap(hsc, (size_t) T * (size_t) d * 4);+ mtl::Buffer bks = wrap(hks, (size_t) ratio * (size_t) d * 4);+ mtl::Buffer bss = wrap(hss, (size_t) ratio * (size_t) d * 4);+ mtl::Buffer bw = wrap(hw, (size_t) d * 4);+ mtl::Buffer bla = wrap(hla, (size_t) T * (size_t) d * 4);+ if (!bkv.handle || !bsc.handle || !bks.handle || !bss.handle || !bw.handle || !bla.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_compress"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bkv, off(hkv)), mtl::Arg::B(bsc, off(hsc)), mtl::Arg::B(bks, off(hks)),+ mtl::Arg::B(bss, off(hss)), mtl::Arg::V(ratio), mtl::Arg::V(p0), mtl::Arg::V(T),+ mtl::Arg::V((unsigned) d), mtl::Arg::B(bw, off(hw)), mtl::Arg::V(eps), mtl::Arg::B(bla, off(hla))};+ dsp.grid[0] = 1;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 512; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}++// ds4_kernels.cu:582 index_score_kernel<128, kfmt> - the indexer's scores: a position's key row against every head's+// query, relu, the heads' weighted sum (rows = ds4_kernels.cuh's Rows, by value there and copied here)+bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand,+ int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt) {+ if (n <= 0 || T <= 0) return true;+ if (hd != 128 || heads <= 0 || heads > 32 || (kfmt != 3 && kfmt != 0)) return false;+ if (q == nullptr || wq == nullptr || kc == nullptr || out == nullptr || rows == nullptr) return false;+ if (!ensure()) return false;+ void* hq = cpu_addr((void*) q);+ void* hwq = cpu_addr((void*) wq);+ void* hkc = cpu_addr((void*) kc);+ void* hout = cpu_addr((void*) out);+ if (!hq || !hwq || !hkc || !hout) {+ std::fprintf(stderr, "dsv41 metal: index_score has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t kb = kfmt == 3 ? (size_t) ((hd / 2 + hd / 32 + 15) / 16 * 16) : (size_t) hd * 2;+ mtl::Buffer bq = wrap(hq, (size_t) T * (size_t) heads * (size_t) hd * 4);+ mtl::Buffer bwq = wrap(hwq, (size_t) T * 64 * 4);+ mtl::Buffer bkc = wrap(hkc, (size_t) n * kb);+ mtl::Buffer bo = wrap(hout, ((size_t) (T - 1) * (size_t) out_stride + (size_t) n) * 4);+ if (!bq.handle || !bwq.handle || !bkc.handle || !bo.handle) return false;+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);+ if (!bc.handle) return false;+ // Rows is a host struct: it travels as a copy (no gpu address)+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: index_score rows buffer failed\n"); return false; }+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_index_score"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::B(bo, off(hout)),+ mtl::Arg::B(br), mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V(cand_stride), mtl::Arg::V(blk),+ mtl::Arg::V((long) out_stride), mtl::Arg::V(kfmt), mtl::Arg::V(cand != nullptr ? 1 : 0)};+ dsp.grid[0] = (uint32_t) ((n + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(br);+ if (cand == nullptr) g_cx->free(bc);+ return true;+}++// ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations, a warp a row of the matrix and 2 tokens+// a block (the indexer's wproj: ds4_model.cu:7743)+bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy) {+ if (T <= 0 || n <= 0 || k <= 0) return true;+ if (k % 8 != 0 || W == nullptr || x == nullptr || y == nullptr) return false;+ if (!ensure()) return false;+ void* hw = cpu_addr((void*) W);+ void* hx = cpu_addr((void*) x);+ void* hy = cpu_addr((void*) y);+ if (!hw || !hx || !hy) {+ std::fprintf(stderr, "dsv41 metal: gemv_f16_rows has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bw = wrap(hw, (size_t) n * (size_t) k * 2);+ mtl::Buffer bx = wrap(hx, ((size_t) (T - 1) * ldx + (size_t) k) * 4);+ mtl::Buffer by = wrap(hy, ((size_t) (T - 1) * ldy + (size_t) n) * 4);+ if (!bw.handle || !bx.handle || !by.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f16_rows"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V(n),+ mtl::Arg::V(k), mtl::Arg::V(T), mtl::Arg::V(alpha), mtl::Arg::B(by, off(hy)),+ mtl::Arg::V((unsigned long) ldy)};+ dsp.grid[0] = (uint32_t) ((n + 7) / 8);+ dsp.grid[1] = (uint32_t) ((T + 1) / 2);+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}++// ds4_kernels.cu:650 topk_kernel - a token's k largest indexer scores (four 8-bit radix passes, then the picks in index+// order); Rows by value there and copied here+bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out,+ int out_stride) {+ if (T <= 0) return true;+ if (vals == nullptr || out == nullptr || rows == nullptr) return false;+ if (!ensure()) return false;+ void* hv = cpu_addr((void*) vals);+ void* ho = cpu_addr((void*) out);+ if (!hv || !ho) {+ std::fprintf(stderr, "dsv41 metal: topk has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bv = wrap(hv, ((size_t) (T - 1) * (size_t) stride + 1) * 4);+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + 1) * 4);+ if (!bv.handle || !bo.handle) return false;+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);+ if (!bc.handle) return false;+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: topk rows buffer failed\n"); return false; }+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_topk"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bv, off(hv)), mtl::Arg::V((long) stride), mtl::Arg::B(br),+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::V(cand_stride),+ mtl::Arg::V(blk), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(out_stride),+ mtl::Arg::V(cand != nullptr ? 1 : 0)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(br);+ if (cand == nullptr) g_cx->free(bc);+ return true;+}++// ds4_kernels.cu:751 block_max_kernel - a block of the indexer's positions and its largest score (the newest pinned at +inf)+bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride,+ int nb_max) {+ if (T <= 0 || nb_max <= 0 || blk <= 0) return true;+ if (score == nullptr || bm == nullptr || rows == nullptr) return false;+ if (!ensure()) return false;+ void* hs = cpu_addr((void*) score);+ void* hb = cpu_addr((void*) bm);+ if (!hs || !hb) {+ std::fprintf(stderr, "dsv41 metal: block_max has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bs = wrap(hs, ((size_t) (T - 1) * (size_t) stride + (size_t) nb_max * (size_t) blk) * 4);+ mtl::Buffer bb = wrap(hb, ((size_t) (T - 1) * (size_t) bm_stride + (size_t) nb_max) * 4);+ if (!bs.handle || !bb.handle) return false;+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: block_max rows buffer failed\n"); return false; }+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_block_max"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((long) stride), mtl::Arg::B(br), mtl::Arg::V(blk),+ mtl::Arg::B(bb, off(hb)), mtl::Arg::V((long) bm_stride)};+ dsp.grid[0] = (uint32_t) ((nb_max + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(br);+ return true;+}++// ds4_kernels.cu:1141 hc_mean_kernel - a token's mean over its hc streams (grid (ceil(dim / 256), T), 256 threads)+bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride) {+ if (T <= 0 || hc <= 0 || dim <= 0) return true;+ if (h == nullptr || out == nullptr) return false;+ if (!ensure()) return false;+ void* hh = cpu_addr((void*) h);+ void* ho = cpu_addr((void*) out);+ if (!hh || !ho) {+ std::fprintf(stderr, "dsv41 metal: hc_mean has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + (size_t) dim) * 4);+ if (!bh.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mean"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::V(hc), mtl::Arg::V(dim), mtl::Arg::B(bo, off(ho)),+ mtl::Arg::V((long) out_stride)};+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}++// ds4_kernels.cu:1150 iota_rows_kernel - out[t][i] = i+bool iota_rows(int* out, int T, int n, int stride) {+ if (T <= 0 || n <= 0) return true;+ if (out == nullptr) return false;+ if (!ensure()) return false;+ void* ho = cpu_addr((void*) out);+ if (!ho) {+ std::fprintf(stderr, "dsv41 metal: iota_rows has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) stride + (size_t) n) * 4);+ if (!bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_iota_rows"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bo, off(ho)), mtl::Arg::V(n), mtl::Arg::V(stride)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 64; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+}++// vendor/sampler.cu:93 sampler_greedy_kernel - the plain argmax over the vocabulary, one block (1024) a token, the+// repetition penalties from the history window's bitmap (the caller takes this path when greedy or temperature <= 0)+bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n,+ float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out) {+ if (n_tokens <= 0 || n_vocab <= 0) return true;+ if (logits == nullptr || out == nullptr) return false;+ if ((n_vocab + 31) / 32 > 4096) return false; // (the kernel's threadgroup bitmap)+ if (!ensure()) return false;+ void* hl = cpu_addr((void*) logits);+ void* ho = cpu_addr((void*) out);+ if (!hl || !ho) {+ std::fprintf(stderr, "dsv41 metal: sample_greedy has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bl = wrap(hl, (size_t) n_tokens * (size_t) n_vocab * 4);+ mtl::Buffer bo = wrap(ho, (size_t) n_tokens * 4);+ if (!bl.handle || !bo.handle) return false;+ const size_t hbytes = (size_t) n_tokens * (size_t) (history_len > 0 ? history_len : 1) * 4;+ mtl::Buffer bh = history != nullptr ? wrap(cpu_addr((void*) history), hbytes) : g_cx->alloc(4);+ mtl::Buffer bb = logit_bias != nullptr ? wrap(cpu_addr((void*) logit_bias), (size_t) n_vocab * 4) : g_cx->alloc(4);+ if (!bh.handle || !bb.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_sampler_greedy"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::V(n_vocab),+ mtl::Arg::B(bh, history != nullptr ? off(cpu_addr((void*) history)) : 0),+ mtl::Arg::V(history_len), mtl::Arg::V(penalty_last_n), mtl::Arg::V(penalty_repeat),+ mtl::Arg::V(penalty_freq), mtl::Arg::V(penalty_present),+ mtl::Arg::B(bb, logit_bias != nullptr ? off(cpu_addr((void*) logit_bias)) : 0),+ mtl::Arg::V(logit_bias != nullptr ? 1 : 0), mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) n_tokens;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ if (history == nullptr) g_cx->free(bh);+ if (logit_bias == nullptr) g_cx->free(bb);+ return true;+}++bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap,+ const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes) {+ if (T <= 0 || sel_stride <= 0 || nsel_count <= 0) return true;+ // nothing selected yet (a prompt at position 0): the scratch stays untouched, which is what the CUDA launch would do with+ // no rows to write+ if (ckv == nullptr || sel == nullptr || out == nullptr) return true;+ if (!ensure()) return false;+ void* hc = cpu_addr((void*) ckv);+ void* hs = cpu_addr((void*) sel);+ void* ho = cpu_addr((void*) out);+ if (!hc || !hs || !ho) { std::fprintf(stderr, "dsv41 metal: unpack_sel has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bc = wrap(hc, (size_t) 4096 * row_bytes );+ mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );+ mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );+ if (!bc.handle || !bsl.handle || !bo.handle) return false;+ mtl::Buffer bn = g_cx->alloc((size_t) nsel_count * 4);+ if (!bn.ptr) return false;+ std::memcpy(bn.ptr, nsel, (size_t) nsel_count * 4);+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_unpack_sel"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bc, off(hc)), mtl::Arg::B(bsl, off(hs)), mtl::Arg::V((unsigned) sel_stride),+ mtl::Arg::V((int) gen), mtl::Arg::V((long) p0), mtl::Arg::V((int) t0), mtl::Arg::V((int) R),+ mtl::Arg::V((int) kcap), mtl::Arg::B(bn), mtl::Arg::V((unsigned) cap), mtl::Arg::B(bo, off(ho)),+ mtl::Arg::V((unsigned long) row_bytes)};+ d.grid[0] = (uint32_t) ((sel_stride + 15) / 16);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bn);+ return true;+}+++// ds4_kernels.cu:1772 - the expert outputs back into the token rows+bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim) {+ if (T <= 0 || dim <= 0) return true;+ if (!ensure()) return false;+ void* ha = cpu_addr(acc);+ void* hy = cpu_addr((void*) y);+ void* hp = cpu_addr((void*) pos);+ void* hw = cpu_addr((void*) w_);+ if (!ha || !hy || !hp || !hw) { std::fprintf(stderr, "dsv41 metal: moe_combine has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer ba = wrap(ha, (size_t) T * dim * 4 );+ mtl::Buffer by = wrap(hy, (size_t) (r1 > r0 ? r1 - r0 : 1) * dim * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );+ if (!ba.handle || !by.handle || !bp.handle || !bw.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_moe_combine"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(by, off(hy)), mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::V((int) k), mtl::Arg::V((int) r0), mtl::Arg::V((int) r1), mtl::Arg::V((unsigned) dim)};+ d.grid[0] = (uint32_t) ((dim + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_kernels.cu:362 - hc_post: out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]+bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out) {+ if (T <= 0 || dim <= 0) return true;+ if (!ensure()) return false;+ void* hy = cpu_addr((void*) y);+ void* hr = cpu_addr((void*) res);+ void* hp = cpu_addr((void*) post);+ void* hco = cpu_addr((void*) comb);+ void* ho = cpu_addr((void*) out);+ if (!hy || !hr || !hp || !hco || !ho) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );+ mtl::Buffer br = wrap(hr, (size_t) T * hc * dim * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );+ mtl::Buffer bc = wrap(hco, (size_t) T * 64 * 4 );+ mtl::Buffer bo = wrap(ho, (size_t) T * hc * dim * 4 );+ if (!by.handle || !br.handle || !bp.handle || !bc.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post_v"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp)),+ mtl::Arg::B(bc, off(hco)), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((unsigned) dim),+ mtl::Arg::V((unsigned) hc)};+ d.grid[0] = (uint32_t) ((dim * hc + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_kernels.cu:441 - the F32 GEMV (residual / dense projections)+bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy) {+ if (T <= 0 || n <= 0 || k <= 0) return true;+ if (!ensure()) return false;+ void* hw = cpu_addr((void*) W);+ void* hx = cpu_addr((void*) x);+ void* hy = cpu_addr((void*) y);+ if (!hw || !hx || !hy) { std::fprintf(stderr, "dsv41 metal: gemv_f32 has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bw = wrap(hw, (size_t) n * k * 4 );+ mtl::Buffer bx = wrap(hx, (size_t) T * k * 4 );+ mtl::Buffer by = wrap(hy, (size_t) T * (ldy > 0 ? ldy : n) * 4 );+ if (!bw.handle || !bx.handle || !by.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f32"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned) n),+ mtl::Arg::V((unsigned) k), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned) ldy)};+ d.grid[0] = (uint32_t) ((n + 7) / 8);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_kernels.cu:1353 - the router's selection (idx, w, plan_ptr). The prefetch requests stay unpublished for now.+bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram,+ int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl) {+ if (T <= 0 || n <= 0) return true;+ if (!ensure()) return false;+ void* hl = cpu_addr((void*) logits);+ void* hb = cpu_addr((void*) bias);+ void* hv = cpu_addr((void*) vram);+ void* hi = cpu_addr((void*) idx);+ void* hw = cpu_addr((void*) w);+ void* hp = cpu_addr((void*) plan_ptr);+ if (!hl || !hb || !hv || !hi || !hw || !hp) { std::fprintf(stderr, "dsv41 metal: route_publish has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4 );+ mtl::Buffer bb = wrap(hb, (size_t) n * 4 );+ mtl::Buffer bv = wrap(hv, (size_t) n * 8 );+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4 );+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 8 );+ if (!bl.handle || !bb.handle || !bv.handle || !bi.handle || !bw.handle || !bp.handle) return false;+ mtl::Buffer bvl = bb;+ size_t vl_off = 0;+ if (bias_vl) {+ void* hvl = cpu_addr((void*) bias_vl);+ if (!hvl) return false;+ bvl = wrap(hvl, (size_t) n * 4 );+ vl_off = off(hvl);+ }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_route_publish"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),+ mtl::Arg::V(route_scale), mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bvl, vl_off), mtl::Arg::V((unsigned) vl),+ mtl::Arg::V((unsigned) (bias_vl ? 1 : 0))};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 512; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_kernels.cu swiglu: h = silu(g) * u (clamped when lim > 0)+bool swiglu(const float* g, const float* u, int n, float lim, float* h) {+ if (n <= 0) return true;+ if (!ensure()) return false;+ void* hg = cpu_addr((void*) g);+ void* hu = cpu_addr((void*) u);+ void* hh = cpu_addr((void*) h);+ if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host) {+ const size_t bytes = (size_t) n * 4 + 65536;+ return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);+ if (!bg.handle || !bu.handle || !bh.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_swiglu"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bg, off(hg)), mtl::Arg::B(bu, off(hu)), mtl::Arg::V((unsigned) n), mtl::Arg::V(lim),+ mtl::Arg::B(bh, off(hh))};+ d.grid[0] = (uint32_t) ((n + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// vendor/moe_win.cu:49 moe_win_group - the expert -> group index for a window. One thread in CUDA; on the Mac every device+// pointer is a CPU address, so the same 25-line loop runs on the host, which keeps the first-appearance order the kernel has and+// avoids mirroring the 17-field MoeWin struct through a constant buffer.+bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k) {+ if (T <= 0 || k <= 0) return true;+ if (T * 8 > 4096) { std::fprintf(stderr, "dsv41 metal: moe_win_group window of %d too wide\n", T); return false; }+ const unsigned long long* plan = (const unsigned long long*) cpu_addr((void*) plan_ptr);+ unsigned long long* gptr = (unsigned long long*) cpu_addr(grp_ptr);+ int* gstart = (int*) cpu_addr(grp_start);+ int* gent = (int*) cpu_addr(grp_ent);+ int* ng_out = (int*) cpu_addr(n_groups);+ if (!plan || !gptr || !gstart || !gent || !ng_out) return false;+ static unsigned long long gp[4096];+ static int cnt[4096], fill[4097], gof[4096];+ int ng = 0;+ for (int t = 0; t < T; ++t) {+ for (int i = 0; i < 8; ++i) {+ const int e = t * 8 + i;+ gof[e] = -1;+ const unsigned long long p = i < k ? plan[e] : 0ull;+ if (p == 0ull) continue;+ int g = 0;+ while (g < ng && gp[g] != p) ++g;+ if (g == ng) {+ gp[ng] = p;+ cnt[ng++] = 0;+ }+ ++cnt[g];+ gof[e] = g;+ }+ }+ fill[0] = 0;+ for (int g = 0; g < ng; ++g) {+ gptr[g] = gp[g];+ gstart[g] = fill[g];+ fill[g + 1] = fill[g] + cnt[g];+ }+ gstart[ng] = fill[ng];+ for (int e = 0; e < T * 8; ++e) {+ if (gof[e] >= 0) gent[fill[gof[e]]++] = e;+ }+ *ng_out = ng;+ return true;+}+} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex c2550903d..668d3ee42 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -77,6 +77,42 @@ static inline float ds4_block_sum(float v, threadgroup float* red, ushort lane,return red[0];}+// ds4_kernels.cu:65 pow2_ceil / :88 fp8_rt / :90 e2m1_rt - the KV quantiser's scale and roundings, shared by finish_pack+// (the packed rows) and ds4_finish's rounding branch+static inline float ds4_pow2_ceil(float x) {+ const uint b = as_type<uint>(x);+ const int e = int((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);+ return as_type<float>(uint(e + 127) << 23);+}++static inline float ds4_e4m3_round(float v) { // E4M3, nearest even (the input is clamped)+ const float a = fabs(v);+ if (!(a == a)) return v;+ float r;+ if (a >= 448.0f) r = 448.0f;+ else {+ const int e = int((as_type<uint>(a) >> 23) & 0xffu) - 127;+ const float step = e < -6 ? 0.001953125f : as_type<float>(uint(e - 3 + 127) << 23);+ r = a == 0.0f ? 0.0f : rint(a / step) * step;+ r = fmin(r, 448.0f);+ }+ return copysign(r, v);+}++static inline float ds4_e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even+ const float a = fabs(v);+ float r;+ if (a <= 0.25f) r = 0.0f;+ else if (a < 0.75f) r = 0.5f;+ else if (a <= 1.25f) r = 1.0f;+ else if (a < 1.75f) r = 1.5f;+ else if (a <= 2.5f) r = 2.0f;+ else if (a < 3.5f) r = 3.0f;+ else if (a <= 5.0f) r = 4.0f;+ else r = 6.0f;+ return copysign(r, v);+}+kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],@@ -101,6 +137,44 @@ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buif (e < len) r[j] = r[j] * inv * f.w[e];}}+ if (f.rd > 0) {+ // mrope: the last rd values move in (re, im) pairs a lane apart (ds4_kernels.cu:161): the partner comes from a+ // lane xor 1, the angle from the token's position and the frequency of the pair+ const uint r0 = len - uint(f.rd);+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ const float other = simd_shuffle_xor(r[j], 1);+ if (e >= r0 && e < len) {+ const uint i = (e - r0) >> 1;+ const float ang = float(pos) * f.freqs[i];+ const float c = cos(ang);+ const float sn = f.inverse ? -sin(ang) : sin(ang);+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;+ }+ }+ }+ if (f.quant != 0) { // the KV rounding without the pack: the group's scale, then the value rounded to the format+ const uint g = f.quant == 2 ? 16u : 32u;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ float a = e < len ? fabs(r[j]) : 0.0f;+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));+ if (e >= len) continue;+ if (f.quant == 1) {+ const float s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));+ r[j] = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f)) * s;+ } else if (f.quant == 2) {+ const float s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;+ } else {+ const float s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;+ }+ }+ }for (uint j = 0; j < 32u; ++j) {if (j >= nj) break;const uint e = tid + j * 256u;@@ -309,3 +383,1139 @@ kernel void ds4_moe_sort_fill(device const int* idx [[buffer(0)]], constant uintthreadgroup_barrier(mem_flags::mem_threadgroup);}}++// ds4_kernels.cu:1690 engram_dequant_tok_kernel - one hash-table row a threadgroup: IQ4_NL blocks to F32.+// fmt 2 (this model's Engram): an 18-byte block is fp16 d then 16 packed nibbles; the low nibble of q[j] is value j and the+// high nibble value j + 16, each an index into ggml-common.h's kvalues_iq4nl (ds4_kernels.cu:1280).+constant int ds4_kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};++kernel void ds4_engram_dequant(device const uchar* src [[buffer(0)]], constant ulong& per [[buffer(1)]],+ constant uint& cols [[buffer(2)]], constant uint& hd [[buffer(3)]],+ device float* out [[buffer(4)]], constant int& fmt [[buffer(5)]],+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint t = r / cols, c = r - t * cols;+ const device uchar* rows = src + (size_t) t * per + (size_t) c * hd;+ if (fmt == 2) {+ for (uint e = tid; e < hd; e += 256u) {+ const device uchar* b = rows + (size_t) (e / 32u) * 18u;+ const uint q = b[2u + (e & 15u)];+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);+ out[(size_t) r * hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);+ }+ return;+ }+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: write NaN, so a wrong path shows as NaN in the output rather than+ // as a silent zero.+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * hd + e] = NAN;+}++// ds4_kernels.cu:341 hc_pre_norm_kernel - per token: a = sum_s mix[s]*h[token][s][e], one RMS over dim, then x = a*inv+// (times w when given). One threadgroup a token, 1024 threads; a thread holds at most 8 values (dim <= 8*1024 = 8192).+kernel void ds4_hc_pre_norm(device const float* h [[buffer(0)]], device const float* mix [[buffer(1)]],+ device const float* w [[buffer(2)]], constant uint& dim [[buffer(3)]],+ constant uint& hc [[buffer(4)]], constant float& eps [[buffer(5)]],+ device float* x [[buffer(6)]], constant uint& has_w [[buffer(7)]],+ uint b [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const device float* hh = h + (size_t) b * hc * dim;+ const device float* mm = mix + (size_t) b * 8u;+ device float* xx = x + (size_t) b * dim;+ float r[8];+ float ss = 0.0f;+ uint n = 0;+ for (uint e = tid; e < dim; e += 1024u, ++n) {+ float a = 0.0f;+ for (uint s = 0; s < 8u; ++s) if (s < hc) a += mm[s] * hh[(size_t) s * dim + e];+ if (n < 8u) r[n] = a;+ ss += a * a;+ }+ // a 1024-thread reduction: 32 simdgroups, each into red[], then the first simdgroup sums those+ float v = ds4_warp_sum(ss);+ if (lane == 0u) red[wg] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wg == 0u) {+ float t = lane < 32u ? red[lane] : 0.0f;+ t = ds4_warp_sum(t);+ if (lane == 0u) red[0] = t;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const float inv = rsqrt(red[0] / float(dim) + eps);+ n = 0;+ for (uint e = tid; e < dim; e += 1024u, ++n) {+ if (n >= 8u) break;+ xx[e] = has_w != 0u ? r[n] * inv * w[e] : r[n];+ }+}++// ds4_gguf.cu:186 q8_quant_kernel: a warp a 32-group - amax across the warp, sc = amax/127, q = rint(v/sc), d = sc.+// grid = (ceil((k/32) / 8 warps), rows), threadgroup = 256 (8 warps of 32).+kernel void ds4_q8_quant(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],+ constant uint& k [[buffer(2)]], device char* q [[buffer(3)]],+ device float* d [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const uint warp = tid / 32u, lane = tid % 32u;+ const uint b = tgp.x * 8u + warp; // the 32-group index+ const uint r = tgp.y;+ if ((size_t) b * 32u >= (size_t) k) return;+ const device float* xr = x + (size_t) r * ldx;+ const float v = xr[(size_t) b * 32u + lane];+ const float am = simd_max(fabs(v));+ const float sc = am / 127.0f;+ q[(size_t) r * k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);+ if (lane == 0u) d[(size_t) r * (k / 32u) + b] = sc;+}++// ds4_gguf.cu:371 mx_mv_kernel - the MXFP4 expert matrix-vector.+// grid = (ceil(n / 8 warps), jobs), threadgroup = 256. A warp a row; a lane strides the 32-value blocks; NC columns of+// activations share each decoded block (the T=8 window), then a warp sum and y[c][row] = alpha*v + bias[row].+// Block layout (kMxBytes = 17 a 32 values): the E8M0 scale byte, then 16 bytes whose low nibble is value j and high nibble+// value j+16, each an index into the 16-entry codebook {0,1,2,3,4,6,8,12} and its negatives.+struct Ds4MxJob {+ device const void* w;+ ulong ld; // bytes a row+ device const float* x;+ ulong xcs;+ device float* y;+ ulong ycs;+ device const float* bias;+ float alpha;+ int k, n, ncol;+ device const char* xq;+ device const float* xd;+};++static inline float ds4_e8m0(uint e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));+}++static inline void ds4_mx_block(device const uchar* p, threadgroup const float* lut, thread float* w) {+ const float d = ds4_e8m0(uint(p[0]));+ for (uint j = 0; j < 16u; ++j) {+ const uint q = uint(p[1u + j]);+ w[j] = lut[q & 15u] * d;+ w[j + 16u] = lut[q >> 4] * d;+ }+}++kernel void ds4_mx_mv(device const uchar* jobs [[buffer(0)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]]) {+ threadgroup float lut[16];+ if (tid < 16u) {+ const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};+ lut[tid] = tid < 8u ? v[tid] : -v[tid - 8u];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;+ thread float acc[8];+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;+ const uint nblk = uint(b.k) / 32u;+ for (uint kb = lane; kb < nblk; kb += 32u) {+ float wv[32];+ ds4_mx_block(w + (size_t) kb * 17u, lut, wv);+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ const device float* x = b.x + c * b.xcs + (size_t) kb * 32u;+ float a = acc[c];+ for (uint j = 0; j < 32u; ++j) a = fma(wv[j], x[j], a);+ acc[c] = a;+ }+ }+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// ds4_gguf.cu:320 q8_mv_kernel - the Q8_0 matvec against quantised activations.+// grid = (ceil(n / 8), jobs), threadgroup 256: a warp a row, a lane strides the 34-byte blocks (fp16 d + 32 int8), the dot is+// integer (CUDA's __dp4a in four-value groups; here the same sum written out), then acc += (dw * xd[c]) * sum.+// The staged-prefetch (ST) and vector variants of the CUDA kernel are prefetch strategies only - this reads the blocks directly.+kernel void ds4_q8_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;+ const uint nb = uint(b.k) / 32u;+ float acc[8];+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;+ for (uint kb = lane; kb < nb; kb += 32u) {+ const device uchar* p = w + (size_t) kb * 34u;+ const ushort dh = ushort(p[0]) | (ushort(p[1]) << 8);+ const float dw = float(as_type<half>(dh));+ int wv[32];+ for (uint j = 0; j < 32u; ++j) wv[j] = int((char) p[2u + j]);+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + (size_t) kb * 32u;+ int sum = 0;+ for (uint j = 0; j < 32u; ++j) sum += wv[j] * int(xq[j]);+ acc[c] = fma(dw * b.xd[(size_t) c * nb + kb], float(sum), acc[c]);+ }+ }+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// ds4_kernels.cu:1092 ring_to_ext_kernel - one ring row of the KV cache to BF16: each value an E4M3 code times its group's+// E8M0 power of two (row[e] and row[d + e/32]); rows before the window start write zero.+static inline float ds4_e4m3(uint b) { // E4M3 (bias 7, no infinities, max 448): sign, 4-bit exp, 3-bit mantissa+ const float s = (b & 0x80u) ? -1.0f : 1.0f;+ const uint ex = (b >> 3) & 0xfu, m = b & 7u;+ if (ex == 0u) return s * float(m) * (1.0f / 8.0f) * (1.0f / 64.0f);+ if (ex == 15u && m == 7u) return NAN;+ return s * exp2(float(ex) - 7.0f) * (1.0f + float(m) * 0.125f);+}+static inline float ds4_ue8m0(uint b) { return as_type<float>(b << 23); } // 2^(b - 127)+static inline ushort ds4_f2bf16(float f) {+ const uint u = as_type<uint>(f);+ return ushort((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);+}+++// ds4_kernels.cu:1104 ext_to_ring_kernel - the window's BF16 rows packed back into the E4M3 ring, each 32-value group at the+// smallest power-of-two scale it fits under (k), so v / 2^k is an E4M3 value times a power of two.+static inline uint ds4_e4m3_code(float v, int k) {+ const uint sign = (v < 0.0f) ? 0x80u : 0u;+ float a = fabs(v) * exp2(-float(k));+ if (a == 0.0f) return sign;+ int e = int(floor(log2(a)));+ int mant = int(round((a / exp2(float(e)) - 1.0f) * 8.0f));+ int ex = e + 7;+ if (mant >= 8) { mant = 0; ++ex; }+ if (ex <= 0) { mant = 0; ex = 0; }+ if (ex > 15) { mant = 7; ex = 15; }+ return sign | (uint(ex) << 3) | uint(mant & 7);+}++kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],+ device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],+ constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],+ constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]],+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ if (T - t > n) return; // overwritten by a later row of the window anyway+ device uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;+ if (backup != nullptr) {+ for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const device ushort* src = ext + (size_t) (n + t) * d;+ for (uint e0 = 0; e0 < d; e0 += 256u) {+ const uint e = e0 + tid;+ const float v = e < d ? float(as_type<half>(src[e])) : 0.0f;+ uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bits+ for (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));+ const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);+ if (e >= d) continue;+ row[e] = uchar(ds4_e4m3_code(v, k));+ if ((e & 31u) == 0u) row[d + e / 32u] = uchar(k + 127);+ }+}+++// ds4_kernels.cu:916 unpack32 - 16 threads expand one MXFP4 KV row (288 B at d=512) into 32 BF16 values a thread:+// 16 code bytes at part*16, two E4M3 scale bytes at 256 + part*2, and the 4-bit code is itself an FP4 bit pattern+// (e2m1_val14: sign and exponent/mantissa bits spliced into a half).+static inline float ds4_e2m1_val14(uint c) {+ const ushort h = (ushort) (((c & 8u) << 12) | ((c & 7u) << 9));+ return float(as_type<half>(h));+}++// ds4_kernels.cu:937 unpack_sel_kernel - the selected rows of the compressed cache into the attention scratch, one row a+// group of 16 threads. n_sel comes either from the generated schedule (gen: min(kcap, (pos+1)/R)) or from nsel[t].+kernel void ds4_unpack_sel(device const uchar* ckv [[buffer(0)]], device const int* sel [[buffer(1)]],+ constant uint& sel_stride [[buffer(2)]], constant int& gen [[buffer(3)]],+ constant long& p0 [[buffer(4)]], constant int& t0 [[buffer(5)]],+ constant int& R [[buffer(6)]], constant int& kcap [[buffer(7)]],+ device const int* nsel [[buffer(8)]], constant uint& cap [[buffer(9)]],+ device ushort* out [[buffer(10)]], constant ulong& rb [[buffer(11)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint t = tgp.y;+ int n_sel;+ if (gen != 0) {+ const long pa = p0 + long(t0) + long(t);+ n_sel = R > 0 ? int(min(long(kcap), (pa + 1) / long(R))) : 0;+ } else {+ n_sel = nsel[t];+ }+ const uint i = tgp.x * 16u + (tid >> 4u), part = tid & 15u;+ if (int(i) >= n_sel) return;+ const size_t r = (size_t) t * cap + i;+ const device uchar* rr = ckv + (size_t) sel[(size_t) t * sel_stride + i] * rb;+ const uint4 cw = *(const device uint4*) (rr + part * 16u);+ const float k0 = ds4_e4m3(uint(rr[256u + part * 2u])) * 16384.0f;+ const float k1 = ds4_e4m3(uint(rr[256u + part * 2u + 1u])) * 16384.0f;+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};+ uint o[16];+ for (uint k = 0; k < 16u; ++k) {+ const uint b = wv[k >> 2] >> (8u * (k & 3u));+ const float ks = k < 8u ? k0 : k1;+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);+ }+ device uint* dst = (device uint*) (out + r * 512u + part * 32u);+ for (uint q = 0; q < 4u; ++q) {+ dst[q * 4u + 0u] = o[4u * q + 0u];+ dst[q * 4u + 1u] = o[4u * q + 1u];+ dst[q * 4u + 2u] = o[4u * q + 2u];+ dst[q * 4u + 3u] = o[4u * q + 3u];+ }+}+++// ds4_kernels.cu:1772 moe_combine_kernel - a token's expert outputs back into its row: for each pick whose routed row is in+// [r0, r1), acc += w[t][i] * y[r - r0][e]; untouched when no pick falls in range.+kernel void ds4_moe_combine(device float* acc [[buffer(0)]], device const float* y [[buffer(1)]],+ device const int* pos [[buffer(2)]], device const float* w [[buffer(3)]],+ constant int& kk [[buffer(4)]], constant int& r0 [[buffer(5)]],+ constant int& r1 [[buffer(6)]], constant uint& dim [[buffer(7)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + tid;+ if (e >= dim) return;+ const uint t = tgp.y;+ float a = acc[(size_t) t * dim + e];+ bool any = false;+ for (int i = 0; i < 8; ++i) {+ if (i >= kk) break;+ const int r = pos[(size_t) t * 8u + i];+ if (r < r0 || r >= r1) continue;+ a = fma(w[(size_t) t * 8u + i], y[(size_t) (r - r0) * dim + e], a);+ any = true;+ }+ if (any) acc[(size_t) t * dim + e] = a;+}+++// ds4_kernels.cu:362 hc_post_kernel - like hc_post_inplace, but reads a separate residual and writes a separate output:+// out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]+kernel void ds4_hc_post_v(device const float* y [[buffer(0)]], device const float* res [[buffer(1)]],+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],+ device float* out [[buffer(4)]], constant uint& dim [[buffer(5)]],+ constant uint& hc [[buffer(6)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const uint i = tgp.x * 256u + tid;+ if (i >= dim * hc) return;+ const uint t = tgp.y, k = i / dim, e = i - k * dim;+ const device float* r = res + (size_t) t * hc * dim;+ float a = post[(size_t) t * 8u + k] * y[(size_t) t * dim + e];+ for (uint j = 0; j < 8u; ++j) {+ if (j >= hc) break;+ a += comb[(size_t) t * 64u + j * hc + k] * r[(size_t) j * dim + e];+ }+ out[(size_t) t * hc * dim + i] = a;+}+++// ds4_kernels.cu:441 gemv_f32_kernel - a warp a row of F32 weights: float4 loads of W[row] and x[t], four FMAs a step,+// lane-strided over k/4, then the warp sum into y[t][row].+kernel void ds4_gemv_f32(device const float* W [[buffer(0)]], device const float* x [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],+ device float* y [[buffer(4)]], constant uint& ldy [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]]) {+ const uint row = tgp.x * 8u + (tid >> 5);+ if (row >= n) return;+ const uint t = tgp.y;+ const device float4* w = (const device float4*) (W + (size_t) row * kk);+ const device float4* xv = (const device float4*) (x + (size_t) t * kk);+ float acc = 0.0f;+ for (uint c = lane; c < kk / 4u; c += 32u) {+ const float4 a = w[c], b = xv[c];+ acc = fma(a.x, b.x, acc);+ acc = fma(a.y, b.y, acc);+ acc = fma(a.z, b.z, acc);+ acc = fma(a.w, b.w, acc);+ }+ acc += simd_shuffle_xor(acc, 16);+ acc += simd_shuffle_xor(acc, 8);+ acc += simd_shuffle_xor(acc, 4);+ acc += simd_shuffle_xor(acc, 2);+ acc += simd_shuffle_xor(acc, 1);+ if (lane == 0u) y[(size_t) t * ldy + row] = acc;+}+++// ds4_kernels.cu:1353 route_publish_kernel - the MoE router: sc = sqrt(softplus(logit)), bs = sc + bias, then k rounds of a+// block-wide argmax (a strided max, a warp reduction, a 16-warp pass), the picks' sc summed into the weights, and each pick's+// device address published to plan_ptr. The prefetch requests (req) are NOT published here yet: that is the schedule, not the+// answer, and it can be turned on separately.+kernel void ds4_route_publish(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],+ constant float& route_scale [[buffer(4)]], device const ulong* vram [[buffer(5)]],+ device int* idx [[buffer(6)]], device float* w [[buffer(7)]],+ device ulong* plan_ptr [[buffer(8)]], device const float* bias_vl [[buffer(9)]],+ constant uint& vl [[buffer(10)]], constant uint& has_vl [[buffer(11)]],+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ threadgroup float sc[1024];+ threadgroup float bs[1024];+ threadgroup float bv[16];+ threadgroup int bi[16];+ threadgroup int pick[8];+ const device float* lg = logits + (size_t) t * 1024u;+ const device float* b = (has_vl != 0u && ((vl >> t) & 1u) != 0u) ? bias_vl : bias;+ for (uint i = tid; i < n; i += 512u) {+ const float l = lg[i];+ const float sp = l > 20.0f ? l : log(1.0f + exp(l));+ sc[i] = sqrt(sp);+ bs[i] = sc[i] + b[i];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const uint lane = tid & 31u, wp = tid >> 5u;+ for (uint r = 0; r < 8u; ++r) {+ if (r >= kk) break;+ float best = -3.402823466e+38f;+ int bix = 0x7fffffff;+ for (uint i = tid; i < n; i += 512u) {+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) { best = bs[i]; bix = int(i); }+ }+ for (uint o = 16u; o > 0u; o >>= 1u) {+ const float ov = simd_shuffle_xor(best, (ushort) o);+ const int oi = simd_shuffle_xor(bix, (ushort) o);+ if (ov > best || (ov == best && oi < bix)) { best = ov; bix = oi; }+ }+ if (lane == 0u && wp < 16u) { bv[wp] = best; bi[wp] = bix; }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) {+ float bb = -3.402823466e+38f;+ int ix = 0x7fffffff;+ for (uint q = 0; q < 16u; ++q) {+ if (bv[q] > bb || (bv[q] == bb && bi[q] < ix)) { bb = bv[q]; ix = bi[q]; }+ }+ pick[r] = ix;+ bs[ix] = -3.402823466e+38f;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (tid == 0u) {+ float sum = 0.0f;+ for (uint r = 0; r < 8u; ++r) { if (r >= kk) break; sum += sc[pick[r]]; }+ for (uint r = 0; r < 8u; ++r) {+ if (r >= kk) break;+ const uint p_ = uint(pick[r]);+ const float wv = sc[p_] / (sum + 1e-20f) * route_scale;+ idx[(size_t) t * 8u + r] = int(p_);+ w[(size_t) t * 8u + r] = wv;+ plan_ptr[(size_t) t * 8u + r] = vram[p_];+ }+ }+}+++// ds4_kernels.cu swiglu_kernel - the expert MLP's activation: h = silu(g) * u, with the clamp when lim > 0+kernel void ds4_swiglu(device const float* g [[buffer(0)]], device const float* u [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant float& lim [[buffer(3)]],+ device float* h [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ if (gid >= n) return;+ float gv = g[gid], uv = u[gid];+ if (lim > 0.0f) { // both inputs clamped, then the gate itself+ gv = fmin(gv, lim);+ uv = fmin(fmax(uv, -lim), lim);+ }+ h[gid] = (gv / (1.0f + exp(-gv))) * uv; // silu(g) * u+}++kernel void ds4_ring_to_ext(device const uchar* ring [[buffer(0)]], device ushort* ext [[buffer(1)]],+ constant long& p0 [[buffer(2)]], constant uint& n [[buffer(3)]],+ constant uint& d [[buffer(4)]], constant ulong& rb [[buffer(5)]],+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const long p = p0 - long(n) + long(r);+ const device uchar* row = ring + (size_t) (p >= 0 ? (p % long(n)) : 0) * rb;+ for (uint e = tid; e < d; e += 256u) {+ ext[(size_t) r * d + e] = p >= 0 ? ds4_f2bf16(ds4_e4m3(uint(row[e])) * ds4_ue8m0(uint(row[d + e / 32u]))) : ushort(0);+ }+}++// ---------------------------------------------------------------- ds4_gguf.cu:577 mx_combine_kernel+//+// A token's expert outputs summed in plan order (an expert off the device adds 0), the CPU lane's part added when its+// flag is set, then the shared expert's - ds4_gguf.cu:169's combine(). Grid (n_embd / 128, T), 128 threads.+struct Ds4Comb {+ device const ulong* plan_ptr; // [T][8]: nonzero when the device holds that expert+ device const float* plan_w; // [T][8]+ device const float* part; // [entries][n_embd]+ device const float* cpu_part; // [T][cpu_stride], or null+ device const int* cpu_flag; // [T], or null+ device const float* sh_out; // [T][n_embd], or null+ device float* out;+ long cpu_stride, out_stride;+ int k, n_embd;+};++kernel void ds4_mx_combine(constant Ds4Comb& a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const uint r = tgp.x * 128u + tid;+ if (int(r) >= a.n_embd) return;+ const int t = int(tgp.y);+ float m = 0.0f;+ for (int i = 0; i < a.k; ++i) { // kPicks <= 8+ const int e = t * 8 + i;+ const float pi = a.plan_ptr[e] != 0ul ? a.part[(size_t) e * (size_t) a.n_embd + r] : 0.0f;+ m = fma(a.plan_w[t * 8 + i], pi, m);+ }+ if (a.cpu_flag != nullptr && a.cpu_flag[t] != 0) m += a.cpu_part[(size_t) t * (size_t) a.cpu_stride + r];+ m += a.sh_out != nullptr ? a.sh_out[(size_t) t * (size_t) a.n_embd + r] : 0.0f;+ a.out[(size_t) t * (size_t) a.out_stride + r] = m;+}++// ---------------------------------------------------------------- ds4_kernels.cu:1286 engram_dequant_kernel / :1311 engram_gate_kernel+//+// The Engram tables' row decoder and gate (this model's tables are fmt 2: IQ4_NL, an 18-byte block of 32 values whose+// nibbles index ds4_kIq4nl above).+kernel void ds4_engram_dequant_rows(device const uchar* rows [[buffer(0)]], constant uint& hd [[buffer(1)]],+ device float* out [[buffer(2)]], constant int& fmt [[buffer(3)]],+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ if (fmt == 2) {+ const device uchar* row = rows + (size_t) r * (size_t) hd;+ for (uint e = tid; e < hd; e += 256u) {+ const device uchar* b = row + (size_t) (e / 32u) * 18u;+ const uint q = b[2u + (e & 15u)];+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);+ out[(size_t) r * (size_t) hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);+ }+ return;+ }+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: NaN rather than a silent zero (as ds4_engram_dequant does)+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * (size_t) hd + e] = NAN;+}++// ds4_kernels.cu's block_sum for a 1024-thread threadgroup: a warp sum a warp, then every warp sums the 32 partials (the+// same result in all of them, so no warp diverges at a barrier)+static inline float ds4_block_sum32(float v, threadgroup float* red, ushort lane, ushort w) {+ v = ds4_warp_sum(v);+ if (lane == 0) red[w] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float t = lane < 32u ? red[lane] : 0.0f;+ t = ds4_warp_sum(t);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return t;+}++kernel void ds4_engram_gate(device float* h [[buffer(0)]], device const float* kv [[buffer(1)]],+ device const float* w [[buffer(2)]], constant uint& hc [[buffer(3)]],+ constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const uint k = tgp.x, t = tgp.y;+ device float* hk = h + ((size_t) t * hc + (size_t) k) * (size_t) dim;+ const device float* kvt = kv + (size_t) t * (size_t) (hc + 1u) * (size_t) dim;+ const device float* key = kvt + (size_t) k * (size_t) dim;+ const device float* val = kvt + (size_t) hc * (size_t) dim;+ const device float* wk = w + (size_t) k * (size_t) dim;+ float sh = 0.0f, sk = 0.0f, sd = 0.0f;+ for (uint e = tid; e < dim; e += 1024u) {+ const float hv = hk[e], kve = key[e];+ sh += hv * hv;+ sk += kve * kve;+ sd += hv * wk[e] * kve;+ }+ sh = ds4_block_sum32(sh, red, lane, wg);+ sk = ds4_block_sum32(sk, red, lane, wg);+ sd = ds4_block_sum32(sd, red, lane, wg);+ const float rstd = rsqrt(sh / float(dim) + eps) * rsqrt(sk / float(dim) + eps);+ const float dot = sd * rstd * rsqrt(float(dim));+ const float sq = copysign(sqrt(fmax(fabs(dot), 1e-6f)), dot);+ const float gate = 1.0f / (1.0f + exp(-sq));+ for (uint e = tid; e < dim; e += 1024u) hk[e] = hk[e] + gate * val[e];+}++// ds4_gguf.cu:72 bf16_mv_kernel<NC, true> - a BF16 matrix's matvec against F32 activations (the engram WKV here, and the+// BF16 dense matrices the jobs' f == 0 names). Same frame as ds4_mx_mv above: a warp a row, a lane strides the row; here a+// lane 8 BF16 weights (16 bytes = one uint4) and 8 activations, so the row is walked in 256-value steps.+// A pair's F32 is bit-exact - the low half zeroed - which is bf16_mv_kernel's __uint_as_float(w << 16).+kernel void ds4_bf16_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const device ushort* w = (const device ushort*) b.w + (size_t) row * b.ld;+ thread float acc[16]; // (the CUDA template's largest NC; ncol cuts the rest)+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;+ for (uint i = uint(lane) * 8u; i < uint(b.k); i += 256u) {+ const uint4 q = *(const device uint4*) (w + i);+ const float wf[8] = {as_type<float>(q.x << 16), as_type<float>(q.x & 0xffff0000u),+ as_type<float>(q.y << 16), as_type<float>(q.y & 0xffff0000u),+ as_type<float>(q.z << 16), as_type<float>(q.z & 0xffff0000u),+ as_type<float>(q.w << 16), as_type<float>(q.w & 0xffff0000u)};+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device float4* xp = (const device float4*) (b.x + c * b.xcs + (size_t) i);+ const float4 x0 = xp[0], x1 = xp[1];+ float a = acc[c];+ a = fma(wf[0], x0.x, a);+ a = fma(wf[1], x0.y, a);+ a = fma(wf[2], x0.z, a);+ a = fma(wf[3], x0.w, a);+ a = fma(wf[4], x1.x, a);+ a = fma(wf[5], x1.y, a);+ a = fma(wf[6], x1.z, a);+ a = fma(wf[7], x1.w, a);+ acc[c] = a;+ }+ }+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// ---------------------------------------------------------------- ds4_kernels.cu:524 compress_kernel / :582 index_score_kernel+//+// The compressor (ratio > 1 layers): each token's kv / score row into its slot, and every ratio'th token pooled by a+// per-value softmax over the slots, scaled by a row RMS and the norm weights. Grid (1), 512 threads (d <= 1024 values).+static inline float ds4_block_sum_n(float v, threadgroup float* red, ushort lane, ushort w, uint nw) {+ v = ds4_warp_sum(v);+ if (lane == 0) red[w] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float t = lane < nw ? red[lane] : 0.0f;+ t = ds4_warp_sum(t); // (every warp sums the same partials: no divergence at a barrier)+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return t;+}++kernel void ds4_compress(device const float* kv [[buffer(0)]], device const float* score [[buffer(1)]],+ device float* kv_state [[buffer(2)]], device float* score_state [[buffer(3)]],+ constant int& ratio [[buffer(4)]], constant int& p0 [[buffer(5)]], constant int& T [[buffer(6)]],+ constant uint& d [[buffer(7)]], device const float* wn [[buffer(8)]],+ constant float& eps [[buffer(9)]], device float* latent [[buffer(10)]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort wg [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const float kfmax = -3.4028234663852886e+38f; // -FLT_MAX, as the CUDA writes it+ int g = 0;+ for (int t = 0; t < T; ++t) {+ const int slot = (p0 + t) % ratio;+ const bool pool = (p0 + t + 1) % ratio == 0;+ for (uint e = tid; e < d; e += 512u) {+ kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];+ score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (!pool) continue;+ float val[2] = {0.0f, 0.0f}; // (d <= 2 * 512)+ float ss = 0.0f;+ uint n = 0;+ for (uint e = tid; e < d; e += 512u, ++n) {+ float m = kfmax;+ for (int s = 0; s < ratio; ++s) m = fmax(m, score_state[(size_t) s * d + e]);+ float den = 0.0f;+ for (int s = 0; s < ratio; ++s) den += exp(score_state[(size_t) s * d + e] - m);+ float acc = 0.0f;+ for (int s = 0; s < ratio; ++s)+ acc += kv_state[(size_t) s * d + e] * (exp(score_state[(size_t) s * d + e] - m) / den);+ val[n] = acc;+ ss += acc * acc;+ }+ const float inv = rsqrt(ds4_block_sum_n(ss, red, lane, wg, 16u) / float(d) + eps);+ n = 0;+ for (uint e = tid; e < d; e += 512u, ++n) latent[(size_t) g * d + e] = val[n] * inv * wn[e];+ ++g;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+}++// ---------------------------------------------------------------- ds4_kernels.cu:582 index_score_kernel<128, KF>+//+// A thread a position: its key row (kfmt 3: 64 bytes of E2M1 codes (value 2i in the low nibble) then 4 UE8M0 scales; kfmt+// 0: BF16) against every head's query (a threadgroup tile), relu, then the heads' weighted sum. Grid (ceil(n / 256), T).+struct Ds4Rows { // ds4_kernels.cuh:115 Rows (kWin = 16)+ int n[16];+ int k[16];+ int gen, R, kcap, ccap, blk;+ long p0;+};++static inline void ds4_rows_at(constant Ds4Rows& r, int t, thread int& n, thread int& k) {+ if (r.gen == 0) {+ n = r.n[t];+ k = r.k[t];+ return;+ }+ const int nn = int((r.p0 + t + 1) / r.R);+ const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);+ switch (r.gen) {+ case 1: n = nn; k = min(r.kcap, nn); break;+ case 2: n = nn; k = cb; break;+ case 3: n = cb * r.blk; k = min(r.kcap, nn); break;+ default: n = (nn + r.blk - 1) / r.blk; k = r.kcap; break;+ }+}++kernel void ds4_index_score(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],+ device const uchar* kc [[buffer(2)]], device const int* cand [[buffer(3)]],+ device float* out [[buffer(4)]], constant Ds4Rows& lim [[buffer(5)]],+ constant int& heads [[buffer(6)]], constant int& n [[buffer(7)]],+ constant int& cand_stride [[buffer(8)]], constant int& blk [[buffer(9)]],+ constant long& out_stride [[buffer(10)]], constant int& kfmt [[buffer(11)]],+ constant int& has_cand [[buffer(12)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ constexpr int HD = 128;+ threadgroup float qs[32 * HD + 32];+ const int t = int(tgp.y);+ for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qs[i] = q[(size_t) t * (size_t) heads * HD + i];+ for (uint i = tid; i < uint(heads); i += 256u) qs[uint(heads) * HD + i] = wq[(size_t) t * 64 + i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int i = int(tgp.x) * 256 + int(tid);+ if (i >= n) return;+ device float* o = out + (long) t * out_stride;+ int ln, lk;+ ds4_rows_at(lim, t, ln, lk);+ int p = i;+ if (has_cand != 0) {+ if (i >= lk * blk) {+ o[i] = -INFINITY;+ return;+ }+ p = cand[(size_t) t * (size_t) cand_stride + (size_t) (i / blk)] * blk + i % blk;+ }+ if (p >= ln) {+ o[i] = -INFINITY;+ return;+ }+ const size_t kb = kfmt == 3 ? (size_t) ((HD / 2 + HD / 32 + 15) / 16 * 16) : (size_t) HD * 2;+ const device uchar* kr = kc + (size_t) p * kb;+ float acc[32];+ for (uint h = 0; h < 32u; ++h) acc[h] = 0.0f;+ for (int c = 0; c < HD; c += 8) {+ float kvv[8];+ if (kfmt == 3) { // 8 codes in 4 bytes, the group's power-of-two scale (2^(b - 127))+ const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |+ (uint(kr[c / 2 + 3]) << 24);+ const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);+ for (uint x = 0; x < 8u; ++x) kvv[x] = ds4_e2m1_val14(cw >> (4 * x)) * 16384.0f * ks;+ } else {+ const device ushort* b = (const device ushort*) (kr + (size_t) c * 2);+ for (uint x = 0; x < 8u; ++x) kvv[x] = as_type<float>(uint(b[x]) << 16);+ }+ for (uint h = 0; h < 32u; ++h) {+ if (int(h) >= heads) break;+ threadgroup const float* qh = qs + h * HD + c;+ for (uint x = 0; x < 8u; ++x) acc[h] += qh[x] * kvv[x];+ }+ }+ float s = 0.0f;+ for (uint h = 0; h < 32u; ++h) {+ if (int(h) >= heads) break;+ s += fmax(acc[h], 0.0f) * qs[uint(heads) * HD + h];+ }+ o[i] = s;+}++// ---------------------------------------------------------------- ds4_kernels.cu:142 finish_kernel<true> (the KV pack)+//+// The same load, RMS and mrope as ds4_finish above, then the row written as the cache's packed codes (kv_row_bytes): the+// quantiser's scale byte a group (32 values, or 16 for the 4-bit MXFP4-style rows), the codes themselves from bit 0 up -+// for the 4-bit formats two values a byte, the second from the next lane. Written from the CUDA of the same name.++++static inline uint ds4_e2m1_code(float q) { // (bit 3 the sign, -0 kept; the magnitudes as 0 .. 7)+ const float a = fabs(q);+ const uint m = a <= 1.5f ? uint(a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;+ return m | ((as_type<uint>(q) >> 31) << 3);+}++kernel void ds4_finish_pack(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {+ threadgroup float red[8];+ device float* v = x + (long) b * stride;+ const uint nj = (len + 255u) / 256u;+ float r[32];+ float ss = 0.0f;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ r[j] = e < len ? v[e] : 0.0f;+ ss += r[j] * r[j];+ }+ if (f.w != nullptr) {+ const float tot = ds4_block_sum(ss, red, lane, w);+ const float inv = rsqrt(tot / float(len) + f.eps);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ if (e < len) r[j] = r[j] * inv * f.w[e];+ }+ }+ if (f.rd > 0) {+ const uint r0 = len - uint(f.rd);+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ const float other = simd_shuffle_xor(r[j], 1);+ if (e >= r0 && e < len) {+ const uint i = (e - r0) >> 1;+ const float ang = float(pos) * f.freqs[i];+ const float c = cos(ang);+ const float sn = f.inverse ? -sin(ang) : sin(ang);+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;+ }+ }+ }+ // the packed row: a group's scale and the group's codes as the format's (quant 1: d E4M3 codes at pr[e] and d/32+ // scale bytes at pr[len + e/32]; 2: E2M1 pairs at pr[e/2] with E4M3 scales at pr[len/2 + e/16]; 3: E2M1 pairs with+ // UE8M0 scales at pr[len/2 + e/32])+ const uint g = f.quant == 2 ? 16u : 32u;+ device uchar* pr = f.pdst + (long) b * f.pdst_stride;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ float a = e < len ? fabs(r[j]) : 0.0f;+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));+ float q = 0.0f, s = 1.0f;+ if (e < len) {+ if (f.quant == 1) {+ s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));+ q = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f));+ } else if (f.quant == 2) {+ s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));+ } else {+ s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));+ }+ r[j] = q * s;+ }+ if (f.quant == 1) {+ if (e < len) pr[e] = uchar(ds4_e4m3_code(q, 0));+ } else {+ const uint c = ds4_e2m1_code(q);+ const uint hi = simd_shuffle_down(c, 1); // (value e + 1: the next lane's)+ if (e < len && (e & 1u) == 0u) pr[e >> 1] = uchar(c | (hi << 4));+ }+ if (e < len && e % g == 0u)+ pr[(f.quant == 1 ? len : len / 2u) + e / g] =+ uchar(f.quant == 2 ? ds4_e4m3_code(s, 0) : (as_type<uint>(s) >> 23));+ }+ device ushort* d = f.dst != nullptr ? f.dst + (long) b * f.dst_stride : nullptr;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ if (e >= len) continue;+ if (d != nullptr) {+ const uint bits = as_type<uint>(r[j]);+ d[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even+ }+ }+}++// ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations (the indexer's wproj here): a warp a row+// of the matrix, a lane 8 halves (16 bytes, the uint4 above) and 8 activations a step, kRowsT = 2 tokens a block. Grid+// (ceil(n / 8), ceil(T / 2)), threadgroup 256.+kernel void ds4_gemv_f16_rows(device const ushort* W [[buffer(0)]], device const float* x [[buffer(1)]],+ constant ulong& ldx [[buffer(2)]], constant int& n [[buffer(3)]],+ constant int& k [[buffer(4)]], constant int& T [[buffer(5)]],+ constant float& alpha [[buffer(6)]], device float* y [[buffer(7)]],+ constant ulong& ldy [[buffer(8)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ constexpr int kRowsT = 2;+ const int row = int(tgp.x) * 8 + int(tid >> 5);+ const int t0 = int(tgp.y) * kRowsT, nt = min(kRowsT, T - t0);+ if (row >= n) return;+ const device ushort* w = W + (size_t) row * (size_t) k;+ float acc[kRowsT];+ for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;+ for (int c = int(lane) * 8; c < k; c += 32 * 8) {+ const uint4 q = *(const device uint4*) (w + c);+ const float2 f0 = float2(as_type<half2>(q.x)), f1 = float2(as_type<half2>(q.y));+ const float2 f2 = float2(as_type<half2>(q.z)), f3 = float2(as_type<half2>(q.w));+ const float2 fs[4] = {f0, f1, f2, f3};+ float xv[kRowsT][8];+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) {+ const device float4* xp = (const device float4*) (x + (size_t) (t0 + t) * ldx + (size_t) c);+ const float4 a4 = xp[0], b4 = xp[1];+ xv[t][0] = a4.x;+ xv[t][1] = a4.y;+ xv[t][2] = a4.z;+ xv[t][3] = a4.w;+ xv[t][4] = b4.x;+ xv[t][5] = b4.y;+ xv[t][6] = b4.z;+ xv[t][7] = b4.w;+ }+ for (int i = 0; i < 4; ++i) {+ const float2 f = fs[i];+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];+ }+ }+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) {+ float a = acc[t];+ a += simd_shuffle_xor(a, 16);+ a += simd_shuffle_xor(a, 8);+ a += simd_shuffle_xor(a, 4);+ a += simd_shuffle_xor(a, 2);+ a += simd_shuffle_xor(a, 1);+ if (lane == 0u) y[(size_t) (t0 + t) * ldy + row] = alpha * a;+ }+}++// ---------------------------------------------------------------- ds4_kernels.cu:650 topk_kernel+//+// A token's k largest scores: four 8-bit radix passes over the order keys of the row (a threadgroup of 1024, shared+// histograms), then the picks written in index order by a scan a pass, so the attention sums them in the same order every+// run and the ties kept are the lowest positions.+static inline uint ds4_ord_key(float f) {+ const uint b = as_type<uint>(f);+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);+}++kernel void ds4_topk(device const float* vals [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant Ds4Rows& r [[buffer(2)]], device const int* cand [[buffer(3)]],+ constant int& cand_stride [[buffer(4)]], constant int& blk [[buffer(5)]],+ device int* out [[buffer(6)]], constant int& out_stride [[buffer(7)]],+ constant int& has_cand [[buffer(8)]], uint tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort wid [[simdgroup_index_in_threadgroup]]) {+ threadgroup uint hist[256];+ threadgroup uint prefix_s, mask_s;+ threadgroup int remaining_s, cnt_gt, cnt_eq;+ threadgroup int wgt[32], weq[32];+ const int t = int(tgp);+ int n, k;+ ds4_rows_at(r, t, n, k);+ device const float* v = vals + (long) t * stride;+ device int* o = out + (size_t) t * (size_t) out_stride;+ device const int* cd = has_cand != 0 ? cand + (size_t) t * (size_t) cand_stride : nullptr;+ if (n <= 0) return;+ if (k >= n) {+ for (uint i = tid; i < uint(n); i += 1024u)+ o[i] = cd != nullptr ? cd[i / uint(blk)] * blk + int(i % uint(blk)) : int(i);+ return;+ }+ if (tid == 0u) {+ prefix_s = 0u;+ mask_s = 0u;+ remaining_s = k;+ cnt_gt = 0;+ cnt_eq = 0;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int shift = 24; shift >= 0; shift -= 8) {+ for (uint i = tid; i < 256u; i += 1024u) hist[i] = 0u;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const uint prefix = prefix_s, mask = mask_s;+ for (int i = int(tid); i < n; i += 1024) {+ const uint u = ds4_ord_key(v[i]);+ if ((u & mask) == prefix) {+ threadgroup atomic_uint* hp = (threadgroup atomic_uint*) &hist[(u >> shift) & 0xffu];+ atomic_fetch_add_explicit(hp, 1u, memory_order_relaxed);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) {+ const int rem = remaining_s;+ int cum = 0;+ for (int d = 255; d >= 0; --d) {+ const int h = int(hist[d]);+ if (cum + h >= rem) {+ prefix_s = prefix | (uint(d) << shift);+ mask_s = mask | (0xffu << shift);+ remaining_s = rem - cum;+ break;+ }+ cum += h;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const uint thr = prefix_s;+ const int remaining = remaining_s;+ const int n_gt = k - remaining;+ for (int i0 = 0; i0 < n; i0 += 1024) {+ const int i = i0 + int(tid);+ const uint u = i < n ? ds4_ord_key(v[i]) : 0u;+ const bool gt = i < n && u > thr, eq = i < n && u == thr;+ const auto bgv = simd_ballot(gt);+ const auto bev = simd_ballot(eq);+ const uint bg = uint((simd_vote::vote_t)bgv), be = uint((simd_vote::vote_t)bev);+ if (lane == 0u) {+ wgt[wid] = int(popcount(bg));+ weq[wid] = int(popcount(be));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wid == 0u) {+ const int g0 = wgt[lane], e0 = weq[lane];+ int gs = g0, es = e0;+ for (uint o2 = 1u; o2 < 32u; o2 <<= 1) {+ const int a = simd_shuffle_up(gs, ushort(o2)), b = simd_shuffle_up(es, ushort(o2));+ if (uint(lane) >= o2) {+ gs += a;+ es += b;+ }+ }+ wgt[lane] = cnt_gt + gs - g0;+ weq[lane] = cnt_eq + es - e0;+ simdgroup_barrier(mem_flags::mem_none);+ if (lane == 31u) {+ cnt_gt += gs;+ cnt_eq += es;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ int slot = -1;+ const uint below = (1u << lane) - 1u;+ if (gt) slot = wgt[wid] + int(popcount(bg & below));+ else if (eq) {+ const int e = weq[wid] + int(popcount(be & below));+ if (e < remaining) slot = n_gt + e;+ }+ if (slot >= 0) o[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+}++// ds4_kernels.cu:751 block_max_kernel - a block of `blk` positions' largest score (the indexer's candidate blocks), the+// newest position's block pinned in at +inf. Grid (ceil(nb_max / 256), T), 256 threads.+kernel void ds4_block_max(device const float* score [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant Ds4Rows& r [[buffer(2)]], constant int& blk [[buffer(3)]],+ device float* bm [[buffer(4)]], constant long& bm_stride [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const int b = int(tgp.x) * 256 + int(tid), t = int(tgp.y);+ int n, k_;+ ds4_rows_at(r, t, n, k_);+ const int nb = (n + blk - 1) / blk;+ if (b >= nb) return;+ float m = -INFINITY;+ for (int j = 0; j < blk; ++j) {+ const int p = b * blk + j;+ if (p < n) m = fmax(m, score[(long) t * stride + p]);+ }+ if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in+ bm[(long) t * bm_stride + b] = m;+}++// ---------------------------------------------------------------- ds4_kernels.cu:1141 hc_mean_kernel / :1150 iota_rows_kernel+//+// The two small shapes the decode loop asks for: a token's mean over its hc streams, and a row's indices.+kernel void ds4_hc_mean(device const float* h [[buffer(0)]], constant int& hc [[buffer(1)]],+ constant int& dim [[buffer(2)]], device float* out [[buffer(3)]],+ constant long& out_stride [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const int e = int(tgp.x) * 256 + int(tid), t = int(tgp.y);+ if (e >= dim) return;+ float a = 0.0f;+ for (int s = 0; s < hc; ++s) a += h[((size_t) t * (size_t) hc + (size_t) s) * (size_t) dim + (size_t) e];+ out[(long) t * out_stride + e] = a / float(hc);+}++kernel void ds4_iota_rows(device int* out [[buffer(0)]], constant int& n [[buffer(1)]], constant int& stride [[buffer(2)]],+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint t = tg;+ for (uint i = tid; i < uint(n); i += 64u) out[(size_t) t * (size_t) stride + i] = int(i);+}++// ---------------------------------------------------------------- vendor/sampler.cu:93 sampler_greedy_kernel+//+// The plain argmax over the vocabulary, ONE BLOCK (1024) a token: a thread's strided candidates with the repetition+// penalties applied (a threadgroup bitmap of the history window says which tokens can be penalised at all), a warp+// reduction, then warp 0's - a tie keeps the larger value and the SMALLER index, so this is the serial scan's answer.+kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constant int& n_vocab [[buffer(1)]],+ device const int* history [[buffer(2)]], constant int& history_len [[buffer(3)]],+ constant int& plen [[buffer(4)]], constant float& pen_repeat [[buffer(5)]],+ constant float& pen_freq [[buffer(6)]], constant float& pen_present [[buffer(7)]],+ device const float* logit_bias [[buffer(8)]], constant int& has_bias [[buffer(9)]],+ device int* out [[buffer(10)]], uint tg [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort wid [[simdgroup_index_in_threadgroup]]) {+ threadgroup uint penal_bits[4096]; // (n_vocab + 31) / 32 words (the host checks it fits)+ threadgroup float sv[32];+ threadgroup int si[32];+ const int t = int(tg);+ device const float* l = logits + (size_t) t * (size_t) n_vocab;+ device const int* hrow = history != nullptr ? history + (size_t) t * (size_t) history_len : nullptr;+ int hlen = 0;+ if (hrow != nullptr) {+ hlen = min(plen, history_len);+ if (hlen < 0) hlen = 0;+ hrow += history_len - hlen; // the window is the tail+ }+ const int bits_words = (n_vocab + 31) / 32;+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;+ if (use_bits) {+ for (int w = int(tid); w < bits_words; w += 1024) penal_bits[w] = 0u;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int i = int(tid); i < hlen; i += 1024)+ if (hrow[i] >= 0 && hrow[i] < n_vocab) {+ threadgroup atomic_uint* pb = (threadgroup atomic_uint*) &penal_bits[hrow[i] >> 5];+ atomic_fetch_or_explicit(pb, 1u << (hrow[i] & 31), memory_order_relaxed);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ float bv = -INFINITY;+ int best = n_vocab; // (the "no candidate" index: it loses every comparison)+ for (int v = int(tid); v < n_vocab; v += 1024) {+ float s = has_bias != 0 ? l[v] + logit_bias[v] : l[v];+ int cnt = 0;+ if (use_bits && (penal_bits[v >> 5] & (1u << (v & 31))) != 0u)+ for (int i = 0; i < hlen; ++i)+ if (hrow[i] == v) ++cnt;+ if (cnt > 0) {+ if (s <= 0.0f) s *= pen_repeat;+ else s /= pen_repeat;+ s -= float(cnt) * pen_freq + 1.0f * pen_present;+ }+ if (s > bv) {+ bv = s;+ best = v;+ }+ }+ for (uint o = 16u; o > 0u; o >>= 1) {+ const float ov = simd_shuffle_down(bv, ushort(o));+ const int oi = simd_shuffle_down(best, ushort(o));+ if (ov > bv || (ov == bv && oi < best)) {+ bv = ov;+ best = oi;+ }+ }+ if (lane == 0u) {+ sv[wid] = bv;+ si[wid] = best;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wid == 0u) {+ float wv = sv[lane];+ int wi = si[lane];+ for (uint o = 16u; o > 0u; o >>= 1) {+ const float ov = simd_shuffle_down(wv, ushort(o));+ const int oi = simd_shuffle_down(wi, ushort(o));+ if (ov > wv || (ov == wv && oi < wi)) {+ wv = ov;+ wi = oi;+ }+ }+ if (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)+ }+}diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metalnew file mode 100644index 000000000..7e9fb5aa5--- /dev/null+++ b/metal/src/engine/msl/ds4_iq.metal@@ -0,0 +1,361 @@+#include <metal_stdlib>+using namespace metal;+// ds4_iq.cu:809 mv_kernel - the I-quant expert matvec. Same frame as the MXFP4/Q8_0 ones (a warp a row, a lane strides the+// 256-value blocks, eight accumulator columns for the token window), with the block decoder chosen per job by b.type:+// 16 IQ2_XXS (66 B), 17 IQ2_XS (74 B), 22 IQ2_S (82 B). The activations are the quantiser's int8 (xq) with their scales (xd),+// so each sub-group's dot is integer and the two scales fold in at the end - the same arithmetic as the float probes, at a+// quarter of the traffic.+struct Ds4MvJob {+ device const uchar* w;+ ulong ld;+ int type, k, n, ncol;+ device const char* xq;+ device const float* xd;+ device float* y;+ ulong ycs;+ device const float* bias;+ float alpha;+};++kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nsb = uint(b.k) / 32u;+ const uint nblk = uint(b.k) / 256u;+ const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : 82u);+ const device uchar* w = b.w + (size_t) row * b.ld;+ float acc[8];+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;+ for (uint blk = lane; blk < nblk; blk += 32u) {+ const device uchar* p = w + (size_t) blk * bs;+ const float d = float(as_type<half>(*(const device ushort*) p));+ for (uint ib = 0; ib < 8u; ++ib) {+ float db0, db1;+ uint idx[4];+ uint sg[4];+ if (b.type == 16) {+ const uint aux0 = uint(p[2 + 4 * ib]) | (uint(p[3 + 4 * ib]) << 8) | (uint(p[4 + 4 * ib]) << 16) |+ (uint(p[5 + 4 * ib]) << 24);+ const uint aux1 = uint(p[6 + 4 * ib]) | (uint(p[7 + 4 * ib]) << 8) | (uint(p[8 + 4 * ib]) << 16) |+ (uint(p[9 + 4 * ib]) << 24);+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ db0 = db1 = db;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = (aux0 >> (8u * l)) & 0xffu;+ sg[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ }+ } else if (b.type == 17) {+ const device ushort* qs = (const device ushort*) (p + 2);+ const device uchar* sc = p + 66;+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[4u * ib + l]);+ idx[l] = q & 511u;+ sg[l] = 0xffffffffu ^ ((q >> 9) * 0x00204081u); // the 7 sign bits, expanded (ggml's sign trick)+ }+ } else {+ const device uchar* qs = p + 2;+ const device uchar* sgb = qs + 32;+ const device uchar* qh = p + 66;+ const device uchar* sc = p + 74;+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = uint(qs[4u * ib + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);+ sg[l] = uint(sgb[4u * ib + l]);+ }+ }+ for (uint l = 0; l < 4u; ++l) {+ ulong g;+ if (b.type == 16) g = iq2xxs_grid[idx[l]];+ else if (b.type == 17) g = iq2xs_grid[idx[l]];+ else g = iq2s_grid[idx[l] & 1023u];+ const float db = (l < 2u) ? db0 : db1;+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sum = 0;+ for (uint j = 0; j < 8u; ++j) {+ const int code = int((g >> (8u * j)) & 0xffu);+ const bool neg = b.type == 16 ? ((sg[l] & uint(kmask_iq2xs[j])) != 0u)+ : ((sg[l] & (1u << j)) != 0u);+ sum += (neg ? -code : code) * int(xq[j]);+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db, float(sum), acc[c]);+ }+ }+ }+ }+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// ---------------------------------------------------------------- ds4_iq.cu:536 gate_up_kernel / :588 down_kernel+//+// The prompt / window path's expert GEMM (ds4_iq.cu's launch_gu / launch_dn): a threadgroup of 256 = 8 warps, a warp+// RW rows of the gate and of the up (down: RW rows of the down), a lane 32 / RW sub-blocks of the row, and the group's+// entries NE at a pass so that a sub-block the weights decode once serves them all. Grid (rows / (8 RW), groups).+//+// The group's blob and its entries are the CUDA kernel's `group` (ds4_iq.cu:512): a plan group (grp_ptr null) is one+// expert whose entry is the group's own index and whose activation row is 0, a window group's entries come from+// grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.+struct Ds4IqJob {+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)+ device const int* grp_ent; // the entries (token * 8 + pick)+ device const int* n_groups;+ device const char* xq; // the int8 activations of a row (n_embd an entry)+ device const float* xd; // their scales (n_embd / 32 a row)+ device const char* hq; // down's int8 input (n_ff an entry)+ device const float* hd;+ device float* h; // [entries][n_ff]: the clamped swiglu+ device float* part; // [entries][n_embd]: each entry's down output+ long tr0, tr1, tr2; // the roles' offsets in a blob+ int n_embd, n_ff;+ int type; // 16 IQ2_XXS, 17 IQ2_XS, 22 IQ2_S (ds4_iq.cuh's kIQ2_*)+ float limit; // swiglu_limit (0: none)+};++// a 32-value sub-block as 32 int8 codes (8 packed ints, values 4 w .. 4 w + 3 in word w) and the two halves' scales+// (ggml's d (2 s + 1) / 8, the .cu's Sub.d * Sub.s0 in one F32: the codes of the second half are scaled by s1)+struct Ds4IqSub {+ int q[8];+ float s0, s1;+};++// the block bytes of the types these kernels take: 16 IQ2_XXS, 17 IQ2_XS, 18 IQ3_XXS, 22 IQ2_S (ds4_iq.cu's Ty<T>::B)+static inline uint ds4_iq_bs(int type) {+ return type == 16 ? 66u : type == 17 ? 74u : type == 18 ? 98u : 82u;+}++// ds4_iq.cu:139 decode<T>: the block's 8 sub-blocks, kb's is the (kb & 7)'th; the grids are ggml-common.h's+static inline Ds4IqSub ds4_iq_sub(device const uchar* row, uint bs, int kb, int type) {+ Ds4IqSub s;+ const int ib = kb & 7;+ const device uchar* p = row + (size_t) (kb >> 3) * bs;+ const float d = float(as_type<half>(*(const device ushort*) p));+ uint idx[4], sm[4];+ float db0, db1;+ if (type == 18) { // IQ3_XXS: d, qs[64] grid bytes (4 a sub-block), scales[8] (the top nibble the group's s)+ const device uchar* qs = p + 2;+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) |+ (uint(qs[8 * ib + 3]) << 24);+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) |+ (uint(qs[8 * ib + 7]) << 24);+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) |+ (uint(qs[67 + 4 * ib]) << 24);+ for (uint l = 0; l < 4u; ++l) {+ const uint w = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sg = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(w >> sh) & 255u], g1 = iq3xxs_grid[(w >> (sh + 8u)) & 255u];+ int a0 = 0, a1 = 0;+ for (uint j = 0; j < 4u; ++j) {+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);+ if ((sg & (1u << j)) != 0u) v0 = -v0;+ if ((sg & (1u << (j + 4u))) != 0u) v1 = -v1;+ a0 |= (v0 & 0xff) << int(8u * j);+ a1 |= (v1 & 0xff) << int(8u * j);+ }+ s.q[2u * l] = a0;+ s.q[2u * l + 1u] = a1;+ }+ const float ds = d * (0.5f + float(aux >> 28)) * 0.5f; // ggml's db: d (0.5 + s) 0.5+ s.s0 = ds;+ s.s1 = ds;+ return s;+ }+ if (type == 16) { // d, qs[8] i16 (4 grid bytes, then 4 sign/scale bytes), scales in the top nibbles+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |+ (uint(p[5 + 8 * ib]) << 24);+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |+ (uint(p[9 + 8 * ib]) << 24);+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ db0 = db1 = db;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = (aux0 >> (8u * l)) & 0xffu;+ sm[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ }+ } else if (type == 17) { // d, qs[16] i16 (9-bit grid, 7 sign bits), scales[8] (two 4-bit)+ const device ushort* qs = (const device ushort*) (p + 2);+ const device uchar* sc = p + 66;+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[4u * uint(ib) + l]);+ idx[l] = q & 511u;+ sm[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]);+ }+ } else { // IQ2_S: d, qs[32] (8-bit grid), signs[32], qh[8] (2 bits more), scales[8]+ const device uchar* qs = p + 2;+ const device uchar* sgb = qs + 32;+ const device uchar* qh = p + 66;+ const device uchar* sc = p + 74;+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = uint(qs[4u * uint(ib) + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);+ sm[l] = uint(sgb[4u * uint(ib) + l]);+ }+ }+ for (uint l = 0; l < 4u; ++l) {+ const ulong g = type == 16 ? iq2xxs_grid[idx[l]] : (type == 17 ? iq2xs_grid[idx[l]] : iq2s_grid[idx[l] & 1023u]);+ int w0 = 0, w1 = 0;+ for (uint j = 0; j < 4u; ++j) {+ int v0 = int((g >> (8u * j)) & 0xffu), v1 = int((g >> (8u * (j + 4u))) & 0xffu);+ if ((sm[l] & (1u << j)) != 0u) v0 = -v0;+ if ((sm[l] & (1u << (j + 4u))) != 0u) v1 = -v1;+ w0 |= (v0 & 0xff) << int(8u * j);+ w1 |= (v1 & 0xff) << int(8u * j);+ }+ s.q[2u * l] = w0;+ s.q[2u * l + 1u] = w1;+ }+ s.s0 = db0;+ s.s1 = db1;+ return s;+}++// 4 int8 codes against the 4 int8 activations packed in x (ds4_iq.cu:350's __dp4a: Metal has no such instruction)+static inline int ds4_iq_dot4(int q, int x) {+ const char4 a = as_type<char4>(q), b = as_type<char4>(x);+ return int(a.x) * int(b.x) + int(a.y) * int(b.y) + int(a.z) * int(b.z) + int(a.w) * int(b.w);+}++// ds4_iq.cu:349 sub_dot: the sub-block against its 32 int8 activations (8 packed ints), the scales in F32+static inline float ds4_iq_dot(Ds4IqSub s, device const int* xp) {+ const int lo = ds4_iq_dot4(s.q[0], xp[0]) + ds4_iq_dot4(s.q[1], xp[1]) + ds4_iq_dot4(s.q[2], xp[2]) ++ ds4_iq_dot4(s.q[3], xp[3]);+ const int hi = ds4_iq_dot4(s.q[4], xp[4]) + ds4_iq_dot4(s.q[5], xp[5]) + ds4_iq_dot4(s.q[6], xp[6]) ++ ds4_iq_dot4(s.q[7], xp[7]);+ return s.s0 * float(lo) + s.s1 * float(hi);+}++// ds4_iq.cu:502 lanes_sum: the row's sum over the 32 / RW lanes that share it+template <int RW>+static inline float ds4_iq_lsum(float v) {+ for (int o = 16 / RW; o > 0; o >>= 1) v += simd_shuffle_xor(v, ushort(o));+ return v;+}++// ds4_iq.cu:497 swiglu: the limit's clamps, then g / (1 + e^-g) * u+static inline float ds4_iq_swiglu(float gv, float uv, float limit) {+ if (limit > 0.0f) {+ gv = min(gv, limit);+ uv = min(max(uv, -limit), limit);+ }+ return (gv / (1.0f + exp(-gv))) * uv;+}++// the group's blob and its entry range (ds4_iq.cu:512 group); false: the threadgroup has nothing to do+static inline bool ds4_iq_group(constant Ds4IqJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {+ if (j.grp_ptr == nullptr) {+ const ulong p = j.plan_ptr[g];+ if (p == 0ul) return false;+ blob = p;+ m0 = int(g);+ m1 = m0 + 1;+ return true;+ }+ if (int(g) >= *j.n_groups) return false;+ blob = j.grp_ptr[g];+ m0 = j.grp_start[g];+ m1 = j.grp_start[g + 1];+ return true;+}++// gate_up_kernel<T, NE> (kGuRW = 1: a warp a row, a lane a sub-block): h = the clamped swiglu of the gate and the up+kernel void ds4_iq_gu(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ ulong blob;+ int m0, m1;+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;+ const int row = int(tgp.x) * 8 + int(tid >> 5);+ if (row >= j.n_ff) return;+ const uint bs = ds4_iq_bs(j.type);+ const uint nsb = uint(j.n_embd) / 32u;+ const size_t rb = (size_t) (j.n_embd / 256) * (size_t) bs;+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += 4) {+ const int ne = min(4, m1 - mm);+ int es[4], xr[4];+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];+ es[i] = e;+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);+ }+ }+ float ag[4] = {0.0f, 0.0f, 0.0f, 0.0f}, au[4] = {0.0f, 0.0f, 0.0f, 0.0f};+ for (uint kb = uint(lane); kb < nsb; kb += 32u) {+ const Ds4IqSub sg = ds4_iq_sub(g, bs, int(kb), j.type);+ const Ds4IqSub su = ds4_iq_sub(u, bs, int(kb), j.type);+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const device int* xp = (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd ++ (size_t) kb * 32u);+ const float dx = j.xd[(size_t) xr[i] * nsb + kb];+ ag[i] += dx * ds4_iq_dot(sg, xp);+ au[i] += dx * ds4_iq_dot(su, xp);+ }+ }+ }+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);+ }+ }+ }+}++// down_kernel<T, NE> (kDnRW = 2: a warp two rows, 16 lanes a row): part = the down against the int8 h+kernel void ds4_iq_dn(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ ulong blob;+ int m0, m1;+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;+ const int row = (int(tgp.x) * 8 + int(tid >> 5)) * 2 + int(lane) / 16;+ if (row >= j.n_embd) return;+ const uint kl = uint(lane) % 16u;+ const uint bs = ds4_iq_bs(j.type);+ const uint nsb = uint(j.n_ff) / 32u;+ const size_t rb = (size_t) (j.n_ff / 256) * (size_t) bs;+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += 4) {+ const int ne = min(4, m1 - mm);+ int es[4];+ for (int i = 0; i < 4; ++i)+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];+ float ad[4] = {0.0f, 0.0f, 0.0f, 0.0f};+ for (uint kb = kl; kb < nsb; kb += 16u) {+ const Ds4IqSub sd = ds4_iq_sub(w, bs, int(kb), j.type);+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const device int* hp = (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff ++ (size_t) kb * 32u);+ ad[i] += j.hd[(size_t) es[i] * nsb + kb] * ds4_iq_dot(sd, hp);+ }+ }+ }+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const float v = ds4_iq_lsum<2>(ad[i]);+ if (kl == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;+ }+ }+ }+}diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex a72bcf7c8..02fadda68 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -2,6 +2,9 @@// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).#include "ds4_gguf.cuh"+#if defined(STRATA_METAL)+#include <ds4_metal_host.h>+#endif#include "ds4_iq.cuh"#include "ds4_numerics.cuh"@@ -198,6 +201,9 @@ __global__ void __launch_bounds__(kThreads) q8_quant_kernel(const float* __restr}void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);+#if defined(STRATA_METAL)+ if (::strata::dsv41::metal_host::q8_quant_rows(x, ldx, rows, k, (int8_t*) q, d)) return;+#endifstrata_ds4_launch(q8_quant_kernel, grid, kThreads, 0, s, x, ldx, k, q, d);}@@ -1040,12 +1046,26 @@ template <int NC>void mv_launch(int f, const BfJobs& J, int nj, int max_n, bool vec, cudaStream_t s) {const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);if (f == 1) {+#if defined(STRATA_METAL)+ // the MXFP4 matvec: the engine's jobs go across as they are (same layout on both sides)+ if (max_n > 0 && nj > 0 &&+ ::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 1)) return;+#endifif (vec) strata_ds4_launch(mx_mv_kernel<NC, true, false>, grid, kThreads, 0, s, J);else strata_ds4_launch(mx_mv_kernel<NC, false, false>, grid, kThreads, 0, s, J);} else if (f == 2) { // (vec here: every row 4-byte aligned, so staged)+#if defined(STRATA_METAL)+ if (max_n > 0 && nj > 0 &&+ ::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 2)) return;+#endifif (vec) strata_ds4_launch(q8_mv_kernel<NC, true>, grid, kThreads, 0, s, J);else strata_ds4_launch(q8_mv_kernel<NC, false>, grid, kThreads, 0, s, J);} else {+#if defined(STRATA_METAL)+ // the BF16 matvec (vec: the rows and activations 16-byte aligned, so the 8-value step is safe)+ if (vec && max_n > 0 && nj > 0 &&+ ::strata::dsv41::metal_host::mx_mv((const void*) &J, nj, max_n, sizeof(BfJob), 0)) return;+#endifif (vec) strata_ds4_launch(bf16_mv_kernel<NC, true>, grid, kThreads, 0, s, J);else strata_ds4_launch(bf16_mv_kernel<NC, false>, grid, kThreads, 0, s, J);}@@ -1265,6 +1285,10 @@ void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float*c.out_stride = (size_t) a.n_embd;c.k = a.k;c.n_embd = a.n_embd;+#if defined(STRATA_METAL)+ if (!::strata::dsv41::metal_host::mx_combine(c.plan_ptr, c.plan_w, c.part, c.cpu_part, c.cpu_stride,+ c.cpu_flag, c.sh_out, c.out, c.out_stride, 1, c.k, c.n_embd))+#endifstrata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s, c);check("moe_down");}@@ -1312,7 +1336,10 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)if (mx) launch_groups<DownK>(m, a.n_embd, groups, a.T, s);else iq::down(ia, groups, a.T, s);+#if !defined(STRATA_METAL)+ // (this port's dispatches are synchronous and nothing is published to wait for: ds4_kernels.cu's wait_seq)if (a.wait_ptr != nullptr) strata_ds4_launch(mx_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);+#endifCombArgs c{};c.plan_ptr = a.plan_ptr;c.plan_w = a.plan_w;@@ -1325,6 +1352,11 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {c.out_stride = a.out_stride;c.k = a.k;c.n_embd = a.n_embd;+#if defined(STRATA_METAL)+ // mx_combine_kernel on the Metal dispatch; the launch below is the fallback+ if (!::strata::dsv41::metal_host::mx_combine(c.plan_ptr, c.plan_w, c.part, c.cpu_part, c.cpu_stride,+ c.cpu_flag, c.sh_out, c.out, c.out_stride, a.T, c.k, c.n_embd))+#endifstrata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s, c);check("win_experts");return true;diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex 1baa84e5d..dfe7503ad 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -3,6 +3,9 @@// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.#include "ds4_iq.cuh"+#if defined(STRATA_METAL)+#include <ds4_metal_host.h>+#endif#include "ds4_numerics.cuh"#include <cuda_fp16.h>@@ -1179,6 +1182,12 @@ const char* name(int t) {}bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {+#if defined(STRATA_METAL)+ // gate_up_kernel on the Metal dispatch (the prompt / window path's expert GEMM); the launch below is the fallback+ if (::strata::dsv41::metal_host::iq_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,+ a.n_ff, a.limit, a.xq, a.xd, a.h, a.tg, groups))+ return true;+#endifif (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);@@ -1189,6 +1198,12 @@ bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {}bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {+#if defined(STRATA_METAL)+ // down_kernel on the Metal dispatch; the launch below is the fallback+ if (::strata::dsv41::metal_host::iq_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,+ a.n_ff, a.hq, a.hd, a.part, a.td, groups))+ return true;+#endifif (is_tq(a.td)) return tq::down(a, groups, ne_max, s);if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);@@ -1240,6 +1255,9 @@ bool set_down(const SetArgs& a, cudaStream_t s) {}bool mv(const MvJob* jobs, int n, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (n > 0 && n <= 64 && ::strata::dsv41::metal_host::iq_mv((const void*) jobs, n)) return true;+#endif{ // the trellis jobs go to their kernels, the others onMvJob tqj[64], oth[64];int nt = 0, no = 0;diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 21343c8fd..c89c9507c 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1826,15 +1826,36 @@ void route_publish(const float* logits, const float* bias, int n, int k, int T,const unsigned long long* vram, const float* x, int dim, int* idx, float* w,unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl,unsigned vl, int na0, int na1) {+#if defined(STRATA_METAL)+ // selection only: the prefetch schedule (req) is published by the CUDA kernel, and is turned on separately here+ if (strata::dsv41::metal_host::route_publish(logits, bias, n, k, T, route_scale, vram, idx, w, plan_ptr,+ vl != 0 ? bias_vl : nullptr, vl)) {+ // no prefetcher on the Mac path, so publish_kernel has no consumer: the selection is the whole job here+ (void) req; (void) seq; (void) na0; (void) na1; (void) x; (void) dim;+ return;+ }+#endifstrata_ds4_launch(route_publish_kernel, T, 512, 0, s, logits, bias, n, k, route_scale, vram, x, dim, idx, w, plan_ptr, req,vl != 0 ? bias_vl : nullptr, vl);if (req != nullptr) strata_ds4_launch(publish_kernel, 1, 1, 0, s, req, T, seq, na0, na1);}void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s) {+#if defined(STRATA_METAL)+ // no prefetch schedule on the Mac path yet: the router publishes no requests, so there is nothing to look ahead for+ (void) logits; (void) bias; (void) n; (void) m; (void) T; (void) req; (void) off; (void) s;+ return;+#endifif (req != nullptr && m > 0 && T > 0 && n <= 1024 && off + m <= kAhead)strata_ds4_launch(route_ahead_kernel, T, 512, 0, s, logits, bias, n, m, req, off);}-void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { strata_ds4_launch(wait_seq_kernel, 1, 32, 0, s, seq_ptr, seq); }+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) {+#if defined(STRATA_METAL)+ // nothing is published, so nothing is waited for: the CUDA path spins until the host's prefetcher acknowledges+ (void) seq_ptr; (void) seq; (void) s;+ return;+#endif+ strata_ds4_launch(wait_seq_kernel, 1, 32, 0, s, seq_ptr, seq);+}// the RAM store's experts for the device (a decode token, window of one): a pick the VRAM table left at 0 whose expert the store holds// (zc_tab[expert] = the slot's device address over PCIe, 0: not) is copied to staging slot i with wide coalesced loads (the PCIe@@ -1887,6 +1908,10 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudstd::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);return;}+#if defined(STRATA_METAL)+ const auto& mp = *reinterpret_cast<const strata::dsv41::metal_host::FinishIn*>(&f);+ if (strata::dsv41::metal_host::finish(x, n_vec, len, stride, mp)) return; // finish_kernel<true> (the KV pack)+#endifstrata_ds4_launch(finish_kernel<true>, n_vec, kThreads, 0, s, x, len, stride, f);} else {#if defined(STRATA_METAL)@@ -1912,10 +1937,16 @@ void hc_split(const float* mixes, const float* scale, const float* base, int hc,}void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_pre_norm(h, mix, w, dim, hc, T, eps, x)) return;+#endifstrata_ds4_launch(hc_pre_norm_kernel, T, 1024, 0, s, h, mix, w, dim, hc, eps, x);}void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,float* out, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_post_v(y, res, post, comb, dim, hc, T, out)) return;+#endifstrata_ds4_launch(hc_post_kernel, dim3(cdiv(dim * hc, kThreads), T), kThreads, 0, s, y, res, post, comb, dim, hc, out);}@@ -1933,6 +1964,9 @@ void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float*strata_ds4_launch(hc_mix_scale_kernel, T, kThreads, 0, s, h, hcdim, n_mix, eps, mixes);}void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int fmt) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::engram_dequant_tok(src, per, cols, T, hd, out, fmt)) return;+#endifif (T > 0) strata_ds4_launch(engram_dequant_tok_kernel, T * cols, kThreads, 0, s, src, per, cols, hd, out, fmt);}void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,@@ -1946,7 +1980,12 @@ void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds}void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,cudaStream_t s) {- if (r1 > r0 && T > 0) strata_ds4_launch(moe_combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, acc, y, pos, w, k, r0, r1, dim);+ if (r1 > r0 && T > 0) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::moe_combine(acc, y, pos, w, T, k, r0, r1, dim)) return;+#endif+ strata_ds4_launch(moe_combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, acc, y, pos, w, k, r0, r1, dim);+ }}void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s) {@@ -1954,12 +1993,19 @@ void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alph}void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,cudaStream_t s) {+#if defined(STRATA_METAL)+ // gemv_f16_rows_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::gemv_f16_rows(W, x, ldx, n, k, T, alpha, y, ldy)) return;+#endifif (T > 0)strata_ds4_launch(gemv_f16_rows_kernel, dim3(cdiv(n, kThreads / 32), cdiv(T, kRowsT)), kThreads, 0, s, W, x, ldx, n, k, T, alpha,y, ldy);}void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (T > 0 && ::strata::dsv41::metal_host::gemv_f32(W, x, n, k, T, y, ldy)) return;+#endifif (T > 0) strata_ds4_launch(gemv_f32_kernel, dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s, W, x, n, k, y, ldy);}@@ -1971,11 +2017,20 @@ void route(const float* logits, const float* bias, int n, int k, int T, float ro}void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s) {+#if defined(STRATA_METAL)+ // declared here rather than in the header: the header carries the other entry points, this one is only used here+ namespace mh = strata::dsv41::metal_host;+ if (n > 0 && mh::swiglu(g, u, n, lim, h)) return;+#endifif (n > 0) strata_ds4_launch(swiglu_kernel, cdiv(n, kThreads), kThreads, 0, s, g, u, n, lim, h);}void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,int d, const float* w, float eps, float* latent, cudaStream_t s) {+#if defined(STRATA_METAL)+ // compress_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::compress_win(kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent)) return;+#endifstrata_ds4_launch(compress_kernel, 1, 512, 0, s, kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent);}@@ -1987,6 +2042,12 @@ void index_score(const float* q, const float* w, const void* k, int heads, int hreturn;}if (n <= 0) return;+#if defined(STRATA_METAL)+ // index_score_kernel<128, kfmt> on the Metal dispatch; the launches below are the fallback+ if (strata::dsv41::metal_host::index_score(q, w, (const uint8_t*) k, heads, hd, T, n, cand, cand_stride, blk, &lim,+ out, out_stride, kfmt))+ return;+#endifconst size_t smem = (size_t) (heads * hd + heads) * sizeof(float);const dim3 grid(cdiv(n, kThreads), T);if (kfmt == kKvFp4p)@@ -1999,6 +2060,10 @@ void index_score(const float* q, const float* w, const void* k, int heads, int hvoid topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,int* out, int out_stride, cudaStream_t s) {+#if defined(STRATA_METAL)+ // topk_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::topk(vals, stride, T, &r, cand, cand_stride, blk, out, out_stride)) return;+#endifstrata_ds4_launch(topk_kernel, T, 1024, 0, s, vals, stride, r, cand, cand_stride, blk, out, out_stride);}@@ -2016,6 +2081,10 @@ void cand_blocks(const float* score, long long stride, int T, const Rows& r, intrb.k[t] = r.k[t];nb_max = rb.n[t] > nb_max ? rb.n[t] : nb_max;}+#if defined(STRATA_METAL)+ // block_max_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::block_max(score, stride, T, &r, blk, bm, bm_stride, nb_max)) return;+#endifstrata_ds4_launch(block_max_kernel, dim3(cdiv(nb_max, kThreads), T), kThreads, 0, s, score, stride, r, blk, bm, bm_stride);topk(bm, bm_stride, T, rb, nullptr, 0, 1, out, out_stride, s);}@@ -2064,6 +2133,10 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int// the BF16 rows' kernel itself (heads_per_block -1: this way at any T)const bool few = heads_per_block == -1 || (heads_per_block == 0 && T <= kAttnScratchT);if (few && scr != nullptr && scr->rows != nullptr && T <= scr->max_t && sel_stride <= scr->cap) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::unpack_sel(ckv, sel, sel_stride, a.gen, a.p0, a.t0, a.R, a.kcap, a.nsel,+ scr->cap, scr->rows, T, a.gen ? 1 : (T < kWin ? T : kWin), kv_row_bytes(kKvFp4, 512))) return;+#endifstrata_ds4_launch(unpack_sel_kernel, dim3(cdiv(sel_stride, kStage), T), kThreads, 0, s, (const uint8_t*) ckv, sel, sel_stride, a,scr->cap, scr->rows);#if !defined(STRATA_USE_HIP)@@ -2092,6 +2165,9 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int}void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::ring_to_ext(ring, ext, (long long) p0, n, d, kv_row_bytes(kKvFp8, d))) return;+#endifstrata_ds4_launch(ring_to_ext_kernel, n, kThreads, 0, s, ring, ext, (long long) p0, n, d);}void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s, uint8_t* backup) {@@ -2099,6 +2175,9 @@ void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, istd::fprintf(stderr, "ds4 ext_to_ring: row of %d not covered\n", d);return;}+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::ext_to_ring(ext, ring, (long long) p0, n, T, d, kv_row_bytes(kKvFp8, d), backup)) return;+#endifstrata_ds4_launch(ext_to_ring_kernel, T, kThreads, 0, s, ext, ring, (long long) p0, n, T, d, backup);}void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {@@ -2111,9 +2190,19 @@ void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T,kv_row_bytes(kKvFp8, d));}void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {+#if defined(STRATA_METAL)+ // hc_mean_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::hc_mean(h, T, hc, dim, out, out_stride)) return;+#endifstrata_ds4_launch(hc_mean_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, h, hc, dim, out, out_stride);}-void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride); }+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) {+#if defined(STRATA_METAL)+ // iota_rows_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::iota_rows(out, T, n, stride)) return;+#endif+ strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride);+}void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {strata_ds4_launch(markov_kernel, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, H, tok, V, R);}@@ -2133,10 +2222,18 @@ void confidence(const float* x, const uint16_t* E, const int* tok, const float*}void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt) {+#if defined(STRATA_METAL)+ // engram_dequant_kernel on the Metal dispatch (fmt 2, this model's IQ4_NL tables); the launch below is the fallback+ if (strata::dsv41::metal_host::engram_dequant(rows, scales, n, hd, out, fmt)) return;+#endifstrata_ds4_launch(engram_dequant_kernel, n, kThreads, 0, s, rows, scales, hd, out, fmt);}void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {+#if defined(STRATA_METAL)+ // engram_gate_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::engram_gate(h, kv, w, hc, dim, T, eps)) return;+#endifstrata_ds4_launch(engram_gate_kernel, dim3(hc, T), 1024, 0, s, h, kv, w, hc, dim, eps);}diff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cuindex 94944c6fb..996c02002 100644--- a/src/dsv41/vendor/sampler.cu+++ b/src/dsv41/vendor/sampler.cu@@ -16,6 +16,9 @@// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).#include "strata/dsv41/sampler.hpp"+#if defined(STRATA_METAL)+#include <ds4_metal_host.h>+#endif#include <cuda_runtime.h>@@ -525,6 +528,13 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hiif (p.greedy || p.temperature <= 0.0f) {// One block per token, 1,024 threads over the vocabulary. See `sampler_greedy_kernel`.const int gthreads = 1024;+#if defined(STRATA_METAL)+ // sampler_greedy_kernel on the Metal dispatch; the launch below is the fallback+ if (::strata::dsv41::metal_host::sample_greedy(logits, n_tokens, n_vocab, history, history_len,+ p.penalty_last_n, p.penalty_repeat, p.penalty_freq,+ p.penalty_present, p.logit_bias, out))+ return;+#endifstrata_ds4_launch(sampler_greedy_kernel, (unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream,logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);} else {