Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// metal/src/engine/ds4_finish.cpp - the host side of ds4_finish (ds4_kernels.cu:1894, finish_kernel<false>).
//
// The kernel takes the engine's own device pointer for x as a bound buffer plus an offset, and the Ds4Finish struct as a constant
// buffer whose pointer fields are the engine's gpuAddresses (Metal lets a kernel dereference them - ss the repo's note that
// "kernels may take raw gpuAddress pointers"). Strides and counts travel as plain values.
//
// The mrope (rd > 0) and rounding (quant != 0) branches are not written yet, and neither is the KV pack (pdst): this function
// returns false for those, so the engine reports the launch line instead of a step being skipped.
#include "mtl_rt.h"
#include <strata/gpu/runtime.hpp>
#include <cuda_runtime.h>
#include <cstdio>
#include <cstdint>
#include <cstring>
#include <algorithm>
#include <map>
#include <vector>
// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read
extern "C" void ds4_metal_flush();
extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);
extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off);
extern "C" void ds4_gpu_sync_all_streams();
extern "C" unsigned long long ds4_dispatch_seq();
static unsigned long long g_q8_deduped = 0;
// The bytes a kernel's wrap() binds from the page-aligned `base`, `need` of them used: whole pages, but never past the end of
// the allocation that holds them. A range that leaves its allocation cannot be bound as the runtime's own buffer and gets a fresh
// no-copy buffer instead: a second GPU mapping of the same pages, through which a kernel read the previous values of what an
// earlier kernel had written through the first (ext_to_ring read the window rows of the layer before; a prompt of a few tokens
// answered differently on every run, with the same result only when the GPU was synchronised after the kernel).
static size_t wrap_len(size_t base, size_t need) {
const size_t pg = 16384, up = (need + pg - 1) / pg * pg;
const size_t rem = ds4_gpu_alloc_remaining((const void*) base);
return rem > 0 && rem < up ? rem : up; // (sparse_attn asks for one row more than the window buffer holds, which it never reads)
}
#define DSV41_PROBE(fmt, ...) do { static bool once_ = false; if (!once_) { once_ = true; std::fprintf(stderr, "[probe] " fmt, __VA_ARGS__); } } while (0)
namespace strata::dsv41::metal_host {
// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Finish (Metal has no 'long long': 64-bit is 'long')
struct GpuFinish {
uint64_t w; // device float* (the engine's gpuAddress, 0 = none)
float eps;
int rd, pos, inverse, vpp, pos_step;
uint64_t freqs; // device float*
int quant;
uint64_t dst; // device ushort*
int64_t dst_stride;
uint64_t pdst; // device uchar*
int64_t pdst_stride;
};
namespace {
mtl::Context* g_cx = nullptr;
int g_lib = -1;
int g_lib_iq = -1;
bool ensure() {
if (std::getenv("DS4_DBG_MEM")) { // does a write through host_view read back through cudaMemcpy?
void* p = nullptr;
const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;
if (cudaMalloc(&p, tsz) == cudaSuccess) {
void* v = strata::gpu::host_view(p);
if (v) {
((float*) v)[0] = 42.0f;
float out = 0.0f;
cudaMemcpy(&out, p, 4, cudaMemcpyDeviceToHost);
std::fprintf(stderr, "DBG mem: gpu=%p view=%p wrote 42 through the view, cudaMemcpy gives %g\n", p, v, out);
} else {
std::fprintf(stderr, "DBG mem: gpu=%p has no host view\n", p);
}
cudaFree(p);
}
}
if (g_cx) return g_lib >= 0;
g_cx = new mtl::Context();
g_cx->start_keepalive();
g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);
if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");
if (g_lib >= 0 && std::getenv("DS4_DBG_BIND")) { // write a constant by each route, copy it back
void* p = nullptr;
const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;
if (cudaMalloc(&p, tsz) == cudaSuccess) {
void* v = strata::gpu::host_view(p);
float got[4] = {0, 0, 0, 0};
{
mtl::Buffer b = g_cx->wrap((void*) (((uintptr_t) v) & ~(uintptr_t) 16383), 16384);
mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write");
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(b, ((uintptr_t) v) & 16383), mtl::Arg::V((unsigned) 4)};
d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->flush_runs();
cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);
std::fprintf(stderr, "DBG bind A (wrap host_view + Arg::B): %g %g %g %g\n", got[0], got[1], got[2], got[3]);
}
{
uint64_t pp = (uint64_t) (uintptr_t) p;
mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write_v");
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::V(pp), mtl::Arg::V((unsigned) 4)};
d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->flush_runs();
cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);
std::fprintf(stderr, "DBG bind B (device pointer by value): %g %g %g %g\n", got[0], got[1], got[2], got[3]);
}
cudaFree(p);
}
}
return g_lib >= 0;
}
} // namespace
// the engine's Finish, as ds4_kernels.cuh declares it
struct FinishIn {
const float* w;
float eps;
int rd, pos, inverse, vpp, pos_step;
const float* freqs;
int quant;
const uint16_t* dst;
long long dst_stride;
uint8_t* pdst;
long long pdst_stride;
};
bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {
if (n_vec <= 0 || len <= 0) return true;
// finish_kernel<true> (the KV pack: the row quantised to codes + group scales) or <false>; the <false> rounding
// branch (quant without pdst) is the one still unwritten
const bool pack = f.pdst != nullptr;
if (!ensure()) return false;
void* hx = strata::gpu::host_view(x);
if (!hx) { std::fprintf(stderr, "dsv41 metal: finish has no host view\n"); return false; }
const size_t page = 16384;
const size_t bytes = (size_t) (n_vec - 1) * (size_t) stride * sizeof(float) + (size_t) len * sizeof(float);
const size_t base = ((uintptr_t) hx) & ~(uintptr_t) (page - 1);
const size_t off = ((uintptr_t) hx) - base;
mtl::Buffer bx = g_cx->wrap((void*) base, wrap_len(base, bytes + off));
if (!bx.handle) { std::fprintf(stderr, "dsv41 metal: finish wrap failed\n"); return false; }
GpuFinish gf{};
gf.w = (uint64_t) (uintptr_t) f.w;
gf.eps = f.eps;
gf.rd = f.rd; gf.pos = f.pos; gf.inverse = f.inverse; gf.vpp = f.vpp; gf.pos_step = f.pos_step;
gf.freqs = (uint64_t) (uintptr_t) f.freqs;
gf.quant = f.quant;
gf.dst = (uint64_t) (uintptr_t) f.dst;
gf.dst_stride = f.dst_stride;
gf.pdst = (uint64_t) (uintptr_t) f.pdst;
gf.pdst_stride = f.pdst_stride;
mtl::Pipeline pl;
try {
pl = g_cx->pipeline(g_lib, pack ? "ds4_finish_pack" : "ds4_finish");
} catch (...) {
return false;
}
// one dispatch, one threadgroup a vector (the kernel's blockIdx.x is threadgroup_position_in_grid)
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bx, off), mtl::Arg::V((long) stride), mtl::Arg::V((unsigned) len), mtl::Arg::V(gf)};
d.grid[0] = (uint32_t) n_vec;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256;
d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
// The fp8-rounding finishes (the KV rows, the attention output) are read stale by the next dispatch when it shares their
// command buffer: end the buffer after them (committed, not waited for: the queue orders the buffers).
// (it once ended the command buffer after every fp8-rounding finish, 2 a layer, against a stale read; each extra buffer costs the GPU ~0.3 ms idle, a third of a decode token.
// The decode-path score is bit-identical without it, the dispatch barrier after every dispatch orders the readers. DS4_FINISH_SPLIT=1 brings it back.)
static const bool split_after = std::getenv("DS4_FINISH_SPLIT") != nullptr;
if (f.quant && split_after) g_cx->split_runs();
if (const char* fm2 = std::getenv("DS4_FLUSH_FINISH_CTX")) { if (f.quant) { if (std::atoi(fm2) == 2) g_cx->split_runs(); else g_cx->flush_runs(); } }
if (const char* fm = std::getenv("DS4_FLUSH_FINISH")) { // debugging: which finish calls need a drain after them
const int m = std::atoi(fm);
if ((m == 1 && f.dst) || (m == 2 && f.rd > 0) || (m == 3 && f.w) || (m == 4 && f.quant) || (m == 5 && !f.dst) ||
(m == 6 && f.dst == nullptr && f.rd == 0) || (m == 7 && f.rd > 0 && f.inverse) || (m == 8 && f.rd > 0 && !f.inverse && !f.dst) || (m == 9 && f.rd > 0 && !f.dst) || (m == 10 && (f.dst || f.inverse)) || (m == 11 && f.rd > 0 && !f.inverse))
ds4_metal_flush();
}
return true;
}
// ds4_kernels.cu:1677 - hc_post_inplace: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]
bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T) {
if (!ensure()) return false;
void* hy = strata::gpu::host_view((void*) y);
void* hh = strata::gpu::host_view(h);
if (!hy || !hh) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
if (!by.handle || !bh.handle) return false;
// post is [T][8], comb is [T][64]: device operands like the others (reached through their host views)
void* hpost = strata::gpu::host_view((void*) post);
void* hcomb = strata::gpu::host_view((void*) comb);
if (!hpost || !hcomb) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view for post / comb\n"); return false; }
mtl::Buffer bp = wrap(hpost, (size_t) T * 8 * 4);
mtl::Buffer bc = wrap(hcomb, (size_t) T * 64 * 4);
if (!bp.handle || !bc.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_post"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp, off(hpost)), mtl::Arg::B(bc, off(hcomb)),
mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
d.grid[0] = (uint32_t) ((dim + 255) / 256);
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
const bool dbg = std::getenv("DS4_DBG_SG") != nullptr;
if (dbg) {
g_cx->flush_runs();
const float* fy = (const float*) hy; const float* fh = (const float*) hh; const float* fp = (const float*) hpost; const float* fc = (const float*) hcomb;
std::fprintf(stderr, "hc_post in: y %g %g h %g %g | %g %g post %g %g %g %g comb %g %g %g %g\n", fy[0], fy[1], fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1], fp[0], fp[1], fp[2], fp[3], fc[0], fc[1], fc[2], fc[3]);
}
g_cx->run({d});
if (dbg) {
g_cx->flush_runs();
const float* fh = (const float*) hh;
std::fprintf(stderr, "hc_post out: h %g %g | %g %g\n", fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1]);
}
g_cx->free(bp);
g_cx->free(bc);
return true;
}
// ds4_kernels.cu:1692 - hc_mix_scale: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps)
bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes) {
if (!ensure()) return false;
void* hh = strata::gpu::host_view((void*) h);
void* hm = strata::gpu::host_view((void*) mixes);
if (!hh || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix_scale has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );
mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
if (!bh.handle || !bm.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_scale"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::V((unsigned) hcdim),
mtl::Arg::V((unsigned) n_mix), mtl::Arg::V(eps)};
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
if (std::getenv("DS4_DBG_SG")) {
g_cx->flush_runs();
const float* fc = (const float*) hm;
std::fprintf(stderr, "hc_mix_scale T %d hcdim %d n_mix %d eps %g -> %g %g %g\n", T, hcdim, n_mix, eps, fc[0], fc[1], fc[2]);
}
return true;
}
// ds4_kernels.cu:1901 - hc_mix: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps)
bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes) {
if (!ensure()) return false;
void* hh = strata::gpu::host_view((void*) h);
void* hf = strata::gpu::host_view((void*) fn);
void* hm = strata::gpu::host_view((void*) mixes);
if (!hh || !hf || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );
mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 );
mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
if (!bh.handle || !bf.handle || !bm.handle) return false;
mtl::Pipeline pl;
const bool all = T >= 16 && n_mix <= 32 && !std::getenv("DS4_NO_HC_ALL"); // a prompt chunk: a threadgroup a token, 4 mixes a pass
try { pl = g_cx->pipeline(g_lib, all ? "ds4_hc_mix_all" : "ds4_hc_mix"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),
mtl::Arg::V((unsigned) hcdim), mtl::Arg::V(eps)};
if (all) d.args.push_back(mtl::Arg::V((unsigned) n_mix));
d.grid[0] = all ? (uint32_t) T : (uint32_t) n_mix;
d.grid[1] = all ? 1u : (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:1908 - hc_split: the sinkhorn normalisation (scalar path; see the MSL comment)
bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
float* pre, float* post, float* comb) {
if (!ensure()) return false;
void* hm = strata::gpu::host_view((void*) mixes);
void* hp = strata::gpu::host_view((void*) pre);
void* ho = strata::gpu::host_view((void*) post);
void* hc_ = strata::gpu::host_view((void*) comb);
if (!hm || !hp || !ho || !hc_) { std::fprintf(stderr, "dsv41 metal: hc_split has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 );
mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 );
if (!bm.handle || !bp.handle || !bo.handle || !bc.handle) return false;
// scale and base come from the engine like every other operand: reach them through host_view, not as raw host pointers
void* hs = strata::gpu::host_view((void*) scale);
void* hb = strata::gpu::host_view((void*) base);
if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }
const size_t nbase = (size_t) (2 * hc + hc * hc);
mtl::Buffer bs = wrap(hs, 3 * sizeof(float));
mtl::Buffer bb = wrap(hb, nbase * sizeof(float));
if (!bs.handle || !bb.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, hc == 4 && !std::getenv("DS4_NO_HCS4") ? "ds4_hc_split4" : "ds4_hc_split"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),
mtl::Arg::B(bo, off(ho)), mtl::Arg::B(bc, off(hc_)), mtl::Arg::V((unsigned) hc),
mtl::Arg::V((unsigned) iters), mtl::Arg::V((float) hc_eps)};
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
if (std::getenv("DS4_DBG_SG")) {
g_cx->flush_runs();
const float* fm = (const float*) hm; const float* fs = (const float*) hs; const float* fb = (const float*) hb;
std::fprintf(stderr, "hc_split in: mix %g %g %g scale %g %g %g base %g %g\n", fm[0], fm[1], fm[2], fs[0], fs[1], fs[2], fb[0], fb[1]);
}
g_cx->run({d});
if (std::getenv("DS4_DBG_SG")) {
g_cx->flush_runs();
const float* fp = (const float*) hp;
std::fprintf(stderr, "hc_split out: pre %g %g %g %g\n", fp[0], fp[1], fp[2], fp[3]);
}
return true;
}
// the hyper-connection mix and split in one dispatch (ds4_hc_mix_split): the engine used to issue hc_mix and hc_split
// back to back for the same token, and the kernel does both in one threadgroup, bit for bit as before.
bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,
const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
float* comb) {
if (!ensure()) return false;
void* hh = strata::gpu::host_view((void*) h);
void* hf = strata::gpu::host_view((void*) fn);
void* hm = strata::gpu::host_view((void*) mixes);
void* hs = strata::gpu::host_view((void*) scale);
void* hb = strata::gpu::host_view((void*) base);
void* hp = strata::gpu::host_view((void*) pre);
void* ho = strata::gpu::host_view((void*) post);
void* hc_ = strata::gpu::host_view((void*) comb);
if (!hh || !hf || !hm || !hs || !hb || !hp || !ho || !hc_) {
std::fprintf(stderr, "dsv41 metal: hc_mix_split has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base_ = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base_, wrap_len(base_, end - base_));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(size_t) (page - 1))); };
mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4);
mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4);
mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4);
mtl::Buffer bs = wrap(hs, 3 * sizeof(float));
mtl::Buffer bb = wrap(hb, (size_t) (2 * hc + hc * hc) * sizeof(float));
mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4);
mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4);
mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4);
if (!bh.handle || !bf.handle || !bm.handle || !bs.handle || !bb.handle || !bp.handle || !bo.handle || !bc.handle)
return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_split"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),
mtl::Arg::V((unsigned) hcdim), mtl::Arg::V((float) eps), mtl::Arg::V((unsigned) n_mix),
mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),
mtl::Arg::B(bo, off(ho)), mtl::Arg::B(bc, off(hc_)), mtl::Arg::V((unsigned) hc),
mtl::Arg::V((unsigned) iters), mtl::Arg::V((float) hc_eps)};
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:1938 - moe_sort: count each virtual slot's picks, prefix-sum the counts, fill in token order.
// Three dispatches on one command buffer, so their order is the order of the calls (serial dispatch).
bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos) {
if (!ensure()) return false;
void* hi = strata::gpu::host_view((void*) idx);
void* hv = strata::gpu::host_view((void*) vs);
void* hb = strata::gpu::host_view((void*) bounds);
void* hr = strata::gpu::host_view((void*) row_tok);
void* hp = strata::gpu::host_view((void*) pos);
if (!hi || !hv || !hb || !hr || !hp) { std::fprintf(stderr, "dsv41 metal: moe_sort has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
const size_t t_bytes = (size_t) T * (size_t) k * 4;
mtl::Buffer bi = wrap(hi, t_bytes);
mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );
mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );
mtl::Buffer br = wrap(hr, t_bytes);
mtl::Buffer bp = wrap(hp, t_bytes);
if (!bi.handle || !bv.handle || !bb.handle || !br.handle || !bp.handle) return false;
mtl::Pipeline pc, pf, ps;
try {
pc = g_cx->pipeline(g_lib, "ds4_moe_sort_count");
pf = g_cx->pipeline(g_lib, "ds4_moe_sort_fill");
ps = g_cx->pipeline(g_lib, "ds4_bounds_scan");
} catch (...) { return false; }
// 1. counts
mtl::Dispatch d1;
d1.p = &pc;
d1.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),
mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb))};
d1.grid[0] = (uint32_t) n_vs;
d1.grid[1] = d1.grid[2] = 1;
d1.tg[0] = 256; d1.tg[1] = d1.tg[2] = 1;
g_cx->run({d1});
// 2. the exclusive prefix over the slots
mtl::Dispatch d2;
d2.p = &ps;
d2.args = {mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n_vs)};
d2.grid[0] = 1;
d2.grid[1] = d2.grid[2] = 1;
d2.tg[0] = 32; d2.tg[1] = d2.tg[2] = 1;
g_cx->run({d2});
// 3. the fill
mtl::Dispatch d3;
d3.p = &pf;
d3.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),
mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp))};
d3.grid[0] = (uint32_t) n_vs;
d3.grid[1] = d3.grid[2] = 1;
d3.tg[0] = 256; d3.tg[1] = d3.tg[2] = 1;
g_cx->run({d3});
return true;
}
// ds4_kernels.cu:1690 - the Engram row decoder (fmt 2 = IQ4_NL, this model's tables)
bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt) {
if (T <= 0) return true;
if (!ensure()) return false;
void* hs = strata::gpu::host_view((void*) src);
void* ho = strata::gpu::host_view((void*) out);
if (!hs || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant_tok has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
const size_t rows = (size_t) T * (size_t) cols;
mtl::Buffer bs = wrap(hs, (size_t) T * per );
mtl::Buffer bo = wrap(ho, rows * (size_t) hd * 4 );
if (!bs.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((unsigned long) per), mtl::Arg::V((unsigned) cols),
mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((int) fmt)};
d.grid[0] = (uint32_t) rows;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:341 - hc_pre_norm: a = sum_s mix[s]*h[s][e]; x = a * rsqrt(mean(a^2) + eps) (x w when given)
bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x) {
if (!ensure()) return false;
void* hh = strata::gpu::host_view((void*) h);
void* hm = strata::gpu::host_view((void*) mix);
void* hx = strata::gpu::host_view((void*) x);
if (!hh || !hm || !hx) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
mtl::Buffer bm = wrap(hm, (size_t) T * 8 * 4 );
mtl::Buffer bx = wrap(hx, (size_t) T * dim * 4 );
if (!bh.handle || !bm.handle || !bx.handle) return false;
mtl::Buffer bw;
size_t w_off = 0;
if (w) {
void* hw = strata::gpu::host_view((void*) w);
if (!hw) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm weight has no host view\n"); return false; }
bw = wrap(hw, (size_t) dim * 4 );
w_off = off(hw);
if (!bw.handle) return false;
} else {
bw = bx; // never read when has_w == 0
}
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_pre_norm"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bw, w_off),
mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc), mtl::Arg::V(eps), mtl::Arg::B(bx, off(hx)),
mtl::Arg::V((unsigned) (w ? 1 : 0))};
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_gguf.cu:186 - the activation quantiser the expert matmuls consume (Q8_0: 32 values, one scale)
bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {
if (rows <= 0 || k <= 0) return true;
if (!ensure()) return false;
void* hx = strata::gpu::host_view((void*) x);
void* hq = strata::gpu::host_view((void*) q);
void* hd = strata::gpu::host_view((void*) d);
if (!hx || !hq || !hd) { std::fprintf(stderr, "dsv41 metal: q8_quant has no host view\n"); return false; }
const size_t page = 16384;
// The pages the buffer itself spans: [base, base + off + bytes). A fixed slack (the +65536 the call sites used to
// carry) walks off the end of the mapping when the buffer is small and sits at an arena chunk's edge, and Metal then
// refuses the no-copy wrap ("mtl: wrap 81920 failed" - the hd scales here, a few pages).
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bx = wrap(hx, (size_t) rows * ldx * 4);
mtl::Buffer bq = wrap(hq, (size_t) rows * k);
mtl::Buffer bd = wrap(hd, (size_t) rows * (k / 32) * 4);
if (!bx.handle || !bq.handle || !bd.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) k),
mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bd, off(hd))};
const uint groups = (uint32_t) ((k / 32 + 7) / 8);
dsp.grid[0] = groups;
dsp.grid[1] = (uint32_t) rows;
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// the GPU address of host weight memory (see mtl::Context::zc_map); 0: not available
uint64_t zc_map(const void* p, size_t bytes) {
if (!ensure()) return 0;
return g_cx->zc_map(p, bytes);
}
// Several ds4_q8_quant calls in one dispatch (specs: {x, ldx, rows, k, q, d} with the engine's device pointers, which are the
// kernels' own addresses, as iq_mv's jobs carry them)
namespace {
struct HostQ8Spec {
const float* x;
uint64_t ldx;
uint32_t rows, k;
int8_t* q;
float* d;
};
} // namespace
bool q8_quant_multi(const void* specs, int n) {
if (n <= 0) return true;
if (!ensure()) return false;
const HostQ8Spec* sp = (const HostQ8Spec*) specs;
uint32_t max_rows = 1, max_k = 32;
for (int i = 0; i < n; ++i) {
max_rows = std::max(max_rows, sp[i].rows);
max_k = std::max(max_k, sp[i].k);
}
mtl::Buffer bj = g_cx->alloc((size_t) n * sizeof(HostQ8Spec));
std::memcpy(bj.ptr, sp, (size_t) n * sizeof(HostQ8Spec));
static mtl::Pipeline pl;
static bool have = false;
if (!have) {
try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant_multi"); } catch (...) { g_cx->free(bj); return false; }
have = true;
}
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (max_k / 32 + 7) / 8;
d.grid[1] = max_rows;
d.grid[2] = (uint32_t) n;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
// ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so
// the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.
namespace {
// mirrors ds4_gguf.cu's BfJob (and the kernels' Ds4MxJob)
struct HostBfJob {
const void* w;
uint64_t ld;
const float* x;
uint64_t xcs;
float* y;
uint64_t ycs;
const float* bias;
float alpha;
int k, n, ncol;
const int8_t* xq;
const float* xd;
};
} // namespace
bool mx_mv(const void* jobs_in, int nj_in, int max_n, size_t job_bytes, int fmt) {
if (nj_in <= 0 || max_n <= 0) return true;
if (!ensure()) return false;
// The matvec kernels keep 8 accumulators (the T=8 window): a job of more activations (a 16-token window) is cut into
// jobs of at most 8, each reading its own activations and writing its own rows of y. (Left whole, the columns past the
// eighth were never computed: rows of zeros in every projection of a prompt window.)
std::vector<HostBfJob> cut;
if (job_bytes == sizeof(HostBfJob)) {
const HostBfJob* src = (const HostBfJob*) jobs_in;
for (int i = 0; i < nj_in; ++i) {
const HostBfJob& q = src[i];
const int ncol = q.ncol > 0 ? q.ncol : 1;
for (int c0 = 0; c0 < ncol; c0 += 8) {
HostBfJob b = q;
b.ncol = std::min(8, ncol - c0);
if (c0 != 0) {
if (q.x) b.x = q.x + (size_t) c0 * (size_t) q.xcs;
b.y = q.y + (size_t) c0 * (size_t) q.ycs;
if (q.xq) b.xq = q.xq + (size_t) c0 * (size_t) q.k;
if (q.xd) b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
}
cut.push_back(b);
}
}
}
const void* jobs = cut.empty() ? jobs_in : (const void*) cut.data();
const int nj = cut.empty() ? nj_in : (int) cut.size();
mtl::Buffer bj = g_cx->alloc((size_t) nj * job_bytes);
if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: mx_mv job buffer failed\n"); return false; }
std::memcpy(bj.ptr, jobs, (size_t) nj * job_bytes);
mtl::Pipeline pl;
// fmt 1 = MXFP4, 2 = Q8_0, 0 = BF16 (a dense matrix: the engram's wkv, ds4_gguf.cu:1064)
const char* fn = fmt == 2 ? "ds4_q8_mv" : (fmt == 1 ? "ds4_mx_mv" : "ds4_bf16_mv");
try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (uint32_t) ((max_n + 7) / 8);
d.grid[1] = (uint32_t) nj;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
// ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they
// travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.
static inline void* cpu_addr(void* p);
namespace {
// mirrors ds4_iq.cuh's MvJob (and ds4_iq.metal's Ds4MvJob): 80 bytes on arm64
struct Ds4IqMvJob {
const uint8_t* w;
uint64_t ld;
int type, k, n, ncol;
const int8_t* xq;
const float* xd;
float* y;
uint64_t ycs;
const float* bias;
float alpha;
};
static_assert(sizeof(Ds4IqMvJob) == 80, "Ds4IqMvJob layout must match ds4_iq.cuh's MvJob");
} // namespace
bool iq_mv(const void* jobs, int nj) {
if (nj <= 0) return true;
if (!ensure()) return false;
const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;
// Each job goes to the kernel of its type and column count: NC = 1, 2, 4 or 8 activations decode the row's weights once
// (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.
std::map<std::string, std::vector<Ds4IqMvJob>> groups;
int max_n = 0;
if (std::getenv("DS4_DBG_JOBS")) {
std::fprintf(stderr, "iq_mv call:");
for (int i = 0; i < nj; ++i) std::fprintf(stderr, " [t%d n%d k%d c%d]", src[i].type, src[i].n, src[i].k, src[i].ncol);
std::fprintf(stderr, "\n");
}
static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;
for (int i = 0; i < nj; ++i) {
const Ds4IqMvJob& q = src[i];
{ static const char* sk = std::getenv("DS4_SKIP_TYPES"); if (sk != nullptr && std::strstr(sk, ("," + std::to_string(q.type) + ",").c_str()) != nullptr) continue; } // (timing only)
if (q.n > max_n) max_n = q.n;
const int ncol = q.ncol > 0 ? q.ncol : 1;
const char* tn = nullptr;
switch (q.type) {
case 13: tn = "ds4_q5k_mvn"; break;
case 16: tn = "ds4_iq2xxs_mvn"; break;
case 17: tn = "ds4_iq2xs_mvn"; break;
case 18: tn = "ds4_iq3xxs_mvn"; break;
case 22: tn = "ds4_iq2s_mvn"; break;
case 29: tn = "ds4_iq1m_mvn"; break; // (the 2.00 bpw pack's IQ1_M: ds4_iq.metal's ds4_w32_iq1m)
default: break;
}
const bool grid = tn != nullptr && !old_only;
const int step = grid ? 8 : (ncol > 16 ? 16 : ncol);
for (int c0 = 0; c0 < ncol; c0 += step) {
Ds4IqMvJob b = q;
b.ncol = (c0 + step < ncol) ? step : ncol - c0;
if (c0 != 0) {
b.xq = q.xq + (size_t) c0 * (size_t) q.k;
b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
b.y = q.y + (size_t) c0 * (size_t) q.ycs;
}
std::string fn;
if (grid) {
const int nc = b.ncol <= 1 ? 1 : (b.ncol <= 2 ? 2 : (b.ncol <= 4 ? 4 : 8));
fn = std::string(tn) + std::to_string(nc);
} else {
fn = (b.type == 13 || b.type == 14) ? "ds4_iq_mv_k" : "ds4_iq_mv";
}
groups[fn].push_back(b);
}
}
if (max_n <= 0) return true;
const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group
for (auto& kv : groups) {
std::vector<Ds4IqMvJob>& v = kv.second;
mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");
return false;
}
std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));
mtl::Pipeline pl;
try {
pl = g_cx->pipeline(g_lib_iq, kv.first);
} catch (...) {
g_cx->free(bj);
return false;
}
std::vector<mtl::Dispatch> ds;
for (size_t base = 0; base < v.size(); base += kBatch) {
const size_t cnt = std::min(kBatch, v.size() - base);
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj, base * sizeof(Ds4IqMvJob))};
d.grid[0] = (uint32_t) ((max_n + 7) / 8);
d.grid[1] = (uint32_t) cnt;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
ds.push_back(d);
}
g_cx->run(ds);
g_cx->free(bj);
}
return true;
}
// ds4_iq.cu:536 gate_up_kernel<T, NE> / :588 down_kernel<T, NE> - the prompt / window path's expert GEMM (the launches at
// :958 launch_gu / :960 launch_dn). A threadgroup of 256 is 8 warps: a warp RW rows (gate / up 1, down 2), a lane the
// row's sub-blocks, the group's entries four at a pass. The group's blob and its entries are read the way the CUDA kernel
// reads them, so the plan / window pointer arrays travel as they are: every pointer here is a CPU address (unified memory)
// and a kernel may take raw gpuAddresses (as ds4_finish's f.w does).
namespace {
// (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4IqJob (Metal has no 'long long': 64-bit is 'long')
struct IqJob {
const void* plan_ptr;
const void* grp_ptr;
const void* grp_start;
const void* grp_ent;
const void* n_groups;
const void* xq;
const void* xd;
const void* hq;
const void* hd;
void* h;
void* part;
int64_t tr0, tr1, tr2;
int n_embd, n_ff, type;
float limit;
};
static bool iq_gemm(const IqJob& j, const char* fn, int grid_x, int groups) {
if (!ensure()) return false;
mtl::Buffer bj = g_cx->alloc(sizeof(j));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);
return false;
}
std::memcpy(bj.ptr, &j, sizeof(j));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (uint32_t) grid_x;
d.grid[1] = (uint32_t) groups;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
} // namespace
// gate / up of the group's entries: h = the clamped swiglu (ds4_iq.cuh's Args: tr[0] gate, tr[1] up, tr[2] down)
bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,
const int8_t* xq, const float* xd, float* h, int type, int groups) {
if (groups <= 0) return true;
if (n_embd <= 0 || n_ff <= 0 || n_embd % 256 != 0 || n_ff % 8 != 0) return false;
if (xq == nullptr || xd == nullptr || h == nullptr) return false;
if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
IqJob j{};
j.plan_ptr = plan_ptr;
j.grp_ptr = grp_ptr;
j.grp_start = grp_start;
j.grp_ent = grp_ent;
j.n_groups = n_groups;
j.xq = xq;
j.xd = xd;
j.h = h;
j.tr0 = (int64_t) tr[0];
j.tr1 = (int64_t) tr[1];
j.tr2 = (int64_t) tr[2];
j.n_embd = n_embd;
j.n_ff = n_ff;
j.type = type;
j.limit = limit;
return iq_gemm(j, "ds4_iq_gu", n_ff / 8, groups);
}
// down of the group's entries against their int8 h: part (a row of n_embd an entry)
bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,
const float* hd, float* part, int type, int groups) {
if (groups <= 0) return true;
if (n_embd <= 0 || n_ff <= 0 || n_ff % 256 != 0 || n_embd % 16 != 0) return false;
if (hq == nullptr || hd == nullptr || part == nullptr) return false;
if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
IqJob j{};
j.plan_ptr = plan_ptr;
j.grp_ptr = grp_ptr;
j.grp_start = grp_start;
j.grp_ent = grp_ent;
j.n_groups = n_groups;
j.hq = hq;
j.hd = hd;
j.part = part;
j.tr0 = (int64_t) tr[0];
j.tr1 = (int64_t) tr[1];
j.tr2 = (int64_t) tr[2];
j.n_embd = n_embd;
j.n_ff = n_ff;
j.type = type;
j.limit = 0.0f;
return iq_gemm(j, "ds4_iq_dn", n_embd / 16, groups);
}
// ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel - the MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down /
// win_experts). The same frame as iq_gu / iq_dn above: a threadgroup of 256 is 8 warps, a warp a row, a lane strides the
// 32-value blocks, the group's entries sixteen at a pass. The job struct is the kernel's own (metal/src/engine/msl/
// ds4_iq.metal's Ds4MxJob); the plan / window pointer arrays travel as they are: every pointer here is a CPU address
// (unified memory) and a kernel may take raw gpuAddresses.
namespace {
// (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4MxJob (Metal
// has no 'long long': 64-bit is 'long')
struct MxJob {
const void* plan_ptr;
const void* grp_ptr;
const void* grp_start;
const void* grp_ent;
const void* n_groups;
const void* xq;
const void* xd;
const void* hq;
const void* hd;
void* h;
void* part;
int64_t tr0, tr1, tr2; // ('long' there, so 8 a field and the float ends the struct on the same byte)
int n_embd, n_ff;
float limit;
};
static_assert(sizeof(MxJob) == 128, "Ds4MxJob's layout");
static bool mx_gemm(const MxJob& j, const char* fn, int grid_x, int groups) {
if (!ensure()) return false;
mtl::Buffer bj = g_cx->alloc(sizeof(j));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);
return false;
}
std::memcpy(bj.ptr, &j, sizeof(j));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (uint32_t) grid_x;
d.grid[1] = (uint32_t) groups;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
} // namespace
// gate / up of the group's experts: h = the clamped swiglu (ds4_gguf.cu's MxArgs: tr[0] gate, tr[1] up, tr[2] down)
bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,
const int8_t* xq, const float* xd, float* h, int groups) {
if (groups <= 0) return true;
if (n_embd <= 0 || n_ff <= 0 || n_embd % 32 != 0 || n_ff % 8 != 0) return false;
if (xq == nullptr || xd == nullptr || h == nullptr) return false;
if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
MxJob j{};
j.plan_ptr = plan_ptr;
j.grp_ptr = grp_ptr;
j.grp_start = grp_start;
j.grp_ent = grp_ent;
j.n_groups = n_groups;
j.xq = xq;
j.xd = xd;
j.h = h;
j.tr0 = (int64_t) tr[0];
j.tr1 = (int64_t) tr[1];
j.tr2 = (int64_t) tr[2];
j.n_embd = n_embd;
j.n_ff = n_ff;
j.limit = limit;
return mx_gemm(j, "ds4_mx_gu", (n_ff + 7) / 8, groups);
}
// down of the group's experts against their int8 h: part (a row of n_embd an entry)
bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,
const float* hd, float* part, int groups) {
if (groups <= 0) return true;
if (n_embd <= 0 || n_ff <= 0 || n_ff % 32 != 0 || n_embd % 8 != 0) return false;
if (hq == nullptr || hd == nullptr || part == nullptr) return false;
if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
MxJob j{};
j.plan_ptr = plan_ptr;
j.grp_ptr = grp_ptr;
j.grp_start = grp_start;
j.grp_ent = grp_ent;
j.n_groups = n_groups;
j.hq = hq;
j.hd = hd;
j.part = part;
j.tr0 = (int64_t) tr[0];
j.tr1 = (int64_t) tr[1];
j.tr2 = (int64_t) tr[2];
j.n_embd = n_embd;
j.n_ff = n_ff;
return mx_gemm(j, "ds4_mx_dn", (n_embd + 7) / 8, groups);
}
// ds4_gguf.cu:577 mx_combine_kernel - a token's expert outputs summed in plan order (the experts off the device add 0),
// the CPU lane's part when its flag is set, then the shared expert's: ds4_gguf.cu:169's combine(), 128 threads a block
namespace {
// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Comb (a 'long' there is int64_t here)
struct MxCombJob {
const void* plan_ptr;
const void* plan_w;
const void* part;
const void* cpu_part;
const int* cpu_flag;
const void* sh_out;
void* out;
int64_t cpu_stride, out_stride;
int k, n_embd;
};
} // namespace
bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride,
const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd) {
if (T <= 0 || n_embd <= 0) return true;
if (k <= 0 || plan_ptr == nullptr || plan_w == nullptr || part == nullptr || out == nullptr) return false;
if (!ensure()) return false;
MxCombJob j{};
j.plan_ptr = plan_ptr;
j.plan_w = plan_w;
j.part = part;
j.cpu_part = cpu_part;
j.cpu_flag = cpu_flag;
j.sh_out = sh_out;
j.out = out;
j.cpu_stride = (int64_t) cpu_stride;
j.out_stride = (int64_t) out_stride;
j.k = k;
j.n_embd = n_embd;
mtl::Buffer bj = g_cx->alloc(sizeof(j));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: mx_combine job buffer failed\n");
return false;
}
std::memcpy(bj.ptr, &j, sizeof(j));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_mx_combine"); } catch (...) { g_cx->free(bj); return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (uint32_t) ((n_embd + 127) / 128);
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
// ds4_kernels.cu:1092 - the KV ring row to BF16 (E4M3 code x E8M0 group scale)
bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes) {
if (n <= 0 || d <= 0) return true;
if (!ensure()) return false;
void* hr = strata::gpu::host_view((void*) ring);
void* he = strata::gpu::host_view(ext);
if (!hr || !he) { std::fprintf(stderr, "dsv41 metal: ring_to_ext has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );
mtl::Buffer be = wrap(he, (size_t) n * d * 2 );
if (!br.handle || !be.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_ring_to_ext"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::B(be, off(he)), mtl::Arg::V((long) p0),
mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};
dsp.grid[0] = (uint32_t) n;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// ds4_kernels.cu:1104 - the window back into the ring (E4M3 codes + E8M0 group scales, optional backup)
bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup) {
if (T <= 0 || n <= 0 || d <= 0) return true;
if (!ensure()) return false;
void* he = strata::gpu::host_view((void*) ext);
void* hr = strata::gpu::host_view(ring);
if (!he || !hr) { std::fprintf(stderr, "dsv41 metal: ext_to_ring has no host view\n"); return false; }
void* hb = backup ? strata::gpu::host_view(backup) : nullptr;
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer be = wrap(he, (size_t) (n + T) * d * 2 );
mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );
if (!be.handle || !br.handle) return false;
mtl::Buffer bb = br; // never written when backup is null
size_t b_off = 0;
if (hb) { bb = wrap(hb, (size_t) T * row_bytes ); b_off = off(hb); if (!bb.handle) return false; }
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_ext_to_ring"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),
mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),
mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes), mtl::Arg::V((unsigned) (hb ? 1 : 0))};
dsp.grid[0] = (uint32_t) T;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)
namespace {
// mirrors metal/src/engine/msl/ds4_init.metal's Ds4AttnRows (ds4_kernels.cuh:149 AttnRows, kWin = 16)
struct Ds4AttnRowsJob {
int win0[16], nwin[16], nsel[16];
int gen, t0, win, R, kcap;
int64_t p0;
};
static_assert(sizeof(Ds4AttnRowsJob) == 224, "Ds4AttnRowsJob layout must match the MSL struct");
} // namespace
// ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false> - the prompt path's sparse attention.
// The packed kernel stages rows once for G heads of a query; the BF16 kernel handles rows already in BF16 (a DSpark block).
bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows,
const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt) {
if (T <= 0 || heads <= 0) return true;
if (d != 512 || rows == nullptr || (ckv_fmt != 0 && ckv_fmt != 2)) { std::fprintf(stderr, "dsv41 metal: sparse_attn unsupported\n"); return false; }
Ds4AttnRowsJob ar{};
ar.t0 = (int) T; // the two-query kernel reads the row count here on the gen == 0 path
std::memcpy(&ar, rows, sizeof(ar));
if (q == nullptr || out == nullptr) return false;
if (ar.gen == 0 && T > 16) { std::fprintf(stderr, "dsv41 metal: sparse_attn gen0 T>16\n"); return false; }
if (!ensure()) return false;
void* hq = cpu_addr((void*) q);
void* hw = cpu_addr((void*) win);
void* hc = cpu_addr((void*) ckv);
void* hs = cpu_addr((void*) sel);
void* hk = cpu_addr((void*) sink);
void* ho = cpu_addr((void*) out);
if (!hq || !ho) {
std::fprintf(stderr, "dsv41 metal: sparse_attn has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
int wmax = 0;
if (ar.gen != 0) {
wmax = ar.win + T + 1;
} else {
for (int t = 0; t < T; ++t) wmax = ar.win0[t] + ar.nwin[t] > wmax ? ar.win0[t] + ar.nwin[t] : wmax;
wmax += 1;
}
if (wmax > 0 && hw == nullptr) { std::fprintf(stderr, "dsv41 metal: sparse_attn win null (wmax %d)\n", wmax); return false; }
const size_t wrows = (size_t) (wmax > 0 ? wmax : 1);
size_t crows = 0;
if (ckv_fmt == 2) {
if (ar.gen != 0 && ar.R > 0) {
crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position
} else {
// (the rows the selection names are on the device and not read back here: an earlier build read them on the
// host while the kernel that wrote them was still queued, and sized the binding from stale zeros)
const size_t rem = ds4_gpu_alloc_remaining(hc);
crows = rem > 0 ? rem / 288 : (size_t) (ar.nsel[0] > 0 ? ar.nsel[0] : 1);
}
if (crows < 1) crows = 1;
} else {
crows = (size_t) (T > 16 ? T : 16);
}
const size_t qbytes = (size_t) T * (size_t) heads * 512 * 4;
const size_t wbytes = wrows * 512 * 2;
const size_t cbytes = crows * (ckv_fmt == 2 ? 288 : 1024);
const size_t obytes = (size_t) T * (size_t) heads * 512 * 4;
const size_t sbytes = (size_t) T * (size_t) (sel_stride > 0 ? sel_stride : 1) * 4;
mtl::Buffer bq = wrap(hq, qbytes);
mtl::Buffer bw = hw != nullptr ? wrap(hw, wbytes) : g_cx->alloc(64);
mtl::Buffer bc = hc != nullptr ? wrap(hc, cbytes) : g_cx->alloc(64);
mtl::Buffer bs = hs != nullptr ? wrap(hs, sbytes) : g_cx->alloc(4);
mtl::Buffer bk = hk != nullptr ? wrap(hk, (size_t) heads * 4) : g_cx->alloc((size_t) heads * 4);
mtl::Buffer bo = wrap(ho, obytes);
if (!bq.handle || !bw.handle || !bc.handle || !bs.handle || !bk.handle || !bo.handle) {
std::fprintf(stderr, "dsv41 metal: sparse_attn wrap failed\n");
return false;
}
int G = 8;
if (heads % 8 != 0) G = heads % 4 == 0 ? 4 : (heads % 2 == 0 ? 2 : 1);
while (G > 1 && (heads / G) * T < 64) G /= 2; // keep enough threadgroups on the GPU (CUDA's 2 * SMs, in effect)
static const int attn_mode = std::getenv("DS4_ATTN_MX") ? std::atoi(std::getenv("DS4_ATTN_MX")) : 1; // 1 (default): matrix-unit kernel; 0: the scalar one; 2: both, compare
const bool use_mx = ckv_fmt == 2 && G == 8 && attn_mode != 0;
// DS4_ATTN2Q: two queries a threadgroup, sharing the staged keys (the staging is ~20 % of a 32K prompt and is per
// key, so sharing halves it). Verified bit-exact in tools/ds4/attn_harness, including the causal shape where the
// two rows have different window lengths.
static const bool attn2q_env = std::getenv("DS4_ATTN2Q") != nullptr && use_mx;
// Two queries a threadgroup need a shared window base and selection list (the kernel stages one key set for both,
// and the causal shape gives the second query a superset only when those agree). The rows carry their own
// win0/nwin/nsel, so the pairing is decided here, per dispatch, and never assumed.
bool attn2q = attn2q_env;
if (attn2q) {
for (int t = 1; t < T; ++t) {
if (ar.win0[t] != ar.win0[0] || ar.nsel[t] != ar.nsel[0] || ar.nwin[t] < ar.nwin[t - 1]) { attn2q = false; break; }
}
}
const int qpt = attn2q ? 2 : 1;
const char* fn = attn2q ? "ds4_sparse_attn_mx2" : use_mx ? "ds4_sparse_attn_mx" : ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"
: G == 2 ? "ds4_sparse_attn_g2" : "ds4_sparse_attn_g1")
: "ds4_sparse_attn_bf16";
mtl::Pipeline pl;
try {
// DS4_ATTN_PROBE=N: the attention timing probe (see ATTN_PROBE in ds4_init.metal): 1 skips P.V, 2 skips QK.
// The numbers it produces are wrong on purpose.
const char* ap = std::getenv("DS4_ATTN_PROBE");
if (ap != nullptr && std::atoi(ap) != 0) {
mtl::Constants cs;
cs.i(25, std::atoi(ap));
pl = g_cx->pipeline_uncached(g_lib, fn, cs);
} else {
pl = g_cx->pipeline(g_lib, fn);
}
} catch (...) {
std::fprintf(stderr, "dsv41 metal: sparse_attn pipeline %s unavailable\n", fn);
return false;
}
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, (hw != nullptr ? off(hw) : (size_t) 0)),
mtl::Arg::B(bc, (hc != nullptr ? off(hc) : (size_t) 0)),
mtl::Arg::B(bs, (hs != nullptr ? off(hs) : (size_t) 0)),
mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),
mtl::Arg::B(bk, (hk != nullptr ? off(hk) : (size_t) 0)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),
mtl::Arg::B(bo, off(ho))};
dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);
dsp.grid[1] = (uint32_t) ((T + qpt - 1) / qpt);
dsp.grid[2] = 1;
dsp.tg[0] = 256;
dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
// the stand-ins for absent inputs are fresh Metal buffers (16 KB of address space each): give them back
if (hw == nullptr) g_cx->free(bw);
if (hc == nullptr) g_cx->free(bc);
if (hs == nullptr) g_cx->free(bs);
if (hk == nullptr) g_cx->free(bk);
if (use_mx && attn_mode == 2) { // debugging: the scalar kernel on the same inputs, then the difference
mtl::Pipeline ps;
try { ps = g_cx->pipeline(g_lib, "ds4_sparse_attn_g8"); } catch (...) { return true; }
mtl::Buffer bo2 = g_cx->alloc(obytes);
mtl::Dispatch d2 = dsp;
d2.p = &ps;
d2.args.back() = mtl::Arg::B(bo2, 0);
g_cx->run({d2});
g_cx->flush_runs();
const float* a = (const float*) ho;
const float* c = (const float*) bo2.ptr;
double mx_abs = 0, mx_ref = 0, sum_sq = 0, sum_ref = 0;
for (size_t i = 0; i < obytes / 4; ++i) {
const double df = std::fabs((double) a[i] - (double) c[i]);
mx_abs = std::max(mx_abs, df);
mx_ref = std::max(mx_ref, (double) std::fabs(c[i]));
sum_sq += df * df;
sum_ref += (double) c[i] * (double) c[i];
}
static double worst_rel = 0; static long calls = 0;
const double rel = std::sqrt(sum_sq / (sum_ref + 1e-30));
worst_rel = std::max(worst_rel, rel);
if (++calls % 50 == 1) std::fprintf(stderr, "attn mx vs scalar: call %ld T %d max abs %.3g (max |ref| %.3g) rms rel %.3g worst rms rel %.3g\n", calls, T, mx_abs, mx_ref, rel, worst_rel);
g_cx->free(bo2);
}
return true;
}
// ds4_kernels.cu:461 route_kernel - sqrt(softplus(logits)) top k with bias, then the unbiased weights. vl is a VlBits
// (1024 bytes) or null; bias_vl applies to the rows its bits select.
bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
const float* bias_vl, const void* vl) {
if (T <= 0) return true;
if (logits == nullptr || bias == nullptr || idx == nullptr || w == nullptr || n <= 0 || n > 1024 || k <= 0 || k > 8)
return false;
if (!ensure()) return false;
void* hl = cpu_addr((void*) logits);
void* hb = cpu_addr((void*) bias);
void* hi = cpu_addr((void*) idx);
void* hw = cpu_addr((void*) w);
void* hv = bias_vl != nullptr ? cpu_addr((void*) bias_vl) : nullptr;
if (!hl || !hb || !hi || !hw) {
std::fprintf(stderr, "dsv41 metal: route has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4);
mtl::Buffer bb = wrap(hb, (size_t) n * 4);
mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4);
mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4);
mtl::Buffer bv = hv != nullptr ? wrap(hv, (size_t) n * 4) : g_cx->alloc(4);
mtl::Buffer bbv = g_cx->alloc(1024);
if (!bl.handle || !bb.handle || !bi.handle || !bw.handle || !bv.handle || !bbv.ptr) return false;
const bool has_vl = vl != nullptr;
if (has_vl) std::memcpy(bbv.ptr, vl, 1024);
mtl::Pipeline pl;
try {
pl = g_cx->pipeline(g_lib, "ds4_route");
} catch (...) {
std::fprintf(stderr, "dsv41 metal: route pipeline unavailable\n");
return false;
}
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),
mtl::Arg::V(route_scale), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),
mtl::Arg::B(bv, hv != nullptr ? off(hv) : 0), mtl::Arg::V(has_vl ? 1u : 0u), mtl::Arg::B(bbv)};
dsp.grid[0] = (uint32_t) T;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 512;
dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
g_cx->free(bbv);
if (hv == nullptr) g_cx->free(bv);
return true;
}
namespace {
struct Ds4GatherJob {
int src[16 * 8];
};
struct Ds4CombineJob {
int n[16];
int row[16][8];
float w[16][8];
int cpu[16];
};
static_assert(sizeof(Ds4GatherJob) == 512, "Ds4GatherJob layout must match the MSL struct");
static_assert(sizeof(Ds4CombineJob) == 1152, "Ds4CombineJob layout must match the MSL struct");
} // namespace
// ds4_kernels.cu:1336 gather_kernel - out[r] = x[g.src[r]] for n rows of cols floats
bool gather_rows(const float* x, const void* g, int n, int cols, float* out) {
if (n <= 0 || cols <= 0) return true;
if (x == nullptr || g == nullptr || out == nullptr || n > 16 * 8) return false;
if (!ensure()) return false;
void* hx = cpu_addr((void*) x);
void* ho = cpu_addr((void*) out);
if (!hx || !ho) return false;
Ds4GatherJob gj{};
std::memcpy(&gj, g, sizeof(gj));
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
int max_src = 0;
for (int i = 0; i < n; ++i) max_src = gj.src[i] > max_src ? gj.src[i] : max_src;
mtl::Buffer bx = wrap(hx, (size_t) (max_src + 1) * cols * 4); // src rows may be beyond the n output rows
mtl::Buffer bo = wrap(ho, (size_t) n * cols * 4);
if (!bx.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_gather"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V(gj), mtl::Arg::V((unsigned) cols), mtl::Arg::B(bo, off(ho))};
dsp.grid[0] = (uint32_t) ((cols + 255) / 256);
dsp.grid[1] = (uint32_t) n;
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// ds4_kernels.cu:1342 combine_kernel - out[t] = sum_j w[t][j] ed[row[t][j]] + cpu[t] (when set) + sh[t]
bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out) {
if (T <= 0 || dim <= 0) return true;
if (ed == nullptr || c == nullptr || out == nullptr || T > 16) return false;
if (!ensure()) return false;
void* he = cpu_addr((void*) ed);
void* hc = cpu != nullptr ? cpu_addr((void*) cpu) : nullptr;
void* hs = sh != nullptr ? cpu_addr((void*) sh) : nullptr;
void* ho = cpu_addr((void*) out);
if (!he || !ho) return false;
Ds4CombineJob cj{};
std::memcpy(&cj, c, sizeof(cj));
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
int max_row = 0;
for (int t = 0; t < T; ++t)
for (int j = 0; j < cj.n[t]; ++j) max_row = cj.row[t][j] > max_row ? cj.row[t][j] : max_row;
mtl::Buffer be = wrap(he, (size_t) (max_row + 1) * dim * 4); // rows may be beyond T * kPicks
mtl::Buffer bc = hc != nullptr ? wrap(hc, (size_t) T * dim * 4) : g_cx->alloc(4);
mtl::Buffer bs = hs != nullptr ? wrap(hs, (size_t) T * dim * 4) : g_cx->alloc(4);
mtl::Buffer bo = wrap(ho, (size_t) T * dim * 4);
if (!be.handle || !bc.handle || !bs.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_combine"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(bc, hc != nullptr ? off(hc) : 0),
mtl::Arg::B(bs, hs != nullptr ? off(hs) : 0), mtl::Arg::V(cj), mtl::Arg::V((unsigned) dim),
mtl::Arg::V(hc != nullptr ? 1u : 0u), mtl::Arg::V(hs != nullptr ? 1u : 0u), mtl::Arg::B(bo, off(ho))};
dsp.grid[0] = (uint32_t) ((dim + 255) / 256);
dsp.grid[1] = (uint32_t) T;
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
if (hc == nullptr) g_cx->free(bc);
if (hs == nullptr) g_cx->free(bs);
return true;
}
// On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about
// (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.
static inline void* cpu_addr(void* p) {
void* h = strata::gpu::host_view(p);
return h ? h : p;
}
// ds4_kernels.cu:1286 engram_dequant_kernel - the Engram rows (fmt 2 = IQ4_NL, the tables this model's pack carries; the
// other formats are left to the CUDA launch, which reports the line)
bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt) {
(void) scales; // (fmt 2 keeps every block's scale in its own 18 bytes)
if (n <= 0 || hd <= 0) return true;
if (fmt != 2 || rows == nullptr || out == nullptr) return false;
if (!ensure()) return false;
void* hr = cpu_addr((void*) rows);
void* ho = cpu_addr((void*) out);
if (!hr || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
// (the kernel strides rows by hd bytes, as the CUDA kernel does, though an IQ4_NL row only fills hd/32*18 of them:
// wrapping n * 18/32 * hd left the last rows outside the buffer - garbage scales, measured as inf at row 257 of 288)
mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);
mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);
if (!br.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant_rows"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(fmt)};
d.grid[0] = (uint32_t) n;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:1311 engram_gate_kernel - the Engram gate over a (token, copy) row: h += sigmoid(normalised dot) * val
bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps) {
if (T <= 0 || hc <= 0 || dim <= 0) return true;
if (h == nullptr || kv == nullptr || w == nullptr) return false;
if (!ensure()) return false;
void* hh = cpu_addr((void*) h);
void* hk = cpu_addr((void*) kv);
void* hw = cpu_addr((void*) w);
if (!hh || !hk || !hw) { std::fprintf(stderr, "dsv41 metal: engram_gate has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);
mtl::Buffer bk = wrap(hk, (size_t) T * (size_t) (hc + 1) * (size_t) dim * 4);
mtl::Buffer bw = wrap(hw, (size_t) hc * (size_t) dim * 4);
if (!bh.handle || !bk.handle || !bw.handle) {
std::fprintf(stderr, "dsv41 metal: engram_gate wrap failed (h %p kv %p w %p; %p %p %p)\n", hh, hk, hw,
bh.handle, bk.handle, bw.handle);
return false;
}
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_engram_gate"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bk, off(hk)), mtl::Arg::B(bw, off(hw)),
mtl::Arg::V((unsigned) hc), mtl::Arg::V((unsigned) dim), mtl::Arg::V(eps)};
d.grid[0] = (uint32_t) hc;
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
namespace {
// mirrors metal/src/engine/msl/ds4_init.metal's Ds4Rows (ds4_kernels.cuh:115 Rows, kWin = 16)
struct Ds4RowsJob {
int n[16];
int k[16];
int gen, R, kcap, ccap, blk;
int64_t p0;
};
} // namespace
// ds4_kernels.cu:524 compress_kernel - the window compressor (ratio > 1): each token's kv / score row into its slot, and
// every ratio'th token pooled by a per-value softmax over the slots, scaled by the row RMS and the norm weights
bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
int d, const float* w, float eps, float* latent) {
if (T <= 0 || d <= 0 || ratio <= 0) return true;
if (d > 2 * 512) return false; // (the kernel holds d <= 2 * blockDim)
if (kv == nullptr || score == nullptr || kv_state == nullptr || score_state == nullptr || w == nullptr ||
latent == nullptr)
return false;
if (!ensure()) return false;
void* hkv = cpu_addr((void*) kv);
void* hsc = cpu_addr((void*) score);
void* hks = cpu_addr((void*) kv_state);
void* hss = cpu_addr((void*) score_state);
void* hw = cpu_addr((void*) w);
void* hla = cpu_addr((void*) latent);
if (!hkv || !hsc || !hks || !hss || !hw || !hla) {
std::fprintf(stderr, "dsv41 metal: compress_win has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bkv = wrap(hkv, (size_t) T * (size_t) d * 4);
mtl::Buffer bsc = wrap(hsc, (size_t) T * (size_t) d * 4);
mtl::Buffer bks = wrap(hks, (size_t) ratio * (size_t) d * 4);
mtl::Buffer bss = wrap(hss, (size_t) ratio * (size_t) d * 4);
mtl::Buffer bw = wrap(hw, (size_t) d * 4);
mtl::Buffer bla = wrap(hla, (size_t) T * (size_t) d * 4);
if (!bkv.handle || !bsc.handle || !bks.handle || !bss.handle || !bw.handle || !bla.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_compress"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bkv, off(hkv)), mtl::Arg::B(bsc, off(hsc)), mtl::Arg::B(bks, off(hks)),
mtl::Arg::B(bss, off(hss)), mtl::Arg::V(ratio), mtl::Arg::V(p0), mtl::Arg::V(T),
mtl::Arg::V((unsigned) d), mtl::Arg::B(bw, off(hw)), mtl::Arg::V(eps), mtl::Arg::B(bla, off(hla))};
dsp.grid[0] = 1;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 512; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// ds4_kernels.cu:582 index_score_kernel<128, kfmt> - the indexer's scores: a position's key row against every head's
// query, relu, the heads' weighted sum (rows = ds4_kernels.cuh's Rows, by value there and copied here)
bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand,
int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt) {
if (n <= 0 || T <= 0) return true;
if (hd != 128 || heads <= 0 || heads > 32 || (kfmt != 3 && kfmt != 0)) return false;
if (q == nullptr || wq == nullptr || kc == nullptr || out == nullptr || rows == nullptr) return false;
if (!ensure()) return false;
void* hq = cpu_addr((void*) q);
void* hwq = cpu_addr((void*) wq);
void* hkc = cpu_addr((void*) kc);
void* hout = cpu_addr((void*) out);
if (!hq || !hwq || !hkc || !hout) {
std::fprintf(stderr, "dsv41 metal: index_score has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
const size_t kb = kfmt == 3 ? (size_t) ((hd / 2 + hd / 32 + 15) / 16 * 16) : (size_t) hd * 2;
mtl::Buffer bq = wrap(hq, (size_t) T * (size_t) heads * (size_t) hd * 4);
mtl::Buffer bwq = wrap(hwq, (size_t) T * 64 * 4);
mtl::Buffer bkc = wrap(hkc, (size_t) n * kb);
mtl::Buffer bo = wrap(hout, ((size_t) (T - 1) * (size_t) out_stride + (size_t) n) * 4);
if (!bq.handle || !bwq.handle || !bkc.handle || !bo.handle) return false;
const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;
mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);
if (!bc.handle) return false;
// Rows is a host struct: it travels as a copy (no gpu address)
mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: index_score rows buffer failed\n"); return false; }
std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_index_score"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),
mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::B(bo, off(hout)),
mtl::Arg::B(br), mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V(cand_stride), mtl::Arg::V(blk),
mtl::Arg::V((long) out_stride), mtl::Arg::V(kfmt), mtl::Arg::V(cand != nullptr ? 1 : 0)};
// DS4_IDX_MMA: the matrix-unit indexer (ds4_index_score_mma). It stages 16 fp4 keys a threadgroup and runs the
// 32 x 128 x 16 head-dot product through simdgroup_matrix - 3.17-6.10x the scalar loop in the standalone harness,
// with every output matching to half precision. Opt-in until the gate and KL say the rounding is acceptable.
// On by default: measured +7.7 to +12 % on the 32K prompt and +2 % at 3K, with the KL unchanged on both packs
// (0.11018 and 0.17087). DS4_NO_IDX_MMA=1 restores the scalar indexer; on the 2.00 pack the score hash moves
// (e81282a3 -> 0800179d) because half rounding breaks token ties, while KL and top-1 are identical.
static const bool idx_mma = std::getenv("DS4_NO_IDX_MMA") == nullptr;
if (idx_mma && cand == nullptr && kfmt == 3) {
mtl::Pipeline pm;
try { pm = g_cx->pipeline(g_lib, "ds4_index_score_mma"); } catch (...) { pm = mtl::Pipeline{}; }
if (pm.handle) {
mtl::Dispatch dm;
dm.p = ±
dm.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),
mtl::Arg::B(bc, 0), mtl::Arg::B(bo, off(hout)), mtl::Arg::B(br),
mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V((int) T), mtl::Arg::V((long) out_stride)};
dm.grid[0] = (uint32_t) ((n + 15) / 16);
dm.grid[1] = (uint32_t) T;
dm.grid[2] = 1;
dm.tg[0] = 256; dm.tg[1] = dm.tg[2] = 1;
g_cx->run({dm});
g_cx->free(br);
if (bc.handle && cand != nullptr) g_cx->free(bc);
return true;
}
}
dsp.grid[0] = (uint32_t) ((n + 255) / 256);
dsp.grid[1] = (uint32_t) T;
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
g_cx->free(br);
if (cand == nullptr) g_cx->free(bc);
return true;
}
// ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations, a warp a row of the matrix and 2 tokens
// a block (the indexer's wproj: ds4_model.cu:7743)
bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy) {
if (T <= 0 || n <= 0 || k <= 0) return true;
if (k % 8 != 0 || W == nullptr || x == nullptr || y == nullptr) return false;
if (!ensure()) return false;
void* hw = cpu_addr((void*) W);
void* hx = cpu_addr((void*) x);
void* hy = cpu_addr((void*) y);
if (!hw || !hx || !hy) {
std::fprintf(stderr, "dsv41 metal: gemv_f16_rows has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bw = wrap(hw, (size_t) n * (size_t) k * 2);
mtl::Buffer bx = wrap(hx, ((size_t) (T - 1) * ldx + (size_t) k) * 4);
mtl::Buffer by = wrap(hy, ((size_t) (T - 1) * ldy + (size_t) n) * 4);
if (!bw.handle || !bx.handle || !by.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f16_rows"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V(n),
mtl::Arg::V(k), mtl::Arg::V(T), mtl::Arg::V(alpha), mtl::Arg::B(by, off(hy)),
mtl::Arg::V((unsigned long) ldy)};
dsp.grid[0] = (uint32_t) ((n + 7) / 8);
dsp.grid[1] = (uint32_t) ((T + 1) / 2);
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// ds4_kernels.cu:650 topk_kernel - a token's k largest indexer scores (four 8-bit radix passes, then the picks in index
// order); Rows by value there and copied here
bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out,
int out_stride) {
if (T <= 0) return true;
if (vals == nullptr || out == nullptr || rows == nullptr) return false;
if (!ensure()) return false;
void* hv = cpu_addr((void*) vals);
void* ho = cpu_addr((void*) out);
if (!hv || !ho) {
std::fprintf(stderr, "dsv41 metal: topk has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bv = wrap(hv, ((size_t) (T - 1) * (size_t) stride + 1) * 4);
mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + 1) * 4);
if (!bv.handle || !bo.handle) return false;
const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;
mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);
if (!bc.handle) return false;
mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: topk rows buffer failed\n"); return false; }
std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_topk"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bv, off(hv)), mtl::Arg::V((long) stride), mtl::Arg::B(br),
mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::V(cand_stride),
mtl::Arg::V(blk), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(out_stride),
mtl::Arg::V(cand != nullptr ? 1 : 0)};
dsp.grid[0] = (uint32_t) T;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
g_cx->free(br);
if (cand == nullptr) g_cx->free(bc);
return true;
}
// ds4_kernels.cu:751 block_max_kernel - a block of the indexer's positions and its largest score (the newest pinned at +inf)
bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride,
int nb_max) {
if (T <= 0 || nb_max <= 0 || blk <= 0) return true;
if (score == nullptr || bm == nullptr || rows == nullptr) return false;
if (!ensure()) return false;
void* hs = cpu_addr((void*) score);
void* hb = cpu_addr((void*) bm);
if (!hs || !hb) {
std::fprintf(stderr, "dsv41 metal: block_max has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bs = wrap(hs, ((size_t) (T - 1) * (size_t) stride + (size_t) nb_max * (size_t) blk) * 4);
mtl::Buffer bb = wrap(hb, ((size_t) (T - 1) * (size_t) bm_stride + (size_t) nb_max) * 4);
if (!bs.handle || !bb.handle) return false;
mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: block_max rows buffer failed\n"); return false; }
std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_block_max"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((long) stride), mtl::Arg::B(br), mtl::Arg::V(blk),
mtl::Arg::B(bb, off(hb)), mtl::Arg::V((long) bm_stride)};
dsp.grid[0] = (uint32_t) ((nb_max + 255) / 256);
dsp.grid[1] = (uint32_t) T;
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
g_cx->free(br);
return true;
}
// ds4_kernels.cu:1141 hc_mean_kernel - a token's mean over its hc streams (grid (ceil(dim / 256), T), 256 threads)
bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride) {
if (T <= 0 || hc <= 0 || dim <= 0) return true;
if (h == nullptr || out == nullptr) return false;
if (!ensure()) return false;
void* hh = cpu_addr((void*) h);
void* ho = cpu_addr((void*) out);
if (!hh || !ho) {
std::fprintf(stderr, "dsv41 metal: hc_mean has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);
mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + (size_t) dim) * 4);
if (!bh.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_mean"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::V(hc), mtl::Arg::V(dim), mtl::Arg::B(bo, off(ho)),
mtl::Arg::V((long) out_stride)};
dsp.grid[0] = (uint32_t) ((dim + 255) / 256);
dsp.grid[1] = (uint32_t) T;
dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// ds4_kernels.cu:1150 iota_rows_kernel - out[t][i] = i
// ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel - the drafter's head. markov_add is a warp a vocabulary
// row (the previous token's E row staged in the threadgroup, then E . H[v] added to logits[v]); confidence a threadgroup
// a token. Both jobs are the kernels' own structs (metal/src/engine/msl/ds4_init.metal's Ds4MarkovJob / Ds4ConfJob).
namespace {
// (not an entry point: the generator takes only the ^bool lines) mirrors ds4_init.metal's Ds4MarkovJob / Ds4ConfJob
struct MarkovJob {
void* logits;
const void* E;
const void* H;
const void* tok;
int V, R;
};
static_assert(sizeof(MarkovJob) == 40, "Ds4MarkovJob's layout");
struct ConfJob {
const void* x;
const void* E;
const void* tok;
const void* w;
void* out;
int dim, R;
};
static_assert(sizeof(ConfJob) == 48, "Ds4ConfJob's layout");
} // namespace
bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R) {
if (logits == nullptr || E == nullptr || H == nullptr || tok == nullptr) return false;
if (V <= 0 || R <= 0 || R > 1024) return false; // (the kernel's staged E row holds 1024 floats)
if (!ensure()) return false;
MarkovJob j{};
j.logits = const_cast<float*>(logits);
j.E = E;
j.H = H;
j.tok = tok;
j.V = V;
j.R = R;
mtl::Buffer bj = g_cx->alloc(sizeof(j));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: markov_add job buffer failed\n");
return false;
}
std::memcpy(bj.ptr, &j, sizeof(j));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_markov_add"); } catch (...) { g_cx->free(bj); return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (uint32_t) ((V + 7) / 8);
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out) {
if (x == nullptr || E == nullptr || tok == nullptr || w == nullptr || out == nullptr) return false;
if (T <= 0 || dim <= 0 || R <= 0) return false;
if (!ensure()) return false;
ConfJob j{};
j.x = x;
j.E = E;
j.tok = tok;
j.w = w;
j.out = out;
j.dim = dim;
j.R = R;
mtl::Buffer bj = g_cx->alloc(sizeof(j));
if (!bj.ptr) {
std::fprintf(stderr, "dsv41 metal: confidence job buffer failed\n");
return false;
}
std::memcpy(bj.ptr, &j, sizeof(j));
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_confidence"); } catch (...) { g_cx->free(bj); return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bj)};
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bj);
return true;
}
bool iota_rows(int* out, int T, int n, int stride) {
if (T <= 0 || n <= 0) return true;
if (out == nullptr) return false;
if (!ensure()) return false;
void* ho = cpu_addr((void*) out);
if (!ho) {
std::fprintf(stderr, "dsv41 metal: iota_rows has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) stride + (size_t) n) * 4);
if (!bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_iota_rows"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bo, off(ho)), mtl::Arg::V(n), mtl::Arg::V(stride)};
dsp.grid[0] = (uint32_t) T;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 64; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
return true;
}
// vendor/sampler.cu:93 sampler_greedy_kernel - the plain argmax over the vocabulary, one block (1024) a token, the
// repetition penalties from the history window's bitmap (the caller takes this path when greedy or temperature <= 0)
bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n,
float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out) {
if (n_tokens <= 0 || n_vocab <= 0) return true;
if (logits == nullptr || out == nullptr) return false;
if ((n_vocab + 31) / 32 > 4096) return false; // (the kernel's threadgroup bitmap)
if (!ensure()) return false;
void* hl = cpu_addr((void*) logits);
void* ho = cpu_addr((void*) out);
if (!hl || !ho) {
std::fprintf(stderr, "dsv41 metal: sample_greedy has no host view\n");
return false;
}
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bl = wrap(hl, (size_t) n_tokens * (size_t) n_vocab * 4);
mtl::Buffer bo = wrap(ho, (size_t) n_tokens * 4);
if (!bl.handle || !bo.handle) return false;
const size_t hbytes = (size_t) n_tokens * (size_t) (history_len > 0 ? history_len : 1) * 4;
mtl::Buffer bh = history != nullptr ? wrap(cpu_addr((void*) history), hbytes) : g_cx->alloc(4);
mtl::Buffer bb = logit_bias != nullptr ? wrap(cpu_addr((void*) logit_bias), (size_t) n_vocab * 4) : g_cx->alloc(4);
if (!bh.handle || !bb.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_sampler_greedy"); } catch (...) { return false; }
mtl::Dispatch dsp;
dsp.p = &pl;
dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::V(n_vocab),
mtl::Arg::B(bh, history != nullptr ? off(cpu_addr((void*) history)) : 0),
mtl::Arg::V(history_len), mtl::Arg::V(penalty_last_n), mtl::Arg::V(penalty_repeat),
mtl::Arg::V(penalty_freq), mtl::Arg::V(penalty_present),
mtl::Arg::B(bb, logit_bias != nullptr ? off(cpu_addr((void*) logit_bias)) : 0),
mtl::Arg::V(logit_bias != nullptr ? 1 : 0), mtl::Arg::B(bo, off(ho))};
dsp.grid[0] = (uint32_t) n_tokens;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;
g_cx->run({dsp});
if (history == nullptr) g_cx->free(bh);
if (logit_bias == nullptr) g_cx->free(bb);
return true;
}
bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap,
const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes) {
if (T <= 0 || sel_stride <= 0 || nsel_count <= 0) return true;
// nothing selected yet (a prompt at position 0): the scratch stays untouched, which is what the CUDA launch would do with
// no rows to write
if (ckv == nullptr || sel == nullptr || out == nullptr) return true;
if (!ensure()) return false;
void* hc = cpu_addr((void*) ckv);
void* hs = cpu_addr((void*) sel);
void* ho = cpu_addr((void*) out);
if (!hc || !hs || !ho) { std::fprintf(stderr, "dsv41 metal: unpack_sel has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
int maxrow = -1;
if (gen != 0 && R > 0) {
maxrow = (int) ((p0 + t0 + T) / R) + 1;
} else if (gen == 0 && nsel != nullptr) {
for (int t = 0; t < T && t < nsel_count; ++t)
for (int i = 0; i < nsel[t]; ++i) {
const int r = sel[(size_t) t * (size_t) sel_stride + (size_t) i];
if (r > maxrow) maxrow = r;
}
}
mtl::Buffer bc = wrap(hc, (size_t) (maxrow + 1 > 0 ? maxrow + 1 : 1) * row_bytes);
mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );
mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );
if (!bc.handle || !bsl.handle || !bo.handle) return false;
mtl::Buffer bn = g_cx->alloc((size_t) nsel_count * 4);
if (!bn.ptr) return false;
std::memcpy(bn.ptr, nsel, (size_t) nsel_count * 4);
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_unpack_sel"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bc, off(hc)), mtl::Arg::B(bsl, off(hs)), mtl::Arg::V((unsigned) sel_stride),
mtl::Arg::V((int) gen), mtl::Arg::V((long) p0), mtl::Arg::V((int) t0), mtl::Arg::V((int) R),
mtl::Arg::V((int) kcap), mtl::Arg::B(bn), mtl::Arg::V((unsigned) cap), mtl::Arg::B(bo, off(ho)),
mtl::Arg::V((unsigned long) row_bytes)};
d.grid[0] = (uint32_t) ((sel_stride + 15) / 16);
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
g_cx->free(bn);
return true;
}
// ds4_kernels.cu:1772 - the expert outputs back into the token rows
bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim) {
if (T <= 0 || dim <= 0) return true;
if (!ensure()) return false;
void* ha = cpu_addr(acc);
void* hy = cpu_addr((void*) y);
void* hp = cpu_addr((void*) pos);
void* hw = cpu_addr((void*) w_);
if (!ha || !hy || !hp || !hw) { std::fprintf(stderr, "dsv41 metal: moe_combine has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer ba = wrap(ha, (size_t) T * dim * 4 );
mtl::Buffer by = wrap(hy, (size_t) (r1 > r0 ? r1 - r0 : 1) * dim * 4 );
mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );
if (!ba.handle || !by.handle || !bp.handle || !bw.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_moe_combine"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(by, off(hy)), mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bw, off(hw)),
mtl::Arg::V((int) k), mtl::Arg::V((int) r0), mtl::Arg::V((int) r1), mtl::Arg::V((unsigned) dim)};
d.grid[0] = (uint32_t) ((dim + 255) / 256);
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:362 - hc_post: out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]
bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out) {
if (T <= 0 || dim <= 0) return true;
if (!ensure()) return false;
void* hy = cpu_addr((void*) y);
void* hr = cpu_addr((void*) res);
void* hp = cpu_addr((void*) post);
void* hco = cpu_addr((void*) comb);
void* ho = cpu_addr((void*) out);
if (!hy || !hr || !hp || !hco || !ho) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
mtl::Buffer br = wrap(hr, (size_t) T * hc * dim * 4 );
mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
mtl::Buffer bc = wrap(hco, (size_t) T * 64 * 4 );
mtl::Buffer bo = wrap(ho, (size_t) T * hc * dim * 4 );
if (!by.handle || !br.handle || !bp.handle || !bc.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_hc_post_v"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp)),
mtl::Arg::B(bc, off(hco)), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((unsigned) dim),
mtl::Arg::V((unsigned) hc)};
d.grid[0] = (uint32_t) ((dim * hc + 255) / 256);
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:441 - the F32 GEMV (residual / dense projections)
bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy) {
if (T <= 0 || n <= 0 || k <= 0) return true;
if (!ensure()) return false;
void* hw = cpu_addr((void*) W);
void* hx = cpu_addr((void*) x);
void* hy = cpu_addr((void*) y);
if (!hw || !hx || !hy) { std::fprintf(stderr, "dsv41 metal: gemv_f32 has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bw = wrap(hw, (size_t) n * k * 4 );
mtl::Buffer bx = wrap(hx, (size_t) T * k * 4 );
mtl::Buffer by = wrap(hy, (size_t) T * (ldy > 0 ? ldy : n) * 4 );
if (!bw.handle || !bx.handle || !by.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f32"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned) n),
mtl::Arg::V((unsigned) k), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned) ldy)};
d.grid[0] = (uint32_t) ((n + 7) / 8);
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu:1353 - the router's selection (idx, w, plan_ptr). The prefetch requests stay unpublished for now.
bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram,
int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl) {
if (T <= 0 || n <= 0) return true;
if (!ensure()) return false;
void* hl = cpu_addr((void*) logits);
void* hb = cpu_addr((void*) bias);
void* hv = cpu_addr((void*) vram);
void* hi = cpu_addr((void*) idx);
void* hw = cpu_addr((void*) w);
void* hp = cpu_addr((void*) plan_ptr);
if (!hl || !hb || !hv || !hi || !hw || !hp) { std::fprintf(stderr, "dsv41 metal: route_publish has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4 );
mtl::Buffer bb = wrap(hb, (size_t) n * 4 );
mtl::Buffer bv = wrap(hv, (size_t) n * 8 );
mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4 );
mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );
mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 8 );
if (!bl.handle || !bb.handle || !bv.handle || !bi.handle || !bw.handle || !bp.handle) return false;
mtl::Buffer bvl = bb;
size_t vl_off = 0;
if (bias_vl) {
void* hvl = cpu_addr((void*) bias_vl);
if (!hvl) return false;
bvl = wrap(hvl, (size_t) n * 4 );
vl_off = off(hvl);
}
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_route_publish"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),
mtl::Arg::V(route_scale), mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),
mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bvl, vl_off), mtl::Arg::V((unsigned) vl),
mtl::Arg::V((unsigned) (bias_vl ? 1 : 0))};
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 512; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// ds4_kernels.cu swiglu: h = silu(g) * u (clamped when lim > 0)
bool swiglu(const float* g, const float* u, int n, float lim, float* h) {
if (n <= 0) return true;
if (!ensure()) return false;
void* hg = cpu_addr((void*) g);
void* hu = cpu_addr((void*) u);
void* hh = cpu_addr((void*) h);
if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + (size_t) n * 4;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);
if (!bg.handle || !bu.handle || !bh.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_swiglu"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bg, off(hg)), mtl::Arg::B(bu, off(hu)), mtl::Arg::V((unsigned) n), mtl::Arg::V(lim),
mtl::Arg::B(bh, off(hh))};
d.grid[0] = (uint32_t) ((n + 255) / 256);
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// vendor/moe_win.cu:49 moe_win_group - the expert -> group index for a window. One thread in CUDA; on the Mac every device
// pointer is a CPU address, so the same 25-line loop runs on the host, which keeps the first-appearance order the kernel has and
// avoids mirroring the 17-field MoeWin struct through a constant buffer.
bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k) {
if (T <= 0 || k <= 0) return true;
if (T * 8 > 4096) { std::fprintf(stderr, "dsv41 metal: moe_win_group window of %d too wide\n", T); return false; }
const unsigned long long* plan = (const unsigned long long*) cpu_addr((void*) plan_ptr);
unsigned long long* gptr = (unsigned long long*) cpu_addr(grp_ptr);
int* gstart = (int*) cpu_addr(grp_start);
int* gent = (int*) cpu_addr(grp_ent);
int* ng_out = (int*) cpu_addr(n_groups);
if (!plan || !gptr || !gstart || !gent || !ng_out) return false;
static unsigned long long gp[4096];
static int cnt[4096], fill[4097], gof[4096];
int ng = 0;
for (int t = 0; t < T; ++t) {
for (int i = 0; i < 8; ++i) {
const int e = t * 8 + i;
gof[e] = -1;
const unsigned long long p = i < k ? plan[e] : 0ull;
if (p == 0ull) continue;
int g = 0;
while (g < ng && gp[g] != p) ++g;
if (g == ng) {
gp[ng] = p;
cnt[ng++] = 0;
}
++cnt[g];
gof[e] = g;
}
}
fill[0] = 0;
for (int g = 0; g < ng; ++g) {
gptr[g] = gp[g];
gstart[g] = fill[g];
fill[g + 1] = fill[g] + cnt[g];
}
gstart[ng] = fill[ng];
for (int e = 0; e < T * 8; ++e) {
if (gof[e] >= 0) gent[fill[gof[e]]++] = e;
}
*ng_out = ng;
return true;
}
// ---- the prompt chunk's dense GEMM: float rows -> half, a quantized matrix -> half rows, MPP matmul2d (kernels/dense_mm.metal)
namespace {
mtl::Buffer g_gm_xs, g_gm_yt; // grow-only scratch: packed activations, the contiguous result
int g_lib_mm = -2;
mtl::Buffer scratch(mtl::Buffer& b, size_t bytes) {
if (!b.handle || b.bytes < bytes) {
if (b.handle) g_cx->free(b);
b = g_cx->alloc(bytes + (bytes >> 2));
}
return b;
}
} // namespace
bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out) {
if (nrows <= 0 || ncols <= 0) return true;
if (!ensure()) return false;
void* hx = strata::gpu::host_view((void*) x);
void* ho = strata::gpu::host_view(out);
void* hi = idx ? strata::gpu::host_view((void*) idx) : nullptr;
if (!hx || !ho) return false;
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bx = wrap(hx, ((size_t) (nrows - 1) * (size_t) ldx + (size_t) ncols) * 4);
mtl::Buffer bo = wrap(ho, (size_t) nrows * (size_t) ncols * 2);
mtl::Buffer bi = hi ? wrap(hi, (size_t) nrows * 4) : bx;
if (!bx.handle || !bo.handle || !bi.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_rows_f16"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bi, hi ? off(hi) : off(hx)),
mtl::Arg::V((unsigned) (hi ? 1 : 0)), mtl::Arg::V((unsigned) ncols), mtl::Arg::B(bo, off(ho))};
d.grid[0] = (uint32_t) ((ncols + 255) / 256);
d.grid[1] = (uint32_t) nrows;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
return true;
}
// kind: 0 BF16 values, 8 Q8_0, 13 Q5_K. n rows of k values, ld the row stride (bytes; values for BF16), out half rows.
bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out) {
if (n <= 0 || k <= 0) return true;
if (!ensure()) return false;
const char* fn = kind == 13 ? "ds4_dq_q5k" : kind == 8 ? "ds4_dq_q8" : kind == 0 ? "ds4_dq_bf16" : nullptr;
if (!fn) return false;
if ((kind == 13 && k % 256 != 0) || (kind == 8 && k % 32 != 0)) return false;
void* hw = strata::gpu::host_view((void*) raw);
void* ho = strata::gpu::host_view(out);
if (!hw || !ho) return false;
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
const size_t wbytes = (size_t) (n - 1) * ld * (kind == 0 ? 2 : 1) + (kind == 0 ? (size_t) k * 2 : (kind == 13 ? (size_t) (k / 256) * 176 : (size_t) (k / 32) * 34));
if (!std::getenv("DS4_NO_TOUCH")) { // (the matrix may be file-backed pages not read yet: the GPU does not page them in, it stalls)
volatile unsigned char sink = 0;
for (size_t o = 0; o < wbytes; o += 4096) sink = sink + ((const volatile unsigned char*) hw)[o];
(void) sink;
}
mtl::Buffer bw = wrap(hw, wbytes);
mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) k * 2);
if (!bw.handle || !bo.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }
size_t items = kind == 13 ? (size_t) n * (k / 256) * 8 : kind == 8 ? (size_t) n * (k / 32) : (size_t) n * k;
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::V((unsigned long) ld), mtl::Arg::V((unsigned) k), mtl::Arg::V((unsigned) n),
mtl::Arg::B(bo, off(ho))};
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
d.grid[0] = (uint32_t) ((items + 255) / 256);
d.grid[1] = d.grid[2] = 1;
g_cx->run({d});
return true;
}
// Y[r][c] = alpha * (X16[r] . W16[c]) + beta * Y[r][c]: X16 rows x K (stride ldx halves), W16 N x K contiguous, Y rows x N float
// (stride ldy). The cuBLAS call's operands in row-major terms; the MPP tile kernel wants contiguous operands and result.
bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha,
float beta) {
if (rows <= 0 || N <= 0) return true;
if (!ensure()) return false;
if (g_lib_mm == -2) {
g_lib_mm = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "dense_mm.metal"});
if (g_lib_mm < 0) std::fprintf(stderr, "dsv41 metal: compiling dense_mm.metal failed\n");
}
if (g_lib_mm < 0) return false;
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
void* hw = strata::gpu::host_view((void*) W16);
void* hx = strata::gpu::host_view((void*) X16);
void* hy = strata::gpu::host_view(Y);
if (!hw || !hx || !hy) return false;
mtl::Buffer bw = wrap(hw, (size_t) N * K * 2);
mtl::Buffer bx = wrap(hx, ((size_t) (rows - 1) * (size_t) ldx + (size_t) K) * 2);
mtl::Buffer by = wrap(hy, ((size_t) (rows - 1) * (size_t) ldy + (size_t) N) * 4);
if (!bw.handle || !bx.handle || !by.handle) return false;
// activations contiguous
mtl::Buffer ax = bx;
size_t ax_off = off(hx);
if (ldx != K) {
ax = scratch(g_gm_xs, (size_t) rows * K * 2);
ax_off = 0;
mtl::Pipeline pp;
try { pp = g_cx->pipeline(g_lib, "ds4_pack_h16"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pp;
d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) K), mtl::Arg::B(ax)};
d.grid[0] = (uint32_t) ((K + 255) / 256);
d.grid[1] = (uint32_t) rows;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
}
const bool direct = ldy == N && alpha == 1.0f && beta == 0.0f;
mtl::Buffer ty = by;
size_t ty_off = off(hy);
if (!direct) {
ty = scratch(g_gm_yt, (size_t) rows * N * 4);
ty_off = 0;
}
const int tm = 64, tn = N <= 64 ? 32 : 64;
const std::string fn = "gemm_f16_" + std::to_string(tm) + "_" + std::to_string(tn) + "_4";
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib_mm, fn); } catch (...) { return false; }
struct GemmArgs { int M, N, K, pad; } ga{rows, N, K, 0};
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(ax, ax_off), mtl::Arg::B(bw, off(hw)), mtl::Arg::B(ty, ty_off), mtl::Arg::V(ga)};
d.grid[0] = (uint32_t) ((N + tn - 1) / tn);
d.grid[1] = (uint32_t) ((rows + tm - 1) / tm);
d.grid[2] = 1;
d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
if (!direct) {
mtl::Pipeline pa;
try { pa = g_cx->pipeline(g_lib, "ds4_axpby_rows"); } catch (...) { return false; }
mtl::Dispatch e;
e.p = &pa;
e.args = {mtl::Arg::B(ty), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned long) ldy), mtl::Arg::V((unsigned) N),
mtl::Arg::V(alpha), mtl::Arg::V(beta)};
e.grid[0] = (uint32_t) ((N + 255) / 256);
e.grid[1] = (uint32_t) rows;
e.grid[2] = 1;
e.tg[0] = 256; e.tg[1] = e.tg[2] = 1;
g_cx->run({e});
}
return true;
}
// ---- the prompt chunk's expert GEMMs (ds4_mm.metal): a set = n_exp consecutive expert slots, their rows sorted by expert ----
namespace {
int g_lib_moe = -2;
mtl::Buffer g_mm_xs, g_mm_hs, g_mm_ys; // grow-only scratch: the sorted rows' half activations; the gate/up's swiglu rows (the down's input)
bool g_mm_hs_ok = false;
// bytes of a 256-value block per type: the tile kernels are instantiated for exactly these (DS4_MM_INST)
size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : type == 29 ? 56 : 0; }
struct DsMmArgsH {
unsigned long gate_off, up_off, down_off, stride;
unsigned gu_row, d_row, n_embd, n_ff;
float limit;
unsigned pad0;
};
bool mm_prepare() {
if (!ensure()) return false;
if (g_lib_moe == -2) {
g_lib_moe = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "moe_mm.metal", "../src/engine/msl/ds4_mm.metal"});
if (g_lib_moe < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_mm.metal failed\n");
}
return g_lib_moe >= 0;
}
} // namespace
static bool mm_set(bool gate_up, const uint8_t* wbase, size_t stride, const size_t* tr, const int* hb, int n_exp, int total_rows,
int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, const uint16_t* hin, uint16_t* hout,
int ldh, float* yout, int ldy, int tg, int td) {
const int type = gate_up ? tg : td;
if (n_exp <= 0 || total_rows <= 0) return true;
if (iq_bs(tg) == 0 || iq_bs(td) == 0 || n_embd % 256 != 0 || n_ff % 256 != 0) return false;
if (ldh != n_ff || (!gate_up && ldy != n_embd)) return false;
if (gate_up ? (n_ff % 128 != 0) : (n_embd % 128 != 0)) return false;
if (!mm_prepare()) return false;
constexpr int TM = 32; // (the tile kernels' TM: ds4_mm.metal DS4_MM_INST)
constexpr int SG = 4, TN = 32; // the tile kernels' simdgroups and columns each:
constexpr int COLS = SG * TN; // ds4_mm.metal DS4_MM_INST must match these
// the tile list: (slot, first row, valid rows) a TM-row piece of one expert
std::vector<int> tiles;
for (int e = 0; e < n_exp; ++e) {
const int lo = hb[e] - hb[0], hi = hb[e + 1] - hb[0];
for (int r = lo; r < hi; r += TM) { tiles.push_back(e); tiles.push_back(r); tiles.push_back(std::min(TM, hi - r)); tiles.push_back(0); }
}
const int ntiles = (int) (tiles.size() / 4);
if (ntiles == 0) return true;
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
const size_t len = ((end - base + page - 1) / page) * page;
if (std::getenv("DS4_MM_DBG")) { size_t o2 = 0; std::fprintf(stderr, "mm wrap %p len %zu own=%d rem=%zu\n", (void*) base, len, ds4_gpu_buffer_for_cpu((void*) base, len, &o2) != nullptr, ds4_gpu_alloc_remaining((void*) base)); }
return g_cx->wrap((void*) base, len);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
void* hw = strata::gpu::host_view((void*) wbase);
if (!hw) return false;
mtl::Buffer bw = wrap(hw, (size_t) n_exp * stride);
mtl::Buffer bt = g_cx->alloc(tiles.size() * 4);
std::memcpy(bt.ptr, tiles.data(), tiles.size() * 4);
DsMmArgsH a{tr[0], tr[1], tr[2], (unsigned long) stride, (unsigned) ((size_t) n_embd / 256 * iq_bs(tg)), (unsigned) ((size_t) n_ff / 256 * iq_bs(td)),
(unsigned) n_embd, (unsigned) n_ff, limit, std::getenv("DS4_MM_ABL") ? (unsigned) std::atoi(std::getenv("DS4_MM_ABL")) : 0u};
if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "mm %s embd %d ff %d tg %d td %d lim %g stride %zu tr %zu %zu %zu gu_row %u d_row %u\n", gate_up ? "gu" : "dn", n_embd, n_ff, tg, td, limit, stride, tr[0], tr[1], tr[2], a.gu_row, a.d_row);
mtl::Pipeline pl;
// DS4_MM_CTAB: QUANT_CTAB (function constant 23) - the i-quant grids and the sign table read from constant
// memory instead of a threadgroup copy. Same values; the decode's two lookups per 8 weights go through the
// constant cache instead of threadgroup memory.
static const bool mm_ctab = std::getenv("DS4_MM_CTAB") != nullptr;
try {
const std::string fn = std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type);
if (mm_ctab) {
mtl::Constants cs;
cs.b(23, true);
pl = g_cx->pipeline_uncached(g_lib_moe, fn, cs);
} else {
pl = g_cx->pipeline(g_lib_moe, fn);
}
} catch (...) { g_cx->free(bt); return false; }
if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "pipe %s max_threads %d tgmem %d width %d\n", pl.name.c_str(), pl.max_threads, pl.static_tg_mem, pl.simd_width);
mtl::Dispatch d;
d.p = &pl;
d.tg[0] = SG * 32; d.tg[1] = d.tg[2] = 1;
d.grid[2] = 1;
d.grid[1] = (uint32_t) ntiles;
if (gate_up) {
// the sorted rows' activations, half
const size_t xs_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 2;
if (!g_mm_xs.handle || g_mm_xs.bytes < xs_bytes) {
if (g_mm_xs.handle) g_cx->free(g_mm_xs);
g_mm_xs = g_cx->alloc(xs_bytes + (xs_bytes >> 2));
}
void* hx = strata::gpu::host_view((void*) xf);
void* hi = strata::gpu::host_view((void*) idx);
void* ho = strata::gpu::host_view(hout);
if (!hx || !hi || !ho) { g_cx->free(bt); return false; }
// (the float rows: the whole token slab the engine's c.x holds; its extent is the allocation's)
const size_t xrem = ds4_gpu_alloc_remaining(hx);
mtl::Buffer bxf = wrap(hx, xrem > 0 ? xrem : (size_t) ldx * 4);
mtl::Buffer bix = wrap(hi, (size_t) total_rows * 4);
mtl::Pipeline pg;
try { pg = g_cx->pipeline(g_lib_moe, "ds4_mm_gather"); } catch (...) { g_cx->free(bt); return false; }
mtl::Dispatch dg;
dg.p = &pg;
dg.args = {mtl::Arg::B(bxf, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bix, off(hi)), mtl::Arg::V((unsigned) n_embd), mtl::Arg::B(g_mm_xs)};
dg.grid[0] = (uint32_t) ((n_embd + 255) / 256);
dg.grid[1] = (uint32_t) total_rows;
dg.grid[2] = 1;
dg.tg[0] = 256; dg.tg[1] = dg.tg[2] = 1;
g_cx->run({dg});
const size_t hs_bytes = ((size_t) total_rows + TM) * (size_t) n_ff * 2;
if (!g_mm_hs.handle || g_mm_hs.bytes < hs_bytes) {
if (g_mm_hs.handle) g_cx->free(g_mm_hs);
g_mm_hs = g_cx->alloc(std::max<size_t>(hs_bytes + (hs_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
}
g_mm_hs_ok = true;
{ // (the down's result rows are made here too)
const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;
if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {
if (g_mm_ys.handle) g_cx->free(g_mm_ys);
g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
}
}
(void) ho;
d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_xs), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};
d.grid[0] = (uint32_t) (n_ff / COLS);
} else {
void* hh = strata::gpu::host_view((void*) hin);
void* hy = strata::gpu::host_view(yout);
void* hy_real = nullptr;
if (!hh || !hy) { g_cx->free(bt); return false; }
const size_t hrem = ds4_gpu_alloc_remaining(hh);
if (!g_mm_hs_ok) { g_cx->free(bt); return false; }
g_mm_hs_ok = false;
(void) hrem;
mtl::Buffer& bh = g_mm_hs;
const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;
if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {
if (g_mm_ys.handle) g_cx->free(g_mm_ys);
g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
}
mtl::Buffer by = g_mm_ys;
hy_real = hy;
hy = by.ptr;
if (std::getenv("DS4_MM_YALT")) { hy = hh; by = wrap(hh, (size_t) total_rows * (size_t) n_ff * 2); }
if (std::getenv("DS4_MM_EXP1")) { by = g_cx->alloc((size_t) total_rows * (size_t) n_embd * 4 + 4096); hy = by.ptr; }
d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_ys), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};
d.grid[0] = (uint32_t) (n_embd / COLS);
}
if (!gate_up && std::getenv("DS4_MM_ASGU")) { // test: the gate/up kernel on the down's buffers
static mtl::Pipeline pgu;
pgu = g_cx->pipeline(g_lib_moe, "ds4_mm_gu_" + std::to_string(tg));
d.p = &pgu;
d.args[1] = mtl::Arg::B(g_mm_xs);
d.args[2] = mtl::Arg::B(g_mm_ys);
if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / COLS);
}
if (!gate_up && std::getenv("DS4_MM_DROP")) {
const char* dk = std::getenv("DS4_MM_DROP");
static mtl::Buffer dummy;
if (!dummy.handle) dummy = g_cx->alloc(1 << 26);
static mtl::Buffer dummy2;
if (!dummy2.handle) dummy2 = g_cx->alloc(1 << 26);
for (int k = 0; k < 4; ++k) if (std::strchr(dk, '0' + k)) d.args[k] = mtl::Arg::B(std::getenv("DS4_MM_D2") && k == std::atoi(std::getenv("DS4_MM_D2")) ? dummy2 : dummy);
}
if (!gate_up && std::getenv("DS4_MM_ONE")) { d.grid[1] = 1; if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "dn tile0 %d %d %d ntiles %d x %u rows %d\n", tiles[0], tiles[1], tiles[2], ntiles, d.grid[0], total_rows); }
if (std::getenv("DS4_MM_DBG")) for (size_t q = 0; q < d.args.size(); ++q) if (d.args[q].buf) std::fprintf(stderr, "arg %s %zu bytes %zu bias %zu off %zu ptr %p\n", gate_up ? "gu" : "dn", q, d.args[q].buf->bytes, (size_t) d.args[q].buf->bias, (size_t) d.args[q].off, d.args[q].buf->ptr);
if (std::getenv(gate_up ? "DS4_MM_NOGU" : "DS4_MM_NODN")) { g_cx->free(bt); return true; }
g_cx->run({d});
if (!std::getenv("DS4_MM_LEAK")) g_cx->free(bt);
if (!gate_up) { // the result into the engine's rows
void* hyr = strata::gpu::host_view(yout);
const size_t n4 = (size_t) total_rows * (size_t) n_embd;
const size_t base = (size_t) ((uintptr_t) hyr) & ~(size_t) (page - 1);
mtl::Buffer bd = g_cx->wrap((void*) base, wrap_len(base, (size_t) (uintptr_t) hyr + n4 * 4 - base));
mtl::Pipeline pc;
try { pc = g_cx->pipeline(g_lib, "ds4_copy4"); } catch (...) { return false; }
mtl::Dispatch c;
c.p = &pc;
c.args = {mtl::Arg::B(g_mm_ys), mtl::Arg::B(bd, (size_t) ((uintptr_t) hyr - base)), mtl::Arg::V((unsigned long) n4)};
c.tg[0] = 256; c.tg[1] = c.tg[2] = 1;
c.grid[0] = (uint32_t) ((n4 + 255) / 256);
c.grid[1] = c.grid[2] = 1;
g_cx->run({c});
}
return true;
}
bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,
float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td) {
return mm_set(true, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, xf, ldx, idx, nullptr, h, ldh, nullptr, 0, tg, td);
}
bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,
float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td) {
return mm_set(false, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, nullptr, 0, nullptr, h16, nullptr, ldh, y, ldy, tg, td);
}
// ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------
// strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched
// command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).
// DS4_COPY_LOG: how much host time the copies take (they wrap host memory into Metal buffers per call)
namespace { struct CopyLog { unsigned long long n = 0, bytes = 0, us = 0;
~CopyLog() { if (n) std::fprintf(stderr, "copy log: %llu copies, %.1f MB, %.1f ms host time (%.1f us a copy)\n",
n, bytes / 1e6, us / 1e3, n ? (double) us / (double) n : 0.0); } };
CopyLog g_copy_us; }
static inline unsigned long long copy_now_us() {
return (unsigned long long) std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) {
if (n == 0) return true;
const bool copy_log = std::getenv("DS4_COPY_LOG") != nullptr;
const unsigned long long copy_t0 = copy_log ? copy_now_us() : 0;
struct CopyDone { bool on; unsigned long long t0; size_t n; ~CopyDone() { if (on) { g_copy_us.n++; g_copy_us.bytes += n; g_copy_us.us += copy_now_us() - t0; } } } copy_done{copy_log, copy_t0, n};
if (!ensure()) return false;
void* hd = strata::gpu::host_view(dst);
void* hs = strata::gpu::host_view((void*) src);
if (!hd || !hs) return false;
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) host) + bytes;
return g_cx->wrap((void*) base, wrap_len(base, end - base));
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer bs = wrap(hs, n);
mtl::Buffer bd = wrap(hd, n);
if (!bs.handle || !bd.handle) return false;
const bool w4 = (((uintptr_t) hd | (uintptr_t) hs | n) & 3u) == 0;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, w4 ? "ds4_copy4" : "ds4_copy1"); } catch (...) { return false; }
const size_t items = w4 ? n / 4 : n;
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bd, off(hd)), mtl::Arg::V((unsigned long) items)};
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
d.grid[0] = (uint32_t) ((items + 255) / 256);
d.grid[1] = d.grid[2] = 1;
g_cx->run({d});
return true;
}
extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) {
if (n == 0) return true;
if (!ensure()) return false;
void* hd = strata::gpu::host_view(dst);
if (!hd) return false;
const size_t page = 16384;
const size_t base = (size_t) ((uintptr_t) hd) & ~(size_t) (page - 1);
const size_t end = (size_t) ((uintptr_t) hd) + n;
mtl::Buffer bd = g_cx->wrap((void*) base, wrap_len(base, end - base));
if (!bd.handle) return false;
mtl::Pipeline pl;
try { pl = g_cx->pipeline(g_lib, "ds4_fill1"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bd, (size_t) ((uintptr_t) hd - base)), mtl::Arg::V((unsigned long) n), mtl::Arg::V((unsigned) (value & 0xff))};
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
d.grid[0] = (uint32_t) ((n + 255) / 256);
d.grid[1] = d.grid[2] = 1;
g_cx->run({d});
return true;
}
extern "C" void* ds4_ctx_mark() {
if (!g_cx) return nullptr;
return g_cx->mark();
}
extern "C" void ds4_ctx_drop(void* token) {
if (g_cx && token) g_cx->drop_mark(token);
}
extern "C" void ds4_ctx_wait(void* token) {
static const bool no_wait_split = std::getenv("DS4_NO_WAIT_SPLIT") != nullptr;
if (g_cx && !no_wait_split) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)
if (g_cx && token) g_cx->wait_mark(token);
}
// commit the batched dispatches without waiting: the GPU starts on them while the host goes on (the CPU lane)
extern "C" void ds4_metal_kick() {
if (g_cx) g_cx->split_runs();
}
extern "C" void ds4_metal_flush() {
// The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default
// stream, its dispatched kernels on mtl_rt's deferred command buffer. Two queues, so committing one alone leaves the
// other's work unordered: a kernel could read a feed whose blit had not run yet (emb_rows is a zeroed buffer, so
// embed wrote exactly 0.0 into h). The stream's queue goes first - it only carries the feeds at a token boundary,
// the readbacks are encoded after this returns.
ds4_gpu_sync_all_streams();
if (g_cx) g_cx->flush_runs();
}
// the one Context every dsv41 kernel dispatches through: two of them means two Metal queues and no ordering between their work
mtl::Context& shared_cx() {
ensure();
return *g_cx;
}
mtl::Context& finish_cx() {
ensure();
return *g_cx;
}
int shared_lib() {
ensure();
return g_lib;
}
} // namespace strata::dsv41::metal_host