Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: drafter path ported, token-boundary commits, and the embedding blocker measured end to end

Niko1221ba9e818c42c6
diff --git a/metal/include/metal_compat/cuda_runtime.h b/metal/include/metal_compat/cuda_runtime.h
index aedad00a5..62cb432ff 100644
--- a/metal/include/metal_compat/cuda_runtime.h
+++ b/metal/include/metal_compat/cuda_runtime.h
@@ -135,6 +135,10 @@ inline cudaError_t cudaDeviceEnablePeerAccess(int p, unsigned f) { return strata
inline cudaError_t cudaDeviceDisablePeerAccess(int) { return cudaSuccess; } // a Mac has one GPU: nothing to disable
// ---- memory and copies
+// the deferred command buffer mtl_rt batches the port's dispatches into: a copy (or any host read of a device result)
+// has to commit it first, or it reads memory the GPU has not written yet. Defined in ds4_finish.cpp.
+extern "C" void ds4_metal_flush();
+
template <class T> inline cudaError_t cudaMalloc(T** p, size_t n) { return strata::gpu::malloc_device(p, n); }
inline cudaError_t cudaFree(void* p) { return strata::gpu::free_device(p); }
template <class T> inline cudaError_t cudaHostAlloc(T** p, size_t n, unsigned f) { return strata::gpu::host_alloc(p, n, f); }
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index 8c42502c2..1d5fa620c 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -56,6 +56,7 @@ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long l
bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);
bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);
bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);
+bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out);
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);
bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);
@@ -78,10 +79,13 @@ bool iota_rows(int* out, int T, int n, int stride);
bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int type, int groups);
bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int type, int groups);
bool iq_mv(const void* jobs, int nj);
+bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R);
bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);
bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);
bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride, const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd);
+bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int groups);
+bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int groups);
bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);
bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);
bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 2ca4f6700..d14bf1748 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -641,6 +641,106 @@ bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr
}
+// ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel - the MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down /
+// win_experts). The same frame as iq_gu / iq_dn above: a threadgroup of 256 is 8 warps, a warp a row, a lane strides the
+// 32-value blocks, the group's entries sixteen at a pass. The job struct is the kernel's own (metal/src/engine/msl/
+// ds4_iq.metal's Ds4MxJob); the plan / window pointer arrays travel as they are: every pointer here is a CPU address
+// (unified memory) and a kernel may take raw gpuAddresses.
+namespace {
+// (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4MxJob (Metal
+// has no 'long long': 64-bit is 'long')
+struct MxJob {
+ const void* plan_ptr;
+ const void* grp_ptr;
+ const void* grp_start;
+ const void* grp_ent;
+ const void* n_groups;
+ const void* xq;
+ const void* xd;
+ const void* hq;
+ const void* hd;
+ void* h;
+ void* part;
+ int64_t tr0, tr1, tr2; // ('long' there, so 8 a field and the float ends the struct on the same byte)
+ int n_embd, n_ff;
+ float limit;
+};
+static_assert(sizeof(MxJob) == 128, "Ds4MxJob's layout");
+
+static bool mx_gemm(const MxJob& j, const char* fn, int grid_x, int groups) {
+ if (!ensure()) return false;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) grid_x;
+ d.grid[1] = (uint32_t) groups;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+} // namespace
+
+// gate / up of the group's experts: h = the clamped swiglu (ds4_gguf.cu's MxArgs: tr[0] gate, tr[1] up, tr[2] down)
+bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,
+ const int8_t* xq, const float* xd, float* h, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 32 != 0 || n_ff % 8 != 0) return false;
+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ MxJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.xq = xq;
+ j.xd = xd;
+ j.h = h;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ j.limit = limit;
+ return mx_gemm(j, "ds4_mx_gu", (n_ff + 7) / 8, groups);
+}
+
+// down of the group's experts against their int8 h: part (a row of n_embd an entry)
+bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,
+ const float* hd, float* part, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 32 != 0 || n_embd % 8 != 0) return false;
+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ MxJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.hq = hq;
+ j.hd = hd;
+ j.part = part;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ return mx_gemm(j, "ds4_mx_dn", (n_embd + 7) / 8, groups);
+}
+
// ds4_gguf.cu:577 mx_combine_kernel - a token's expert outputs summed in plan order (the experts off the device add 0),
// the CPU lane's part when its flag is set, then the shared expert's: ds4_gguf.cu:169's combine(), 128 threads a block
namespace {
@@ -1377,6 +1477,92 @@ bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_s
}
// ds4_kernels.cu:1150 iota_rows_kernel - out[t][i] = i
+// ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel - the drafter's head. markov_add is a warp a vocabulary
+// row (the previous token's E row staged in the threadgroup, then E . H[v] added to logits[v]); confidence a threadgroup
+// a token. Both jobs are the kernels' own structs (metal/src/engine/msl/ds4_init.metal's Ds4MarkovJob / Ds4ConfJob).
+namespace {
+// (not an entry point: the generator takes only the ^bool lines) mirrors ds4_init.metal's Ds4MarkovJob / Ds4ConfJob
+struct MarkovJob {
+ void* logits;
+ const void* E;
+ const void* H;
+ const void* tok;
+ int V, R;
+};
+static_assert(sizeof(MarkovJob) == 40, "Ds4MarkovJob's layout");
+
+struct ConfJob {
+ const void* x;
+ const void* E;
+ const void* tok;
+ const void* w;
+ void* out;
+ int dim, R;
+};
+static_assert(sizeof(ConfJob) == 48, "Ds4ConfJob's layout");
+} // namespace
+
+bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R) {
+ if (logits == nullptr || E == nullptr || H == nullptr || tok == nullptr) return false;
+ if (V <= 0 || R <= 0 || R > 1024) return false; // (the kernel's staged E row holds 1024 floats)
+ if (!ensure()) return false;
+ MarkovJob j{};
+ j.logits = const_cast<float*>(logits);
+ j.E = E;
+ j.H = H;
+ j.tok = tok;
+ j.V = V;
+ j.R = R;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: markov_add job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_markov_add"); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((V + 7) / 8);
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+
+bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out) {
+ if (x == nullptr || E == nullptr || tok == nullptr || w == nullptr || out == nullptr) return false;
+ if (T <= 0 || dim <= 0 || R <= 0) return false;
+ if (!ensure()) return false;
+ ConfJob j{};
+ j.x = x;
+ j.E = E;
+ j.tok = tok;
+ j.w = w;
+ j.out = out;
+ j.dim = dim;
+ j.R = R;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: confidence job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_confidence"); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+
bool iota_rows(int* out, int T, int n, int stride) {
if (T <= 0 || n <= 0) return true;
if (out == nullptr) return false;
@@ -1744,7 +1930,20 @@ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, voi
extern "C" void ds4_metal_flush() {
+ // The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default
+ // stream, its dispatched kernels on mtl_rt's deferred command buffer. Two queues, so committing one alone leaves the
+ // other's work unordered: a kernel could read a feed whose blit had not run yet (emb_rows is a zeroed buffer, so
+ // embed wrote exactly 0.0 into h). The stream's queue goes first - it only carries the feeds at a token boundary,
+ // the readbacks are encoded after this returns.
+ strata::gpu::stream_synchronize(nullptr);
if (g_cx) g_cx->flush_runs();
}
+
+// the one Context every dsv41 kernel dispatches through: two of them means two Metal queues and no ordering between their work
+mtl::Context& shared_cx() {
+ ensure();
+ return *g_cx;
+}
+
} // namespace strata::dsv41::metal_host
\ No newline at end of file
diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cpp
index b3dbdcb0c..a434eef8d 100644
--- a/metal/src/engine/ds4_host.cpp
+++ b/metal/src/engine/ds4_host.cpp
@@ -8,6 +8,8 @@
#include <strata/gpu/runtime.hpp>
+
+
#include <cstdio>
#include <cstdlib>
#include <cstring>
@@ -70,7 +72,7 @@ namespace {
mtl::Context* g_init_cx = nullptr;
int g_lib_init = -1;
-mtl::Context& init_cx() {
+mtl::Context& shared_cx() {
if (!g_init_cx) {
g_init_cx = new mtl::Context();
g_lib_init = g_init_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
@@ -82,19 +84,19 @@ mtl::Context& init_cx() {
// ds4_kernels.cu:1477 - pre[i] = (i % 8) == 0 ? 1 : 0 for i < T*8
bool init_pre(float* pre, int T, void*) {
- init_cx();
+ strata::dsv41::metal_host::shared_cx();
if (g_lib_init < 0) return false;
void* host = strata::gpu::host_view(pre);
if (!host) { std::fprintf(stderr, "dsv41 metal: init_pre has no host view\n"); return false; }
const size_t bytes = (size_t) T * 8 * sizeof(float);
const size_t page = 16384;
- mtl::Buffer bp = init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)),
+ mtl::Buffer bp = strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)),
((bytes + page - 1) / page) * page);
if (!bp.handle) { std::fprintf(stderr, "dsv41 metal: init_pre wrap failed\n"); return false; }
const uintptr_t delta = (uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1));
mtl::Pipeline pl;
try {
- pl = init_cx().pipeline(g_lib_init, "ds4_init_pre");
+ pl = strata::dsv41::metal_host::shared_cx().pipeline(g_lib_init, "ds4_init_pre");
} catch (...) {
return false;
}
@@ -105,21 +107,21 @@ bool init_pre(float* pre, int T, void*) {
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 128;
d.tg[1] = d.tg[2] = 1;
- init_cx().run({d});
+ strata::dsv41::metal_host::shared_cx().run({d});
return true;
}
// ds4_kernels.cu:130 - the BF16 embedding gather, hc copies a token
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
- init_cx();
+ strata::dsv41::metal_host::shared_cx();
if (g_lib_init < 0) { std::fprintf(stderr, "dsv41 metal: embed: no library\n"); return false; }
void* hrows = strata::gpu::host_view((void*) rows);
void* hh = strata::gpu::host_view(h);
if (!hrows || !hh) { std::fprintf(stderr, "dsv41 metal: embed has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
- return init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
};
mtl::Buffer br = wrap(hrows, (size_t) T * dim * 2);
mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4);
@@ -127,7 +129,7 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
const size_t dro = (size_t) (((uintptr_t) hrows) - (((uintptr_t) hrows) & ~(uintptr_t) (page - 1)));
const size_t dho = (size_t) (((uintptr_t) hh) - (((uintptr_t) hh) & ~(uintptr_t) (page - 1)));
mtl::Pipeline pl;
- try { pl = init_cx().pipeline(g_lib_init, "ds4_embed"); } catch (...) { return false; }
+ try { pl = strata::dsv41::metal_host::shared_cx().pipeline(g_lib_init, "ds4_embed"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(br, dro), mtl::Arg::B(bh, dho), mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
@@ -135,14 +137,14 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
- init_cx().run({d});
+ strata::dsv41::metal_host::shared_cx().run({d});
return true;
}
// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {
- init_cx();
+ strata::dsv41::metal_host::shared_cx();
if (g_lib_init < 0) return false;
void* ha = strata::gpu::host_view((void*) A);
void* hb = strata::gpu::host_view((void*) B);
@@ -150,7 +152,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
if (!ha || !hb || !hc) { std::fprintf(stderr, "dsv41 metal: sgemm_tn has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
- return init_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer ba = wrap(ha, (size_t) m * lda * 4);
@@ -158,7 +160,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
mtl::Buffer bc = wrap(hc, (size_t) nt * ldc * 4);
if (!ba.handle || !bb.handle || !bc.handle) return false;
mtl::Pipeline pl;
- try { pl = init_cx().pipeline(g_lib_init, "ds4_sgemm_tn"); } catch (...) { return false; }
+ try { pl = strata::dsv41::metal_host::shared_cx().pipeline(g_lib_init, "ds4_sgemm_tn"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bc, off(hc)),
@@ -168,7 +170,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
d.grid[0] = (uint32_t) ((total + 127) / 128);
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
- init_cx().run({d});
+ strata::dsv41::metal_host::shared_cx().run({d});
return true;
}
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index c12ea9108..710ece886 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -1932,3 +1932,61 @@ kernel void ds4_combine(device const float* ed [[buffer(0)]], device const float
if (has_sh != 0u) a += sh[(size_t) t * dim + e];
out[(size_t) t * dim + e] = a;
}
+
+// ---------------------------------------------------------------- ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel
+//
+// The drafter's head (DSpark's Markov chain and its confidence): markov_add is a warp a vocabulary row - the previous
+// token's E row (BF16) staged as floats in the threadgroup, then E . H[v] added to logits[v] (H is F16, the pack's
+// markov_head.head.weight); confidence is a threadgroup a token over x and the same E row against the head's weights,
+// its block sum through the logistic.
+struct Ds4MarkovJob {
+ device float* logits; // [V]: += the bias
+ device const ushort* E; // [vocab][R] BF16
+ device const ushort* H; // [vocab][R] F16
+ device const int* tok; // the previous token (one int)
+ int V, R;
+};
+
+struct Ds4ConfJob {
+ device const float* x; // [T][dim]
+ device const ushort* E; // [vocab][R] BF16
+ device const int* tok; // [T]
+ device const float* w; // [dim + R]
+ device float* out; // [T]
+ int dim, R;
+};
+
+// a BF16 bit pattern as its float (ds4_kernels.cu:28 bf16f: the top half, no rounding)
+static inline float ds4_bf16f(ushort b) { return as_type<float>(uint(b) << 16u); }
+
+// a warp a vocabulary row: logits[v] += E[*tok] . H[v]
+kernel void ds4_markov_add(constant Ds4MarkovJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ threadgroup float er[1024]; // (R <= 1024: the host wrapper refuses more)
+ const device ushort* erow = j.E + (size_t) (*j.tok) * (size_t) j.R;
+ for (int i = int(tid); i < j.R; i += 256) er[i] = ds4_bf16f(erow[i]);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int v = int(tg) * 8 + int(tid >> 5);
+ if (v >= j.V) return;
+ const device ushort* hr = j.H + (size_t) v * (size_t) j.R;
+ float a = 0.0f;
+ for (int c = int(lane) * 8; c < j.R; c += 256) {
+ for (int i = 0; i < 8; ++i) a += float(as_type<half>(hr[c + i])) * er[c + i];
+ }
+ a = ds4_warp_sum(a);
+ if (lane == 0u) j.logits[v] += a;
+}
+
+// a threadgroup a token: the logistic of the head's sum over x and the token's E row
+kernel void ds4_confidence(constant Ds4ConfJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const int t = int(tg);
+ const device float* xt = j.x + (size_t) t * (size_t) j.dim;
+ const device ushort* er = j.E + (size_t) j.tok[t] * (size_t) j.R;
+ float a = 0.0f;
+ for (int i = int(tid); i < j.dim + j.R; i += 1024) a += j.w[i] * (i < j.dim ? xt[i] : ds4_bf16f(er[i - j.dim]));
+ a = ds4_block_sum_n(a, red, lane, w, 32u);
+ if (tid == 0u) j.out[t] = 1.0f / (1.0f + exp(-a));
+}
diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metal
index c89e9a8e6..f76d59f9f 100644
--- a/metal/src/engine/msl/ds4_iq.metal
+++ b/metal/src/engine/msl/ds4_iq.metal
@@ -459,3 +459,174 @@ kernel void ds4_iq_dn(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgrou
}
}
}
+
+// ---------------------------------------------------------------- ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel
+//
+// The MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down / win_experts): the same frame as ds4_iq_gu / ds4_iq_dn
+// above - a threadgroup of 256 is 8 warps, a warp a row of the gate and of the up (down: a row of the down), a lane
+// strides the 32-value blocks of the row, and the group's entries go sixteen at a pass so that a block the weights decode
+// once serves them all. A block is 17 bytes (kMxBytes): the E8M0 scale byte, then 16 bytes whose low nibble is value j and
+// whose high nibble is value j + 16, each an index into the codebook {0, 1, 2, 3, 4, 6, 8, 12} and its negatives - the same
+// decode as ds4_init.metal's ds4_mx_block / ds4_e8m0 (ds4_gguf.cu:143 / :154), kept as int8 codes here because the
+// activations are the quantiser's int8, so the dot is integer (ds4_gguf.cu:225 mx_dot) and the two scales fold in at the
+// end.
+//
+// The group's blob and its entries are read as ds4_iq_group above reads them (ds4_gguf.cu:429 mx_group): a plan group
+// (grp_ptr null) is one expert whose entry is the group's own index and whose activation row is 0, a window group's
+// entries come from grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.
+struct Ds4MxJob {
+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)
+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob
+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)
+ device const int* grp_ent; // the entries (token * 8 + pick)
+ device const int* n_groups;
+ device const char* xq; // gate / up's int8 activations (n_embd an entry)
+ device const float* xd; // their scales (n_embd / 32 a row)
+ device const char* hq; // down's int8 input (n_ff an entry)
+ device const float* hd; // its scales
+ device float* h; // [entries][n_ff]: the clamped swiglu
+ device float* part; // [entries][n_embd]: each entry's down output
+ long tr0, tr1, tr2; // the roles' offsets in a blob
+ int n_embd, n_ff;
+ float limit; // swiglu_limit (0: none)
+};
+
+// the codebook of a nibble, in index order (ds4_init.metal's lut_init: {0,1,2,3,4,6,8,12}, then its negatives)
+constant char ds4_mx_code[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+
+// 2^(e - 128) (ds4_init.metal:489 ds4_e8m0, ggml's GGML_E8M0_TO_FP32_HALF)
+static inline float ds4_mx_e8m0(uint e) {
+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));
+}
+
+// a block's 16 code bytes as 8 int8 words: word j the low nibbles of bytes 4 j .. (values 4 j ..), word 4 + j their high
+// nibbles (values 16 + 4 j ..) - the pairs ds4_gguf.cu:225 mx_dot reads xp[j] and xp[j + 4] with
+static inline void ds4_mx_words(device const uchar* p, thread int* q) {
+ for (uint j = 0; j < 4u; ++j) {
+ uint lo = 0u, hi = 0u;
+ for (uint b = 0; b < 4u; ++b) {
+ const uint v = uint(p[1u + 4u * j + b]);
+ lo |= uint(uchar(ds4_mx_code[v & 15u])) << (8u * b);
+ hi |= uint(uchar(ds4_mx_code[v >> 4u])) << (8u * b);
+ }
+ q[j] = int(lo);
+ q[4u + j] = int(hi);
+ }
+}
+
+// a block's 32 int8 codes against its 32 int8 activations (8 packed words, 4 values a word)
+static inline int ds4_mx_dot(thread const int* q, device const int* xp) {
+ int s = 0;
+ for (uint t = 0; t < 8u; ++t) s += ds4_iq_dot4(q[t], xp[t]);
+ return s;
+}
+
+// the group's blob and its entry range (ds4_iq_group's MX form, ds4_gguf.cu:429 mx_group); false: nothing to do
+static inline bool ds4_mx_group(constant Ds4MxJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {
+ if (j.grp_ptr == nullptr) {
+ const ulong p = j.plan_ptr[g];
+ if (p == 0ul) return false;
+ blob = p;
+ m0 = int(g);
+ m1 = m0 + 1;
+ return true;
+ }
+ if (int(g) >= *j.n_groups) return false;
+ blob = j.grp_ptr[g];
+ m0 = j.grp_start[g];
+ m1 = j.grp_start[g + 1];
+ return true;
+}
+
+// mx_gate_up_kernel<NE> (a warp a row of the gate and of the up, a lane a block a step): h = the clamped swiglu
+kernel void ds4_mx_gu(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_mx_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ if (row >= j.n_ff) return;
+ const uint nblk = uint(j.n_embd) / 32u;
+ const size_t rb = (size_t) nblk * 17u;
+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;
+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 16) {
+ const int ne = min(16, m1 - mm);
+ int es[16], xr[16];
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ es[i] = e;
+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);
+ }
+ }
+ float ag[16], au[16];
+ for (int i = 0; i < 16; ++i) {
+ ag[i] = 0.0f;
+ au[i] = 0.0f;
+ }
+ for (uint b = uint(lane); b < nblk; b += 32u) {
+ const device uchar* pg = g + (size_t) b * 17u;
+ const device uchar* pu = u + (size_t) b * 17u;
+ int wg[8], wu[8];
+ ds4_mx_words(pg, wg);
+ ds4_mx_words(pu, wu);
+ const float dg = ds4_mx_e8m0(uint(pg[0])), du = ds4_mx_e8m0(uint(pu[0]));
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const device int* xp =
+ (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd + (size_t) b * 32u);
+ const float dx = j.xd[(size_t) xr[i] * nblk + b];
+ ag[i] = fma(dg * dx, float(ds4_mx_dot(wg, xp)), ag[i]);
+ au[i] = fma(du * dx, float(ds4_mx_dot(wu, xp)), au[i]);
+ }
+ }
+ }
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);
+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);
+ }
+ }
+ }
+}
+
+// mx_down_kernel<NE> (a warp a row of the down, a lane a block a step): part = the down against the int8 h
+kernel void ds4_mx_dn(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_mx_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ if (row >= j.n_embd) return;
+ const uint nblk = uint(j.n_ff) / 32u;
+ const size_t rb = (size_t) nblk * 17u;
+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 16) {
+ const int ne = min(16, m1 - mm);
+ int es[16];
+ for (int i = 0; i < 16; ++i)
+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ float ad[16];
+ for (int i = 0; i < 16; ++i) ad[i] = 0.0f;
+ for (uint b = uint(lane); b < nblk; b += 32u) {
+ const device uchar* pw = w + (size_t) b * 17u;
+ int ww[8];
+ ds4_mx_words(pw, ww);
+ const float dd = ds4_mx_e8m0(uint(pw[0]));
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const device int* hp =
+ (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff + (size_t) b * 32u);
+ ad[i] = fma(dd * j.hd[(size_t) es[i] * nblk + b], float(ds4_mx_dot(ww, hp)), ad[i]);
+ }
+ }
+ }
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const float v = ds4_iq_lsum<1>(ad[i]);
+ if (lane == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;
+ }
+ }
+ }
+}
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 8f3a141cb..892f6f9d8 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1250,6 +1250,14 @@ void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s)
m.xq = sc->xq;
m.xd = sc->xd;
q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);
+#if defined(STRATA_METAL)
+ // mx_gate_up_kernel on the Metal dispatch (the MXFP4 expert GEMM); the launch below is the fallback
+ if (::strata::dsv41::metal_host::mx_gu(a.plan_ptr, nullptr, nullptr, nullptr, nullptr, a.lay.trellis, a.n_embd,
+ a.n_ff, a.limit, sc->xq, sc->xd, h, a.k)) {
+ check("moe_gate_up");
+ return;
+ }
+#endif
launch_groups<GateUpK>(m, a.n_ff, a.k, 1, s);
check("moe_gate_up");
}
@@ -1274,7 +1282,12 @@ void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float*
ia.hd = sc->hd;
iq::down(ia, a.k, 1, s);
} else {
- launch_groups<DownK>(m, a.n_embd, a.k, 1, s);
+#if defined(STRATA_METAL)
+ // mx_down_kernel on the Metal dispatch; the launch below is the fallback
+ if (!::strata::dsv41::metal_host::mx_dn(a.plan_ptr, nullptr, nullptr, nullptr, nullptr, a.lay.trellis, a.n_embd,
+ a.n_ff, sc->hq, sc->hd, sc->part, a.k))
+#endif
+ launch_groups<DownK>(m, a.n_embd, a.k, 1, s);
}
CombArgs c{};
c.plan_ptr = a.plan_ptr;
@@ -1334,11 +1347,27 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {
ia.hd = sc->hd;
}
q8_quant(a.x, a.x_stride, a.T, a.n_embd, sc->xq, sc->xd, s);
- if (mx) launch_groups<GateUpK>(m, a.n_ff, groups, a.T, s);
- else iq::gate_up(ia, groups, a.T, s);
+ if (mx) {
+#if defined(STRATA_METAL)
+ // mx_gate_up_kernel on the Metal dispatch; the launch below is the fallback
+ if (!::strata::dsv41::metal_host::mx_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups,
+ a.lay.trellis, a.n_embd, a.n_ff, a.limit, sc->xq, sc->xd, a.h, groups))
+#endif
+ launch_groups<GateUpK>(m, a.n_ff, groups, a.T, s);
+ } else {
+ iq::gate_up(ia, groups, a.T, s);
+ }
q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)
- if (mx) launch_groups<DownK>(m, a.n_embd, groups, a.T, s);
- else iq::down(ia, groups, a.T, s);
+ if (mx) {
+#if defined(STRATA_METAL)
+ // mx_down_kernel on the Metal dispatch; the launch below is the fallback
+ if (!::strata::dsv41::metal_host::mx_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups,
+ a.lay.trellis, a.n_embd, a.n_ff, sc->hq, sc->hd, a.part, groups))
+#endif
+ launch_groups<DownK>(m, a.n_embd, groups, a.T, s);
+ } else {
+ iq::down(ia, groups, a.T, s);
+ }
#if !defined(STRATA_METAL)
// (this port's dispatches are synchronous and nothing is published to wait for: ds4_kernels.cu's wait_seq)
if (a.wait_ptr != nullptr) strata_ds4_launch(mx_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index c2a86dbb8..c05b0f736 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -2213,6 +2213,10 @@ void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) {
strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride);
}
void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // markov_kernel on the Metal dispatch (the drafter's Markov bias); the launch below is the fallback
+ if (strata::dsv41::metal_host::markov_add(logits, E, H, tok, V, R)) return;
+#endif
strata_ds4_launch(markov_kernel, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, H, tok, V, R);
}
void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {
@@ -2227,6 +2231,10 @@ void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const ui
}
void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,
cudaStream_t s) {
+#if defined(STRATA_METAL)
+ // confidence_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::confidence(x, E, tok, w, T, dim, R, out)) return;
+#endif
strata_ds4_launch(confidence_kernel, T, 1024, 0, s, x, E, tok, w, dim, R, out);
}
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 154a8af48..5b0594b70 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1141,6 +1141,9 @@ struct Model::Impl {
void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {
std::vector<float> v((size_t) T * g.hc * g.dim);
cudaSetDevice(d.id);
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)
+#endif
cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
cudaStreamSynchronize(d.st);
std::lock_guard<std::mutex> lk(dump_mu);
@@ -2336,6 +2339,12 @@ struct Model::Impl {
err = "embed.weight: read failed";
return false;
}
+ if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,
+ "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",
+ (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],
+ (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],
+ (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);
+
}
// the host side of the CPU lane and the Engram rows
const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
@@ -5781,8 +5790,21 @@ struct Model::Impl {
Dev& d = devs[(size_t) d_dev];
cudaSetDevice(d.id);
for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);
- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));
+ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
embed(d.emb_rows, T, d.h, D, H, d.st);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = d.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float hh[4] = {0, 0, 0, 0};
+ cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);
+ }
init_pre(d.pre, T, d.st);
WinPlan dp; // (window attention alone: the mailbox seqs)
dp.async = !sync_decode && T <= async_max_t;
@@ -5887,6 +5909,9 @@ struct Model::Impl {
confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);
int tok[kWin + 1];
float cf[kWin];
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));
CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
CK(cudaStreamSynchronize(hd.st));
@@ -5931,6 +5956,9 @@ struct Model::Impl {
sp.logit_bias = bias;
strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);
int t = -1;
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
cudaStreamSynchronize(d.st);
return t;
@@ -5963,6 +5991,9 @@ struct Model::Impl {
sp.greedy = true;
sp.temperature = 0.0f;
strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
CK(cudaStreamSynchronize(hd.st));
while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
@@ -5994,6 +6025,9 @@ struct Model::Impl {
sp.seed = seed;
sp.counter = counter;
strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));
CK(cudaStreamSynchronize(hd.st));
while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
@@ -6491,6 +6525,13 @@ struct Model::Impl {
CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
CK(cudaEventRecord(c.emb_ev[slot], s));
embed(c.emb, T, c.h, D, H, s);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = c.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
init_pre(c.pre, T, s);
} else { // or its state after the layer before
const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);
@@ -6648,6 +6689,13 @@ struct Model::Impl {
CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
CK(cudaEventRecord(c.emb_ev[slot], s));
embed(c.emb, T, c.h, D, H, s);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = c.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
init_pre(c.pre, T, s);
} else {
Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];
@@ -7432,8 +7480,15 @@ struct Model::Impl {
{
Dev& d = devs[(size_t) cur];
cudaSetDevice(d.id);
- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));
+ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
embed(d.emb_rows, T, d.h, D, H, d.st);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = d.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
init_pre(d.pre, T, d.st);
}
std::vector<float> dump;
@@ -7533,8 +7588,15 @@ struct Model::Impl {
if (!started) { // the window's input: the embedding rows, or the previous group's hand-off
started = true;
if (il == 0) {
- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
embed(d.emb_rows, T, d.h, D, H, s);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = d.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
init_pre(d.pre, T, s);
} else {
const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];
@@ -7605,6 +7667,23 @@ struct Model::Impl {
const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
const ArGroup& ag = groups[(size_t) L0.grp].ar;
const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ if (std::getenv("DS4_DBG_H") && mi == 0 && l.il < 2) { // DBG
+ std::vector<float> hv((size_t) T * H * D);
+ cudaSetDevice(d.id);
+ ds4_metal_flush();
+ cudaMemcpy(hv.data(), d.h, hv.size() * 4, cudaMemcpyDeviceToHost);
+ int nf = 0, nzv = 0;
+ float mx = 0.0f;
+ for (size_t i = 0; i < hv.size(); ++i) {
+ const float v = hv[i];
+ if (!(v == v)) { ++nf; continue; }
+ if (v != 0.0f) ++nzv;
+ if (std::fabs(v) > mx) mx = std::fabs(v);
+ }
+ std::fprintf(stderr, "DBG h: il=%d p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g\n",
+ l.il, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],
+ hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f);
+ }
prof_mark(d, 0);
// Engram
if (l.eg >= 0) {
@@ -8070,6 +8149,9 @@ struct Model::Impl {
strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,
sp, d.tok, d.st);
int t = -1;
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
cudaStreamSynchronize(d.st);
return t;
@@ -8083,6 +8165,9 @@ struct Model::Impl {
Dev& d = devs[(size_t) head_dev];
cudaSetDevice(d.id);
out.resize((size_t) std::max(1, n_logits) * g.vocab);
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;
}