Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek Metal: chunked prompt path correct (sgemm on the one Context, KEEP dump syncs, debug switches); 2988-token prompt 251-277 tok/s vs 32 in windows

Niko1221700def65a2a4
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 966eb491f..9b29796d1 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -210,7 +210,18 @@ bool hc_post(const float* y, float* h, const float* post, const float* comb, int
d.grid[1] = (uint32_t) T;
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ const bool dbg = std::getenv("DS4_DBG_SG") != nullptr;
+ if (dbg) {
+ g_cx->flush_runs();
+ const float* fy = (const float*) hy; const float* fh = (const float*) hh; const float* fp = (const float*) hpost; const float* fc = (const float*) hcomb;
+ std::fprintf(stderr, "hc_post in: y %g %g h %g %g | %g %g post %g %g %g %g comb %g %g %g %g\n", fy[0], fy[1], fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1], fp[0], fp[1], fp[2], fp[3], fc[0], fc[1], fc[2], fc[3]);
+ }
g_cx->run({d});
+ if (dbg) {
+ g_cx->flush_runs();
+ const float* fh = (const float*) hh;
+ std::fprintf(stderr, "hc_post out: h %g %g | %g %g\n", fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1]);
+ }
g_cx->free(bp);
g_cx->free(bc);
return true;
@@ -243,6 +254,11 @@ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float*
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
+ if (std::getenv("DS4_DBG_SG")) {
+ g_cx->flush_runs();
+ const float* fc = (const float*) hm;
+ std::fprintf(stderr, "hc_mix_scale T %d hcdim %d n_mix %d eps %g -> %g %g %g\n", T, hcdim, n_mix, eps, fc[0], fc[1], fc[2]);
+ }
return true;
}
@@ -319,7 +335,17 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,
d.grid[0] = (uint32_t) T;
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
+ if (std::getenv("DS4_DBG_SG")) {
+ g_cx->flush_runs();
+ const float* fm = (const float*) hm; const float* fs = (const float*) hs; const float* fb = (const float*) hb;
+ std::fprintf(stderr, "hc_split in: mix %g %g %g scale %g %g %g base %g %g\n", fm[0], fm[1], fm[2], fs[0], fs[1], fs[2], fb[0], fb[1]);
+ }
g_cx->run({d});
+ if (std::getenv("DS4_DBG_SG")) {
+ g_cx->flush_runs();
+ const float* fp = (const float*) hp;
+ std::fprintf(stderr, "hc_split out: pre %g %g %g %g\n", fp[0], fp[1], fp[2], fp[3]);
+ }
return true;
}
@@ -2474,6 +2500,10 @@ mtl::Context& shared_cx() {
ensure();
return *g_cx;
}
+mtl::Context& finish_cx() {
+ ensure();
+ return *g_cx;
+}
int shared_lib() {
ensure();
diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cpp
index cc9129cb3..2d4251745 100644
--- a/metal/src/engine/ds4_host.cpp
+++ b/metal/src/engine/ds4_host.cpp
@@ -19,6 +19,7 @@
#include <vector>
namespace strata::dsv41::metal_host {
+mtl::Context& finish_cx(); // ds4_finish.cpp's Context: every kernel that orders with the chunk path's dispatches uses it
int shared_lib(); // from ds4_finish.cpp: the library the shared Context compiled
@@ -145,7 +146,7 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {
- strata::dsv41::metal_host::shared_cx();
+ finish_cx();
const int lib = shared_lib();
if (lib < 0) return false;
void* ha = strata::gpu::host_view((void*) A);
@@ -154,7 +155,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
if (!ha || !hb || !hc) { std::fprintf(stderr, "dsv41 metal: sgemm_tn has no host view\n"); return false; }
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
- return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ return finish_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
mtl::Buffer ba = wrap(ha, (size_t) m * lda * 4);
@@ -162,7 +163,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
mtl::Buffer bc = wrap(hc, (size_t) nt * ldc * 4);
if (!ba.handle || !bb.handle || !bc.handle) return false;
mtl::Pipeline pl;
- try { pl = strata::dsv41::metal_host::shared_cx().pipeline(g_lib_init, "ds4_sgemm_tn"); } catch (...) { return false; }
+ try { pl = finish_cx().pipeline(lib, "ds4_sgemm_tn"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bc, off(hc)),
@@ -172,7 +173,12 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
d.grid[0] = (uint32_t) ((total + 127) / 128);
d.grid[1] = d.grid[2] = 1;
d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
- strata::dsv41::metal_host::shared_cx().run({d});
+ finish_cx().run({d});
+ if (std::getenv("DS4_DBG_SG")) {
+ finish_cx().flush_runs();
+ const float* fa = (const float*) ha; const float* fb = (const float*) hb; const float* fc = (const float*) hc;
+ std::fprintf(stderr, "sgemm m %d nt %d k %d lda %d ldb %d ldc %d A %g %g B %g %g C %g %g %g\n", m, nt, k, lda, ldb, ldc, fa[0], fa[1], fb[0], fb[1], fc[0], fc[1], fc[2]);
+ }
return true;
}
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 6bd5d6bdf..bb0cdea75 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1165,6 +1165,7 @@ struct Model::Impl {
cudaSetDevice(d.id);
#if defined(STRATA_METAL)
ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)
+ cudaDeviceSynchronize();
#endif
cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
cudaStreamSynchronize(d.st);
@@ -1191,6 +1192,7 @@ struct Model::Impl {
cudaSetDevice(d.id);
#if defined(STRATA_METAL)
ds4_metal_flush();
+ cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)
#endif
cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);
cudaStreamSynchronize(d.st);
@@ -6198,14 +6200,22 @@ struct Model::Impl {
}
// attention
if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;
+ dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
+ dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
+ dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);
+ dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);
+ dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);
+ dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);
x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
!gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
return false;
if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
+ dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);
+ dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);
Finish fq;
fq.w = l.q_norm;
fq.eps = g.eps;
@@ -6369,6 +6379,7 @@ struct Model::Impl {
fr.vpp = NHL;
fr.freqs = freqs;
finish(c.q, na * NHL, HD, HD, fr, s);
+ if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);
AttnRows ar;
ar.gen = 1;
ar.p0 = p0;
@@ -6378,6 +6389,7 @@ struct Model::Impl {
ar.kcap = g.ix_topk;
sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,
na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)
+ if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);
Finish fo;
fo.rd = g.rd;
fo.pos = (int) (p0 + a0);
@@ -6386,6 +6398,7 @@ struct Model::Impl {
fo.inverse = 1;
fo.quant = 1;
finish(c.o, na * NHL, HD, HD, fo, s);
+ if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);
x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);
const int per = HQ / OG;
for (int gi = 0; gi < OG; ++gi)
@@ -6394,9 +6407,12 @@ struct Model::Impl {
return false;
x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);
if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;
+ if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);
}
if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);
hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)
prof_mark(d, 1);
// the MoE
if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;
@@ -7208,6 +7224,7 @@ struct Model::Impl {
return false;
}
const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
+ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);
Run run;
for (int a = 0; a < n; a += per) {
run.p0.push_back(pos + a);
@@ -8317,6 +8334,7 @@ struct Model::Impl {
hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
std::swap(d.h, d.h2);
dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)
CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
// (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)
return true;
@@ -8421,6 +8439,7 @@ bool Model::step(int token, bool logits, std::string& err) { return p_->step(tok
bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
Impl& p = *p_;
p.pm.last = -1;
+ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);
if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)