Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: chunked prompt path correct (sgemm on the one Context, KEEP dump syncs, debug switches); 2988-token prompt 251-277 tok/s vs 32 in windows
Niko1221700def65a2a4
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 966eb491f..9b29796d1 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -210,7 +210,18 @@ bool hc_post(const float* y, float* h, const float* post, const float* comb, intd.grid[1] = (uint32_t) T;d.grid[2] = 1;d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ const bool dbg = std::getenv("DS4_DBG_SG") != nullptr;+ if (dbg) {+ g_cx->flush_runs();+ const float* fy = (const float*) hy; const float* fh = (const float*) hh; const float* fp = (const float*) hpost; const float* fc = (const float*) hcomb;+ std::fprintf(stderr, "hc_post in: y %g %g h %g %g | %g %g post %g %g %g %g comb %g %g %g %g\n", fy[0], fy[1], fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1], fp[0], fp[1], fp[2], fp[3], fc[0], fc[1], fc[2], fc[3]);+ }g_cx->run({d});+ if (dbg) {+ g_cx->flush_runs();+ const float* fh = (const float*) hh;+ std::fprintf(stderr, "hc_post out: h %g %g | %g %g\n", fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1]);+ }g_cx->free(bp);g_cx->free(bc);return true;@@ -243,6 +254,11 @@ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float*d.grid[1] = d.grid[2] = 1;d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;g_cx->run({d});+ if (std::getenv("DS4_DBG_SG")) {+ g_cx->flush_runs();+ const float* fc = (const float*) hm;+ std::fprintf(stderr, "hc_mix_scale T %d hcdim %d n_mix %d eps %g -> %g %g %g\n", T, hcdim, n_mix, eps, fc[0], fc[1], fc[2]);+ }return true;}@@ -319,7 +335,17 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,d.grid[0] = (uint32_t) T;d.grid[1] = d.grid[2] = 1;d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;+ if (std::getenv("DS4_DBG_SG")) {+ g_cx->flush_runs();+ const float* fm = (const float*) hm; const float* fs = (const float*) hs; const float* fb = (const float*) hb;+ std::fprintf(stderr, "hc_split in: mix %g %g %g scale %g %g %g base %g %g\n", fm[0], fm[1], fm[2], fs[0], fs[1], fs[2], fb[0], fb[1]);+ }g_cx->run({d});+ if (std::getenv("DS4_DBG_SG")) {+ g_cx->flush_runs();+ const float* fp = (const float*) hp;+ std::fprintf(stderr, "hc_split out: pre %g %g %g %g\n", fp[0], fp[1], fp[2], fp[3]);+ }return true;}@@ -2474,6 +2500,10 @@ mtl::Context& shared_cx() {ensure();return *g_cx;}+mtl::Context& finish_cx() {+ ensure();+ return *g_cx;+}int shared_lib() {ensure();diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cppindex cc9129cb3..2d4251745 100644--- a/metal/src/engine/ds4_host.cpp+++ b/metal/src/engine/ds4_host.cpp@@ -19,6 +19,7 @@#include <vector>namespace strata::dsv41::metal_host {+mtl::Context& finish_cx(); // ds4_finish.cpp's Context: every kernel that orders with the chunk path's dispatches uses itint shared_lib(); // from ds4_finish.cpp: the library the shared Context compiled@@ -145,7 +146,7 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {- strata::dsv41::metal_host::shared_cx();+ finish_cx();const int lib = shared_lib();if (lib < 0) return false;void* ha = strata::gpu::host_view((void*) A);@@ -154,7 +155,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, inif (!ha || !hb || !hc) { std::fprintf(stderr, "dsv41 metal: sgemm_tn has no host view\n"); return false; }const size_t page = 16384;auto wrap = [&](void* host, size_t bytes) {- return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ return finish_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };mtl::Buffer ba = wrap(ha, (size_t) m * lda * 4);@@ -162,7 +163,7 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, inmtl::Buffer bc = wrap(hc, (size_t) nt * ldc * 4);if (!ba.handle || !bb.handle || !bc.handle) return false;mtl::Pipeline pl;- try { pl = strata::dsv41::metal_host::shared_cx().pipeline(g_lib_init, "ds4_sgemm_tn"); } catch (...) { return false; }+ try { pl = finish_cx().pipeline(lib, "ds4_sgemm_tn"); } catch (...) { return false; }mtl::Dispatch d;d.p = &pl;d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bc, off(hc)),@@ -172,7 +173,12 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, ind.grid[0] = (uint32_t) ((total + 127) / 128);d.grid[1] = d.grid[2] = 1;d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;- strata::dsv41::metal_host::shared_cx().run({d});+ finish_cx().run({d});+ if (std::getenv("DS4_DBG_SG")) {+ finish_cx().flush_runs();+ const float* fa = (const float*) ha; const float* fb = (const float*) hb; const float* fc = (const float*) hc;+ std::fprintf(stderr, "sgemm m %d nt %d k %d lda %d ldb %d ldc %d A %g %g B %g %g C %g %g %g\n", m, nt, k, lda, ldb, ldc, fa[0], fa[1], fb[0], fb[1], fc[0], fc[1], fc[2]);+ }return true;}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 6bd5d6bdf..bb0cdea75 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1165,6 +1165,7 @@ struct Model::Impl {cudaSetDevice(d.id);#if defined(STRATA_METAL)ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)+ cudaDeviceSynchronize();#endifcudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);cudaStreamSynchronize(d.st);@@ -1191,6 +1192,7 @@ struct Model::Impl {cudaSetDevice(d.id);#if defined(STRATA_METAL)ds4_metal_flush();+ cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)#endifcudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);cudaStreamSynchronize(d.st);@@ -6198,14 +6200,22 @@ struct Model::Impl {}// attentionif (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;+ dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);+ dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);+ dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);+ dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);+ dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);+ dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||!gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))return false;if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;+ dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);+ dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);Finish fq;fq.w = l.q_norm;fq.eps = g.eps;@@ -6369,6 +6379,7 @@ struct Model::Impl {fr.vpp = NHL;fr.freqs = freqs;finish(c.q, na * NHL, HD, HD, fr, s);+ if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);AttnRows ar;ar.gen = 1;ar.p0 = p0;@@ -6378,6 +6389,7 @@ struct Model::Impl {ar.kcap = g.ix_topk;sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)+ if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);Finish fo;fo.rd = g.rd;fo.pos = (int) (p0 + a0);@@ -6386,6 +6398,7 @@ struct Model::Impl {fo.inverse = 1;fo.quant = 1;finish(c.o, na * NHL, HD, HD, fo, s);+ if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);const int per = HQ / OG;for (int gi = 0; gi < OG; ++gi)@@ -6394,9 +6407,12 @@ struct Model::Impl {return false;x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;+ if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);}if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);+ dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)prof_mark(d, 1);// the MoEif (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;@@ -7208,6 +7224,7 @@ struct Model::Impl {return false;}const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;+ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);Run run;for (int a = 0; a < n; a += per) {run.p0.push_back(pos + a);@@ -8317,6 +8334,7 @@ struct Model::Impl {hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);std::swap(d.h, d.h2);dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));// (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)return true;@@ -8421,6 +8439,7 @@ bool Model::step(int token, bool logits, std::string& err) { return p_->step(tokbool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {Impl& p = *p_;p.pm.last = -1;+ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path hereconst double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)