Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
ds4: DS4_DUMP_S sub-step dump inside the layers (same records on CUDA and Metal), DS4_FLUSH_EACH, per-layer debug prints
Niko1221da9cf6354fa2
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 0a72f5c06..132195d83 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -300,6 +300,7 @@ static void ds4_report_counts() {extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }+extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));void Context::flush_runs() {std::lock_guard<std::recursive_mutex> lk_(impl_->mu);if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }@@ -328,6 +329,11 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,encode(impl_->enc_def, ds[0]);++impl_->def_n;++g_disp_count;+ static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in order+ if (each) {+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();+ flush_runs();+ }return 0.0;}flush_runs();diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 1f306544a..240422119 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1068,6 +1068,22 @@ struct Model::Impl {// DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or// chunk path (tests: where the two part)std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;+ // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records+ // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)+ std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;+ std::vector<int> dump_s_layers = [] {+ std::vector<int> v;+ const char* e = std::getenv("DS4_DUMP_S_LAYERS");+ std::string str = e ? e : "0,1,2,3";+ size_t a = 0;+ while (a < str.size()) {+ size_t b = str.find(',', a);+ if (b == std::string::npos) b = str.size();+ if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));+ a = b + 1;+ }+ return v;+ }();// DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),// then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the// slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias@@ -1153,6 +1169,35 @@ struct Model::Impl {std::fflush(dump_l);}+ bool dump_s_on(int il) const {+ if (dump_s == nullptr) return false;+ for (int x : dump_s_layers)+ if (x == il) return true;+ return false;+ }+ void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {+ if (!dump_s_on(il)) return;+ std::vector<uint8_t> v(nbytes);+ cudaSetDevice(d.id);+#if defined(STRATA_METAL)+ ds4_metal_flush();+#endif+ cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ dump_sub_host(name, il, p0, T, v.data(), nbytes);+ }+ void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {+ if (!dump_s_on(il)) return;+ std::lock_guard<std::mutex> lk(dump_mu);+ char nm[16] = {};+ std::strncpy(nm, name, 15);+ const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};+ std::fwrite(nm, 1, 16, dump_s);+ std::fwrite(hdr, 8, 4, dump_s);+ std::fwrite(host, 1, nbytes, dump_s);+ std::fflush(dump_s);+ }+~Impl() {eg_io.stop(); // (its threads write into h_eg)for (auto& w : workers) w->stop();@@ -7754,10 +7799,17 @@ struct Model::Impl {engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);}+ dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);// attentionhc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);+ dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);+ dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);+ dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);+ dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);+ dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);+ dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);{x3::MvJob j[4];int nj = 0;@@ -7767,10 +7819,13 @@ struct Model::Impl {if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);if (!mv_n(d, j, nj, err)) return false;}+ dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);+ dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);Finish fq;fq.w = l.q_norm;fq.eps = g.eps;finish(d.qr, T, g.q_lora, g.q_lora, fq, s);+ dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);if (NHL > 0) {if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;Finish fr;@@ -7780,6 +7835,7 @@ struct Model::Impl {fr.freqs = freqs;finish(d.q, T * NHL, HD, HD, fr, s);}+ if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);// the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark// block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of// that ring and to the whole block, and leaves the ring as it was@@ -7795,6 +7851,7 @@ struct Model::Impl {fk.dst_stride = HD;finish(d.kvr, T, HD, HD, fk, s);if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);+ dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);AttnRows ar;for (int t = 0; t < T; ++t) {if (l.mtp) {@@ -7948,6 +8005,7 @@ struct Model::Impl {if (NHL > 0) {sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);+ dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);Finish fo;fo.rd = g.rd;fo.pos = (int) p0;@@ -7956,6 +8014,7 @@ struct Model::Impl {fo.inverse = 1;fo.quant = 1;finish(d.o, T * NHL, HD, HD, fo, s);+ dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);x3::MvJob j[16];const int per = HQ / OG;for (int gi = 0; gi < OG; ++gi)@@ -7963,15 +8022,21 @@ struct Model::Impl {if (!mv_n(d, j, OG, err)) return false;if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;} else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)+ dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);std::swap(d.h, d.h2);if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)+ dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);prof_mark(d, 1);// the MoEhc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);+ dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);+ dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);+ dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);+ dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);if (wp.async) {// the routing goes to the service thread through the mailbox (a group's lead publishes it, the members// make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the@@ -8084,11 +8149,18 @@ struct Model::Impl {CK(cudaEventRecord(d.ev, s));// the shared expert while the host looks at the picksif (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;+ if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);+ std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);+ if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;const double tw = now_ms();CK(cudaEventSynchronize(d.ev));st.gpu_wait_ms += now_ms() - tw;+ dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);+ dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);+ dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);+ dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);// the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the// CPU lane's entriesCombine cm;@@ -8177,11 +8249,21 @@ struct Model::Impl {for (int e = 0; e < D; ++e) o[e] += wv * y[e];}st.cpu_ms += now_ms() - tc;+ dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));}combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);+ dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);+ if (std::getenv("DS4_DBG_MOE")) {+ float hp = 0.0f;+ for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));+ std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);+ dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);+ dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);+ }hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);std::swap(d.h, d.h2);+ dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));// (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)return true;