Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: finer DS4_PROF for the chunked prompt (stages, expert kernels by type)
Niko1221427830491467
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex a9e370441..41304d8af 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -16,6 +16,7 @@#include "ds4_engram_io.hpp"#include "ds4_gguf.cuh"+#include "ds4_iq.cuh"#include "ds4_gguf_cpu.hpp"#include "ds4_kernels.cuh"#include "ds4_os.hpp"@@ -1121,7 +1122,30 @@ struct Model::Impl {// ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared// expert, after its experts (the streamed ones' waits for their reads in it); printed after the runbool prof_chunk = false;- double cprof_ms[4] = {};+ // DS4_PROF stages of the chunked prompt (finer ones than the decode's 0..5): 0 between layers, 1 attention rest (wq_b, wo,+ // hc_post), 2 router-side tail (shared expert), 3 experts (+ waits), 4 Engram, 5 attn mHC (mix, split, norm), 6 attention+ // projections (wq_a, wkv, compressor), 7 compressor + indexer, 8 q_b + rope, 9 sparse attention, 10 ffn mHC + router + sort+ double cprof_ms[16] = {};+ // DS4_PROF: the experts' kernels alone, by type (events around the gate/up and down calls of run_set)+ struct EkMark {+ cudaEvent_t e0, e1, e2;+ int tg, td, nrows, nexp, maxrows, dev, ff;+ };+ std::vector<EkMark> ek_marks;+ std::map<int, std::vector<cudaEvent_t>> ek_pool; // by device+ cudaEvent_t ek_ev() {+ int cur = 0;+ cudaGetDevice(&cur);+ std::vector<cudaEvent_t>& pl = ek_pool[cur];+ if (!pl.empty()) {+ cudaEvent_t e = pl.back();+ pl.pop_back();+ return e;+ }+ cudaEvent_t e = nullptr;+ cudaEventCreate(&e);+ return e;+ }void prof_mark(const Dev& d, int stage) {if (!prof_window && !prof_chunk) return;std::lock_guard<std::mutex> lk(prof_mu);@@ -3421,8 +3445,30 @@ struct Model::Impl {gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);gg::set_down(sa, c.h16, FF, c.y, D, d.st);} else {+ EkMark ek{};+ const bool ekp = prof_chunk;+ if (ekp) {+ std::lock_guard<std::mutex> lk(prof_mu);+ ek.e0 = ek_ev();+ ek.e1 = ek_ev();+ ek.e2 = ek_ev();+ ek.tg = gg::expert_type(sa.lay, 0);+ ek.td = gg::expert_type(sa.lay, 2);+ ek.nrows = nrows;+ ek.nexp = j - i0;+ ek.maxrows = max_rows;+ ek.ff = FF;+ cudaGetDevice(&ek.dev);+ cudaEventRecord(ek.e0, d.st);+ }x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);+ if (ekp) cudaEventRecord(ek.e1, d.st);x3::set_down(sa, c.h16, FF, c.y, D, d.st);+ if (ekp) {+ cudaEventRecord(ek.e2, d.st);+ std::lock_guard<std::mutex> lk(prof_mu);+ ek_marks.push_back(ek);+ }}if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {std::vector<int> bb(b + i0, b + j + 1);@@ -6247,16 +6293,19 @@ struct Model::Impl {engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);}}+ prof_mark(d, 4);// attentionif (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);+ prof_mark(d, 5);if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||!gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))return false;if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;+ prof_mark(d, 6);Finish fq;fq.w = l.q_norm;fq.eps = g.eps;@@ -6409,6 +6458,7 @@ struct Model::Impl {}ckv = sr.ckv[(size_t) l.dev];}+ prof_mark(d, 7);// the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into accif (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {@@ -6420,6 +6470,7 @@ struct Model::Impl {fr.vpp = NHL;fr.freqs = freqs;finish(c.q, na * NHL, HD, HD, fr, s);+ prof_mark(d, 8);AttnRows ar;ar.gen = 1;ar.p0 = p0;@@ -6429,6 +6480,7 @@ struct Model::Impl {ar.kcap = g.ix_topk;sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)+ prof_mark(d, 9);Finish fo;fo.rd = g.rd;fo.pos = (int) (p0 + a0);@@ -6466,6 +6518,7 @@ struct Model::Impl {CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));+ prof_mark(d, 10);// the shared expert into acc while the host waits for the boundsif (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))return false;@@ -7373,6 +7426,50 @@ struct Model::Impl {"ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms ""(with the waits for streamed ones), between layers %.0f ms\n",n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);+ { // the finer table (ms, % of the sum)+ static const char* nm_[16] = {"between layers", "attn rest (q_b, wo_a/b, hc_post)", "shared expert", "experts (incl waits)",+ "engram", "attn mHC (mix/split/norm)", "attn projections", "compressor + indexer",+ "q_b + rope", "sparse attention", "ffn mHC + router + sort", "", "", "", "", ""};+ double sum = 0;+ for (double v : cprof_ms) sum += v;+ for (int i = 0; i < 11; ++i)+ std::fprintf(stderr, "ds4 prof %-34s %9.1f ms %5.1f%%\n", nm_[i], cprof_ms[i], sum > 0 ? 100.0 * cprof_ms[i] / sum : 0.0);+ struct Agg {+ double ms = 0, flops = 0;+ long calls = 0, rows = 0, experts = 0;+ };+ std::map<std::pair<int, int>, Agg> agg; // (type, role 0 gate+up, 1 down)+ {+ std::lock_guard<std::mutex> lk(prof_mu);+ for (const EkMark& m : ek_marks) {+ float a = 0, b = 0;+ cudaSetDevice(m.dev);+ cudaEventElapsedTime(&a, m.e0, m.e1);+ cudaEventElapsedTime(&b, m.e1, m.e2);+ Agg& x = agg[{m.tg, 0}];+ x.ms += a;+ x.flops += 2.0 * 2.0 * m.nrows * (double) g.dim * (double) m.ff;+ x.calls++, x.rows += m.nrows, x.experts += m.nexp;+ Agg& y = agg[{m.td, 1}];+ y.ms += b;+ y.flops += 2.0 * m.nrows * (double) g.dim * (double) m.ff;+ y.calls++, y.rows += m.nrows, y.experts += m.nexp;+ ek_pool[m.dev].push_back(m.e0);+ ek_pool[m.dev].push_back(m.e1);+ ek_pool[m.dev].push_back(m.e2);+ }+ ek_marks.clear();+ }+ double tot = 0;+ for (auto& kv : agg) tot += kv.second.ms;+ for (auto& kv : agg)+ std::fprintf(stderr,+ "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld rows/expert %6.1f ~%5.1f TFLOPS (2048-wide ffn)\n",+ gg::iq::name(kv.first.first), kv.first.second ? "down" : "gate+up", kv.second.ms, kv.second.calls,+ kv.second.experts > 0 ? (double) kv.second.rows / (double) kv.second.experts : 0.0,+ kv.second.ms > 0 ? kv.second.flops / kv.second.ms / 1e9 : 0.0);+ std::fprintf(stderr, "ds4 prof expert kernels total %.1f ms\n", tot);+ }for (double& v : cprof_ms) v = 0;prof_chunk = false;}