Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
verify: STRATA_ROUTE_STATS (distinct and new-per-row routed experts per layer per window width) and a per-width GPU stage profile line for STRATA_SPEC_T_CYCLE runs (bench only, off by default)
Niko1221d276adf27778
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hppindex b2ea81923..c36be8a5c 100644--- a/include/strata/core/verify.hpp+++ b/include/strata/core/verify.hpp@@ -357,6 +357,10 @@ public:/// STRATA_VERIFY_PROFILE=1 - GPU stage times of the windows since the last call (ms per/// window), as one line; empty when off.std::string profile_report();+ /// STRATA_VERIFY_PROFILE: the same stage split, one line per window size T (bench: with STRATA_SPEC_T_CYCLE). Resets.+ std::string profile_by_t_report();+ /// STRATA_ROUTE_STATS=1: per window size, the routed experts per layer: distinct, and new per row (not in the rows before it). Resets.+ std::string route_stats_report();private:RemoteExpertOpt* remote_opt_ = nullptr;@@ -415,7 +419,8 @@ private:bool instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key, const char* what,std::string& err);void collect_profile(); ///< STRATA_VERIFY_PROFILE: add the last window's stamps to prof_sum_- void accumulate_profile(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_+ void accumulate_profile(const unsigned long long* stamps);+ void accumulate_profile_inner(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_// pipelined windows (pl_launch ...)cudaStream_t ext_stream_ = nullptr; ///< set_stream: the stage's shared stream (not destroyed here)bool always_publish_ = false;@@ -506,6 +511,11 @@ private:std::vector<unsigned long long> prof_h_;double prof_sum_[2][kProfPer] = {}; // [GDN / QSA layers][stage]int64_t prof_windows_ = 0;+ double prof_t_sum_[9][2][kProfPer] = {}; // the same per window size T (1..8), for profile_by_t_report+ int64_t prof_t_win_[9] = {};+ bool rs_on_ = false; // STRATA_ROUTE_STATS+ unsigned long long* rs_dev_ = nullptr; // [9 T][n_layers][8 rows]: experts of row r that no earlier row of the window routed to+ int64_t rs_win_[9] = {};const WeightTable* wt_ = nullptr;const ModelGeometry* g_ = nullptr;diff --git a/include/strata/kernels/verify_kernels.hpp b/include/strata/kernels/verify_kernels.hppindex e7dd1392c..a5e35181e 100644--- a/include/strata/kernels/verify_kernels.hpp+++ b/include/strata/kernels/verify_kernels.hpp@@ -126,6 +126,9 @@ void broadcast_streams(const float* x, float* R, int64_t n_embd, int hc, int n_t/// out[:n_embd] = the mean of the hc streams of ONE row R[hc][n_embd] (the DFlash2 drafter's target features; only the/// STRATA_TAP_FILE hook calls it).void hc_mean_streams(const float* R, float* out, int64_t n_embd, int hc, void* stream);+/// STRATA_ROUTE_STATS only: acc[r] += the number of ids of row r (k per row, n rows <= 8) that no earlier row of the window holds+/// (duplicates inside a row count once).+void route_marginal_stats(const int32_t* ids, int n, int k, unsigned long long* acc, void* stream);/// dst[:n] = src[(*index) * stride + :n] (index read from device memory; a negative index copies nothing).void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* index, int64_t n, void* stream);diff --git a/src/core/verify.cpp b/src/core/verify.cppindex c9f48a747..078bbd5fa 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -923,6 +923,16 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&#endif}prof_on_ = std::getenv("STRATA_VERIFY_PROFILE") != nullptr;+ if (const char* rs = std::getenv("STRATA_ROUTE_STATS"); rs != nullptr && rs[0] == '1') {+ const size_t nrs = (size_t) 9 * (size_t) g.n_layers * 8;+ if (cudaMalloc((void**) &rs_dev_, nrs * 8) == cudaSuccess) {+ cudaMemset(rs_dev_, 0, nrs * 8);+ rs_on_ = true;+ } else {+ rs_dev_ = nullptr;+ cudaGetLastError();+ }+ }if (prof_on_) {const size_t np = (size_t) g.n_layers * kProfPer + 4;if (cudaMalloc((void**) &prof_, np * 8) != cudaSuccess) { prof_on_ = false; prof_ = nullptr; cudaGetLastError(); }@@ -1959,6 +1969,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {rr_stats_ != nullptr ? rr_stats_ + (n <= 8 ? 4 : 0) : nullptr, cs);} catch (const std::exception& e) { err = "verify route-resident: " + std::string(e.what()); return false; }}+ if (rs_on_ && G == 1 && n <= 8 && T <= 8)+ strata::kernels::route_marginal_stats(ids_ + tb * K, n, (int) K, rs_dev_ + ((size_t) T * (size_t) g.n_layers + (size_t) l) * 8, cs);if (ar_on()) {resident_plan(ids_ + tb * K, n * (int) K, (int) K, hits_.d_res + l * g.n_expert, (int) g.n_expert,hits_.cache_base, slot_off_d_, (long long) hits_.blob,@@ -2318,6 +2330,66 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {return true;}+std::string Verifier::profile_by_t_report() {+ if (!prof_on_) return std::string();+ static const char* names[kProfPer] = {"-", "hc-read0", "qkv gemv", "conv", "ab", "z", "rec", "q8+kv-idx",+ "k/v+norm-rope", "kv+idx append", "q+q-idx", "scores+topk", "kv-resolve",+ "attention", "gate", "", "out-proj", "router+plan", "shared+quant",+ "waitA", "VRAM hits", "waitB", "PCIe grp", "waitCPU", "copy+combine",+ "(gap)", "head", "hc0 norm", "hc0 down", "hc0 up", "hc1 norm", "hc1 down", "hc1 up"};+ std::string out;+ char b[96];+ for (int T = 1; T <= 8; ++T) {+ if (prof_t_win_[T] == 0) continue;+ double total = 0;+ std::snprintf(b, sizeof b, "T%d (n=%lld):", T, (long long) prof_t_win_[T]);+ out += b;+ for (int k = 0; k < 2; ++k) {+ out += k == 0 ? " GDN[" : " | QSA[";+ for (int i = 0; i < kProfPer; ++i) {+ if (prof_t_sum_[T][k][i] <= 0) continue;+ total += prof_t_sum_[T][k][i];+ std::snprintf(b, sizeof b, " %s %.3f", names[i], prof_t_sum_[T][k][i] / 1e6 / (double) prof_t_win_[T]);+ out += b;+ }+ out += " ]";+ }+ std::snprintf(b, sizeof b, " | total %.2f ms\n", total / 1e6 / (double) prof_t_win_[T]);+ out += b;+ }+ std::memset(prof_t_sum_, 0, sizeof prof_t_sum_);+ std::memset(prof_t_win_, 0, sizeof prof_t_win_);+ return out;+}++std::string Verifier::route_stats_report() {+ if (!rs_on_ || rs_dev_ == nullptr || g_ == nullptr) return std::string();+ const int64_t L = g_->n_layers;+ std::vector<unsigned long long> h((size_t) 9 * (size_t) L * 8);+ cudaMemcpy(h.data(), rs_dev_, h.size() * 8, cudaMemcpyDeviceToHost);+ std::string out;+ char b[160];+ for (int T = 1; T <= 8; ++T) {+ if (rs_win_[T] == 0) continue;+ double row[8] = {};+ for (int64_t l = 0; l < L; ++l)+ for (int r = 0; r < 8; ++r) row[r] += (double) h[((size_t) T * (size_t) L + (size_t) l) * 8 + (size_t) r];+ const double denom = (double) rs_win_[T] * (double) L;+ double sum = 0;+ for (int r = 0; r < T; ++r) sum += row[r] / denom;+ std::snprintf(b, sizeof b, "T%d (n=%lld): distinct experts per layer %.2f; new per row:", T, (long long) rs_win_[T], sum);+ out += b;+ for (int r = 0; r < T; ++r) {+ std::snprintf(b, sizeof b, " %.2f", row[r] / denom);+ out += b;+ }+ out += "\n";+ }+ cudaMemset(rs_dev_, 0, h.size() * 8);+ std::memset(rs_win_, 0, sizeof rs_win_);+ return out;+}+std::string Verifier::profile_report() {if (!prof_on_ || prof_windows_ == 0) return std::string();static const char* names[kProfPer] = {"-", "hc-read0", "q8+qkv/q-idx gemv", "conv", "ab", "z", "rec", "q8+kv-idx",@@ -2651,6 +2723,7 @@ void Verifier::stage_inputs(int T, const int32_t* tokens, int64_t pos0) {std::atomic_thread_fence(std::memory_order_seq_cst);_mm_sfence();last_t_ = T;+ if (rs_on_ && T >= 1 && T <= 8) ++rs_win_[T];last_pos0_ = pos0;for (int t = 0; t < T; ++t) last_tokens_[t] = tokens[t];staged_ = true;@@ -2662,6 +2735,17 @@ void Verifier::collect_profile() {}void Verifier::accumulate_profile(const unsigned long long* stamps) {+ double before[2][kProfPer];+ std::memcpy(before, prof_sum_, sizeof before);+ accumulate_profile_inner(stamps);+ if (last_t_ >= 1 && last_t_ <= 8) {+ for (int k = 0; k < 2; ++k)+ for (int i = 0; i < kProfPer; ++i) prof_t_sum_[last_t_][k][i] += prof_sum_[k][i] - before[k][i];+ ++prof_t_win_[last_t_];+ }+}++void Verifier::accumulate_profile_inner(const unsigned long long* stamps) {const ModelGeometry& g = *g_;const int64_t L = g.n_layers;auto at = [&](int64_t l, int i) { return stamps[(size_t) (l * kProfPer + i)]; };diff --git a/src/kernels/cuda/verify_kernels.cu b/src/kernels/cuda/verify_kernels.cuindex dba0f1016..a9f570b9e 100644--- a/src/kernels/cuda/verify_kernels.cu+++ b/src/kernels/cuda/verify_kernels.cu@@ -588,6 +588,21 @@ __global__ void hc_mean_streams_kernel(const float* __restrict__ R, float* __resout[i] = s / (float) hc;}+__global__ void route_marginal_stats_kernel(const int32_t* __restrict__ ids, int n, int k, unsigned long long* acc) {+ const int r = threadIdx.x;+ if (r >= n) return;+ int fresh = 0;+ for (int j = 0; j < k; ++j) {+ const int32_t e = ids[r * k + j];+ bool seen = false;+ for (int j2 = 0; j2 < j && !seen; ++j2) seen = ids[r * k + j2] == e;+ for (int r2 = 0; r2 < r && !seen; ++r2)+ for (int j2 = 0; j2 < k && !seen; ++j2) seen = ids[r2 * k + j2] == e;+ if (!seen) ++fresh;+ }+ atomicAdd(acc + r, (unsigned long long) fresh);+}+__global__ void copy_indexed_kernel(float* __restrict__ dst, const float* __restrict__ src, int64_t stride,const int32_t* __restrict__ index, int64_t n) {const int idx = *index;@@ -1527,6 +1542,11 @@ void hc_mean_streams(const float* R, float* out, int64_t n_embd, int hc, void* scheck("hc_mean_streams");}+void route_marginal_stats(const int32_t* ids, int n, int k, unsigned long long* acc, void* stream) {+ route_marginal_stats_kernel<<<1, 32, 0, (cudaStream_t) stream>>>(ids, n, k, acc);+ check("route_marginal_stats");+}+void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* index, int64_t n, void* stream) {const unsigned blocks = (unsigned) ((n + 255) / 256 < 64 ? (n + 255) / 256 : 64);copy_indexed_kernel<<<blocks, 256, 0, (cudaStream_t) stream>>>(dst, src, stride, index, n);diff --git a/src/program/generate.cpp b/src/program/generate.cppindex d7116d2df..cfb576651 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -16454,6 +16454,12 @@ int main(int argc, char** argv) {for (size_t i = 1; i < t_cycle_ms.size(); ++i)if (t_cycle_n[i] > 0) std::printf(" T%zu:%.2f(n=%lld)", i, t_cycle_ms[i] / (double) t_cycle_n[i], (long long) t_cycle_n[i]);std::printf("\n");+ { // STRATA_VERIFY_PROFILE / STRATA_ROUTE_STATS: the same split per window width (bench only)+ const std::string pt = ver.profile_by_t_report();+ if (!pt.empty()) std::printf("GPU stages per window width (ms):\n%s", pt.c_str());+ const std::string rs = ver.route_stats_report();+ if (!rs.empty()) std::printf("routed experts per window width:\n%s", rs.c_str());+ }}std::printf("%-24s", "accepted per round");for (size_t i = 0; i < accepted_hist.size(); ++i) std::printf(" %zu:%lld", i, (long long) accepted_hist[i]);