Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

verify: STRATA_ROUTE_STATS (distinct and new-per-row routed experts per layer per window width) and a per-width GPU stage profile line for STRATA_SPEC_T_CYCLE runs (bench only, off by default)

Niko1221d276adf27778
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index b2ea81923..c36be8a5c 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -357,6 +357,10 @@ public:
/// STRATA_VERIFY_PROFILE=1 - GPU stage times of the windows since the last call (ms per
/// window), as one line; empty when off.
std::string profile_report();
+ /// STRATA_VERIFY_PROFILE: the same stage split, one line per window size T (bench: with STRATA_SPEC_T_CYCLE). Resets.
+ std::string profile_by_t_report();
+ /// STRATA_ROUTE_STATS=1: per window size, the routed experts per layer: distinct, and new per row (not in the rows before it). Resets.
+ std::string route_stats_report();
private:
RemoteExpertOpt* remote_opt_ = nullptr;
@@ -415,7 +419,8 @@ private:
bool instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key, const char* what,
std::string& err);
void collect_profile(); ///< STRATA_VERIFY_PROFILE: add the last window's stamps to prof_sum_
- void accumulate_profile(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_
+ void accumulate_profile(const unsigned long long* stamps);
+ void accumulate_profile_inner(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_
// pipelined windows (pl_launch ...)
cudaStream_t ext_stream_ = nullptr; ///< set_stream: the stage's shared stream (not destroyed here)
bool always_publish_ = false;
@@ -506,6 +511,11 @@ private:
std::vector<unsigned long long> prof_h_;
double prof_sum_[2][kProfPer] = {}; // [GDN / QSA layers][stage]
int64_t prof_windows_ = 0;
+ double prof_t_sum_[9][2][kProfPer] = {}; // the same per window size T (1..8), for profile_by_t_report
+ int64_t prof_t_win_[9] = {};
+ bool rs_on_ = false; // STRATA_ROUTE_STATS
+ unsigned long long* rs_dev_ = nullptr; // [9 T][n_layers][8 rows]: experts of row r that no earlier row of the window routed to
+ int64_t rs_win_[9] = {};
const WeightTable* wt_ = nullptr;
const ModelGeometry* g_ = nullptr;
diff --git a/include/strata/kernels/verify_kernels.hpp b/include/strata/kernels/verify_kernels.hpp
index e7dd1392c..a5e35181e 100644
--- a/include/strata/kernels/verify_kernels.hpp
+++ b/include/strata/kernels/verify_kernels.hpp
@@ -126,6 +126,9 @@ void broadcast_streams(const float* x, float* R, int64_t n_embd, int hc, int n_t
/// out[:n_embd] = the mean of the hc streams of ONE row R[hc][n_embd] (the DFlash2 drafter's target features; only the
/// STRATA_TAP_FILE hook calls it).
void hc_mean_streams(const float* R, float* out, int64_t n_embd, int hc, void* stream);
+/// STRATA_ROUTE_STATS only: acc[r] += the number of ids of row r (k per row, n rows <= 8) that no earlier row of the window holds
+/// (duplicates inside a row count once).
+void route_marginal_stats(const int32_t* ids, int n, int k, unsigned long long* acc, void* stream);
/// dst[:n] = src[(*index) * stride + :n] (index read from device memory; a negative index copies nothing).
void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* index, int64_t n, void* stream);
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index c9f48a747..078bbd5fa 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -923,6 +923,16 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
#endif
}
prof_on_ = std::getenv("STRATA_VERIFY_PROFILE") != nullptr;
+ if (const char* rs = std::getenv("STRATA_ROUTE_STATS"); rs != nullptr && rs[0] == '1') {
+ const size_t nrs = (size_t) 9 * (size_t) g.n_layers * 8;
+ if (cudaMalloc((void**) &rs_dev_, nrs * 8) == cudaSuccess) {
+ cudaMemset(rs_dev_, 0, nrs * 8);
+ rs_on_ = true;
+ } else {
+ rs_dev_ = nullptr;
+ cudaGetLastError();
+ }
+ }
if (prof_on_) {
const size_t np = (size_t) g.n_layers * kProfPer + 4;
if (cudaMalloc((void**) &prof_, np * 8) != cudaSuccess) { prof_on_ = false; prof_ = nullptr; cudaGetLastError(); }
@@ -1959,6 +1969,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
rr_stats_ != nullptr ? rr_stats_ + (n <= 8 ? 4 : 0) : nullptr, cs);
} catch (const std::exception& e) { err = "verify route-resident: " + std::string(e.what()); return false; }
}
+ if (rs_on_ && G == 1 && n <= 8 && T <= 8)
+ strata::kernels::route_marginal_stats(ids_ + tb * K, n, (int) K, rs_dev_ + ((size_t) T * (size_t) g.n_layers + (size_t) l) * 8, cs);
if (ar_on()) {
resident_plan(ids_ + tb * K, n * (int) K, (int) K, hits_.d_res + l * g.n_expert, (int) g.n_expert,
hits_.cache_base, slot_off_d_, (long long) hits_.blob,
@@ -2318,6 +2330,66 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
return true;
}
+std::string Verifier::profile_by_t_report() {
+ if (!prof_on_) return std::string();
+ static const char* names[kProfPer] = {"-", "hc-read0", "qkv gemv", "conv", "ab", "z", "rec", "q8+kv-idx",
+ "k/v+norm-rope", "kv+idx append", "q+q-idx", "scores+topk", "kv-resolve",
+ "attention", "gate", "", "out-proj", "router+plan", "shared+quant",
+ "waitA", "VRAM hits", "waitB", "PCIe grp", "waitCPU", "copy+combine",
+ "(gap)", "head", "hc0 norm", "hc0 down", "hc0 up", "hc1 norm", "hc1 down", "hc1 up"};
+ std::string out;
+ char b[96];
+ for (int T = 1; T <= 8; ++T) {
+ if (prof_t_win_[T] == 0) continue;
+ double total = 0;
+ std::snprintf(b, sizeof b, "T%d (n=%lld):", T, (long long) prof_t_win_[T]);
+ out += b;
+ for (int k = 0; k < 2; ++k) {
+ out += k == 0 ? " GDN[" : " | QSA[";
+ for (int i = 0; i < kProfPer; ++i) {
+ if (prof_t_sum_[T][k][i] <= 0) continue;
+ total += prof_t_sum_[T][k][i];
+ std::snprintf(b, sizeof b, " %s %.3f", names[i], prof_t_sum_[T][k][i] / 1e6 / (double) prof_t_win_[T]);
+ out += b;
+ }
+ out += " ]";
+ }
+ std::snprintf(b, sizeof b, " | total %.2f ms\n", total / 1e6 / (double) prof_t_win_[T]);
+ out += b;
+ }
+ std::memset(prof_t_sum_, 0, sizeof prof_t_sum_);
+ std::memset(prof_t_win_, 0, sizeof prof_t_win_);
+ return out;
+}
+
+std::string Verifier::route_stats_report() {
+ if (!rs_on_ || rs_dev_ == nullptr || g_ == nullptr) return std::string();
+ const int64_t L = g_->n_layers;
+ std::vector<unsigned long long> h((size_t) 9 * (size_t) L * 8);
+ cudaMemcpy(h.data(), rs_dev_, h.size() * 8, cudaMemcpyDeviceToHost);
+ std::string out;
+ char b[160];
+ for (int T = 1; T <= 8; ++T) {
+ if (rs_win_[T] == 0) continue;
+ double row[8] = {};
+ for (int64_t l = 0; l < L; ++l)
+ for (int r = 0; r < 8; ++r) row[r] += (double) h[((size_t) T * (size_t) L + (size_t) l) * 8 + (size_t) r];
+ const double denom = (double) rs_win_[T] * (double) L;
+ double sum = 0;
+ for (int r = 0; r < T; ++r) sum += row[r] / denom;
+ std::snprintf(b, sizeof b, "T%d (n=%lld): distinct experts per layer %.2f; new per row:", T, (long long) rs_win_[T], sum);
+ out += b;
+ for (int r = 0; r < T; ++r) {
+ std::snprintf(b, sizeof b, " %.2f", row[r] / denom);
+ out += b;
+ }
+ out += "\n";
+ }
+ cudaMemset(rs_dev_, 0, h.size() * 8);
+ std::memset(rs_win_, 0, sizeof rs_win_);
+ return out;
+}
+
std::string Verifier::profile_report() {
if (!prof_on_ || prof_windows_ == 0) return std::string();
static const char* names[kProfPer] = {"-", "hc-read0", "q8+qkv/q-idx gemv", "conv", "ab", "z", "rec", "q8+kv-idx",
@@ -2651,6 +2723,7 @@ void Verifier::stage_inputs(int T, const int32_t* tokens, int64_t pos0) {
std::atomic_thread_fence(std::memory_order_seq_cst);
_mm_sfence();
last_t_ = T;
+ if (rs_on_ && T >= 1 && T <= 8) ++rs_win_[T];
last_pos0_ = pos0;
for (int t = 0; t < T; ++t) last_tokens_[t] = tokens[t];
staged_ = true;
@@ -2662,6 +2735,17 @@ void Verifier::collect_profile() {
}
void Verifier::accumulate_profile(const unsigned long long* stamps) {
+ double before[2][kProfPer];
+ std::memcpy(before, prof_sum_, sizeof before);
+ accumulate_profile_inner(stamps);
+ if (last_t_ >= 1 && last_t_ <= 8) {
+ for (int k = 0; k < 2; ++k)
+ for (int i = 0; i < kProfPer; ++i) prof_t_sum_[last_t_][k][i] += prof_sum_[k][i] - before[k][i];
+ ++prof_t_win_[last_t_];
+ }
+}
+
+void Verifier::accumulate_profile_inner(const unsigned long long* stamps) {
const ModelGeometry& g = *g_;
const int64_t L = g.n_layers;
auto at = [&](int64_t l, int i) { return stamps[(size_t) (l * kProfPer + i)]; };
diff --git a/src/kernels/cuda/verify_kernels.cu b/src/kernels/cuda/verify_kernels.cu
index dba0f1016..a9f570b9e 100644
--- a/src/kernels/cuda/verify_kernels.cu
+++ b/src/kernels/cuda/verify_kernels.cu
@@ -588,6 +588,21 @@ __global__ void hc_mean_streams_kernel(const float* __restrict__ R, float* __res
out[i] = s / (float) hc;
}
+__global__ void route_marginal_stats_kernel(const int32_t* __restrict__ ids, int n, int k, unsigned long long* acc) {
+ const int r = threadIdx.x;
+ if (r >= n) return;
+ int fresh = 0;
+ for (int j = 0; j < k; ++j) {
+ const int32_t e = ids[r * k + j];
+ bool seen = false;
+ for (int j2 = 0; j2 < j && !seen; ++j2) seen = ids[r * k + j2] == e;
+ for (int r2 = 0; r2 < r && !seen; ++r2)
+ for (int j2 = 0; j2 < k && !seen; ++j2) seen = ids[r2 * k + j2] == e;
+ if (!seen) ++fresh;
+ }
+ atomicAdd(acc + r, (unsigned long long) fresh);
+}
+
__global__ void copy_indexed_kernel(float* __restrict__ dst, const float* __restrict__ src, int64_t stride,
const int32_t* __restrict__ index, int64_t n) {
const int idx = *index;
@@ -1527,6 +1542,11 @@ void hc_mean_streams(const float* R, float* out, int64_t n_embd, int hc, void* s
check("hc_mean_streams");
}
+void route_marginal_stats(const int32_t* ids, int n, int k, unsigned long long* acc, void* stream) {
+ route_marginal_stats_kernel<<<1, 32, 0, (cudaStream_t) stream>>>(ids, n, k, acc);
+ check("route_marginal_stats");
+}
+
void copy_indexed(float* dst, const float* src, int64_t stride, const int32_t* index, int64_t n, void* stream) {
const unsigned blocks = (unsigned) ((n + 255) / 256 < 64 ? (n + 255) / 256 : 64);
copy_indexed_kernel<<<blocks, 256, 0, (cudaStream_t) stream>>>(dst, src, stride, index, n);
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index d7116d2df..cfb576651 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -16454,6 +16454,12 @@ int main(int argc, char** argv) {
for (size_t i = 1; i < t_cycle_ms.size(); ++i)
if (t_cycle_n[i] > 0) std::printf(" T%zu:%.2f(n=%lld)", i, t_cycle_ms[i] / (double) t_cycle_n[i], (long long) t_cycle_n[i]);
std::printf("\n");
+ { // STRATA_VERIFY_PROFILE / STRATA_ROUTE_STATS: the same split per window width (bench only)
+ const std::string pt = ver.profile_by_t_report();
+ if (!pt.empty()) std::printf("GPU stages per window width (ms):\n%s", pt.c_str());
+ const std::string rs = ver.route_stats_report();
+ if (!rs.empty()) std::printf("routed experts per window width:\n%s", rs.c_str());
+ }
}
std::printf("%-24s", "accepted per round");
for (size_t i = 0; i < accepted_hist.size(); ++i) std::printf(" %zu:%lld", i, (long long) accepted_hist[i]);