Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Verify and batch windows of up to 16 rows: HIP builds default to 16, narrow builds unchanged
Niko12219d3c8c3664f1
diff --git a/CMakeLists.txt b/CMakeLists.txtindex ee0599211..5946ce639 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -92,6 +92,31 @@ if(STRATA_HIP_GFX906)endif()option(STRATA_ENABLE_HIP "Build the HIP targets for wave32 AMD GPUs (gfx1100, gfx1201; gfx1012/gfx1030 unvalidated)" OFF)option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)+# The rows one verify / batch window holds (include/strata/kernels/window_limits.hpp): 8, the 0.2.7 window, or 16 (four pipelined+# groups of 16 slots = 64 slots on a four-card split, docs/BATCHING.md). Only the CUDA and HIP engines are built from+# these sources with a wide window; the SYCL and Metal builds never read this variable and keep 8. A window of up to+# 8 rows runs the kernels it always ran, whichever is chosen (the wide instantiations are extra code). "auto" = 16 for+# the HIP engines (a HIP code object loads on first use, so the extra kernels cost nothing until a wide window forms)+# and 8 for CUDA, which loads every kernel at start (CUDA_MODULE_LOADING=EAGER: the wide kernels would take VRAM from+# the expert cache of every card); -DSTRATA_WINDOW_MAX_T=16 builds the wide CUDA engine.+set(STRATA_WINDOW_MAX_T "auto" CACHE STRING "rows of a verify / batch window: 8, 16 or auto (HIP 16, CUDA 8)")+set_property(CACHE STRATA_WINDOW_MAX_T PROPERTY STRINGS auto 8 16)+if(NOT STRATA_WINDOW_MAX_T STREQUAL "auto" AND NOT STRATA_WINDOW_MAX_T STREQUAL "8" AND NOT STRATA_WINDOW_MAX_T STREQUAL "16")+ message(FATAL_ERROR "STRATA_WINDOW_MAX_T must be auto, 8 or 16 (got '${STRATA_WINDOW_MAX_T}')")+endif()+if(STRATA_ENABLE_HIP OR STRATA_ENABLE_CUDA)+ if(STRATA_WINDOW_MAX_T STREQUAL "auto")+ if(STRATA_ENABLE_HIP)+ set(_strata_window_max_t 16)+ else()+ set(_strata_window_max_t 8)+ endif()+ else()+ set(_strata_window_max_t ${STRATA_WINDOW_MAX_T})+ endif()+ add_compile_definitions(STRATA_VERIFY_MAX_T=${_strata_window_max_t})+ message(STATUS "Strata: verify / batch windows hold up to ${_strata_window_max_t} rows")+endif()# MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K (Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF# projections, STRATA_DENSE_MMQ). On by default since MODEL-UPDATE Phase 1: on CUDA they are their own library# (strata_kq, loaded only for a model with these formats: the engine loads every kernel of its own binary at start,@@ -1736,6 +1761,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)add_executable(adapt_decay_test src/program/adapt_decay_test.cpp)target_include_directories(adapt_decay_test PRIVATE include)add_test(NAME adapt_decay_test COMMAND adapt_decay_test)+ # the window limits (8 rows, or 16 in a wide CUDA / HIP build) and the pipelined-window probe (header-only, CPU)+ add_executable(window_limits_test src/core/window_limits_test.cpp)+ target_include_directories(window_limits_test PRIVATE include)+ add_test(NAME window_limits_test COMMAND window_limits_test)# which window-size graph is freed when the next one does not fit (#3702; header-only, CPU)add_executable(window_evict_test src/core/window_evict_test.cpp)target_include_directories(window_evict_test PRIVATE include)diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hppindex 78609014a..1b67dd3ca 100644--- a/include/strata/core/verify.hpp+++ b/include/strata/core/verify.hpp@@ -27,6 +27,7 @@#include "strata/core/layer.hpp"#include "strata/core/session.hpp"#include "strata/kernels/sampler.hpp"+#include "strata/kernels/window_limits.hpp"#include <cuda_runtime.h>@@ -404,13 +405,13 @@ private:std::vector<SessionState*> slots_;bool batch_rec_ = false; ///< record_window is capturing a batch windowint row_base_ = 0; ///< ... its hand-off rows start here (a pipeline group's own rows)- int brow_[8] = {}; ///< ... and row t is slot brow_[t]+ int brow_[strata::kernels::kWindowMaxT] = {}; ///< ... and row t is slot brow_[t]bool last_batch_ = false; ///< the last run was a batch window (set_plan_slot: one group)std::map<std::vector<int>, cudaGraphExec_t> exec_bm_, commit_bm_; ///< full row layout avoids slot-ID collisionsstd::map<std::vector<int>, uint64_t> bm_used_; ///< last use of each captured layout (LRU)uint64_t bm_tick_ = 0;size_t batch_graph_limit_ = 0; ///< 0: keep every captured batch graph (0.1.39); N: LRU-evict beyond N layouts- int last_rows_[8] = {}; ///< the slots of the last batch window's rows+ int last_rows_[strata::kernels::kWindowMaxT] = {}; ///< the slots of the last batch window's rowsstd::vector<int> bkey(const int* rows, int S, int hbase) const { // #871: the doorbell variant has its own graphsstd::vector<int> k = batch_key(rows, S, hbase);k.push_back(ar_off_ ? 1 : 0);@@ -439,14 +440,14 @@ private:std::chrono::steady_clock::duration(b_last_.load(std::memory_order_acquire)));}std::chrono::steady_clock::time_point ext_checked_{}; ///< batch_poll's last check with external serving- int32_t b_out_[8] = {};+ int32_t b_out_[strata::kernels::kWindowMaxT] = {};std::vector<strata::kernels::SamplerParams> slot_sp_;bool sample_rows(int S, std::string& err); ///< the sampled slots' rows of the last batch windowint32_t* h_commitb_ = nullptr; int32_t* m_commitb_ = nullptr; // per slot [1, 0, pos, -1 ..], stride 2 + max_tint32_t* commitb_ = nullptr;float* tail_snap_b_ = nullptr; ///< per (slot, QSA layer) indexer tail snapshotvoid* arena_b_ = nullptr;- int64_t last_pos_b_[8] = {};+ int64_t last_pos_b_[strata::kernels::kWindowMaxT] = {};bool capture_batch(const int* rows, int S, int hbase, std::string& err);// STRATA_PIPED_MTP (init_deferred_commit): the saved rows, laid out as the window scratch (global layer index) with// dc_rows_ rows per layer instead of max_t_; the slots' accepted prefixes (mapped and device, 2 + max_t_ per slot)@@ -534,7 +535,7 @@ private:unsigned* qcnt_ = nullptr; ///< S26 STRATA_QFUSE: the HC read's q8_1 group counters (n_embd / 32)unsigned long long* slot_off_d_ = nullptr; ///< E-6: the slot offsets on the deviceint64_t lb_ = 0, le_ = -1; ///< set_stage: the layers this verifier runs (-1: to the last)- int handoff_rows_ = 0; ///< set_handoff_rows (0: kVerifyMaxT)+ int handoff_rows_ = 0; ///< set_handoff_rows (0: kVerifyNarrowT)const float* hand_in_ = nullptr;float* hand_out_ = nullptr;uint32_t* hflag_in_h_ = nullptr; ///< set_handoff_flags: the incoming hand-off's ready word (host / device)@@ -573,7 +574,7 @@ private:std::vector<unsigned long long> prof_h_;double prof_sum_[2][kProfPer] = {}; // [GDN / QSA layers][stage]int64_t prof_windows_ = 0;- double prof_t_sum_[9][2][kProfPer] = {}; // the same per window size T (1..8), for profile_by_t_report+ double prof_t_sum_[9][2][kProfPer] = {}; // the same per window size T (1..8: wider windows are not profiled), for profile_by_t_reportint64_t prof_t_win_[9] = {};bool rs_on_ = false; // STRATA_ROUTE_STATSunsigned long long* rs_dev_ = nullptr; // [9 T][n_layers][8 rows]: experts of row r that no earlier row of the window routed to@@ -589,7 +590,7 @@ private:float* ple_val_ = nullptr;int last_t_ = 0;int64_t last_pos0_ = 0;- int32_t last_tokens_[8] = {};+ int32_t last_tokens_[strata::kernels::kWindowMaxT] = {};int64_t n_vocab_ = 0;cudaStream_t cs_ = nullptr;cudaStream_t sh_cs_ = nullptr;@@ -602,9 +603,10 @@ private:cudaEvent_t df_join_[2] = {};// [sel_long_][T]: 1 is the long-context QSA select (STRATA_SELECT_LONG=1: multi-CTA top-k, shuffle-free decode// scores; see run()); without it only [0] is ever captured- cudaGraphExec_t exec_[2][9] = {};- cudaGraphExec_t exec_nr_[2][9] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise- bool launched_[2][9] = {}, launched_nr_[2][9] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)+ // (per window size T = 1..kWindowMaxT: a solo window is at most 8 rows, but a merge window of a wide batch is not)+ cudaGraphExec_t exec_[2][strata::kernels::kWindowMaxT + 1] = {};+ cudaGraphExec_t exec_nr_[2][strata::kernels::kWindowMaxT + 1] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise+ bool launched_[2][strata::kernels::kWindowMaxT + 1] = {}, launched_nr_[2][strata::kernels::kWindowMaxT + 1] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)int sel_long_ = 0;// fork 34a35db (STRATA_SIDE_MIXER=1): the GDN gates and the QSA indexer on a branch of the window's graph beside the// mixer's projections, joined before the recurrence / the attention@@ -672,7 +674,7 @@ private:void set_plan_slot(int grp);bool split_ = false; // opt-in (--spec-split): exact but slower, see the overlap studystd::function<bool(cudaStream_t, std::string&)> pre_launch_; // set_pre_launch- int groups_[9] = {};+ int groups_[strata::kernels::kWindowMaxT + 1] = {};float* h_ymiss_ = nullptr; float* m_ymiss_ = nullptr; // T * k * n_embd// devicediff --git a/include/strata/kernels/fused_gr.hpp b/include/strata/kernels/fused_gr.hppindex 648380278..cc264fad7 100644--- a/include/strata/kernels/fused_gr.hpp+++ b/include/strata/kernels/fused_gr.hpp@@ -17,6 +17,8 @@// must be different buffers (every block reads the former while one block writes the latter).#pragma once+#include "strata/kernels/window_limits.hpp"+#include <cstdint>namespace strata::kernels {@@ -66,6 +68,9 @@ void fused_gr_quantize_q8_0(const uint16_t* bf16, uint8_t* out, int64_t n_blocks/// weight pointers and eps must be the same for every t); `xn_scratch` is n_tok * hc * n_embd floats. Every/// token's outputs are bitwise `fused_gr_read(a[t])`.constexpr int kFusedGrMaxT = 8;+/// A window of more than kFusedGrMaxT tokens is read in slices of at most kFusedGrMaxT (every token's outputs are+/// strictly its own, so the slices are bitwise the one read); `a` and `xn_scratch` hold up to kFusedGrWideT tokens.+constexpr int kFusedGrWideT = kWindowMaxT;/// Returns true when it also wrote the q8_1 images (every token's q8_mixed set and this read supports it).bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,unsigned long long* stamp_buf = nullptr, int stamp_i0 = 0);diff --git a/include/strata/kernels/s2_expert_grouped.hpp b/include/strata/kernels/s2_expert_grouped.hppindex af6f52f99..b11dec9e1 100644--- a/include/strata/kernels/s2_expert_grouped.hpp+++ b/include/strata/kernels/s2_expert_grouped.hpp@@ -23,6 +23,8 @@// can keep one captured graph per layer with the maximum grid and a device-side count.#pragma once+#include "strata/kernels/window_limits.hpp"+#include <cstdint>namespace strata::kernels {@@ -100,6 +102,29 @@ void moe_hit_add(float* parts, const float* hit_out, const int32_t* dst, const ivoid moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream);+#if STRATA_VERIFY_MAX_T > 8+/// The same for a window of 9..16 rows (CUDA / HIP engines built with a wide window, window_limits.hpp): `max_group` is the+/// most entries one group can hold = the window's rows (a token routes to an expert once). Up to 8 it is+/// `moe_grouped_s2` (one pass of each kernel); 9..16 adds a second pass over the entries from the 9th on.+void moe_grouped_s2_wide(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,+ int max_group);+#endif+/// What a verify window calls: `max_group` is the window's rows. A build with narrow windows (the default, and every SYCL / Metal+/// build, whose kernel libraries have only `moe_grouped_s2`) runs `moe_grouped_s2`: its windows hold at most 8 rows.+inline void moe_grouped_s2_window(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,+ int max_group) {+#if STRATA_VERIFY_MAX_T > 8+ moe_grouped_s2_wide(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out,+ stream, max_group);+#else+ (void) max_group;+ moe_grouped_s2(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream);+#endif+}/// Plan v0.3 P6: the groups for `moe_grouped_s2` when every expert id is resident at base + id * blob (the MTP/// layer's 512 experts): counts[0] groups, counts[1] entries; entry dst = routing index, tok = index / k_per_tok.void moe_group_resident(const int32_t* ids, int n, int k_per_tok, const uint8_t* base, int64_t blob,diff --git a/include/strata/kernels/verify_kernels.hpp b/include/strata/kernels/verify_kernels.hppindex b2ff4a0d4..4d8f2e590 100644--- a/include/strata/kernels/verify_kernels.hpp+++ b/include/strata/kernels/verify_kernels.hpp@@ -14,11 +14,15 @@// Everything that varies per window (token ids, `n_keep`) is read from DEVICE memory so the kernels can be captured.#pragma once+#include "strata/kernels/window_limits.hpp"+#include <cstdint>namespace strata::kernels {-inline constexpr int kVerifyMaxT = 8;+/// The most rows one verify / batch window holds (8; 16 in the CUDA / HIP builds, window_limits.hpp). A window of up to+/// kVerifyNarrowT rows runs the kernels it always ran.+inline constexpr int kVerifyMaxT = kWindowMaxT;/// For token t of T: conv over [history(3) | qkv_0 .. qkv_t] -> SiLU -> L2 norm of the q/k heads -> h[t]./// `history` is NOT written, but with `commit` (one token): it then keeps the token, as gdn_conv_commit. Bitwisediff --git a/include/strata/kernels/window_limits.hpp b/include/strata/kernels/window_limits.hppnew file mode 100644index 000000000..8520ec2e4--- /dev/null+++ b/include/strata/kernels/window_limits.hpp@@ -0,0 +1,27 @@+// include/strata/kernels/window_limits.hpp - how many rows one verify / batch window can hold.+//+// Until 0.2.7 a window held at most 8 rows. The CUDA and HIP engines now hold up to 16 (four pipelined groups of 16+// slots = 64 slots on a four-card split: docs/BATCHING.md), so every compile-time maximum that follows the window+// (the verifier's per-row arrays, the expert pool's per-token tables, the resident plan's one-thread-per-entry block)+// is derived from ONE number. It is a macro because the SYCL and Metal builds compile the same shared headers and+// must not change: they get the default, 8. The CUDA / HIP CMake sets STRATA_VERIFY_MAX_T from the cache variable+// STRATA_WINDOW_MAX_T (8 or 16; -DSTRATA_WINDOW_MAX_T=8 builds an engine whose windows are the 0.2.7 ones).+//+// The rule every user of the number follows: a window of up to kVerifyNarrowT rows runs EXACTLY the code it ran at 8+// (the same kernel instantiations, the same buffer sizes); the wider code is reached only by a window of 9..16 rows.+#pragma once++#ifndef STRATA_VERIFY_MAX_T+#define STRATA_VERIFY_MAX_T 8+#endif++namespace strata::kernels {++/// The window every 0.2.7 kernel was built for: windows of up to this many rows keep their instantiations.+inline constexpr int kVerifyNarrowT = 8;+/// The most rows one window can hold in this build (8 or 16).+inline constexpr int kWindowMaxT = STRATA_VERIFY_MAX_T;++static_assert(kWindowMaxT == 8 || kWindowMaxT == 16, "STRATA_VERIFY_MAX_T is 8 or 16 (the wide kernels are built for 16)");++} // namespace strata::kernelsdiff --git a/include/strata/program/pipe_probe.hpp b/include/strata/program/pipe_probe.hppnew file mode 100644index 000000000..9bd30b2ca--- /dev/null+++ b/include/strata/program/pipe_probe.hpp@@ -0,0 +1,148 @@+// include/strata/program/pipe_probe.hpp - STRATA_P4_TIMING / STRATA_P4_TRACE: where the wall time of a pipelined batch window goes.+//+// Opt-in measurement for the layer-split batch loop (generate.cpp, `pump`). Off, every hook is one predictable branch and+// reads no clock. On:+// STRATA_P4_TIMING=1 per stage: the window's time from launch to done, the cycle between launches, the gap between a+// window done and the next launch, the stage's busy share, the host launch cost; the pump's cost+// per tick, the verdict work per window, how many groups were in flight, the ticks a free stage had+// a group waiting or was starved. Printed to stderr with the batch summary line.+// STRATA_P4_TRACE=PATH one line per window event ("L stage time_us group" at a launch, "D stage time_us 0" when the stage+// is done), flushed on every line so a killed run keeps its trace. "1" or an empty value writes+// ./strata-p4-trace.csv.+// Written for the four-card R9700 investigation (docs/BATCHING.md): it found that the stages run 4 of 4 concurrently in the+// decode phase and that the serial admissions were the idle time.+#pragma once++#include <algorithm>+#include <chrono>+#include <cstdio>+#include <cstdlib>+#include <cstring>++namespace strata::program {++class PipeProbe {+public:+ using Clock = std::chrono::steady_clock;+ using Time = Clock::time_point;+ static constexpr int kMaxStages = 16;+ static constexpr int kMaxGroups = 64;++ PipeProbe(int n_stages, size_t n_groups)+ : n_(std::min(n_stages, kMaxStages)), n_groups_(n_groups) {+ timing_ = std::getenv("STRATA_P4_TIMING") != nullptr;+ const char* tr = std::getenv("STRATA_P4_TRACE");+ if (tr != nullptr) {+ const char* path = (tr[0] == '\0' || std::strcmp(tr, "1") == 0) ? "strata-p4-trace.csv" : tr;+ trace_ = std::fopen(path, "w");+ if (trace_ != nullptr) std::setvbuf(trace_, nullptr, _IONBF, 0);+ else std::fprintf(stderr, "strata p4: cannot write the trace %s\n", path);+ }+ }+ ~PipeProbe() { if (trace_ != nullptr) std::fclose(trace_); }+ PipeProbe(const PipeProbe&) = delete;+ PipeProbe& operator=(const PipeProbe&) = delete;++ bool on() const { return timing_ || trace_ != nullptr; }+ bool timing() const { return timing_; }+ Time now() const { return on() ? Clock::now() : Time{}; }++ /// The pump's cost: constructed at the top of `pump`, adds its life to the per-tick figure.+ struct PumpScope {+ PipeProbe* p = nullptr;+ Time t0{};+ explicit PumpScope(PipeProbe& probe) {+ ++probe.ticks_;+ if (probe.on()) { p = &probe; t0 = Clock::now(); }+ }+ ~PumpScope() { if (p != nullptr) p->pump_ms_ += ms(Clock::now() - t0); }+ };+ /// The verdict work of a window that reached the last stage.+ struct VerdictScope {+ PipeProbe* p = nullptr;+ Time t0{};+ explicit VerdictScope(PipeProbe& probe) {+ if (probe.timing_) { p = &probe; t0 = Clock::now(); ++probe.verdict_n_; }+ }+ ~VerdictScope() { if (p != nullptr) p->verdict_ms_ += ms(Clock::now() - t0); }+ };++ /// Per tick: stages with a window in flight, groups in flight.+ void busy(int stages, int groups) { busy_ += stages; inflight_ += groups; }+ /// A stage k >= 1 that is free: whether a group is waiting at it (else it is starved by the stage before).+ void free_stage(int k, bool group_waiting) {+ if (k < 0 || k >= n_) return;+ if (group_waiting) ++starve_idle_[k]; else ++starve_none_[k];+ }+ void span_start(Time t) { span_start_ = t; span_ms_ = 0; }+ void span_end(double wall_ms) { span_ms_ = wall_ms; }+ void polled(Time before) { if (on()) poll_ms_ += ms(Clock::now() - before); }+ /// Stage k reported its window done.+ void done(int k, int group) {+ if (!on() || k < 0 || k >= n_) return;+ const Time t = Clock::now();+ if (timing_ && launched_[k]) { win_ms_[k] += ms(t - at_[k]); ++win_n_[k]; }+ if (trace_ != nullptr) std::fprintf(trace_, "D %d %.3f 0\n", k, us(t));+ done_at_[k] = t;+ if (timing_ && group >= 0 && group < kMaxGroups && left_[group] != Time{}) { q_wait_ += ms(t - left_[group]); ++q_n_; }+ }+ /// Stage k launched `group`'s window; `before` is now() taken before the launch call.+ void launched(int k, int group, Time before) {+ if (!on() || k < 0 || k >= n_) return;+ const Time t = Clock::now();+ launch_ms_ += ms(t - before);+ at_[k] = t;+ launched_[k] = true;+ if (group >= 0 && group < kMaxGroups) left_[group] = t;+ if (trace_ != nullptr) std::fprintf(trace_, "L %d %.3f %d\n", k, us(t), group);+ if (timing_) {+ if (last_launch_[k] != Time{}) cycle_ms_[k] += ms(t - last_launch_[k]);+ if (done_at_[k] != Time{}) gap_ms_[k] += ms(t - done_at_[k]);+ last_launch_[k] = t;+ }+ }+ /// Print and reset (the batch summary line calls it when the loop goes idle).+ void flush() {+ if (!timing_) return;+ const double ticks = (double) std::max(1LL, ticks_);+ std::fprintf(stderr, "strata p4: pump %.3f ms/tick over %lld ticks; stage-busy %.2f of %d; group wait %.1f ms;\n",+ pump_ms_ / ticks, ticks_, busy_ / ticks, n_, q_n_ ? q_wait_ / (double) q_n_ : 0.0);+ for (int k = 0; k < n_; ++k) {+ const double n = win_n_[k] ? (double) win_n_[k] : 1.0;+ std::fprintf(stderr, "strata p4: s%d windows %lld win %.2f ms cycle %.2f ms gap %.2f ms busy %.0f%% launch %.2f ms\n",+ k, win_n_[k], win_ms_[k] / n, cycle_ms_[k] / n, gap_ms_[k] / n,+ 100.0 * win_ms_[k] / std::max(1e-9, span_ms_), launch_ms_ / n);+ }+ std::fprintf(stderr, "strata p4: verdict work %.2f ms/window; inflight groups avg %.2f of %zu\n",+ verdict_ms_ / std::max(1.0, (double) verdict_n_), inflight_ / ticks, n_groups_);+ for (int k = 1; k < n_; ++k)+ std::fprintf(stderr, "strata p4: s%d free ticks: with a group waiting %lld, starved %lld\n", k, starve_idle_[k],+ starve_none_[k]);+ pump_ms_ = poll_ms_ = launch_ms_ = verdict_ms_ = busy_ = inflight_ = q_wait_ = 0;+ ticks_ = verdict_n_ = q_n_ = 0;+ for (int k = 0; k < kMaxStages; ++k) {+ win_ms_[k] = cycle_ms_[k] = gap_ms_[k] = 0;+ win_n_[k] = starve_idle_[k] = starve_none_[k] = 0;+ }+ span_ms_ = 0;+ }++private:+ static double ms(Clock::duration d) { return std::chrono::duration<double, std::milli>(d).count(); }+ static double us(Time t) { return std::chrono::duration<double, std::micro>(t.time_since_epoch()).count(); }++ int n_ = 0;+ size_t n_groups_ = 0;+ bool timing_ = false;+ std::FILE* trace_ = nullptr;+ double pump_ms_ = 0, poll_ms_ = 0, launch_ms_ = 0, verdict_ms_ = 0, busy_ = 0, inflight_ = 0, q_wait_ = 0, span_ms_ = 0;+ long long ticks_ = 0, verdict_n_ = 0, q_n_ = 0;+ double win_ms_[kMaxStages] = {}, cycle_ms_[kMaxStages] = {}, gap_ms_[kMaxStages] = {};+ long long win_n_[kMaxStages] = {}, starve_idle_[kMaxStages] = {}, starve_none_[kMaxStages] = {};+ Time at_[kMaxStages] = {}, last_launch_[kMaxStages] = {}, done_at_[kMaxStages] = {};+ bool launched_[kMaxStages] = {};+ Time left_[kMaxGroups] = {};+ Time span_start_{};+};++} // namespace strata::programdiff --git a/src/core/expert_source.cpp b/src/core/expert_source.cppindex 1b5b11552..912e207ba 100644--- a/src/core/expert_source.cpp+++ b/src/core/expert_source.cpp@@ -7,6 +7,7 @@#include "strata/core/remote_experts.hpp"#include "strata/core/peer_experts.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/window_limits.hpp"#include "strata/artifact/gguf_reader.hpp"#include "strata/core/pinned.hpp"@@ -4786,10 +4787,10 @@ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, consnamespace {// the verify window's per-entry tables in `expert_pool_dispatch_multi` (`kind`, `distinct`, `first_of`)-// are fixed arrays of this many entries: MAXT tokens of the model's 10 routed experts must fit, and a larger k is-// refused at run time rather than written past them.-constexpr int64_t kMaxWindowEntries = 128;-static_assert(strata::kernels::cpu::MAXT * 10 <= kMaxWindowEntries, "a verify window's entries overflow the tables");+// are fixed arrays of this many entries: a window's rows (kWindowMaxT) of the model's 10 routed experts must fit, and a+// larger k is refused at run time rather than written past them. 128 for the 8-row window (as it was), 256 for 16.+constexpr int64_t kMaxWindowEntries = strata::kernels::kWindowMaxT <= 8 ? 128 : 256;+static_assert(strata::kernels::kWindowMaxT * 10 <= kMaxWindowEntries, "a verify window's entries overflow the tables");} // namespacenamespace {@@ -4813,7 +4814,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32float* out, strata::kernels::cpu::NativeArithmetic arithmetic) {using namespace strata::kernels::cpu;if (d.failed) return;- if (n_tok < 1 || n_tok > MAXT) {+ if (n_tok < 1 || n_tok > strata::kernels::kWindowMaxT) {d.failed = true;d.fail = "a verify window has more tokens than the multi-token expert kernel takes";d.fail_layer = d.layers;@@ -4832,12 +4833,13 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32d.fail_layer = d.layers;return;}- if ((int64_t) d.act_multi.size() < n_tok) d.act_multi.resize((size_t) MAXT);+ if ((int64_t) d.act_multi.size() < n_tok) d.act_multi.resize((size_t) strata::kernels::kWindowMaxT);const ExpertLayout& lay = expert_layout();const bool native = lay.native;- if (native && d.nact_multi.size() < (size_t) MAXT * kNativeActBytes) d.nact_multi.resize((size_t) MAXT * kNativeActBytes);+ if (native && d.nact_multi.size() < (size_t) strata::kernels::kWindowMaxT * kNativeActBytes)+ d.nact_multi.resize((size_t) strata::kernels::kWindowMaxT * kNativeActBytes);if (d.job_of.size() != (size_t) d.n_expert) d.job_of.assign((size_t) d.n_expert, (int16_t) -1);- if (d.jobs_multi.size() < (size_t) (n_tok * k)) d.jobs_multi.resize((size_t) (MAXT * k));+ if (d.jobs_multi.size() < (size_t) (n_tok * k)) d.jobs_multi.resize((size_t) (strata::kernels::kWindowMaxT * k));static const bool ptrace = std::getenv("STRATA_POOL_TRACE") != nullptr;auto pt = [&](const char* what, long long a = -1) {if (ptrace) { std::fprintf(stderr, "pool trace: layer %lld %s %lld\n", (long long) d.layers, what, a); std::fflush(stderr); }@@ -5151,6 +5153,17 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32++d.cold_zc;}}+ if (d.jobs_multi[(size_t) jo].nt == MAXT) {+ // a window of 9..16 rows routes more than MAXT tokens to this expert: the kernels and the pool's scratch+ // take MAXT tokens a job, so the rest are a further job over the same blob (each token's rows are its own,+ // so the split changes no value; the blob is read once more)+ const int16_t prev = jo;+ jo = (int16_t) njobs++;+ ExpertJobMulti& nj = d.jobs_multi[(size_t) jo];+ nj.blob = d.jobs_multi[(size_t) prev].blob;+ nj.fmt = d.jobs_multi[(size_t) prev].fmt;+ nj.nt = 0;+ }ExpertJobMulti& jb = d.jobs_multi[(size_t) jo];jb.act[jb.nt] = &d.act_multi[(size_t) t];jb.nact[jb.nt] = native ? d.nact_multi.data() + (size_t) t * kNativeActBytes : nullptr;diff --git a/src/core/mtp.cpp b/src/core/mtp.cppindex 64ad715ad..c7f31db14 100644--- a/src/core/mtp.cpp+++ b/src/core/mtp.cpp@@ -277,7 +277,8 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Sessionif (ple_ss_ == nullptr) ple_ss_ = &ss;max_t_ = max_t;rt_dir_ = rt_dir;- if (max_t < 1 || max_t > strata::kernels::kVerifyMaxT) { err = "mtp: max_t out of range"; return false; }+ // (the draft layer's kernels - the resident grouping, the grouped experts, the fused reads - are built for the narrow window)+ if (max_t < 1 || max_t > strata::kernels::kVerifyNarrowT) { err = "mtp: max_t out of range"; return false; }if (shared != nullptr) {if (shared->device_ != device_ || shared->g_ != &g || shared->dense_ == nullptr ||shared->experts_ == nullptr || shared->rt_dir_ != rt_dir) {diff --git a/src/core/peer_experts.cpp b/src/core/peer_experts.cppindex 0320b75e5..4f589b788 100644--- a/src/core/peer_experts.cpp+++ b/src/core/peer_experts.cpp@@ -8,6 +8,7 @@#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/quantize_act.hpp"#include "strata/kernels/s2_expert_grouped.hpp"+#include "strata/kernels/window_limits.hpp"#include <algorithm>#include <chrono>@@ -26,7 +27,7 @@ bool peer_portable() { return g_peer_portable; }namespace {constexpr int64_t H = strata::kernels::cpu::H;-constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10; // entries per layer: MAXT tokens x top-10+constexpr int64_t CAP = strata::kernels::kWindowMaxT * 10; // entries per layer: a window's rows x top-10struct Meta { // one block, uploaded with one copy per layerunsigned long long ptr[CAP];diff --git a/src/core/remote_expert_opt.cu b/src/core/remote_expert_opt.cuindex c92542f9e..531008e9e 100644--- a/src/core/remote_expert_opt.cu+++ b/src/core/remote_expert_opt.cu@@ -4,6 +4,7 @@#include "strata/kernels/cpu/expert_layout.hpp"#include "strata/kernels/elementwise.hpp"#include "strata/kernels/verify_kernels.hpp"+#include "strata/kernels/window_limits.hpp"#include <algorithm>#include <cstring>@@ -11,7 +12,7 @@namespace strata::core {namespace {constexpr int64_t H = strata::kernels::cpu::H;-constexpr int K = 10, MAXT = strata::kernels::cpu::MAXT, CAP = MAXT * K;+constexpr int K = 10, MAXT = strata::kernels::kWindowMaxT, CAP = MAXT * K; // (a window's rows)struct ReduceMeta { int32_t row[CAP]; float weight[CAP]; };bool check(cudaError_t status, std::string& err) {diff --git a/src/core/remote_experts.cpp b/src/core/remote_experts.cppindex 6e6849595..c2791540c 100644--- a/src/core/remote_experts.cpp+++ b/src/core/remote_experts.cpp@@ -7,6 +7,7 @@#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/quantize_act.hpp"#include "strata/kernels/s2_expert_grouped.hpp"+#include "strata/kernels/window_limits.hpp"#include <algorithm>#include <chrono>@@ -18,7 +19,7 @@ namespace strata::core {namespace {constexpr int64_t H = strata::kernels::cpu::H;constexpr int64_t FF = strata::kernels::cpu::FF;-constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10;+constexpr int64_t CAP = strata::kernels::kWindowMaxT * 10; // a window's rows x top-10// Small enough to copy as one pinned buffer per layer. The kernels read the// individual arrays through pointers into the same device allocation.@@ -268,7 +269,7 @@ bool RemoteExperts::begin(int64_t layer, const float* x, const int32_t* ids, intstruct Timer { double& acc; std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now();~Timer() { acc += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count(); } } timer{ms_begin_};const int64_t n = n_tok * k;- if (n <= 0 || n > CAP || n_tok > strata::kernels::cpu::MAXT || k != 10 || layer < 0 ||+ if (n <= 0 || n > CAP || n_tok > strata::kernels::kWindowMaxT || k != 10 || layer < 0 ||(size_t) layer >= layers_present_.size() || device_ < 0) {err = "CUDA" + std::to_string(device_) + " experts: invalid layer, routing width or window size";return false;@@ -342,8 +343,9 @@ bool RemoteExperts::begin(int64_t layer, const float* x, const int32_t* ids, intgroups_, (int64_t) dst_.size(), d_q8_, d_scratch_, zero_copy_ && !reduce ? z_out_ : d_out_, s);} else {strata::kernels::quantize_q8_0_scaled(zero_copy_ ? z_x_ : d_x_, d_q8_, d_scales_, n_tok * H, s);- strata::kernels::moe_grouped_s2(d_ptr_, d_start_, d_count_, d_dst_, d_tok_,- groups_, (int64_t) dst_.size(), d_q8_, d_scales_, d_scratch_, zero_copy_ && !reduce ? z_out_ : d_out_, s);+ strata::kernels::moe_grouped_s2_window(d_ptr_, d_start_, d_count_, d_dst_, d_tok_,+ groups_, (int64_t) dst_.size(), d_q8_, d_scales_, d_scratch_, zero_copy_ && !reduce ? z_out_ : d_out_, s,+ (int) n_tok);}const uint64_t compact_bytes = (uint64_t) (reduce ? n_tok : dst_.size()) * H * sizeof(float);if (reduce) {diff --git a/src/core/verify.cpp b/src/core/verify.cppindex c8eef3f30..85b62e4ff 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -135,7 +135,7 @@ struct RouteResidentCfg {RouteResidentCfg& c = route_resident_cfg();for (const auto& kv : c.blocks) {if (kv.second == nullptr) continue;- unsigned long long h[8] = {};+ unsigned long long h[8] = {}; // (the counter block allocated above: 8 words)if (cudaSetDevice(kv.first) != cudaSuccess || cudaDeviceSynchronize() != cudaSuccess ||cudaMemcpy(h, kv.second, sizeof(h), cudaMemcpyDeviceToHost) != cudaSuccess) {(void) cudaGetLastError();@@ -824,7 +824,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&return strata::metal_engine::window_init(metal_, g, ss, max_t, ss.qsa_states[ss.qsa_primary()].max_cells, &n_vocab_,err);#endif- if (max_t < 2 || max_t > strata::kernels::kVerifyMaxT || max_t > strata::kernels::cpu::MAXT) {+ if (max_t < 2 || max_t > strata::kernels::kVerifyMaxT) { // (the CPU experts take a wider window as jobs of at most cpu::MAXT tokens)err = "verify: the window must hold 2.." + std::to_string(strata::kernels::kVerifyMaxT) + " tokens";return false;}@@ -1476,7 +1476,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {pending = false;}auto gr_read_group = [&](int half, bool apply, float* inj_prev, float* inj_out) {- FusedGrArgs fa[kFusedGrMaxT];+ FusedGrArgs fa[kFusedGrWideT];for (int t = tb; t < te; ++t) {FusedGrArgs& a = fa[t - tb];a.R = Rt(t); a.R_out = Rt(t); a.apply = apply;@@ -2255,8 +2255,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {native_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, dst_buf, cs, gy);} else {- moe_grouped_s2(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,- hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, dst_buf, cs);+ moe_grouped_s2_window(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,+ hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, dst_buf, cs, n); // (a group holds <= n entries)}};bool hits_in_combine = false;@@ -2413,7 +2413,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {// (no pending write, no inject)const bool mix_q8 = hd->hc_q8 != nullptr && hu->hc_q8 != nullptr && head_ != nullptr && head_->loaded();if (mix_q8) {- FusedGrArgs fa[kFusedGrMaxT];+ FusedGrArgs fa[kFusedGrWideT];for (int t = 0; t < T; ++t) {FusedGrArgs& a = fa[t];a.R = Rt(t); a.R_out = Rt(t); a.apply = false;@@ -2426,7 +2426,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {fused_gr_read_multi(fa, T, xn_, cs);}else if (fuse_head_gr) {- FusedGrArgs fa[kFusedGrMaxT];+ FusedGrArgs fa[kFusedGrWideT];for (int t = 0; t < T; ++t) {FusedGrArgs& a = fa[t];a.R = Rt(t); a.R_out = Rt(t); a.apply = true;@@ -2449,7 +2449,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {err = "verify: the output_hc_* weights have the wrong engine forms";return false;}- FusedGrArgs fa[kFusedGrMaxT];+ FusedGrArgs fa[kFusedGrWideT];for (int t = 0; t < T; ++t) {fa[t].R = Rt(t); fa[t].R_out = Rt(t); fa[t].apply = false;fa[t].w_norm = (const float*) hn->data; fa[t].w_down = (const uint16_t*) hd->data;@@ -4157,9 +4157,10 @@ bool Verifier::evict_batch_graph(const std::vector<int>& keep, bool& evicted, stbool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {evicted = false;if (no_window_eviction_ || keep_t < 1) return true;- bool have[9] = {};- for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[sel_long_][t] : exec_[sel_long_][t]) != nullptr;- const int victim = strata::core::pick_window_victim(have, std::min(max_t_, 8), keep_t, prelaunched_);+ bool have[strata::kernels::kWindowMaxT + 1] = {};+ const int top = std::min<int>(max_t_, strata::kernels::kWindowMaxT);+ for (int t = 1; t <= top; ++t) have[t] = (ar_off_ ? exec_nr_[sel_long_][t] : exec_[sel_long_][t]) != nullptr;+ const int victim = strata::core::pick_window_victim(have, top, keep_t, prelaunched_);if (victim == 0) return true;if (cudaStreamSynchronize(cs_) != cudaSuccess) {err = "verify: synchronizing before window graph eviction failed";@@ -4317,9 +4318,9 @@ bool Verifier::stage_batch(const int* rows, int S, int hbase, const int32_t* tokstd::string& err) {using namespace strata::kernels;// the row layout: a slot may own a contiguous group of rows (--batch-mtp), also across the stages of a layer split,- // whose hand-off buffers hold kVerifyMaxT rows, or set_handoff_rows' (a stage that hands rows on or takes them)+ // whose hand-off buffers hold kVerifyNarrowT rows, or set_handoff_rows' (a stage that hands rows on or takes them)if (const char* why = batch_rows_error(rows, S, pos, (int) slots_.size(), max_t_, hbase,- next_ != nullptr || lb_ > 0, handoff_rows_ > 0 ? handoff_rows_ : kVerifyMaxT)) {+ next_ != nullptr || lb_ > 0, handoff_rows_ > 0 ? handoff_rows_ : kVerifyNarrowT)) {err = why;return false;}@@ -4394,8 +4395,8 @@ bool Verifier::stage_batch(const int* rows, int S, int hbase, const int32_t* tokbool Verifier::run_slots(int S, const int32_t* tokens, const int64_t* pos, PoolMultiFn pool, void* user, int32_t* out,std::string& err) {- int rows[8] = {};- for (int t = 0; t < S && t < 8; ++t) rows[t] = t;+ int rows[strata::kernels::kVerifyMaxT] = {};+ for (int t = 0; t < S && t < strata::kernels::kVerifyMaxT; ++t) rows[t] = t;return run_slot_rows(rows, S, tokens, pos, pool, user, out, err);}@@ -4571,8 +4572,8 @@ bool Verifier::sample_rows(int S, std::string& err) {bool Verifier::batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err) {const OnDevice on_device(device_);if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; }- int rows[8] = {};- for (int t = 0; t < S && t < 8; ++t) rows[t] = base + t;+ int rows[strata::kernels::kVerifyMaxT] = {};+ for (int t = 0; t < S && t < strata::kernels::kVerifyMaxT; ++t) rows[t] = base + t;if (pre_launch_ && !pre_launch_(cs_, err)) return false;if (!stage_batch(rows, S, base, tokens, pos, err)) return false;cudaError_t le = cudaGraphLaunch(exec_bm_[bkey(rows, S, base)], cs_);diff --git a/src/core/window_limits_test.cpp b/src/core/window_limits_test.cppnew file mode 100644index 000000000..3c89cc400--- /dev/null+++ b/src/core/window_limits_test.cpp@@ -0,0 +1,64 @@+// The window limits (include/strata/kernels/window_limits.hpp) and the pipelined-window probe+// (include/strata/program/pipe_probe.hpp): CPU only, header-only.+#include "strata/kernels/cpu/expert.hpp"+#include "strata/kernels/fused_gr.hpp"+#include "strata/kernels/verify_kernels.hpp"+#include "strata/program/pipe_probe.hpp"++#include <cstdio>+#include <cstdlib>++namespace {+int checks = 0;+void check(bool ok, const char* label) {+ ++checks;+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }+}+} // namespace++int main() {+ using namespace strata::kernels;+ check(kVerifyNarrowT == 8, "the narrow window is the 0.2.7 one: 8 rows");+ check(kWindowMaxT == 8 || kWindowMaxT == 16, "a window holds 8 or 16 rows");+ check(kVerifyMaxT == kWindowMaxT, "kVerifyMaxT is the window limit");+ check(kVerifyMaxT >= kVerifyNarrowT, "the wide window is not narrower than the narrow one");+ check(cpu::MAXT == 8, "the CPU kernels' tokens per job stay 8 (a wider window is split into jobs)");+ check(kFusedGrMaxT == 8, "the hyper-connection kernels are built for 8 tokens (a wider window is read in slices)");+ check(kFusedGrWideT == kWindowMaxT, "the caller's token arrays follow the window");+ check(kVerifyMaxT * 10 <= 256, "the resident plan's block holds a full window of 10 routed experts a row");++ // the probe: off without the environment (no clock read), and a full cycle of events does not fail when it is on+ unsetenv("STRATA_P4_TIMING");+ unsetenv("STRATA_P4_TRACE");+ {+ strata::program::PipeProbe off(4, 4);+ check(!off.on() && !off.timing(), "the probe is off by default");+ check(off.now() == strata::program::PipeProbe::Time{}, "an off probe reads no clock");+ { strata::program::PipeProbe::PumpScope s(off); }+ off.launched(0, 1, off.now());+ off.done(0, 1);+ off.flush();+ }+ setenv("STRATA_P4_TIMING", "1", 1);+ {+ strata::program::PipeProbe on(4, 4);+ check(on.timing(), "STRATA_P4_TIMING turns the timing on");+ on.span_start(on.now());+ for (int w = 0; w < 3; ++w)+ for (int k = 0; k < 4; ++k) {+ strata::program::PipeProbe::PumpScope s(on);+ on.busy(2, 3);+ on.free_stage(1, k % 2 == 0);+ const auto before = on.now();+ on.launched(k, w, before);+ on.polled(before);+ on.done(k, w);+ strata::program::PipeProbe::VerdictScope v(on);+ }+ on.span_end(10.0);+ on.flush();+ }+ unsetenv("STRATA_P4_TIMING");+ std::printf("window_limits_test: %d checks passed (window %d rows)\n", checks, (int) kVerifyMaxT);+ return 0;+}diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cuindex 12c74a5dc..24b001431 100644--- a/src/kernels/cuda/fused_gr.cu+++ b/src/kernels/cuda/fused_gr.cu@@ -2330,8 +2330,10 @@ static bool gr_v3_default_on(int dev) {#endif}-bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,- int stamp_i0) {+// One read of at most kFusedGrMaxT tokens: the kernels below are built for that many (the instantiations, the shared+// memory, the by-value launch argument), whatever the window is.+static bool fused_gr_read_multi_slice(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,+ unsigned long long* stamp_buf, int stamp_i0) {if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {std::fprintf(stderr, "fused_gr_read_multi: invalid arguments\n");std::exit(1);@@ -2466,6 +2468,30 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voireturn q8;}+// The multi read for a window of up to kFusedGrWideT tokens. Up to kFusedGrMaxT tokens it is the read above untouched+// (the same kernels, the same bits). A wider window (9..16 rows: the batch groups of a four-card split) is read in+// slices of at most kFusedGrMaxT tokens, in order on the stream. Every token's outputs are strictly its own (no value+// crosses tokens), so the slices are bitwise the one wide read; the cost is the hyper-connection weights read once per+// slice. The STRATA_QFUSE counters belong to token 0: each slice's first token carries them (the kernels zero them+// again for the next launch); the profile stamps are the first slice's.+bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,+ int stamp_i0) {+ if (n_tok <= kFusedGrMaxT) return fused_gr_read_multi_slice(a, n_tok, xn_scratch, stream, stamp_buf, stamp_i0);+ if (n_tok > kFusedGrWideT || a == nullptr || xn_scratch == nullptr) {+ std::fprintf(stderr, "fused_gr_read_multi: invalid arguments (%d tokens)\n", n_tok);+ std::exit(1);+ }+ bool q8 = false;+ for (int c0 = 0; c0 < n_tok; c0 += kFusedGrMaxT) {+ const int ct = n_tok - c0 < kFusedGrMaxT ? n_tok - c0 : kFusedGrMaxT;+ FusedGrArgs s[kFusedGrMaxT];+ for (int k = 0; k < ct; ++k) s[k] = a[c0 + k];+ s[0].q8_cnt = a[0].q8_cnt;+ q8 = fused_gr_read_multi_slice(s, ct, xn_scratch + (size_t) c0 * D, stream, c0 == 0 ? stamp_buf : nullptr, stamp_i0);+ }+ return q8;+}+// STRATA_HC_Q8 on a bf16 pack: one warp a Q8_0 block, ggml's reference rounding (d = amax / 127 as fp16, q = round(x * (1 / d)))__global__ void __launch_bounds__(256) gr_quantize_q8_0_kernel(const uint16_t* __restrict__ w, uint8_t* __restrict__ out, int64_t nb) {const int64_t b = (int64_t) blockIdx.x * 8 + (threadIdx.x >> 5);diff --git a/src/kernels/cuda/native_bf16.cu b/src/kernels/cuda/native_bf16.cuindex 1f4b98acc..ff6c439ef 100644--- a/src/kernels/cuda/native_bf16.cu+++ b/src/kernels/cuda/native_bf16.cu@@ -1,4 +1,5 @@#include "strata/kernels/bf16_gemv.hpp"+#include "strata/kernels/window_limits.hpp"#include "s26_tsum.cuh"#include "strata/kernels/bf16_bits.hpp"#include "strata/kernels/pdl.hpp"@@ -295,12 +296,12 @@ bool bf16_gemv_fp32_mmvf_multi_aux(const float* x, int64_t ldx, const uint16_t*void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,int64_t n_in, int64_t n_out, int n_tok, void* stream) {if (n_tok == 1 && ldy >= n_out) { bf16_gemv_fp32_mmvf(x, w, y, n_in, n_out, stream); return; }- if (n_tok < 1 || n_tok > 8 || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 || x == nullptr ||- w == nullptr || y == nullptr || (reinterpret_cast<uintptr_t>(x) & 7u) != 0)- throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows, even n_in/ldx, aligned pointers");+ if (n_tok < 1 || n_tok > STRATA_VERIFY_MAX_T || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 ||+ x == nullptr || w == nullptr || y == nullptr || (reinterpret_cast<uintptr_t>(x) & 7u) != 0)+ throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows (16 in a wide build), even n_in/ldx, aligned pointers");const cudaStream_t st = (cudaStream_t) stream;static const bool rows = [] { const char* v = std::getenv("STRATA_MMVF_ROWS"); return v && v[0] == '1'; }();- if (rows && n_out >= 64) {+ if (rows && n_out >= 64 && n_tok <= 8) { // (the opt-in rows kernels hold 8 columns: a wider window takes the switch below)const int ni = (int) n_in, no = (int) n_out;switch (mmvf_block_size(n_in)) {case 32: launch_rows<32>(x, ldx, w, y, ldy, ni, no, n_tok, st); break;@@ -317,8 +318,19 @@ void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, fthrow std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result));return;}+ // 9..16 rows (wide builds only): one exact instantiation per row count, each row's dot independent of the others. 7 and 8+ // rows stay with the default case below, the instantiation they always had.+#if STRATA_VERIFY_MAX_T > 8+#define STRATA_MMVF_W(N, T) case T: launch_pdl(bf16_f32_mmvf_multi_kernel<N, T, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break;+#define STRATA_MMVF_WIDE(N) \+ STRATA_MMVF_W(N, 9) STRATA_MMVF_W(N, 10) STRATA_MMVF_W(N, 11) STRATA_MMVF_W(N, 12) \+ STRATA_MMVF_W(N, 13) STRATA_MMVF_W(N, 14) STRATA_MMVF_W(N, 15) STRATA_MMVF_W(N, 16)+#else+#define STRATA_MMVF_WIDE(N)+#endif#define STRATA_MMVF_M(N) case N: \switch (n_tok) { \+ STRATA_MMVF_WIDE(N) \case 1: launch_pdl(bf16_f32_mmvf_multi_kernel<N, 1, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \case 2: launch_pdl(bf16_f32_mmvf_multi_kernel<N, 2, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \case 3: launch_pdl(bf16_f32_mmvf_multi_kernel<N, 3, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \@@ -332,6 +344,10 @@ void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, fSTRATA_MMVF_M(160); STRATA_MMVF_M(192); STRATA_MMVF_M(224); STRATA_MMVF_M(256);}#undef STRATA_MMVF_M+#undef STRATA_MMVF_WIDE+#ifdef STRATA_MMVF_W+#undef STRATA_MMVF_W+#endifconst cudaError_t result = cudaGetLastError();if (result != cudaSuccess)throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result));diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cuindex c806e1630..0f2bdf7dc 100644--- a/src/kernels/cuda/native_mmvq.cu+++ b/src/kernels/cuda/native_mmvq.cu@@ -888,7 +888,11 @@ __global__ void native_small_mmvq_kernel(const Weight* __restrict__ w,// row) value is accumulated over kbx in the same order, summed across warps in the same order and reduced with// the same warp tree as the ncols = 1 kernel, so every column is BITWISE equal to a single-column call on that// column (checked by bench/micro/native_mmvq_multi.cpp). The ncols = 1 kernels are untouched.-constexpr int MAX_NCOLS = 8;+// Columns one launch takes: 8, the 0.2.7 limit, or 16 where windows are wide (window_limits.hpp: the cases 9..16 are+// compiled only then, so an engine built with STRATA_WINDOW_MAX_T=8 has exactly the kernels it had). A window has one+// column per row. The wide launches read every weight once for all their columns; running 16 columns as two 8-column+// passes (native_mmvq below does it for more than MAX_NCOLS) reads the dense weights twice and measured slower than 8 rows.+constexpr int MAX_NCOLS = STRATA_VERIFY_MAX_T > 8 ? 16 : 8;// Each format splits its dot product into `load` (everything that depends only on the weight block: codes,// unpacked scales, block scale) and `apply` (the activation loads and the original *_impl expression). `load` runs@@ -1583,7 +1587,17 @@ void launch_multi(const void* weights, const void* x_q8_1, float* y, int n_in, icase 6: launch_multi_n<F, 6>(weights, x_q8_1, y, n_in, n_out, s); break;case 7: launch_multi_n<F, 7>(weights, x_q8_1, y, n_in, n_out, s); break;case 8: launch_multi_n<F, 8>(weights, x_q8_1, y, n_in, n_out, s); break;- default: throw std::invalid_argument("native MMVQ multi-column launch requires 2 <= ncols <= 8");+#if STRATA_VERIFY_MAX_T > 8+ case 9: launch_multi_n<F, 9>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 10: launch_multi_n<F, 10>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 11: launch_multi_n<F, 11>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 12: launch_multi_n<F, 12>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 13: launch_multi_n<F, 13>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 14: launch_multi_n<F, 14>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 15: launch_multi_n<F, 15>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 16: launch_multi_n<F, 16>(weights, x_q8_1, y, n_in, n_out, s); break;+#endif+ default: throw std::invalid_argument("native MMVQ multi-column launch requires 2 <= ncols <= MAX_NCOLS");}}@@ -2307,7 +2321,7 @@ void validate_shape(int n_in, int ncols, int block_elems = Q8K) {if (n_in <= 0 || n_in % block_elems != 0) {throw std::invalid_argument("native MMVQ requires n_in > 0 and divisible by its block element count");}- if (ncols < 1 || ncols > MAX_NCOLS) throw std::invalid_argument("native MMVQ requires 1 <= ncols <= 8");+ if (ncols < 1 || ncols > MAX_NCOLS) throw std::invalid_argument("native MMVQ requires 1 <= ncols <= MAX_NCOLS");}void validate_pointer(const void* p) {if (!p || reinterpret_cast<std::uintptr_t>(p) % 4 != 0) {@@ -2386,7 +2400,17 @@ void wave_launch(const void* weights, const void* x_q8_1, float* y, int n_in, incase 6: wave_launch_n<F, 6>(weights, x_q8_1, y, n_in, n_out, s); break;case 7: wave_launch_n<F, 7>(weights, x_q8_1, y, n_in, n_out, s); break;case 8: wave_launch_n<F, 8>(weights, x_q8_1, y, n_in, n_out, s); break;- default: throw std::invalid_argument("native MMVQ (wave) requires 1 <= ncols <= 8");+#if STRATA_VERIFY_MAX_T > 8+ case 9: wave_launch_n<F, 9>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 10: wave_launch_n<F, 10>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 11: wave_launch_n<F, 11>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 12: wave_launch_n<F, 12>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 13: wave_launch_n<F, 13>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 14: wave_launch_n<F, 14>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 15: wave_launch_n<F, 15>(weights, x_q8_1, y, n_in, n_out, s); break;+ case 16: wave_launch_n<F, 16>(weights, x_q8_1, y, n_in, n_out, s); break;+#endif+ default: throw std::invalid_argument("native MMVQ (wave) requires 1 <= ncols <= MAX_NCOLS");}}#define STRATA_WAVE_MMVQ(...) \@@ -2569,8 +2593,16 @@ bool q8_packed_mmvq(const void* weights, const void* x_q8_1, float* y, int n_in,validate_pointer(x_q8_1);validate_pointer(y);validate_stream(stream);- if (q8sm60::launch(found->second.qs, found->second.d, static_cast<const Q81Block*>(x_q8_1), y, n_in,- n_out, ncols, static_cast<cudaStream_t>(stream))) {+ // (the kernel takes at most q8sm60::MAX_COLS columns; a 9..16-row window is run in passes, each column bitwise+ // the one-column result, as native_mmvq does for more than MAX_NCOLS)+ bool all = true;+ for (int j = 0; all && j < ncols; j += q8sm60::MAX_COLS) {+ const int nj = ncols - j < q8sm60::MAX_COLS ? ncols - j : q8sm60::MAX_COLS;+ all = q8sm60::launch(found->second.qs, found->second.d,+ static_cast<const Q81Block*>(x_q8_1) + std::size_t(j) * std::size_t(n_in / Q8K),+ y + std::size_t(j) * std::size_t(n_out), n_in, n_out, nj, static_cast<cudaStream_t>(stream));+ }+ if (all) {launch_check();return true;}@@ -2597,7 +2629,17 @@ bool q8_packed_mmvq(const void* weights, const void* x_q8_1, float* y, int n_in,case 6: q8_packed_launch<6>(found->second, x, y, s); break;case 7: q8_packed_launch<7>(found->second, x, y, s); break;case 8: q8_packed_launch<8>(found->second, x, y, s); break;- default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= 8");+#if STRATA_VERIFY_MAX_T > 8+ case 9: q8_packed_launch<9>(found->second, x, y, s); break;+ case 10: q8_packed_launch<10>(found->second, x, y, s); break;+ case 11: q8_packed_launch<11>(found->second, x, y, s); break;+ case 12: q8_packed_launch<12>(found->second, x, y, s); break;+ case 13: q8_packed_launch<13>(found->second, x, y, s); break;+ case 14: q8_packed_launch<14>(found->second, x, y, s); break;+ case 15: q8_packed_launch<15>(found->second, x, y, s); break;+ case 16: q8_packed_launch<16>(found->second, x, y, s); break;+#endif+ default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= MAX_NCOLS");}launch_check();return true;@@ -2732,7 +2774,17 @@ bool q6_packed_mmvq(const void* weights, const void* x_q8_1, float* y, int n_in,case 6: q6_packed_launch<6>(found->second, x, y, s); break;case 7: q6_packed_launch<7>(found->second, x, y, s); break;case 8: q6_packed_launch<8>(found->second, x, y, s); break;- default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= 8");+#if STRATA_VERIFY_MAX_T > 8+ case 9: q6_packed_launch<9>(found->second, x, y, s); break;+ case 10: q6_packed_launch<10>(found->second, x, y, s); break;+ case 11: q6_packed_launch<11>(found->second, x, y, s); break;+ case 12: q6_packed_launch<12>(found->second, x, y, s); break;+ case 13: q6_packed_launch<13>(found->second, x, y, s); break;+ case 14: q6_packed_launch<14>(found->second, x, y, s); break;+ case 15: q6_packed_launch<15>(found->second, x, y, s); break;+ case 16: q6_packed_launch<16>(found->second, x, y, s); break;+#endif+ default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= MAX_NCOLS");}launch_check();return true;@@ -2844,7 +2896,7 @@ bool native_q8_0_sm60_pack_in_place(void* weights, int n_in, int n_out, const chif (q8_packed_registry().count(weights)) return true;// The GGUF blocks are gone afterwards, so only where q8_packed_mmvq can never fall back to the GGUF-layout kernels// (every column count fits the kernel's shared-memory planes) and its 16-byte weight loads stay aligned- if (q8sm60::smem_bytes(MAX_NCOLS, n_in) > q8sm60::MAX_SMEM || reinterpret_cast<std::uintptr_t>(weights) % 16 != 0)+ if (q8sm60::smem_bytes(q8sm60::MAX_COLS, n_in) > q8sm60::MAX_SMEM || reinterpret_cast<std::uintptr_t>(weights) % 16 != 0)return native_q8_0_sm60_pack(weights, n_in, n_out, what);const std::size_t bytes = std::size_t(n_in / 32) * std::size_t(n_out) * 34; // both layouts: 34 bytes per 32std::vector<uint8_t> blocks(bytes), planes(bytes);@@ -3476,6 +3528,19 @@ std::size_t native_mmvq_weight_bytes(int ggml_type, int n_in, int n_out) {void native_mmvq(int ggml_type, const void* weights, const void* x_q8_1, float* y,int n_in, int n_out, int ncols, void* stream) {+ // more columns than one launch takes: ceil(ncols / MAX_NCOLS) passes over the same weights. Every column of a+ // multi-column pass is bitwise the single-column result (bench/micro/native_mmvq_multi.cpp), so the split changes no+ // arithmetic. Column j starts at activation block j * (n_in / Q8K) and writes y + j * n_out. (Never reached by a+ // window of up to 16 rows in a wide build, nor of up to 8 in a narrow one: a guard for a caller that asks for more.)+ if (ncols > MAX_NCOLS) {+ const std::size_t col_blocks = std::size_t(n_in / Q8K);+ for (int j = 0; j < ncols; j += MAX_NCOLS) {+ const int nj = (ncols - j < MAX_NCOLS) ? (ncols - j) : MAX_NCOLS;+ native_mmvq(ggml_type, weights, static_cast<const Q81Block*>(x_q8_1) + std::size_t(j) * col_blocks,+ y + std::size_t(j) * std::size_t(n_out), n_in, n_out, nj, stream);+ }+ return;+ }switch (ggml_type) {case 2: native_q4_0_mmvq(weights, x_q8_1, y, n_in, n_out, ncols, stream); break;case 6: native_q5_0_mmvq(weights, x_q8_1, y, n_in, n_out, ncols, stream); break;diff --git a/src/kernels/cuda/native_moe.cu b/src/kernels/cuda/native_moe.cuindex fc28c4a91..3e0117abd 100644--- a/src/kernels/cuda/native_moe.cu+++ b/src/kernels/cuda/native_moe.cu@@ -21,6 +21,7 @@// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE// SOFTWARE.#include "strata/kernels/native_moe.hpp"+#include "strata/kernels/verify_kernels.hpp"#include <cuda_runtime.h>#include <atomic>#include <cstddef>@@ -175,14 +176,14 @@ void native_moe_combine(const float* parts, const float* weights, const float* sif (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));}bool native_moe_combine_hits_supported(const float* parts, const float* shared, const float* output, int64_t n_embd, int64_t k, int n_tok) {- return k == 10 && shared != nullptr && n_tok >= 1 && n_tok <= 8 && n_embd > 0 && (n_embd & 3) == 0 &&+ return k == 10 && shared != nullptr && n_tok >= 1 && n_tok <= kVerifyMaxT && n_embd > 0 && (n_embd & 3) == 0 &&(((uintptr_t) parts | (uintptr_t) shared | (uintptr_t) output) & 15u) == 0;}void native_moe_combine_multi_hits_added(const float* parts, const float* hits, const int32_t* dst, const int32_t* count,const float* weights, const float* shared, float* output, int64_t n_embd, int64_t k,int n_tok, void* stream) {if (!stream || !native_moe_combine_hits_supported(parts, shared, output, n_embd, k, n_tok) || ((uintptr_t) hits & 15u) != 0)- throw std::invalid_argument("native MoE combine (hits added) requires k = 10, aligned spans, 1..8 tokens");+ throw std::invalid_argument("native MoE combine (hits added) requires k = 10, aligned spans, 1..8 tokens (16 in a wide build)");const int64_t n4 = n_embd >> 2;combine_k10_vec4_hits<<<dim3(unsigned((n4 + 127) / 128), (unsigned) n_tok), 128, 0, static_cast<cudaStream_t>(stream)>>>(reinterpret_cast<const float4*>(parts), reinterpret_cast<const float4*>(hits), dst, count, weights,diff --git a/src/kernels/cuda/q8_sm60.cuh b/src/kernels/cuda/q8_sm60.cuhindex a45b84095..ef1d9fd22 100644--- a/src/kernels/cuda/q8_sm60.cuh+++ b/src/kernels/cuda/q8_sm60.cuh@@ -22,6 +22,7 @@ namespace q8sm60 {constexpr int THREADS = 256;constexpr std::size_t MAX_SMEM = 48 * 1024; // a Pascal workgroup's static + dynamic limit+constexpr int MAX_COLS = 8; // columns of one launch (a window of 9..16 rows runs in passes)inline std::size_t smem_bytes(int ncols, int n_in) {return std::size_t(ncols) * std::size_t(n_in) + std::size_t(ncols) * std::size_t(n_in / 32) * sizeof(float);@@ -118,7 +119,7 @@ void launch_cols(const int8_t* qs, const half* d, const Q81Block* x, float* y, i// false: a shape this kernel does not take (the caller runs its usual kernel)inline bool launch(const int8_t* qs, const half* d, const Q81Block* x, float* y, int n_in, int n_out, int ncols,cudaStream_t s) {- if (n_in % 32 != 0 || n_in < 128 || ncols < 1 || ncols > 8 || smem_bytes(ncols, n_in) > MAX_SMEM) return false;+ if (n_in % 32 != 0 || n_in < 128 || ncols < 1 || ncols > MAX_COLS || smem_bytes(ncols, n_in) > MAX_SMEM) return false;switch (ncols) {case 1: launch_cols<1>(qs, d, x, y, n_in, n_out, s); break;case 2: launch_cols<2>(qs, d, x, y, n_in, n_out, s); break;diff --git a/src/kernels/cuda/s2_expert_grouped.cu b/src/kernels/cuda/s2_expert_grouped.cuindex bff8b21e6..1fe69788d 100644--- a/src/kernels/cuda/s2_expert_grouped.cu+++ b/src/kernels/cuda/s2_expert_grouped.cu@@ -824,9 +824,14 @@ namespace {// hit kernel's.constexpr int GU_CHUNKS = (H / 32 + 31) / 32; // 3: chunks of a gate/up row per lane (80 chunks / 32 lanes)constexpr int GMAX = 8; // entries per group (tokens routed to one expert in a window)-// a group holds one entry per token of the window routed to its expert, and the kernels below keep at-// most GMAX of them (`min(..., GMAX)`): a longer window would drop entries without a word.-static_assert(GMAX >= kVerifyMaxT, "a verify window's group can exceed GMAX entries");+// a group holds one entry per token of the window routed to its expert, and a kernel keeps at most GMAX of them+// (`min(..., GMAX)`): a longer group would drop entries without a word. A window of 9..16 rows can route more tokens+// than that to one expert, so `moe_grouped_s2` takes the window's largest group and runs the kernel again with+// SKIP = GMAX over the entries from the GMAX-th on (the staging is GMAX entries wide in shared memory: 51 KB at 16+// would not compile on CUDA, whose static limit is 48 KB). Every entry's output is its own dot products, so the+// passes are bitwise the one wide pass; SKIP = 0 is the kernel as it was.+constexpr int kGroupPasses = (kVerifyMaxT + GMAX - 1) / GMAX;+static_assert(kGroupPasses * GMAX >= kVerifyMaxT, "a verify window's group can exceed the passes' entries");constexpr int GU_ROWS = 32; // gate/up rows per block: 4 per warpconstexpr int D_ROWS = 64; // down rows per block: 8 per warp@@ -850,6 +855,7 @@ __device__ __forceinline__ float chunk_dot(uint2 cb, const int* xw, float dw, fl// Gate/up: a block = GU_ROWS rows of ONE group. The group's activations (each entry's token row of x_q8_0 and// its fp32 scales) are staged once into shared memory as aligned words; each warp then walks its rows, loading// each lane's code chunks once and dotting them with every entry.+template <int SKIP>__global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long long* __restrict__ grp_ptr,const int32_t* __restrict__ grp_start,const int32_t* __restrict__ n_groups,@@ -861,7 +867,8 @@ __global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long lon__shared__ float xs_d[GMAX][H / 32];const int g = blockIdx.y;if (g >= *n_groups) return;- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);+ if (SKIP > 0 && ne <= 0) return;const int t = threadIdx.x, lane = t & 31, warp = t >> 5;for (int i = t; i < ne * (H / 32); i += blockDim.x) {const int k = i / (H / 32), c = i - k * (H / 32);@@ -912,6 +919,7 @@ __global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long lon// Down: a block = D_ROWS rows of ONE group; the entries' quantized intermediates staged once. A down row is 20// chunks, so lanes 0..19 each hold one chunk, as in the per-entry kernel.+template <int SKIP>__global__ void __launch_bounds__(256) down_grouped_kernel(const unsigned long long* __restrict__ grp_ptr,const int32_t* __restrict__ grp_start,const int32_t* __restrict__ n_groups,@@ -923,7 +931,8 @@ __global__ void __launch_bounds__(256) down_grouped_kernel(const unsigned long l__shared__ float hs_d[GMAX][FF / 32];const int g = blockIdx.y;if (g >= *n_groups) return;- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);+ if (SKIP > 0 && ne <= 0) return;const int t = threadIdx.x, lane = t & 31, warp = t >> 5;for (int i = t; i < ne * (FF / 32); i += blockDim.x) {const int k = i / (FF / 32), c = i - k * (FF / 32);@@ -985,6 +994,7 @@ __device__ __forceinline__ void stage_chunk(const uint8_t* __restrict__ xb, floaxs_dh[i] = make_int2(__float_as_int(dx), hx);}+template <int SKIP>__global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long long* __restrict__ grp_ptr,const int32_t* __restrict__ grp_start,const int32_t* __restrict__ n_groups,@@ -997,7 +1007,8 @@ __global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long l__shared__ int2 xs_dh[GMAX * NC]; // 5 KB: (dx as bits, hx)const int g = blockIdx.y;if (g >= *n_groups) return;- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);+ if (SKIP > 0 && ne <= 0) return;const int t = threadIdx.x, lane = t & 31, warp = t >> 5;for (int i = t; i < ne * NC; i += blockDim.x) {const int k = i / NC, c = i - k * NC;@@ -1051,6 +1062,7 @@ __global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long l}}+template <int SKIP>__global__ void __launch_bounds__(256) down_grouped_t_kernel(const unsigned long long* __restrict__ grp_ptr,const int32_t* __restrict__ grp_start,const int32_t* __restrict__ n_groups,@@ -1063,7 +1075,8 @@ __global__ void __launch_bounds__(256) down_grouped_t_kernel(const unsigned long__shared__ int2 hs_dh[GMAX * NC];const int g = blockIdx.y;if (g >= *n_groups) return;- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);+ if (SKIP > 0 && ne <= 0) return;const int t = threadIdx.x, lane = t & 31, warp = t >> 5;for (int i = t; i < ne * NC; i += blockDim.x) {const int k = i / NC, c = i - k * NC;@@ -1166,10 +1179,18 @@ void moe_group_resident(const int32_t* ids, int n, int k_per_tok, const uint8_t*check("moe_group_resident", stream);}-void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,- const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,- const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream) {+static void grouped_s2_impl(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,+ int max_group) {if (cap_groups <= 0 || cap_entries <= 0) return;+ if (max_group < 1 || max_group > kGroupPasses * GMAX) {+ std::fprintf(stderr, "moe_grouped_s2: a group of %d entries, at most %d\n", max_group, kGroupPasses * GMAX);+ std::exit(1);+ }+ // a group of more than GMAX entries (a window of 9..16 rows, wide builds only): the entries from the GMAX-th on, too+ constexpr bool kWide = kGroupPasses > 1;+ const bool second = kWide && max_group > GMAX;cudaStream_t cs = (cudaStream_t) stream;const uint64_t gu_bytes = ((uint64_t) cap_entries * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;const uint64_t q8_bytes = ((uint64_t) cap_entries * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;@@ -1179,12 +1200,21 @@ void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start,const bool fast = new_grouped(x_q8_0, scratch);{const dim3 grid((unsigned) (2 * FF / GU_ROWS), (unsigned) cap_groups);- if (fast)- gu_grouped_t_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,- gate_up, (int) cap_entries);- else- gu_grouped_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales, gate_up,- (int) cap_entries);+ if (fast) {+ gu_grouped_t_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,+ gate_up, (int) cap_entries);+ if constexpr (kWide)+ if (second)+ gu_grouped_t_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,+ gate_up, (int) cap_entries);+ } else {+ gu_grouped_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales, gate_up,+ (int) cap_entries);+ if constexpr (kWide)+ if (second)+ gu_grouped_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,+ gate_up, (int) cap_entries);+ }check("moe_grouped_s2/gu", stream);}// #783 PR-k (stuchapin909): SwiGLU and the scaled Q8_0 quantize of its result in one kernel (gate_up still gets the@@ -1221,14 +1251,36 @@ void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start,{const dim3 grid((unsigned) (H / D_ROWS), (unsigned) cap_groups);const float* hs = x_scales != nullptr ? h_scales : nullptr;- if (fast)- down_grouped_t_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);- else- down_grouped_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);+ if (fast) {+ down_grouped_t_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);+ if constexpr (kWide)+ if (second) down_grouped_t_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);+ } else {+ down_grouped_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);+ if constexpr (kWide)+ if (second) down_grouped_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);+ }check("moe_grouped_s2/down", stream);}}+void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream) {+ grouped_s2_impl(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream,+ GMAX);+}++#if STRATA_VERIFY_MAX_T > 8+void moe_grouped_s2_wide(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,+ int max_group) {+ grouped_s2_impl(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream,+ max_group);+}+#endif+void moe_hit_add(float* parts, const float* hit_out, const int32_t* dst, const int32_t* count, int64_t cap,int64_t n_embd, void* stream) {if (cap <= 0) return;diff --git a/src/kernels/cuda/shared_expert.cu b/src/kernels/cuda/shared_expert.cuindex bee0da978..36568f042 100644--- a/src/kernels/cuda/shared_expert.cu+++ b/src/kernels/cuda/shared_expert.cu@@ -27,6 +27,7 @@#include "strata/kernels/s2_gemv_q8.hpp"#include "strata/kernels/s_gemv.hpp"#include "strata/kernels/native_mmvq.hpp"+#include "strata/kernels/verify_kernels.hpp"#include <cuda_fp16.h>#include <cuda_runtime.h>@@ -216,8 +217,8 @@ void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, consconst uint16_t* gate_inp_bf16, float* gate, float* up, float* g, float* out, int64_t n_embd,int64_t n_ff, void* stream, const void* x_q8_1_ready, int lfuse) {const bool gate_deferred = (lfuse & 1) != 0, pair = (lfuse & 2) != 0;- if (n_tok < 1 || n_tok > 8 || !nw.q8_1 || !nw.gate_data || !nw.up_data || !nw.down_data || !stream)- throw std::invalid_argument("shared_expert_multi: needs 1..8 tokens, native weights, scratch and a stream");+ if (n_tok < 1 || n_tok > kVerifyMaxT || !nw.q8_1 || !nw.gate_data || !nw.up_data || !nw.down_data || !stream)+ throw std::invalid_argument("shared_expert_multi: needs 1..8 tokens (16 in a wide build), native weights, scratch and a stream");cudaStream_t cs = (cudaStream_t) stream;// the activation's q8_1: the caller's (STRATA_VERIFY_QDEDUP, x_q8_1_ready) or the descriptor's, else quantized hereconst void* x_q8_1 = x_q8_1_ready ? x_q8_1_ready : nw.x_q8_1;diff --git a/src/kernels/cuda/verify_kernels.cu b/src/kernels/cuda/verify_kernels.cuindex 3da7ac639..f2aba7378 100644--- a/src/kernels/cuda/verify_kernels.cu+++ b/src/kernels/cuda/verify_kernels.cu@@ -92,7 +92,7 @@ __global__ void gdn_conv_commit_kernel(float* __restrict__ hist, const float* __hist[c * 3 + 2] = seq[2];}-template <int MAX_T = kVerifyMaxT, bool EXACT_T = false>+template <int MAX_T = kVerifyNarrowT, bool EXACT_T = false>__device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restrict__ x, const uint16_t* __restrict__ wa,const uint16_t* __restrict__ wb,const float* __restrict__ dt,@@ -143,7 +143,7 @@ __device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restric}}-template <int MAX_T = kVerifyMaxT, bool EXACT_T = false>+template <int MAX_T = kVerifyNarrowT, bool EXACT_T = false>__global__ void __launch_bounds__(64) gdn_ab_multi_kernel(const float* __restrict__ x, const uint16_t* __restrict__ wa,const uint16_t* __restrict__ wb,const float* __restrict__ dt,@@ -1079,6 +1079,7 @@ void gdn_conv_ab_multi(const float* history, const float* qkv, const float* convcase 4: gdn_conv_ab_kernel<4, true><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;case 5: gdn_conv_ab_kernel<5, true><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;case 6: gdn_conv_ab_kernel<6, true><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;+ case 7: case 8: gdn_conv_ab_kernel<kVerifyNarrowT, false><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;default: gdn_conv_ab_kernel<kVerifyMaxT, false><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;}check("gdn_conv_ab_multi");@@ -1128,6 +1129,7 @@ void gdn_ab_multi(const float* x, const uint16_t* w_alpha, const uint16_t* w_betcase 4: gdn_ab_multi_kernel<4, true><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;case 5: gdn_ab_multi_kernel<5, true><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;case 6: gdn_ab_multi_kernel<6, true><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;+ case 7: case 8: gdn_ab_multi_kernel<kVerifyNarrowT, false><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;default: gdn_ab_multi_kernel<kVerifyMaxT, false><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;}check("gdn_ab_multi");@@ -1172,8 +1174,12 @@ void gdn_step_norm_multi(float* state, const float* h, int conv_channels, constif (split && (t_out_begin >= n_tok || n_tok - t_out_begin >= 2)) {const dim3 grid((unsigned) h_v, S / GS_COLS), block(GS_COLS, RG);if (t_out_begin >= n_tok) { // no outputs (the commit): the state only- gdn_state_replay_kernel<kVerifyMaxT><<<grid, block, 0, (cudaStream_t) stream>>>(- state, h, conv_channels, gate, beta, h_k, h_v, n_tok, n_keep);+ if (n_tok > kVerifyNarrowT)+ gdn_state_replay_kernel<kVerifyMaxT><<<grid, block, 0, (cudaStream_t) stream>>>(+ state, h, conv_channels, gate, beta, h_k, h_v, n_tok, n_keep);+ else+ gdn_state_replay_kernel<kVerifyNarrowT><<<grid, block, 0, (cudaStream_t) stream>>>(+ state, h, conv_channels, gate, beta, h_k, h_v, n_tok, n_keep);} else {gdn_step_split_kernel<true><<<grid, block, 0, (cudaStream_t) stream>>>(state, h, conv_channels, gate, beta, y, h_k, h_v, n_tok, n_keep, t_out_begin);@@ -1301,20 +1307,25 @@ void handoff_publish(float* dst, const float* a, int64_t na, const float* b, int}namespace {-// one thread per window entry: up to kVerifyMaxT tokens x 10 routed experts (80), so 128 (#646 had 64: a window of-// 7+ tokens lost its last entries)-constexpr int kResidentPlanMax = 128;+// one thread per window entry: up to kVerifyNarrowT tokens x 10 routed experts (80), so 128 (#646 had 64: a window of+// 7+ tokens lost its last entries). A window of 9..16 rows (up to 160 entries) runs the same kernel on 256 threads;+// one block size per window class, so a narrow window launches exactly the 128-thread kernel it always did.+constexpr int kResidentPlanNarrow = 128;+constexpr int kResidentPlanMax = kVerifyMaxT > kVerifyNarrowT ? 256 : kResidentPlanNarrow;+static_assert(kVerifyNarrowT * 10 <= kResidentPlanNarrow, "resident_plan: one thread per entry");static_assert(kVerifyMaxT * 10 <= kResidentPlanMax, "resident_plan: one thread per entry");-// the parallel scan keeps one partial sum per warp in s_wsum[4] and packs (entries << 16 | groups) in an int-static_assert(kResidentPlanMax <= 128 && kResidentPlanMax % 32 == 0, "resident_plan: at most 4 warps");+// the parallel scan keeps one partial sum per warp in s_wsum[NT / 32] and packs (entries << 16 | groups) in an int+static_assert(kResidentPlanMax <= 256 && kResidentPlanMax % 32 == 0, "resident_plan: at most 8 warps");static_assert(kResidentPlanMax < 32768, "resident_plan: the packed entry count must stay below 2^15");-__global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const int32_t* __restrict__ ids, int n, int k, const int32_t* __restrict__ res,+template <int NT>+__global__ void __launch_bounds__(NT) resident_plan_kernel(const int32_t* __restrict__ ids, int n, int k, const int32_t* __restrict__ res,int n_expert, const uint8_t* cache_base, const unsigned long long* slot_off,long long blob, int32_t* __restrict__ pl, long long capx, uint32_t* skip,uint32_t ring, volatile uint32_t* plan_err) {- __shared__ int32_t s_ids[kResidentPlanMax];- __shared__ int32_t s_excl[kResidentPlanMax];- __shared__ int32_t s_wsum[4];+ constexpr int NW = NT / 32;+ __shared__ int32_t s_ids[NT];+ __shared__ int32_t s_excl[NT];+ __shared__ int32_t s_wsum[NW];__shared__ int s_bad;const int tid = threadIdx.x;if (tid == 0) s_bad = 0;@@ -1378,7 +1389,7 @@ __global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const iif (is_first) {int tot = s_excl[tid];#pragma unroll- for (int w = 0; w < 4; ++w) {+ for (int w = 0; w < NW; ++w) {if (w < warp) tot += s_wsum[w];}const int grp_idx = tot & 0xffff;@@ -1390,7 +1401,7 @@ __global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const iconst int fj_warp = first_j >> 5;int fj_tot = s_excl[first_j];#pragma unroll- for (int w = 0; w < 4; ++w) {+ for (int w = 0; w < NW; ++w) {if (w < fj_warp) fj_tot += s_wsum[w];}const int out_idx = (fj_tot >> 16) + rank_in_group;@@ -1398,7 +1409,14 @@ __global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const itok[out_idx] = tid / k;}if (tid == 0) {- const int groups = (s_wsum[0] + s_wsum[1] + s_wsum[2] + s_wsum[3]) & 0xffff;+ int groups;+ if constexpr (NW == 4) groups = (s_wsum[0] + s_wsum[1] + s_wsum[2] + s_wsum[3]) & 0xffff; // (the 128-thread kernel as it was)+ else {+ groups = 0;+#pragma unroll+ for (int w = 0; w < NW; ++w) groups += s_wsum[w];+ groups &= 0xffff;+ }start[groups] = n;start2[0] = n;counts[0] = groups;@@ -1496,8 +1514,18 @@ __global__ void copy_or_zero_kernel(float4* __restrict__ dst, const volatile flovoid resident_plan(const int32_t* ids, int n_entries, int k, const int32_t* res_layer, int n_expert,const uint8_t* cache_base, const unsigned long long* slot_off, long long blob, int32_t* plan,long long capx, uint32_t* skip, uint32_t ring, void* stream, uint32_t* plan_err) {- resident_plan_kernel<<<1, kResidentPlanMax, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,+ if (n_entries > kResidentPlanNarrow) { // a 9..16-row window (kResidentPlanMax == 256 only in a wide build)+ if (kResidentPlanMax > kResidentPlanNarrow && n_entries <= kResidentPlanMax)+ resident_plan_kernel<kResidentPlanMax><<<1, kResidentPlanMax, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,blob, plan, capx, skip, ring, plan_err);+ else {+ std::fprintf(stderr, "resident_plan: %d entries, at most %d\n", n_entries, kResidentPlanMax);+ std::exit(1);+ }+ } else {+ resident_plan_kernel<kResidentPlanNarrow><<<1, kResidentPlanNarrow, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,+ blob, plan, capx, skip, ring, plan_err);+ }check("resident_plan");}void wait_flag_ge_or(const uint32_t* flag, uint32_t value, const uint32_t* skip, void* stream) {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 824bd19db..7de185b2c 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -100,6 +100,7 @@#include "strata/spec/suffix_drafter.hpp"#include "strata/spec/spec_start.hpp"#include "strata/program/batch_groups.hpp"+#include "strata/program/pipe_probe.hpp"#include "strata/kernels/cvec.hpp"#include "strata/core/progress.hpp"#include "strata/core/device.hpp"@@ -3755,7 +3756,14 @@ int main(int argc, char** argv) {if (o.lookup_chain > 0 && o.spec >= 2) {if (o.mtp_max_t == 0) o.mtp_max_t = o.spec;const int chain_base = o.spec;- o.spec = std::max(chain_base, std::min(std::max(chain_base, o.mtp_max_t) + o.lookup_chain, 8)); // kVerifyMaxT+ o.spec = std::max(chain_base, std::min(std::max(chain_base, o.mtp_max_t) + o.lookup_chain, 8)); // kVerifyNarrowT+ }+ // A speculative (solo) window holds at most kVerifyNarrowT rows: the hand-offs, the penalty history and the drafters'+ // buffers are sized for that. Only a plain --batch forms the wider windows of a build with kVerifyMaxT > 8.+ if (o.spec > (int) strata::kernels::kVerifyNarrowT) {+ std::fprintf(stderr, "strata generate: --spec %d: a speculative window holds 2..%d tokens\n", o.spec,+ (int) strata::kernels::kVerifyNarrowT);+ return 1;}// dflash2-kern: right after the start the resident experts are the profile's, not this text's; a speculative round carries// 2-5 tokens, so the adaptive tier (one swap round per adapt_every VERIFY rounds) learned the text 2-5 times slower per@@ -6000,17 +6008,19 @@ int main(int argc, char** argv) {int pm_drafts = 0;if (piped_mtp) {pm_drafts = std::clamp(std::atoi(std::getenv("STRATA_PIPED_MTP")), 1,- std::max(1, std::min(o.spec - 1, (int) strata::kernels::kVerifyMaxT - 1)));+ std::max(1, std::min(o.spec - 1, (int) strata::kernels::kVerifyNarrowT - 1)));}// The verifiers' window: the solo path's drafts (--spec rows), the plain batch windows (the slots) and, with// STRATA_PIPED_MTP, a group's rows: its slots, each its token and its proposals ((slots) * (1 + proposals), one slot// per group = at most --spec rows). A function, not a number: --batch (the slots that fit) and the groups are final// only after this point, at the verifiers' init, and the plain value must stay what it was there.auto batch_max_t = [&]() -> int {- if (batch_mtp && !piped_mtp) return (int) strata::kernels::kVerifyMaxT;+ // (--batch-mtp and STRATA_PIPED_MTP draft with the draft layer's kernels, built for the narrow window: their windows+ // stay at most kVerifyNarrowT rows; only the plain batch windows are wide)+ if (batch_mtp && !piped_mtp) return (int) strata::kernels::kVerifyNarrowT;const int plain = std::max(o.spec, std::min(o.batch, (int) strata::kernels::kVerifyMaxT));const int group_rows = (piped_mtp && o.batch_groups > 1 && o.batch > 0)- ? std::min<int>(strata::kernels::kVerifyMaxT, (o.batch / o.batch_groups) * (1 + pm_drafts)) : 0;+ ? std::min<int>(strata::kernels::kVerifyNarrowT, (o.batch / o.batch_groups) * (1 + pm_drafts)) : 0;return std::max(plain, group_rows);};// The most slots, up to n, whose windows hold at most kVerifyMaxT rows each: n itself up to that, more only as@@ -6146,10 +6156,10 @@ int main(int argc, char** argv) {why = buf;} else if (sc != nullptr && sc[0] != '\0' && sc[0] != '0') {why = "it does not combine with STRATA_BATCH_SELF_COMMIT (the window would keep its rows itself)";- } else if ((o.batch / o.batch_groups) * (1 + pm_drafts) > (int) strata::kernels::kVerifyMaxT) {+ } else if ((o.batch / o.batch_groups) * (1 + pm_drafts) > (int) strata::kernels::kVerifyNarrowT) {std::snprintf(buf, sizeof buf, "a group's window would hold %d rows (%d slots x %d), more than %d",(o.batch / o.batch_groups) * (1 + pm_drafts), o.batch / o.batch_groups, 1 + pm_drafts,- (int) strata::kernels::kVerifyMaxT);+ (int) strata::kernels::kVerifyNarrowT);why = buf;}if (why != nullptr) {@@ -6212,7 +6222,7 @@ int main(int argc, char** argv) {const strata::core::OnDevice on_mtp(last_st && !mtp_first ? last_st->dev : -1);// --pipeline-windows 2: 8 rows (the chain runs on past the window in flight: its drafts, its bonus token and// the next window's drafts), and the round/step graphs carry the forcing kernel- const int mtp_t = o.pipeline_windows >= 2 ? strata::kernels::kVerifyMaxT : o.spec;+ const int mtp_t = o.pipeline_windows >= 2 ? strata::kernels::kVerifyNarrowT : o.spec;if (o.pipeline_windows >= 2) mtp.set_force_capture(true);if (!o.mtp.empty() && !mtp.load(o.mtp, draft_geometry, last_st && !mtp_first ? last_st->ss : ss, mtp_t, err, o.mtp_window)) { std::fprintf(stderr, "strata generate: %s%s\n", err.c_str(), vram_free_note().c_str()); return 1; }mtp.set_ple_session(&ss);@@ -9867,7 +9877,9 @@ int main(int argc, char** argv) {// the widest window; a request without penalties gets a null buffer and takes the byte-for-byte// neutral path (no upload, no buffer handed to the sampler).constexpr int kPenaltyWindowCap = 4096;- constexpr size_t kHistSlots = (size_t) kPenaltyWindowCap * (size_t) strata::kernels::kVerifyMaxT;+ // (the widest window the run can have: a batch of more than kVerifyNarrowT slots is the only way to a wider one)+ const size_t kHistSlots = (size_t) kPenaltyWindowCap *+ (size_t) (o.batch > strata::kernels::kVerifyNarrowT ? strata::kernels::kVerifyMaxT : strata::kernels::kVerifyNarrowT);int32_t* d_hist = nullptr;std::vector<int32_t> hist_stage(kHistSlots, -1);const int hist_dev = last_st ? last_st->dev : -1; // with the head: the last stage's device@@ -9935,11 +9947,15 @@ int main(int argc, char** argv) {}if (n_stages > 1) {// pipelined slot groups write their rows at hbase = group * rows-per-group, so with more than- // kVerifyMaxT slots in groups the hand-off holds one row per slot (otherwise kVerifyMaxT, as before)+ // kVerifyNarrowT slots in groups the hand-off holds one row per slot (otherwise kVerifyNarrowT, as before; a+ // wide build holds more only for a batch of more than that: the buffer is not made larger for a run that+ // never forms a window of 9..16 rows)// (STRATA_PIPED_MTP: per slot its token and up to pm_drafts proposals)- const int hrows = piped_mtp ? std::max<int>(strata::kernels::kVerifyMaxT, (1 + pm_drafts) * o.batch)- : !batch_mtp && o.batch_groups > 1 ? std::max<int>(strata::kernels::kVerifyMaxT, o.batch)- : strata::kernels::kVerifyMaxT;+ const int hrows = piped_mtp ? std::max<int>(strata::kernels::kVerifyNarrowT, (1 + pm_drafts) * o.batch)+ : !batch_mtp && o.batch_groups > 1 ? std::max<int>(strata::kernels::kVerifyNarrowT, o.batch)+ : !batch_mtp ? std::max<int>(strata::kernels::kVerifyNarrowT,+ std::min<int>(o.batch, strata::kernels::kVerifyMaxT))+ : strata::kernels::kVerifyNarrowT;const size_t hb = (size_t) hrows * (size_t) strata::core::Verifier::handoff_floats(g) * sizeof(float);std::vector<float*> hand((size_t) n_stages - 1, nullptr);for (float*& h : hand) {@@ -9978,7 +9994,7 @@ int main(int argc, char** argv) {for (int st = 0; st < n_stages; ++st) {stage_ver(st).set_stage(st == 0 ? 0 : split_at[(size_t) st - 1], st + 1 < n_stages ? split_at[(size_t) st] : -1,st == 0 ? nullptr : hand[(size_t) st - 1], st + 1 < n_stages ? hand[(size_t) st] : nullptr);- if (hrows > strata::kernels::kVerifyMaxT) stage_ver(st).set_handoff_rows(hrows);+ if (hrows > strata::kernels::kVerifyNarrowT) stage_ver(st).set_handoff_rows(hrows);if (overlap)stage_ver(st).set_handoff_flags(st == 0 ? nullptr : hflag_h[(size_t) st - 1],st == 0 ? nullptr : hflag_d[(size_t) st - 1],@@ -10031,7 +10047,7 @@ int main(int argc, char** argv) {// watchdog's verifier), and the drafter's own row buffer, which either parity's rows are copied intofloat* pl_mtp_R = nullptr;if (pipe) {- const size_t hb = (size_t) strata::kernels::kVerifyMaxT *+ const size_t hb = (size_t) strata::kernels::kVerifyNarrowT *(size_t) strata::core::Verifier::handoff_floats(g) * sizeof(float);std::vector<float*> hand_b((size_t) n_stages - 1, nullptr); // the odd windows' hand-off per boundaryfor (float*& h : hand_b) {@@ -10063,7 +10079,7 @@ int main(int argc, char** argv) {gs.dev, st + 1, err.c_str());return 1;}- if (last && !mtp_first && cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyMaxT * (size_t) (g.hc * g.n_embd) *+ if (last && !mtp_first && cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyNarrowT * (size_t) (g.hc * g.n_embd) *sizeof(float)) != cudaSuccess) {std::fprintf(stderr, "strata serve: --pipeline-windows: the drafter's row buffer does not fit\n");return 1;@@ -10071,7 +10087,7 @@ int main(int argc, char** argv) {}if (mtp_first) { // the drafter's row buffer on its own card (the first)const strata::core::OnDevice on_m(mtp.device());- if (cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyMaxT * (size_t) (g.hc * g.n_embd) *+ if (cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyNarrowT * (size_t) (g.hc * g.n_embd) *sizeof(float)) != cudaSuccess) {std::fprintf(stderr, "strata serve: --pipeline-windows: the drafter's row buffer does not fit\n");return 1;@@ -10134,7 +10150,7 @@ int main(int argc, char** argv) {}// STRATA_PIPED_MTP: the rows a group's commit replays after the verdict, one hand-off row eachif (piped_mtp && bslot_ss.size() > 1 &&- !vk.init_deferred_commit(std::max<int>(strata::kernels::kVerifyMaxT, (1 + pm_drafts) * o.batch), err)) {+ !vk.init_deferred_commit(std::max<int>(strata::kernels::kVerifyNarrowT, (1 + pm_drafts) * o.batch), err)) {std::fprintf(stderr, "strata serve: STRATA_PIPED_MTP, stage %zu: %s\n", k + 1, err.c_str());return 1;}@@ -12391,7 +12407,7 @@ int main(int argc, char** argv) {struct BSlot {bool active = false, stop = false;int32_t x = 0; ///< the token the next window feeds (not yet in the slot's state)- std::array<int32_t, strata::kernels::kVerifyMaxT> draft{}; ///< the slot's next MTP proposal+ std::array<int32_t, strata::kernels::kVerifyNarrowT> draft{}; ///< the slot's next MTP proposalbool draft_ready = false;int n_drafts = 1; ///< STRATA_PIPED_MTP: how many of `draft` its next window checksint64_t p = 0; ///< its position@@ -12655,7 +12671,9 @@ int main(int argc, char** argv) {static size_t next_slot = 0;// Each MTP slot uses two rows; rotate slots when more than four are active.int A = 0;- for (size_t offset = 0; offset < bs.size() && S + (batch_mtp ? 2 : 1) <= strata::kernels::kVerifyMaxT; ++offset) {+ // (--batch-mtp: two rows a slot, at most kVerifyNarrowT; the plain windows hold up to kVerifyMaxT)+ const int win_cap = batch_mtp ? (int) strata::kernels::kVerifyNarrowT : (int) strata::kernels::kVerifyMaxT;+ for (size_t offset = 0; offset < bs.size() && S + (batch_mtp ? 2 : 1) <= win_cap; ++offset) {const int b = (int) ((next_slot + offset) % bs.size());if (bs[(size_t) b].active) {first[A] = S;@@ -13099,11 +13117,28 @@ int main(int argc, char** argv) {p.last = pa_last;return strata::program::batch_groups::pipe_adapt_due(p);};+ // STRATA_P4_TIMING=1 / STRATA_P4_TRACE=PATH: where a pipelined window's time goes (pipe_probe.hpp; off: a branch)+ strata::program::PipeProbe p4(n_pipe, pg.size());// one turn: serve every running stage, hand finished groups on, start what can startauto pump = [&](bool may_start) -> bool {++pipe_tick;+ const strata::program::PipeProbe::PumpScope p4_scope(p4);+ if (p4.timing()) {+ int nb = 0, nf = 0;+ for (int k = 0; k < n_pipe; ++k) if (stage_group[(size_t) k] >= 0) ++nb;+ for (const PGroup& x : pg) if (x.inflight) ++nf;+ p4.busy(nb, nf);+ // a stage free with no group waiting at it = starved by the stage before, else it was only idle a moment+ for (int k = 1; k < n_pipe; ++k) {+ if (stage_group[(size_t) k] >= 0) continue;+ bool ready = false;+ for (size_t g2 = 0; g2 < pg.size() && !ready; ++g2) ready = pg[g2].inflight && pg[g2].stage == k;+ p4.free_stage(k, ready);+ }+ }if (bt_windows == 0 && !pipe_inflight()) {bt_start = Clock::now();+ p4.span_start(bt_start);bt_wait0 = ver.ms_wait; bt_pool0 = ver.ms_pool;bt_miss0 = drive.d.multi_misses; bt_hits0 = drive.d.cache_hits; bt_pcie0 = drive.d.pcie_experts;}@@ -13111,9 +13146,12 @@ int main(int argc, char** argv) {const int gi = stage_group[(size_t) k];if (gi < 0) continue;strata::core::Verifier& vk = stage_verifier(k);+ const auto p4_p0 = p4.now();const int r = vk.batch_poll(win_pool_fn, win_pool_user, err);+ p4.polled(p4_p0);if (r < 0) { std::printf("ERR %s\n", err.c_str()); return false; }if (r == 0) continue;+ p4.done(k, gi);if (batch_telemetry_enabled) stage_in_flight.complete(k, Clock::now());stage_group[(size_t) k] = -1;PGroup& G = pg[(size_t) gi];@@ -13121,6 +13159,7 @@ int main(int argc, char** argv) {// the last stage: the group's picksconst int32_t* outb = vk.batch_out();if (pmtp && !pmtp_verdict(G, vk, outb)) return false;+ const strata::program::PipeProbe::VerdictScope p4_verdict(p4);for (int t = 0; t < G.S && !pmtp; ++t) {BSlot& sl = bs[(size_t) (gi * GS + t)];if (!sl.active) continue;@@ -13251,11 +13290,13 @@ int main(int argc, char** argv) {PGroup& G = pg[(size_t) pick];strata::core::progress().busy.store(true);if (batch_telemetry_enabled) stage_in_flight.launch(k, Clock::now());+ const auto p4_l0 = p4.now();if (!(pmtp ? stage_verifier(k).batch_launch_rows(G.rows, G.S, G.hbase, G.tok, G.pos, err): stage_verifier(k).batch_launch(pick * GS, G.S, G.tok, G.pos, err))) {std::printf("ERR %s\n", err.c_str());return false;}+ p4.launched(k, pick, p4_l0);stage_group[(size_t) k] = pick;if (conc_serving) { // the doorbell thread may be waiting between windows: the launch above is seenstd::atomic_thread_fence(std::memory_order_seq_cst);@@ -13292,6 +13333,7 @@ int main(int argc, char** argv) {if (pmtp && !pipe_inflight() && !batch_on() && !pmtp_settle()) return false;if (!pipe_inflight() && !batch_on() && bt_windows > 0) { // all idle: the timing line, as batch_stepconst double w = (double) bt_windows, wall = std::chrono::duration<double, std::milli>(Clock::now() - bt_start).count();+ p4.span_end(wall);char pa_txt[160] = "";if (pipe_adapt_every > 0)std::snprintf(pa_txt, sizeof pa_txt, "; adaptive tier (STRATA_PIPE_ADAPT_EVERY): %lld rounds, %lld experts "@@ -13302,6 +13344,7 @@ int main(int argc, char** argv) {if (pmtp)std::fprintf(stderr, " (%lld of %lld MTP proposals kept)", (long long) bt_accepted, (long long) bt_tokens);std::fprintf(stderr, "\n");+ p4.flush();pa_rounds = pa_swaps = 0;pa_ms = 0;for (int k = 0; k < n_pipe; ++k) {@@ -16307,7 +16350,7 @@ int main(int argc, char** argv) {++pl_on;// the chain over A, forced through B's drafts: B's bonus guess and the next window's draftsmtp.set_source_R(V1(A).final_R(0));- chain_n = std::min(strata::kernels::kVerifyMaxT - 1, B.T + S_mtp - 1);+ chain_n = std::min(strata::kernels::kVerifyNarrowT - 1, B.T + S_mtp - 1);if (!mtp.chain_launch(A.T, outp.data(), A.p, a, B.tok + 1, B.T - 1, chain_n, chain_n, err))return die(err);chain_kind = 1;@@ -16336,7 +16379,7 @@ int main(int argc, char** argv) {B = PW{};if (!last) {mtp.set_source_R(V1(A).final_R(0));- chain_n = strata::kernels::kVerifyMaxT - 1;+ chain_n = strata::kernels::kVerifyNarrowT - 1;if (!mtp.chain_launch(A.T, outp.data(), A.p, a, nullptr, 0, chain_n, chain_n, err))return die(err);chain_kind = 2;