Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Verify and batch windows of up to 16 rows: HIP builds default to 16, narrow builds unchanged

Niko12219d3c8c3664f1
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ee0599211..5946ce639 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -92,6 +92,31 @@ if(STRATA_HIP_GFX906)
endif()
option(STRATA_ENABLE_HIP "Build the HIP targets for wave32 AMD GPUs (gfx1100, gfx1201; gfx1012/gfx1030 unvalidated)" OFF)
option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
+# The rows one verify / batch window holds (include/strata/kernels/window_limits.hpp): 8, the 0.2.7 window, or 16 (four pipelined
+# groups of 16 slots = 64 slots on a four-card split, docs/BATCHING.md). Only the CUDA and HIP engines are built from
+# these sources with a wide window; the SYCL and Metal builds never read this variable and keep 8. A window of up to
+# 8 rows runs the kernels it always ran, whichever is chosen (the wide instantiations are extra code). "auto" = 16 for
+# the HIP engines (a HIP code object loads on first use, so the extra kernels cost nothing until a wide window forms)
+# and 8 for CUDA, which loads every kernel at start (CUDA_MODULE_LOADING=EAGER: the wide kernels would take VRAM from
+# the expert cache of every card); -DSTRATA_WINDOW_MAX_T=16 builds the wide CUDA engine.
+set(STRATA_WINDOW_MAX_T "auto" CACHE STRING "rows of a verify / batch window: 8, 16 or auto (HIP 16, CUDA 8)")
+set_property(CACHE STRATA_WINDOW_MAX_T PROPERTY STRINGS auto 8 16)
+if(NOT STRATA_WINDOW_MAX_T STREQUAL "auto" AND NOT STRATA_WINDOW_MAX_T STREQUAL "8" AND NOT STRATA_WINDOW_MAX_T STREQUAL "16")
+ message(FATAL_ERROR "STRATA_WINDOW_MAX_T must be auto, 8 or 16 (got '${STRATA_WINDOW_MAX_T}')")
+endif()
+if(STRATA_ENABLE_HIP OR STRATA_ENABLE_CUDA)
+ if(STRATA_WINDOW_MAX_T STREQUAL "auto")
+ if(STRATA_ENABLE_HIP)
+ set(_strata_window_max_t 16)
+ else()
+ set(_strata_window_max_t 8)
+ endif()
+ else()
+ set(_strata_window_max_t ${STRATA_WINDOW_MAX_T})
+ endif()
+ add_compile_definitions(STRATA_VERIFY_MAX_T=${_strata_window_max_t})
+ message(STATUS "Strata: verify / batch windows hold up to ${_strata_window_max_t} rows")
+endif()
# MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K (Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF
# projections, STRATA_DENSE_MMQ). On by default since MODEL-UPDATE Phase 1: on CUDA they are their own library
# (strata_kq, loaded only for a model with these formats: the engine loads every kernel of its own binary at start,
@@ -1736,6 +1761,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
add_executable(adapt_decay_test src/program/adapt_decay_test.cpp)
target_include_directories(adapt_decay_test PRIVATE include)
add_test(NAME adapt_decay_test COMMAND adapt_decay_test)
+ # the window limits (8 rows, or 16 in a wide CUDA / HIP build) and the pipelined-window probe (header-only, CPU)
+ add_executable(window_limits_test src/core/window_limits_test.cpp)
+ target_include_directories(window_limits_test PRIVATE include)
+ add_test(NAME window_limits_test COMMAND window_limits_test)
# which window-size graph is freed when the next one does not fit (#3702; header-only, CPU)
add_executable(window_evict_test src/core/window_evict_test.cpp)
target_include_directories(window_evict_test PRIVATE include)
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index 78609014a..1b67dd3ca 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -27,6 +27,7 @@
#include "strata/core/layer.hpp"
#include "strata/core/session.hpp"
#include "strata/kernels/sampler.hpp"
+#include "strata/kernels/window_limits.hpp"
#include <cuda_runtime.h>
@@ -404,13 +405,13 @@ private:
std::vector<SessionState*> slots_;
bool batch_rec_ = false; ///< record_window is capturing a batch window
int row_base_ = 0; ///< ... its hand-off rows start here (a pipeline group's own rows)
- int brow_[8] = {}; ///< ... and row t is slot brow_[t]
+ int brow_[strata::kernels::kWindowMaxT] = {}; ///< ... and row t is slot brow_[t]
bool last_batch_ = false; ///< the last run was a batch window (set_plan_slot: one group)
std::map<std::vector<int>, cudaGraphExec_t> exec_bm_, commit_bm_; ///< full row layout avoids slot-ID collisions
std::map<std::vector<int>, uint64_t> bm_used_; ///< last use of each captured layout (LRU)
uint64_t bm_tick_ = 0;
size_t batch_graph_limit_ = 0; ///< 0: keep every captured batch graph (0.1.39); N: LRU-evict beyond N layouts
- int last_rows_[8] = {}; ///< the slots of the last batch window's rows
+ int last_rows_[strata::kernels::kWindowMaxT] = {}; ///< the slots of the last batch window's rows
std::vector<int> bkey(const int* rows, int S, int hbase) const { // #871: the doorbell variant has its own graphs
std::vector<int> k = batch_key(rows, S, hbase);
k.push_back(ar_off_ ? 1 : 0);
@@ -439,14 +440,14 @@ private:
std::chrono::steady_clock::duration(b_last_.load(std::memory_order_acquire)));
}
std::chrono::steady_clock::time_point ext_checked_{}; ///< batch_poll's last check with external serving
- int32_t b_out_[8] = {};
+ int32_t b_out_[strata::kernels::kWindowMaxT] = {};
std::vector<strata::kernels::SamplerParams> slot_sp_;
bool sample_rows(int S, std::string& err); ///< the sampled slots' rows of the last batch window
int32_t* h_commitb_ = nullptr; int32_t* m_commitb_ = nullptr; // per slot [1, 0, pos, -1 ..], stride 2 + max_t
int32_t* commitb_ = nullptr;
float* tail_snap_b_ = nullptr; ///< per (slot, QSA layer) indexer tail snapshot
void* arena_b_ = nullptr;
- int64_t last_pos_b_[8] = {};
+ int64_t last_pos_b_[strata::kernels::kWindowMaxT] = {};
bool capture_batch(const int* rows, int S, int hbase, std::string& err);
// STRATA_PIPED_MTP (init_deferred_commit): the saved rows, laid out as the window scratch (global layer index) with
// dc_rows_ rows per layer instead of max_t_; the slots' accepted prefixes (mapped and device, 2 + max_t_ per slot)
@@ -534,7 +535,7 @@ private:
unsigned* qcnt_ = nullptr; ///< S26 STRATA_QFUSE: the HC read's q8_1 group counters (n_embd / 32)
unsigned long long* slot_off_d_ = nullptr; ///< E-6: the slot offsets on the device
int64_t lb_ = 0, le_ = -1; ///< set_stage: the layers this verifier runs (-1: to the last)
- int handoff_rows_ = 0; ///< set_handoff_rows (0: kVerifyMaxT)
+ int handoff_rows_ = 0; ///< set_handoff_rows (0: kVerifyNarrowT)
const float* hand_in_ = nullptr;
float* hand_out_ = nullptr;
uint32_t* hflag_in_h_ = nullptr; ///< set_handoff_flags: the incoming hand-off's ready word (host / device)
@@ -573,7 +574,7 @@ private:
std::vector<unsigned long long> prof_h_;
double prof_sum_[2][kProfPer] = {}; // [GDN / QSA layers][stage]
int64_t prof_windows_ = 0;
- double prof_t_sum_[9][2][kProfPer] = {}; // the same per window size T (1..8), for profile_by_t_report
+ double prof_t_sum_[9][2][kProfPer] = {}; // the same per window size T (1..8: wider windows are not profiled), for profile_by_t_report
int64_t prof_t_win_[9] = {};
bool rs_on_ = false; // STRATA_ROUTE_STATS
unsigned long long* rs_dev_ = nullptr; // [9 T][n_layers][8 rows]: experts of row r that no earlier row of the window routed to
@@ -589,7 +590,7 @@ private:
float* ple_val_ = nullptr;
int last_t_ = 0;
int64_t last_pos0_ = 0;
- int32_t last_tokens_[8] = {};
+ int32_t last_tokens_[strata::kernels::kWindowMaxT] = {};
int64_t n_vocab_ = 0;
cudaStream_t cs_ = nullptr;
cudaStream_t sh_cs_ = nullptr;
@@ -602,9 +603,10 @@ private:
cudaEvent_t df_join_[2] = {};
// [sel_long_][T]: 1 is the long-context QSA select (STRATA_SELECT_LONG=1: multi-CTA top-k, shuffle-free decode
// scores; see run()); without it only [0] is ever captured
- cudaGraphExec_t exec_[2][9] = {};
- cudaGraphExec_t exec_nr_[2][9] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise
- bool launched_[2][9] = {}, launched_nr_[2][9] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)
+ // (per window size T = 1..kWindowMaxT: a solo window is at most 8 rows, but a merge window of a wide batch is not)
+ cudaGraphExec_t exec_[2][strata::kernels::kWindowMaxT + 1] = {};
+ cudaGraphExec_t exec_nr_[2][strata::kernels::kWindowMaxT + 1] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise
+ bool launched_[2][strata::kernels::kWindowMaxT + 1] = {}, launched_nr_[2][strata::kernels::kWindowMaxT + 1] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)
int sel_long_ = 0;
// fork 34a35db (STRATA_SIDE_MIXER=1): the GDN gates and the QSA indexer on a branch of the window's graph beside the
// mixer's projections, joined before the recurrence / the attention
@@ -672,7 +674,7 @@ private:
void set_plan_slot(int grp);
bool split_ = false; // opt-in (--spec-split): exact but slower, see the overlap study
std::function<bool(cudaStream_t, std::string&)> pre_launch_; // set_pre_launch
- int groups_[9] = {};
+ int groups_[strata::kernels::kWindowMaxT + 1] = {};
float* h_ymiss_ = nullptr; float* m_ymiss_ = nullptr; // T * k * n_embd
// device
diff --git a/include/strata/kernels/fused_gr.hpp b/include/strata/kernels/fused_gr.hpp
index 648380278..cc264fad7 100644
--- a/include/strata/kernels/fused_gr.hpp
+++ b/include/strata/kernels/fused_gr.hpp
@@ -17,6 +17,8 @@
// must be different buffers (every block reads the former while one block writes the latter).
#pragma once
+#include "strata/kernels/window_limits.hpp"
+
#include <cstdint>
namespace strata::kernels {
@@ -66,6 +68,9 @@ void fused_gr_quantize_q8_0(const uint16_t* bf16, uint8_t* out, int64_t n_blocks
/// weight pointers and eps must be the same for every t); `xn_scratch` is n_tok * hc * n_embd floats. Every
/// token's outputs are bitwise `fused_gr_read(a[t])`.
constexpr int kFusedGrMaxT = 8;
+/// A window of more than kFusedGrMaxT tokens is read in slices of at most kFusedGrMaxT (every token's outputs are
+/// strictly its own, so the slices are bitwise the one read); `a` and `xn_scratch` hold up to kFusedGrWideT tokens.
+constexpr int kFusedGrWideT = kWindowMaxT;
/// Returns true when it also wrote the q8_1 images (every token's q8_mixed set and this read supports it).
bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,
unsigned long long* stamp_buf = nullptr, int stamp_i0 = 0);
diff --git a/include/strata/kernels/s2_expert_grouped.hpp b/include/strata/kernels/s2_expert_grouped.hpp
index af6f52f99..b11dec9e1 100644
--- a/include/strata/kernels/s2_expert_grouped.hpp
+++ b/include/strata/kernels/s2_expert_grouped.hpp
@@ -23,6 +23,8 @@
// can keep one captured graph per layer with the maximum grid and a device-side count.
#pragma once
+#include "strata/kernels/window_limits.hpp"
+
#include <cstdint>
namespace strata::kernels {
@@ -100,6 +102,29 @@ void moe_hit_add(float* parts, const float* hit_out, const int32_t* dst, const i
void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream);
+#if STRATA_VERIFY_MAX_T > 8
+/// The same for a window of 9..16 rows (CUDA / HIP engines built with a wide window, window_limits.hpp): `max_group` is the
+/// most entries one group can hold = the window's rows (a token routes to an expert once). Up to 8 it is
+/// `moe_grouped_s2` (one pass of each kernel); 9..16 adds a second pass over the entries from the 9th on.
+void moe_grouped_s2_wide(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,
+ int max_group);
+#endif
+/// What a verify window calls: `max_group` is the window's rows. A build with narrow windows (the default, and every SYCL / Metal
+/// build, whose kernel libraries have only `moe_grouped_s2`) runs `moe_grouped_s2`: its windows hold at most 8 rows.
+inline void moe_grouped_s2_window(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,
+ int max_group) {
+#if STRATA_VERIFY_MAX_T > 8
+ moe_grouped_s2_wide(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out,
+ stream, max_group);
+#else
+ (void) max_group;
+ moe_grouped_s2(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream);
+#endif
+}
/// Plan v0.3 P6: the groups for `moe_grouped_s2` when every expert id is resident at base + id * blob (the MTP
/// layer's 512 experts): counts[0] groups, counts[1] entries; entry dst = routing index, tok = index / k_per_tok.
void moe_group_resident(const int32_t* ids, int n, int k_per_tok, const uint8_t* base, int64_t blob,
diff --git a/include/strata/kernels/verify_kernels.hpp b/include/strata/kernels/verify_kernels.hpp
index b2ff4a0d4..4d8f2e590 100644
--- a/include/strata/kernels/verify_kernels.hpp
+++ b/include/strata/kernels/verify_kernels.hpp
@@ -14,11 +14,15 @@
// Everything that varies per window (token ids, `n_keep`) is read from DEVICE memory so the kernels can be captured.
#pragma once
+#include "strata/kernels/window_limits.hpp"
+
#include <cstdint>
namespace strata::kernels {
-inline constexpr int kVerifyMaxT = 8;
+/// The most rows one verify / batch window holds (8; 16 in the CUDA / HIP builds, window_limits.hpp). A window of up to
+/// kVerifyNarrowT rows runs the kernels it always ran.
+inline constexpr int kVerifyMaxT = kWindowMaxT;
/// For token t of T: conv over [history(3) | qkv_0 .. qkv_t] -> SiLU -> L2 norm of the q/k heads -> h[t].
/// `history` is NOT written, but with `commit` (one token): it then keeps the token, as gdn_conv_commit. Bitwise
diff --git a/include/strata/kernels/window_limits.hpp b/include/strata/kernels/window_limits.hpp
new file mode 100644
index 000000000..8520ec2e4
--- /dev/null
+++ b/include/strata/kernels/window_limits.hpp
@@ -0,0 +1,27 @@
+// include/strata/kernels/window_limits.hpp - how many rows one verify / batch window can hold.
+//
+// Until 0.2.7 a window held at most 8 rows. The CUDA and HIP engines now hold up to 16 (four pipelined groups of 16
+// slots = 64 slots on a four-card split: docs/BATCHING.md), so every compile-time maximum that follows the window
+// (the verifier's per-row arrays, the expert pool's per-token tables, the resident plan's one-thread-per-entry block)
+// is derived from ONE number. It is a macro because the SYCL and Metal builds compile the same shared headers and
+// must not change: they get the default, 8. The CUDA / HIP CMake sets STRATA_VERIFY_MAX_T from the cache variable
+// STRATA_WINDOW_MAX_T (8 or 16; -DSTRATA_WINDOW_MAX_T=8 builds an engine whose windows are the 0.2.7 ones).
+//
+// The rule every user of the number follows: a window of up to kVerifyNarrowT rows runs EXACTLY the code it ran at 8
+// (the same kernel instantiations, the same buffer sizes); the wider code is reached only by a window of 9..16 rows.
+#pragma once
+
+#ifndef STRATA_VERIFY_MAX_T
+#define STRATA_VERIFY_MAX_T 8
+#endif
+
+namespace strata::kernels {
+
+/// The window every 0.2.7 kernel was built for: windows of up to this many rows keep their instantiations.
+inline constexpr int kVerifyNarrowT = 8;
+/// The most rows one window can hold in this build (8 or 16).
+inline constexpr int kWindowMaxT = STRATA_VERIFY_MAX_T;
+
+static_assert(kWindowMaxT == 8 || kWindowMaxT == 16, "STRATA_VERIFY_MAX_T is 8 or 16 (the wide kernels are built for 16)");
+
+} // namespace strata::kernels
diff --git a/include/strata/program/pipe_probe.hpp b/include/strata/program/pipe_probe.hpp
new file mode 100644
index 000000000..9bd30b2ca
--- /dev/null
+++ b/include/strata/program/pipe_probe.hpp
@@ -0,0 +1,148 @@
+// include/strata/program/pipe_probe.hpp - STRATA_P4_TIMING / STRATA_P4_TRACE: where the wall time of a pipelined batch window goes.
+//
+// Opt-in measurement for the layer-split batch loop (generate.cpp, `pump`). Off, every hook is one predictable branch and
+// reads no clock. On:
+// STRATA_P4_TIMING=1 per stage: the window's time from launch to done, the cycle between launches, the gap between a
+// window done and the next launch, the stage's busy share, the host launch cost; the pump's cost
+// per tick, the verdict work per window, how many groups were in flight, the ticks a free stage had
+// a group waiting or was starved. Printed to stderr with the batch summary line.
+// STRATA_P4_TRACE=PATH one line per window event ("L stage time_us group" at a launch, "D stage time_us 0" when the stage
+// is done), flushed on every line so a killed run keeps its trace. "1" or an empty value writes
+// ./strata-p4-trace.csv.
+// Written for the four-card R9700 investigation (docs/BATCHING.md): it found that the stages run 4 of 4 concurrently in the
+// decode phase and that the serial admissions were the idle time.
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+namespace strata::program {
+
+class PipeProbe {
+public:
+ using Clock = std::chrono::steady_clock;
+ using Time = Clock::time_point;
+ static constexpr int kMaxStages = 16;
+ static constexpr int kMaxGroups = 64;
+
+ PipeProbe(int n_stages, size_t n_groups)
+ : n_(std::min(n_stages, kMaxStages)), n_groups_(n_groups) {
+ timing_ = std::getenv("STRATA_P4_TIMING") != nullptr;
+ const char* tr = std::getenv("STRATA_P4_TRACE");
+ if (tr != nullptr) {
+ const char* path = (tr[0] == '\0' || std::strcmp(tr, "1") == 0) ? "strata-p4-trace.csv" : tr;
+ trace_ = std::fopen(path, "w");
+ if (trace_ != nullptr) std::setvbuf(trace_, nullptr, _IONBF, 0);
+ else std::fprintf(stderr, "strata p4: cannot write the trace %s\n", path);
+ }
+ }
+ ~PipeProbe() { if (trace_ != nullptr) std::fclose(trace_); }
+ PipeProbe(const PipeProbe&) = delete;
+ PipeProbe& operator=(const PipeProbe&) = delete;
+
+ bool on() const { return timing_ || trace_ != nullptr; }
+ bool timing() const { return timing_; }
+ Time now() const { return on() ? Clock::now() : Time{}; }
+
+ /// The pump's cost: constructed at the top of `pump`, adds its life to the per-tick figure.
+ struct PumpScope {
+ PipeProbe* p = nullptr;
+ Time t0{};
+ explicit PumpScope(PipeProbe& probe) {
+ ++probe.ticks_;
+ if (probe.on()) { p = &probe; t0 = Clock::now(); }
+ }
+ ~PumpScope() { if (p != nullptr) p->pump_ms_ += ms(Clock::now() - t0); }
+ };
+ /// The verdict work of a window that reached the last stage.
+ struct VerdictScope {
+ PipeProbe* p = nullptr;
+ Time t0{};
+ explicit VerdictScope(PipeProbe& probe) {
+ if (probe.timing_) { p = &probe; t0 = Clock::now(); ++probe.verdict_n_; }
+ }
+ ~VerdictScope() { if (p != nullptr) p->verdict_ms_ += ms(Clock::now() - t0); }
+ };
+
+ /// Per tick: stages with a window in flight, groups in flight.
+ void busy(int stages, int groups) { busy_ += stages; inflight_ += groups; }
+ /// A stage k >= 1 that is free: whether a group is waiting at it (else it is starved by the stage before).
+ void free_stage(int k, bool group_waiting) {
+ if (k < 0 || k >= n_) return;
+ if (group_waiting) ++starve_idle_[k]; else ++starve_none_[k];
+ }
+ void span_start(Time t) { span_start_ = t; span_ms_ = 0; }
+ void span_end(double wall_ms) { span_ms_ = wall_ms; }
+ void polled(Time before) { if (on()) poll_ms_ += ms(Clock::now() - before); }
+ /// Stage k reported its window done.
+ void done(int k, int group) {
+ if (!on() || k < 0 || k >= n_) return;
+ const Time t = Clock::now();
+ if (timing_ && launched_[k]) { win_ms_[k] += ms(t - at_[k]); ++win_n_[k]; }
+ if (trace_ != nullptr) std::fprintf(trace_, "D %d %.3f 0\n", k, us(t));
+ done_at_[k] = t;
+ if (timing_ && group >= 0 && group < kMaxGroups && left_[group] != Time{}) { q_wait_ += ms(t - left_[group]); ++q_n_; }
+ }
+ /// Stage k launched `group`'s window; `before` is now() taken before the launch call.
+ void launched(int k, int group, Time before) {
+ if (!on() || k < 0 || k >= n_) return;
+ const Time t = Clock::now();
+ launch_ms_ += ms(t - before);
+ at_[k] = t;
+ launched_[k] = true;
+ if (group >= 0 && group < kMaxGroups) left_[group] = t;
+ if (trace_ != nullptr) std::fprintf(trace_, "L %d %.3f %d\n", k, us(t), group);
+ if (timing_) {
+ if (last_launch_[k] != Time{}) cycle_ms_[k] += ms(t - last_launch_[k]);
+ if (done_at_[k] != Time{}) gap_ms_[k] += ms(t - done_at_[k]);
+ last_launch_[k] = t;
+ }
+ }
+ /// Print and reset (the batch summary line calls it when the loop goes idle).
+ void flush() {
+ if (!timing_) return;
+ const double ticks = (double) std::max(1LL, ticks_);
+ std::fprintf(stderr, "strata p4: pump %.3f ms/tick over %lld ticks; stage-busy %.2f of %d; group wait %.1f ms;\n",
+ pump_ms_ / ticks, ticks_, busy_ / ticks, n_, q_n_ ? q_wait_ / (double) q_n_ : 0.0);
+ for (int k = 0; k < n_; ++k) {
+ const double n = win_n_[k] ? (double) win_n_[k] : 1.0;
+ std::fprintf(stderr, "strata p4: s%d windows %lld win %.2f ms cycle %.2f ms gap %.2f ms busy %.0f%% launch %.2f ms\n",
+ k, win_n_[k], win_ms_[k] / n, cycle_ms_[k] / n, gap_ms_[k] / n,
+ 100.0 * win_ms_[k] / std::max(1e-9, span_ms_), launch_ms_ / n);
+ }
+ std::fprintf(stderr, "strata p4: verdict work %.2f ms/window; inflight groups avg %.2f of %zu\n",
+ verdict_ms_ / std::max(1.0, (double) verdict_n_), inflight_ / ticks, n_groups_);
+ for (int k = 1; k < n_; ++k)
+ std::fprintf(stderr, "strata p4: s%d free ticks: with a group waiting %lld, starved %lld\n", k, starve_idle_[k],
+ starve_none_[k]);
+ pump_ms_ = poll_ms_ = launch_ms_ = verdict_ms_ = busy_ = inflight_ = q_wait_ = 0;
+ ticks_ = verdict_n_ = q_n_ = 0;
+ for (int k = 0; k < kMaxStages; ++k) {
+ win_ms_[k] = cycle_ms_[k] = gap_ms_[k] = 0;
+ win_n_[k] = starve_idle_[k] = starve_none_[k] = 0;
+ }
+ span_ms_ = 0;
+ }
+
+private:
+ static double ms(Clock::duration d) { return std::chrono::duration<double, std::milli>(d).count(); }
+ static double us(Time t) { return std::chrono::duration<double, std::micro>(t.time_since_epoch()).count(); }
+
+ int n_ = 0;
+ size_t n_groups_ = 0;
+ bool timing_ = false;
+ std::FILE* trace_ = nullptr;
+ double pump_ms_ = 0, poll_ms_ = 0, launch_ms_ = 0, verdict_ms_ = 0, busy_ = 0, inflight_ = 0, q_wait_ = 0, span_ms_ = 0;
+ long long ticks_ = 0, verdict_n_ = 0, q_n_ = 0;
+ double win_ms_[kMaxStages] = {}, cycle_ms_[kMaxStages] = {}, gap_ms_[kMaxStages] = {};
+ long long win_n_[kMaxStages] = {}, starve_idle_[kMaxStages] = {}, starve_none_[kMaxStages] = {};
+ Time at_[kMaxStages] = {}, last_launch_[kMaxStages] = {}, done_at_[kMaxStages] = {};
+ bool launched_[kMaxStages] = {};
+ Time left_[kMaxGroups] = {};
+ Time span_start_{};
+};
+
+} // namespace strata::program
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index 1b5b11552..912e207ba 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -7,6 +7,7 @@
#include "strata/core/remote_experts.hpp"
#include "strata/core/peer_experts.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/window_limits.hpp"
#include "strata/artifact/gguf_reader.hpp"
#include "strata/core/pinned.hpp"
@@ -4786,10 +4787,10 @@ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, cons
namespace {
// the verify window's per-entry tables in `expert_pool_dispatch_multi` (`kind`, `distinct`, `first_of`)
-// are fixed arrays of this many entries: MAXT tokens of the model's 10 routed experts must fit, and a larger k is
-// refused at run time rather than written past them.
-constexpr int64_t kMaxWindowEntries = 128;
-static_assert(strata::kernels::cpu::MAXT * 10 <= kMaxWindowEntries, "a verify window's entries overflow the tables");
+// are fixed arrays of this many entries: a window's rows (kWindowMaxT) of the model's 10 routed experts must fit, and a
+// larger k is refused at run time rather than written past them. 128 for the 8-row window (as it was), 256 for 16.
+constexpr int64_t kMaxWindowEntries = strata::kernels::kWindowMaxT <= 8 ? 128 : 256;
+static_assert(strata::kernels::kWindowMaxT * 10 <= kMaxWindowEntries, "a verify window's entries overflow the tables");
} // namespace
namespace {
@@ -4813,7 +4814,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
float* out, strata::kernels::cpu::NativeArithmetic arithmetic) {
using namespace strata::kernels::cpu;
if (d.failed) return;
- if (n_tok < 1 || n_tok > MAXT) {
+ if (n_tok < 1 || n_tok > strata::kernels::kWindowMaxT) {
d.failed = true;
d.fail = "a verify window has more tokens than the multi-token expert kernel takes";
d.fail_layer = d.layers;
@@ -4832,12 +4833,13 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
d.fail_layer = d.layers;
return;
}
- if ((int64_t) d.act_multi.size() < n_tok) d.act_multi.resize((size_t) MAXT);
+ if ((int64_t) d.act_multi.size() < n_tok) d.act_multi.resize((size_t) strata::kernels::kWindowMaxT);
const ExpertLayout& lay = expert_layout();
const bool native = lay.native;
- if (native && d.nact_multi.size() < (size_t) MAXT * kNativeActBytes) d.nact_multi.resize((size_t) MAXT * kNativeActBytes);
+ if (native && d.nact_multi.size() < (size_t) strata::kernels::kWindowMaxT * kNativeActBytes)
+ d.nact_multi.resize((size_t) strata::kernels::kWindowMaxT * kNativeActBytes);
if (d.job_of.size() != (size_t) d.n_expert) d.job_of.assign((size_t) d.n_expert, (int16_t) -1);
- if (d.jobs_multi.size() < (size_t) (n_tok * k)) d.jobs_multi.resize((size_t) (MAXT * k));
+ if (d.jobs_multi.size() < (size_t) (n_tok * k)) d.jobs_multi.resize((size_t) (strata::kernels::kWindowMaxT * k));
static const bool ptrace = std::getenv("STRATA_POOL_TRACE") != nullptr;
auto pt = [&](const char* what, long long a = -1) {
if (ptrace) { std::fprintf(stderr, "pool trace: layer %lld %s %lld\n", (long long) d.layers, what, a); std::fflush(stderr); }
@@ -5151,6 +5153,17 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
++d.cold_zc;
}
}
+ if (d.jobs_multi[(size_t) jo].nt == MAXT) {
+ // a window of 9..16 rows routes more than MAXT tokens to this expert: the kernels and the pool's scratch
+ // take MAXT tokens a job, so the rest are a further job over the same blob (each token's rows are its own,
+ // so the split changes no value; the blob is read once more)
+ const int16_t prev = jo;
+ jo = (int16_t) njobs++;
+ ExpertJobMulti& nj = d.jobs_multi[(size_t) jo];
+ nj.blob = d.jobs_multi[(size_t) prev].blob;
+ nj.fmt = d.jobs_multi[(size_t) prev].fmt;
+ nj.nt = 0;
+ }
ExpertJobMulti& jb = d.jobs_multi[(size_t) jo];
jb.act[jb.nt] = &d.act_multi[(size_t) t];
jb.nact[jb.nt] = native ? d.nact_multi.data() + (size_t) t * kNativeActBytes : nullptr;
diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp
index 64ad715ad..c7f31db14 100644
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@ -277,7 +277,8 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
if (ple_ss_ == nullptr) ple_ss_ = &ss;
max_t_ = max_t;
rt_dir_ = rt_dir;
- if (max_t < 1 || max_t > strata::kernels::kVerifyMaxT) { err = "mtp: max_t out of range"; return false; }
+ // (the draft layer's kernels - the resident grouping, the grouped experts, the fused reads - are built for the narrow window)
+ if (max_t < 1 || max_t > strata::kernels::kVerifyNarrowT) { err = "mtp: max_t out of range"; return false; }
if (shared != nullptr) {
if (shared->device_ != device_ || shared->g_ != &g || shared->dense_ == nullptr ||
shared->experts_ == nullptr || shared->rt_dir_ != rt_dir) {
diff --git a/src/core/peer_experts.cpp b/src/core/peer_experts.cpp
index 0320b75e5..4f589b788 100644
--- a/src/core/peer_experts.cpp
+++ b/src/core/peer_experts.cpp
@@ -8,6 +8,7 @@
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/quantize_act.hpp"
#include "strata/kernels/s2_expert_grouped.hpp"
+#include "strata/kernels/window_limits.hpp"
#include <algorithm>
#include <chrono>
@@ -26,7 +27,7 @@ bool peer_portable() { return g_peer_portable; }
namespace {
constexpr int64_t H = strata::kernels::cpu::H;
-constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10; // entries per layer: MAXT tokens x top-10
+constexpr int64_t CAP = strata::kernels::kWindowMaxT * 10; // entries per layer: a window's rows x top-10
struct Meta { // one block, uploaded with one copy per layer
unsigned long long ptr[CAP];
diff --git a/src/core/remote_expert_opt.cu b/src/core/remote_expert_opt.cu
index c92542f9e..531008e9e 100644
--- a/src/core/remote_expert_opt.cu
+++ b/src/core/remote_expert_opt.cu
@@ -4,6 +4,7 @@
#include "strata/kernels/cpu/expert_layout.hpp"
#include "strata/kernels/elementwise.hpp"
#include "strata/kernels/verify_kernels.hpp"
+#include "strata/kernels/window_limits.hpp"
#include <algorithm>
#include <cstring>
@@ -11,7 +12,7 @@
namespace strata::core {
namespace {
constexpr int64_t H = strata::kernels::cpu::H;
-constexpr int K = 10, MAXT = strata::kernels::cpu::MAXT, CAP = MAXT * K;
+constexpr int K = 10, MAXT = strata::kernels::kWindowMaxT, CAP = MAXT * K; // (a window's rows)
struct ReduceMeta { int32_t row[CAP]; float weight[CAP]; };
bool check(cudaError_t status, std::string& err) {
diff --git a/src/core/remote_experts.cpp b/src/core/remote_experts.cpp
index 6e6849595..c2791540c 100644
--- a/src/core/remote_experts.cpp
+++ b/src/core/remote_experts.cpp
@@ -7,6 +7,7 @@
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/quantize_act.hpp"
#include "strata/kernels/s2_expert_grouped.hpp"
+#include "strata/kernels/window_limits.hpp"
#include <algorithm>
#include <chrono>
@@ -18,7 +19,7 @@ namespace strata::core {
namespace {
constexpr int64_t H = strata::kernels::cpu::H;
constexpr int64_t FF = strata::kernels::cpu::FF;
-constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10;
+constexpr int64_t CAP = strata::kernels::kWindowMaxT * 10; // a window's rows x top-10
// Small enough to copy as one pinned buffer per layer. The kernels read the
// individual arrays through pointers into the same device allocation.
@@ -268,7 +269,7 @@ bool RemoteExperts::begin(int64_t layer, const float* x, const int32_t* ids, int
struct Timer { double& acc; std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now();
~Timer() { acc += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count(); } } timer{ms_begin_};
const int64_t n = n_tok * k;
- if (n <= 0 || n > CAP || n_tok > strata::kernels::cpu::MAXT || k != 10 || layer < 0 ||
+ if (n <= 0 || n > CAP || n_tok > strata::kernels::kWindowMaxT || k != 10 || layer < 0 ||
(size_t) layer >= layers_present_.size() || device_ < 0) {
err = "CUDA" + std::to_string(device_) + " experts: invalid layer, routing width or window size";
return false;
@@ -342,8 +343,9 @@ bool RemoteExperts::begin(int64_t layer, const float* x, const int32_t* ids, int
groups_, (int64_t) dst_.size(), d_q8_, d_scratch_, zero_copy_ && !reduce ? z_out_ : d_out_, s);
} else {
strata::kernels::quantize_q8_0_scaled(zero_copy_ ? z_x_ : d_x_, d_q8_, d_scales_, n_tok * H, s);
- strata::kernels::moe_grouped_s2(d_ptr_, d_start_, d_count_, d_dst_, d_tok_,
- groups_, (int64_t) dst_.size(), d_q8_, d_scales_, d_scratch_, zero_copy_ && !reduce ? z_out_ : d_out_, s);
+ strata::kernels::moe_grouped_s2_window(d_ptr_, d_start_, d_count_, d_dst_, d_tok_,
+ groups_, (int64_t) dst_.size(), d_q8_, d_scales_, d_scratch_, zero_copy_ && !reduce ? z_out_ : d_out_, s,
+ (int) n_tok);
}
const uint64_t compact_bytes = (uint64_t) (reduce ? n_tok : dst_.size()) * H * sizeof(float);
if (reduce) {
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index c8eef3f30..85b62e4ff 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -135,7 +135,7 @@ struct RouteResidentCfg {
RouteResidentCfg& c = route_resident_cfg();
for (const auto& kv : c.blocks) {
if (kv.second == nullptr) continue;
- unsigned long long h[8] = {};
+ unsigned long long h[8] = {}; // (the counter block allocated above: 8 words)
if (cudaSetDevice(kv.first) != cudaSuccess || cudaDeviceSynchronize() != cudaSuccess ||
cudaMemcpy(h, kv.second, sizeof(h), cudaMemcpyDeviceToHost) != cudaSuccess) {
(void) cudaGetLastError();
@@ -824,7 +824,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
return strata::metal_engine::window_init(metal_, g, ss, max_t, ss.qsa_states[ss.qsa_primary()].max_cells, &n_vocab_,
err);
#endif
- if (max_t < 2 || max_t > strata::kernels::kVerifyMaxT || max_t > strata::kernels::cpu::MAXT) {
+ if (max_t < 2 || max_t > strata::kernels::kVerifyMaxT) { // (the CPU experts take a wider window as jobs of at most cpu::MAXT tokens)
err = "verify: the window must hold 2.." + std::to_string(strata::kernels::kVerifyMaxT) + " tokens";
return false;
}
@@ -1476,7 +1476,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
pending = false;
}
auto gr_read_group = [&](int half, bool apply, float* inj_prev, float* inj_out) {
- FusedGrArgs fa[kFusedGrMaxT];
+ FusedGrArgs fa[kFusedGrWideT];
for (int t = tb; t < te; ++t) {
FusedGrArgs& a = fa[t - tb];
a.R = Rt(t); a.R_out = Rt(t); a.apply = apply;
@@ -2255,8 +2255,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
native_expert_grouped(L, gp, gs, gn, p_dst, p_tok, cap, cap,
nat_xq_ + (size_t) tb * (N / 32) * 36, hit_scratch_, dst_buf, cs, gy);
} else {
- moe_grouped_s2(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,
- hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, dst_buf, cs);
+ moe_grouped_s2_window(gp, gs, gn, p_dst, p_tok, cap, cap, hit_xq_ + (size_t) tb * (N / 32) * 34,
+ hit_xs_ + (size_t) tb * (N / 32), hit_scratch_, dst_buf, cs, n); // (a group holds <= n entries)
}
};
bool hits_in_combine = false;
@@ -2413,7 +2413,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
// (no pending write, no inject)
const bool mix_q8 = hd->hc_q8 != nullptr && hu->hc_q8 != nullptr && head_ != nullptr && head_->loaded();
if (mix_q8) {
- FusedGrArgs fa[kFusedGrMaxT];
+ FusedGrArgs fa[kFusedGrWideT];
for (int t = 0; t < T; ++t) {
FusedGrArgs& a = fa[t];
a.R = Rt(t); a.R_out = Rt(t); a.apply = false;
@@ -2426,7 +2426,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
fused_gr_read_multi(fa, T, xn_, cs);
}
else if (fuse_head_gr) {
- FusedGrArgs fa[kFusedGrMaxT];
+ FusedGrArgs fa[kFusedGrWideT];
for (int t = 0; t < T; ++t) {
FusedGrArgs& a = fa[t];
a.R = Rt(t); a.R_out = Rt(t); a.apply = true;
@@ -2449,7 +2449,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
err = "verify: the output_hc_* weights have the wrong engine forms";
return false;
}
- FusedGrArgs fa[kFusedGrMaxT];
+ FusedGrArgs fa[kFusedGrWideT];
for (int t = 0; t < T; ++t) {
fa[t].R = Rt(t); fa[t].R_out = Rt(t); fa[t].apply = false;
fa[t].w_norm = (const float*) hn->data; fa[t].w_down = (const uint16_t*) hd->data;
@@ -4157,9 +4157,10 @@ bool Verifier::evict_batch_graph(const std::vector<int>& keep, bool& evicted, st
bool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {
evicted = false;
if (no_window_eviction_ || keep_t < 1) return true;
- bool have[9] = {};
- for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[sel_long_][t] : exec_[sel_long_][t]) != nullptr;
- const int victim = strata::core::pick_window_victim(have, std::min(max_t_, 8), keep_t, prelaunched_);
+ bool have[strata::kernels::kWindowMaxT + 1] = {};
+ const int top = std::min<int>(max_t_, strata::kernels::kWindowMaxT);
+ for (int t = 1; t <= top; ++t) have[t] = (ar_off_ ? exec_nr_[sel_long_][t] : exec_[sel_long_][t]) != nullptr;
+ const int victim = strata::core::pick_window_victim(have, top, keep_t, prelaunched_);
if (victim == 0) return true;
if (cudaStreamSynchronize(cs_) != cudaSuccess) {
err = "verify: synchronizing before window graph eviction failed";
@@ -4317,9 +4318,9 @@ bool Verifier::stage_batch(const int* rows, int S, int hbase, const int32_t* tok
std::string& err) {
using namespace strata::kernels;
// the row layout: a slot may own a contiguous group of rows (--batch-mtp), also across the stages of a layer split,
- // whose hand-off buffers hold kVerifyMaxT rows, or set_handoff_rows' (a stage that hands rows on or takes them)
+ // whose hand-off buffers hold kVerifyNarrowT rows, or set_handoff_rows' (a stage that hands rows on or takes them)
if (const char* why = batch_rows_error(rows, S, pos, (int) slots_.size(), max_t_, hbase,
- next_ != nullptr || lb_ > 0, handoff_rows_ > 0 ? handoff_rows_ : kVerifyMaxT)) {
+ next_ != nullptr || lb_ > 0, handoff_rows_ > 0 ? handoff_rows_ : kVerifyNarrowT)) {
err = why;
return false;
}
@@ -4394,8 +4395,8 @@ bool Verifier::stage_batch(const int* rows, int S, int hbase, const int32_t* tok
bool Verifier::run_slots(int S, const int32_t* tokens, const int64_t* pos, PoolMultiFn pool, void* user, int32_t* out,
std::string& err) {
- int rows[8] = {};
- for (int t = 0; t < S && t < 8; ++t) rows[t] = t;
+ int rows[strata::kernels::kVerifyMaxT] = {};
+ for (int t = 0; t < S && t < strata::kernels::kVerifyMaxT; ++t) rows[t] = t;
return run_slot_rows(rows, S, tokens, pos, pool, user, out, err);
}
@@ -4571,8 +4572,8 @@ bool Verifier::sample_rows(int S, std::string& err) {
bool Verifier::batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err) {
const OnDevice on_device(device_);
if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; }
- int rows[8] = {};
- for (int t = 0; t < S && t < 8; ++t) rows[t] = base + t;
+ int rows[strata::kernels::kVerifyMaxT] = {};
+ for (int t = 0; t < S && t < strata::kernels::kVerifyMaxT; ++t) rows[t] = base + t;
if (pre_launch_ && !pre_launch_(cs_, err)) return false;
if (!stage_batch(rows, S, base, tokens, pos, err)) return false;
cudaError_t le = cudaGraphLaunch(exec_bm_[bkey(rows, S, base)], cs_);
diff --git a/src/core/window_limits_test.cpp b/src/core/window_limits_test.cpp
new file mode 100644
index 000000000..3c89cc400
--- /dev/null
+++ b/src/core/window_limits_test.cpp
@@ -0,0 +1,64 @@
+// The window limits (include/strata/kernels/window_limits.hpp) and the pipelined-window probe
+// (include/strata/program/pipe_probe.hpp): CPU only, header-only.
+#include "strata/kernels/cpu/expert.hpp"
+#include "strata/kernels/fused_gr.hpp"
+#include "strata/kernels/verify_kernels.hpp"
+#include "strata/program/pipe_probe.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+ ++checks;
+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+} // namespace
+
+int main() {
+ using namespace strata::kernels;
+ check(kVerifyNarrowT == 8, "the narrow window is the 0.2.7 one: 8 rows");
+ check(kWindowMaxT == 8 || kWindowMaxT == 16, "a window holds 8 or 16 rows");
+ check(kVerifyMaxT == kWindowMaxT, "kVerifyMaxT is the window limit");
+ check(kVerifyMaxT >= kVerifyNarrowT, "the wide window is not narrower than the narrow one");
+ check(cpu::MAXT == 8, "the CPU kernels' tokens per job stay 8 (a wider window is split into jobs)");
+ check(kFusedGrMaxT == 8, "the hyper-connection kernels are built for 8 tokens (a wider window is read in slices)");
+ check(kFusedGrWideT == kWindowMaxT, "the caller's token arrays follow the window");
+ check(kVerifyMaxT * 10 <= 256, "the resident plan's block holds a full window of 10 routed experts a row");
+
+ // the probe: off without the environment (no clock read), and a full cycle of events does not fail when it is on
+ unsetenv("STRATA_P4_TIMING");
+ unsetenv("STRATA_P4_TRACE");
+ {
+ strata::program::PipeProbe off(4, 4);
+ check(!off.on() && !off.timing(), "the probe is off by default");
+ check(off.now() == strata::program::PipeProbe::Time{}, "an off probe reads no clock");
+ { strata::program::PipeProbe::PumpScope s(off); }
+ off.launched(0, 1, off.now());
+ off.done(0, 1);
+ off.flush();
+ }
+ setenv("STRATA_P4_TIMING", "1", 1);
+ {
+ strata::program::PipeProbe on(4, 4);
+ check(on.timing(), "STRATA_P4_TIMING turns the timing on");
+ on.span_start(on.now());
+ for (int w = 0; w < 3; ++w)
+ for (int k = 0; k < 4; ++k) {
+ strata::program::PipeProbe::PumpScope s(on);
+ on.busy(2, 3);
+ on.free_stage(1, k % 2 == 0);
+ const auto before = on.now();
+ on.launched(k, w, before);
+ on.polled(before);
+ on.done(k, w);
+ strata::program::PipeProbe::VerdictScope v(on);
+ }
+ on.span_end(10.0);
+ on.flush();
+ }
+ unsetenv("STRATA_P4_TIMING");
+ std::printf("window_limits_test: %d checks passed (window %d rows)\n", checks, (int) kVerifyMaxT);
+ return 0;
+}
diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cu
index 12c74a5dc..24b001431 100644
--- a/src/kernels/cuda/fused_gr.cu
+++ b/src/kernels/cuda/fused_gr.cu
@@ -2330,8 +2330,10 @@ static bool gr_v3_default_on(int dev) {
#endif
}
-bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,
- int stamp_i0) {
+// One read of at most kFusedGrMaxT tokens: the kernels below are built for that many (the instantiations, the shared
+// memory, the by-value launch argument), whatever the window is.
+static bool fused_gr_read_multi_slice(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,
+ unsigned long long* stamp_buf, int stamp_i0) {
if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {
std::fprintf(stderr, "fused_gr_read_multi: invalid arguments\n");
std::exit(1);
@@ -2466,6 +2468,30 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
return q8;
}
+// The multi read for a window of up to kFusedGrWideT tokens. Up to kFusedGrMaxT tokens it is the read above untouched
+// (the same kernels, the same bits). A wider window (9..16 rows: the batch groups of a four-card split) is read in
+// slices of at most kFusedGrMaxT tokens, in order on the stream. Every token's outputs are strictly its own (no value
+// crosses tokens), so the slices are bitwise the one wide read; the cost is the hyper-connection weights read once per
+// slice. The STRATA_QFUSE counters belong to token 0: each slice's first token carries them (the kernels zero them
+// again for the next launch); the profile stamps are the first slice's.
+bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,
+ int stamp_i0) {
+ if (n_tok <= kFusedGrMaxT) return fused_gr_read_multi_slice(a, n_tok, xn_scratch, stream, stamp_buf, stamp_i0);
+ if (n_tok > kFusedGrWideT || a == nullptr || xn_scratch == nullptr) {
+ std::fprintf(stderr, "fused_gr_read_multi: invalid arguments (%d tokens)\n", n_tok);
+ std::exit(1);
+ }
+ bool q8 = false;
+ for (int c0 = 0; c0 < n_tok; c0 += kFusedGrMaxT) {
+ const int ct = n_tok - c0 < kFusedGrMaxT ? n_tok - c0 : kFusedGrMaxT;
+ FusedGrArgs s[kFusedGrMaxT];
+ for (int k = 0; k < ct; ++k) s[k] = a[c0 + k];
+ s[0].q8_cnt = a[0].q8_cnt;
+ q8 = fused_gr_read_multi_slice(s, ct, xn_scratch + (size_t) c0 * D, stream, c0 == 0 ? stamp_buf : nullptr, stamp_i0);
+ }
+ return q8;
+}
+
// STRATA_HC_Q8 on a bf16 pack: one warp a Q8_0 block, ggml's reference rounding (d = amax / 127 as fp16, q = round(x * (1 / d)))
__global__ void __launch_bounds__(256) gr_quantize_q8_0_kernel(const uint16_t* __restrict__ w, uint8_t* __restrict__ out, int64_t nb) {
const int64_t b = (int64_t) blockIdx.x * 8 + (threadIdx.x >> 5);
diff --git a/src/kernels/cuda/native_bf16.cu b/src/kernels/cuda/native_bf16.cu
index 1f4b98acc..ff6c439ef 100644
--- a/src/kernels/cuda/native_bf16.cu
+++ b/src/kernels/cuda/native_bf16.cu
@@ -1,4 +1,5 @@
#include "strata/kernels/bf16_gemv.hpp"
+#include "strata/kernels/window_limits.hpp"
#include "s26_tsum.cuh"
#include "strata/kernels/bf16_bits.hpp"
#include "strata/kernels/pdl.hpp"
@@ -295,12 +296,12 @@ bool bf16_gemv_fp32_mmvf_multi_aux(const float* x, int64_t ldx, const uint16_t*
void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,
int64_t n_in, int64_t n_out, int n_tok, void* stream) {
if (n_tok == 1 && ldy >= n_out) { bf16_gemv_fp32_mmvf(x, w, y, n_in, n_out, stream); return; }
- if (n_tok < 1 || n_tok > 8 || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 || x == nullptr ||
- w == nullptr || y == nullptr || (reinterpret_cast<uintptr_t>(x) & 7u) != 0)
- throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows, even n_in/ldx, aligned pointers");
+ if (n_tok < 1 || n_tok > STRATA_VERIFY_MAX_T || n_in <= 0 || (n_in & 1) != 0 || n_out <= 0 || (ldx & 1) != 0 ||
+ x == nullptr || w == nullptr || y == nullptr || (reinterpret_cast<uintptr_t>(x) & 7u) != 0)
+ throw std::invalid_argument("bf16_gemv_fp32_mmvf_multi: 1..8 rows (16 in a wide build), even n_in/ldx, aligned pointers");
const cudaStream_t st = (cudaStream_t) stream;
static const bool rows = [] { const char* v = std::getenv("STRATA_MMVF_ROWS"); return v && v[0] == '1'; }();
- if (rows && n_out >= 64) {
+ if (rows && n_out >= 64 && n_tok <= 8) { // (the opt-in rows kernels hold 8 columns: a wider window takes the switch below)
const int ni = (int) n_in, no = (int) n_out;
switch (mmvf_block_size(n_in)) {
case 32: launch_rows<32>(x, ldx, w, y, ldy, ni, no, n_tok, st); break;
@@ -317,8 +318,19 @@ void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, f
throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result));
return;
}
+ // 9..16 rows (wide builds only): one exact instantiation per row count, each row's dot independent of the others. 7 and 8
+ // rows stay with the default case below, the instantiation they always had.
+#if STRATA_VERIFY_MAX_T > 8
+#define STRATA_MMVF_W(N, T) case T: launch_pdl(bf16_f32_mmvf_multi_kernel<N, T, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break;
+#define STRATA_MMVF_WIDE(N) \
+ STRATA_MMVF_W(N, 9) STRATA_MMVF_W(N, 10) STRATA_MMVF_W(N, 11) STRATA_MMVF_W(N, 12) \
+ STRATA_MMVF_W(N, 13) STRATA_MMVF_W(N, 14) STRATA_MMVF_W(N, 15) STRATA_MMVF_W(N, 16)
+#else
+#define STRATA_MMVF_WIDE(N)
+#endif
#define STRATA_MMVF_M(N) case N: \
switch (n_tok) { \
+ STRATA_MMVF_WIDE(N) \
case 1: launch_pdl(bf16_f32_mmvf_multi_kernel<N, 1, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \
case 2: launch_pdl(bf16_f32_mmvf_multi_kernel<N, 2, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \
case 3: launch_pdl(bf16_f32_mmvf_multi_kernel<N, 3, true>, dim3((unsigned) n_out), dim3(N), 0, st, x, ldx, w, y, ldy, (int) n_in, n_tok); break; \
@@ -332,6 +344,10 @@ void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, f
STRATA_MMVF_M(160); STRATA_MMVF_M(192); STRATA_MMVF_M(224); STRATA_MMVF_M(256);
}
#undef STRATA_MMVF_M
+#undef STRATA_MMVF_WIDE
+#ifdef STRATA_MMVF_W
+#undef STRATA_MMVF_W
+#endif
const cudaError_t result = cudaGetLastError();
if (result != cudaSuccess)
throw std::runtime_error(std::string("bf16_gemv_fp32_mmvf_multi launch: ") + cudaGetErrorString(result));
diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cu
index c806e1630..0f2bdf7dc 100644
--- a/src/kernels/cuda/native_mmvq.cu
+++ b/src/kernels/cuda/native_mmvq.cu
@@ -888,7 +888,11 @@ __global__ void native_small_mmvq_kernel(const Weight* __restrict__ w,
// row) value is accumulated over kbx in the same order, summed across warps in the same order and reduced with
// the same warp tree as the ncols = 1 kernel, so every column is BITWISE equal to a single-column call on that
// column (checked by bench/micro/native_mmvq_multi.cpp). The ncols = 1 kernels are untouched.
-constexpr int MAX_NCOLS = 8;
+// Columns one launch takes: 8, the 0.2.7 limit, or 16 where windows are wide (window_limits.hpp: the cases 9..16 are
+// compiled only then, so an engine built with STRATA_WINDOW_MAX_T=8 has exactly the kernels it had). A window has one
+// column per row. The wide launches read every weight once for all their columns; running 16 columns as two 8-column
+// passes (native_mmvq below does it for more than MAX_NCOLS) reads the dense weights twice and measured slower than 8 rows.
+constexpr int MAX_NCOLS = STRATA_VERIFY_MAX_T > 8 ? 16 : 8;
// Each format splits its dot product into `load` (everything that depends only on the weight block: codes,
// unpacked scales, block scale) and `apply` (the activation loads and the original *_impl expression). `load` runs
@@ -1583,7 +1587,17 @@ void launch_multi(const void* weights, const void* x_q8_1, float* y, int n_in, i
case 6: launch_multi_n<F, 6>(weights, x_q8_1, y, n_in, n_out, s); break;
case 7: launch_multi_n<F, 7>(weights, x_q8_1, y, n_in, n_out, s); break;
case 8: launch_multi_n<F, 8>(weights, x_q8_1, y, n_in, n_out, s); break;
- default: throw std::invalid_argument("native MMVQ multi-column launch requires 2 <= ncols <= 8");
+#if STRATA_VERIFY_MAX_T > 8
+ case 9: launch_multi_n<F, 9>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 10: launch_multi_n<F, 10>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 11: launch_multi_n<F, 11>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 12: launch_multi_n<F, 12>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 13: launch_multi_n<F, 13>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 14: launch_multi_n<F, 14>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 15: launch_multi_n<F, 15>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 16: launch_multi_n<F, 16>(weights, x_q8_1, y, n_in, n_out, s); break;
+#endif
+ default: throw std::invalid_argument("native MMVQ multi-column launch requires 2 <= ncols <= MAX_NCOLS");
}
}
@@ -2307,7 +2321,7 @@ void validate_shape(int n_in, int ncols, int block_elems = Q8K) {
if (n_in <= 0 || n_in % block_elems != 0) {
throw std::invalid_argument("native MMVQ requires n_in > 0 and divisible by its block element count");
}
- if (ncols < 1 || ncols > MAX_NCOLS) throw std::invalid_argument("native MMVQ requires 1 <= ncols <= 8");
+ if (ncols < 1 || ncols > MAX_NCOLS) throw std::invalid_argument("native MMVQ requires 1 <= ncols <= MAX_NCOLS");
}
void validate_pointer(const void* p) {
if (!p || reinterpret_cast<std::uintptr_t>(p) % 4 != 0) {
@@ -2386,7 +2400,17 @@ void wave_launch(const void* weights, const void* x_q8_1, float* y, int n_in, in
case 6: wave_launch_n<F, 6>(weights, x_q8_1, y, n_in, n_out, s); break;
case 7: wave_launch_n<F, 7>(weights, x_q8_1, y, n_in, n_out, s); break;
case 8: wave_launch_n<F, 8>(weights, x_q8_1, y, n_in, n_out, s); break;
- default: throw std::invalid_argument("native MMVQ (wave) requires 1 <= ncols <= 8");
+#if STRATA_VERIFY_MAX_T > 8
+ case 9: wave_launch_n<F, 9>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 10: wave_launch_n<F, 10>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 11: wave_launch_n<F, 11>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 12: wave_launch_n<F, 12>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 13: wave_launch_n<F, 13>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 14: wave_launch_n<F, 14>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 15: wave_launch_n<F, 15>(weights, x_q8_1, y, n_in, n_out, s); break;
+ case 16: wave_launch_n<F, 16>(weights, x_q8_1, y, n_in, n_out, s); break;
+#endif
+ default: throw std::invalid_argument("native MMVQ (wave) requires 1 <= ncols <= MAX_NCOLS");
}
}
#define STRATA_WAVE_MMVQ(...) \
@@ -2569,8 +2593,16 @@ bool q8_packed_mmvq(const void* weights, const void* x_q8_1, float* y, int n_in,
validate_pointer(x_q8_1);
validate_pointer(y);
validate_stream(stream);
- if (q8sm60::launch(found->second.qs, found->second.d, static_cast<const Q81Block*>(x_q8_1), y, n_in,
- n_out, ncols, static_cast<cudaStream_t>(stream))) {
+ // (the kernel takes at most q8sm60::MAX_COLS columns; a 9..16-row window is run in passes, each column bitwise
+ // the one-column result, as native_mmvq does for more than MAX_NCOLS)
+ bool all = true;
+ for (int j = 0; all && j < ncols; j += q8sm60::MAX_COLS) {
+ const int nj = ncols - j < q8sm60::MAX_COLS ? ncols - j : q8sm60::MAX_COLS;
+ all = q8sm60::launch(found->second.qs, found->second.d,
+ static_cast<const Q81Block*>(x_q8_1) + std::size_t(j) * std::size_t(n_in / Q8K),
+ y + std::size_t(j) * std::size_t(n_out), n_in, n_out, nj, static_cast<cudaStream_t>(stream));
+ }
+ if (all) {
launch_check();
return true;
}
@@ -2597,7 +2629,17 @@ bool q8_packed_mmvq(const void* weights, const void* x_q8_1, float* y, int n_in,
case 6: q8_packed_launch<6>(found->second, x, y, s); break;
case 7: q8_packed_launch<7>(found->second, x, y, s); break;
case 8: q8_packed_launch<8>(found->second, x, y, s); break;
- default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= 8");
+#if STRATA_VERIFY_MAX_T > 8
+ case 9: q8_packed_launch<9>(found->second, x, y, s); break;
+ case 10: q8_packed_launch<10>(found->second, x, y, s); break;
+ case 11: q8_packed_launch<11>(found->second, x, y, s); break;
+ case 12: q8_packed_launch<12>(found->second, x, y, s); break;
+ case 13: q8_packed_launch<13>(found->second, x, y, s); break;
+ case 14: q8_packed_launch<14>(found->second, x, y, s); break;
+ case 15: q8_packed_launch<15>(found->second, x, y, s); break;
+ case 16: q8_packed_launch<16>(found->second, x, y, s); break;
+#endif
+ default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= MAX_NCOLS");
}
launch_check();
return true;
@@ -2732,7 +2774,17 @@ bool q6_packed_mmvq(const void* weights, const void* x_q8_1, float* y, int n_in,
case 6: q6_packed_launch<6>(found->second, x, y, s); break;
case 7: q6_packed_launch<7>(found->second, x, y, s); break;
case 8: q6_packed_launch<8>(found->second, x, y, s); break;
- default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= 8");
+#if STRATA_VERIFY_MAX_T > 8
+ case 9: q6_packed_launch<9>(found->second, x, y, s); break;
+ case 10: q6_packed_launch<10>(found->second, x, y, s); break;
+ case 11: q6_packed_launch<11>(found->second, x, y, s); break;
+ case 12: q6_packed_launch<12>(found->second, x, y, s); break;
+ case 13: q6_packed_launch<13>(found->second, x, y, s); break;
+ case 14: q6_packed_launch<14>(found->second, x, y, s); break;
+ case 15: q6_packed_launch<15>(found->second, x, y, s); break;
+ case 16: q6_packed_launch<16>(found->second, x, y, s); break;
+#endif
+ default: throw std::invalid_argument("native MMVQ requires 1 <= ncols <= MAX_NCOLS");
}
launch_check();
return true;
@@ -2844,7 +2896,7 @@ bool native_q8_0_sm60_pack_in_place(void* weights, int n_in, int n_out, const ch
if (q8_packed_registry().count(weights)) return true;
// The GGUF blocks are gone afterwards, so only where q8_packed_mmvq can never fall back to the GGUF-layout kernels
// (every column count fits the kernel's shared-memory planes) and its 16-byte weight loads stay aligned
- if (q8sm60::smem_bytes(MAX_NCOLS, n_in) > q8sm60::MAX_SMEM || reinterpret_cast<std::uintptr_t>(weights) % 16 != 0)
+ if (q8sm60::smem_bytes(q8sm60::MAX_COLS, n_in) > q8sm60::MAX_SMEM || reinterpret_cast<std::uintptr_t>(weights) % 16 != 0)
return native_q8_0_sm60_pack(weights, n_in, n_out, what);
const std::size_t bytes = std::size_t(n_in / 32) * std::size_t(n_out) * 34; // both layouts: 34 bytes per 32
std::vector<uint8_t> blocks(bytes), planes(bytes);
@@ -3476,6 +3528,19 @@ std::size_t native_mmvq_weight_bytes(int ggml_type, int n_in, int n_out) {
void native_mmvq(int ggml_type, const void* weights, const void* x_q8_1, float* y,
int n_in, int n_out, int ncols, void* stream) {
+ // more columns than one launch takes: ceil(ncols / MAX_NCOLS) passes over the same weights. Every column of a
+ // multi-column pass is bitwise the single-column result (bench/micro/native_mmvq_multi.cpp), so the split changes no
+ // arithmetic. Column j starts at activation block j * (n_in / Q8K) and writes y + j * n_out. (Never reached by a
+ // window of up to 16 rows in a wide build, nor of up to 8 in a narrow one: a guard for a caller that asks for more.)
+ if (ncols > MAX_NCOLS) {
+ const std::size_t col_blocks = std::size_t(n_in / Q8K);
+ for (int j = 0; j < ncols; j += MAX_NCOLS) {
+ const int nj = (ncols - j < MAX_NCOLS) ? (ncols - j) : MAX_NCOLS;
+ native_mmvq(ggml_type, weights, static_cast<const Q81Block*>(x_q8_1) + std::size_t(j) * col_blocks,
+ y + std::size_t(j) * std::size_t(n_out), n_in, n_out, nj, stream);
+ }
+ return;
+ }
switch (ggml_type) {
case 2: native_q4_0_mmvq(weights, x_q8_1, y, n_in, n_out, ncols, stream); break;
case 6: native_q5_0_mmvq(weights, x_q8_1, y, n_in, n_out, ncols, stream); break;
diff --git a/src/kernels/cuda/native_moe.cu b/src/kernels/cuda/native_moe.cu
index fc28c4a91..3e0117abd 100644
--- a/src/kernels/cuda/native_moe.cu
+++ b/src/kernels/cuda/native_moe.cu
@@ -21,6 +21,7 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "strata/kernels/native_moe.hpp"
+#include "strata/kernels/verify_kernels.hpp"
#include <cuda_runtime.h>
#include <atomic>
#include <cstddef>
@@ -175,14 +176,14 @@ void native_moe_combine(const float* parts, const float* weights, const float* s
if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error));
}
bool native_moe_combine_hits_supported(const float* parts, const float* shared, const float* output, int64_t n_embd, int64_t k, int n_tok) {
- return k == 10 && shared != nullptr && n_tok >= 1 && n_tok <= 8 && n_embd > 0 && (n_embd & 3) == 0 &&
+ return k == 10 && shared != nullptr && n_tok >= 1 && n_tok <= kVerifyMaxT && n_embd > 0 && (n_embd & 3) == 0 &&
(((uintptr_t) parts | (uintptr_t) shared | (uintptr_t) output) & 15u) == 0;
}
void native_moe_combine_multi_hits_added(const float* parts, const float* hits, const int32_t* dst, const int32_t* count,
const float* weights, const float* shared, float* output, int64_t n_embd, int64_t k,
int n_tok, void* stream) {
if (!stream || !native_moe_combine_hits_supported(parts, shared, output, n_embd, k, n_tok) || ((uintptr_t) hits & 15u) != 0)
- throw std::invalid_argument("native MoE combine (hits added) requires k = 10, aligned spans, 1..8 tokens");
+ throw std::invalid_argument("native MoE combine (hits added) requires k = 10, aligned spans, 1..8 tokens (16 in a wide build)");
const int64_t n4 = n_embd >> 2;
combine_k10_vec4_hits<<<dim3(unsigned((n4 + 127) / 128), (unsigned) n_tok), 128, 0, static_cast<cudaStream_t>(stream)>>>(
reinterpret_cast<const float4*>(parts), reinterpret_cast<const float4*>(hits), dst, count, weights,
diff --git a/src/kernels/cuda/q8_sm60.cuh b/src/kernels/cuda/q8_sm60.cuh
index a45b84095..ef1d9fd22 100644
--- a/src/kernels/cuda/q8_sm60.cuh
+++ b/src/kernels/cuda/q8_sm60.cuh
@@ -22,6 +22,7 @@ namespace q8sm60 {
constexpr int THREADS = 256;
constexpr std::size_t MAX_SMEM = 48 * 1024; // a Pascal workgroup's static + dynamic limit
+constexpr int MAX_COLS = 8; // columns of one launch (a window of 9..16 rows runs in passes)
inline std::size_t smem_bytes(int ncols, int n_in) {
return std::size_t(ncols) * std::size_t(n_in) + std::size_t(ncols) * std::size_t(n_in / 32) * sizeof(float);
@@ -118,7 +119,7 @@ void launch_cols(const int8_t* qs, const half* d, const Q81Block* x, float* y, i
// false: a shape this kernel does not take (the caller runs its usual kernel)
inline bool launch(const int8_t* qs, const half* d, const Q81Block* x, float* y, int n_in, int n_out, int ncols,
cudaStream_t s) {
- if (n_in % 32 != 0 || n_in < 128 || ncols < 1 || ncols > 8 || smem_bytes(ncols, n_in) > MAX_SMEM) return false;
+ if (n_in % 32 != 0 || n_in < 128 || ncols < 1 || ncols > MAX_COLS || smem_bytes(ncols, n_in) > MAX_SMEM) return false;
switch (ncols) {
case 1: launch_cols<1>(qs, d, x, y, n_in, n_out, s); break;
case 2: launch_cols<2>(qs, d, x, y, n_in, n_out, s); break;
diff --git a/src/kernels/cuda/s2_expert_grouped.cu b/src/kernels/cuda/s2_expert_grouped.cu
index bff8b21e6..1fe69788d 100644
--- a/src/kernels/cuda/s2_expert_grouped.cu
+++ b/src/kernels/cuda/s2_expert_grouped.cu
@@ -824,9 +824,14 @@ namespace {
// hit kernel's.
constexpr int GU_CHUNKS = (H / 32 + 31) / 32; // 3: chunks of a gate/up row per lane (80 chunks / 32 lanes)
constexpr int GMAX = 8; // entries per group (tokens routed to one expert in a window)
-// a group holds one entry per token of the window routed to its expert, and the kernels below keep at
-// most GMAX of them (`min(..., GMAX)`): a longer window would drop entries without a word.
-static_assert(GMAX >= kVerifyMaxT, "a verify window's group can exceed GMAX entries");
+// a group holds one entry per token of the window routed to its expert, and a kernel keeps at most GMAX of them
+// (`min(..., GMAX)`): a longer group would drop entries without a word. A window of 9..16 rows can route more tokens
+// than that to one expert, so `moe_grouped_s2` takes the window's largest group and runs the kernel again with
+// SKIP = GMAX over the entries from the GMAX-th on (the staging is GMAX entries wide in shared memory: 51 KB at 16
+// would not compile on CUDA, whose static limit is 48 KB). Every entry's output is its own dot products, so the
+// passes are bitwise the one wide pass; SKIP = 0 is the kernel as it was.
+constexpr int kGroupPasses = (kVerifyMaxT + GMAX - 1) / GMAX;
+static_assert(kGroupPasses * GMAX >= kVerifyMaxT, "a verify window's group can exceed the passes' entries");
constexpr int GU_ROWS = 32; // gate/up rows per block: 4 per warp
constexpr int D_ROWS = 64; // down rows per block: 8 per warp
@@ -850,6 +855,7 @@ __device__ __forceinline__ float chunk_dot(uint2 cb, const int* xw, float dw, fl
// Gate/up: a block = GU_ROWS rows of ONE group. The group's activations (each entry's token row of x_q8_0 and
// its fp32 scales) are staged once into shared memory as aligned words; each warp then walks its rows, loading
// each lane's code chunks once and dotting them with every entry.
+template <int SKIP>
__global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
@@ -861,7 +867,8 @@ __global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long lon
__shared__ float xs_d[GMAX][H / 32];
const int g = blockIdx.y;
if (g >= *n_groups) return;
- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);
+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
+ if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * (H / 32); i += blockDim.x) {
const int k = i / (H / 32), c = i - k * (H / 32);
@@ -912,6 +919,7 @@ __global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long lon
// Down: a block = D_ROWS rows of ONE group; the entries' quantized intermediates staged once. A down row is 20
// chunks, so lanes 0..19 each hold one chunk, as in the per-entry kernel.
+template <int SKIP>
__global__ void __launch_bounds__(256) down_grouped_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
@@ -923,7 +931,8 @@ __global__ void __launch_bounds__(256) down_grouped_kernel(const unsigned long l
__shared__ float hs_d[GMAX][FF / 32];
const int g = blockIdx.y;
if (g >= *n_groups) return;
- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);
+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
+ if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * (FF / 32); i += blockDim.x) {
const int k = i / (FF / 32), c = i - k * (FF / 32);
@@ -985,6 +994,7 @@ __device__ __forceinline__ void stage_chunk(const uint8_t* __restrict__ xb, floa
xs_dh[i] = make_int2(__float_as_int(dx), hx);
}
+template <int SKIP>
__global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
@@ -997,7 +1007,8 @@ __global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long l
__shared__ int2 xs_dh[GMAX * NC]; // 5 KB: (dx as bits, hx)
const int g = blockIdx.y;
if (g >= *n_groups) return;
- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);
+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
+ if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * NC; i += blockDim.x) {
const int k = i / NC, c = i - k * NC;
@@ -1051,6 +1062,7 @@ __global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long l
}
}
+template <int SKIP>
__global__ void __launch_bounds__(256) down_grouped_t_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
@@ -1063,7 +1075,8 @@ __global__ void __launch_bounds__(256) down_grouped_t_kernel(const unsigned long
__shared__ int2 hs_dh[GMAX * NC];
const int g = blockIdx.y;
if (g >= *n_groups) return;
- const int e0 = grp_start[g], ne = min(grp_start[g + 1] - e0, GMAX);
+ const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
+ if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * NC; i += blockDim.x) {
const int k = i / NC, c = i - k * NC;
@@ -1166,10 +1179,18 @@ void moe_group_resident(const int32_t* ids, int n, int k_per_tok, const uint8_t*
check("moe_group_resident", stream);
}
-void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
- const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
- const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream) {
+static void grouped_s2_impl(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,
+ int max_group) {
if (cap_groups <= 0 || cap_entries <= 0) return;
+ if (max_group < 1 || max_group > kGroupPasses * GMAX) {
+ std::fprintf(stderr, "moe_grouped_s2: a group of %d entries, at most %d\n", max_group, kGroupPasses * GMAX);
+ std::exit(1);
+ }
+ // a group of more than GMAX entries (a window of 9..16 rows, wide builds only): the entries from the GMAX-th on, too
+ constexpr bool kWide = kGroupPasses > 1;
+ const bool second = kWide && max_group > GMAX;
cudaStream_t cs = (cudaStream_t) stream;
const uint64_t gu_bytes = ((uint64_t) cap_entries * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap_entries * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
@@ -1179,12 +1200,21 @@ void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start,
const bool fast = new_grouped(x_q8_0, scratch);
{
const dim3 grid((unsigned) (2 * FF / GU_ROWS), (unsigned) cap_groups);
- if (fast)
- gu_grouped_t_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
- gate_up, (int) cap_entries);
- else
- gu_grouped_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales, gate_up,
- (int) cap_entries);
+ if (fast) {
+ gu_grouped_t_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
+ gate_up, (int) cap_entries);
+ if constexpr (kWide)
+ if (second)
+ gu_grouped_t_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
+ gate_up, (int) cap_entries);
+ } else {
+ gu_grouped_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales, gate_up,
+ (int) cap_entries);
+ if constexpr (kWide)
+ if (second)
+ gu_grouped_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
+ gate_up, (int) cap_entries);
+ }
check("moe_grouped_s2/gu", stream);
}
// #783 PR-k (stuchapin909): SwiGLU and the scaled Q8_0 quantize of its result in one kernel (gate_up still gets the
@@ -1221,14 +1251,36 @@ void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start,
{
const dim3 grid((unsigned) (H / D_ROWS), (unsigned) cap_groups);
const float* hs = x_scales != nullptr ? h_scales : nullptr;
- if (fast)
- down_grouped_t_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
- else
- down_grouped_kernel<<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
+ if (fast) {
+ down_grouped_t_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
+ if constexpr (kWide)
+ if (second) down_grouped_t_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
+ } else {
+ down_grouped_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
+ if constexpr (kWide)
+ if (second) down_grouped_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
+ }
check("moe_grouped_s2/down", stream);
}
}
+void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream) {
+ grouped_s2_impl(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream,
+ GMAX);
+}
+
+#if STRATA_VERIFY_MAX_T > 8
+void moe_grouped_s2_wide(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
+ const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
+ const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,
+ int max_group) {
+ grouped_s2_impl(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream,
+ max_group);
+}
+#endif
+
void moe_hit_add(float* parts, const float* hit_out, const int32_t* dst, const int32_t* count, int64_t cap,
int64_t n_embd, void* stream) {
if (cap <= 0) return;
diff --git a/src/kernels/cuda/shared_expert.cu b/src/kernels/cuda/shared_expert.cu
index bee0da978..36568f042 100644
--- a/src/kernels/cuda/shared_expert.cu
+++ b/src/kernels/cuda/shared_expert.cu
@@ -27,6 +27,7 @@
#include "strata/kernels/s2_gemv_q8.hpp"
#include "strata/kernels/s_gemv.hpp"
#include "strata/kernels/native_mmvq.hpp"
+#include "strata/kernels/verify_kernels.hpp"
#include <cuda_fp16.h>
#include <cuda_runtime.h>
@@ -216,8 +217,8 @@ void shared_expert_multi(int n_tok, const float* x, const uint16_t* x_bf16, cons
const uint16_t* gate_inp_bf16, float* gate, float* up, float* g, float* out, int64_t n_embd,
int64_t n_ff, void* stream, const void* x_q8_1_ready, int lfuse) {
const bool gate_deferred = (lfuse & 1) != 0, pair = (lfuse & 2) != 0;
- if (n_tok < 1 || n_tok > 8 || !nw.q8_1 || !nw.gate_data || !nw.up_data || !nw.down_data || !stream)
- throw std::invalid_argument("shared_expert_multi: needs 1..8 tokens, native weights, scratch and a stream");
+ if (n_tok < 1 || n_tok > kVerifyMaxT || !nw.q8_1 || !nw.gate_data || !nw.up_data || !nw.down_data || !stream)
+ throw std::invalid_argument("shared_expert_multi: needs 1..8 tokens (16 in a wide build), native weights, scratch and a stream");
cudaStream_t cs = (cudaStream_t) stream;
// the activation's q8_1: the caller's (STRATA_VERIFY_QDEDUP, x_q8_1_ready) or the descriptor's, else quantized here
const void* x_q8_1 = x_q8_1_ready ? x_q8_1_ready : nw.x_q8_1;
diff --git a/src/kernels/cuda/verify_kernels.cu b/src/kernels/cuda/verify_kernels.cu
index 3da7ac639..f2aba7378 100644
--- a/src/kernels/cuda/verify_kernels.cu
+++ b/src/kernels/cuda/verify_kernels.cu
@@ -92,7 +92,7 @@ __global__ void gdn_conv_commit_kernel(float* __restrict__ hist, const float* __
hist[c * 3 + 2] = seq[2];
}
-template <int MAX_T = kVerifyMaxT, bool EXACT_T = false>
+template <int MAX_T = kVerifyNarrowT, bool EXACT_T = false>
__device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restrict__ x, const uint16_t* __restrict__ wa,
const uint16_t* __restrict__ wb,
const float* __restrict__ dt,
@@ -143,7 +143,7 @@ __device__ __forceinline__ void gdn_ab_body(const int bx, const float* __restric
}
}
-template <int MAX_T = kVerifyMaxT, bool EXACT_T = false>
+template <int MAX_T = kVerifyNarrowT, bool EXACT_T = false>
__global__ void __launch_bounds__(64) gdn_ab_multi_kernel(const float* __restrict__ x, const uint16_t* __restrict__ wa,
const uint16_t* __restrict__ wb,
const float* __restrict__ dt,
@@ -1079,6 +1079,7 @@ void gdn_conv_ab_multi(const float* history, const float* qkv, const float* conv
case 4: gdn_conv_ab_kernel<4, true><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
case 5: gdn_conv_ab_kernel<5, true><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
case 6: gdn_conv_ab_kernel<6, true><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
+ case 7: case 8: gdn_conv_ab_kernel<kVerifyNarrowT, false><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
default: gdn_conv_ab_kernel<kVerifyMaxT, false><<<grid, S, 0, st>>>(conv_blocks, hist, qkv, conv_w, h, channels, qk_heads, eps, t_begin, commit, x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
}
check("gdn_conv_ab_multi");
@@ -1128,6 +1129,7 @@ void gdn_ab_multi(const float* x, const uint16_t* w_alpha, const uint16_t* w_bet
case 4: gdn_ab_multi_kernel<4, true><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
case 5: gdn_ab_multi_kernel<5, true><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
case 6: gdn_ab_multi_kernel<6, true><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
+ case 7: case 8: gdn_ab_multi_kernel<kVerifyNarrowT, false><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
default: gdn_ab_multi_kernel<kVerifyMaxT, false><<<blocks, 64, 0, st>>>(x, w_alpha, w_beta, dt, ssm_a, gate, beta, n_embd, h_v, n_tok); break;
}
check("gdn_ab_multi");
@@ -1172,8 +1174,12 @@ void gdn_step_norm_multi(float* state, const float* h, int conv_channels, const
if (split && (t_out_begin >= n_tok || n_tok - t_out_begin >= 2)) {
const dim3 grid((unsigned) h_v, S / GS_COLS), block(GS_COLS, RG);
if (t_out_begin >= n_tok) { // no outputs (the commit): the state only
- gdn_state_replay_kernel<kVerifyMaxT><<<grid, block, 0, (cudaStream_t) stream>>>(
- state, h, conv_channels, gate, beta, h_k, h_v, n_tok, n_keep);
+ if (n_tok > kVerifyNarrowT)
+ gdn_state_replay_kernel<kVerifyMaxT><<<grid, block, 0, (cudaStream_t) stream>>>(
+ state, h, conv_channels, gate, beta, h_k, h_v, n_tok, n_keep);
+ else
+ gdn_state_replay_kernel<kVerifyNarrowT><<<grid, block, 0, (cudaStream_t) stream>>>(
+ state, h, conv_channels, gate, beta, h_k, h_v, n_tok, n_keep);
} else {
gdn_step_split_kernel<true><<<grid, block, 0, (cudaStream_t) stream>>>(
state, h, conv_channels, gate, beta, y, h_k, h_v, n_tok, n_keep, t_out_begin);
@@ -1301,20 +1307,25 @@ void handoff_publish(float* dst, const float* a, int64_t na, const float* b, int
}
namespace {
-// one thread per window entry: up to kVerifyMaxT tokens x 10 routed experts (80), so 128 (#646 had 64: a window of
-// 7+ tokens lost its last entries)
-constexpr int kResidentPlanMax = 128;
+// one thread per window entry: up to kVerifyNarrowT tokens x 10 routed experts (80), so 128 (#646 had 64: a window of
+// 7+ tokens lost its last entries). A window of 9..16 rows (up to 160 entries) runs the same kernel on 256 threads;
+// one block size per window class, so a narrow window launches exactly the 128-thread kernel it always did.
+constexpr int kResidentPlanNarrow = 128;
+constexpr int kResidentPlanMax = kVerifyMaxT > kVerifyNarrowT ? 256 : kResidentPlanNarrow;
+static_assert(kVerifyNarrowT * 10 <= kResidentPlanNarrow, "resident_plan: one thread per entry");
static_assert(kVerifyMaxT * 10 <= kResidentPlanMax, "resident_plan: one thread per entry");
-// the parallel scan keeps one partial sum per warp in s_wsum[4] and packs (entries << 16 | groups) in an int
-static_assert(kResidentPlanMax <= 128 && kResidentPlanMax % 32 == 0, "resident_plan: at most 4 warps");
+// the parallel scan keeps one partial sum per warp in s_wsum[NT / 32] and packs (entries << 16 | groups) in an int
+static_assert(kResidentPlanMax <= 256 && kResidentPlanMax % 32 == 0, "resident_plan: at most 8 warps");
static_assert(kResidentPlanMax < 32768, "resident_plan: the packed entry count must stay below 2^15");
-__global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const int32_t* __restrict__ ids, int n, int k, const int32_t* __restrict__ res,
+template <int NT>
+__global__ void __launch_bounds__(NT) resident_plan_kernel(const int32_t* __restrict__ ids, int n, int k, const int32_t* __restrict__ res,
int n_expert, const uint8_t* cache_base, const unsigned long long* slot_off,
long long blob, int32_t* __restrict__ pl, long long capx, uint32_t* skip,
uint32_t ring, volatile uint32_t* plan_err) {
- __shared__ int32_t s_ids[kResidentPlanMax];
- __shared__ int32_t s_excl[kResidentPlanMax];
- __shared__ int32_t s_wsum[4];
+ constexpr int NW = NT / 32;
+ __shared__ int32_t s_ids[NT];
+ __shared__ int32_t s_excl[NT];
+ __shared__ int32_t s_wsum[NW];
__shared__ int s_bad;
const int tid = threadIdx.x;
if (tid == 0) s_bad = 0;
@@ -1378,7 +1389,7 @@ __global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const i
if (is_first) {
int tot = s_excl[tid];
#pragma unroll
- for (int w = 0; w < 4; ++w) {
+ for (int w = 0; w < NW; ++w) {
if (w < warp) tot += s_wsum[w];
}
const int grp_idx = tot & 0xffff;
@@ -1390,7 +1401,7 @@ __global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const i
const int fj_warp = first_j >> 5;
int fj_tot = s_excl[first_j];
#pragma unroll
- for (int w = 0; w < 4; ++w) {
+ for (int w = 0; w < NW; ++w) {
if (w < fj_warp) fj_tot += s_wsum[w];
}
const int out_idx = (fj_tot >> 16) + rank_in_group;
@@ -1398,7 +1409,14 @@ __global__ void __launch_bounds__(kResidentPlanMax) resident_plan_kernel(const i
tok[out_idx] = tid / k;
}
if (tid == 0) {
- const int groups = (s_wsum[0] + s_wsum[1] + s_wsum[2] + s_wsum[3]) & 0xffff;
+ int groups;
+ if constexpr (NW == 4) groups = (s_wsum[0] + s_wsum[1] + s_wsum[2] + s_wsum[3]) & 0xffff; // (the 128-thread kernel as it was)
+ else {
+ groups = 0;
+#pragma unroll
+ for (int w = 0; w < NW; ++w) groups += s_wsum[w];
+ groups &= 0xffff;
+ }
start[groups] = n;
start2[0] = n;
counts[0] = groups;
@@ -1496,8 +1514,18 @@ __global__ void copy_or_zero_kernel(float4* __restrict__ dst, const volatile flo
void resident_plan(const int32_t* ids, int n_entries, int k, const int32_t* res_layer, int n_expert,
const uint8_t* cache_base, const unsigned long long* slot_off, long long blob, int32_t* plan,
long long capx, uint32_t* skip, uint32_t ring, void* stream, uint32_t* plan_err) {
- resident_plan_kernel<<<1, kResidentPlanMax, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,
+ if (n_entries > kResidentPlanNarrow) { // a 9..16-row window (kResidentPlanMax == 256 only in a wide build)
+ if (kResidentPlanMax > kResidentPlanNarrow && n_entries <= kResidentPlanMax)
+ resident_plan_kernel<kResidentPlanMax><<<1, kResidentPlanMax, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,
blob, plan, capx, skip, ring, plan_err);
+ else {
+ std::fprintf(stderr, "resident_plan: %d entries, at most %d\n", n_entries, kResidentPlanMax);
+ std::exit(1);
+ }
+ } else {
+ resident_plan_kernel<kResidentPlanNarrow><<<1, kResidentPlanNarrow, 0, (cudaStream_t) stream>>>(ids, n_entries, k, res_layer, n_expert, cache_base, slot_off,
+ blob, plan, capx, skip, ring, plan_err);
+ }
check("resident_plan");
}
void wait_flag_ge_or(const uint32_t* flag, uint32_t value, const uint32_t* skip, void* stream) {
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 824bd19db..7de185b2c 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -100,6 +100,7 @@
#include "strata/spec/suffix_drafter.hpp"
#include "strata/spec/spec_start.hpp"
#include "strata/program/batch_groups.hpp"
+#include "strata/program/pipe_probe.hpp"
#include "strata/kernels/cvec.hpp"
#include "strata/core/progress.hpp"
#include "strata/core/device.hpp"
@@ -3755,7 +3756,14 @@ int main(int argc, char** argv) {
if (o.lookup_chain > 0 && o.spec >= 2) {
if (o.mtp_max_t == 0) o.mtp_max_t = o.spec;
const int chain_base = o.spec;
- o.spec = std::max(chain_base, std::min(std::max(chain_base, o.mtp_max_t) + o.lookup_chain, 8)); // kVerifyMaxT
+ o.spec = std::max(chain_base, std::min(std::max(chain_base, o.mtp_max_t) + o.lookup_chain, 8)); // kVerifyNarrowT
+ }
+ // A speculative (solo) window holds at most kVerifyNarrowT rows: the hand-offs, the penalty history and the drafters'
+ // buffers are sized for that. Only a plain --batch forms the wider windows of a build with kVerifyMaxT > 8.
+ if (o.spec > (int) strata::kernels::kVerifyNarrowT) {
+ std::fprintf(stderr, "strata generate: --spec %d: a speculative window holds 2..%d tokens\n", o.spec,
+ (int) strata::kernels::kVerifyNarrowT);
+ return 1;
}
// dflash2-kern: right after the start the resident experts are the profile's, not this text's; a speculative round carries
// 2-5 tokens, so the adaptive tier (one swap round per adapt_every VERIFY rounds) learned the text 2-5 times slower per
@@ -6000,17 +6008,19 @@ int main(int argc, char** argv) {
int pm_drafts = 0;
if (piped_mtp) {
pm_drafts = std::clamp(std::atoi(std::getenv("STRATA_PIPED_MTP")), 1,
- std::max(1, std::min(o.spec - 1, (int) strata::kernels::kVerifyMaxT - 1)));
+ std::max(1, std::min(o.spec - 1, (int) strata::kernels::kVerifyNarrowT - 1)));
}
// The verifiers' window: the solo path's drafts (--spec rows), the plain batch windows (the slots) and, with
// STRATA_PIPED_MTP, a group's rows: its slots, each its token and its proposals ((slots) * (1 + proposals), one slot
// per group = at most --spec rows). A function, not a number: --batch (the slots that fit) and the groups are final
// only after this point, at the verifiers' init, and the plain value must stay what it was there.
auto batch_max_t = [&]() -> int {
- if (batch_mtp && !piped_mtp) return (int) strata::kernels::kVerifyMaxT;
+ // (--batch-mtp and STRATA_PIPED_MTP draft with the draft layer's kernels, built for the narrow window: their windows
+ // stay at most kVerifyNarrowT rows; only the plain batch windows are wide)
+ if (batch_mtp && !piped_mtp) return (int) strata::kernels::kVerifyNarrowT;
const int plain = std::max(o.spec, std::min(o.batch, (int) strata::kernels::kVerifyMaxT));
const int group_rows = (piped_mtp && o.batch_groups > 1 && o.batch > 0)
- ? std::min<int>(strata::kernels::kVerifyMaxT, (o.batch / o.batch_groups) * (1 + pm_drafts)) : 0;
+ ? std::min<int>(strata::kernels::kVerifyNarrowT, (o.batch / o.batch_groups) * (1 + pm_drafts)) : 0;
return std::max(plain, group_rows);
};
// The most slots, up to n, whose windows hold at most kVerifyMaxT rows each: n itself up to that, more only as
@@ -6146,10 +6156,10 @@ int main(int argc, char** argv) {
why = buf;
} else if (sc != nullptr && sc[0] != '\0' && sc[0] != '0') {
why = "it does not combine with STRATA_BATCH_SELF_COMMIT (the window would keep its rows itself)";
- } else if ((o.batch / o.batch_groups) * (1 + pm_drafts) > (int) strata::kernels::kVerifyMaxT) {
+ } else if ((o.batch / o.batch_groups) * (1 + pm_drafts) > (int) strata::kernels::kVerifyNarrowT) {
std::snprintf(buf, sizeof buf, "a group's window would hold %d rows (%d slots x %d), more than %d",
(o.batch / o.batch_groups) * (1 + pm_drafts), o.batch / o.batch_groups, 1 + pm_drafts,
- (int) strata::kernels::kVerifyMaxT);
+ (int) strata::kernels::kVerifyNarrowT);
why = buf;
}
if (why != nullptr) {
@@ -6212,7 +6222,7 @@ int main(int argc, char** argv) {
const strata::core::OnDevice on_mtp(last_st && !mtp_first ? last_st->dev : -1);
// --pipeline-windows 2: 8 rows (the chain runs on past the window in flight: its drafts, its bonus token and
// the next window's drafts), and the round/step graphs carry the forcing kernel
- const int mtp_t = o.pipeline_windows >= 2 ? strata::kernels::kVerifyMaxT : o.spec;
+ const int mtp_t = o.pipeline_windows >= 2 ? strata::kernels::kVerifyNarrowT : o.spec;
if (o.pipeline_windows >= 2) mtp.set_force_capture(true);
if (!o.mtp.empty() && !mtp.load(o.mtp, draft_geometry, last_st && !mtp_first ? last_st->ss : ss, mtp_t, err, o.mtp_window)) { std::fprintf(stderr, "strata generate: %s%s\n", err.c_str(), vram_free_note().c_str()); return 1; }
mtp.set_ple_session(&ss);
@@ -9867,7 +9877,9 @@ int main(int argc, char** argv) {
// the widest window; a request without penalties gets a null buffer and takes the byte-for-byte
// neutral path (no upload, no buffer handed to the sampler).
constexpr int kPenaltyWindowCap = 4096;
- constexpr size_t kHistSlots = (size_t) kPenaltyWindowCap * (size_t) strata::kernels::kVerifyMaxT;
+ // (the widest window the run can have: a batch of more than kVerifyNarrowT slots is the only way to a wider one)
+ const size_t kHistSlots = (size_t) kPenaltyWindowCap *
+ (size_t) (o.batch > strata::kernels::kVerifyNarrowT ? strata::kernels::kVerifyMaxT : strata::kernels::kVerifyNarrowT);
int32_t* d_hist = nullptr;
std::vector<int32_t> hist_stage(kHistSlots, -1);
const int hist_dev = last_st ? last_st->dev : -1; // with the head: the last stage's device
@@ -9935,11 +9947,15 @@ int main(int argc, char** argv) {
}
if (n_stages > 1) {
// pipelined slot groups write their rows at hbase = group * rows-per-group, so with more than
- // kVerifyMaxT slots in groups the hand-off holds one row per slot (otherwise kVerifyMaxT, as before)
+ // kVerifyNarrowT slots in groups the hand-off holds one row per slot (otherwise kVerifyNarrowT, as before; a
+ // wide build holds more only for a batch of more than that: the buffer is not made larger for a run that
+ // never forms a window of 9..16 rows)
// (STRATA_PIPED_MTP: per slot its token and up to pm_drafts proposals)
- const int hrows = piped_mtp ? std::max<int>(strata::kernels::kVerifyMaxT, (1 + pm_drafts) * o.batch)
- : !batch_mtp && o.batch_groups > 1 ? std::max<int>(strata::kernels::kVerifyMaxT, o.batch)
- : strata::kernels::kVerifyMaxT;
+ const int hrows = piped_mtp ? std::max<int>(strata::kernels::kVerifyNarrowT, (1 + pm_drafts) * o.batch)
+ : !batch_mtp && o.batch_groups > 1 ? std::max<int>(strata::kernels::kVerifyNarrowT, o.batch)
+ : !batch_mtp ? std::max<int>(strata::kernels::kVerifyNarrowT,
+ std::min<int>(o.batch, strata::kernels::kVerifyMaxT))
+ : strata::kernels::kVerifyNarrowT;
const size_t hb = (size_t) hrows * (size_t) strata::core::Verifier::handoff_floats(g) * sizeof(float);
std::vector<float*> hand((size_t) n_stages - 1, nullptr);
for (float*& h : hand) {
@@ -9978,7 +9994,7 @@ int main(int argc, char** argv) {
for (int st = 0; st < n_stages; ++st) {
stage_ver(st).set_stage(st == 0 ? 0 : split_at[(size_t) st - 1], st + 1 < n_stages ? split_at[(size_t) st] : -1,
st == 0 ? nullptr : hand[(size_t) st - 1], st + 1 < n_stages ? hand[(size_t) st] : nullptr);
- if (hrows > strata::kernels::kVerifyMaxT) stage_ver(st).set_handoff_rows(hrows);
+ if (hrows > strata::kernels::kVerifyNarrowT) stage_ver(st).set_handoff_rows(hrows);
if (overlap)
stage_ver(st).set_handoff_flags(st == 0 ? nullptr : hflag_h[(size_t) st - 1],
st == 0 ? nullptr : hflag_d[(size_t) st - 1],
@@ -10031,7 +10047,7 @@ int main(int argc, char** argv) {
// watchdog's verifier), and the drafter's own row buffer, which either parity's rows are copied into
float* pl_mtp_R = nullptr;
if (pipe) {
- const size_t hb = (size_t) strata::kernels::kVerifyMaxT *
+ const size_t hb = (size_t) strata::kernels::kVerifyNarrowT *
(size_t) strata::core::Verifier::handoff_floats(g) * sizeof(float);
std::vector<float*> hand_b((size_t) n_stages - 1, nullptr); // the odd windows' hand-off per boundary
for (float*& h : hand_b) {
@@ -10063,7 +10079,7 @@ int main(int argc, char** argv) {
gs.dev, st + 1, err.c_str());
return 1;
}
- if (last && !mtp_first && cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyMaxT * (size_t) (g.hc * g.n_embd) *
+ if (last && !mtp_first && cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyNarrowT * (size_t) (g.hc * g.n_embd) *
sizeof(float)) != cudaSuccess) {
std::fprintf(stderr, "strata serve: --pipeline-windows: the drafter's row buffer does not fit\n");
return 1;
@@ -10071,7 +10087,7 @@ int main(int argc, char** argv) {
}
if (mtp_first) { // the drafter's row buffer on its own card (the first)
const strata::core::OnDevice on_m(mtp.device());
- if (cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyMaxT * (size_t) (g.hc * g.n_embd) *
+ if (cudaMalloc((void**) &pl_mtp_R, (size_t) strata::kernels::kVerifyNarrowT * (size_t) (g.hc * g.n_embd) *
sizeof(float)) != cudaSuccess) {
std::fprintf(stderr, "strata serve: --pipeline-windows: the drafter's row buffer does not fit\n");
return 1;
@@ -10134,7 +10150,7 @@ int main(int argc, char** argv) {
}
// STRATA_PIPED_MTP: the rows a group's commit replays after the verdict, one hand-off row each
if (piped_mtp && bslot_ss.size() > 1 &&
- !vk.init_deferred_commit(std::max<int>(strata::kernels::kVerifyMaxT, (1 + pm_drafts) * o.batch), err)) {
+ !vk.init_deferred_commit(std::max<int>(strata::kernels::kVerifyNarrowT, (1 + pm_drafts) * o.batch), err)) {
std::fprintf(stderr, "strata serve: STRATA_PIPED_MTP, stage %zu: %s\n", k + 1, err.c_str());
return 1;
}
@@ -12391,7 +12407,7 @@ int main(int argc, char** argv) {
struct BSlot {
bool active = false, stop = false;
int32_t x = 0; ///< the token the next window feeds (not yet in the slot's state)
- std::array<int32_t, strata::kernels::kVerifyMaxT> draft{}; ///< the slot's next MTP proposal
+ std::array<int32_t, strata::kernels::kVerifyNarrowT> draft{}; ///< the slot's next MTP proposal
bool draft_ready = false;
int n_drafts = 1; ///< STRATA_PIPED_MTP: how many of `draft` its next window checks
int64_t p = 0; ///< its position
@@ -12655,7 +12671,9 @@ int main(int argc, char** argv) {
static size_t next_slot = 0;
// Each MTP slot uses two rows; rotate slots when more than four are active.
int A = 0;
- for (size_t offset = 0; offset < bs.size() && S + (batch_mtp ? 2 : 1) <= strata::kernels::kVerifyMaxT; ++offset) {
+ // (--batch-mtp: two rows a slot, at most kVerifyNarrowT; the plain windows hold up to kVerifyMaxT)
+ const int win_cap = batch_mtp ? (int) strata::kernels::kVerifyNarrowT : (int) strata::kernels::kVerifyMaxT;
+ for (size_t offset = 0; offset < bs.size() && S + (batch_mtp ? 2 : 1) <= win_cap; ++offset) {
const int b = (int) ((next_slot + offset) % bs.size());
if (bs[(size_t) b].active) {
first[A] = S;
@@ -13099,11 +13117,28 @@ int main(int argc, char** argv) {
p.last = pa_last;
return strata::program::batch_groups::pipe_adapt_due(p);
};
+ // STRATA_P4_TIMING=1 / STRATA_P4_TRACE=PATH: where a pipelined window's time goes (pipe_probe.hpp; off: a branch)
+ strata::program::PipeProbe p4(n_pipe, pg.size());
// one turn: serve every running stage, hand finished groups on, start what can start
auto pump = [&](bool may_start) -> bool {
++pipe_tick;
+ const strata::program::PipeProbe::PumpScope p4_scope(p4);
+ if (p4.timing()) {
+ int nb = 0, nf = 0;
+ for (int k = 0; k < n_pipe; ++k) if (stage_group[(size_t) k] >= 0) ++nb;
+ for (const PGroup& x : pg) if (x.inflight) ++nf;
+ p4.busy(nb, nf);
+ // a stage free with no group waiting at it = starved by the stage before, else it was only idle a moment
+ for (int k = 1; k < n_pipe; ++k) {
+ if (stage_group[(size_t) k] >= 0) continue;
+ bool ready = false;
+ for (size_t g2 = 0; g2 < pg.size() && !ready; ++g2) ready = pg[g2].inflight && pg[g2].stage == k;
+ p4.free_stage(k, ready);
+ }
+ }
if (bt_windows == 0 && !pipe_inflight()) {
bt_start = Clock::now();
+ p4.span_start(bt_start);
bt_wait0 = ver.ms_wait; bt_pool0 = ver.ms_pool;
bt_miss0 = drive.d.multi_misses; bt_hits0 = drive.d.cache_hits; bt_pcie0 = drive.d.pcie_experts;
}
@@ -13111,9 +13146,12 @@ int main(int argc, char** argv) {
const int gi = stage_group[(size_t) k];
if (gi < 0) continue;
strata::core::Verifier& vk = stage_verifier(k);
+ const auto p4_p0 = p4.now();
const int r = vk.batch_poll(win_pool_fn, win_pool_user, err);
+ p4.polled(p4_p0);
if (r < 0) { std::printf("ERR %s\n", err.c_str()); return false; }
if (r == 0) continue;
+ p4.done(k, gi);
if (batch_telemetry_enabled) stage_in_flight.complete(k, Clock::now());
stage_group[(size_t) k] = -1;
PGroup& G = pg[(size_t) gi];
@@ -13121,6 +13159,7 @@ int main(int argc, char** argv) {
// the last stage: the group's picks
const int32_t* outb = vk.batch_out();
if (pmtp && !pmtp_verdict(G, vk, outb)) return false;
+ const strata::program::PipeProbe::VerdictScope p4_verdict(p4);
for (int t = 0; t < G.S && !pmtp; ++t) {
BSlot& sl = bs[(size_t) (gi * GS + t)];
if (!sl.active) continue;
@@ -13251,11 +13290,13 @@ int main(int argc, char** argv) {
PGroup& G = pg[(size_t) pick];
strata::core::progress().busy.store(true);
if (batch_telemetry_enabled) stage_in_flight.launch(k, Clock::now());
+ const auto p4_l0 = p4.now();
if (!(pmtp ? stage_verifier(k).batch_launch_rows(G.rows, G.S, G.hbase, G.tok, G.pos, err)
: stage_verifier(k).batch_launch(pick * GS, G.S, G.tok, G.pos, err))) {
std::printf("ERR %s\n", err.c_str());
return false;
}
+ p4.launched(k, pick, p4_l0);
stage_group[(size_t) k] = pick;
if (conc_serving) { // the doorbell thread may be waiting between windows: the launch above is seen
std::atomic_thread_fence(std::memory_order_seq_cst);
@@ -13292,6 +13333,7 @@ int main(int argc, char** argv) {
if (pmtp && !pipe_inflight() && !batch_on() && !pmtp_settle()) return false;
if (!pipe_inflight() && !batch_on() && bt_windows > 0) { // all idle: the timing line, as batch_step
const double w = (double) bt_windows, wall = std::chrono::duration<double, std::milli>(Clock::now() - bt_start).count();
+ p4.span_end(wall);
char pa_txt[160] = "";
if (pipe_adapt_every > 0)
std::snprintf(pa_txt, sizeof pa_txt, "; adaptive tier (STRATA_PIPE_ADAPT_EVERY): %lld rounds, %lld experts "
@@ -13302,6 +13344,7 @@ int main(int argc, char** argv) {
if (pmtp)
std::fprintf(stderr, " (%lld of %lld MTP proposals kept)", (long long) bt_accepted, (long long) bt_tokens);
std::fprintf(stderr, "\n");
+ p4.flush();
pa_rounds = pa_swaps = 0;
pa_ms = 0;
for (int k = 0; k < n_pipe; ++k) {
@@ -16307,7 +16350,7 @@ int main(int argc, char** argv) {
++pl_on;
// the chain over A, forced through B's drafts: B's bonus guess and the next window's drafts
mtp.set_source_R(V1(A).final_R(0));
- chain_n = std::min(strata::kernels::kVerifyMaxT - 1, B.T + S_mtp - 1);
+ chain_n = std::min(strata::kernels::kVerifyNarrowT - 1, B.T + S_mtp - 1);
if (!mtp.chain_launch(A.T, outp.data(), A.p, a, B.tok + 1, B.T - 1, chain_n, chain_n, err))
return die(err);
chain_kind = 1;
@@ -16336,7 +16379,7 @@ int main(int argc, char** argv) {
B = PW{};
if (!last) {
mtp.set_source_R(V1(A).final_R(0));
- chain_n = strata::kernels::kVerifyMaxT - 1;
+ chain_n = strata::kernels::kVerifyNarrowT - 1;
if (!mtp.chain_launch(A.T, outp.data(), A.p, a, nullptr, 0, chain_n, chain_n, err))
return die(err);
chain_kind = 2;