Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dflash2: tiled Q8 GEMM, fused quantizers, head-major K/V ring with a prefetching attention, forked context update, PLE read-ahead, benches
Niko12215bc7f8cab348
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 4ee0af934..0a918bc5e 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -735,6 +735,21 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(tune_rocblas PRIVATE hip::host roc::rocblas)endif()+ # DFlash2 drafter tools (not built by default: cmake --build <dir> --target dflash2_parity dflash2_gemm_bench dflash2_attn_bench): the drafter against+ # the torch port on a synthetic case (needs the pack and a case file, tools/dflash2/mk_parity_case.py), and the timing of its Q8 GEMM and attention+ add_executable(dflash2_parity EXCLUDE_FROM_ALL src/core/dflash2_parity.cpp)+ target_include_directories(dflash2_parity PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_parity PRIVATE strata_engine strata_kernels ${_strata_gpu_runtime_target})+ add_executable(dflash2_gemm_bench EXCLUDE_FROM_ALL src/core/dflash2_gemm_bench.cpp)+ target_include_directories(dflash2_gemm_bench PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_gemm_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})+ add_executable(dflash2_launch_bench EXCLUDE_FROM_ALL src/core/dflash2_launch_bench.cu)+ target_include_directories(dflash2_launch_bench PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_launch_bench PRIVATE ${_strata_gpu_runtime_target})+ add_executable(dflash2_attn_bench EXCLUDE_FROM_ALL src/core/dflash2_attn_bench.cpp)+ target_include_directories(dflash2_attn_bench PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_attn_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})+add_executable(strata src/program/generate.cpp)strata_utf8_manifest(strata)target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})@@ -1084,11 +1099,6 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(verify_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME verify_parity COMMAND verify_parity --selftest)endif()- # the DFlash2 drafter against the torch port on a synthetic case: needs the pack and a case file (tools/dflash2/mk_parity_case.py), so not a ctest- add_executable(dflash2_parity src/core/dflash2_parity.cpp)- target_include_directories(dflash2_parity PRIVATE ${_strata_gpu_include_directories})- target_link_libraries(dflash2_parity PRIVATE strata_engine strata_kernels ${_strata_gpu_runtime_target})-add_executable(fused_gr_bench src/kernels/fused_gr_bench.cpp)target_include_directories(fused_gr_bench PRIVATE ${_strata_gpu_include_directories})target_link_libraries(fused_gr_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hppindex 75aadb075..205bff3d0 100644--- a/include/strata/core/dflash2.hpp+++ b/include/strata/core/dflash2.hpp@@ -22,6 +22,7 @@#include <cstdint>#include <functional>+#include <initializer_list>#include <string>#include <vector>@@ -57,6 +58,9 @@ public:/// selector's pick and its probability). The taps pointer must be the same device buffer on every call (graphs are captured on it).bool draft(const float* taps_dev, int n_new, int64_t pos0, int32_t anchor, int32_t* drafts, float* probs, std::string& err);+ /// Captures the eight round graphs (n_new = 1..8) on `taps` now, so no round pays for a capture.+ bool prepare(const float* taps_dev, std::string& err);+uint64_t vram_bytes() const { return vram_; }int64_t n_draft_vocab() const { return n_dvocab_; }int n_embd() const { return N_; }@@ -65,7 +69,7 @@ public:/// Diagnostics (STRATA_DF2_DUMP): the final normalized block rows of the last draft(), host copy.bool debug_rows(std::vector<float>& rows, std::string& err);- double ms_draft = 0, ms_gpu = 0, ms_ctx = 0;+ double ms_draft = 0, ms_gpu = 0, ms_ctx = 0, ms_capture = 0;int64_t rounds = 0;private:@@ -73,8 +77,8 @@ private:const Tensor& find(const std::string& name) const;const void* q8(const std::string& name) const { return blob_ + find(name).off; }const float* f32(const std::string& name) const { return reinterpret_cast<const float*>(blob_ + find(name).off); }- void mm(const std::string& name, const void* xq, float* y, int ncols); ///< Q8_0 matrix x ncols activation columns (quantized in xq)- void quant(const float* x, int n_in, int ncols);+ /// planar Q8 matrices (1..3 of them, same input width) times the planar activations in xqp_ / xdp_ (ncols columns)+ void gemm(std::initializer_list<const char*> names, std::initializer_list<float*> outs, int ncols, cudaStream_t cs, const uint8_t* xq, const float* xd);bool record_context(const float* taps, int n, cudaStream_t cs, std::string& err);bool record_block(cudaStream_t cs, std::string& err);bool capture(int n, const float* taps, std::string& err);@@ -84,6 +88,7 @@ private:void stamp(const char* name);void prof_collect();void prof_report() const;+ void prof_free();int device_ = -1;cudaStream_t cs_ = nullptr;@@ -112,7 +117,14 @@ private:float* hp_out_ = nullptr;float *kring_[8] = {}, *vring_[8] = {};float *rt_ctx_ = nullptr, *rt_blk_ = nullptr;- uint8_t* xq_ = nullptr;+ uint8_t* xq_ = nullptr; ///< the head's q8_1 activations+ uint8_t* xqp_ = nullptr; ///< planar activations of the drafter's own GEMMs (kernels/dflash2.hpp)+ float* xdp_ = nullptr;+ uint8_t* xqc_ = nullptr; ///< the same for the context update, which runs beside the block+ float* xdc_ = nullptr;+ cudaStream_t cs2_ = nullptr; ///< the context update's branch of the round graph+ cudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr, ev_ctx_[8] = {};+ bool fork_ = false; ///< recording the forked graph: layer l's attention waits for ev_ctx_[l]float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;float *ao_ = nullptr, *o_ = nullptr, *gate_ = nullptr, *up_ = nullptr, *mlp_ = nullptr, *part_ = nullptr, *fn_ = nullptr;diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hppindex 8a544f298..0ce4775b3 100644--- a/include/strata/kernels/dflash2.hpp+++ b/include/strata/kernels/dflash2.hpp@@ -14,8 +14,8 @@ constexpr int kDf2Rows = 8; ///< block rows (the anchor + 7 masks) anconstexpr int kDf2Kv = 8; ///< K/V headsconstexpr int kDf2Gqa = 4; ///< query heads per K/V headconstexpr int kDf2Hd = 128; ///< head dim-constexpr int kDf2Chunk = 256; ///< keys per attention block-constexpr int kDf2MaxChunks = 9; ///< ceil((2047 context keys + 8 block keys) / kDf2Chunk)+constexpr int kDf2Chunk = 128; ///< keys per attention block+constexpr int kDf2MaxChunks = 17; ///< ceil((2047 context keys + 8 block keys) / kDf2Chunk)constexpr int kDf2TopK = 16;constexpr int kDf2TopBlocks = 64; ///< partial top-k blocks per row (a block holds at most 4096 logits)@@ -38,12 +38,33 @@ void df2_norm_rope(const float* in, int in_stride, const float* gamma, const intvoid df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod, int rows, int cols, void* stream);/// Attention of the block's 8 query rows (q: [8][32][128], already normed and rotated) over the context ring (kring, vring:-/// [ring_slots][8][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when+/// head-major [8][ring_slots][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when/// s + i - j < window (j in [max(0, s - window + 1), s - 1], slot j % ring_slots) and every block key. `s` is read from *anchor_pos/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined into out [8][32*128].void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,int ring_slots, int window, float* part, float* out, void* stream);+/// The Q8 GEMM the drafter's matrices run on. A weight matrix is TILED (see the layout note in dflash2.cu; df2_q8_tile makes it from GGUF Q8_0 blocks):+/// the int8 plane at `w` (n_out * n_in bytes) then the fp16 scale plane. Activations are planar (df2_quant_rows and the fused quantizers below):+/// xq is int8 [col][n_in] (8 columns of n_in bytes), xd float [col][n_in / 32]. Up to 3 matrices that share the input run in one launch; y (matrix i)+/// is [ncols][n_out]. n_in a multiple of 512, n_out of 16, ncols 1..8 (the other columns of xq are read and dropped).+/// Bench only: pick a GEMM kernel variant (0 = the default).+void df2_gemm_variant(int v);+/// Bench only: attention variants (bit 0 skips Q K^T, bit 1 skips P V).+void df2_attention_variant(int v);+struct Df2Mat { const void* w; float* y; int n_out; };+void df2_gemm(const Df2Mat* mats, int n, const void* xq, const float* xd, int n_in, int ncols, void* stream);+/// Planar quantization of rows x[ncols][K].+void df2_quant_rows(const float* x, int K, int ncols, void* xq, float* xd, void* stream);+/// RMS norm (weights gamma) of `rows` rows of N, written to xn (optional) and quantized.+void df2_rms_quant(const float* h, const float* gamma, float* xn, int rows, int N, float eps, void* xq, float* xd, void* stream);+/// df2_conv without residual, then quantized.+void df2_conv_quant(const float* x, const float* dyn, int dyn_stride, const float* base, int rows, int H, int group, void* xq, float* xd, void* stream);+/// silu(gate) * up of `rows` rows of I, quantized.+void df2_swiglu_quant(const float* gate, const float* up, int rows, int I, void* xq, float* xd, void* stream);+/// GGUF Q8_0 blocks (34 bytes) of a matrix with rows of n_in -> the tiled qs plane and the tiled d plane (count = rows * n_in / 32 blocks).+void df2_q8_tile(const void* blocks, void* qs, void* dpl, int n_in, int64_t count, void* stream);+/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id./// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cppindex 80687d3fa..6a04146bc 100644--- a/src/core/dflash2.cpp+++ b/src/core/dflash2.cpp@@ -17,6 +17,7 @@#include <cstring>#include <exception>#include <atomic>+#include <initializer_list>#include <stdexcept>namespace strata::core {@@ -38,13 +39,13 @@ constexpr int GGML_Q8_0 = 8;Dflash2Drafter::~Dflash2Drafter() {if (device_ < 0) return;prof_report();- delete prof_;+ prof_free();const OnDevice on_device(device_);if (cs_) cudaStreamSynchronize(cs_);for (cudaGraphExec_t& e : exec_)if (e) cudaGraphExecDestroy(e);auto fr = [](void* p) { if (p) cudaFree(p); };- fr(blob_); fr(d_meta_); fr(xq_); fr(rt_ctx_); fr(rt_blk_);+ fr(blob_); fr(d_meta_); fr(xq_); fr(xqp_); fr(xdp_); fr(xqc_); fr(xdc_); fr(rt_ctx_); fr(rt_blk_);for (int l = 0; l < 8; ++l) { fr(kring_[l]); fr(vring_[l]); }fr(fcy_); fr(hctx_); fr(kc_); fr(vc_); fr(h_); fr(xn_); fr(xc_); fr(dyn_); fr(q_); fr(qn_); fr(k_); fr(v_); fr(kblk_); fr(ao_); fr(o_);fr(gate_); fr(up_); fr(mlp_); fr(part_); fr(fn_); fr(logits_); fr(topval_); fr(hsel_); fr(toptok_); fr(topscratch_);@@ -52,6 +53,10 @@ Dflash2Drafter::~Dflash2Drafter() {if (h_meta_) cudaFreeHost(h_meta_);if (h_out_) cudaFreeHost(h_out_);if (cs_) cudaStreamDestroy(cs_);+ if (cs2_) cudaStreamDestroy(cs2_);+ if (ev_fork_) cudaEventDestroy(ev_fork_);+ if (ev_join_) cudaEventDestroy(ev_join_);+ for (cudaEvent_t e : ev_ctx_) if (e) cudaEventDestroy(e);if (ev_a_) cudaEventDestroy(ev_a_);if (ev_b_) cudaEventDestroy(ev_b_);}@@ -64,6 +69,17 @@ struct Dflash2Drafter::Prof {int64_t rounds = 0;};+void Dflash2Drafter::prof_free() {+ delete prof_;+ prof_ = nullptr;+}++// STRATA_DF2_SKIP=attn,head,ctx,mlp,qkvo (measurement only): leaves those parts out of the round, so the graph's GPU time without them can be read+static bool skip_part(const char* what) {+ static const std::string list = [] { const char* e = std::getenv("STRATA_DF2_SKIP"); return std::string(e ? e : ""); }();+ return !list.empty() && list.find(what) != std::string::npos;+}+void Dflash2Drafter::stamp(const char* name) {if (!prof_) return;cudaEvent_t e;@@ -154,19 +170,38 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)}if (cudaMalloc((void**) &blob_, total) != cudaSuccess) { err = "dflash2: the pack does not fit in VRAM (" + std::to_string(total >> 20) + " MiB)"; return false; }vram_ += total;- std::vector<uint8_t> stage(64u << 20);+ uint64_t max_bytes = 0;+ for (const Tensor& t : tensors_) max_bytes = std::max(max_bytes, t.bytes);+ std::vector<uint8_t> stage(max_bytes);+ uint8_t* tmp = nullptr;+ if (cudaMalloc((void**) &tmp, max_bytes) != cudaSuccess) { err = "dflash2: upload scratch"; return false; }+ struct TmpFree { uint8_t* p; ~TmpFree() { cudaFree(p); } } tmp_free{tmp};+ for (const Tensor& t : tensors_) {#if defined(_WIN32)- _fseeki64(f, data_off, SEEK_SET);+ _fseeki64(f, data_off + (long long) t.off, SEEK_SET);#else- fseeko(f, data_off, SEEK_SET);+ fseeko(f, data_off + (off_t) t.off, SEEK_SET);#endif- for (uint64_t o = 0; o < total;) {- const size_t n = (size_t) std::min<uint64_t>(stage.size(), total - o);- if (std::fread(stage.data(), 1, n, f) != n) { err = "dflash2: the pack is shorter than its index"; return false; }- if (cudaMemcpy(blob_ + o, stage.data(), n, cudaMemcpyHostToDevice) != cudaSuccess) { err = "dflash2: weight upload failed"; return false; }- o += n;+ if (std::fread(stage.data(), 1, t.bytes, f) != t.bytes) { err = "dflash2: the pack is shorter than its index"; return false; }+ if (t.dtype == "q8_0") { // GGUF blocks -> the tiled qs plane at the tensor's offset, then the d plane (same total bytes)+ if (cudaMemcpy(tmp, stage.data(), t.bytes, cudaMemcpyHostToDevice) != cudaSuccess) { err = "dflash2: weight upload failed"; return false; }+ kernels::df2_q8_tile(tmp, blob_ + t.off, blob_ + t.off + (uint64_t) t.rows * (uint64_t) t.cols, (int) t.cols, t.rows * t.cols / 32, nullptr);+ if (cudaDeviceSynchronize() != cudaSuccess) { err = "dflash2: weight conversion failed"; return false; }+ } else if (cudaMemcpy(blob_ + t.off, stage.data(), t.bytes, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = "dflash2: weight upload failed";+ return false;+ }}- if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "dflash2: stream"; return false; }+ if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess || cudaStreamCreateWithFlags(&cs2_, cudaStreamNonBlocking) != cudaSuccess) {+ err = "dflash2: stream";+ return false;+ }+ if (cudaEventCreateWithFlags(&ev_fork_, cudaEventDisableTiming) != cudaSuccess || cudaEventCreateWithFlags(&ev_join_, cudaEventDisableTiming) != cudaSuccess) {+ err = "dflash2: events";+ return false;+ }+ for (int l = 0; l < 8; ++l)+ if (cudaEventCreateWithFlags(&ev_ctx_[l], cudaEventDisableTiming) != cudaSuccess) { err = "dflash2: events"; return false; }if (const char* e = std::getenv("STRATA_DF2_PROFILE"); e && e[0] == '1') prof_ = new Prof();if (const char* e = std::getenv("STRATA_DF2_TIMING"); e && e[0] == '1') {timing_ = true;@@ -184,7 +219,7 @@ bool Dflash2Drafter::alloc(std::string& err) {return true;};const size_t B = kRows;- bool ok = dev(&d_meta_, 32) && dev(&xq_, kernels::native_q8_1_bytes(std::max(N_ * 5, I_), kRows) + 4096) && dev(&rt_ctx_, B * 128) && dev(&rt_blk_, B * 128) &&+ bool ok = dev(&d_meta_, 32) && dev(&xq_, kernels::native_q8_1_bytes(N_, kRows) + 4096) && dev(&xqp_, (size_t) 5 * N_ * 8 + 4096) && dev(&xdp_, (size_t) 5 * N_ / 32 * 8 + 64) && dev(&xqc_, (size_t) 5 * N_ * 8 + 4096) && dev(&xdc_, (size_t) 5 * N_ / 32 * 8 + 64) && dev(&rt_ctx_, B * 128) && dev(&rt_blk_, B * 128) &&dev(&fcy_, B * N_) && dev(&hctx_, B * N_) && dev(&kc_, B * NKV_ * HD_) && dev(&vc_, B * NKV_ * HD_) && dev(&h_, B * N_) && dev(&xn_, B * N_) &&dev(&xc_, B * N_) && dev(&dyn_, B * 640) && dev(&q_, B * NH_ * HD_) && dev(&qn_, B * NH_ * HD_) && dev(&k_, B * NKV_ * HD_) && dev(&v_, B * NKV_ * HD_) &&dev(&kblk_, B * NKV_ * HD_) && dev(&ao_, B * NH_ * HD_) && dev(&o_, B * N_) && dev(&gate_, B * I_) && dev(&up_, B * I_) && dev(&mlp_, B * N_) &&@@ -202,6 +237,10 @@ bool Dflash2Drafter::alloc(std::string& err) {out_tok_ = (int32_t*) dd; // the select kernel writes the 7 picks here (host-visible) ...out_prob_ = (float*) dd + 16; // ... and their probabilities at +16 intscudaMemset(rt_ctx_, 0, B * 128 * 4);+ cudaMemset(xqp_, 0, (size_t) 5 * N_ * 8 + 4096);+ cudaMemset(xdp_, 0, ((size_t) 5 * N_ / 32 * 8 + 64) * 4);+ cudaMemset(xqc_, 0, (size_t) 5 * N_ * 8 + 4096);+ cudaMemset(xdc_, 0, ((size_t) 5 * N_ / 32 * 8 + 64) * 4);for (int l = 0; l < L_; ++l) { cudaMemset(kring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); cudaMemset(vring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); }return cudaDeviceSynchronize() == cudaSuccess;}@@ -278,32 +317,43 @@ bool Dflash2Drafter::bind(const WeightTable& wt, const NativeHead* head, const sreturn true;}-void Dflash2Drafter::quant(const float* x, int n_in, int ncols) { kernels::native_quantize_q8_1(x, xq_, n_in, ncols, cs_); }--void Dflash2Drafter::mm(const std::string& name, const void* xq, float* y, int ncols) {- const Tensor& t = find(name);- kernels::native_q8_0_mmvq(blob_ + t.off, xq, y, (int) t.cols, (int) t.rows, ncols, cs_);+void Dflash2Drafter::gemm(std::initializer_list<const char*> names, std::initializer_list<float*> outs, int ncols, cudaStream_t cs, const uint8_t* xq, const float* xd) {+ kernels::Df2Mat mats[3];+ int n = 0, n_in = 0;+ auto o = outs.begin();+ for (const char* name : names) {+ const Tensor& t = find(name);+ mats[n] = kernels::Df2Mat{blob_ + t.off, *o++, (int) t.rows};+ if (n == 0) n_in = (int) t.cols;+ ++n;+ }+ kernels::df2_gemm(mats, n, xq, xd, n_in, ncols, cs);}bool Dflash2Drafter::record_context(const float* taps, int n, cudaStream_t cs, std::string& err) {using namespace kernels;+ if (skip_part("ctx")) return true;try {const int tap_n = (int) taps_.size() * N_;df2_rope_table(d_meta_ + 0, n, theta_, rt_ctx_, cs);- stamp("ctx-start");- quant(taps, tap_n, n);- mm("fc.weight", xq_, fcy_, n);- stamp("fc");- native_qsa_rms_norm_weighted(fcy_, f32("hidden_norm.weight"), hctx_, N_, n, eps_, cs);- quant(hctx_, N_, n);+ stamp("df2_rope_table");+ df2_quant_rows(taps, tap_n, n, xqc_, xdc_, cs);+ stamp("df2_quant_rows");+ gemm({"fc.weight"}, {fcy_}, n, cs, xqc_, xdc_);+ stamp("gemm:fc");+ df2_rms_quant(fcy_, f32("hidden_norm.weight"), nullptr, n, N_, eps_, xqc_, xdc_, cs);+ stamp("df2_rms_quant");for (int l = 0; l < L_; ++l) {const std::string p = "layers." + std::to_string(l) + ".";- mm(p + "self_attn.k_proj.weight", xq_, kc_, n);- mm(p + "self_attn.v_proj.weight", xq_, vc_, n);+ const std::string kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";+ gemm({kp.c_str(), vp.c_str()}, {kc_, vc_}, n, cs, xqc_, xdc_);+ stamp("gemm:kp");df2_norm_rope(kc_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], kRing, n, NKV_, eps_, cs);+ stamp("df2_norm_rope");df2_copy_rows(vc_, vring_[l], d_meta_ + 0, kRing, n, NKV_ * HD_, cs);+ stamp("df2_copy_rows");+ if (fork_) cudaEventRecord(ev_ctx_[l], cs); // layer l's keys and values are in the ring}- stamp("ctx-kv");} catch (const std::exception& e) {err = e.what();return false;@@ -316,56 +366,65 @@ bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {try {const int B = kRows;df2_rope_table(d_meta_ + 8, B, theta_, rt_blk_, cs);+ stamp("df2_rope_table");embed_(d_meta_ + 24, B, h_, cs);- stamp("embed");+ stamp("embed_");for (int l = 0; l < L_; ++l) {const std::string p = "layers." + std::to_string(l) + ".";const float* base_a = f32(p + "attention_conv.base_kernel");const float* base_m = f32(p + "mlp_conv.base_kernel");- native_qsa_rms_norm_weighted(h_, f32(p + "input_layernorm.weight"), xn_, N_, B, eps_, cs);- quant(xn_, N_, B);- mm(p + "attention_conv.kernel_projection.weight", xq_, dyn_, B);- df2_conv(xn_, dyn_, 640, base_a, nullptr, xc_, B, N_, group_, cs);- quant(xc_, N_, B);- stamp("attn-front");- mm(p + "self_attn.q_proj.weight", xq_, q_, B);- mm(p + "self_attn.k_proj.weight", xq_, k_, B);- mm(p + "self_attn.v_proj.weight", xq_, v_, B);- stamp("qkv");+ const std::string akp = p + "attention_conv.kernel_projection.weight", mkp = p + "mlp_conv.kernel_projection.weight";+ const std::string qp = p + "self_attn.q_proj.weight", kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";+ const std::string gp = p + "mlp.gate_proj.weight", up = p + "mlp.up_proj.weight";+ df2_rms_quant(h_, f32(p + "input_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);+ stamp("df2_rms_quant");+ gemm({akp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);+ stamp("gemm:akp");+ df2_conv_quant(xn_, dyn_, 640, base_a, B, N_, group_, xqp_, xdp_, cs);+ stamp("df2_conv_quant");+ if (!skip_part("qkvo")) gemm({qp.c_str(), kp.c_str(), vp.c_str()}, {q_, k_, v_}, B, cs, xqp_, xdp_);+ stamp("gemm:qp");df2_norm_rope(q_, NH_ * HD_, f32(p + "self_attn.q_norm.weight"), d_meta_ + 8, rt_blk_, qn_, 0, B, NH_, eps_, cs);+ stamp("df2_norm_rope");df2_norm_rope(k_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 8, rt_blk_, kblk_, 0, B, NKV_, eps_, cs);- df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);- stamp("attention");- quant(ao_, NH_ * HD_, B);- mm(p + "self_attn.o_proj.weight", xq_, o_, B);- stamp("o_proj");+ stamp("df2_norm_rope");+ if (fork_) cudaStreamWaitEvent(cs, ev_ctx_[l], 0); // the context update has written this layer's ring+ if (!skip_part("attn")) df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);+ stamp("df2_attention");+ df2_quant_rows(ao_, NH_ * HD_, B, xqp_, xdp_, cs);+ stamp("df2_quant_rows");+ if (!skip_part("qkvo")) gemm({(p + "self_attn.o_proj.weight").c_str()}, {o_}, B, cs, xqp_, xdp_);+ stamp("gemm:o_proj");df2_conv(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, h_, B, N_, group_, cs);- native_qsa_rms_norm_weighted(h_, f32(p + "post_attention_layernorm.weight"), xn_, N_, B, eps_, cs);- quant(xn_, N_, B);- mm(p + "mlp_conv.kernel_projection.weight", xq_, dyn_, B);- df2_conv(xn_, dyn_, 640, base_m, nullptr, xc_, B, N_, group_, cs);- quant(xc_, N_, B);- stamp("mlp-front");- mm(p + "mlp.gate_proj.weight", xq_, gate_, B);- mm(p + "mlp.up_proj.weight", xq_, up_, B);- stamp("gate-up");- native_swiglu_quantize_q8_1(gate_, up_, xq_, I_, B, cs);- mm(p + "mlp.down_proj.weight", xq_, mlp_, B);- stamp("down");+ stamp("df2_conv");+ df2_rms_quant(h_, f32(p + "post_attention_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);+ stamp("df2_rms_quant");+ gemm({mkp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);+ stamp("gemm:mkp");+ df2_conv_quant(xn_, dyn_, 640, base_m, B, N_, group_, xqp_, xdp_, cs);+ stamp("df2_conv_quant");+ if (!skip_part("mlp")) gemm({gp.c_str(), up.c_str()}, {gate_, up_}, B, cs, xqp_, xdp_);+ stamp("gemm:gp");+ df2_swiglu_quant(gate_, up_, B, I_, xqp_, xdp_, cs);+ stamp("df2_swiglu_quant");+ if (!skip_part("mlp")) gemm({(p + "mlp.down_proj.weight").c_str()}, {mlp_}, B, cs, xqp_, xdp_);+ stamp("gemm:mlp.down_proj");df2_conv(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, h_, B, N_, group_, cs);+ stamp("df2_conv");}native_qsa_rms_norm_weighted(h_, f32("norm.weight"), fn_, N_, B, eps_, cs);- quant(fn_ + N_, N_, kDrafts); // rows 1..7- stamp("final-norm");- native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);- stamp("head");+ stamp("native_qsa_rms_norm_weighted");+ native_quantize_q8_1(fn_ + N_, xq_, N_, kDrafts, cs); // rows 1..7, the head's format+ df2_quant_rows(fn_ + N_, N_, kDrafts, xqp_, xdp_, cs); // and the selector projection's+ if (!skip_part("head")) native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);+ stamp("native_mmvq");df2_topk(logits_, (int) n_dvocab_, dvocab_, kDrafts, topscratch_, topval_, toptok_, cs);- stamp("topk");- mm("candidate_selector.hidden_projection.weight", xq_, hsel_, kDrafts);+ stamp("df2_topk");+ gemm({"candidate_selector.hidden_projection.weight"}, {hsel_}, kDrafts, cs, xqp_, xdp_);+ stamp("gemm:candidate_selector.hidden_projection");df2_select(hsel_, topval_, toptok_, reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.predecessor_codebook").off),reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.successor_codebook").off), rank_, kDrafts, d_meta_ + 16, out_tok_,out_prob_, cs);- stamp("selector");} catch (const std::exception& e) {err = e.what();return false;@@ -377,8 +436,17 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {if (exec_[n]) return true;if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "dflash2: begin capture"; return false; }kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);- bool ok = record_context(taps, n, cs_, err);+ // the context update (fc, the keys and values) and the block's front run side by side: they meet before each layer's attention+ fork_ = prof_ == nullptr && cs2_ != nullptr;+ if (fork_) {+ cudaEventRecord(ev_fork_, cs_);+ cudaStreamWaitEvent(cs2_, ev_fork_, 0);+ }+ bool ok = record_context(taps, n, fork_ ? cs2_ : cs_, err);+ if (fork_) cudaEventRecord(ev_join_, cs2_);if (ok) ok = record_block(cs_, err);+ if (fork_) cudaStreamWaitEvent(cs_, ev_join_, 0);+ fork_ = false;cudaGraph_t graph = nullptr;const cudaError_t ce = cudaStreamEndCapture(cs_, &graph);if (!ok) { if (graph) cudaGraphDestroy(graph); return false; }@@ -393,6 +461,19 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {return true;}+bool Dflash2Drafter::prepare(const float* taps, std::string& err) {+ if (!bound_) { err = "dflash2: not bound"; return false; }+ if (prof_ != nullptr) return true; // the profile runs without graphs+ const OnDevice on_device(device_);+ const Clock::time_point t0 = Clock::now();+ for (int i = 0; i < 8; ++i) h_meta_[i] = i;+ for (int n = 1; n <= kRows; ++n)+ if (!capture(n, taps, err)) return false;+ graph_taps_ = taps;+ ms_capture = ms_since(t0);+ return true;+}+bool Dflash2Drafter::context(const float* taps, int64_t n, int64_t pos0, std::string& err) {const OnDevice on_device(device_);const int tap_n = (int) taps_.size() * N_;diff --git a/src/core/dflash2_attn_bench.cpp b/src/core/dflash2_attn_bench.cppnew file mode 100644index 000000000..897b5b823--- /dev/null+++ b/src/core/dflash2_attn_bench.cpp@@ -0,0 +1,58 @@+// src/core/dflash2_attn_bench.cpp - the drafter's attention (kernels/dflash2.hpp df2_attention) on random K/V: time per call and GB/s of K/V read.+//+// dflash2_attn_bench [iters] [context]+#include "strata/kernels/dflash2.hpp"++#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <vector>++int main(int argc, char** argv) {+ const int iters = argc > 1 ? std::atoi(argv[1]) : 100;+ const int ctx = argc > 2 ? std::atoi(argv[2]) : 2047;+ const int ring = 2064;+ using namespace strata::kernels;+ float *q, *kr, *vr, *kb, *vb, *part, *out;+ int32_t* pos;+ cudaMalloc((void**) &q, 8 * 32 * 128 * 4);+ cudaMalloc((void**) &kr, (size_t) ring * 1024 * 4);+ cudaMalloc((void**) &vr, (size_t) ring * 1024 * 4);+ cudaMalloc((void**) &kb, 8 * 1024 * 4);+ cudaMalloc((void**) &vb, 8 * 1024 * 4);+ cudaMalloc((void**) &part, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);+ cudaMalloc((void**) &out, 8 * 32 * 128 * 4);+ cudaMalloc((void**) &pos, 4);+ std::vector<float> h((size_t) ring * 1024);+ for (size_t i = 0; i < h.size(); ++i) h[i] = (float) ((int) (i * 2654435761u >> 20) % 2001 - 1000) / 1000.0f;+ cudaMemcpy(kr, h.data(), h.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(vr, h.data(), h.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(q, h.data(), 8 * 32 * 128 * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(kb, h.data(), 8 * 1024 * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(vb, h.data(), 8 * 1024 * 4, cudaMemcpyHostToDevice);+ const int32_t s = ctx + 5000;+ cudaMemcpy(pos, &s, 4, cudaMemcpyHostToDevice);+ cudaEvent_t a, b;+ cudaEventCreate(&a);+ cudaEventCreate(&b);+ for (int v : {0, 1, 2, 3}) {+ df2_attention_variant(v);+ df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);+ cudaDeviceSynchronize();+ cudaEventRecord(a);+ for (int i = 0; i < iters; ++i) df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);+ cudaEventRecord(b);+ cudaEventSynchronize(b);+ float ms;+ cudaEventElapsedTime(&ms, a, b);+ std::vector<float> o(8 * 32 * 128);+ cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost);+ double sum = 0;+ for (float x : o) sum += x;+ const double kv_bytes = 2.0 * (ctx + 8) * 1024 * 4;+ std::printf("variant %d: %.1f us per call, K/V %.1f GB/s (checksum %.6f)\n", v, 1000.0 * ms / iters, kv_bytes * iters / (ms / 1000.0) / 1e9, sum);+ }+ return 0;+}diff --git a/src/core/dflash2_gemm_bench.cpp b/src/core/dflash2_gemm_bench.cppnew file mode 100644index 000000000..26a556853--- /dev/null+++ b/src/core/dflash2_gemm_bench.cpp@@ -0,0 +1,62 @@+// src/core/dflash2_gemm_bench.cpp - the drafter's Q8 GEMM (kernels/dflash2.hpp df2_gemm) on random planar weights: GB/s per shape and column count.+//+// dflash2_gemm_bench [iters]+//+// Weight sets are cycled through 2 GiB of distinct matrices so the L2 cache cannot hold them (the drafter reads each matrix once per round).+#include "strata/kernels/dflash2.hpp"++#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <vector>++int main(int argc, char** argv) {+ const int iters = argc > 1 ? std::atoi(argv[1]) : 50;+ struct Shape { const char* name; int n_in, n_out; int mats; };+ const Shape shapes[] = {{"gate+up 2560->7680 x2", 2560, 7680, 2}, {"down 7680->2560", 7680, 2560, 1}, {"qkv 2560->(4096,1024,1024)", 2560, 4096, 3},+ {"o 4096->2560", 4096, 2560, 1}, {"fc 12800->2560", 12800, 2560, 1}, {"kv 2560->1024 x2", 2560, 1024, 2}};+ cudaEvent_t a, b;+ cudaEventCreate(&a);+ cudaEventCreate(&b);+ for (const Shape& sh : shapes) {+ const size_t wbytes = (size_t) sh.n_out * sh.n_in * 34 / 32;+ const size_t per_set = wbytes * sh.mats;+ const int sets = (int) std::max<size_t>(2, ((size_t) 1 << 30) / per_set);+ std::vector<uint8_t*> w((size_t) sets * sh.mats);+ for (auto& p : w) {+ if (cudaMalloc((void**) &p, wbytes) != cudaSuccess) { std::fprintf(stderr, "alloc\n"); return 1; }+ cudaMemset(p, 0x11, wbytes);+ }+ uint8_t* xq;+ float *xd, *y;+ cudaMalloc((void**) &xq, (size_t) sh.n_in * 8);+ cudaMalloc((void**) &xd, (size_t) sh.n_in / 32 * 8 * 4);+ cudaMalloc((void**) &y, (size_t) 8 * sh.n_out * 3 * 4);+ cudaMemset(xq, 3, (size_t) sh.n_in * 8);+ std::vector<float> ones((size_t) sh.n_in / 32 * 8, 0.01f);+ cudaMemcpy(xd, ones.data(), ones.size() * 4, cudaMemcpyHostToDevice);+ for (int var : {1, 2, 3, 4, 5, 6, 7, 8, 9}) {+ strata::kernels::df2_gemm_variant(var);+ const int ncols = 8;+ cudaDeviceSynchronize();+ cudaEventRecord(a);+ for (int it = 0; it < iters; ++it) {+ strata::kernels::Df2Mat m[3];+ for (int i = 0; i < sh.mats; ++i) m[i] = {w[(size_t) (it % sets) * sh.mats + i], y + (size_t) i * 8 * sh.n_out, sh.n_out};+ if (sh.mats == 3) { m[1].n_out = 1024; m[2].n_out = 1024; }+ strata::kernels::df2_gemm(m, sh.mats, xq, xd, sh.n_in, ncols, nullptr);+ }+ cudaEventRecord(b);+ cudaEventSynchronize(b);+ float ms;+ cudaEventElapsedTime(&ms, a, b);+ const double bytes = sh.mats == 3 ? (double) (4096 + 1024 + 1024) * sh.n_in * 34 / 32 : (double) per_set;+ std::printf("%-30s variant %d: %7.1f us %6.1f GB/s\n", sh.name, var, 1000.0 * ms / iters, bytes * iters / (ms / 1000.0) / 1e9);+ }+ for (auto p : w) cudaFree(p);+ cudaFree(xq); cudaFree(xd); cudaFree(y);+ }+ return 0;+}diff --git a/src/core/dflash2_launch_bench.cu b/src/core/dflash2_launch_bench.cunew file mode 100644index 000000000..d4fae689e--- /dev/null+++ b/src/core/dflash2_launch_bench.cu@@ -0,0 +1,68 @@+// src/core/dflash2_launch_bench.cpp - what a dependent kernel costs inside a captured graph on this GPU: N tiny kernels in a chain, and N kernels of 256 blocks.+//+// dflash2_launch_bench [n=100]+#include <cuda_runtime.h>++#include <chrono>+#include <cstdio>+#include <cstdlib>++__global__ void tiny(float* x, int n) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ if (i < n) x[i] = x[i] * 1.0001f + 1.0f;+}++int main(int argc, char** argv) {+ const int n = argc > 1 ? std::atoi(argv[1]) : 100;+ float* x;+ cudaMalloc((void**) &x, 1 << 24);+ cudaMemset(x, 0, 1 << 24);+ cudaStream_t cs;+ cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking);+ cudaEvent_t a, b;+ cudaEventCreate(&a);+ cudaEventCreate(&b);+ for (int blocks : {1, 64, 256, 1024}) {+ cudaGraph_t g;+ cudaGraphExec_t ge;+ cudaStreamBeginCapture(cs, cudaStreamCaptureModeThreadLocal);+ for (int i = 0; i < n; ++i) tiny<<<blocks, 256, 0, cs>>>(x, blocks * 256);+ cudaStreamEndCapture(cs, &g);+ cudaGraphInstantiate(&ge, g, 0);+ cudaGraphLaunch(ge, cs);+ cudaStreamSynchronize(cs);+ cudaEventRecord(a, cs);+ for (int it = 0; it < 50; ++it) cudaGraphLaunch(ge, cs);+ cudaEventRecord(b, cs);+ cudaEventSynchronize(b);+ float ms;+ cudaEventElapsedTime(&ms, a, b);+ std::printf("graph of %d chained kernels, %4d blocks: %.2f us per kernel\n", n, blocks, 1000.0 * ms / 50 / n);+ // one launch from an idle stream: event, launch, event, wait (what the drafter does each round)+ {+ double tot = 0, wall = 0;+ for (int it = 0; it < 20; ++it) {+ cudaStreamSynchronize(cs);+ auto t0 = std::chrono::steady_clock::now();+ cudaEventRecord(a, cs);+ cudaGraphLaunch(ge, cs);+ cudaEventRecord(b, cs);+ cudaStreamSynchronize(cs);+ wall += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();+ float m;+ cudaEventElapsedTime(&m, a, b);+ tot += m;+ }+ std::printf("single launch from idle, %d kernels, %4d blocks: events %.3f ms, wall %.3f ms\n", n, blocks, tot / 20, wall / 20);+ }+ // the same without a graph+ cudaEventRecord(a, cs);+ for (int it = 0; it < 5; ++it)+ for (int i = 0; i < n; ++i) tiny<<<blocks, 256, 0, cs>>>(x, blocks * 256);+ cudaEventRecord(b, cs);+ cudaEventSynchronize(b);+ cudaEventElapsedTime(&ms, a, b);+ std::printf("stream of %d chained kernels, %4d blocks: %.2f us per kernel\n", n, blocks, 1000.0 * ms / 5 / n);+ }+ return 0;+}diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cuindex 82c40ad63..fb98c4e25 100644--- a/src/kernels/cuda/dflash2.cu+++ b/src/kernels/cuda/dflash2.cu@@ -1,6 +1,9 @@// src/kernels/cuda/dflash2.cu - the small kernels of the DFlash2 drafter. See include/strata/kernels/dflash2.hpp.#include "strata/kernels/dflash2.hpp"+#include "strata/kernels/dp4a.hpp"++#include <cuda_fp16.h>#include <cuda_runtime.h>#include <cfloat>@@ -14,6 +17,8 @@ namespace strata::kernels {namespace {constexpr int HD = kDf2Hd;+int g_gemm_variant = 0; // bench only (df2_gemm_variant)+int g_attn_variant = 0; // bench only (df2_attention_variant)void check(const char* what) {const cudaError_t e = cudaGetLastError();@@ -77,7 +82,8 @@ __global__ void norm_rope_kernel(const float* __restrict__ in, int in_stride, cores = d < 64 ? v * c - y[d + 64] * s : v * c + y[d - 64] * s;}const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;- out[((size_t) slot * heads + h) * HD + d] = res;+ // a ring (slot_mod > 0) is head-major [head][slot][128], the block's own keys row-major [row][head][128]+ out[slot_mod > 0 ? ((size_t) h * slot_mod + slot) * HD + d : ((size_t) slot * heads + h) * HD + d] = res;}__global__ void copy_rows_kernel(const float* __restrict__ in, float* __restrict__ out, const int32_t* __restrict__ pos, int slot_mod, int rows, int cols) {@@ -85,106 +91,160 @@ __global__ void copy_rows_kernel(const float* __restrict__ in, float* __restrictif (i >= rows * cols) return;const int r = i / cols, c = i % cols;const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;- out[(size_t) slot * cols + c] = in[i];+ out[slot_mod > 0 ? ((size_t) (c / HD) * slot_mod + slot) * HD + c % HD : (size_t) slot * cols + c] = in[i];}-// ---- attention. grid (8 K/V heads, kDf2MaxChunks), 256 threads. 32 queries (4 heads x 8 rows) per block share each key tile.-constexpr int KS = 129; // smem row stride of the K / V / Q tiles (conflict-free)-__global__ void __launch_bounds__(256) attn_part_kernel(const float* __restrict__ q, const float* __restrict__ kring, const float* __restrict__ vring,+// ---- attention. grid (8 K/V heads, kDf2MaxChunks), 128 threads. The 32 queries (4 heads x 8 rows) of a K/V head share each key tile;+// S = Q K^T in 4 x 4 register tiles (64 threads), the online softmax per query, O += P V in 4 x 8 register tiles (128 threads).+constexpr int AS = 132; // smem row stride of the Q / K / V tiles (float4 aligned, conflict-free for the 4 x 4 tiles)+constexpr int PS = 36; // stride of the transposed probabilities P[key][query]+__global__ void __launch_bounds__(128) attn_part_kernel(const float* __restrict__ q, const float* __restrict__ kring, const float* __restrict__ vring,const float* __restrict__ kblk, const float* __restrict__ vblk,- const int32_t* __restrict__ anchor_pos, int ring_slots, int window, float* __restrict__ part) {- __shared__ float Qs[32 * KS];- __shared__ float Ks[32 * KS]; // the K tile, then the V tile- __shared__ float Ss[32 * 33];+ const int32_t* __restrict__ anchor_pos, int ring_slots, int window, float* __restrict__ part, int variant) {+ __shared__ __align__(16) float Qs[32 * AS];+ __shared__ __align__(16) float Ks[32 * AS]; // the K tile, then the V tile+ __shared__ __align__(16) float Ps[32 * PS];+ __shared__ float Sc[32 * 33];+ __shared__ float m_s[32], l_s[32], corr_s[32];const int kvh = blockIdx.x, chunk = blockIdx.y, tid = threadIdx.x;const int s = *anchor_pos;const int lo = s - (window - 1) > 0 ? s - (window - 1) : 0;const int nctx = s - lo;const int nkeys = nctx + kDf2Rows;- const int qi = tid >> 3, tg = tid & 7;- const int qrow = qi >> 2, qg = qi & 3;- float* const pout = part + (((size_t) chunk * kDf2Kv + kvh) * 32 + qi) * 130;const int k_begin = chunk * kDf2Chunk;+ float* const pbase = part + (((size_t) chunk * kDf2Kv + kvh) * 32) * 130;if (k_begin >= nkeys) {- if (tg == 0) { pout[0] = -INFINITY; pout[1] = 0.0f; }- for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = 0.0f;+ for (int i = tid; i < 32 * 130; i += 128) pbase[i] = (i % 130 == 0) ? -INFINITY : 0.0f;return;}const float scale = rsqrtf((float) HD);- for (int i = tid; i < 32 * HD; i += 256) {- const int a = i / HD, d = i % HD;+ for (int i = tid; i < 32 * HD / 4; i += 128) { // Q: 32 queries x 128, float4+ const int a = i / (HD / 4), d4 = i % (HD / 4);const int row = a >> 2, g = a & 3;- Qs[a * KS + d] = q[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] * scale;+ float4 v = *reinterpret_cast<const float4*>(q + ((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d4 * 4);+ v.x *= scale; v.y *= scale; v.z *= scale; v.w *= scale;+ *reinterpret_cast<float4*>(&Qs[a * AS + d4 * 4]) = v;}- float m = -INFINITY, l = 0.0f;- float O[16];+ if (tid < 32) { m_s[tid] = -INFINITY; l_s[tid] = 0.0f; }+ float O[4][8];+#pragma unroll+ for (int i = 0; i < 4; ++i)#pragma unroll- for (int e = 0; e < 16; ++e) O[e] = 0.0f;+ for (int j = 0; j < 8; ++j) O[i][j] = 0.0f;+ const int qg_pv = tid >> 4, dg_pv = tid & 15; // PV: queries qg*4 .. +3, dims dg*8 .. +7const int k_end = min(k_begin + kDf2Chunk, nkeys);- for (int k0 = k_begin; k0 < k_end; k0 += 32) {- __syncthreads(); // Ks free (previous tile's V reads done), Qs written (first tile)- for (int i = tid; i < 32 * HD; i += 256) {- const int t = i / HD, d = i % HD, k = k0 + t;- float v = 0.0f;- if (k < nctx) v = kring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];- else if (k < nkeys) v = kblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];- Ks[t * KS + d] = v;- }- __syncthreads();- float sc[4] = {0.f, 0.f, 0.f, 0.f};-#pragma unroll 8- for (int d = 0; d < HD; ++d) {- const float qv = Qs[qi * KS + d];+ // a tile is 32 keys x 128 dims = 1024 float4: eight per thread, loaded into registers ahead of their use+ auto fetch = [&](int k0, float4 (®)[8], const float* ring, const float* blk) {#pragma unroll- for (int u = 0; u < 4; ++u) sc[u] = fmaf(qv, Ks[(tg * 4 + u) * KS + d], sc[u]);+ for (int u = 0; u < 8; ++u) {+ const int i = tid + u * 128, t = i / (HD / 4), d4 = i % (HD / 4), k = k0 + t;+ float4 v = make_float4(0.f, 0.f, 0.f, 0.f);+ if (k < nctx) v = *reinterpret_cast<const float4*>(ring + ((size_t) kvh * ring_slots + (lo + k) % ring_slots) * HD + d4 * 4);+ else if (k < nkeys) v = *reinterpret_cast<const float4*>(blk + ((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d4 * 4);+ reg[u] = v;}- float tmax = -INFINITY;+ };+ auto put = [&](const float4 (®)[8]) {#pragma unroll- for (int u = 0; u < 4; ++u) {- const int k = k0 + tg * 4 + u;- bool ok = k < nkeys;- if (ok && k < nctx) ok = (lo + k) > (s + qrow - window); // s + row - j < window- sc[u] = ok ? sc[u] : -INFINITY;- tmax = fmaxf(tmax, sc[u]);+ for (int u = 0; u < 8; ++u) {+ const int i = tid + u * 128, t = i / (HD / 4), d4 = i % (HD / 4);+ *reinterpret_cast<float4*>(&Ks[t * AS + d4 * 4]) = reg[u];}- tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 1));- tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 2));- tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 4));- const float mn = fmaxf(m, tmax);- const float corr = (m == -INFINITY) ? 0.0f : __expf(m - mn);- float psum = 0.0f;+ };+ float4 kreg[8], vreg[8];+ fetch(k_begin, kreg, kring, kblk);+ for (int k0 = k_begin; k0 < k_end; k0 += 32) {+ __syncthreads(); // Ks free (the last tile's V reads done); Qs / m_s written on the first pass+ put(kreg);+ fetch(k0, vreg, vring, vblk); // this tile's V, hidden behind the S stage and the softmax+ __syncthreads();+ if (tid < 64) { // S tile: 4 queries x 4 keys per thread+ const int qg = tid >> 3, kg = tid & 7;+ float acc[4][4];+#pragma unroll+ for (int i = 0; i < 4; ++i)+#pragma unroll+ for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f;+#pragma unroll 4+ for (int d = 0; d < ((variant & 1) ? 0 : HD); d += 4) {+ float4 qv[4], kv[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) qv[i] = *reinterpret_cast<const float4*>(&Qs[(qg * 4 + i) * AS + d]);#pragma unroll- for (int u = 0; u < 4; ++u) {- const float p = sc[u] == -INFINITY ? 0.0f : __expf(sc[u] - mn);- Ss[qi * 33 + tg * 4 + u] = p;- psum += p;+ for (int j = 0; j < 4; ++j) kv[j] = *reinterpret_cast<const float4*>(&Ks[(kg * 4 + j) * AS + d]);+#pragma unroll+ for (int i = 0; i < 4; ++i)+#pragma unroll+ for (int j = 0; j < 4; ++j)+ acc[i][j] += qv[i].x * kv[j].x + qv[i].y * kv[j].y + qv[i].z * kv[j].z + qv[i].w * kv[j].w;+ }+#pragma unroll+ for (int i = 0; i < 4; ++i)+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int qi = qg * 4 + i, t = kg * 4 + j, k = k0 + t, row = qi >> 2;+ bool ok = k < nkeys;+ if (ok && k < nctx) ok = (lo + k) > (s + row - window); // s + row - j < window+ Sc[qi * 33 + t] = ok ? acc[i][j] : -INFINITY;+ }}- psum += __shfl_xor_sync(0xffffffffu, psum, 1);- psum += __shfl_xor_sync(0xffffffffu, psum, 2);- psum += __shfl_xor_sync(0xffffffffu, psum, 4);- l = l * corr + psum;- m = mn;+ __syncthreads();+ { // online softmax: query = tid / 4, 8 keys each, 4 lanes per query+ const int qi = tid >> 2, sub = tid & 3;+ float v[8], tmax = -INFINITY;+#pragma unroll+ for (int u = 0; u < 8; ++u) { v[u] = Sc[qi * 33 + sub * 8 + u]; tmax = fmaxf(tmax, v[u]); }+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 1));+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 2));+ const float m_old = m_s[qi];+ const float mn = fmaxf(m_old, tmax);+ const float corr = (m_old == -INFINITY) ? 0.0f : __expf(m_old - mn);+ float psum = 0.0f;#pragma unroll- for (int e = 0; e < 16; ++e) O[e] *= corr;- __syncthreads(); // Ss complete; every read of Ks done- for (int i = tid; i < 32 * HD; i += 256) {- const int t = i / HD, d = i % HD, k = k0 + t;- float v = 0.0f;- if (k < nctx) v = vring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];- else if (k < nkeys) v = vblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];- Ks[t * KS + d] = v;+ for (int u = 0; u < 8; ++u) {+ const float p = v[u] == -INFINITY ? 0.0f : __expf(v[u] - mn);+ Ps[(sub * 8 + u) * PS + qi] = p;+ psum += p;+ }+ psum += __shfl_xor_sync(0xffffffffu, psum, 1);+ psum += __shfl_xor_sync(0xffffffffu, psum, 2);+ if (sub == 0) { l_s[qi] = l_s[qi] * corr + psum; m_s[qi] = mn; corr_s[qi] = corr; }}+ __syncthreads(); // Ps, corr_s complete; every read of Ks (the S tile) done+ put(vreg);+ if (k0 + 32 < k_end) fetch(k0 + 32, kreg, kring, kblk); // the next tile's K, hidden behind the PV stage__syncthreads();- for (int t = 0; t < 32; ++t) {- const float p = Ss[qi * 33 + t];+ {+ const float4 c4 = *reinterpret_cast<const float4*>(&corr_s[qg_pv * 4]);+ const float cr[4] = {c4.x, c4.y, c4.z, c4.w};+#pragma unroll+ for (int i = 0; i < 4; ++i)+#pragma unroll+ for (int j = 0; j < 8; ++j) O[i][j] *= cr[i];+#pragma unroll 4+ for (int t = 0; t < ((variant & 2) ? 0 : 32); ++t) {+ const float4 p4 = *reinterpret_cast<const float4*>(&Ps[t * PS + qg_pv * 4]);+ const float4 va = *reinterpret_cast<const float4*>(&Ks[t * AS + dg_pv * 8]);+ const float4 vb = *reinterpret_cast<const float4*>(&Ks[t * AS + dg_pv * 8 + 4]);+ const float pv[4] = {p4.x, p4.y, p4.z, p4.w};+ const float vv[8] = {va.x, va.y, va.z, va.w, vb.x, vb.y, vb.z, vb.w};#pragma unroll- for (int e = 0; e < 16; ++e) O[e] = fmaf(p, Ks[t * KS + e * 8 + tg], O[e]);+ for (int i = 0; i < 4; ++i)+#pragma unroll+ for (int j = 0; j < 8; ++j) O[i][j] = fmaf(pv[i], vv[j], O[i][j]);+ }}}- if (tg == 0) { pout[0] = m; pout[1] = l; }+ __syncthreads();+ if (tid < 32) { pbase[tid * 130] = m_s[tid]; pbase[tid * 130 + 1] = l_s[tid]; }#pragma unroll- for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = O[e];- (void) qg;+ for (int i = 0; i < 4; ++i) {+ float* o = pbase + (size_t) (qg_pv * 4 + i) * 130 + 2 + dg_pv * 8;+ *reinterpret_cast<float2*>(o) = make_float2(O[i][0], O[i][1]);+ *reinterpret_cast<float2*>(o + 2) = make_float2(O[i][2], O[i][3]);+ *reinterpret_cast<float2*>(o + 4) = make_float2(O[i][4], O[i][5]);+ *reinterpret_cast<float2*>(o + 6) = make_float2(O[i][6], O[i][7]);+ }}__global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrict__ out) {@@ -204,6 +264,157 @@ __global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrout[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] = L > 0.0f ? o / L : 0.0f;}+// ---- Q8 GEMM for up to 8 activation columns (the drafter's matrices)+//+// Weights are TILED (df2_q8_tile makes them from GGUF Q8_0 blocks): a group of 8 rows is one unit; per 512 values of the reduction axis (a "step",+// 16 Q8 blocks) the group stores the 8 rows' 512 int8 one after the other (4 KB), so a warp that works on a group reads 4 KB contiguous per step+// (lane = block * 2 + half: one 16-byte load per row). The fp16 block scales follow all the int8 of the matrix, in the same tile order (8 rows x 16+// blocks = 256 bytes a step). Activations: x_q (int8 [col][n_in]) and x_d (float [col][n_in / 32]); a column's 512 bytes of a step are contiguous.+struct Df2Seg { const int8_t* q; const __half* d; float* y; int n_out; int row0; };+struct Df2Segs { Df2Seg s[3]; int n; };++template <int WARPS, bool COMPUTE, int UNR = 2>+__global__ void __launch_bounds__(WARPS * 32) gemm8_kernel(Df2Segs segs, const int8_t* __restrict__ xq, const float* __restrict__ xd, int n_in, int ncols) {+ constexpr int R = 8;+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;+ const int row_g = (blockIdx.x * WARPS + warp) * R; // first row of this warp's group, over all the launch's matrices+ int si = 0;+ while (si + 1 < segs.n && row_g >= segs.s[si + 1].row0) ++si;+ const Df2Seg sg = segs.s[si];+ const int row0 = row_g - sg.row0;+ const int nb = n_in / 32, steps = nb / 16;+ const int4* wq = reinterpret_cast<const int4*>(sg.q + (size_t) row0 * n_in) + lane;+ const __half* wd = sg.d + (size_t) row0 * nb + (lane >> 1);+ const int4* xv = reinterpret_cast<const int4*>(xq) + lane;+ float acc[R][8];+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < 8; ++c) acc[r][c] = 0.0f;+#pragma unroll UNR+ for (int st = 0; st < steps; ++st) {+ int4 w[R];+ float dw[R];+#pragma unroll+ for (int r = 0; r < R; ++r) {+ w[r] = wq[(size_t) st * 256 + r * 32];+ dw[r] = __half2float(wd[(size_t) st * 128 + r * 16]);+ }+ if (!COMPUTE) { // bench only: the same loads, almost no arithmetic+ for (int r = 0; r < R; ++r) acc[r][0] += (float) (w[r].x ^ w[r].y ^ w[r].z ^ w[r].w) * dw[r];+ continue;+ }+ float xs[8];+#pragma unroll+ for (int c = 0; c < 8; ++c) xs[c] = xd[(size_t) c * nb + st * 16 + (lane >> 1)];+#pragma unroll+ for (int c = 0; c < 8; ++c) {+ const int4 x = xv[(size_t) c * (n_in / 16) + st * 32];+#pragma unroll+ for (int r = 0; r < R; ++r) {+ int isum = STRATA_DP4A(w[r].x, x.x, 0);+ isum = STRATA_DP4A(w[r].y, x.y, isum);+ isum = STRATA_DP4A(w[r].z, x.z, isum);+ isum = STRATA_DP4A(w[r].w, x.w, isum);+ acc[r][c] = fmaf(dw[r] * xs[c], (float) isum, acc[r][c]);+ }+ }+ }+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < 8; ++c) {+ float v = acc[r][c];+ v += __shfl_xor_sync(0xffffffffu, v, 16);+ v += __shfl_xor_sync(0xffffffffu, v, 8);+ v += __shfl_xor_sync(0xffffffffu, v, 4);+ v += __shfl_xor_sync(0xffffffffu, v, 2);+ v += __shfl_xor_sync(0xffffffffu, v, 1);+ if (lane == 0 && c < ncols) sg.y[(size_t) c * sg.n_out + row0 + r] = v;+ }+}++// quantize one 32-block of one row to the planar activation layout (a warp: lane = element)+__device__ __forceinline__ void quant_block(float v, int blk, int col, int K, int8_t* __restrict__ xq, float* __restrict__ xd) {+ const int lane = threadIdx.x & 31;+ float amax = fabsf(v);+ for (int o = 16; o > 0; o >>= 1) amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, o));+ const float d = amax / 127.0f;+ const float inv = amax > 0.0f ? 127.0f / amax : 0.0f;+ const int q = __float2int_rn(v * inv);+ xq[(size_t) col * K + blk * 32 + lane] = (int8_t) q;+ if (lane == 0) xd[(size_t) col * (K / 32) + blk] = d;+}++// rows [ncols][K] -> planar; grid (ceil(K / 32 / 8), ncols), 256 threads (a warp per block)+__global__ void quant_rows_kernel(const float* __restrict__ x, int K, int8_t* __restrict__ xq, float* __restrict__ xd) {+ const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), col = blockIdx.y, lane = threadIdx.x & 31;+ if (blk * 32 >= K) return;+ quant_block(x[(size_t) col * K + blk * 32 + lane], blk, col, K, xq, xd);+}++// RMS norm of row `col` (N values) then the planar quantization of the normalized values; also writes them when xn != null. One block of 256 per row.+__global__ void rms_quant_kernel(const float* __restrict__ h, const float* __restrict__ gamma, float* __restrict__ xn, int N, float eps,+ int8_t* __restrict__ xq, float* __restrict__ xd) {+ const int col = blockIdx.x, tid = threadIdx.x, warp = tid >> 5, lane = tid & 31;+ __shared__ float red[8];+ float ss = 0.0f;+ for (int i = tid; i < N; i += 256) { const float v = h[(size_t) col * N + i]; ss += v * v; }+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);+ if (lane == 0) red[warp] = ss;+ __syncthreads();+ float tot = 0.0f;+ for (int w = 0; w < 8; ++w) tot += red[w];+ const float scale = rsqrtf(tot / (float) N + eps);+ for (int blk = warp; blk * 32 < N; blk += 8) {+ const int i = blk * 32 + lane;+ const float v = h[(size_t) col * N + i] * scale * gamma[i];+ if (xn) xn[(size_t) col * N + i] = v;+ quant_block(v, blk, col, N, xq, xd);+ }+}++// conv (as conv_kernel, no residual) then the planar quantization: grid (ceil(H / 32 / 8), rows), 256 threads+__global__ void conv_quant_kernel(const float* __restrict__ x, const float* __restrict__ dyn, int dyn_stride, const float* __restrict__ base, int H, int group,+ int8_t* __restrict__ xq, float* __restrict__ xd) {+ const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), b = blockIdx.y, lane = threadIdx.x & 31;+ if (blk * 32 >= H) return;+ const int c = blk * 32 + lane, groups = H / group;+ float acc = 0.0f;+#pragma unroll+ for (int o = 0; o < 2; ++o) {+ if (b - o < 0) continue;+ const float k = base[o * H + c] + dyn[(size_t) b * dyn_stride + o * groups + c / group];+ acc = fmaf(k, x[(size_t) (b - o) * H + c], acc);+ }+ quant_block(acc, blk, b, H, xq, xd);+}++// silu(gate) * up then the planar quantization: grid (ceil(I / 32 / 8), rows), 256 threads+__global__ void swiglu_quant_kernel(const float* __restrict__ gate, const float* __restrict__ up, int I, int8_t* __restrict__ xq, float* __restrict__ xd) {+ const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), b = blockIdx.y, lane = threadIdx.x & 31;+ if (blk * 32 >= I) return;+ const size_t i = (size_t) b * I + blk * 32 + lane;+ const float g = gate[i];+ quant_block(g / (1.0f + __expf(-g)) * up[i], blk, b, I, xq, xd);+}++// GGUF Q8_0 blocks (34 bytes: fp16 d, 32 int8) of a [n_out][n_in] matrix -> the tiled qs plane (n_out * n_in bytes) and the tiled d plane; one thread per block+__global__ void q8_tile_kernel(const uint8_t* __restrict__ blocks, int8_t* __restrict__ qs, __half* __restrict__ dpl, int n_in, int64_t count) {+ const int64_t b = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (b >= count) return;+ const int nb = n_in / 32;+ const int64_t row = b / nb;+ const int blk = (int) (b % nb);+ const int64_t g = row / 8;+ const int r = (int) (row % 8), st = blk / 16, bi = blk % 16;+ const uint8_t* src = blocks + b * 34;+ const uint16_t dh = (uint16_t) src[0] | ((uint16_t) src[1] << 8);+ dpl[(g * 8 * nb) + (int64_t) st * 128 + r * 16 + bi] = __ushort_as_half(dh);+ int8_t* o = qs + g * 8 * n_in + (int64_t) st * 4096 + r * 512 + bi * 32;+ for (int i = 0; i < 32; ++i) o[i] = (int8_t) src[2 + i];+}+// ---- top-16constexpr int TK_CHUNK_MAX = 4096;__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }@@ -381,12 +592,69 @@ void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_modvoid df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,int ring_slots, int window, float* part, float* out, void* stream) {- attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 256, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part);+ attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 128, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part, g_attn_variant);check("attn_part");attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, out);check("attn_merge");}+void df2_gemm_variant(int v) { g_gemm_variant = v; }+void df2_attention_variant(int v) { g_attn_variant = v; }++void df2_gemm(const Df2Mat* mats, int n, const void* xq, const float* xd, int n_in, int ncols, void* stream) {+ if (n < 1 || n > 3 || n_in % 512 != 0 || ncols < 1 || ncols > 8) throw std::invalid_argument("df2_gemm: unsupported shape");+ Df2Segs segs;+ int rows = 0;+ for (int i = 0; i < n; ++i) {+ if (mats[i].n_out % 16 != 0) throw std::invalid_argument("df2_gemm: n_out must be a multiple of 16");+ const int8_t* q = static_cast<const int8_t*>(mats[i].w);+ segs.s[i] = Df2Seg{q, reinterpret_cast<const __half*>(q + (size_t) mats[i].n_out * n_in), mats[i].y, mats[i].n_out, rows};+ rows += mats[i].n_out;+ }+ segs.n = n;+ const int8_t* xqp = static_cast<const int8_t*>(xq);+ cudaStream_t st = (cudaStream_t) stream;+ const int variant = g_gemm_variant != 0 ? g_gemm_variant : (rows % 32 == 0 ? 1 : 2);+ switch (variant) {+ case 3: gemm8_kernel<2, false><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 4: gemm8_kernel<1, true, 2><<<rows / 8, 32, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 5: gemm8_kernel<2, true, 1><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 6: gemm8_kernel<2, true, 4><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 7: gemm8_kernel<4, true, 4><<<rows / 32, 128, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 8: gemm8_kernel<8, true, 2><<<rows / 64, 256, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 9: gemm8_kernel<4, false><<<rows / 32, 128, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ case 2: gemm8_kernel<2, true><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break; // 16 rows a block: every n_out is a multiple of 16+ default: gemm8_kernel<4, true><<<rows / 32, 128, 0, st>>>(segs, xqp, xd, n_in, ncols); break;+ }+ check("gemm8");+}++void df2_quant_rows(const float* x, int K, int ncols, void* xq, float* xd, void* stream) {+ quant_rows_kernel<<<dim3((K / 32 + 7) / 8, ncols), 256, 0, (cudaStream_t) stream>>>(x, K, static_cast<int8_t*>(xq), xd);+ check("quant_rows");+}++void df2_rms_quant(const float* h, const float* gamma, float* xn, int rows, int N, float eps, void* xq, float* xd, void* stream) {+ rms_quant_kernel<<<rows, 256, 0, (cudaStream_t) stream>>>(h, gamma, xn, N, eps, static_cast<int8_t*>(xq), xd);+ check("rms_quant");+}++void df2_conv_quant(const float* x, const float* dyn, int dyn_stride, const float* base, int rows, int H, int group, void* xq, float* xd, void* stream) {+ conv_quant_kernel<<<dim3((H / 32 + 7) / 8, rows), 256, 0, (cudaStream_t) stream>>>(x, dyn, dyn_stride, base, H, group, static_cast<int8_t*>(xq), xd);+ check("conv_quant");+}++void df2_swiglu_quant(const float* gate, const float* up, int rows, int I, void* xq, float* xd, void* stream) {+ swiglu_quant_kernel<<<dim3((I / 32 + 7) / 8, rows), 256, 0, (cudaStream_t) stream>>>(gate, up, I, static_cast<int8_t*>(xq), xd);+ check("swiglu_quant");+}++void df2_q8_tile(const void* blocks, void* qs, void* dpl, int n_in, int64_t count, void* stream) {+ q8_tile_kernel<<<(unsigned) ((count + 255) / 256), 256, 0, (cudaStream_t) stream>>>(static_cast<const uint8_t*>(blocks), static_cast<int8_t*>(qs),+ static_cast<__half*>(dpl), n_in, count);+ check("q8_tile");+}+void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream) {if ((n + kDf2TopBlocks - 1) / kDf2TopBlocks > TK_CHUNK_MAX) throw std::invalid_argument("df2_topk: too many logits per row");float* cval = reinterpret_cast<float*>(scratch);diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 4f75c9fe1..d7ec9a694 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -16027,7 +16027,11 @@ int main(int argc, char** argv) {std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());use_df2 = false;}- if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s), the MTP layer stays as the fallback\n", o.draft_model.c_str());+ if (use_df2 && !df2.prepare(ver.tap_dev(), err)) {+ std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());+ use_df2 = false;+ }+ if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s, graphs captured in %.0f ms), the MTP layer stays as the fallback\n", o.draft_model.c_str(), df2.ms_capture);#endifmem_mark("the verifier and the drafter's binding");ver.set_sampling(sp); // the CLI's own sampling (until 0.1.19 this loop was always greedy); no penalties here@@ -16332,6 +16336,23 @@ int main(int argc, char** argv) {float pp[strata::core::Dflash2Drafter::kDrafts];if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;for (size_t i = 0; i < drafts.size() && i < (size_t) strata::core::Dflash2Drafter::kDrafts; ++i) { drafts[i] = dd[i]; dprob[i] = pp[i]; }+ // what the MTP does while it drafts: the PLE rows of the tokens the next window will hold are read ahead+ // (the window's gather would otherwise wait for them: 1.8 ms a round on strix)+ if (ss.ple.ready() && ss.ple.table != nullptr) {+ int n_pf = 0;+ const int cap = std::min<int>(S_mtp, (int) drafts.size() + 1);+ for (int t = 1; t < cap && (o.spec_min_p <= 0.0 || dprob[(size_t) t - 1] >= (float) o.spec_min_p); ++t) ++n_pf;+ int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};+ auto pf = [&](int32_t tok) {+ uint32_t rows16[strata::kernels::PLE_N_HEADS];+ strata::kernels::ngram_rows(&tok, prev, 1, ss.ple.consts, rows16);+ prev[0] = prev[1];+ prev[1] = tok;+ ss.ple.table->prefetch_rows(rows16);+ };+ pf(outv[(size_t) a_]);+ for (int i = 0; i < n_pf; ++i) pf(drafts[(size_t) i]);+ }return true;}#endif