Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dflash2: tiled Q8 GEMM, fused quantizers, head-major K/V ring with a prefetching attention, forked context update, PLE read-ahead, benches

Niko12215bc7f8cab348
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4ee0af934..0a918bc5e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -735,6 +735,21 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(tune_rocblas PRIVATE hip::host roc::rocblas)
endif()
+ # DFlash2 drafter tools (not built by default: cmake --build <dir> --target dflash2_parity dflash2_gemm_bench dflash2_attn_bench): the drafter against
+ # the torch port on a synthetic case (needs the pack and a case file, tools/dflash2/mk_parity_case.py), and the timing of its Q8 GEMM and attention
+ add_executable(dflash2_parity EXCLUDE_FROM_ALL src/core/dflash2_parity.cpp)
+ target_include_directories(dflash2_parity PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_parity PRIVATE strata_engine strata_kernels ${_strata_gpu_runtime_target})
+ add_executable(dflash2_gemm_bench EXCLUDE_FROM_ALL src/core/dflash2_gemm_bench.cpp)
+ target_include_directories(dflash2_gemm_bench PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_gemm_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+ add_executable(dflash2_launch_bench EXCLUDE_FROM_ALL src/core/dflash2_launch_bench.cu)
+ target_include_directories(dflash2_launch_bench PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_launch_bench PRIVATE ${_strata_gpu_runtime_target})
+ add_executable(dflash2_attn_bench EXCLUDE_FROM_ALL src/core/dflash2_attn_bench.cpp)
+ target_include_directories(dflash2_attn_bench PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_attn_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+
add_executable(strata src/program/generate.cpp)
strata_utf8_manifest(strata)
target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})
@@ -1084,11 +1099,6 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(verify_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_test(NAME verify_parity COMMAND verify_parity --selftest)
endif()
- # the DFlash2 drafter against the torch port on a synthetic case: needs the pack and a case file (tools/dflash2/mk_parity_case.py), so not a ctest
- add_executable(dflash2_parity src/core/dflash2_parity.cpp)
- target_include_directories(dflash2_parity PRIVATE ${_strata_gpu_include_directories})
- target_link_libraries(dflash2_parity PRIVATE strata_engine strata_kernels ${_strata_gpu_runtime_target})
-
add_executable(fused_gr_bench src/kernels/fused_gr_bench.cpp)
target_include_directories(fused_gr_bench PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(fused_gr_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})
diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hpp
index 75aadb075..205bff3d0 100644
--- a/include/strata/core/dflash2.hpp
+++ b/include/strata/core/dflash2.hpp
@@ -22,6 +22,7 @@
#include <cstdint>
#include <functional>
+#include <initializer_list>
#include <string>
#include <vector>
@@ -57,6 +58,9 @@ public:
/// selector's pick and its probability). The taps pointer must be the same device buffer on every call (graphs are captured on it).
bool draft(const float* taps_dev, int n_new, int64_t pos0, int32_t anchor, int32_t* drafts, float* probs, std::string& err);
+ /// Captures the eight round graphs (n_new = 1..8) on `taps` now, so no round pays for a capture.
+ bool prepare(const float* taps_dev, std::string& err);
+
uint64_t vram_bytes() const { return vram_; }
int64_t n_draft_vocab() const { return n_dvocab_; }
int n_embd() const { return N_; }
@@ -65,7 +69,7 @@ public:
/// Diagnostics (STRATA_DF2_DUMP): the final normalized block rows of the last draft(), host copy.
bool debug_rows(std::vector<float>& rows, std::string& err);
- double ms_draft = 0, ms_gpu = 0, ms_ctx = 0;
+ double ms_draft = 0, ms_gpu = 0, ms_ctx = 0, ms_capture = 0;
int64_t rounds = 0;
private:
@@ -73,8 +77,8 @@ private:
const Tensor& find(const std::string& name) const;
const void* q8(const std::string& name) const { return blob_ + find(name).off; }
const float* f32(const std::string& name) const { return reinterpret_cast<const float*>(blob_ + find(name).off); }
- void mm(const std::string& name, const void* xq, float* y, int ncols); ///< Q8_0 matrix x ncols activation columns (quantized in xq)
- void quant(const float* x, int n_in, int ncols);
+ /// planar Q8 matrices (1..3 of them, same input width) times the planar activations in xqp_ / xdp_ (ncols columns)
+ void gemm(std::initializer_list<const char*> names, std::initializer_list<float*> outs, int ncols, cudaStream_t cs, const uint8_t* xq, const float* xd);
bool record_context(const float* taps, int n, cudaStream_t cs, std::string& err);
bool record_block(cudaStream_t cs, std::string& err);
bool capture(int n, const float* taps, std::string& err);
@@ -84,6 +88,7 @@ private:
void stamp(const char* name);
void prof_collect();
void prof_report() const;
+ void prof_free();
int device_ = -1;
cudaStream_t cs_ = nullptr;
@@ -112,7 +117,14 @@ private:
float* hp_out_ = nullptr;
float *kring_[8] = {}, *vring_[8] = {};
float *rt_ctx_ = nullptr, *rt_blk_ = nullptr;
- uint8_t* xq_ = nullptr;
+ uint8_t* xq_ = nullptr; ///< the head's q8_1 activations
+ uint8_t* xqp_ = nullptr; ///< planar activations of the drafter's own GEMMs (kernels/dflash2.hpp)
+ float* xdp_ = nullptr;
+ uint8_t* xqc_ = nullptr; ///< the same for the context update, which runs beside the block
+ float* xdc_ = nullptr;
+ cudaStream_t cs2_ = nullptr; ///< the context update's branch of the round graph
+ cudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr, ev_ctx_[8] = {};
+ bool fork_ = false; ///< recording the forked graph: layer l's attention waits for ev_ctx_[l]
float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;
float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;
float *ao_ = nullptr, *o_ = nullptr, *gate_ = nullptr, *up_ = nullptr, *mlp_ = nullptr, *part_ = nullptr, *fn_ = nullptr;
diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hpp
index 8a544f298..0ce4775b3 100644
--- a/include/strata/kernels/dflash2.hpp
+++ b/include/strata/kernels/dflash2.hpp
@@ -14,8 +14,8 @@ constexpr int kDf2Rows = 8; ///< block rows (the anchor + 7 masks) an
constexpr int kDf2Kv = 8; ///< K/V heads
constexpr int kDf2Gqa = 4; ///< query heads per K/V head
constexpr int kDf2Hd = 128; ///< head dim
-constexpr int kDf2Chunk = 256; ///< keys per attention block
-constexpr int kDf2MaxChunks = 9; ///< ceil((2047 context keys + 8 block keys) / kDf2Chunk)
+constexpr int kDf2Chunk = 128; ///< keys per attention block
+constexpr int kDf2MaxChunks = 17; ///< ceil((2047 context keys + 8 block keys) / kDf2Chunk)
constexpr int kDf2TopK = 16;
constexpr int kDf2TopBlocks = 64; ///< partial top-k blocks per row (a block holds at most 4096 logits)
@@ -38,12 +38,33 @@ void df2_norm_rope(const float* in, int in_stride, const float* gamma, const int
void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod, int rows, int cols, void* stream);
/// Attention of the block's 8 query rows (q: [8][32][128], already normed and rotated) over the context ring (kring, vring:
-/// [ring_slots][8][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when
+/// head-major [8][ring_slots][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when
/// s + i - j < window (j in [max(0, s - window + 1), s - 1], slot j % ring_slots) and every block key. `s` is read from *anchor_pos
/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined into out [8][32*128].
void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,
int ring_slots, int window, float* part, float* out, void* stream);
+/// The Q8 GEMM the drafter's matrices run on. A weight matrix is TILED (see the layout note in dflash2.cu; df2_q8_tile makes it from GGUF Q8_0 blocks):
+/// the int8 plane at `w` (n_out * n_in bytes) then the fp16 scale plane. Activations are planar (df2_quant_rows and the fused quantizers below):
+/// xq is int8 [col][n_in] (8 columns of n_in bytes), xd float [col][n_in / 32]. Up to 3 matrices that share the input run in one launch; y (matrix i)
+/// is [ncols][n_out]. n_in a multiple of 512, n_out of 16, ncols 1..8 (the other columns of xq are read and dropped).
+/// Bench only: pick a GEMM kernel variant (0 = the default).
+void df2_gemm_variant(int v);
+/// Bench only: attention variants (bit 0 skips Q K^T, bit 1 skips P V).
+void df2_attention_variant(int v);
+struct Df2Mat { const void* w; float* y; int n_out; };
+void df2_gemm(const Df2Mat* mats, int n, const void* xq, const float* xd, int n_in, int ncols, void* stream);
+/// Planar quantization of rows x[ncols][K].
+void df2_quant_rows(const float* x, int K, int ncols, void* xq, float* xd, void* stream);
+/// RMS norm (weights gamma) of `rows` rows of N, written to xn (optional) and quantized.
+void df2_rms_quant(const float* h, const float* gamma, float* xn, int rows, int N, float eps, void* xq, float* xd, void* stream);
+/// df2_conv without residual, then quantized.
+void df2_conv_quant(const float* x, const float* dyn, int dyn_stride, const float* base, int rows, int H, int group, void* xq, float* xd, void* stream);
+/// silu(gate) * up of `rows` rows of I, quantized.
+void df2_swiglu_quant(const float* gate, const float* up, int rows, int I, void* xq, float* xd, void* stream);
+/// GGUF Q8_0 blocks (34 bytes) of a matrix with rows of n_in -> the tiled qs plane and the tiled d plane (count = rows * n_in / 32 blocks).
+void df2_q8_tile(const void* blocks, void* qs, void* dpl, int n_in, int64_t count, void* stream);
+
/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id.
/// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.
void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);
diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cpp
index 80687d3fa..6a04146bc 100644
--- a/src/core/dflash2.cpp
+++ b/src/core/dflash2.cpp
@@ -17,6 +17,7 @@
#include <cstring>
#include <exception>
#include <atomic>
+#include <initializer_list>
#include <stdexcept>
namespace strata::core {
@@ -38,13 +39,13 @@ constexpr int GGML_Q8_0 = 8;
Dflash2Drafter::~Dflash2Drafter() {
if (device_ < 0) return;
prof_report();
- delete prof_;
+ prof_free();
const OnDevice on_device(device_);
if (cs_) cudaStreamSynchronize(cs_);
for (cudaGraphExec_t& e : exec_)
if (e) cudaGraphExecDestroy(e);
auto fr = [](void* p) { if (p) cudaFree(p); };
- fr(blob_); fr(d_meta_); fr(xq_); fr(rt_ctx_); fr(rt_blk_);
+ fr(blob_); fr(d_meta_); fr(xq_); fr(xqp_); fr(xdp_); fr(xqc_); fr(xdc_); fr(rt_ctx_); fr(rt_blk_);
for (int l = 0; l < 8; ++l) { fr(kring_[l]); fr(vring_[l]); }
fr(fcy_); fr(hctx_); fr(kc_); fr(vc_); fr(h_); fr(xn_); fr(xc_); fr(dyn_); fr(q_); fr(qn_); fr(k_); fr(v_); fr(kblk_); fr(ao_); fr(o_);
fr(gate_); fr(up_); fr(mlp_); fr(part_); fr(fn_); fr(logits_); fr(topval_); fr(hsel_); fr(toptok_); fr(topscratch_);
@@ -52,6 +53,10 @@ Dflash2Drafter::~Dflash2Drafter() {
if (h_meta_) cudaFreeHost(h_meta_);
if (h_out_) cudaFreeHost(h_out_);
if (cs_) cudaStreamDestroy(cs_);
+ if (cs2_) cudaStreamDestroy(cs2_);
+ if (ev_fork_) cudaEventDestroy(ev_fork_);
+ if (ev_join_) cudaEventDestroy(ev_join_);
+ for (cudaEvent_t e : ev_ctx_) if (e) cudaEventDestroy(e);
if (ev_a_) cudaEventDestroy(ev_a_);
if (ev_b_) cudaEventDestroy(ev_b_);
}
@@ -64,6 +69,17 @@ struct Dflash2Drafter::Prof {
int64_t rounds = 0;
};
+void Dflash2Drafter::prof_free() {
+ delete prof_;
+ prof_ = nullptr;
+}
+
+// STRATA_DF2_SKIP=attn,head,ctx,mlp,qkvo (measurement only): leaves those parts out of the round, so the graph's GPU time without them can be read
+static bool skip_part(const char* what) {
+ static const std::string list = [] { const char* e = std::getenv("STRATA_DF2_SKIP"); return std::string(e ? e : ""); }();
+ return !list.empty() && list.find(what) != std::string::npos;
+}
+
void Dflash2Drafter::stamp(const char* name) {
if (!prof_) return;
cudaEvent_t e;
@@ -154,19 +170,38 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)
}
if (cudaMalloc((void**) &blob_, total) != cudaSuccess) { err = "dflash2: the pack does not fit in VRAM (" + std::to_string(total >> 20) + " MiB)"; return false; }
vram_ += total;
- std::vector<uint8_t> stage(64u << 20);
+ uint64_t max_bytes = 0;
+ for (const Tensor& t : tensors_) max_bytes = std::max(max_bytes, t.bytes);
+ std::vector<uint8_t> stage(max_bytes);
+ uint8_t* tmp = nullptr;
+ if (cudaMalloc((void**) &tmp, max_bytes) != cudaSuccess) { err = "dflash2: upload scratch"; return false; }
+ struct TmpFree { uint8_t* p; ~TmpFree() { cudaFree(p); } } tmp_free{tmp};
+ for (const Tensor& t : tensors_) {
#if defined(_WIN32)
- _fseeki64(f, data_off, SEEK_SET);
+ _fseeki64(f, data_off + (long long) t.off, SEEK_SET);
#else
- fseeko(f, data_off, SEEK_SET);
+ fseeko(f, data_off + (off_t) t.off, SEEK_SET);
#endif
- for (uint64_t o = 0; o < total;) {
- const size_t n = (size_t) std::min<uint64_t>(stage.size(), total - o);
- if (std::fread(stage.data(), 1, n, f) != n) { err = "dflash2: the pack is shorter than its index"; return false; }
- if (cudaMemcpy(blob_ + o, stage.data(), n, cudaMemcpyHostToDevice) != cudaSuccess) { err = "dflash2: weight upload failed"; return false; }
- o += n;
+ if (std::fread(stage.data(), 1, t.bytes, f) != t.bytes) { err = "dflash2: the pack is shorter than its index"; return false; }
+ if (t.dtype == "q8_0") { // GGUF blocks -> the tiled qs plane at the tensor's offset, then the d plane (same total bytes)
+ if (cudaMemcpy(tmp, stage.data(), t.bytes, cudaMemcpyHostToDevice) != cudaSuccess) { err = "dflash2: weight upload failed"; return false; }
+ kernels::df2_q8_tile(tmp, blob_ + t.off, blob_ + t.off + (uint64_t) t.rows * (uint64_t) t.cols, (int) t.cols, t.rows * t.cols / 32, nullptr);
+ if (cudaDeviceSynchronize() != cudaSuccess) { err = "dflash2: weight conversion failed"; return false; }
+ } else if (cudaMemcpy(blob_ + t.off, stage.data(), t.bytes, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = "dflash2: weight upload failed";
+ return false;
+ }
}
- if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "dflash2: stream"; return false; }
+ if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess || cudaStreamCreateWithFlags(&cs2_, cudaStreamNonBlocking) != cudaSuccess) {
+ err = "dflash2: stream";
+ return false;
+ }
+ if (cudaEventCreateWithFlags(&ev_fork_, cudaEventDisableTiming) != cudaSuccess || cudaEventCreateWithFlags(&ev_join_, cudaEventDisableTiming) != cudaSuccess) {
+ err = "dflash2: events";
+ return false;
+ }
+ for (int l = 0; l < 8; ++l)
+ if (cudaEventCreateWithFlags(&ev_ctx_[l], cudaEventDisableTiming) != cudaSuccess) { err = "dflash2: events"; return false; }
if (const char* e = std::getenv("STRATA_DF2_PROFILE"); e && e[0] == '1') prof_ = new Prof();
if (const char* e = std::getenv("STRATA_DF2_TIMING"); e && e[0] == '1') {
timing_ = true;
@@ -184,7 +219,7 @@ bool Dflash2Drafter::alloc(std::string& err) {
return true;
};
const size_t B = kRows;
- bool ok = dev(&d_meta_, 32) && dev(&xq_, kernels::native_q8_1_bytes(std::max(N_ * 5, I_), kRows) + 4096) && dev(&rt_ctx_, B * 128) && dev(&rt_blk_, B * 128) &&
+ bool ok = dev(&d_meta_, 32) && dev(&xq_, kernels::native_q8_1_bytes(N_, kRows) + 4096) && dev(&xqp_, (size_t) 5 * N_ * 8 + 4096) && dev(&xdp_, (size_t) 5 * N_ / 32 * 8 + 64) && dev(&xqc_, (size_t) 5 * N_ * 8 + 4096) && dev(&xdc_, (size_t) 5 * N_ / 32 * 8 + 64) && dev(&rt_ctx_, B * 128) && dev(&rt_blk_, B * 128) &&
dev(&fcy_, B * N_) && dev(&hctx_, B * N_) && dev(&kc_, B * NKV_ * HD_) && dev(&vc_, B * NKV_ * HD_) && dev(&h_, B * N_) && dev(&xn_, B * N_) &&
dev(&xc_, B * N_) && dev(&dyn_, B * 640) && dev(&q_, B * NH_ * HD_) && dev(&qn_, B * NH_ * HD_) && dev(&k_, B * NKV_ * HD_) && dev(&v_, B * NKV_ * HD_) &&
dev(&kblk_, B * NKV_ * HD_) && dev(&ao_, B * NH_ * HD_) && dev(&o_, B * N_) && dev(&gate_, B * I_) && dev(&up_, B * I_) && dev(&mlp_, B * N_) &&
@@ -202,6 +237,10 @@ bool Dflash2Drafter::alloc(std::string& err) {
out_tok_ = (int32_t*) dd; // the select kernel writes the 7 picks here (host-visible) ...
out_prob_ = (float*) dd + 16; // ... and their probabilities at +16 ints
cudaMemset(rt_ctx_, 0, B * 128 * 4);
+ cudaMemset(xqp_, 0, (size_t) 5 * N_ * 8 + 4096);
+ cudaMemset(xdp_, 0, ((size_t) 5 * N_ / 32 * 8 + 64) * 4);
+ cudaMemset(xqc_, 0, (size_t) 5 * N_ * 8 + 4096);
+ cudaMemset(xdc_, 0, ((size_t) 5 * N_ / 32 * 8 + 64) * 4);
for (int l = 0; l < L_; ++l) { cudaMemset(kring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); cudaMemset(vring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); }
return cudaDeviceSynchronize() == cudaSuccess;
}
@@ -278,32 +317,43 @@ bool Dflash2Drafter::bind(const WeightTable& wt, const NativeHead* head, const s
return true;
}
-void Dflash2Drafter::quant(const float* x, int n_in, int ncols) { kernels::native_quantize_q8_1(x, xq_, n_in, ncols, cs_); }
-
-void Dflash2Drafter::mm(const std::string& name, const void* xq, float* y, int ncols) {
- const Tensor& t = find(name);
- kernels::native_q8_0_mmvq(blob_ + t.off, xq, y, (int) t.cols, (int) t.rows, ncols, cs_);
+void Dflash2Drafter::gemm(std::initializer_list<const char*> names, std::initializer_list<float*> outs, int ncols, cudaStream_t cs, const uint8_t* xq, const float* xd) {
+ kernels::Df2Mat mats[3];
+ int n = 0, n_in = 0;
+ auto o = outs.begin();
+ for (const char* name : names) {
+ const Tensor& t = find(name);
+ mats[n] = kernels::Df2Mat{blob_ + t.off, *o++, (int) t.rows};
+ if (n == 0) n_in = (int) t.cols;
+ ++n;
+ }
+ kernels::df2_gemm(mats, n, xq, xd, n_in, ncols, cs);
}
bool Dflash2Drafter::record_context(const float* taps, int n, cudaStream_t cs, std::string& err) {
using namespace kernels;
+ if (skip_part("ctx")) return true;
try {
const int tap_n = (int) taps_.size() * N_;
df2_rope_table(d_meta_ + 0, n, theta_, rt_ctx_, cs);
- stamp("ctx-start");
- quant(taps, tap_n, n);
- mm("fc.weight", xq_, fcy_, n);
- stamp("fc");
- native_qsa_rms_norm_weighted(fcy_, f32("hidden_norm.weight"), hctx_, N_, n, eps_, cs);
- quant(hctx_, N_, n);
+ stamp("df2_rope_table");
+ df2_quant_rows(taps, tap_n, n, xqc_, xdc_, cs);
+ stamp("df2_quant_rows");
+ gemm({"fc.weight"}, {fcy_}, n, cs, xqc_, xdc_);
+ stamp("gemm:fc");
+ df2_rms_quant(fcy_, f32("hidden_norm.weight"), nullptr, n, N_, eps_, xqc_, xdc_, cs);
+ stamp("df2_rms_quant");
for (int l = 0; l < L_; ++l) {
const std::string p = "layers." + std::to_string(l) + ".";
- mm(p + "self_attn.k_proj.weight", xq_, kc_, n);
- mm(p + "self_attn.v_proj.weight", xq_, vc_, n);
+ const std::string kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";
+ gemm({kp.c_str(), vp.c_str()}, {kc_, vc_}, n, cs, xqc_, xdc_);
+ stamp("gemm:kp");
df2_norm_rope(kc_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], kRing, n, NKV_, eps_, cs);
+ stamp("df2_norm_rope");
df2_copy_rows(vc_, vring_[l], d_meta_ + 0, kRing, n, NKV_ * HD_, cs);
+ stamp("df2_copy_rows");
+ if (fork_) cudaEventRecord(ev_ctx_[l], cs); // layer l's keys and values are in the ring
}
- stamp("ctx-kv");
} catch (const std::exception& e) {
err = e.what();
return false;
@@ -316,56 +366,65 @@ bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {
try {
const int B = kRows;
df2_rope_table(d_meta_ + 8, B, theta_, rt_blk_, cs);
+ stamp("df2_rope_table");
embed_(d_meta_ + 24, B, h_, cs);
- stamp("embed");
+ stamp("embed_");
for (int l = 0; l < L_; ++l) {
const std::string p = "layers." + std::to_string(l) + ".";
const float* base_a = f32(p + "attention_conv.base_kernel");
const float* base_m = f32(p + "mlp_conv.base_kernel");
- native_qsa_rms_norm_weighted(h_, f32(p + "input_layernorm.weight"), xn_, N_, B, eps_, cs);
- quant(xn_, N_, B);
- mm(p + "attention_conv.kernel_projection.weight", xq_, dyn_, B);
- df2_conv(xn_, dyn_, 640, base_a, nullptr, xc_, B, N_, group_, cs);
- quant(xc_, N_, B);
- stamp("attn-front");
- mm(p + "self_attn.q_proj.weight", xq_, q_, B);
- mm(p + "self_attn.k_proj.weight", xq_, k_, B);
- mm(p + "self_attn.v_proj.weight", xq_, v_, B);
- stamp("qkv");
+ const std::string akp = p + "attention_conv.kernel_projection.weight", mkp = p + "mlp_conv.kernel_projection.weight";
+ const std::string qp = p + "self_attn.q_proj.weight", kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";
+ const std::string gp = p + "mlp.gate_proj.weight", up = p + "mlp.up_proj.weight";
+ df2_rms_quant(h_, f32(p + "input_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);
+ stamp("df2_rms_quant");
+ gemm({akp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);
+ stamp("gemm:akp");
+ df2_conv_quant(xn_, dyn_, 640, base_a, B, N_, group_, xqp_, xdp_, cs);
+ stamp("df2_conv_quant");
+ if (!skip_part("qkvo")) gemm({qp.c_str(), kp.c_str(), vp.c_str()}, {q_, k_, v_}, B, cs, xqp_, xdp_);
+ stamp("gemm:qp");
df2_norm_rope(q_, NH_ * HD_, f32(p + "self_attn.q_norm.weight"), d_meta_ + 8, rt_blk_, qn_, 0, B, NH_, eps_, cs);
+ stamp("df2_norm_rope");
df2_norm_rope(k_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 8, rt_blk_, kblk_, 0, B, NKV_, eps_, cs);
- df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);
- stamp("attention");
- quant(ao_, NH_ * HD_, B);
- mm(p + "self_attn.o_proj.weight", xq_, o_, B);
- stamp("o_proj");
+ stamp("df2_norm_rope");
+ if (fork_) cudaStreamWaitEvent(cs, ev_ctx_[l], 0); // the context update has written this layer's ring
+ if (!skip_part("attn")) df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);
+ stamp("df2_attention");
+ df2_quant_rows(ao_, NH_ * HD_, B, xqp_, xdp_, cs);
+ stamp("df2_quant_rows");
+ if (!skip_part("qkvo")) gemm({(p + "self_attn.o_proj.weight").c_str()}, {o_}, B, cs, xqp_, xdp_);
+ stamp("gemm:o_proj");
df2_conv(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, h_, B, N_, group_, cs);
- native_qsa_rms_norm_weighted(h_, f32(p + "post_attention_layernorm.weight"), xn_, N_, B, eps_, cs);
- quant(xn_, N_, B);
- mm(p + "mlp_conv.kernel_projection.weight", xq_, dyn_, B);
- df2_conv(xn_, dyn_, 640, base_m, nullptr, xc_, B, N_, group_, cs);
- quant(xc_, N_, B);
- stamp("mlp-front");
- mm(p + "mlp.gate_proj.weight", xq_, gate_, B);
- mm(p + "mlp.up_proj.weight", xq_, up_, B);
- stamp("gate-up");
- native_swiglu_quantize_q8_1(gate_, up_, xq_, I_, B, cs);
- mm(p + "mlp.down_proj.weight", xq_, mlp_, B);
- stamp("down");
+ stamp("df2_conv");
+ df2_rms_quant(h_, f32(p + "post_attention_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);
+ stamp("df2_rms_quant");
+ gemm({mkp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);
+ stamp("gemm:mkp");
+ df2_conv_quant(xn_, dyn_, 640, base_m, B, N_, group_, xqp_, xdp_, cs);
+ stamp("df2_conv_quant");
+ if (!skip_part("mlp")) gemm({gp.c_str(), up.c_str()}, {gate_, up_}, B, cs, xqp_, xdp_);
+ stamp("gemm:gp");
+ df2_swiglu_quant(gate_, up_, B, I_, xqp_, xdp_, cs);
+ stamp("df2_swiglu_quant");
+ if (!skip_part("mlp")) gemm({(p + "mlp.down_proj.weight").c_str()}, {mlp_}, B, cs, xqp_, xdp_);
+ stamp("gemm:mlp.down_proj");
df2_conv(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, h_, B, N_, group_, cs);
+ stamp("df2_conv");
}
native_qsa_rms_norm_weighted(h_, f32("norm.weight"), fn_, N_, B, eps_, cs);
- quant(fn_ + N_, N_, kDrafts); // rows 1..7
- stamp("final-norm");
- native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);
- stamp("head");
+ stamp("native_qsa_rms_norm_weighted");
+ native_quantize_q8_1(fn_ + N_, xq_, N_, kDrafts, cs); // rows 1..7, the head's format
+ df2_quant_rows(fn_ + N_, N_, kDrafts, xqp_, xdp_, cs); // and the selector projection's
+ if (!skip_part("head")) native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);
+ stamp("native_mmvq");
df2_topk(logits_, (int) n_dvocab_, dvocab_, kDrafts, topscratch_, topval_, toptok_, cs);
- stamp("topk");
- mm("candidate_selector.hidden_projection.weight", xq_, hsel_, kDrafts);
+ stamp("df2_topk");
+ gemm({"candidate_selector.hidden_projection.weight"}, {hsel_}, kDrafts, cs, xqp_, xdp_);
+ stamp("gemm:candidate_selector.hidden_projection");
df2_select(hsel_, topval_, toptok_, reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.predecessor_codebook").off),
reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.successor_codebook").off), rank_, kDrafts, d_meta_ + 16, out_tok_,
out_prob_, cs);
- stamp("selector");
} catch (const std::exception& e) {
err = e.what();
return false;
@@ -377,8 +436,17 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {
if (exec_[n]) return true;
if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "dflash2: begin capture"; return false; }
kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);
- bool ok = record_context(taps, n, cs_, err);
+ // the context update (fc, the keys and values) and the block's front run side by side: they meet before each layer's attention
+ fork_ = prof_ == nullptr && cs2_ != nullptr;
+ if (fork_) {
+ cudaEventRecord(ev_fork_, cs_);
+ cudaStreamWaitEvent(cs2_, ev_fork_, 0);
+ }
+ bool ok = record_context(taps, n, fork_ ? cs2_ : cs_, err);
+ if (fork_) cudaEventRecord(ev_join_, cs2_);
if (ok) ok = record_block(cs_, err);
+ if (fork_) cudaStreamWaitEvent(cs_, ev_join_, 0);
+ fork_ = false;
cudaGraph_t graph = nullptr;
const cudaError_t ce = cudaStreamEndCapture(cs_, &graph);
if (!ok) { if (graph) cudaGraphDestroy(graph); return false; }
@@ -393,6 +461,19 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {
return true;
}
+bool Dflash2Drafter::prepare(const float* taps, std::string& err) {
+ if (!bound_) { err = "dflash2: not bound"; return false; }
+ if (prof_ != nullptr) return true; // the profile runs without graphs
+ const OnDevice on_device(device_);
+ const Clock::time_point t0 = Clock::now();
+ for (int i = 0; i < 8; ++i) h_meta_[i] = i;
+ for (int n = 1; n <= kRows; ++n)
+ if (!capture(n, taps, err)) return false;
+ graph_taps_ = taps;
+ ms_capture = ms_since(t0);
+ return true;
+}
+
bool Dflash2Drafter::context(const float* taps, int64_t n, int64_t pos0, std::string& err) {
const OnDevice on_device(device_);
const int tap_n = (int) taps_.size() * N_;
diff --git a/src/core/dflash2_attn_bench.cpp b/src/core/dflash2_attn_bench.cpp
new file mode 100644
index 000000000..897b5b823
--- /dev/null
+++ b/src/core/dflash2_attn_bench.cpp
@@ -0,0 +1,58 @@
+// src/core/dflash2_attn_bench.cpp - the drafter's attention (kernels/dflash2.hpp df2_attention) on random K/V: time per call and GB/s of K/V read.
+//
+// dflash2_attn_bench [iters] [context]
+#include "strata/kernels/dflash2.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+int main(int argc, char** argv) {
+ const int iters = argc > 1 ? std::atoi(argv[1]) : 100;
+ const int ctx = argc > 2 ? std::atoi(argv[2]) : 2047;
+ const int ring = 2064;
+ using namespace strata::kernels;
+ float *q, *kr, *vr, *kb, *vb, *part, *out;
+ int32_t* pos;
+ cudaMalloc((void**) &q, 8 * 32 * 128 * 4);
+ cudaMalloc((void**) &kr, (size_t) ring * 1024 * 4);
+ cudaMalloc((void**) &vr, (size_t) ring * 1024 * 4);
+ cudaMalloc((void**) &kb, 8 * 1024 * 4);
+ cudaMalloc((void**) &vb, 8 * 1024 * 4);
+ cudaMalloc((void**) &part, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);
+ cudaMalloc((void**) &out, 8 * 32 * 128 * 4);
+ cudaMalloc((void**) &pos, 4);
+ std::vector<float> h((size_t) ring * 1024);
+ for (size_t i = 0; i < h.size(); ++i) h[i] = (float) ((int) (i * 2654435761u >> 20) % 2001 - 1000) / 1000.0f;
+ cudaMemcpy(kr, h.data(), h.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(vr, h.data(), h.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(q, h.data(), 8 * 32 * 128 * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(kb, h.data(), 8 * 1024 * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(vb, h.data(), 8 * 1024 * 4, cudaMemcpyHostToDevice);
+ const int32_t s = ctx + 5000;
+ cudaMemcpy(pos, &s, 4, cudaMemcpyHostToDevice);
+ cudaEvent_t a, b;
+ cudaEventCreate(&a);
+ cudaEventCreate(&b);
+ for (int v : {0, 1, 2, 3}) {
+ df2_attention_variant(v);
+ df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);
+ cudaDeviceSynchronize();
+ cudaEventRecord(a);
+ for (int i = 0; i < iters; ++i) df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);
+ cudaEventRecord(b);
+ cudaEventSynchronize(b);
+ float ms;
+ cudaEventElapsedTime(&ms, a, b);
+ std::vector<float> o(8 * 32 * 128);
+ cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost);
+ double sum = 0;
+ for (float x : o) sum += x;
+ const double kv_bytes = 2.0 * (ctx + 8) * 1024 * 4;
+ std::printf("variant %d: %.1f us per call, K/V %.1f GB/s (checksum %.6f)\n", v, 1000.0 * ms / iters, kv_bytes * iters / (ms / 1000.0) / 1e9, sum);
+ }
+ return 0;
+}
diff --git a/src/core/dflash2_gemm_bench.cpp b/src/core/dflash2_gemm_bench.cpp
new file mode 100644
index 000000000..26a556853
--- /dev/null
+++ b/src/core/dflash2_gemm_bench.cpp
@@ -0,0 +1,62 @@
+// src/core/dflash2_gemm_bench.cpp - the drafter's Q8 GEMM (kernels/dflash2.hpp df2_gemm) on random planar weights: GB/s per shape and column count.
+//
+// dflash2_gemm_bench [iters]
+//
+// Weight sets are cycled through 2 GiB of distinct matrices so the L2 cache cannot hold them (the drafter reads each matrix once per round).
+#include "strata/kernels/dflash2.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+int main(int argc, char** argv) {
+ const int iters = argc > 1 ? std::atoi(argv[1]) : 50;
+ struct Shape { const char* name; int n_in, n_out; int mats; };
+ const Shape shapes[] = {{"gate+up 2560->7680 x2", 2560, 7680, 2}, {"down 7680->2560", 7680, 2560, 1}, {"qkv 2560->(4096,1024,1024)", 2560, 4096, 3},
+ {"o 4096->2560", 4096, 2560, 1}, {"fc 12800->2560", 12800, 2560, 1}, {"kv 2560->1024 x2", 2560, 1024, 2}};
+ cudaEvent_t a, b;
+ cudaEventCreate(&a);
+ cudaEventCreate(&b);
+ for (const Shape& sh : shapes) {
+ const size_t wbytes = (size_t) sh.n_out * sh.n_in * 34 / 32;
+ const size_t per_set = wbytes * sh.mats;
+ const int sets = (int) std::max<size_t>(2, ((size_t) 1 << 30) / per_set);
+ std::vector<uint8_t*> w((size_t) sets * sh.mats);
+ for (auto& p : w) {
+ if (cudaMalloc((void**) &p, wbytes) != cudaSuccess) { std::fprintf(stderr, "alloc\n"); return 1; }
+ cudaMemset(p, 0x11, wbytes);
+ }
+ uint8_t* xq;
+ float *xd, *y;
+ cudaMalloc((void**) &xq, (size_t) sh.n_in * 8);
+ cudaMalloc((void**) &xd, (size_t) sh.n_in / 32 * 8 * 4);
+ cudaMalloc((void**) &y, (size_t) 8 * sh.n_out * 3 * 4);
+ cudaMemset(xq, 3, (size_t) sh.n_in * 8);
+ std::vector<float> ones((size_t) sh.n_in / 32 * 8, 0.01f);
+ cudaMemcpy(xd, ones.data(), ones.size() * 4, cudaMemcpyHostToDevice);
+ for (int var : {1, 2, 3, 4, 5, 6, 7, 8, 9}) {
+ strata::kernels::df2_gemm_variant(var);
+ const int ncols = 8;
+ cudaDeviceSynchronize();
+ cudaEventRecord(a);
+ for (int it = 0; it < iters; ++it) {
+ strata::kernels::Df2Mat m[3];
+ for (int i = 0; i < sh.mats; ++i) m[i] = {w[(size_t) (it % sets) * sh.mats + i], y + (size_t) i * 8 * sh.n_out, sh.n_out};
+ if (sh.mats == 3) { m[1].n_out = 1024; m[2].n_out = 1024; }
+ strata::kernels::df2_gemm(m, sh.mats, xq, xd, sh.n_in, ncols, nullptr);
+ }
+ cudaEventRecord(b);
+ cudaEventSynchronize(b);
+ float ms;
+ cudaEventElapsedTime(&ms, a, b);
+ const double bytes = sh.mats == 3 ? (double) (4096 + 1024 + 1024) * sh.n_in * 34 / 32 : (double) per_set;
+ std::printf("%-30s variant %d: %7.1f us %6.1f GB/s\n", sh.name, var, 1000.0 * ms / iters, bytes * iters / (ms / 1000.0) / 1e9);
+ }
+ for (auto p : w) cudaFree(p);
+ cudaFree(xq); cudaFree(xd); cudaFree(y);
+ }
+ return 0;
+}
diff --git a/src/core/dflash2_launch_bench.cu b/src/core/dflash2_launch_bench.cu
new file mode 100644
index 000000000..d4fae689e
--- /dev/null
+++ b/src/core/dflash2_launch_bench.cu
@@ -0,0 +1,68 @@
+// src/core/dflash2_launch_bench.cpp - what a dependent kernel costs inside a captured graph on this GPU: N tiny kernels in a chain, and N kernels of 256 blocks.
+//
+// dflash2_launch_bench [n=100]
+#include <cuda_runtime.h>
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+
+__global__ void tiny(float* x, int n) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) x[i] = x[i] * 1.0001f + 1.0f;
+}
+
+int main(int argc, char** argv) {
+ const int n = argc > 1 ? std::atoi(argv[1]) : 100;
+ float* x;
+ cudaMalloc((void**) &x, 1 << 24);
+ cudaMemset(x, 0, 1 << 24);
+ cudaStream_t cs;
+ cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking);
+ cudaEvent_t a, b;
+ cudaEventCreate(&a);
+ cudaEventCreate(&b);
+ for (int blocks : {1, 64, 256, 1024}) {
+ cudaGraph_t g;
+ cudaGraphExec_t ge;
+ cudaStreamBeginCapture(cs, cudaStreamCaptureModeThreadLocal);
+ for (int i = 0; i < n; ++i) tiny<<<blocks, 256, 0, cs>>>(x, blocks * 256);
+ cudaStreamEndCapture(cs, &g);
+ cudaGraphInstantiate(&ge, g, 0);
+ cudaGraphLaunch(ge, cs);
+ cudaStreamSynchronize(cs);
+ cudaEventRecord(a, cs);
+ for (int it = 0; it < 50; ++it) cudaGraphLaunch(ge, cs);
+ cudaEventRecord(b, cs);
+ cudaEventSynchronize(b);
+ float ms;
+ cudaEventElapsedTime(&ms, a, b);
+ std::printf("graph of %d chained kernels, %4d blocks: %.2f us per kernel\n", n, blocks, 1000.0 * ms / 50 / n);
+ // one launch from an idle stream: event, launch, event, wait (what the drafter does each round)
+ {
+ double tot = 0, wall = 0;
+ for (int it = 0; it < 20; ++it) {
+ cudaStreamSynchronize(cs);
+ auto t0 = std::chrono::steady_clock::now();
+ cudaEventRecord(a, cs);
+ cudaGraphLaunch(ge, cs);
+ cudaEventRecord(b, cs);
+ cudaStreamSynchronize(cs);
+ wall += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+ float m;
+ cudaEventElapsedTime(&m, a, b);
+ tot += m;
+ }
+ std::printf("single launch from idle, %d kernels, %4d blocks: events %.3f ms, wall %.3f ms\n", n, blocks, tot / 20, wall / 20);
+ }
+ // the same without a graph
+ cudaEventRecord(a, cs);
+ for (int it = 0; it < 5; ++it)
+ for (int i = 0; i < n; ++i) tiny<<<blocks, 256, 0, cs>>>(x, blocks * 256);
+ cudaEventRecord(b, cs);
+ cudaEventSynchronize(b);
+ cudaEventElapsedTime(&ms, a, b);
+ std::printf("stream of %d chained kernels, %4d blocks: %.2f us per kernel\n", n, blocks, 1000.0 * ms / 5 / n);
+ }
+ return 0;
+}
diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cu
index 82c40ad63..fb98c4e25 100644
--- a/src/kernels/cuda/dflash2.cu
+++ b/src/kernels/cuda/dflash2.cu
@@ -1,6 +1,9 @@
// src/kernels/cuda/dflash2.cu - the small kernels of the DFlash2 drafter. See include/strata/kernels/dflash2.hpp.
#include "strata/kernels/dflash2.hpp"
+#include "strata/kernels/dp4a.hpp"
+
+#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <cfloat>
@@ -14,6 +17,8 @@ namespace strata::kernels {
namespace {
constexpr int HD = kDf2Hd;
+int g_gemm_variant = 0; // bench only (df2_gemm_variant)
+int g_attn_variant = 0; // bench only (df2_attention_variant)
void check(const char* what) {
const cudaError_t e = cudaGetLastError();
@@ -77,7 +82,8 @@ __global__ void norm_rope_kernel(const float* __restrict__ in, int in_stride, co
res = d < 64 ? v * c - y[d + 64] * s : v * c + y[d - 64] * s;
}
const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;
- out[((size_t) slot * heads + h) * HD + d] = res;
+ // a ring (slot_mod > 0) is head-major [head][slot][128], the block's own keys row-major [row][head][128]
+ out[slot_mod > 0 ? ((size_t) h * slot_mod + slot) * HD + d : ((size_t) slot * heads + h) * HD + d] = res;
}
__global__ void copy_rows_kernel(const float* __restrict__ in, float* __restrict__ out, const int32_t* __restrict__ pos, int slot_mod, int rows, int cols) {
@@ -85,106 +91,160 @@ __global__ void copy_rows_kernel(const float* __restrict__ in, float* __restrict
if (i >= rows * cols) return;
const int r = i / cols, c = i % cols;
const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;
- out[(size_t) slot * cols + c] = in[i];
+ out[slot_mod > 0 ? ((size_t) (c / HD) * slot_mod + slot) * HD + c % HD : (size_t) slot * cols + c] = in[i];
}
-// ---- attention. grid (8 K/V heads, kDf2MaxChunks), 256 threads. 32 queries (4 heads x 8 rows) per block share each key tile.
-constexpr int KS = 129; // smem row stride of the K / V / Q tiles (conflict-free)
-__global__ void __launch_bounds__(256) attn_part_kernel(const float* __restrict__ q, const float* __restrict__ kring, const float* __restrict__ vring,
+// ---- attention. grid (8 K/V heads, kDf2MaxChunks), 128 threads. The 32 queries (4 heads x 8 rows) of a K/V head share each key tile;
+// S = Q K^T in 4 x 4 register tiles (64 threads), the online softmax per query, O += P V in 4 x 8 register tiles (128 threads).
+constexpr int AS = 132; // smem row stride of the Q / K / V tiles (float4 aligned, conflict-free for the 4 x 4 tiles)
+constexpr int PS = 36; // stride of the transposed probabilities P[key][query]
+__global__ void __launch_bounds__(128) attn_part_kernel(const float* __restrict__ q, const float* __restrict__ kring, const float* __restrict__ vring,
const float* __restrict__ kblk, const float* __restrict__ vblk,
- const int32_t* __restrict__ anchor_pos, int ring_slots, int window, float* __restrict__ part) {
- __shared__ float Qs[32 * KS];
- __shared__ float Ks[32 * KS]; // the K tile, then the V tile
- __shared__ float Ss[32 * 33];
+ const int32_t* __restrict__ anchor_pos, int ring_slots, int window, float* __restrict__ part, int variant) {
+ __shared__ __align__(16) float Qs[32 * AS];
+ __shared__ __align__(16) float Ks[32 * AS]; // the K tile, then the V tile
+ __shared__ __align__(16) float Ps[32 * PS];
+ __shared__ float Sc[32 * 33];
+ __shared__ float m_s[32], l_s[32], corr_s[32];
const int kvh = blockIdx.x, chunk = blockIdx.y, tid = threadIdx.x;
const int s = *anchor_pos;
const int lo = s - (window - 1) > 0 ? s - (window - 1) : 0;
const int nctx = s - lo;
const int nkeys = nctx + kDf2Rows;
- const int qi = tid >> 3, tg = tid & 7;
- const int qrow = qi >> 2, qg = qi & 3;
- float* const pout = part + (((size_t) chunk * kDf2Kv + kvh) * 32 + qi) * 130;
const int k_begin = chunk * kDf2Chunk;
+ float* const pbase = part + (((size_t) chunk * kDf2Kv + kvh) * 32) * 130;
if (k_begin >= nkeys) {
- if (tg == 0) { pout[0] = -INFINITY; pout[1] = 0.0f; }
- for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = 0.0f;
+ for (int i = tid; i < 32 * 130; i += 128) pbase[i] = (i % 130 == 0) ? -INFINITY : 0.0f;
return;
}
const float scale = rsqrtf((float) HD);
- for (int i = tid; i < 32 * HD; i += 256) {
- const int a = i / HD, d = i % HD;
+ for (int i = tid; i < 32 * HD / 4; i += 128) { // Q: 32 queries x 128, float4
+ const int a = i / (HD / 4), d4 = i % (HD / 4);
const int row = a >> 2, g = a & 3;
- Qs[a * KS + d] = q[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] * scale;
+ float4 v = *reinterpret_cast<const float4*>(q + ((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d4 * 4);
+ v.x *= scale; v.y *= scale; v.z *= scale; v.w *= scale;
+ *reinterpret_cast<float4*>(&Qs[a * AS + d4 * 4]) = v;
}
- float m = -INFINITY, l = 0.0f;
- float O[16];
+ if (tid < 32) { m_s[tid] = -INFINITY; l_s[tid] = 0.0f; }
+ float O[4][8];
+#pragma unroll
+ for (int i = 0; i < 4; ++i)
#pragma unroll
- for (int e = 0; e < 16; ++e) O[e] = 0.0f;
+ for (int j = 0; j < 8; ++j) O[i][j] = 0.0f;
+ const int qg_pv = tid >> 4, dg_pv = tid & 15; // PV: queries qg*4 .. +3, dims dg*8 .. +7
const int k_end = min(k_begin + kDf2Chunk, nkeys);
- for (int k0 = k_begin; k0 < k_end; k0 += 32) {
- __syncthreads(); // Ks free (previous tile's V reads done), Qs written (first tile)
- for (int i = tid; i < 32 * HD; i += 256) {
- const int t = i / HD, d = i % HD, k = k0 + t;
- float v = 0.0f;
- if (k < nctx) v = kring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];
- else if (k < nkeys) v = kblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];
- Ks[t * KS + d] = v;
- }
- __syncthreads();
- float sc[4] = {0.f, 0.f, 0.f, 0.f};
-#pragma unroll 8
- for (int d = 0; d < HD; ++d) {
- const float qv = Qs[qi * KS + d];
+ // a tile is 32 keys x 128 dims = 1024 float4: eight per thread, loaded into registers ahead of their use
+ auto fetch = [&](int k0, float4 (&reg)[8], const float* ring, const float* blk) {
#pragma unroll
- for (int u = 0; u < 4; ++u) sc[u] = fmaf(qv, Ks[(tg * 4 + u) * KS + d], sc[u]);
+ for (int u = 0; u < 8; ++u) {
+ const int i = tid + u * 128, t = i / (HD / 4), d4 = i % (HD / 4), k = k0 + t;
+ float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+ if (k < nctx) v = *reinterpret_cast<const float4*>(ring + ((size_t) kvh * ring_slots + (lo + k) % ring_slots) * HD + d4 * 4);
+ else if (k < nkeys) v = *reinterpret_cast<const float4*>(blk + ((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d4 * 4);
+ reg[u] = v;
}
- float tmax = -INFINITY;
+ };
+ auto put = [&](const float4 (&reg)[8]) {
#pragma unroll
- for (int u = 0; u < 4; ++u) {
- const int k = k0 + tg * 4 + u;
- bool ok = k < nkeys;
- if (ok && k < nctx) ok = (lo + k) > (s + qrow - window); // s + row - j < window
- sc[u] = ok ? sc[u] : -INFINITY;
- tmax = fmaxf(tmax, sc[u]);
+ for (int u = 0; u < 8; ++u) {
+ const int i = tid + u * 128, t = i / (HD / 4), d4 = i % (HD / 4);
+ *reinterpret_cast<float4*>(&Ks[t * AS + d4 * 4]) = reg[u];
}
- tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 1));
- tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 2));
- tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 4));
- const float mn = fmaxf(m, tmax);
- const float corr = (m == -INFINITY) ? 0.0f : __expf(m - mn);
- float psum = 0.0f;
+ };
+ float4 kreg[8], vreg[8];
+ fetch(k_begin, kreg, kring, kblk);
+ for (int k0 = k_begin; k0 < k_end; k0 += 32) {
+ __syncthreads(); // Ks free (the last tile's V reads done); Qs / m_s written on the first pass
+ put(kreg);
+ fetch(k0, vreg, vring, vblk); // this tile's V, hidden behind the S stage and the softmax
+ __syncthreads();
+ if (tid < 64) { // S tile: 4 queries x 4 keys per thread
+ const int qg = tid >> 3, kg = tid & 7;
+ float acc[4][4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i)
+#pragma unroll
+ for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f;
+#pragma unroll 4
+ for (int d = 0; d < ((variant & 1) ? 0 : HD); d += 4) {
+ float4 qv[4], kv[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) qv[i] = *reinterpret_cast<const float4*>(&Qs[(qg * 4 + i) * AS + d]);
#pragma unroll
- for (int u = 0; u < 4; ++u) {
- const float p = sc[u] == -INFINITY ? 0.0f : __expf(sc[u] - mn);
- Ss[qi * 33 + tg * 4 + u] = p;
- psum += p;
+ for (int j = 0; j < 4; ++j) kv[j] = *reinterpret_cast<const float4*>(&Ks[(kg * 4 + j) * AS + d]);
+#pragma unroll
+ for (int i = 0; i < 4; ++i)
+#pragma unroll
+ for (int j = 0; j < 4; ++j)
+ acc[i][j] += qv[i].x * kv[j].x + qv[i].y * kv[j].y + qv[i].z * kv[j].z + qv[i].w * kv[j].w;
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i)
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int qi = qg * 4 + i, t = kg * 4 + j, k = k0 + t, row = qi >> 2;
+ bool ok = k < nkeys;
+ if (ok && k < nctx) ok = (lo + k) > (s + row - window); // s + row - j < window
+ Sc[qi * 33 + t] = ok ? acc[i][j] : -INFINITY;
+ }
}
- psum += __shfl_xor_sync(0xffffffffu, psum, 1);
- psum += __shfl_xor_sync(0xffffffffu, psum, 2);
- psum += __shfl_xor_sync(0xffffffffu, psum, 4);
- l = l * corr + psum;
- m = mn;
+ __syncthreads();
+ { // online softmax: query = tid / 4, 8 keys each, 4 lanes per query
+ const int qi = tid >> 2, sub = tid & 3;
+ float v[8], tmax = -INFINITY;
+#pragma unroll
+ for (int u = 0; u < 8; ++u) { v[u] = Sc[qi * 33 + sub * 8 + u]; tmax = fmaxf(tmax, v[u]); }
+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 1));
+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 2));
+ const float m_old = m_s[qi];
+ const float mn = fmaxf(m_old, tmax);
+ const float corr = (m_old == -INFINITY) ? 0.0f : __expf(m_old - mn);
+ float psum = 0.0f;
#pragma unroll
- for (int e = 0; e < 16; ++e) O[e] *= corr;
- __syncthreads(); // Ss complete; every read of Ks done
- for (int i = tid; i < 32 * HD; i += 256) {
- const int t = i / HD, d = i % HD, k = k0 + t;
- float v = 0.0f;
- if (k < nctx) v = vring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];
- else if (k < nkeys) v = vblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];
- Ks[t * KS + d] = v;
+ for (int u = 0; u < 8; ++u) {
+ const float p = v[u] == -INFINITY ? 0.0f : __expf(v[u] - mn);
+ Ps[(sub * 8 + u) * PS + qi] = p;
+ psum += p;
+ }
+ psum += __shfl_xor_sync(0xffffffffu, psum, 1);
+ psum += __shfl_xor_sync(0xffffffffu, psum, 2);
+ if (sub == 0) { l_s[qi] = l_s[qi] * corr + psum; m_s[qi] = mn; corr_s[qi] = corr; }
}
+ __syncthreads(); // Ps, corr_s complete; every read of Ks (the S tile) done
+ put(vreg);
+ if (k0 + 32 < k_end) fetch(k0 + 32, kreg, kring, kblk); // the next tile's K, hidden behind the PV stage
__syncthreads();
- for (int t = 0; t < 32; ++t) {
- const float p = Ss[qi * 33 + t];
+ {
+ const float4 c4 = *reinterpret_cast<const float4*>(&corr_s[qg_pv * 4]);
+ const float cr[4] = {c4.x, c4.y, c4.z, c4.w};
+#pragma unroll
+ for (int i = 0; i < 4; ++i)
+#pragma unroll
+ for (int j = 0; j < 8; ++j) O[i][j] *= cr[i];
+#pragma unroll 4
+ for (int t = 0; t < ((variant & 2) ? 0 : 32); ++t) {
+ const float4 p4 = *reinterpret_cast<const float4*>(&Ps[t * PS + qg_pv * 4]);
+ const float4 va = *reinterpret_cast<const float4*>(&Ks[t * AS + dg_pv * 8]);
+ const float4 vb = *reinterpret_cast<const float4*>(&Ks[t * AS + dg_pv * 8 + 4]);
+ const float pv[4] = {p4.x, p4.y, p4.z, p4.w};
+ const float vv[8] = {va.x, va.y, va.z, va.w, vb.x, vb.y, vb.z, vb.w};
#pragma unroll
- for (int e = 0; e < 16; ++e) O[e] = fmaf(p, Ks[t * KS + e * 8 + tg], O[e]);
+ for (int i = 0; i < 4; ++i)
+#pragma unroll
+ for (int j = 0; j < 8; ++j) O[i][j] = fmaf(pv[i], vv[j], O[i][j]);
+ }
}
}
- if (tg == 0) { pout[0] = m; pout[1] = l; }
+ __syncthreads();
+ if (tid < 32) { pbase[tid * 130] = m_s[tid]; pbase[tid * 130 + 1] = l_s[tid]; }
#pragma unroll
- for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = O[e];
- (void) qg;
+ for (int i = 0; i < 4; ++i) {
+ float* o = pbase + (size_t) (qg_pv * 4 + i) * 130 + 2 + dg_pv * 8;
+ *reinterpret_cast<float2*>(o) = make_float2(O[i][0], O[i][1]);
+ *reinterpret_cast<float2*>(o + 2) = make_float2(O[i][2], O[i][3]);
+ *reinterpret_cast<float2*>(o + 4) = make_float2(O[i][4], O[i][5]);
+ *reinterpret_cast<float2*>(o + 6) = make_float2(O[i][6], O[i][7]);
+ }
}
__global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrict__ out) {
@@ -204,6 +264,157 @@ __global__ void attn_merge_kernel(const float* __restrict__ part, float* __restr
out[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] = L > 0.0f ? o / L : 0.0f;
}
+// ---- Q8 GEMM for up to 8 activation columns (the drafter's matrices)
+//
+// Weights are TILED (df2_q8_tile makes them from GGUF Q8_0 blocks): a group of 8 rows is one unit; per 512 values of the reduction axis (a "step",
+// 16 Q8 blocks) the group stores the 8 rows' 512 int8 one after the other (4 KB), so a warp that works on a group reads 4 KB contiguous per step
+// (lane = block * 2 + half: one 16-byte load per row). The fp16 block scales follow all the int8 of the matrix, in the same tile order (8 rows x 16
+// blocks = 256 bytes a step). Activations: x_q (int8 [col][n_in]) and x_d (float [col][n_in / 32]); a column's 512 bytes of a step are contiguous.
+struct Df2Seg { const int8_t* q; const __half* d; float* y; int n_out; int row0; };
+struct Df2Segs { Df2Seg s[3]; int n; };
+
+template <int WARPS, bool COMPUTE, int UNR = 2>
+__global__ void __launch_bounds__(WARPS * 32) gemm8_kernel(Df2Segs segs, const int8_t* __restrict__ xq, const float* __restrict__ xd, int n_in, int ncols) {
+ constexpr int R = 8;
+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
+ const int row_g = (blockIdx.x * WARPS + warp) * R; // first row of this warp's group, over all the launch's matrices
+ int si = 0;
+ while (si + 1 < segs.n && row_g >= segs.s[si + 1].row0) ++si;
+ const Df2Seg sg = segs.s[si];
+ const int row0 = row_g - sg.row0;
+ const int nb = n_in / 32, steps = nb / 16;
+ const int4* wq = reinterpret_cast<const int4*>(sg.q + (size_t) row0 * n_in) + lane;
+ const __half* wd = sg.d + (size_t) row0 * nb + (lane >> 1);
+ const int4* xv = reinterpret_cast<const int4*>(xq) + lane;
+ float acc[R][8];
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < 8; ++c) acc[r][c] = 0.0f;
+#pragma unroll UNR
+ for (int st = 0; st < steps; ++st) {
+ int4 w[R];
+ float dw[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) {
+ w[r] = wq[(size_t) st * 256 + r * 32];
+ dw[r] = __half2float(wd[(size_t) st * 128 + r * 16]);
+ }
+ if (!COMPUTE) { // bench only: the same loads, almost no arithmetic
+ for (int r = 0; r < R; ++r) acc[r][0] += (float) (w[r].x ^ w[r].y ^ w[r].z ^ w[r].w) * dw[r];
+ continue;
+ }
+ float xs[8];
+#pragma unroll
+ for (int c = 0; c < 8; ++c) xs[c] = xd[(size_t) c * nb + st * 16 + (lane >> 1)];
+#pragma unroll
+ for (int c = 0; c < 8; ++c) {
+ const int4 x = xv[(size_t) c * (n_in / 16) + st * 32];
+#pragma unroll
+ for (int r = 0; r < R; ++r) {
+ int isum = STRATA_DP4A(w[r].x, x.x, 0);
+ isum = STRATA_DP4A(w[r].y, x.y, isum);
+ isum = STRATA_DP4A(w[r].z, x.z, isum);
+ isum = STRATA_DP4A(w[r].w, x.w, isum);
+ acc[r][c] = fmaf(dw[r] * xs[c], (float) isum, acc[r][c]);
+ }
+ }
+ }
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < 8; ++c) {
+ float v = acc[r][c];
+ v += __shfl_xor_sync(0xffffffffu, v, 16);
+ v += __shfl_xor_sync(0xffffffffu, v, 8);
+ v += __shfl_xor_sync(0xffffffffu, v, 4);
+ v += __shfl_xor_sync(0xffffffffu, v, 2);
+ v += __shfl_xor_sync(0xffffffffu, v, 1);
+ if (lane == 0 && c < ncols) sg.y[(size_t) c * sg.n_out + row0 + r] = v;
+ }
+}
+
+// quantize one 32-block of one row to the planar activation layout (a warp: lane = element)
+__device__ __forceinline__ void quant_block(float v, int blk, int col, int K, int8_t* __restrict__ xq, float* __restrict__ xd) {
+ const int lane = threadIdx.x & 31;
+ float amax = fabsf(v);
+ for (int o = 16; o > 0; o >>= 1) amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, o));
+ const float d = amax / 127.0f;
+ const float inv = amax > 0.0f ? 127.0f / amax : 0.0f;
+ const int q = __float2int_rn(v * inv);
+ xq[(size_t) col * K + blk * 32 + lane] = (int8_t) q;
+ if (lane == 0) xd[(size_t) col * (K / 32) + blk] = d;
+}
+
+// rows [ncols][K] -> planar; grid (ceil(K / 32 / 8), ncols), 256 threads (a warp per block)
+__global__ void quant_rows_kernel(const float* __restrict__ x, int K, int8_t* __restrict__ xq, float* __restrict__ xd) {
+ const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), col = blockIdx.y, lane = threadIdx.x & 31;
+ if (blk * 32 >= K) return;
+ quant_block(x[(size_t) col * K + blk * 32 + lane], blk, col, K, xq, xd);
+}
+
+// RMS norm of row `col` (N values) then the planar quantization of the normalized values; also writes them when xn != null. One block of 256 per row.
+__global__ void rms_quant_kernel(const float* __restrict__ h, const float* __restrict__ gamma, float* __restrict__ xn, int N, float eps,
+ int8_t* __restrict__ xq, float* __restrict__ xd) {
+ const int col = blockIdx.x, tid = threadIdx.x, warp = tid >> 5, lane = tid & 31;
+ __shared__ float red[8];
+ float ss = 0.0f;
+ for (int i = tid; i < N; i += 256) { const float v = h[(size_t) col * N + i]; ss += v * v; }
+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);
+ if (lane == 0) red[warp] = ss;
+ __syncthreads();
+ float tot = 0.0f;
+ for (int w = 0; w < 8; ++w) tot += red[w];
+ const float scale = rsqrtf(tot / (float) N + eps);
+ for (int blk = warp; blk * 32 < N; blk += 8) {
+ const int i = blk * 32 + lane;
+ const float v = h[(size_t) col * N + i] * scale * gamma[i];
+ if (xn) xn[(size_t) col * N + i] = v;
+ quant_block(v, blk, col, N, xq, xd);
+ }
+}
+
+// conv (as conv_kernel, no residual) then the planar quantization: grid (ceil(H / 32 / 8), rows), 256 threads
+__global__ void conv_quant_kernel(const float* __restrict__ x, const float* __restrict__ dyn, int dyn_stride, const float* __restrict__ base, int H, int group,
+ int8_t* __restrict__ xq, float* __restrict__ xd) {
+ const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), b = blockIdx.y, lane = threadIdx.x & 31;
+ if (blk * 32 >= H) return;
+ const int c = blk * 32 + lane, groups = H / group;
+ float acc = 0.0f;
+#pragma unroll
+ for (int o = 0; o < 2; ++o) {
+ if (b - o < 0) continue;
+ const float k = base[o * H + c] + dyn[(size_t) b * dyn_stride + o * groups + c / group];
+ acc = fmaf(k, x[(size_t) (b - o) * H + c], acc);
+ }
+ quant_block(acc, blk, b, H, xq, xd);
+}
+
+// silu(gate) * up then the planar quantization: grid (ceil(I / 32 / 8), rows), 256 threads
+__global__ void swiglu_quant_kernel(const float* __restrict__ gate, const float* __restrict__ up, int I, int8_t* __restrict__ xq, float* __restrict__ xd) {
+ const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), b = blockIdx.y, lane = threadIdx.x & 31;
+ if (blk * 32 >= I) return;
+ const size_t i = (size_t) b * I + blk * 32 + lane;
+ const float g = gate[i];
+ quant_block(g / (1.0f + __expf(-g)) * up[i], blk, b, I, xq, xd);
+}
+
+// GGUF Q8_0 blocks (34 bytes: fp16 d, 32 int8) of a [n_out][n_in] matrix -> the tiled qs plane (n_out * n_in bytes) and the tiled d plane; one thread per block
+__global__ void q8_tile_kernel(const uint8_t* __restrict__ blocks, int8_t* __restrict__ qs, __half* __restrict__ dpl, int n_in, int64_t count) {
+ const int64_t b = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (b >= count) return;
+ const int nb = n_in / 32;
+ const int64_t row = b / nb;
+ const int blk = (int) (b % nb);
+ const int64_t g = row / 8;
+ const int r = (int) (row % 8), st = blk / 16, bi = blk % 16;
+ const uint8_t* src = blocks + b * 34;
+ const uint16_t dh = (uint16_t) src[0] | ((uint16_t) src[1] << 8);
+ dpl[(g * 8 * nb) + (int64_t) st * 128 + r * 16 + bi] = __ushort_as_half(dh);
+ int8_t* o = qs + g * 8 * n_in + (int64_t) st * 4096 + r * 512 + bi * 32;
+ for (int i = 0; i < 32; ++i) o[i] = (int8_t) src[2 + i];
+}
+
// ---- top-16
constexpr int TK_CHUNK_MAX = 4096;
__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }
@@ -381,12 +592,69 @@ void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod
void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,
int ring_slots, int window, float* part, float* out, void* stream) {
- attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 256, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part);
+ attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 128, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part, g_attn_variant);
check("attn_part");
attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, out);
check("attn_merge");
}
+void df2_gemm_variant(int v) { g_gemm_variant = v; }
+void df2_attention_variant(int v) { g_attn_variant = v; }
+
+void df2_gemm(const Df2Mat* mats, int n, const void* xq, const float* xd, int n_in, int ncols, void* stream) {
+ if (n < 1 || n > 3 || n_in % 512 != 0 || ncols < 1 || ncols > 8) throw std::invalid_argument("df2_gemm: unsupported shape");
+ Df2Segs segs;
+ int rows = 0;
+ for (int i = 0; i < n; ++i) {
+ if (mats[i].n_out % 16 != 0) throw std::invalid_argument("df2_gemm: n_out must be a multiple of 16");
+ const int8_t* q = static_cast<const int8_t*>(mats[i].w);
+ segs.s[i] = Df2Seg{q, reinterpret_cast<const __half*>(q + (size_t) mats[i].n_out * n_in), mats[i].y, mats[i].n_out, rows};
+ rows += mats[i].n_out;
+ }
+ segs.n = n;
+ const int8_t* xqp = static_cast<const int8_t*>(xq);
+ cudaStream_t st = (cudaStream_t) stream;
+ const int variant = g_gemm_variant != 0 ? g_gemm_variant : (rows % 32 == 0 ? 1 : 2);
+ switch (variant) {
+ case 3: gemm8_kernel<2, false><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 4: gemm8_kernel<1, true, 2><<<rows / 8, 32, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 5: gemm8_kernel<2, true, 1><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 6: gemm8_kernel<2, true, 4><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 7: gemm8_kernel<4, true, 4><<<rows / 32, 128, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 8: gemm8_kernel<8, true, 2><<<rows / 64, 256, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 9: gemm8_kernel<4, false><<<rows / 32, 128, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ case 2: gemm8_kernel<2, true><<<rows / 16, 64, 0, st>>>(segs, xqp, xd, n_in, ncols); break; // 16 rows a block: every n_out is a multiple of 16
+ default: gemm8_kernel<4, true><<<rows / 32, 128, 0, st>>>(segs, xqp, xd, n_in, ncols); break;
+ }
+ check("gemm8");
+}
+
+void df2_quant_rows(const float* x, int K, int ncols, void* xq, float* xd, void* stream) {
+ quant_rows_kernel<<<dim3((K / 32 + 7) / 8, ncols), 256, 0, (cudaStream_t) stream>>>(x, K, static_cast<int8_t*>(xq), xd);
+ check("quant_rows");
+}
+
+void df2_rms_quant(const float* h, const float* gamma, float* xn, int rows, int N, float eps, void* xq, float* xd, void* stream) {
+ rms_quant_kernel<<<rows, 256, 0, (cudaStream_t) stream>>>(h, gamma, xn, N, eps, static_cast<int8_t*>(xq), xd);
+ check("rms_quant");
+}
+
+void df2_conv_quant(const float* x, const float* dyn, int dyn_stride, const float* base, int rows, int H, int group, void* xq, float* xd, void* stream) {
+ conv_quant_kernel<<<dim3((H / 32 + 7) / 8, rows), 256, 0, (cudaStream_t) stream>>>(x, dyn, dyn_stride, base, H, group, static_cast<int8_t*>(xq), xd);
+ check("conv_quant");
+}
+
+void df2_swiglu_quant(const float* gate, const float* up, int rows, int I, void* xq, float* xd, void* stream) {
+ swiglu_quant_kernel<<<dim3((I / 32 + 7) / 8, rows), 256, 0, (cudaStream_t) stream>>>(gate, up, I, static_cast<int8_t*>(xq), xd);
+ check("swiglu_quant");
+}
+
+void df2_q8_tile(const void* blocks, void* qs, void* dpl, int n_in, int64_t count, void* stream) {
+ q8_tile_kernel<<<(unsigned) ((count + 255) / 256), 256, 0, (cudaStream_t) stream>>>(static_cast<const uint8_t*>(blocks), static_cast<int8_t*>(qs),
+ static_cast<__half*>(dpl), n_in, count);
+ check("q8_tile");
+}
+
void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream) {
if ((n + kDf2TopBlocks - 1) / kDf2TopBlocks > TK_CHUNK_MAX) throw std::invalid_argument("df2_topk: too many logits per row");
float* cval = reinterpret_cast<float*>(scratch);
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 4f75c9fe1..d7ec9a694 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -16027,7 +16027,11 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
use_df2 = false;
}
- if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s), the MTP layer stays as the fallback\n", o.draft_model.c_str());
+ if (use_df2 && !df2.prepare(ver.tap_dev(), err)) {
+ std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
+ use_df2 = false;
+ }
+ if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s, graphs captured in %.0f ms), the MTP layer stays as the fallback\n", o.draft_model.c_str(), df2.ms_capture);
#endif
mem_mark("the verifier and the drafter's binding");
ver.set_sampling(sp); // the CLI's own sampling (until 0.1.19 this loop was always greedy); no penalties here
@@ -16332,6 +16336,23 @@ int main(int argc, char** argv) {
float pp[strata::core::Dflash2Drafter::kDrafts];
if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;
for (size_t i = 0; i < drafts.size() && i < (size_t) strata::core::Dflash2Drafter::kDrafts; ++i) { drafts[i] = dd[i]; dprob[i] = pp[i]; }
+ // what the MTP does while it drafts: the PLE rows of the tokens the next window will hold are read ahead
+ // (the window's gather would otherwise wait for them: 1.8 ms a round on strix)
+ if (ss.ple.ready() && ss.ple.table != nullptr) {
+ int n_pf = 0;
+ const int cap = std::min<int>(S_mtp, (int) drafts.size() + 1);
+ for (int t = 1; t < cap && (o.spec_min_p <= 0.0 || dprob[(size_t) t - 1] >= (float) o.spec_min_p); ++t) ++n_pf;
+ int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};
+ auto pf = [&](int32_t tok) {
+ uint32_t rows16[strata::kernels::PLE_N_HEADS];
+ strata::kernels::ngram_rows(&tok, prev, 1, ss.ple.consts, rows16);
+ prev[0] = prev[1];
+ prev[1] = tok;
+ ss.ple.table->prefetch_rows(rows16);
+ };
+ pf(outv[(size_t) a_]);
+ for (int i = 0; i < n_pf; ++i) pf(drafts[(size_t) i]);
+ }
return true;
}
#endif