Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dflash2: fused conv+residual+norm+quant, q/k norm+rope, K/V ring write and attention-output quantization; stage skip switches and small benches
Niko122189799596a638
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 0a918bc5e..77671a8b4 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -746,6 +746,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_executable(dflash2_launch_bench EXCLUDE_FROM_ALL src/core/dflash2_launch_bench.cu)target_include_directories(dflash2_launch_bench PRIVATE ${_strata_gpu_include_directories})target_link_libraries(dflash2_launch_bench PRIVATE ${_strata_gpu_runtime_target})+ add_executable(dflash2_bw_bench EXCLUDE_FROM_ALL src/core/dflash2_bw_bench.cu)+ target_include_directories(dflash2_bw_bench PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_bw_bench PRIVATE ${_strata_gpu_runtime_target})+ add_executable(dflash2_small_bench EXCLUDE_FROM_ALL src/core/dflash2_small_bench.cpp)+ target_include_directories(dflash2_small_bench PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_small_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_executable(dflash2_attn_bench EXCLUDE_FROM_ALL src/core/dflash2_attn_bench.cpp)target_include_directories(dflash2_attn_bench PRIVATE ${_strata_gpu_include_directories})target_link_libraries(dflash2_attn_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hppindex 205bff3d0..bad2bf5e9 100644--- a/include/strata/core/dflash2.hpp+++ b/include/strata/core/dflash2.hpp@@ -124,6 +124,9 @@ private:float* xdc_ = nullptr;cudaStream_t cs2_ = nullptr; ///< the context update's branch of the round graphcudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr, ev_ctx_[8] = {};+ const float* touch_p_[48] = {};+ int touch_c_[48] = {};+ int touch_n_ = 0;bool fork_ = false; ///< recording the forked graph: layer l's attention waits for ev_ctx_[l]float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hppindex 0ce4775b3..286174981 100644--- a/include/strata/kernels/dflash2.hpp+++ b/include/strata/kernels/dflash2.hpp@@ -40,9 +40,9 @@ void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod/// Attention of the block's 8 query rows (q: [8][32][128], already normed and rotated) over the context ring (kring, vring:/// head-major [8][ring_slots][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when/// s + i - j < window (j in [max(0, s - window + 1), s - 1], slot j % ring_slots) and every block key. `s` is read from *anchor_pos-/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined into out [8][32*128].+/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined and the 8 output rows (4096 wide) written planar-quantized to xq / xd.void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,- int ring_slots, int window, float* part, float* out, void* stream);+ int ring_slots, int window, float* part, void* xq, float* xd, void* stream);/// The Q8 GEMM the drafter's matrices run on. A weight matrix is TILED (see the layout note in dflash2.cu; df2_q8_tile makes it from GGUF Q8_0 blocks):/// the int8 plane at `w` (n_out * n_in bytes) then the fp16 scale plane. Activations are planar (df2_quant_rows and the fused quantizers below):@@ -65,6 +65,20 @@ void df2_swiglu_quant(const float* gate, const float* up, int rows, int I, void*/// GGUF Q8_0 blocks (34 bytes) of a matrix with rows of n_in -> the tiled qs plane and the tiled d plane (count = rows * n_in / 32 blocks).void df2_q8_tile(const void* blocks, void* qs, void* dpl, int n_in, int64_t count, void* stream);+/// h += conv(x) (the finish convolution with the residual), then the RMS norm of the new h (weights gamma) -> xn (optional) and the planar quantization.+/// rows <= 8, H <= 3072 (H a multiple of 32).+void df2_conv_rms_quant(const float* x, const float* dyn, int dyn_stride, const float* base, float* h, const float* gamma, float* xn, int rows, int H, int group,+ float eps, void* xq, float* xd, void* stream);+/// The block's q (q_heads x 128) and k (k_heads x 128) per row: RMS norm and rope with the table of the block's positions -> qo, ko (row-major).+void df2_norm_rope_qk(const float* q, const float* k, const float* gq, const float* gk, const float* table, float* qo, float* ko, int rows, int q_heads, int k_heads,+ float eps, void* stream);+/// The context update's write: K (norm + rope) and V of `rows` rows into the head-major rings at slot pos[r] % slots.+void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int32_t* pos, const float* table, float* kring, float* vring, int slots, int rows,+ int heads, float eps, void* stream);++/// Reads n small f32 tensors (ptrs[i], counts[i] floats; n <= 48) once to bring them into the caches; sink: one float of scratch.+void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream);+/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id./// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cppindex 6a04146bc..3da70987d 100644--- a/src/core/dflash2.cpp+++ b/src/core/dflash2.cpp@@ -208,6 +208,12 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)cudaEventCreate(&ev_a_);cudaEventCreate(&ev_b_);}+ // the small f32 tensors the round touches first (see df2_touch)+ for (const Tensor& t : tensors_)+ if (t.dtype == "f32" && t.rows * t.cols * 4 <= (1 << 20) && touch_n_ < 48) {+ touch_p_[touch_n_] = reinterpret_cast<const float*>(blob_ + t.off);+ touch_c_[touch_n_++] = (int) (t.rows * t.cols);+ }return alloc(err);}@@ -348,10 +354,8 @@ bool Dflash2Drafter::record_context(const float* taps, int n, cudaStream_t cs, sconst std::string kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";gemm({kp.c_str(), vp.c_str()}, {kc_, vc_}, n, cs, xqc_, xdc_);stamp("gemm:kp");- df2_norm_rope(kc_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], kRing, n, NKV_, eps_, cs);- stamp("df2_norm_rope");- df2_copy_rows(vc_, vring_[l], d_meta_ + 0, kRing, n, NKV_ * HD_, cs);- stamp("df2_copy_rows");+ df2_kv_ring(kc_, vc_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], vring_[l], kRing, n, NKV_, eps_, cs);+ stamp("df2_kv_ring");if (fork_) cudaEventRecord(ev_ctx_[l], cs); // layer l's keys and values are in the ring}} catch (const std::exception& e) {@@ -369,6 +373,8 @@ bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {stamp("df2_rope_table");embed_(d_meta_ + 24, B, h_, cs);stamp("embed_");+ df2_rms_quant(h_, f32("layers.0.input_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);+ stamp("df2_rms_quant");for (int l = 0; l < L_; ++l) {const std::string p = "layers." + std::to_string(l) + ".";const float* base_a = f32(p + "attention_conv.base_kernel");@@ -376,44 +382,36 @@ bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {const std::string akp = p + "attention_conv.kernel_projection.weight", mkp = p + "mlp_conv.kernel_projection.weight";const std::string qp = p + "self_attn.q_proj.weight", kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";const std::string gp = p + "mlp.gate_proj.weight", up = p + "mlp.up_proj.weight";- df2_rms_quant(h_, f32(p + "input_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);- stamp("df2_rms_quant");- gemm({akp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);+ if (!skip_part("tiny")) gemm({akp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);stamp("gemm:akp");- df2_conv_quant(xn_, dyn_, 640, base_a, B, N_, group_, xqp_, xdp_, cs);+ if (!skip_part("small")) df2_conv_quant(xn_, dyn_, 640, base_a, B, N_, group_, xqp_, xdp_, cs);stamp("df2_conv_quant");if (!skip_part("qkvo")) gemm({qp.c_str(), kp.c_str(), vp.c_str()}, {q_, k_, v_}, B, cs, xqp_, xdp_);stamp("gemm:qp");- df2_norm_rope(q_, NH_ * HD_, f32(p + "self_attn.q_norm.weight"), d_meta_ + 8, rt_blk_, qn_, 0, B, NH_, eps_, cs);- stamp("df2_norm_rope");- df2_norm_rope(k_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 8, rt_blk_, kblk_, 0, B, NKV_, eps_, cs);- stamp("df2_norm_rope");+ if (!skip_part("small")) df2_norm_rope_qk(q_, k_, f32(p + "self_attn.q_norm.weight"), f32(p + "self_attn.k_norm.weight"), rt_blk_, qn_, kblk_, B, NH_, NKV_, eps_, cs);+ stamp("df2_norm_rope_qk");if (fork_) cudaStreamWaitEvent(cs, ev_ctx_[l], 0); // the context update has written this layer's ring- if (!skip_part("attn")) df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);+ if (!skip_part("attn")) df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, xqp_, xdp_, cs);stamp("df2_attention");- df2_quant_rows(ao_, NH_ * HD_, B, xqp_, xdp_, cs);- stamp("df2_quant_rows");if (!skip_part("qkvo")) gemm({(p + "self_attn.o_proj.weight").c_str()}, {o_}, B, cs, xqp_, xdp_);stamp("gemm:o_proj");- df2_conv(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, h_, B, N_, group_, cs);- stamp("df2_conv");- df2_rms_quant(h_, f32(p + "post_attention_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);- stamp("df2_rms_quant");- gemm({mkp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);+ if (!skip_part("small")) df2_conv_rms_quant(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, f32(p + "post_attention_layernorm.weight"), xn_, B, N_, group_, eps_, xqp_, xdp_, cs);+ stamp("df2_conv_rms_quant");+ if (!skip_part("tiny")) gemm({mkp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);stamp("gemm:mkp");- df2_conv_quant(xn_, dyn_, 640, base_m, B, N_, group_, xqp_, xdp_, cs);+ if (!skip_part("small")) df2_conv_quant(xn_, dyn_, 640, base_m, B, N_, group_, xqp_, xdp_, cs);stamp("df2_conv_quant");if (!skip_part("mlp")) gemm({gp.c_str(), up.c_str()}, {gate_, up_}, B, cs, xqp_, xdp_);stamp("gemm:gp");- df2_swiglu_quant(gate_, up_, B, I_, xqp_, xdp_, cs);+ if (!skip_part("small")) df2_swiglu_quant(gate_, up_, B, I_, xqp_, xdp_, cs);stamp("df2_swiglu_quant");if (!skip_part("mlp")) gemm({(p + "mlp.down_proj.weight").c_str()}, {mlp_}, B, cs, xqp_, xdp_);stamp("gemm:mlp.down_proj");- df2_conv(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, h_, B, N_, group_, cs);- stamp("df2_conv");+ // the next layer's input norm, or after the last layer the final norm (-> fn_, the rows the head and the selector read)+ const std::string next_gamma = l + 1 < L_ ? "layers." + std::to_string(l + 1) + ".input_layernorm.weight" : std::string("norm.weight");+ if (!skip_part("small")) df2_conv_rms_quant(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, f32(next_gamma), l + 1 < L_ ? xn_ : fn_, B, N_, group_, eps_, xqp_, xdp_, cs);+ stamp("df2_conv_rms_quant");}- native_qsa_rms_norm_weighted(h_, f32("norm.weight"), fn_, N_, B, eps_, cs);- stamp("native_qsa_rms_norm_weighted");native_quantize_q8_1(fn_ + N_, xq_, N_, kDrafts, cs); // rows 1..7, the head's formatdf2_quant_rows(fn_ + N_, N_, kDrafts, xqp_, xdp_, cs); // and the selector projection'sif (!skip_part("head")) native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);@@ -437,11 +435,19 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "dflash2: begin capture"; return false; }kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);// the context update (fc, the keys and values) and the block's front run side by side: they meet before each layer's attention- fork_ = prof_ == nullptr && cs2_ != nullptr;+ // measured: -0.3 ms a round on an RTX 3060, +0.0..0.7 on a Strix Halo (gfx1151), so the HIP builds keep one stream (STRATA_DF2_FORK=1 / 0 overrides)+#if defined(STRATA_USE_HIP)+ static const bool fork_default = false;+#else+ static const bool fork_default = true;+#endif+ static const bool fork_on = [] { const char* e = std::getenv("STRATA_DF2_FORK"); return e && e[0] ? e[0] == '1' : fork_default; }();+ fork_ = fork_on && prof_ == nullptr && cs2_ != nullptr;if (fork_) {cudaEventRecord(ev_fork_, cs_);cudaStreamWaitEvent(cs2_, ev_fork_, 0);}+ if (touch_n_ > 0 && prof_ == nullptr && !skip_part("notouch")) kernels::df2_touch(touch_p_, touch_c_, touch_n_, hsel_, cs_);bool ok = record_context(taps, n, fork_ ? cs2_ : cs_, err);if (fork_) cudaEventRecord(ev_join_, cs2_);if (ok) ok = record_block(cs_, err);diff --git a/src/core/dflash2_attn_bench.cpp b/src/core/dflash2_attn_bench.cppindex 897b5b823..17253171b 100644--- a/src/core/dflash2_attn_bench.cpp+++ b/src/core/dflash2_attn_bench.cpp@@ -24,6 +24,8 @@ int main(int argc, char** argv) {cudaMalloc((void**) &vb, 8 * 1024 * 4);cudaMalloc((void**) &part, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);cudaMalloc((void**) &out, 8 * 32 * 128 * 4);+ float* xdb;+ cudaMalloc((void**) &xdb, 8 * 128 * 4);cudaMalloc((void**) &pos, 4);std::vector<float> h((size_t) ring * 1024);for (size_t i = 0; i < h.size(); ++i) h[i] = (float) ((int) (i * 2654435761u >> 20) % 2001 - 1000) / 1000.0f;@@ -39,16 +41,16 @@ int main(int argc, char** argv) {cudaEventCreate(&b);for (int v : {0, 1, 2, 3}) {df2_attention_variant(v);- df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);+ df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, xdb, nullptr);cudaDeviceSynchronize();cudaEventRecord(a);- for (int i = 0; i < iters; ++i) df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);+ for (int i = 0; i < iters; ++i) df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, xdb, nullptr);cudaEventRecord(b);cudaEventSynchronize(b);float ms;cudaEventElapsedTime(&ms, a, b);- std::vector<float> o(8 * 32 * 128);- cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost);+ std::vector<float> o(8 * 128);+ cudaMemcpy(o.data(), xdb, o.size() * 4, cudaMemcpyDeviceToHost);double sum = 0;for (float x : o) sum += x;const double kv_bytes = 2.0 * (ctx + 8) * 1024 * 4;diff --git a/src/core/dflash2_bw_bench.cu b/src/core/dflash2_bw_bench.cunew file mode 100644index 000000000..974a579ae--- /dev/null+++ b/src/core/dflash2_bw_bench.cu@@ -0,0 +1,41 @@+// read-bandwidth probe: sum of a 256 MB buffer with float4 loads, a few launch shapes+#include <cstdio>+#include <cuda_runtime.h>+__global__ void rd(const float4* __restrict__ p, size_t n, float* out) {+ float s = 0;+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (size_t) gridDim.x * blockDim.x) {+ float4 v = p[i];+ s += v.x + v.y + v.z + v.w;+ }+ if (s == 1234.5f) out[0] = s;+}+__global__ void rd_unroll(const int4* __restrict__ p, size_t n, int* out) {+ int s = 0;+ size_t stride = (size_t) gridDim.x * blockDim.x;+ size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x;+ for (; i + 3 * stride < n; i += 4 * stride) {+ int4 a = p[i], b = p[i + stride], c = p[i + 2 * stride], d = p[i + 3 * stride];+ s += a.x ^ a.y ^ a.z ^ a.w ^ b.x ^ b.y ^ b.z ^ b.w ^ c.x ^ c.y ^ c.z ^ c.w ^ d.x ^ d.y ^ d.z ^ d.w;+ }+ if (s == 1234) out[0] = s;+}+int main() {+ const size_t bytes = 256u << 20, n = bytes / 16;+ void* p; float* o;+ cudaMalloc(&p, bytes); cudaMalloc(&o, 4);+ cudaMemset(p, 1, bytes);+ cudaEvent_t a, b; cudaEventCreate(&a); cudaEventCreate(&b);+ for (int blocks : {112, 224, 448, 896, 3584, 14336}) {+ for (int k = 0; k < 2; ++k) {+ cudaEventRecord(a);+ for (int it = 0; it < 20; ++it) {+ if (k == 0) rd<<<blocks, 256>>>((const float4*) p, n, o);+ else rd_unroll<<<blocks, 256>>>((const int4*) p, n, (int*) o);+ }+ cudaEventRecord(b); cudaEventSynchronize(b);+ float ms; cudaEventElapsedTime(&ms, a, b);+ printf("%s blocks %5d: %.1f GB/s\n", k == 0 ? "loop " : "unroll", blocks, 20.0 * bytes / (ms / 1000) / 1e9);+ }+ }+ return 0;+}diff --git a/src/core/dflash2_small_bench.cpp b/src/core/dflash2_small_bench.cppnew file mode 100644index 000000000..e608410d2--- /dev/null+++ b/src/core/dflash2_small_bench.cpp@@ -0,0 +1,82 @@+// src/core/dflash2_small_bench.cpp - time of each of the drafter's small kernels, back to back (graph-free), on dummy data.+//+// dflash2_small_bench [iters]+#include "strata/kernels/dflash2.hpp"++#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <functional>+#include <vector>++using namespace strata::kernels;++static float* dev(size_t n, float v = 0.5f) {+ float* p;+ cudaMalloc((void**) &p, n * 4);+ std::vector<float> h(n);+ for (size_t i = 0; i < n; ++i) h[i] = v * (float) ((i * 2654435761u >> 12) % 1000) / 1000.0f + 0.01f;+ cudaMemcpy(p, h.data(), n * 4, cudaMemcpyHostToDevice);+ return p;+}++int main(int argc, char** argv) {+ const int iters = argc > 1 ? std::atoi(argv[1]) : 200;+ const int N = 2560, B = 8, I = 7680;+ float *h = dev(B * N), *xn = dev(B * N), *o = dev(B * N), *dyn = dev(B * 640), *base = dev(4 * N), *gamma = dev(I), *gate = dev(B * I), *up = dev(B * I);+ float *q = dev(B * 4096), *k = dev(B * 1024), *qn = dev(B * 4096), *kb = dev(B * 1024), *tab = dev(B * 128), *xd = dev(5 * N / 32 * 8);+ float *kr = dev((size_t) 2064 * 1024), *vr = dev((size_t) 2064 * 1024), *lg = dev((size_t) 7 * 106299), *tv = dev(7 * 16), *scr = dev(7 * 64 * 32), *hp = dev(7 * 256);+ uint8_t* xq;+ cudaMalloc((void**) &xq, 5 * N * 8 + 4096);+ cudaMemset(xq, 1, 5 * N * 8);+ int32_t *pos, *tok, *outt;+ float* outp;+ cudaMalloc((void**) &pos, 64);+ cudaMalloc((void**) &tok, 7 * 16 * 4);+ cudaMalloc((void**) &outt, 64);+ cudaMalloc((void**) &outp, 64);+ int32_t hpos[16] = {100, 101, 102, 103, 104, 105, 106, 107};+ cudaMemcpy(pos, hpos, 64, cudaMemcpyHostToDevice);+ cudaMemset(tok, 0, 7 * 16 * 4);+ uint16_t* cb;+ cudaMalloc((void**) &cb, (size_t) 1000 * 256 * 2);+ cudaMemset(cb, 0, (size_t) 1000 * 256 * 2);+ float* part;+ cudaMalloc((void**) &part, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);+ cudaMemset(part, 0, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);+ uint8_t* w;+ cudaMalloc((void**) &w, (size_t) 640 * N * 34 / 32);+ cudaMemset(w, 1, (size_t) 640 * N * 34 / 32);+ float* y = dev(8 * 640);+ cudaEvent_t a, b;+ cudaEventCreate(&a);+ cudaEventCreate(&b);+ auto time = [&](const char* name, const std::function<void()>& f) {+ f();+ cudaDeviceSynchronize();+ cudaEventRecord(a);+ for (int i = 0; i < iters; ++i) f();+ cudaEventRecord(b);+ cudaEventSynchronize(b);+ float ms;+ cudaEventElapsedTime(&ms, a, b);+ std::printf("%-28s %7.2f us\n", name, 1000.0 * ms / iters);+ };+ time("rms_quant (8 x 2560)", [&] { df2_rms_quant(h, gamma, xn, B, N, 1e-6f, xq, xd, nullptr); });+ time("conv_quant", [&] { df2_conv_quant(xn, dyn, 640, base, B, N, 16, xq, xd, nullptr); });+ time("conv_rms_quant", [&] { df2_conv_rms_quant(o, dyn, 640, base, h, gamma, xn, B, N, 16, 1e-6f, xq, xd, nullptr); });+ time("swiglu_quant (8 x 7680)", [&] { df2_swiglu_quant(gate, up, B, I, xq, xd, nullptr); });+ time("quant_rows (8 x 4096)", [&] { df2_quant_rows(q, 4096, B, xq, xd, nullptr); });+ time("norm_rope_qk", [&] { df2_norm_rope_qk(q, k, gamma, gamma, tab, qn, kb, B, 32, 8, 1e-6f, nullptr); });+ time("kv_ring (3 rows)", [&] { df2_kv_ring(k, k, gamma, pos, tab, kr, vr, 2064, 3, 8, 1e-6f, nullptr); });+ time("rope_table (8)", [&] { df2_rope_table(pos, 8, 10000.0f, tab, nullptr); });+ time("topk (7 x 106299)", [&] { df2_topk(lg, 106299, nullptr, 7, reinterpret_cast<uint32_t*>(scr), tv, tok, nullptr); });+ time("select (7 rows)", [&] { df2_select(hp, tv, tok, cb, cb, 256, 7, tok, outt, outp, nullptr); });+ time("conv (resid)", [&] { df2_conv(o, dyn, 640, base, h, h, B, N, 16, nullptr); });+ time("attention merge+quant only", [&] { df2_attention(q, kr, vr, kr, vr, pos, 2064, 2048, part, xq, xd, nullptr); });+ Df2Mat m{w, y, 640};+ time("gemm 640 x 2560 (tiny)", [&] { df2_gemm(&m, 1, xq, xd, N, 8, nullptr); });+ return 0;+}diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cuindex fb98c4e25..43f101457 100644--- a/src/kernels/cuda/dflash2.cu+++ b/src/kernels/cuda/dflash2.cu@@ -247,7 +247,9 @@ __global__ void __launch_bounds__(128) attn_part_kernel(const float* __restrict_}}-__global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrict__ out) {+__device__ __forceinline__ void quant_block(float v, int blk, int col, int K, int8_t* __restrict__ xq, float* __restrict__ xd);++__global__ void attn_merge_kernel(const float* __restrict__ part, int8_t* __restrict__ xq, float* __restrict__ xd) {const int a = blockIdx.x; // kvh * 32 + qiconst int kvh = a >> 5, qi = a & 31, d = threadIdx.x;float M = -INFINITY;@@ -261,7 +263,8 @@ __global__ void attn_merge_kernel(const float* __restrict__ part, float* __restro = fmaf(p[2 + d], w, o);}const int row = qi >> 2, g = qi & 3;- out[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] = L > 0.0f ? o / L : 0.0f;+ // the output row `row`, head kvh * 4 + g, dims d: quant block (kvh * 4 + g) * 4 + d / 32 of the 4096-wide row+ quant_block(L > 0.0f ? o / L : 0.0f, (kvh * kDf2Gqa + g) * (HD / 32) + (d >> 5), row, 32 * HD, xq, xd);}// ---- Q8 GEMM for up to 8 activation columns (the drafter's matrices)@@ -390,6 +393,98 @@ __global__ void conv_quant_kernel(const float* __restrict__ x, const float* __requant_block(acc, blk, b, H, xq, xd);}+// h += conv(x) (the finish convolution with its residual), then RMS norm of the new h with `gamma` -> xn (optional) and the planar quantization.+// One block of 1024 threads per row: a warp owns whole 32-blocks (80 of them for 2560), so the quantization needs no exchange.+__global__ void __launch_bounds__(1024) conv_rms_quant_kernel(const float* __restrict__ x, const float* __restrict__ dyn, int dyn_stride, const float* __restrict__ base,+ float* __restrict__ h, const float* __restrict__ gamma, float* __restrict__ xn, int H, int group,+ float eps, int8_t* __restrict__ xq, float* __restrict__ xd) {+ const int b = blockIdx.x, warp = threadIdx.x >> 5, lane = threadIdx.x & 31;+ const int groups = H / group, nb = H / 32;+ __shared__ float red[32];+ float v[3];+ float ss = 0.0f;+#pragma unroll+ for (int j = 0; j < 3; ++j) {+ const int blk = warp + j * 32;+ v[j] = 0.0f;+ if (blk < nb) {+ const int c = blk * 32 + lane;+ float acc = 0.0f;+#pragma unroll+ for (int o = 0; o < 2; ++o) {+ if (b - o < 0) continue;+ const float k = base[o * H + c] + dyn[(size_t) b * dyn_stride + o * groups + c / group];+ acc = fmaf(k, x[(size_t) (b - o) * H + c], acc);+ }+ v[j] = h[(size_t) b * H + c] + acc;+ h[(size_t) b * H + c] = v[j];+ ss += v[j] * v[j];+ }+ }+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);+ if (lane == 0) red[warp] = ss;+ __syncthreads();+ float tot = 0.0f;+ for (int w = 0; w < 32; ++w) tot += red[w];+ const float scale = rsqrtf(tot / (float) H + eps);+#pragma unroll+ for (int j = 0; j < 3; ++j) {+ const int blk = warp + j * 32;+ if (blk < nb) {+ const int c = blk * 32 + lane;+ const float y = v[j] * scale * gamma[c];+ if (xn) xn[(size_t) b * H + c] = y;+ quant_block(y, blk, b, H, xq, xd);+ }+ }+}++// q and k of the block: per (row, head) RMS norm and rope; blockIdx.y < q_heads is a q head (-> qo, row-major rows), the rest k heads (-> ko)+__global__ void norm_rope_qk_kernel(const float* __restrict__ q, const float* __restrict__ k, const float* __restrict__ gq, const float* __restrict__ gk,+ const float* __restrict__ table, float* __restrict__ qo, float* __restrict__ ko, int q_heads, int k_heads, float eps) {+ const int r = blockIdx.x, hh = blockIdx.y, d = threadIdx.x;+ const bool isq = hh < q_heads;+ const int h = isq ? hh : hh - q_heads, heads = isq ? q_heads : k_heads;+ const float* in = isq ? q : k;+ const float* gamma = isq ? gq : gk;+ float* out = isq ? qo : ko;+ __shared__ float red[4];+ __shared__ float y[HD];+ const float x = in[((size_t) r * heads + h) * HD + d];+ float ss = x * x;+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);+ if ((d & 31) == 0) red[d >> 5] = ss;+ __syncthreads();+ const float v = x * rsqrtf((red[0] + red[1] + red[2] + red[3]) / (float) HD + eps) * gamma[d];+ y[d] = v;+ __syncthreads();+ const int i = d & 63;+ const float c = table[(r * 2 + 0) * 64 + i], sn = table[(r * 2 + 1) * 64 + i];+ out[((size_t) r * heads + h) * HD + d] = d < 64 ? v * c - y[d + 64] * sn : v * c + y[d - 64] * sn;+}++// the context update's ring write: K norm + rope of (row, head) and the V copy, both into the head-major ring slot pos % slots+__global__ void kv_ring_kernel(const float* __restrict__ kin, const float* __restrict__ vin, const float* __restrict__ gk, const int32_t* __restrict__ pos,+ const float* __restrict__ table, float* __restrict__ kring, float* __restrict__ vring, int slots, int heads, float eps) {+ const int r = blockIdx.x, h = blockIdx.y, d = threadIdx.x;+ __shared__ float red[4];+ __shared__ float y[HD];+ const float x = kin[((size_t) r * heads + h) * HD + d];+ float ss = x * x;+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);+ if ((d & 31) == 0) red[d >> 5] = ss;+ __syncthreads();+ const float v = x * rsqrtf((red[0] + red[1] + red[2] + red[3]) / (float) HD + eps) * gk[d];+ y[d] = v;+ __syncthreads();+ const int i = d & 63;+ const float c = table[(r * 2 + 0) * 64 + i], sn = table[(r * 2 + 1) * 64 + i];+ const int slot = (int) (pos[r] % slots);+ const size_t at = ((size_t) h * slots + slot) * HD + d;+ kring[at] = d < 64 ? v * c - y[d + 64] * sn : v * c + y[d - 64] * sn;+ vring[at] = vin[((size_t) r * heads + h) * HD + d];+}+// silu(gate) * up then the planar quantization: grid (ceil(I / 32 / 8), rows), 256 threads__global__ void swiglu_quant_kernel(const float* __restrict__ gate, const float* __restrict__ up, int I, int8_t* __restrict__ xq, float* __restrict__ xd) {const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), b = blockIdx.y, lane = threadIdx.x & 31;@@ -415,6 +510,16 @@ __global__ void q8_tile_kernel(const uint8_t* __restrict__ blocks, int8_t* __resfor (int i = 0; i < 32; ++i) o[i] = (int8_t) src[2 + i];}+// Reads a list of small tensors (norm weights, the convolutions' base kernels, ...) once, so they sit in the caches before the first kernel that needs them+// (the verify window before the drafter streams gigabytes through them, and every dependent small kernel would otherwise wait for DRAM).+struct Df2Touch { const float* p[48]; int n[48]; int count; };+__global__ void touch_kernel(Df2Touch t, float* __restrict__ sink) {+ float s = 0.0f;+ for (int i = 0; i < t.count; ++i)+ for (int j = blockIdx.x * blockDim.x + threadIdx.x; j < t.n[i]; j += gridDim.x * blockDim.x) s += t.p[i][j];+ if (s == 1.2345e30f) sink[0] = s;+}+// ---- top-16constexpr int TK_CHUNK_MAX = 4096;__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }@@ -591,13 +696,41 @@ void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod}void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,- int ring_slots, int window, float* part, float* out, void* stream) {+ int ring_slots, int window, float* part, void* xq, float* xd, void* stream) {attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 128, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part, g_attn_variant);check("attn_part");- attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, out);+ attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, static_cast<int8_t*>(xq), xd);check("attn_merge");}+void df2_conv_rms_quant(const float* x, const float* dyn, int dyn_stride, const float* base, float* h, const float* gamma, float* xn, int rows, int H, int group,+ float eps, void* xq, float* xd, void* stream) {+ if (H > 3 * 1024) throw std::invalid_argument("df2_conv_rms_quant: H too large");+ conv_rms_quant_kernel<<<rows, 1024, 0, (cudaStream_t) stream>>>(x, dyn, dyn_stride, base, h, gamma, xn, H, group, eps, static_cast<int8_t*>(xq), xd);+ check("conv_rms_quant");+}++void df2_norm_rope_qk(const float* q, const float* k, const float* gq, const float* gk, const float* table, float* qo, float* ko, int rows, int q_heads, int k_heads,+ float eps, void* stream) {+ norm_rope_qk_kernel<<<dim3(rows, q_heads + k_heads), HD, 0, (cudaStream_t) stream>>>(q, k, gq, gk, table, qo, ko, q_heads, k_heads, eps);+ check("norm_rope_qk");+}++void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int32_t* pos, const float* table, float* kring, float* vring, int slots, int rows,+ int heads, float eps, void* stream) {+ kv_ring_kernel<<<dim3(rows, heads), HD, 0, (cudaStream_t) stream>>>(kin, vin, gk, pos, table, kring, vring, slots, heads, eps);+ check("kv_ring");+}++void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream) {+ Df2Touch t;+ if (n > 48) n = 48;+ for (int i = 0; i < n; ++i) { t.p[i] = ptrs[i]; t.n[i] = counts[i]; }+ t.count = n;+ touch_kernel<<<40, 256, 0, (cudaStream_t) stream>>>(t, sink);+ check("touch");+}+void df2_gemm_variant(int v) { g_gemm_variant = v; }void df2_attention_variant(int v) { g_attn_variant = v; }diff --git a/src/program/generate.cpp b/src/program/generate.cppindex d7ec9a694..42e1c7799 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -16332,6 +16332,8 @@ int main(int argc, char** argv) {auto draft_next = [&](int T_, int a_) -> bool {#if defined(STRATA_HAVE_DFLASH2)if (use_df2) {+ static const bool wait_commit_first = std::getenv("STRATA_DF2_WAITCOMMIT") != nullptr; // measurement: no overlap with the async commit+ if (wait_commit_first && !ver.wait_commit(err)) return false;int32_t dd[strata::core::Dflash2Drafter::kDrafts];float pp[strata::core::Dflash2Drafter::kDrafts];if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;