Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dflash2: fused conv+residual+norm+quant, q/k norm+rope, K/V ring write and attention-output quantization; stage skip switches and small benches

Niko122189799596a638
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0a918bc5e..77671a8b4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -746,6 +746,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_executable(dflash2_launch_bench EXCLUDE_FROM_ALL src/core/dflash2_launch_bench.cu)
target_include_directories(dflash2_launch_bench PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(dflash2_launch_bench PRIVATE ${_strata_gpu_runtime_target})
+ add_executable(dflash2_bw_bench EXCLUDE_FROM_ALL src/core/dflash2_bw_bench.cu)
+ target_include_directories(dflash2_bw_bench PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_bw_bench PRIVATE ${_strata_gpu_runtime_target})
+ add_executable(dflash2_small_bench EXCLUDE_FROM_ALL src/core/dflash2_small_bench.cpp)
+ target_include_directories(dflash2_small_bench PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_small_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_executable(dflash2_attn_bench EXCLUDE_FROM_ALL src/core/dflash2_attn_bench.cpp)
target_include_directories(dflash2_attn_bench PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(dflash2_attn_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})
diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hpp
index 205bff3d0..bad2bf5e9 100644
--- a/include/strata/core/dflash2.hpp
+++ b/include/strata/core/dflash2.hpp
@@ -124,6 +124,9 @@ private:
float* xdc_ = nullptr;
cudaStream_t cs2_ = nullptr; ///< the context update's branch of the round graph
cudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr, ev_ctx_[8] = {};
+ const float* touch_p_[48] = {};
+ int touch_c_[48] = {};
+ int touch_n_ = 0;
bool fork_ = false; ///< recording the forked graph: layer l's attention waits for ev_ctx_[l]
float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;
float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;
diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hpp
index 0ce4775b3..286174981 100644
--- a/include/strata/kernels/dflash2.hpp
+++ b/include/strata/kernels/dflash2.hpp
@@ -40,9 +40,9 @@ void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod
/// Attention of the block's 8 query rows (q: [8][32][128], already normed and rotated) over the context ring (kring, vring:
/// head-major [8][ring_slots][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when
/// s + i - j < window (j in [max(0, s - window + 1), s - 1], slot j % ring_slots) and every block key. `s` is read from *anchor_pos
-/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined into out [8][32*128].
+/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined and the 8 output rows (4096 wide) written planar-quantized to xq / xd.
void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,
- int ring_slots, int window, float* part, float* out, void* stream);
+ int ring_slots, int window, float* part, void* xq, float* xd, void* stream);
/// The Q8 GEMM the drafter's matrices run on. A weight matrix is TILED (see the layout note in dflash2.cu; df2_q8_tile makes it from GGUF Q8_0 blocks):
/// the int8 plane at `w` (n_out * n_in bytes) then the fp16 scale plane. Activations are planar (df2_quant_rows and the fused quantizers below):
@@ -65,6 +65,20 @@ void df2_swiglu_quant(const float* gate, const float* up, int rows, int I, void*
/// GGUF Q8_0 blocks (34 bytes) of a matrix with rows of n_in -> the tiled qs plane and the tiled d plane (count = rows * n_in / 32 blocks).
void df2_q8_tile(const void* blocks, void* qs, void* dpl, int n_in, int64_t count, void* stream);
+/// h += conv(x) (the finish convolution with the residual), then the RMS norm of the new h (weights gamma) -> xn (optional) and the planar quantization.
+/// rows <= 8, H <= 3072 (H a multiple of 32).
+void df2_conv_rms_quant(const float* x, const float* dyn, int dyn_stride, const float* base, float* h, const float* gamma, float* xn, int rows, int H, int group,
+ float eps, void* xq, float* xd, void* stream);
+/// The block's q (q_heads x 128) and k (k_heads x 128) per row: RMS norm and rope with the table of the block's positions -> qo, ko (row-major).
+void df2_norm_rope_qk(const float* q, const float* k, const float* gq, const float* gk, const float* table, float* qo, float* ko, int rows, int q_heads, int k_heads,
+ float eps, void* stream);
+/// The context update's write: K (norm + rope) and V of `rows` rows into the head-major rings at slot pos[r] % slots.
+void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int32_t* pos, const float* table, float* kring, float* vring, int slots, int rows,
+ int heads, float eps, void* stream);
+
+/// Reads n small f32 tensors (ptrs[i], counts[i] floats; n <= 48) once to bring them into the caches; sink: one float of scratch.
+void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream);
+
/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id.
/// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.
void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);
diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cpp
index 6a04146bc..3da70987d 100644
--- a/src/core/dflash2.cpp
+++ b/src/core/dflash2.cpp
@@ -208,6 +208,12 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)
cudaEventCreate(&ev_a_);
cudaEventCreate(&ev_b_);
}
+ // the small f32 tensors the round touches first (see df2_touch)
+ for (const Tensor& t : tensors_)
+ if (t.dtype == "f32" && t.rows * t.cols * 4 <= (1 << 20) && touch_n_ < 48) {
+ touch_p_[touch_n_] = reinterpret_cast<const float*>(blob_ + t.off);
+ touch_c_[touch_n_++] = (int) (t.rows * t.cols);
+ }
return alloc(err);
}
@@ -348,10 +354,8 @@ bool Dflash2Drafter::record_context(const float* taps, int n, cudaStream_t cs, s
const std::string kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";
gemm({kp.c_str(), vp.c_str()}, {kc_, vc_}, n, cs, xqc_, xdc_);
stamp("gemm:kp");
- df2_norm_rope(kc_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], kRing, n, NKV_, eps_, cs);
- stamp("df2_norm_rope");
- df2_copy_rows(vc_, vring_[l], d_meta_ + 0, kRing, n, NKV_ * HD_, cs);
- stamp("df2_copy_rows");
+ df2_kv_ring(kc_, vc_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], vring_[l], kRing, n, NKV_, eps_, cs);
+ stamp("df2_kv_ring");
if (fork_) cudaEventRecord(ev_ctx_[l], cs); // layer l's keys and values are in the ring
}
} catch (const std::exception& e) {
@@ -369,6 +373,8 @@ bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {
stamp("df2_rope_table");
embed_(d_meta_ + 24, B, h_, cs);
stamp("embed_");
+ df2_rms_quant(h_, f32("layers.0.input_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);
+ stamp("df2_rms_quant");
for (int l = 0; l < L_; ++l) {
const std::string p = "layers." + std::to_string(l) + ".";
const float* base_a = f32(p + "attention_conv.base_kernel");
@@ -376,44 +382,36 @@ bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {
const std::string akp = p + "attention_conv.kernel_projection.weight", mkp = p + "mlp_conv.kernel_projection.weight";
const std::string qp = p + "self_attn.q_proj.weight", kp = p + "self_attn.k_proj.weight", vp = p + "self_attn.v_proj.weight";
const std::string gp = p + "mlp.gate_proj.weight", up = p + "mlp.up_proj.weight";
- df2_rms_quant(h_, f32(p + "input_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);
- stamp("df2_rms_quant");
- gemm({akp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);
+ if (!skip_part("tiny")) gemm({akp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);
stamp("gemm:akp");
- df2_conv_quant(xn_, dyn_, 640, base_a, B, N_, group_, xqp_, xdp_, cs);
+ if (!skip_part("small")) df2_conv_quant(xn_, dyn_, 640, base_a, B, N_, group_, xqp_, xdp_, cs);
stamp("df2_conv_quant");
if (!skip_part("qkvo")) gemm({qp.c_str(), kp.c_str(), vp.c_str()}, {q_, k_, v_}, B, cs, xqp_, xdp_);
stamp("gemm:qp");
- df2_norm_rope(q_, NH_ * HD_, f32(p + "self_attn.q_norm.weight"), d_meta_ + 8, rt_blk_, qn_, 0, B, NH_, eps_, cs);
- stamp("df2_norm_rope");
- df2_norm_rope(k_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 8, rt_blk_, kblk_, 0, B, NKV_, eps_, cs);
- stamp("df2_norm_rope");
+ if (!skip_part("small")) df2_norm_rope_qk(q_, k_, f32(p + "self_attn.q_norm.weight"), f32(p + "self_attn.k_norm.weight"), rt_blk_, qn_, kblk_, B, NH_, NKV_, eps_, cs);
+ stamp("df2_norm_rope_qk");
if (fork_) cudaStreamWaitEvent(cs, ev_ctx_[l], 0); // the context update has written this layer's ring
- if (!skip_part("attn")) df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);
+ if (!skip_part("attn")) df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, xqp_, xdp_, cs);
stamp("df2_attention");
- df2_quant_rows(ao_, NH_ * HD_, B, xqp_, xdp_, cs);
- stamp("df2_quant_rows");
if (!skip_part("qkvo")) gemm({(p + "self_attn.o_proj.weight").c_str()}, {o_}, B, cs, xqp_, xdp_);
stamp("gemm:o_proj");
- df2_conv(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, h_, B, N_, group_, cs);
- stamp("df2_conv");
- df2_rms_quant(h_, f32(p + "post_attention_layernorm.weight"), xn_, B, N_, eps_, xqp_, xdp_, cs);
- stamp("df2_rms_quant");
- gemm({mkp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);
+ if (!skip_part("small")) df2_conv_rms_quant(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, f32(p + "post_attention_layernorm.weight"), xn_, B, N_, group_, eps_, xqp_, xdp_, cs);
+ stamp("df2_conv_rms_quant");
+ if (!skip_part("tiny")) gemm({mkp.c_str()}, {dyn_}, B, cs, xqp_, xdp_);
stamp("gemm:mkp");
- df2_conv_quant(xn_, dyn_, 640, base_m, B, N_, group_, xqp_, xdp_, cs);
+ if (!skip_part("small")) df2_conv_quant(xn_, dyn_, 640, base_m, B, N_, group_, xqp_, xdp_, cs);
stamp("df2_conv_quant");
if (!skip_part("mlp")) gemm({gp.c_str(), up.c_str()}, {gate_, up_}, B, cs, xqp_, xdp_);
stamp("gemm:gp");
- df2_swiglu_quant(gate_, up_, B, I_, xqp_, xdp_, cs);
+ if (!skip_part("small")) df2_swiglu_quant(gate_, up_, B, I_, xqp_, xdp_, cs);
stamp("df2_swiglu_quant");
if (!skip_part("mlp")) gemm({(p + "mlp.down_proj.weight").c_str()}, {mlp_}, B, cs, xqp_, xdp_);
stamp("gemm:mlp.down_proj");
- df2_conv(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, h_, B, N_, group_, cs);
- stamp("df2_conv");
+ // the next layer's input norm, or after the last layer the final norm (-> fn_, the rows the head and the selector read)
+ const std::string next_gamma = l + 1 < L_ ? "layers." + std::to_string(l + 1) + ".input_layernorm.weight" : std::string("norm.weight");
+ if (!skip_part("small")) df2_conv_rms_quant(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, f32(next_gamma), l + 1 < L_ ? xn_ : fn_, B, N_, group_, eps_, xqp_, xdp_, cs);
+ stamp("df2_conv_rms_quant");
}
- native_qsa_rms_norm_weighted(h_, f32("norm.weight"), fn_, N_, B, eps_, cs);
- stamp("native_qsa_rms_norm_weighted");
native_quantize_q8_1(fn_ + N_, xq_, N_, kDrafts, cs); // rows 1..7, the head's format
df2_quant_rows(fn_ + N_, N_, kDrafts, xqp_, xdp_, cs); // and the selector projection's
if (!skip_part("head")) native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);
@@ -437,11 +435,19 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {
if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "dflash2: begin capture"; return false; }
kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);
// the context update (fc, the keys and values) and the block's front run side by side: they meet before each layer's attention
- fork_ = prof_ == nullptr && cs2_ != nullptr;
+ // measured: -0.3 ms a round on an RTX 3060, +0.0..0.7 on a Strix Halo (gfx1151), so the HIP builds keep one stream (STRATA_DF2_FORK=1 / 0 overrides)
+#if defined(STRATA_USE_HIP)
+ static const bool fork_default = false;
+#else
+ static const bool fork_default = true;
+#endif
+ static const bool fork_on = [] { const char* e = std::getenv("STRATA_DF2_FORK"); return e && e[0] ? e[0] == '1' : fork_default; }();
+ fork_ = fork_on && prof_ == nullptr && cs2_ != nullptr;
if (fork_) {
cudaEventRecord(ev_fork_, cs_);
cudaStreamWaitEvent(cs2_, ev_fork_, 0);
}
+ if (touch_n_ > 0 && prof_ == nullptr && !skip_part("notouch")) kernels::df2_touch(touch_p_, touch_c_, touch_n_, hsel_, cs_);
bool ok = record_context(taps, n, fork_ ? cs2_ : cs_, err);
if (fork_) cudaEventRecord(ev_join_, cs2_);
if (ok) ok = record_block(cs_, err);
diff --git a/src/core/dflash2_attn_bench.cpp b/src/core/dflash2_attn_bench.cpp
index 897b5b823..17253171b 100644
--- a/src/core/dflash2_attn_bench.cpp
+++ b/src/core/dflash2_attn_bench.cpp
@@ -24,6 +24,8 @@ int main(int argc, char** argv) {
cudaMalloc((void**) &vb, 8 * 1024 * 4);
cudaMalloc((void**) &part, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);
cudaMalloc((void**) &out, 8 * 32 * 128 * 4);
+ float* xdb;
+ cudaMalloc((void**) &xdb, 8 * 128 * 4);
cudaMalloc((void**) &pos, 4);
std::vector<float> h((size_t) ring * 1024);
for (size_t i = 0; i < h.size(); ++i) h[i] = (float) ((int) (i * 2654435761u >> 20) % 2001 - 1000) / 1000.0f;
@@ -39,16 +41,16 @@ int main(int argc, char** argv) {
cudaEventCreate(&b);
for (int v : {0, 1, 2, 3}) {
df2_attention_variant(v);
- df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);
+ df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, xdb, nullptr);
cudaDeviceSynchronize();
cudaEventRecord(a);
- for (int i = 0; i < iters; ++i) df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, nullptr);
+ for (int i = 0; i < iters; ++i) df2_attention(q, kr, vr, kb, vb, pos, ring, 2048, part, out, xdb, nullptr);
cudaEventRecord(b);
cudaEventSynchronize(b);
float ms;
cudaEventElapsedTime(&ms, a, b);
- std::vector<float> o(8 * 32 * 128);
- cudaMemcpy(o.data(), out, o.size() * 4, cudaMemcpyDeviceToHost);
+ std::vector<float> o(8 * 128);
+ cudaMemcpy(o.data(), xdb, o.size() * 4, cudaMemcpyDeviceToHost);
double sum = 0;
for (float x : o) sum += x;
const double kv_bytes = 2.0 * (ctx + 8) * 1024 * 4;
diff --git a/src/core/dflash2_bw_bench.cu b/src/core/dflash2_bw_bench.cu
new file mode 100644
index 000000000..974a579ae
--- /dev/null
+++ b/src/core/dflash2_bw_bench.cu
@@ -0,0 +1,41 @@
+// read-bandwidth probe: sum of a 256 MB buffer with float4 loads, a few launch shapes
+#include <cstdio>
+#include <cuda_runtime.h>
+__global__ void rd(const float4* __restrict__ p, size_t n, float* out) {
+ float s = 0;
+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (size_t) gridDim.x * blockDim.x) {
+ float4 v = p[i];
+ s += v.x + v.y + v.z + v.w;
+ }
+ if (s == 1234.5f) out[0] = s;
+}
+__global__ void rd_unroll(const int4* __restrict__ p, size_t n, int* out) {
+ int s = 0;
+ size_t stride = (size_t) gridDim.x * blockDim.x;
+ size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x;
+ for (; i + 3 * stride < n; i += 4 * stride) {
+ int4 a = p[i], b = p[i + stride], c = p[i + 2 * stride], d = p[i + 3 * stride];
+ s += a.x ^ a.y ^ a.z ^ a.w ^ b.x ^ b.y ^ b.z ^ b.w ^ c.x ^ c.y ^ c.z ^ c.w ^ d.x ^ d.y ^ d.z ^ d.w;
+ }
+ if (s == 1234) out[0] = s;
+}
+int main() {
+ const size_t bytes = 256u << 20, n = bytes / 16;
+ void* p; float* o;
+ cudaMalloc(&p, bytes); cudaMalloc(&o, 4);
+ cudaMemset(p, 1, bytes);
+ cudaEvent_t a, b; cudaEventCreate(&a); cudaEventCreate(&b);
+ for (int blocks : {112, 224, 448, 896, 3584, 14336}) {
+ for (int k = 0; k < 2; ++k) {
+ cudaEventRecord(a);
+ for (int it = 0; it < 20; ++it) {
+ if (k == 0) rd<<<blocks, 256>>>((const float4*) p, n, o);
+ else rd_unroll<<<blocks, 256>>>((const int4*) p, n, (int*) o);
+ }
+ cudaEventRecord(b); cudaEventSynchronize(b);
+ float ms; cudaEventElapsedTime(&ms, a, b);
+ printf("%s blocks %5d: %.1f GB/s\n", k == 0 ? "loop " : "unroll", blocks, 20.0 * bytes / (ms / 1000) / 1e9);
+ }
+ }
+ return 0;
+}
diff --git a/src/core/dflash2_small_bench.cpp b/src/core/dflash2_small_bench.cpp
new file mode 100644
index 000000000..e608410d2
--- /dev/null
+++ b/src/core/dflash2_small_bench.cpp
@@ -0,0 +1,82 @@
+// src/core/dflash2_small_bench.cpp - time of each of the drafter's small kernels, back to back (graph-free), on dummy data.
+//
+// dflash2_small_bench [iters]
+#include "strata/kernels/dflash2.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+#include <vector>
+
+using namespace strata::kernels;
+
+static float* dev(size_t n, float v = 0.5f) {
+ float* p;
+ cudaMalloc((void**) &p, n * 4);
+ std::vector<float> h(n);
+ for (size_t i = 0; i < n; ++i) h[i] = v * (float) ((i * 2654435761u >> 12) % 1000) / 1000.0f + 0.01f;
+ cudaMemcpy(p, h.data(), n * 4, cudaMemcpyHostToDevice);
+ return p;
+}
+
+int main(int argc, char** argv) {
+ const int iters = argc > 1 ? std::atoi(argv[1]) : 200;
+ const int N = 2560, B = 8, I = 7680;
+ float *h = dev(B * N), *xn = dev(B * N), *o = dev(B * N), *dyn = dev(B * 640), *base = dev(4 * N), *gamma = dev(I), *gate = dev(B * I), *up = dev(B * I);
+ float *q = dev(B * 4096), *k = dev(B * 1024), *qn = dev(B * 4096), *kb = dev(B * 1024), *tab = dev(B * 128), *xd = dev(5 * N / 32 * 8);
+ float *kr = dev((size_t) 2064 * 1024), *vr = dev((size_t) 2064 * 1024), *lg = dev((size_t) 7 * 106299), *tv = dev(7 * 16), *scr = dev(7 * 64 * 32), *hp = dev(7 * 256);
+ uint8_t* xq;
+ cudaMalloc((void**) &xq, 5 * N * 8 + 4096);
+ cudaMemset(xq, 1, 5 * N * 8);
+ int32_t *pos, *tok, *outt;
+ float* outp;
+ cudaMalloc((void**) &pos, 64);
+ cudaMalloc((void**) &tok, 7 * 16 * 4);
+ cudaMalloc((void**) &outt, 64);
+ cudaMalloc((void**) &outp, 64);
+ int32_t hpos[16] = {100, 101, 102, 103, 104, 105, 106, 107};
+ cudaMemcpy(pos, hpos, 64, cudaMemcpyHostToDevice);
+ cudaMemset(tok, 0, 7 * 16 * 4);
+ uint16_t* cb;
+ cudaMalloc((void**) &cb, (size_t) 1000 * 256 * 2);
+ cudaMemset(cb, 0, (size_t) 1000 * 256 * 2);
+ float* part;
+ cudaMalloc((void**) &part, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);
+ cudaMemset(part, 0, (size_t) kDf2MaxChunks * 8 * 32 * 130 * 4);
+ uint8_t* w;
+ cudaMalloc((void**) &w, (size_t) 640 * N * 34 / 32);
+ cudaMemset(w, 1, (size_t) 640 * N * 34 / 32);
+ float* y = dev(8 * 640);
+ cudaEvent_t a, b;
+ cudaEventCreate(&a);
+ cudaEventCreate(&b);
+ auto time = [&](const char* name, const std::function<void()>& f) {
+ f();
+ cudaDeviceSynchronize();
+ cudaEventRecord(a);
+ for (int i = 0; i < iters; ++i) f();
+ cudaEventRecord(b);
+ cudaEventSynchronize(b);
+ float ms;
+ cudaEventElapsedTime(&ms, a, b);
+ std::printf("%-28s %7.2f us\n", name, 1000.0 * ms / iters);
+ };
+ time("rms_quant (8 x 2560)", [&] { df2_rms_quant(h, gamma, xn, B, N, 1e-6f, xq, xd, nullptr); });
+ time("conv_quant", [&] { df2_conv_quant(xn, dyn, 640, base, B, N, 16, xq, xd, nullptr); });
+ time("conv_rms_quant", [&] { df2_conv_rms_quant(o, dyn, 640, base, h, gamma, xn, B, N, 16, 1e-6f, xq, xd, nullptr); });
+ time("swiglu_quant (8 x 7680)", [&] { df2_swiglu_quant(gate, up, B, I, xq, xd, nullptr); });
+ time("quant_rows (8 x 4096)", [&] { df2_quant_rows(q, 4096, B, xq, xd, nullptr); });
+ time("norm_rope_qk", [&] { df2_norm_rope_qk(q, k, gamma, gamma, tab, qn, kb, B, 32, 8, 1e-6f, nullptr); });
+ time("kv_ring (3 rows)", [&] { df2_kv_ring(k, k, gamma, pos, tab, kr, vr, 2064, 3, 8, 1e-6f, nullptr); });
+ time("rope_table (8)", [&] { df2_rope_table(pos, 8, 10000.0f, tab, nullptr); });
+ time("topk (7 x 106299)", [&] { df2_topk(lg, 106299, nullptr, 7, reinterpret_cast<uint32_t*>(scr), tv, tok, nullptr); });
+ time("select (7 rows)", [&] { df2_select(hp, tv, tok, cb, cb, 256, 7, tok, outt, outp, nullptr); });
+ time("conv (resid)", [&] { df2_conv(o, dyn, 640, base, h, h, B, N, 16, nullptr); });
+ time("attention merge+quant only", [&] { df2_attention(q, kr, vr, kr, vr, pos, 2064, 2048, part, xq, xd, nullptr); });
+ Df2Mat m{w, y, 640};
+ time("gemm 640 x 2560 (tiny)", [&] { df2_gemm(&m, 1, xq, xd, N, 8, nullptr); });
+ return 0;
+}
diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cu
index fb98c4e25..43f101457 100644
--- a/src/kernels/cuda/dflash2.cu
+++ b/src/kernels/cuda/dflash2.cu
@@ -247,7 +247,9 @@ __global__ void __launch_bounds__(128) attn_part_kernel(const float* __restrict_
}
}
-__global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrict__ out) {
+__device__ __forceinline__ void quant_block(float v, int blk, int col, int K, int8_t* __restrict__ xq, float* __restrict__ xd);
+
+__global__ void attn_merge_kernel(const float* __restrict__ part, int8_t* __restrict__ xq, float* __restrict__ xd) {
const int a = blockIdx.x; // kvh * 32 + qi
const int kvh = a >> 5, qi = a & 31, d = threadIdx.x;
float M = -INFINITY;
@@ -261,7 +263,8 @@ __global__ void attn_merge_kernel(const float* __restrict__ part, float* __restr
o = fmaf(p[2 + d], w, o);
}
const int row = qi >> 2, g = qi & 3;
- out[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] = L > 0.0f ? o / L : 0.0f;
+ // the output row `row`, head kvh * 4 + g, dims d: quant block (kvh * 4 + g) * 4 + d / 32 of the 4096-wide row
+ quant_block(L > 0.0f ? o / L : 0.0f, (kvh * kDf2Gqa + g) * (HD / 32) + (d >> 5), row, 32 * HD, xq, xd);
}
// ---- Q8 GEMM for up to 8 activation columns (the drafter's matrices)
@@ -390,6 +393,98 @@ __global__ void conv_quant_kernel(const float* __restrict__ x, const float* __re
quant_block(acc, blk, b, H, xq, xd);
}
+// h += conv(x) (the finish convolution with its residual), then RMS norm of the new h with `gamma` -> xn (optional) and the planar quantization.
+// One block of 1024 threads per row: a warp owns whole 32-blocks (80 of them for 2560), so the quantization needs no exchange.
+__global__ void __launch_bounds__(1024) conv_rms_quant_kernel(const float* __restrict__ x, const float* __restrict__ dyn, int dyn_stride, const float* __restrict__ base,
+ float* __restrict__ h, const float* __restrict__ gamma, float* __restrict__ xn, int H, int group,
+ float eps, int8_t* __restrict__ xq, float* __restrict__ xd) {
+ const int b = blockIdx.x, warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
+ const int groups = H / group, nb = H / 32;
+ __shared__ float red[32];
+ float v[3];
+ float ss = 0.0f;
+#pragma unroll
+ for (int j = 0; j < 3; ++j) {
+ const int blk = warp + j * 32;
+ v[j] = 0.0f;
+ if (blk < nb) {
+ const int c = blk * 32 + lane;
+ float acc = 0.0f;
+#pragma unroll
+ for (int o = 0; o < 2; ++o) {
+ if (b - o < 0) continue;
+ const float k = base[o * H + c] + dyn[(size_t) b * dyn_stride + o * groups + c / group];
+ acc = fmaf(k, x[(size_t) (b - o) * H + c], acc);
+ }
+ v[j] = h[(size_t) b * H + c] + acc;
+ h[(size_t) b * H + c] = v[j];
+ ss += v[j] * v[j];
+ }
+ }
+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);
+ if (lane == 0) red[warp] = ss;
+ __syncthreads();
+ float tot = 0.0f;
+ for (int w = 0; w < 32; ++w) tot += red[w];
+ const float scale = rsqrtf(tot / (float) H + eps);
+#pragma unroll
+ for (int j = 0; j < 3; ++j) {
+ const int blk = warp + j * 32;
+ if (blk < nb) {
+ const int c = blk * 32 + lane;
+ const float y = v[j] * scale * gamma[c];
+ if (xn) xn[(size_t) b * H + c] = y;
+ quant_block(y, blk, b, H, xq, xd);
+ }
+ }
+}
+
+// q and k of the block: per (row, head) RMS norm and rope; blockIdx.y < q_heads is a q head (-> qo, row-major rows), the rest k heads (-> ko)
+__global__ void norm_rope_qk_kernel(const float* __restrict__ q, const float* __restrict__ k, const float* __restrict__ gq, const float* __restrict__ gk,
+ const float* __restrict__ table, float* __restrict__ qo, float* __restrict__ ko, int q_heads, int k_heads, float eps) {
+ const int r = blockIdx.x, hh = blockIdx.y, d = threadIdx.x;
+ const bool isq = hh < q_heads;
+ const int h = isq ? hh : hh - q_heads, heads = isq ? q_heads : k_heads;
+ const float* in = isq ? q : k;
+ const float* gamma = isq ? gq : gk;
+ float* out = isq ? qo : ko;
+ __shared__ float red[4];
+ __shared__ float y[HD];
+ const float x = in[((size_t) r * heads + h) * HD + d];
+ float ss = x * x;
+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);
+ if ((d & 31) == 0) red[d >> 5] = ss;
+ __syncthreads();
+ const float v = x * rsqrtf((red[0] + red[1] + red[2] + red[3]) / (float) HD + eps) * gamma[d];
+ y[d] = v;
+ __syncthreads();
+ const int i = d & 63;
+ const float c = table[(r * 2 + 0) * 64 + i], sn = table[(r * 2 + 1) * 64 + i];
+ out[((size_t) r * heads + h) * HD + d] = d < 64 ? v * c - y[d + 64] * sn : v * c + y[d - 64] * sn;
+}
+
+// the context update's ring write: K norm + rope of (row, head) and the V copy, both into the head-major ring slot pos % slots
+__global__ void kv_ring_kernel(const float* __restrict__ kin, const float* __restrict__ vin, const float* __restrict__ gk, const int32_t* __restrict__ pos,
+ const float* __restrict__ table, float* __restrict__ kring, float* __restrict__ vring, int slots, int heads, float eps) {
+ const int r = blockIdx.x, h = blockIdx.y, d = threadIdx.x;
+ __shared__ float red[4];
+ __shared__ float y[HD];
+ const float x = kin[((size_t) r * heads + h) * HD + d];
+ float ss = x * x;
+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);
+ if ((d & 31) == 0) red[d >> 5] = ss;
+ __syncthreads();
+ const float v = x * rsqrtf((red[0] + red[1] + red[2] + red[3]) / (float) HD + eps) * gk[d];
+ y[d] = v;
+ __syncthreads();
+ const int i = d & 63;
+ const float c = table[(r * 2 + 0) * 64 + i], sn = table[(r * 2 + 1) * 64 + i];
+ const int slot = (int) (pos[r] % slots);
+ const size_t at = ((size_t) h * slots + slot) * HD + d;
+ kring[at] = d < 64 ? v * c - y[d + 64] * sn : v * c + y[d - 64] * sn;
+ vring[at] = vin[((size_t) r * heads + h) * HD + d];
+}
+
// silu(gate) * up then the planar quantization: grid (ceil(I / 32 / 8), rows), 256 threads
__global__ void swiglu_quant_kernel(const float* __restrict__ gate, const float* __restrict__ up, int I, int8_t* __restrict__ xq, float* __restrict__ xd) {
const int blk = blockIdx.x * 8 + (threadIdx.x >> 5), b = blockIdx.y, lane = threadIdx.x & 31;
@@ -415,6 +510,16 @@ __global__ void q8_tile_kernel(const uint8_t* __restrict__ blocks, int8_t* __res
for (int i = 0; i < 32; ++i) o[i] = (int8_t) src[2 + i];
}
+// Reads a list of small tensors (norm weights, the convolutions' base kernels, ...) once, so they sit in the caches before the first kernel that needs them
+// (the verify window before the drafter streams gigabytes through them, and every dependent small kernel would otherwise wait for DRAM).
+struct Df2Touch { const float* p[48]; int n[48]; int count; };
+__global__ void touch_kernel(Df2Touch t, float* __restrict__ sink) {
+ float s = 0.0f;
+ for (int i = 0; i < t.count; ++i)
+ for (int j = blockIdx.x * blockDim.x + threadIdx.x; j < t.n[i]; j += gridDim.x * blockDim.x) s += t.p[i][j];
+ if (s == 1.2345e30f) sink[0] = s;
+}
+
// ---- top-16
constexpr int TK_CHUNK_MAX = 4096;
__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }
@@ -591,13 +696,41 @@ void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod
}
void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,
- int ring_slots, int window, float* part, float* out, void* stream) {
+ int ring_slots, int window, float* part, void* xq, float* xd, void* stream) {
attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 128, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part, g_attn_variant);
check("attn_part");
- attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, out);
+ attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, static_cast<int8_t*>(xq), xd);
check("attn_merge");
}
+void df2_conv_rms_quant(const float* x, const float* dyn, int dyn_stride, const float* base, float* h, const float* gamma, float* xn, int rows, int H, int group,
+ float eps, void* xq, float* xd, void* stream) {
+ if (H > 3 * 1024) throw std::invalid_argument("df2_conv_rms_quant: H too large");
+ conv_rms_quant_kernel<<<rows, 1024, 0, (cudaStream_t) stream>>>(x, dyn, dyn_stride, base, h, gamma, xn, H, group, eps, static_cast<int8_t*>(xq), xd);
+ check("conv_rms_quant");
+}
+
+void df2_norm_rope_qk(const float* q, const float* k, const float* gq, const float* gk, const float* table, float* qo, float* ko, int rows, int q_heads, int k_heads,
+ float eps, void* stream) {
+ norm_rope_qk_kernel<<<dim3(rows, q_heads + k_heads), HD, 0, (cudaStream_t) stream>>>(q, k, gq, gk, table, qo, ko, q_heads, k_heads, eps);
+ check("norm_rope_qk");
+}
+
+void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int32_t* pos, const float* table, float* kring, float* vring, int slots, int rows,
+ int heads, float eps, void* stream) {
+ kv_ring_kernel<<<dim3(rows, heads), HD, 0, (cudaStream_t) stream>>>(kin, vin, gk, pos, table, kring, vring, slots, heads, eps);
+ check("kv_ring");
+}
+
+void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream) {
+ Df2Touch t;
+ if (n > 48) n = 48;
+ for (int i = 0; i < n; ++i) { t.p[i] = ptrs[i]; t.n[i] = counts[i]; }
+ t.count = n;
+ touch_kernel<<<40, 256, 0, (cudaStream_t) stream>>>(t, sink);
+ check("touch");
+}
+
void df2_gemm_variant(int v) { g_gemm_variant = v; }
void df2_attention_variant(int v) { g_attn_variant = v; }
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index d7ec9a694..42e1c7799 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -16332,6 +16332,8 @@ int main(int argc, char** argv) {
auto draft_next = [&](int T_, int a_) -> bool {
#if defined(STRATA_HAVE_DFLASH2)
if (use_df2) {
+ static const bool wait_commit_first = std::getenv("STRATA_DF2_WAITCOMMIT") != nullptr; // measurement: no overlap with the async commit
+ if (wait_commit_first && !ver.wait_commit(err)) return false;
int32_t dd[strata::core::Dflash2Drafter::kDrafts];
float pp[strata::core::Dflash2Drafter::kDrafts];
if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;