Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DFlash2 drafter for Qwen3.8-Flash-Next: Q8_0 pack tool, GPU forward (conv, K/V ring, windowed attention, top-16 selector), permanent verify/prefill taps, --draft dflash2 --draft-model (opt-in, MTP stays the fallback)

Niko12217f97c35acaf3
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7c6867668..4ee0af934 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -336,6 +336,8 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cpp
src/core/layout.cpp)
target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+ # the DFlash2 drafter (src/core/dflash2.cpp, src/kernels/cuda/dflash2.cu) exists in the CUDA and HIP builds; the Metal and SYCL builds compile without it
+ target_compile_definitions(strata_core PUBLIC STRATA_HAVE_DFLASH2=1)
if(STRATA_ENABLE_CUDA AND STRATA_EXPERIMENTAL_SM60)
# the runtime device check (src/core/device.cu) admits the cards this build was made for (#236)
target_compile_definitions(strata_core PRIVATE STRATA_EXPERIMENTAL_SM60=1)
@@ -418,6 +420,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
src/kernels/cuda/dequant_bf16.cu
src/kernels/cuda/qsa_select.cu
src/kernels/cuda/verify_kernels.cu
+ src/kernels/cuda/dflash2.cu
src/kernels/cuda/iq_kernels.cu
src/kernels/ngram.cpp
src/ngram/ple_reader.cpp
@@ -579,7 +582,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
# about arithmetic, and this one knows the ORDER the arithmetic happens in.
add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
src/core/remote_expert_opt.cu
- src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
+ src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp src/core/dflash2.cpp
src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp
src/core/arch_defaults.cpp src/core/nv_defaults.cpp src/core/vmm.cpp src/core/conversation_file.cpp)
target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -1081,6 +1084,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(verify_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_test(NAME verify_parity COMMAND verify_parity --selftest)
endif()
+ # the DFlash2 drafter against the torch port on a synthetic case: needs the pack and a case file (tools/dflash2/mk_parity_case.py), so not a ctest
+ add_executable(dflash2_parity src/core/dflash2_parity.cpp)
+ target_include_directories(dflash2_parity PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(dflash2_parity PRIVATE strata_engine strata_kernels ${_strata_gpu_runtime_target})
add_executable(fused_gr_bench src/kernels/fused_gr_bench.cpp)
target_include_directories(fused_gr_bench PRIVATE ${_strata_gpu_include_directories})
diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hpp
new file mode 100644
index 000000000..75aadb075
--- /dev/null
+++ b/include/strata/core/dflash2.hpp
@@ -0,0 +1,129 @@
+// include/strata/core/dflash2.hpp - the DFlash2 drafter for Qwen3.8-Flash-Next on the GPU (CUDA and HIP).
+//
+// incoai/Qwen3.8-Flash-Next-DFlash2 (Apache-2.0) is a block drafter: 5 transformer layers of hidden 2560 that read the target's hidden
+// states (the mean of the 4 hyper-connection streams after layers 5, 15, 25, 35, 45) as context and, from the last accepted token plus
+// 7 mask tokens, produce 7 draft tokens in one forward pass. A candidate selector (the head's top-16 per row, a rank-256 bilinear
+// score between the previous pick and each candidate) picks the path left to right. tools/dflash2_pack.py packs it as Q8_0.
+//
+// context taps (5 x 2560 per accepted row) -> fc (12800 -> 2560) -> hidden_norm -> per layer K (norm + rope) and V -> ring slot pos % R
+// block 8 rows: embed(anchor), embed(mask) x 7; per layer
+// rms -> dynamic 2-tap conv (prepare) -> q, k, v -> q/k norm + rope -> attention over the ring + the block's own keys
+// -> o -> conv (finish) + residual -> rms -> conv (prepare) -> gate/up -> silu*up -> down -> conv (finish) + residual
+// selector final norm -> head over the draft vocabulary (rows 1..7) -> top-16 -> walk
+//
+// The ring holds only COMMITTED rows (an accepted window row's taps arrive after the verify window), and the block's keys stay outside
+// it, so a rejected draft leaves nothing to roll back. One CUDA graph per context-row count n = 1..8 holds the context update and the
+// block; the host stages the positions and the anchor and waits for the 7 drafts.
+#pragma once
+
+#include "strata/core/session.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace strata::core {
+
+class NativeHead;
+
+class Dflash2Drafter {
+public:
+ static constexpr int kRows = 8; ///< block rows: the anchor + 7 masks
+ static constexpr int kDrafts = kRows - 1;
+ static constexpr int kRing = 2064; ///< K/V slots per layer: the 2048 window + slack for rows written ahead of a rejection
+
+ Dflash2Drafter() = default;
+ ~Dflash2Drafter();
+ Dflash2Drafter(const Dflash2Drafter&) = delete;
+ Dflash2Drafter& operator=(const Dflash2Drafter&) = delete;
+
+ /// Reads the pack (tools/dflash2_pack.py) to the GPU `device` and allocates the ring and the buffers (about 0.8 GB).
+ bool load(const std::string& pack_path, int device, std::string& err);
+ /// The target's embedding and head (and the draft vocabulary file, "" or missing: the whole head).
+ bool bind(const WeightTable& wt, const NativeHead* head, const std::string& vocab_file, std::string& err);
+ /// Tests: an embedding callback and a head given as raw rows of `head_type` (a native_mmvq type).
+ using EmbedFn = std::function<void(const int32_t* tokens_dev, int n, float* out, cudaStream_t)>;
+ bool bind_raw(EmbedFn embed, const void* head_w, int head_type, int64_t head_rows, const std::vector<int32_t>& vocab, std::string& err);
+
+ /// A new request: the ring is empty again.
+ void reset() { have_ctx_ = 0; }
+ /// Context rows without a draft: `n` rows of taps (device, [n][5 * n_embd]) for positions pos0 ... pos0 + n - 1.
+ bool context(const float* taps_dev, int64_t n, int64_t pos0, std::string& err);
+ /// One round: the new context rows (n_new in 1..8: the committed window rows, taps device [n_new][5 * n_embd], positions pos0 ...),
+ /// then the block for `anchor` (the last committed token, at pos0 + n_new). `drafts` / `probs` get kDrafts entries (the
+ /// selector's pick and its probability). The taps pointer must be the same device buffer on every call (graphs are captured on it).
+ bool draft(const float* taps_dev, int n_new, int64_t pos0, int32_t anchor, int32_t* drafts, float* probs, std::string& err);
+
+ uint64_t vram_bytes() const { return vram_; }
+ int64_t n_draft_vocab() const { return n_dvocab_; }
+ int n_embd() const { return N_; }
+ const std::vector<int>& tap_layers() const { return taps_; }
+ cudaStream_t stream() const { return cs_; }
+ /// Diagnostics (STRATA_DF2_DUMP): the final normalized block rows of the last draft(), host copy.
+ bool debug_rows(std::vector<float>& rows, std::string& err);
+
+ double ms_draft = 0, ms_gpu = 0, ms_ctx = 0;
+ int64_t rounds = 0;
+
+private:
+ struct Tensor { std::string name, dtype; int64_t rows = 0, cols = 0; uint64_t off = 0, bytes = 0; };
+ const Tensor& find(const std::string& name) const;
+ const void* q8(const std::string& name) const { return blob_ + find(name).off; }
+ const float* f32(const std::string& name) const { return reinterpret_cast<const float*>(blob_ + find(name).off); }
+ void mm(const std::string& name, const void* xq, float* y, int ncols); ///< Q8_0 matrix x ncols activation columns (quantized in xq)
+ void quant(const float* x, int n_in, int ncols);
+ bool record_context(const float* taps, int n, cudaStream_t cs, std::string& err);
+ bool record_block(cudaStream_t cs, std::string& err);
+ bool capture(int n, const float* taps, std::string& err);
+ bool alloc(std::string& err);
+ struct Prof;
+ Prof* prof_ = nullptr;
+ void stamp(const char* name);
+ void prof_collect();
+ void prof_report() const;
+
+ int device_ = -1;
+ cudaStream_t cs_ = nullptr;
+ std::vector<Tensor> tensors_;
+ uint8_t* blob_ = nullptr;
+ uint64_t vram_ = 0;
+ int N_ = 2560, L_ = 5, NH_ = 32, NKV_ = 8, HD_ = 128, I_ = 7680, window_ = 2048, group_ = 16, rank_ = 256, mask_token_ = 0;
+ float theta_ = 10000.0f, eps_ = 1e-6f;
+ std::vector<int> taps_;
+ int64_t vocab_ = 0;
+
+ // target pieces
+ EmbedFn embed_;
+ const void* head_w_ = nullptr;
+ int head_type_ = -1;
+ int64_t n_dvocab_ = 0;
+ int32_t* dvocab_ = nullptr;
+ uint8_t* dhead_ = nullptr; ///< the draft vocabulary's head rows (owned), or null with head_w_ the whole head
+ bool bound_ = false;
+
+ // device buffers
+ int32_t* d_meta_ = nullptr; ///< [0..7] context positions, [8..15] block positions, [16] anchor token, [17] anchor position, [24..31] block tokens
+ int32_t* h_meta_ = nullptr; ///< host staging of d_meta_ (mapped; m_meta_ is its device alias)
+ int32_t* m_meta_ = nullptr;
+ int32_t* h_out_ = nullptr; ///< pinned: 7 draft tokens then 7 probabilities
+ float* hp_out_ = nullptr;
+ float *kring_[8] = {}, *vring_[8] = {};
+ float *rt_ctx_ = nullptr, *rt_blk_ = nullptr;
+ uint8_t* xq_ = nullptr;
+ float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;
+ float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;
+ float *ao_ = nullptr, *o_ = nullptr, *gate_ = nullptr, *up_ = nullptr, *mlp_ = nullptr, *part_ = nullptr, *fn_ = nullptr;
+ float *logits_ = nullptr, *topval_ = nullptr, *hsel_ = nullptr, *out_prob_ = nullptr;
+ int32_t *toptok_ = nullptr, *out_tok_ = nullptr;
+ uint32_t* topscratch_ = nullptr;
+ cudaGraphExec_t exec_[kRows + 1] = {};
+ const float* graph_taps_ = nullptr;
+ int64_t have_ctx_ = 0;
+ cudaEvent_t ev_a_ = nullptr, ev_b_ = nullptr;
+ bool timing_ = false;
+};
+
+} // namespace strata::core
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index b2ea81923..db19db664 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -323,6 +323,12 @@ public:
/// Token t's residual after the last layer, (hc, n_embd) on the device, valid until the next `run`.
const float* final_R(int t) const;
const float* final_R_all() const { return next_ ? next_->final_R_all() : R_; }
+ /// DFlash2 (before init): keep the completed outputs of these layers (zero-based; the mean of the hc streams) for every row of
+ /// each window in a device buffer, tap_dev() = [max_t][n_taps][n_embd] floats, valid until the next window. Single-stage
+ /// verifiers only; STRATA_TAP_FILE (research dump) turns the same buffer on without this call.
+ void set_taps(const std::vector<int>& layers) { tap_req_ = layers; }
+ const float* tap_dev() const { return tap_dev_; }
+ bool tapping() const { return tap_on_; }
/// The GPU plan the pool writes each layer (VRAM hits + the PCIe share of the misses); give it to the
/// dispatch (`ExpertDispatch::plan`) before the first `run`.
@@ -552,6 +558,7 @@ private:
// (the residual with its pending FFN write applied, the mean of the hc streams) per row; run() appends
// (pos, token, taps) records to the file.
bool tap_on_ = false;
+ std::vector<int> tap_req_; ///< set_taps
std::vector<int> tap_layers_; ///< the tapped layer ids (zero-based, completed outputs)
std::vector<int> tap_slot_; ///< layer -> slot in tap_layers_, or -1
float* tap_dev_ = nullptr; ///< [max_t][n_taps][n_embd]
diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hpp
new file mode 100644
index 000000000..8a544f298
--- /dev/null
+++ b/include/strata/kernels/dflash2.hpp
@@ -0,0 +1,66 @@
+// include/strata/kernels/dflash2.hpp - the small kernels of the DFlash2 drafter (core/dflash2.hpp) for Qwen3.8-Flash-Next.
+//
+// The drafter's matrices run through the engine's multi-column MMVQ (native_q8_0_mmvq); what is here is everything between
+// them: the grouped dynamic 2-tap convolution, the per-head RMS norm + rotary rotation, the windowed non-causal attention over a
+// K/V ring, the top-16 selection over the head's logits and the selector's left-to-right walk. All launches are asynchronous on
+// the given stream, allocate nothing and are graph-capturable. Layouts are row-major f32 unless noted.
+#pragma once
+
+#include <cstdint>
+
+namespace strata::kernels {
+
+constexpr int kDf2Rows = 8; ///< block rows (the anchor + 7 masks) and the most context rows one update takes
+constexpr int kDf2Kv = 8; ///< K/V heads
+constexpr int kDf2Gqa = 4; ///< query heads per K/V head
+constexpr int kDf2Hd = 128; ///< head dim
+constexpr int kDf2Chunk = 256; ///< keys per attention block
+constexpr int kDf2MaxChunks = 9; ///< ceil((2047 context keys + 8 block keys) / kDf2Chunk)
+constexpr int kDf2TopK = 16;
+constexpr int kDf2TopBlocks = 64; ///< partial top-k blocks per row (a block holds at most 4096 logits)
+
+/// out[b][c] = sum_{o < 2, b - o >= 0} (base[o][c] + dyn[b * dyn_stride + o * groups + c / group]) * x[(b - o) * H + c] (+ resid[b][c] when
+/// resid != null; out may equal resid). x is [rows][H]; base is [2][H]; `dyn` points at this convolution's [2][groups] slice of row 0.
+void df2_conv(const float* x, const float* dyn, int dyn_stride, const float* base, const float* resid, float* out, int rows, int H,
+ int group, void* stream);
+
+/// table[r][0][i], table[r][1][i] = cos, sin of pos[r] * theta^(-2 i / 128) for i < 64 (the angle reduced in double first, so a large
+/// position keeps its precision under fast math). table: [rows][2][64].
+void df2_rope_table(const int32_t* pos, int rows, float theta, float* table, void* stream);
+
+/// Per (row, head): out = rope(rmsnorm(in) * gamma). `in` rows are `in_stride` floats apart (heads * 128 contiguous per row). The
+/// rotation is the neox/rotate_half one over the whole head dim with row r's entry of `rope_table` (null: the norm only). The result of row r goes to
+/// out + slot(r) * heads * 128 with slot(r) = (slot_mod > 0 ? pos[r] % slot_mod : r). `rope` false: the norm only.
+void df2_norm_rope(const float* in, int in_stride, const float* gamma, const int32_t* pos, const float* rope_table, float* out, int slot_mod,
+ int rows, int heads, float eps, void* stream);
+
+/// out[slot(r)][c] = in[r][c], slot as above (the value half of the ring write).
+void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod, int rows, int cols, void* stream);
+
+/// Attention of the block's 8 query rows (q: [8][32][128], already normed and rotated) over the context ring (kring, vring:
+/// [ring_slots][8][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when
+/// s + i - j < window (j in [max(0, s - window + 1), s - 1], slot j % ring_slots) and every block key. `s` is read from *anchor_pos
+/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined into out [8][32*128].
+void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,
+ int ring_slots, int window, float* part, float* out, void* stream);
+
+/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id.
+/// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.
+void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);
+
+/// The candidate selector's walk: for row i = 0..rows-1 the 16 candidates' scores are unary[i][j] + dot(succ[tok[i][j]],
+/// pred[prev] * hp[i]) over `rank` values (codebooks are bf16 rows of `rank`), the pick is the argmax, prev = the pick. Writes
+/// out_tok[i] and out_prob[i] (the softmax probability of the pick among the 16). `prev0` is read from *anchor_tok on the device.
+void df2_select(const float* hp, const float* unary, const int32_t* tok, const uint16_t* pred_cb, const uint16_t* succ_cb, int rank, int rows,
+ const int32_t* anchor_tok, int32_t* out_tok, float* out_prob, void* stream);
+
+/// The verify window's tap: out[r][d] = mean over the hc streams c of (R[r][c][d] + bo[r][d] * 2 * sigmoid(inject[r][c] / hc)) - the layer output with
+/// its pending FFN write (gr_write) applied, averaged over the streams. R rows are `R_stride` floats apart, bo rows `bo_stride`, inject rows
+/// `inj_stride`, out rows `out_stride`.
+void df2_tap_gr(const float* R, int64_t R_stride, const float* bo, int64_t bo_stride, const float* inject, int64_t inj_stride, float* out, int64_t out_stride,
+ int rows, int64_t n_embd, int hc, void* stream);
+
+/// out[r][:n_embd] = the mean of the hc streams of row r of R ([rows][hc][n_embd]) written to out + r * out_stride (a tap slot).
+void df2_tap_rows(const float* R, float* out, int64_t out_stride, int64_t n_embd, int hc, int rows, void* stream);
+
+} // namespace strata::kernels
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hpp
index 7aadc0726..f9bbca027 100644
--- a/include/strata/prefill/prefill.hpp
+++ b/include/strata/prefill/prefill.hpp
@@ -184,6 +184,13 @@ public:
/// T x hc*n_embd, valid until the next chunk) and the chunk's first position; the MTP draft layer builds its
/// K/V from them. The prefill stream is synchronized before the call.
std::function<bool(const float* R_rows, int64_t T, int64_t pos0, std::string& err)> on_chunk;
+ /// DFlash2 prompt taps (CUDA / HIP, single stage): before `run`, ask for the completed output of each layer in `layers` (zero-based; the
+ /// mean of the hc streams) for the prompt rows at positions >= `first_pos`, at most `cap_rows` rows per chunk (a staging buffer of
+ /// cap_rows x layers x n_embd floats is allocated at the first chunk). After each chunk that has such rows, on_taps(taps
+ /// [rows][layers][n_embd] device, rows, first position, err) runs with the stream synchronized. clear_taps frees the staging.
+ void set_taps(const std::vector<int>& layers, int64_t first_pos, int64_t cap_rows) { tap_layers_ = layers; tap_first_ = first_pos; tap_cap_ = cap_rows; }
+ void clear_taps();
+ std::function<bool(const float* taps, int64_t rows, int64_t pos0, std::string& err)> on_taps;
/// Layer split: called by every stage when it has read a chunk, with the position reached, while its own state
/// is still at that chunk's end (its stream synchronized; the last stage calls it just before `on_chunk`). An
@@ -256,6 +263,9 @@ private:
bool drain_pipeline(std::string& err);
int64_t stage_lb_ = 0, stage_le_ = -1;
+ std::vector<int> tap_layers_;
+ int64_t tap_first_ = 0, tap_cap_ = 0;
+ float* tap_dev_ = nullptr;
Prefill* next_ = nullptr;
Prefill* helper_ = nullptr; ///< set_stage_helper
bool single_chunk_ = false; ///< a later stage: the prompt is one chunk (set by the stage before)
diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cpp
new file mode 100644
index 000000000..80687d3fa
--- /dev/null
+++ b/src/core/dflash2.cpp
@@ -0,0 +1,466 @@
+// src/core/dflash2.cpp - see include/strata/core/dflash2.hpp.
+#include "strata/core/dflash2.hpp"
+
+#include "strata/core/native_head.hpp"
+#include "strata/core/on_device.hpp"
+#include "strata/core/weights.hpp"
+#include "strata/kernels/dflash2.hpp"
+#include "strata/kernels/elementwise.hpp"
+#include "strata/kernels/native_mmvq.hpp"
+#include "strata/kernels/native_qsa.hpp"
+#include "strata/kernels/verify_kernels.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <atomic>
+#include <stdexcept>
+
+namespace strata::core {
+namespace {
+
+using Clock = std::chrono::steady_clock;
+double ms_since(Clock::time_point t) { return std::chrono::duration<double, std::milli>(Clock::now() - t).count(); }
+
+bool mapped(size_t bytes, void** h, void** d) {
+ if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;
+ std::memset(*h, 0, bytes);
+ return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;
+}
+
+constexpr int GGML_Q8_0 = 8;
+
+} // namespace
+
+Dflash2Drafter::~Dflash2Drafter() {
+ if (device_ < 0) return;
+ prof_report();
+ delete prof_;
+ const OnDevice on_device(device_);
+ if (cs_) cudaStreamSynchronize(cs_);
+ for (cudaGraphExec_t& e : exec_)
+ if (e) cudaGraphExecDestroy(e);
+ auto fr = [](void* p) { if (p) cudaFree(p); };
+ fr(blob_); fr(d_meta_); fr(xq_); fr(rt_ctx_); fr(rt_blk_);
+ for (int l = 0; l < 8; ++l) { fr(kring_[l]); fr(vring_[l]); }
+ fr(fcy_); fr(hctx_); fr(kc_); fr(vc_); fr(h_); fr(xn_); fr(xc_); fr(dyn_); fr(q_); fr(qn_); fr(k_); fr(v_); fr(kblk_); fr(ao_); fr(o_);
+ fr(gate_); fr(up_); fr(mlp_); fr(part_); fr(fn_); fr(logits_); fr(topval_); fr(hsel_); fr(toptok_); fr(topscratch_);
+ fr(dvocab_); fr(dhead_);
+ if (h_meta_) cudaFreeHost(h_meta_);
+ if (h_out_) cudaFreeHost(h_out_);
+ if (cs_) cudaStreamDestroy(cs_);
+ if (ev_a_) cudaEventDestroy(ev_a_);
+ if (ev_b_) cudaEventDestroy(ev_b_);
+}
+
+// STRATA_DF2_PROFILE=1 (measurement only): the round runs without its graph and an event follows every stage; the per-stage averages print at exit
+struct Dflash2Drafter::Prof {
+ std::vector<cudaEvent_t> ev;
+ std::vector<std::string> name;
+ std::vector<std::pair<std::string, double>> total;
+ int64_t rounds = 0;
+};
+
+void Dflash2Drafter::stamp(const char* name) {
+ if (!prof_) return;
+ cudaEvent_t e;
+ cudaEventCreate(&e);
+ cudaEventRecord(e, cs_);
+ prof_->ev.push_back(e);
+ prof_->name.push_back(name);
+}
+
+void Dflash2Drafter::prof_collect() {
+ if (!prof_ || prof_->ev.size() < 2) return;
+ cudaStreamSynchronize(cs_);
+ for (size_t i = 1; i < prof_->ev.size(); ++i) {
+ float ms = 0;
+ cudaEventElapsedTime(&ms, prof_->ev[i - 1], prof_->ev[i]);
+ auto it = std::find_if(prof_->total.begin(), prof_->total.end(), [&](const auto& t) { return t.first == prof_->name[i]; });
+ if (it == prof_->total.end()) prof_->total.emplace_back(prof_->name[i], ms); else it->second += ms;
+ }
+ for (cudaEvent_t e : prof_->ev) cudaEventDestroy(e);
+ prof_->ev.clear();
+ prof_->name.clear();
+ ++prof_->rounds;
+}
+
+void Dflash2Drafter::prof_report() const {
+ if (!prof_ || prof_->rounds == 0) return;
+ double sum = 0;
+ for (const auto& t : prof_->total) sum += t.second;
+ std::fprintf(stderr, "strata dflash2 profile (%lld rounds, ms per round, total %.3f):", (long long) prof_->rounds, sum / (double) prof_->rounds);
+ for (const auto& t : prof_->total) std::fprintf(stderr, " %s %.3f", t.first.c_str(), t.second / (double) prof_->rounds);
+ std::fputc(10, stderr);
+}
+
+const Dflash2Drafter::Tensor& Dflash2Drafter::find(const std::string& name) const {
+ for (const Tensor& t : tensors_)
+ if (t.name == name) return t;
+ throw std::runtime_error("dflash2: tensor " + name + " is missing from the pack");
+}
+
+bool Dflash2Drafter::load(const std::string& path, int device, std::string& err) {
+ device_ = device;
+ const OnDevice on_device(device_);
+ std::FILE* f = std::fopen(path.c_str(), "rb");
+ if (f == nullptr) { err = "dflash2: cannot open " + path; return false; }
+ struct Closer { std::FILE* f; ~Closer() { std::fclose(f); } } closer{f};
+ uint32_t hdr[3];
+ if (std::fread(hdr, 4, 3, f) != 3 || std::memcmp(hdr, "DF2P", 4) != 0 || hdr[1] != 1) { err = "dflash2: " + path + " is not a DF2P v1 pack"; return false; }
+ std::string index(hdr[2], ' ');
+ if (std::fread(index.data(), 1, hdr[2], f) != hdr[2]) { err = "dflash2: the pack's index is cut short"; return false; }
+ const long data_off = 12 + (long) hdr[2];
+ uint64_t total = 0;
+ for (size_t i = 0; i < index.size();) {
+ const size_t j = index.find('\n', i);
+ if (j == std::string::npos) break;
+ const std::string line = index.substr(i, j - i);
+ i = j + 1;
+ char a[256], b[64], c[256];
+ long long v1, v2, v3, v4;
+ if (std::sscanf(line.c_str(), "cfg %63s %255s", b, c) == 2) {
+ const std::string key = b;
+ if (key == "hidden") N_ = std::atoi(c);
+ else if (key == "layers") L_ = std::atoi(c);
+ else if (key == "heads") NH_ = std::atoi(c);
+ else if (key == "kv_heads") NKV_ = std::atoi(c);
+ else if (key == "head_dim") HD_ = std::atoi(c);
+ else if (key == "inter") I_ = std::atoi(c);
+ else if (key == "window") window_ = std::atoi(c);
+ else if (key == "rope_theta") theta_ = (float) std::atof(c);
+ else if (key == "eps") eps_ = (float) std::atof(c);
+ else if (key == "conv_group") group_ = std::atoi(c);
+ else if (key == "rank") rank_ = std::atoi(c);
+ else if (key == "mask_token") mask_token_ = std::atoi(c);
+ else if (key == "vocab") vocab_ = std::atoll(c);
+ else if (key == "taps") {
+ taps_.clear();
+ for (const char* q = c; *q;) { taps_.push_back(std::atoi(q)); while (*q && *q != ',') ++q; if (*q == ',') ++q; }
+ }
+ } else if (std::sscanf(line.c_str(), "t %255s %63s %lld %lld %lld %lld", a, b, &v1, &v2, &v3, &v4) == 6) {
+ Tensor t;
+ t.name = a; t.dtype = b; t.rows = v1; t.cols = v2; t.off = (uint64_t) v3; t.bytes = (uint64_t) v4;
+ tensors_.push_back(t);
+ total = std::max(total, t.off + t.bytes);
+ }
+ }
+ if (tensors_.empty() || NH_ != 32 || NKV_ != 8 || HD_ != 128 || rank_ != 256 || N_ != 2560 || L_ > 8 || window_ + 16 > kRing) {
+ err = "dflash2: the pack's shape is not the Flash-Next drafter this engine was written for";
+ return false;
+ }
+ if (cudaMalloc((void**) &blob_, total) != cudaSuccess) { err = "dflash2: the pack does not fit in VRAM (" + std::to_string(total >> 20) + " MiB)"; return false; }
+ vram_ += total;
+ std::vector<uint8_t> stage(64u << 20);
+#if defined(_WIN32)
+ _fseeki64(f, data_off, SEEK_SET);
+#else
+ fseeko(f, data_off, SEEK_SET);
+#endif
+ for (uint64_t o = 0; o < total;) {
+ const size_t n = (size_t) std::min<uint64_t>(stage.size(), total - o);
+ if (std::fread(stage.data(), 1, n, f) != n) { err = "dflash2: the pack is shorter than its index"; return false; }
+ if (cudaMemcpy(blob_ + o, stage.data(), n, cudaMemcpyHostToDevice) != cudaSuccess) { err = "dflash2: weight upload failed"; return false; }
+ o += n;
+ }
+ if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "dflash2: stream"; return false; }
+ if (const char* e = std::getenv("STRATA_DF2_PROFILE"); e && e[0] == '1') prof_ = new Prof();
+ if (const char* e = std::getenv("STRATA_DF2_TIMING"); e && e[0] == '1') {
+ timing_ = true;
+ cudaEventCreate(&ev_a_);
+ cudaEventCreate(&ev_b_);
+ }
+ return alloc(err);
+}
+
+bool Dflash2Drafter::alloc(std::string& err) {
+ auto dev = [&](auto** p, size_t n) {
+ using T = std::remove_pointer_t<std::remove_reference_t<decltype(*p)>>;
+ if (cudaMalloc((void**) p, n * sizeof(T)) != cudaSuccess) return false;
+ vram_ += n * sizeof(T);
+ return true;
+ };
+ const size_t B = kRows;
+ bool ok = dev(&d_meta_, 32) && dev(&xq_, kernels::native_q8_1_bytes(std::max(N_ * 5, I_), kRows) + 4096) && dev(&rt_ctx_, B * 128) && dev(&rt_blk_, B * 128) &&
+ dev(&fcy_, B * N_) && dev(&hctx_, B * N_) && dev(&kc_, B * NKV_ * HD_) && dev(&vc_, B * NKV_ * HD_) && dev(&h_, B * N_) && dev(&xn_, B * N_) &&
+ dev(&xc_, B * N_) && dev(&dyn_, B * 640) && dev(&q_, B * NH_ * HD_) && dev(&qn_, B * NH_ * HD_) && dev(&k_, B * NKV_ * HD_) && dev(&v_, B * NKV_ * HD_) &&
+ dev(&kblk_, B * NKV_ * HD_) && dev(&ao_, B * NH_ * HD_) && dev(&o_, B * N_) && dev(&gate_, B * I_) && dev(&up_, B * I_) && dev(&mlp_, B * N_) &&
+ dev(&part_, (size_t) kernels::kDf2MaxChunks * 8 * 32 * 130) && dev(&fn_, B * N_) && dev(&topval_, kDrafts * 16) && dev(&toptok_, kDrafts * 16) &&
+ dev(&hsel_, kDrafts * rank_) && dev(&topscratch_, (size_t) kDrafts * kernels::kDf2TopBlocks * kernels::kDf2TopK * 2);
+ for (int l = 0; ok && l < L_; ++l) ok = dev(&kring_[l], (size_t) kRing * NKV_ * HD_) && dev(&vring_[l], (size_t) kRing * NKV_ * HD_);
+ if (!ok) { err = "dflash2: the drafter's buffers do not fit in VRAM"; return false; }
+ void* hd = nullptr;
+ void* dd = nullptr;
+ if (!mapped(32 * 4, (void**) &hd, &dd)) { err = "dflash2: mapped staging"; return false; }
+ h_meta_ = (int32_t*) hd;
+ m_meta_ = (int32_t*) dd;
+ if (!mapped(64 * 4, &hd, &dd)) { err = "dflash2: mapped output"; return false; }
+ h_out_ = (int32_t*) hd;
+ out_tok_ = (int32_t*) dd; // the select kernel writes the 7 picks here (host-visible) ...
+ out_prob_ = (float*) dd + 16; // ... and their probabilities at +16 ints
+ cudaMemset(rt_ctx_, 0, B * 128 * 4);
+ for (int l = 0; l < L_; ++l) { cudaMemset(kring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); cudaMemset(vring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); }
+ return cudaDeviceSynchronize() == cudaSuccess;
+}
+
+bool Dflash2Drafter::bind_raw(EmbedFn embed, const void* head_w, int head_type, int64_t head_rows, const std::vector<int32_t>& vocab, std::string& err) {
+ const OnDevice on_device(device_);
+ embed_ = std::move(embed);
+ head_w_ = head_w;
+ head_type_ = head_type;
+ n_dvocab_ = head_rows;
+ if (!vocab.empty()) {
+ n_dvocab_ = (int64_t) vocab.size();
+ if (cudaMalloc((void**) &dvocab_, vocab.size() * 4) != cudaSuccess) { err = "dflash2: draft vocabulary"; return false; }
+ vram_ += vocab.size() * 4;
+ cudaMemcpy(dvocab_, vocab.data(), vocab.size() * 4, cudaMemcpyHostToDevice);
+ }
+ if ((n_dvocab_ + kernels::kDf2TopBlocks - 1) / kernels::kDf2TopBlocks > 4096) { err = "dflash2: the head has too many rows for the top-k"; return false; }
+ if (!logits_ && cudaMalloc((void**) &logits_, (size_t) kDrafts * (size_t) n_dvocab_ * 4) != cudaSuccess) { err = "dflash2: the draft logits do not fit"; return false; }
+ vram_ += (uint64_t) kDrafts * (uint64_t) n_dvocab_ * 4;
+ bound_ = true;
+ return true;
+}
+
+bool Dflash2Drafter::bind(const WeightTable& wt, const NativeHead* head, const std::string& vocab_file, std::string& err) {
+ const OnDevice on_device(device_);
+ if (head == nullptr || !head->loaded()) { err = "dflash2: the drafter needs the native head (--native)"; return false; }
+ const WeightRef* we = wt.find("token_embd.weight");
+ if (we == nullptr) { err = "dflash2: token_embd.weight is missing"; return false; }
+ EmbedFn embed = [we](const int32_t* tok, int n, float* out, cudaStream_t cs) {
+ if (const NativeEmbed* ne = native_embed()) {
+ ne->gather_dev(tok, n, out, cs);
+ } else {
+ const auto* codes = (const uint8_t*) we->data;
+ const auto* scales = (const float*) (codes + we->codes_bytes);
+ const auto* offsets = we->has_offset ? (const float*) (codes + we->codes_bytes + we->scales_bytes) : nullptr;
+ kernels::embedding_gather_dev(codes, scales, offsets, tok, n, we->ne0, we->code_bits, we->code_bias, we->group_elems,
+ (uint64_t) (we->ne0 / (8 / we->code_bits)), (uint64_t) (we->ne0 / we->group_elems), out, cs);
+ }
+ };
+ std::vector<int32_t> vocab;
+ if (!vocab_file.empty()) {
+ if (std::FILE* vf = std::fopen(vocab_file.c_str(), "rb")) {
+ std::fseek(vf, 0, SEEK_END);
+ const long size = std::ftell(vf);
+ std::fseek(vf, 0, SEEK_SET);
+ if (size >= 4 && size % 4 == 0) {
+ vocab.resize((size_t) size / 4);
+ if (std::fread(vocab.data(), 4, vocab.size(), vf) != vocab.size()) vocab.clear();
+ }
+ std::fclose(vf);
+ }
+ }
+ const void* hw = head->weights();
+ const int64_t row_bytes = (int64_t) head->row_bytes();
+ int64_t head_rows = row_bytes > 0 ? (int64_t) (head->weight_bytes() / (uint64_t) row_bytes) : 0;
+ if (!vocab.empty()) {
+ int32_t* dv = nullptr;
+ if (cudaMalloc((void**) &dv, vocab.size() * 4) != cudaSuccess || cudaMalloc((void**) &dhead_, (size_t) ((int64_t) vocab.size() * row_bytes)) != cudaSuccess) {
+ if (dv) cudaFree(dv);
+ err = "dflash2: the draft vocabulary's head rows do not fit";
+ return false;
+ }
+ cudaMemcpy(dv, vocab.data(), vocab.size() * 4, cudaMemcpyHostToDevice);
+ kernels::gather_rows((const uint8_t*) hw, row_bytes, dv, (int64_t) vocab.size(), dhead_, nullptr);
+ cudaDeviceSynchronize();
+ cudaFree(dv);
+ vram_ += (uint64_t) ((int64_t) vocab.size() * row_bytes);
+ hw = dhead_;
+ head_rows = (int64_t) vocab.size();
+ }
+ if (!bind_raw(embed, hw, head->type(), head_rows, vocab, err)) return false;
+ std::fprintf(stderr, "strata dflash2: %lld-token draft head (%.1f MiB), drafter %.0f MiB of VRAM\n", (long long) n_dvocab_,
+ (double) (n_dvocab_ * row_bytes) / 1048576.0, (double) vram_ / 1048576.0);
+ return true;
+}
+
+void Dflash2Drafter::quant(const float* x, int n_in, int ncols) { kernels::native_quantize_q8_1(x, xq_, n_in, ncols, cs_); }
+
+void Dflash2Drafter::mm(const std::string& name, const void* xq, float* y, int ncols) {
+ const Tensor& t = find(name);
+ kernels::native_q8_0_mmvq(blob_ + t.off, xq, y, (int) t.cols, (int) t.rows, ncols, cs_);
+}
+
+bool Dflash2Drafter::record_context(const float* taps, int n, cudaStream_t cs, std::string& err) {
+ using namespace kernels;
+ try {
+ const int tap_n = (int) taps_.size() * N_;
+ df2_rope_table(d_meta_ + 0, n, theta_, rt_ctx_, cs);
+ stamp("ctx-start");
+ quant(taps, tap_n, n);
+ mm("fc.weight", xq_, fcy_, n);
+ stamp("fc");
+ native_qsa_rms_norm_weighted(fcy_, f32("hidden_norm.weight"), hctx_, N_, n, eps_, cs);
+ quant(hctx_, N_, n);
+ for (int l = 0; l < L_; ++l) {
+ const std::string p = "layers." + std::to_string(l) + ".";
+ mm(p + "self_attn.k_proj.weight", xq_, kc_, n);
+ mm(p + "self_attn.v_proj.weight", xq_, vc_, n);
+ df2_norm_rope(kc_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], kRing, n, NKV_, eps_, cs);
+ df2_copy_rows(vc_, vring_[l], d_meta_ + 0, kRing, n, NKV_ * HD_, cs);
+ }
+ stamp("ctx-kv");
+ } catch (const std::exception& e) {
+ err = e.what();
+ return false;
+ }
+ return true;
+}
+
+bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {
+ using namespace kernels;
+ try {
+ const int B = kRows;
+ df2_rope_table(d_meta_ + 8, B, theta_, rt_blk_, cs);
+ embed_(d_meta_ + 24, B, h_, cs);
+ stamp("embed");
+ for (int l = 0; l < L_; ++l) {
+ const std::string p = "layers." + std::to_string(l) + ".";
+ const float* base_a = f32(p + "attention_conv.base_kernel");
+ const float* base_m = f32(p + "mlp_conv.base_kernel");
+ native_qsa_rms_norm_weighted(h_, f32(p + "input_layernorm.weight"), xn_, N_, B, eps_, cs);
+ quant(xn_, N_, B);
+ mm(p + "attention_conv.kernel_projection.weight", xq_, dyn_, B);
+ df2_conv(xn_, dyn_, 640, base_a, nullptr, xc_, B, N_, group_, cs);
+ quant(xc_, N_, B);
+ stamp("attn-front");
+ mm(p + "self_attn.q_proj.weight", xq_, q_, B);
+ mm(p + "self_attn.k_proj.weight", xq_, k_, B);
+ mm(p + "self_attn.v_proj.weight", xq_, v_, B);
+ stamp("qkv");
+ df2_norm_rope(q_, NH_ * HD_, f32(p + "self_attn.q_norm.weight"), d_meta_ + 8, rt_blk_, qn_, 0, B, NH_, eps_, cs);
+ df2_norm_rope(k_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 8, rt_blk_, kblk_, 0, B, NKV_, eps_, cs);
+ df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);
+ stamp("attention");
+ quant(ao_, NH_ * HD_, B);
+ mm(p + "self_attn.o_proj.weight", xq_, o_, B);
+ stamp("o_proj");
+ df2_conv(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, h_, B, N_, group_, cs);
+ native_qsa_rms_norm_weighted(h_, f32(p + "post_attention_layernorm.weight"), xn_, N_, B, eps_, cs);
+ quant(xn_, N_, B);
+ mm(p + "mlp_conv.kernel_projection.weight", xq_, dyn_, B);
+ df2_conv(xn_, dyn_, 640, base_m, nullptr, xc_, B, N_, group_, cs);
+ quant(xc_, N_, B);
+ stamp("mlp-front");
+ mm(p + "mlp.gate_proj.weight", xq_, gate_, B);
+ mm(p + "mlp.up_proj.weight", xq_, up_, B);
+ stamp("gate-up");
+ native_swiglu_quantize_q8_1(gate_, up_, xq_, I_, B, cs);
+ mm(p + "mlp.down_proj.weight", xq_, mlp_, B);
+ stamp("down");
+ df2_conv(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, h_, B, N_, group_, cs);
+ }
+ native_qsa_rms_norm_weighted(h_, f32("norm.weight"), fn_, N_, B, eps_, cs);
+ quant(fn_ + N_, N_, kDrafts); // rows 1..7
+ stamp("final-norm");
+ native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);
+ stamp("head");
+ df2_topk(logits_, (int) n_dvocab_, dvocab_, kDrafts, topscratch_, topval_, toptok_, cs);
+ stamp("topk");
+ mm("candidate_selector.hidden_projection.weight", xq_, hsel_, kDrafts);
+ df2_select(hsel_, topval_, toptok_, reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.predecessor_codebook").off),
+ reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.successor_codebook").off), rank_, kDrafts, d_meta_ + 16, out_tok_,
+ out_prob_, cs);
+ stamp("selector");
+ } catch (const std::exception& e) {
+ err = e.what();
+ return false;
+ }
+ return true;
+}
+
+bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {
+ if (exec_[n]) return true;
+ if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "dflash2: begin capture"; return false; }
+ kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);
+ bool ok = record_context(taps, n, cs_, err);
+ if (ok) ok = record_block(cs_, err);
+ cudaGraph_t graph = nullptr;
+ const cudaError_t ce = cudaStreamEndCapture(cs_, &graph);
+ if (!ok) { if (graph) cudaGraphDestroy(graph); return false; }
+ if (ce != cudaSuccess || cudaGraphInstantiate(&exec_[n], graph, 0) != cudaSuccess) {
+ if (graph) cudaGraphDestroy(graph);
+ err = std::string("dflash2: capture: ") + cudaGetErrorString(ce);
+ return false;
+ }
+ cudaGraphDestroy(graph);
+ cudaGraphUpload(exec_[n], cs_);
+ cudaStreamSynchronize(cs_);
+ return true;
+}
+
+bool Dflash2Drafter::context(const float* taps, int64_t n, int64_t pos0, std::string& err) {
+ const OnDevice on_device(device_);
+ const int tap_n = (int) taps_.size() * N_;
+ for (int64_t r0 = 0; r0 < n; r0 += kRows) {
+ const int m = (int) std::min<int64_t>(kRows, n - r0);
+ for (int i = 0; i < m; ++i) h_meta_[i] = (int32_t) (pos0 + r0 + i);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);
+ if (!record_context(taps + (size_t) r0 * (size_t) tap_n, m, cs_, err)) return false;
+ if (cudaStreamSynchronize(cs_) != cudaSuccess) { err = std::string("dflash2: ") + cudaGetErrorString(cudaGetLastError()); return false; }
+ }
+ have_ctx_ = std::max<int64_t>(have_ctx_, pos0 + n);
+ return true;
+}
+
+bool Dflash2Drafter::draft(const float* taps, int n_new, int64_t pos0, int32_t anchor, int32_t* drafts, float* probs, std::string& err) {
+ if (!bound_) { err = "dflash2: not bound"; return false; }
+ if (n_new < 1 || n_new > kRows) { err = "dflash2: draft rows out of range"; return false; }
+ const OnDevice on_device(device_);
+ const Clock::time_point t0 = Clock::now();
+ if (graph_taps_ != nullptr && graph_taps_ != taps) { err = "dflash2: the taps buffer moved after capture"; return false; }
+ if (!prof_ && !exec_[n_new]) {
+ // the staging must hold valid positions while the capture runs the (empty) kernels' parameter checks
+ for (int i = 0; i < 8; ++i) h_meta_[i] = (int32_t) (pos0 + i);
+ if (!capture(n_new, taps, err)) return false;
+ graph_taps_ = taps;
+ }
+ const int64_t s = pos0 + n_new;
+ for (int i = 0; i < 8; ++i) { h_meta_[i] = (int32_t) (pos0 + std::min(i, n_new - 1)); h_meta_[8 + i] = (int32_t) (s + i); }
+ h_meta_[16] = anchor;
+ h_meta_[17] = (int32_t) s;
+ h_meta_[24] = anchor;
+ for (int i = 1; i < 8; ++i) h_meta_[24 + i] = mask_token_;
+ for (int i = 0; i < kDrafts; ++i) { ((volatile int32_t*) h_out_)[i] = -1; }
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ if (timing_) cudaEventRecord(ev_a_, cs_);
+ if (prof_) { // the same round without its graph, an event after every stage
+ cudaStreamSynchronize(cs_);
+ for (cudaEvent_t e : prof_->ev) cudaEventDestroy(e); // the stamps of context() calls are not a round
+ prof_->ev.clear();
+ prof_->name.clear();
+ kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);
+ stamp("start");
+ if (!record_context(taps, n_new, cs_, err) || !record_block(cs_, err)) return false;
+ prof_collect();
+ } else
+ if (cudaGraphLaunch(exec_[n_new], cs_) != cudaSuccess) { err = std::string("dflash2 draft: ") + cudaGetErrorString(cudaGetLastError()); return false; }
+ if (timing_) cudaEventRecord(ev_b_, cs_);
+ (void) cudaStreamQuery(cs_);
+ if (cudaStreamSynchronize(cs_) != cudaSuccess) { err = std::string("dflash2 draft: ") + cudaGetErrorString(cudaGetLastError()); return false; }
+ for (int i = 0; i < kDrafts; ++i) {
+ drafts[i] = ((volatile int32_t*) h_out_)[i];
+ if (probs) probs[i] = ((volatile float*) h_out_)[16 + i];
+ }
+ if (timing_) {
+ float ms = 0;
+ if (cudaEventElapsedTime(&ms, ev_a_, ev_b_) == cudaSuccess) ms_gpu += ms;
+ }
+ have_ctx_ = std::max<int64_t>(have_ctx_, s);
+ ms_draft += ms_since(t0);
+ ++rounds;
+ return true;
+}
+
+bool Dflash2Drafter::debug_rows(std::vector<float>& rows, std::string& err) {
+ rows.resize((size_t) kRows * (size_t) N_);
+ if (cudaMemcpy(rows.data(), fn_, rows.size() * 4, cudaMemcpyDeviceToHost) != cudaSuccess) { err = "dflash2: debug copy"; return false; }
+ return true;
+}
+
+} // namespace strata::core
diff --git a/src/core/dflash2_parity.cpp b/src/core/dflash2_parity.cpp
new file mode 100644
index 000000000..bf97887e2
--- /dev/null
+++ b/src/core/dflash2_parity.cpp
@@ -0,0 +1,95 @@
+// src/core/dflash2_parity.cpp - the engine's DFlash2 drafter against the torch port, on a synthetic case (GPU, no model).
+//
+// dflash2_parity <pack> <case.bin> [rounds]
+//
+// case.bin comes from tools/dflash2/mk_parity_case.py: n_ctx random tap rows, an anchor, a random Q8_0 head over a draft-vocabulary subset, the
+// target embedding rows of the anchor and the mask token, and the torch answer (tokens, probabilities, the 8 final normalized rows). The engine's
+// drafter reads the same pack (Q8_0 weights, Q8_1 activations), so the rows agree to quantization noise, not bitwise: the test checks the rows'
+// cosine similarity, the walk's tokens and the time of a round.
+#include "strata/core/dflash2.hpp"
+
+#include <cuda_runtime.h>
+
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+using strata::core::Dflash2Drafter;
+
+int main(int argc, char** argv) {
+ if (argc < 3) { std::fprintf(stderr, "usage: dflash2_parity <pack> <case.bin> [rounds]\n"); return 2; }
+ const int rounds = argc > 3 ? std::atoi(argv[3]) : 20;
+ std::FILE* f = std::fopen(argv[2], "rb");
+ if (!f) { std::fprintf(stderr, "cannot open %s\n", argv[2]); return 2; }
+ int32_t hdr[4];
+ if (std::fread(hdr, 4, 4, f) != 4) return 2;
+ const int n_ctx = hdr[0], anchor = hdr[1], vp = hdr[2];
+ const int H = 2560, tapn = 5 * H;
+ std::vector<float> taps((size_t) n_ctx * tapn), emb(2 * H), fn_ref(8 * H);
+ std::vector<int32_t> dvocab(vp), tok_ref(7);
+ std::vector<float> prob_ref(7);
+ const size_t row_bytes = (size_t) (H / 32) * 34;
+ std::vector<uint8_t> head((size_t) vp * row_bytes);
+ bool ok = std::fread(taps.data(), 4, taps.size(), f) == taps.size() && std::fread(emb.data(), 4, emb.size(), f) == emb.size() &&
+ std::fread(dvocab.data(), 4, dvocab.size(), f) == dvocab.size() && std::fread(head.data(), 1, head.size(), f) == head.size() &&
+ std::fread(tok_ref.data(), 4, 7, f) == 7 && std::fread(prob_ref.data(), 4, 7, f) == 7 && std::fread(fn_ref.data(), 4, fn_ref.size(), f) == fn_ref.size();
+ std::fclose(f);
+ if (!ok) { std::fprintf(stderr, "case file is cut short\n"); return 2; }
+
+ std::string err;
+ Dflash2Drafter d;
+ if (!d.load(argv[1], 0, err)) { std::fprintf(stderr, "load: %s\n", err.c_str()); return 1; }
+ float *d_taps = nullptr, *d_emb = nullptr;
+ uint8_t* d_head = nullptr;
+ cudaMalloc((void**) &d_taps, taps.size() * 4);
+ cudaMalloc((void**) &d_emb, emb.size() * 4);
+ cudaMalloc((void**) &d_head, head.size());
+ cudaMemcpy(d_taps, taps.data(), taps.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(d_emb, emb.data(), emb.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(d_head, head.data(), head.size(), cudaMemcpyHostToDevice);
+ auto embed = [&](const int32_t*, int n, float* out, cudaStream_t cs) {
+ cudaMemcpyAsync(out, d_emb, (size_t) H * 4, cudaMemcpyDeviceToDevice, cs);
+ for (int i = 1; i < n; ++i) cudaMemcpyAsync(out + (size_t) i * H, d_emb + H, (size_t) H * 4, cudaMemcpyDeviceToDevice, cs);
+ };
+ if (!d.bind_raw(embed, d_head, 8, vp, dvocab, err)) { std::fprintf(stderr, "bind: %s\n", err.c_str()); return 1; }
+ std::printf("drafter VRAM %.0f MiB\n", (double) d.vram_bytes() / 1048576.0);
+
+ // the context but the last row, then one draft round that adds the last row
+ if (!d.context(d_taps, n_ctx - 1, 0, err)) { std::fprintf(stderr, "context: %s\n", err.c_str()); return 1; }
+ int32_t drafts[7];
+ float probs[7];
+ const float* last = d_taps + (size_t) (n_ctx - 1) * tapn;
+ if (!d.draft(last, 1, n_ctx - 1, anchor, drafts, probs, err)) { std::fprintf(stderr, "draft: %s\n", err.c_str()); return 1; }
+ std::vector<float> rows;
+ d.debug_rows(rows, err);
+ int bad = 0;
+ double min_cos = 1.0;
+ for (int r = 0; r < 8; ++r) {
+ double dot = 0, na = 0, nb = 0;
+ for (int c = 0; c < H; ++c) {
+ const double a = rows[(size_t) r * H + c], b = fn_ref[(size_t) r * H + c];
+ dot += a * b; na += a * a; nb += b * b;
+ }
+ const double cs = dot / std::sqrt(na * nb + 1e-30);
+ min_cos = std::min(min_cos, cs);
+ std::printf("row %d: cos %.5f norm ratio %.4f\n", r, cs, std::sqrt(na / (nb + 1e-30)));
+ }
+ int same = 0;
+ for (int i = 0; i < 7; ++i) {
+ same += drafts[i] == tok_ref[i];
+ std::printf("draft %d: engine %d (p %.3f) torch %d (p %.3f)\n", i, drafts[i], probs[i], tok_ref[i], prob_ref[i]);
+ }
+ if (min_cos < 0.98) { std::printf("FAIL: a final row's cosine similarity %.4f is below 0.98\n", min_cos); ++bad; }
+ // timing: the same round again (the graph replays; the ring rewrites the same slot)
+ cudaDeviceSynchronize();
+ const auto t0 = std::chrono::steady_clock::now();
+ for (int i = 0; i < rounds; ++i)
+ if (!d.draft(last, 1, n_ctx - 1, anchor, drafts, probs, err)) { std::fprintf(stderr, "draft: %s\n", err.c_str()); return 1; }
+ const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count() / rounds;
+ std::printf("tokens equal %d of 7; %.3f ms per round (1 context row + the block), GPU %.3f ms\n", same, ms, d.rounds > 0 ? d.ms_gpu / (double) d.rounds : 0.0);
+ std::printf(bad ? "dflash2_parity: FAIL\n" : "dflash2_parity: ok\n");
+ return bad ? 1 : 0;
+}
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index c9f48a747..5b898f9da 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -56,6 +56,7 @@
#include "strata/core/progress.hpp"
#include "strata/kernels/shared_expert.hpp"
#include "strata/kernels/verify_kernels.hpp"
+#include "strata/kernels/dflash2.hpp"
#include <algorithm>
#include <atomic>
@@ -779,35 +780,42 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
mapped(64, (void**) &h_plan_err_, (void**) &m_plan_err_) &&
mapped(T * K * N * 4, (void**) &h_ymiss_, (void**) &m_ymiss_);
if (!ok) { err = "verify: mapped staging allocation failed " + strata::core::pin_exhausted_hint(); return false; }
- // STRATA_TAP_FILE: the DFlash2 experiment's hook (see verify.hpp); off unless the variable is set
- if (const char* tap_path = std::getenv("STRATA_TAP_FILE"); tap_path != nullptr && tap_path[0] != '\0' && lb_ == 0 &&
- le_ == g.n_layers) {
+ // DFlash2 taps: set_taps (the drafter) or STRATA_TAP_FILE (the research dump); off, and nothing is allocated or captured, otherwise
+ const char* tap_path = std::getenv("STRATA_TAP_FILE");
+ if (tap_path != nullptr && tap_path[0] == '\0') tap_path = nullptr;
+ if ((tap_path != nullptr || !tap_req_.empty()) && lb_ == 0 && le_ == g.n_layers) {
tap_layers_.clear();
- const char* tl = std::getenv("STRATA_TAP_LAYERS");
- std::string list = tl != nullptr && tl[0] != '\0' ? tl : "5,15,25,35,45";
- for (size_t i = 0; i < list.size();) {
- const size_t j = list.find(',', i);
- const int l = std::atoi(list.substr(i, j == std::string::npos ? std::string::npos : j - i).c_str());
- if (l >= 0 && l < g.n_layers - 1) tap_layers_.push_back(l);
- if (j == std::string::npos) break;
- i = j + 1;
+ if (tap_path == nullptr) {
+ for (int l : tap_req_) if (l >= 0 && l < g.n_layers - 1) tap_layers_.push_back(l);
+ } else {
+ const char* tl = std::getenv("STRATA_TAP_LAYERS");
+ std::string list = tl != nullptr && tl[0] != '\0' ? tl : "5,15,25,35,45";
+ for (size_t i = 0; i < list.size();) {
+ const size_t j = list.find(',', i);
+ const int l = std::atoi(list.substr(i, j == std::string::npos ? std::string::npos : j - i).c_str());
+ if (l >= 0 && l < g.n_layers - 1) tap_layers_.push_back(l);
+ if (j == std::string::npos) break;
+ i = j + 1;
+ }
}
tap_slot_.assign((size_t) g.n_layers, -1);
for (size_t i = 0; i < tap_layers_.size(); ++i) tap_slot_[(size_t) tap_layers_[i]] = (int) i;
const size_t nt = tap_layers_.size();
if (nt == 0 || cudaMalloc((void**) &tap_dev_, (size_t) T * nt * N * 4) != cudaSuccess ||
- cudaMalloc((void**) &tap_scratch_, (size_t) HC * N * 4) != cudaSuccess ||
- !mapped((size_t) T * nt * N * 4, (void**) &h_tap_, (void**) &m_tap_)) {
- err = "verify: the STRATA_TAP_FILE buffers could not be allocated";
+ (tap_path != nullptr && (cudaMalloc((void**) &tap_scratch_, (size_t) HC * N * 4) != cudaSuccess ||
+ !mapped((size_t) T * nt * N * 4, (void**) &h_tap_, (void**) &m_tap_)))) {
+ err = "verify: the DFlash2 tap buffers could not be allocated";
return false;
}
- tap_file_ = std::fopen(tap_path, "wb");
- if (tap_file_ == nullptr) { err = std::string("verify: cannot write STRATA_TAP_FILE ") + tap_path; return false; }
- const int32_t hdr[4] = {0x50415453 /* 'STAP' */, 1, (int32_t) N, (int32_t) nt};
- std::fwrite(hdr, sizeof hdr, 1, tap_file_);
- for (int l : tap_layers_) { const int32_t v = l; std::fwrite(&v, sizeof v, 1, tap_file_); }
+ if (tap_path != nullptr) {
+ tap_file_ = std::fopen(tap_path, "wb");
+ if (tap_file_ == nullptr) { err = std::string("verify: cannot write STRATA_TAP_FILE ") + tap_path; return false; }
+ const int32_t hdr[4] = {0x50415453 /* 'STAP' */, 1, (int32_t) N, (int32_t) nt};
+ std::fwrite(hdr, sizeof hdr, 1, tap_file_);
+ for (int l : tap_layers_) { const int32_t v = l; std::fwrite(&v, sizeof v, 1, tap_file_); }
+ std::fprintf(stderr, "strata: STRATA_TAP_FILE %s: %zu taps of %lld floats per row\n", tap_path, nt, (long long) N);
+ }
tap_on_ = true;
- std::fprintf(stderr, "strata: STRATA_TAP_FILE %s: %zu taps of %lld floats per row\n", tap_path, nt, (long long) N);
}
if (bal_ != nullptr) { // set_pcie_balance: four stamp words per (layer, group)
if (!mapped((size_t) g.n_layers * 2 * 4 * 8, (void**) &h_bal_, (void**) &m_bal_)) {
@@ -2179,13 +2187,11 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
device_plan_ ? skip_ + grp : nullptr, ring, cs);
stamp(l, 24, grp);
if (tap_on_ && !batch_rec_ && l < (int64_t) tap_slot_.size() && tap_slot_[(size_t) l] >= 0) {
- // the layer's completed output: R with this FFN half's write applied (the next layer's read would fold it)
+ // the layer's completed output: R with this FFN half's write applied (the next layer's read would fold it), the mean of
+ // the hc streams, all rows of the group in one launch
const int64_t nt = (int64_t) tap_layers_.size();
- for (int t = tb; t < te; ++t) {
- cudaMemcpyAsync(tap_scratch_, Rt(t), (size_t) HC * N * 4, cudaMemcpyDeviceToDevice, cs);
- gr_write(tap_scratch_, bo_ + t * N, inj2_ + t * HC, gs, tap_scratch_, cs);
- hc_mean_streams(tap_scratch_, tap_dev_ + ((size_t) t * nt + (size_t) tap_slot_[(size_t) l]) * N, N, (int) HC, cs);
- }
+ strata::kernels::df2_tap_gr(Rt(tb), HC * N, bo_ + tb * N, N, inj2_ + tb * HC, HC, tap_dev_ + ((size_t) tb * nt + (size_t) tap_slot_[(size_t) l]) * N,
+ nt * N, n, N, (int) HC, cs);
}
if (l == g.n_layers - 1) {
if (!fuse_head_gr) {
@@ -2314,7 +2320,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
if (nan_guard_on()) strata::kernels::logits_nonfinite_rows(head_logits_, T, (int) n_vocab_, m_nf_, cs);
}
stamp(g.n_layers, 1, 0);
- if (tap_on_ && !batch_rec_) copy_from_mapped(m_tap_, tap_dev_, (int64_t) T * (int64_t) tap_layers_.size() * N, cs);
+ if (tap_on_ && !batch_rec_ && tap_file_ != nullptr) copy_from_mapped(m_tap_, tap_dev_, (int64_t) T * (int64_t) tap_layers_.size() * N, cs);
return true;
}
@@ -3096,7 +3102,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
}
}
for (int t = 0; t < T; ++t) out[t] = ((volatile int32_t*) h_out_)[t];
- if (tap_on_ && !batch_rec_) tap_write(T, tokens, pos0);
+ if (tap_file_ != nullptr && !batch_rec_) tap_write(T, tokens, pos0);
note_nonfinite(T, nullptr);
if (static const bool dbg = std::getenv("STRATA_DBG_NAN") != nullptr; dbg) { // debug: the first non-finite head
static bool reported = false;
diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cu
new file mode 100644
index 000000000..82c40ad63
--- /dev/null
+++ b/src/kernels/cuda/dflash2.cu
@@ -0,0 +1,412 @@
+// src/kernels/cuda/dflash2.cu - the small kernels of the DFlash2 drafter. See include/strata/kernels/dflash2.hpp.
+#include "strata/kernels/dflash2.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cfloat>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <stdexcept>
+#include <string>
+
+namespace strata::kernels {
+namespace {
+
+constexpr int HD = kDf2Hd;
+
+void check(const char* what) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "dflash2 kernel %s: %s\n", what, cudaGetErrorString(e));
+ throw std::runtime_error(std::string("dflash2 kernel ") + what + ": " + cudaGetErrorString(e));
+ }
+}
+
+__device__ __forceinline__ float bf16f(uint16_t h) { return __uint_as_float((uint32_t) h << 16); }
+
+// ---- the grouped dynamic 2-tap convolution over the block's rows
+__global__ void conv_kernel(const float* __restrict__ x, const float* __restrict__ dyn, int dyn_stride, const float* __restrict__ base,
+ const float* __restrict__ resid, float* __restrict__ out, int rows, int H, int group) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= rows * H) return;
+ const int b = i / H, c = i % H;
+ const int groups = H / group;
+ float acc = 0.0f;
+#pragma unroll
+ for (int o = 0; o < 2; ++o) {
+ if (b - o < 0) continue;
+ const float k = base[o * H + c] + dyn[(size_t) b * dyn_stride + o * groups + c / group];
+ acc = fmaf(k, x[(size_t) (b - o) * H + c], acc);
+ }
+ out[i] = resid ? resid[i] + acc : acc;
+}
+
+__global__ void rope_table_kernel(const int32_t* __restrict__ pos, int rows, float theta, float* __restrict__ table) {
+ const int r = blockIdx.x, i = threadIdx.x; // 64 threads
+ if (r >= rows) return;
+ const double inv = pow((double) theta, -(double) (2 * i) / (double) HD);
+ double ang = (double) pos[r] * inv;
+ const double two_pi = 6.283185307179586476925286766559;
+ ang -= two_pi * rint(ang / two_pi); // [-pi, pi]
+ float s, c;
+ sincosf((float) ang, &s, &c);
+ table[(r * 2 + 0) * 64 + i] = c;
+ table[(r * 2 + 1) * 64 + i] = s;
+}
+
+// one block per (row, head), 128 threads: out = rope(rmsnorm(in) * gamma)
+__global__ void norm_rope_kernel(const float* __restrict__ in, int in_stride, const float* __restrict__ gamma, const int32_t* __restrict__ pos,
+ const float* __restrict__ table, float* __restrict__ out, int slot_mod, int heads, float eps) {
+ const int r = blockIdx.x, h = blockIdx.y, d = threadIdx.x;
+ __shared__ float red[4];
+ __shared__ float y[HD];
+ const float x = in[(size_t) r * in_stride + h * HD + d];
+ float ss = x * x;
+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);
+ if ((d & 31) == 0) red[d >> 5] = ss;
+ __syncthreads();
+ const float tot = red[0] + red[1] + red[2] + red[3];
+ const float v = x * rsqrtf(tot / (float) HD + eps) * gamma[d];
+ float res = v;
+ if (table != nullptr) {
+ y[d] = v;
+ __syncthreads();
+ const int i = d & 63;
+ const float c = table[(r * 2 + 0) * 64 + i], s = table[(r * 2 + 1) * 64 + i];
+ res = d < 64 ? v * c - y[d + 64] * s : v * c + y[d - 64] * s;
+ }
+ const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;
+ out[((size_t) slot * heads + h) * HD + d] = res;
+}
+
+__global__ void copy_rows_kernel(const float* __restrict__ in, float* __restrict__ out, const int32_t* __restrict__ pos, int slot_mod, int rows, int cols) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= rows * cols) return;
+ const int r = i / cols, c = i % cols;
+ const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;
+ out[(size_t) slot * cols + c] = in[i];
+}
+
+// ---- attention. grid (8 K/V heads, kDf2MaxChunks), 256 threads. 32 queries (4 heads x 8 rows) per block share each key tile.
+constexpr int KS = 129; // smem row stride of the K / V / Q tiles (conflict-free)
+__global__ void __launch_bounds__(256) attn_part_kernel(const float* __restrict__ q, const float* __restrict__ kring, const float* __restrict__ vring,
+ const float* __restrict__ kblk, const float* __restrict__ vblk,
+ const int32_t* __restrict__ anchor_pos, int ring_slots, int window, float* __restrict__ part) {
+ __shared__ float Qs[32 * KS];
+ __shared__ float Ks[32 * KS]; // the K tile, then the V tile
+ __shared__ float Ss[32 * 33];
+ const int kvh = blockIdx.x, chunk = blockIdx.y, tid = threadIdx.x;
+ const int s = *anchor_pos;
+ const int lo = s - (window - 1) > 0 ? s - (window - 1) : 0;
+ const int nctx = s - lo;
+ const int nkeys = nctx + kDf2Rows;
+ const int qi = tid >> 3, tg = tid & 7;
+ const int qrow = qi >> 2, qg = qi & 3;
+ float* const pout = part + (((size_t) chunk * kDf2Kv + kvh) * 32 + qi) * 130;
+ const int k_begin = chunk * kDf2Chunk;
+ if (k_begin >= nkeys) {
+ if (tg == 0) { pout[0] = -INFINITY; pout[1] = 0.0f; }
+ for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = 0.0f;
+ return;
+ }
+ const float scale = rsqrtf((float) HD);
+ for (int i = tid; i < 32 * HD; i += 256) {
+ const int a = i / HD, d = i % HD;
+ const int row = a >> 2, g = a & 3;
+ Qs[a * KS + d] = q[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] * scale;
+ }
+ float m = -INFINITY, l = 0.0f;
+ float O[16];
+#pragma unroll
+ for (int e = 0; e < 16; ++e) O[e] = 0.0f;
+ const int k_end = min(k_begin + kDf2Chunk, nkeys);
+ for (int k0 = k_begin; k0 < k_end; k0 += 32) {
+ __syncthreads(); // Ks free (previous tile's V reads done), Qs written (first tile)
+ for (int i = tid; i < 32 * HD; i += 256) {
+ const int t = i / HD, d = i % HD, k = k0 + t;
+ float v = 0.0f;
+ if (k < nctx) v = kring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];
+ else if (k < nkeys) v = kblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];
+ Ks[t * KS + d] = v;
+ }
+ __syncthreads();
+ float sc[4] = {0.f, 0.f, 0.f, 0.f};
+#pragma unroll 8
+ for (int d = 0; d < HD; ++d) {
+ const float qv = Qs[qi * KS + d];
+#pragma unroll
+ for (int u = 0; u < 4; ++u) sc[u] = fmaf(qv, Ks[(tg * 4 + u) * KS + d], sc[u]);
+ }
+ float tmax = -INFINITY;
+#pragma unroll
+ for (int u = 0; u < 4; ++u) {
+ const int k = k0 + tg * 4 + u;
+ bool ok = k < nkeys;
+ if (ok && k < nctx) ok = (lo + k) > (s + qrow - window); // s + row - j < window
+ sc[u] = ok ? sc[u] : -INFINITY;
+ tmax = fmaxf(tmax, sc[u]);
+ }
+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 1));
+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 2));
+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 4));
+ const float mn = fmaxf(m, tmax);
+ const float corr = (m == -INFINITY) ? 0.0f : __expf(m - mn);
+ float psum = 0.0f;
+#pragma unroll
+ for (int u = 0; u < 4; ++u) {
+ const float p = sc[u] == -INFINITY ? 0.0f : __expf(sc[u] - mn);
+ Ss[qi * 33 + tg * 4 + u] = p;
+ psum += p;
+ }
+ psum += __shfl_xor_sync(0xffffffffu, psum, 1);
+ psum += __shfl_xor_sync(0xffffffffu, psum, 2);
+ psum += __shfl_xor_sync(0xffffffffu, psum, 4);
+ l = l * corr + psum;
+ m = mn;
+#pragma unroll
+ for (int e = 0; e < 16; ++e) O[e] *= corr;
+ __syncthreads(); // Ss complete; every read of Ks done
+ for (int i = tid; i < 32 * HD; i += 256) {
+ const int t = i / HD, d = i % HD, k = k0 + t;
+ float v = 0.0f;
+ if (k < nctx) v = vring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];
+ else if (k < nkeys) v = vblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];
+ Ks[t * KS + d] = v;
+ }
+ __syncthreads();
+ for (int t = 0; t < 32; ++t) {
+ const float p = Ss[qi * 33 + t];
+#pragma unroll
+ for (int e = 0; e < 16; ++e) O[e] = fmaf(p, Ks[t * KS + e * 8 + tg], O[e]);
+ }
+ }
+ if (tg == 0) { pout[0] = m; pout[1] = l; }
+#pragma unroll
+ for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = O[e];
+ (void) qg;
+}
+
+__global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrict__ out) {
+ const int a = blockIdx.x; // kvh * 32 + qi
+ const int kvh = a >> 5, qi = a & 31, d = threadIdx.x;
+ float M = -INFINITY;
+ for (int c = 0; c < kDf2MaxChunks; ++c) M = fmaxf(M, part[(((size_t) c * kDf2Kv + kvh) * 32 + qi) * 130]);
+ float L = 0.0f, o = 0.0f;
+ for (int c = 0; c < kDf2MaxChunks; ++c) {
+ const float* p = part + (((size_t) c * kDf2Kv + kvh) * 32 + qi) * 130;
+ const float mc = p[0];
+ const float w = (mc == -INFINITY) ? 0.0f : __expf(mc - M);
+ L += p[1] * w;
+ o = fmaf(p[2 + d], w, o);
+ }
+ const int row = qi >> 2, g = qi & 3;
+ out[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] = L > 0.0f ? o / L : 0.0f;
+}
+
+// ---- top-16
+constexpr int TK_CHUNK_MAX = 4096;
+__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }
+
+__global__ void topk_part_kernel(const float* __restrict__ logits, int n, float* __restrict__ cval, int32_t* __restrict__ cidx) {
+ __shared__ float vals[TK_CHUNK_MAX];
+ __shared__ float wv[8];
+ __shared__ int wi[8];
+ __shared__ int win;
+ const int row = blockIdx.x, blk = blockIdx.y, tid = threadIdx.x;
+ const int cs = (n + kDf2TopBlocks - 1) / kDf2TopBlocks;
+ const int c0 = blk * cs, c1 = min(c0 + cs, n);
+ for (int i = tid; i < cs; i += 256) vals[i] = (c0 + i < c1) ? logits[(size_t) row * n + c0 + i] : -INFINITY;
+ __syncthreads();
+ for (int k = 0; k < kDf2TopK; ++k) {
+ float bv = -INFINITY;
+ int bi = 0x7fffffff;
+ for (int i = tid; i < cs; i += 256)
+ if (better(vals[i], i, bv, bi)) { bv = vals[i]; bi = i; }
+ for (int o = 16; o > 0; o >>= 1) {
+ const float ov = __shfl_xor_sync(0xffffffffu, bv, o);
+ const int oi = __shfl_xor_sync(0xffffffffu, bi, o);
+ if (better(ov, oi, bv, bi)) { bv = ov; bi = oi; }
+ }
+ if ((tid & 31) == 0) { wv[tid >> 5] = bv; wi[tid >> 5] = bi; }
+ __syncthreads();
+ if (tid == 0) {
+ float v = wv[0];
+ int ix = wi[0];
+ for (int w = 1; w < 8; ++w)
+ if (better(wv[w], wi[w], v, ix)) { v = wv[w]; ix = wi[w]; }
+ const size_t o = ((size_t) row * kDf2TopBlocks + blk) * kDf2TopK + k;
+ cval[o] = v;
+ cidx[o] = v == -INFINITY ? -1 : c0 + ix;
+ win = ix;
+ if (v != -INFINITY) vals[ix] = -INFINITY;
+ }
+ __syncthreads();
+ }
+}
+
+__global__ void topk_merge_kernel(const float* __restrict__ cval, const int32_t* __restrict__ cidx, const int32_t* __restrict__ idx_map,
+ float* __restrict__ val, int32_t* __restrict__ tok) {
+ constexpr int NC = kDf2TopBlocks * kDf2TopK;
+ __shared__ float sv[NC];
+ __shared__ int si[NC];
+ __shared__ float wv[8];
+ __shared__ int wi[8], wp[8];
+ const int row = blockIdx.x, tid = threadIdx.x;
+ for (int i = tid; i < NC; i += 256) { sv[i] = cval[(size_t) row * NC + i]; si[i] = cidx[(size_t) row * NC + i]; }
+ __syncthreads();
+ for (int k = 0; k < kDf2TopK; ++k) {
+ float bv = -INFINITY;
+ int bi = 0x7fffffff, bp = -1;
+ for (int i = tid; i < NC; i += 256)
+ if (si[i] >= 0 && better(sv[i], si[i], bv, bi)) { bv = sv[i]; bi = si[i]; bp = i; }
+ for (int o = 16; o > 0; o >>= 1) {
+ const float ov = __shfl_xor_sync(0xffffffffu, bv, o);
+ const int oi = __shfl_xor_sync(0xffffffffu, bi, o);
+ const int op = __shfl_xor_sync(0xffffffffu, bp, o);
+ if (better(ov, oi, bv, bi)) { bv = ov; bi = oi; bp = op; }
+ }
+ if ((tid & 31) == 0) { wv[tid >> 5] = bv; wi[tid >> 5] = bi; wp[tid >> 5] = bp; }
+ __syncthreads();
+ if (tid == 0) {
+ float v = wv[0];
+ int ix = wi[0], p = wp[0];
+ for (int w = 1; w < 8; ++w)
+ if (better(wv[w], wi[w], v, ix)) { v = wv[w]; ix = wi[w]; p = wp[w]; }
+ if (p >= 0) { si[p] = -1; }
+ val[row * kDf2TopK + k] = v;
+ tok[row * kDf2TopK + k] = p >= 0 ? (idx_map ? idx_map[ix] : ix) : 0;
+ }
+ __syncthreads();
+ }
+}
+
+// ---- the selector's walk: one block of 256 threads
+__global__ void select_kernel(const float* __restrict__ hp, const float* __restrict__ unary, const int32_t* __restrict__ tok,
+ const uint16_t* __restrict__ pred_cb, const uint16_t* __restrict__ succ_cb, int rank, int rows,
+ const int32_t* __restrict__ anchor_tok, int32_t* __restrict__ out_tok, float* __restrict__ out_prob) {
+ __shared__ float vp[256];
+ __shared__ float sc[kDf2TopK];
+ __shared__ int prev;
+ const int tid = threadIdx.x, w = tid >> 5, lane = tid & 31;
+ if (tid == 0) prev = *anchor_tok;
+ __syncthreads();
+ for (int i = 0; i < rows; ++i) {
+ vp[tid] = bf16f(pred_cb[(size_t) prev * rank + tid]) * hp[(size_t) i * rank + tid];
+ __syncthreads();
+ for (int j = w * 2; j < w * 2 + 2; ++j) {
+ const int t = tok[i * kDf2TopK + j];
+ const uint4 raw = *reinterpret_cast<const uint4*>(succ_cb + (size_t) t * rank + lane * 8);
+ const uint32_t u[4] = {raw.x, raw.y, raw.z, raw.w};
+ float acc = 0.0f;
+#pragma unroll
+ for (int e = 0; e < 4; ++e) {
+ acc = fmaf(__uint_as_float(u[e] << 16), vp[lane * 8 + e * 2], acc);
+ acc = fmaf(__uint_as_float(u[e] & 0xffff0000u), vp[lane * 8 + e * 2 + 1], acc);
+ }
+ for (int o = 16; o > 0; o >>= 1) acc += __shfl_xor_sync(0xffffffffu, acc, o);
+ if (lane == 0) sc[j] = unary[i * kDf2TopK + j] + acc;
+ }
+ __syncthreads();
+ if (tid == 0) {
+ int bj = 0;
+ float bv = sc[0];
+ for (int j = 1; j < kDf2TopK; ++j)
+ if (sc[j] > bv) { bv = sc[j]; bj = j; }
+ float z = 0.0f;
+ for (int j = 0; j < kDf2TopK; ++j) z += __expf(sc[j] - bv);
+ out_tok[i] = tok[i * kDf2TopK + bj];
+ out_prob[i] = 1.0f / z;
+ prev = out_tok[i];
+ }
+ __syncthreads();
+ }
+}
+
+__global__ void tap_rows_kernel(const float* __restrict__ R, float* __restrict__ out, int64_t out_stride, int64_t n, int hc) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ const int r = blockIdx.y;
+ if (i >= n) return;
+ float s = 0.0f;
+ for (int c = 0; c < hc; ++c) s += R[((size_t) r * hc + c) * n + i];
+ out[(size_t) r * out_stride + i] = s / (float) hc;
+}
+
+__global__ void tap_gr_kernel(const float* __restrict__ R, int64_t R_stride, const float* __restrict__ bo, int64_t bo_stride,
+ const float* __restrict__ inject, int64_t inj_stride, float* __restrict__ out, int64_t out_stride, int64_t n, int hc) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ const int r = blockIdx.y;
+ if (i >= n) return;
+ float wsum = 0.0f, rs = 0.0f;
+ for (int c = 0; c < hc; ++c) {
+ const float x = inject[(size_t) r * inj_stride + c] / (float) hc;
+ wsum += 2.0f / (1.0f + expf(-x));
+ rs += R[(size_t) r * R_stride + (size_t) c * n + i];
+ }
+ out[(size_t) r * out_stride + i] = (rs + bo[(size_t) r * bo_stride + i] * wsum) / (float) hc;
+}
+
+} // namespace
+
+void df2_tap_gr(const float* R, int64_t R_stride, const float* bo, int64_t bo_stride, const float* inject, int64_t inj_stride, float* out, int64_t out_stride,
+ int rows, int64_t n_embd, int hc, void* stream) {
+ tap_gr_kernel<<<dim3((unsigned) ((n_embd + 255) / 256), rows), 256, 0, (cudaStream_t) stream>>>(R, R_stride, bo, bo_stride, inject, inj_stride, out,
+ out_stride, n_embd, hc);
+ check("tap_gr");
+}
+
+void df2_conv(const float* x, const float* dyn, int dyn_stride, const float* base, const float* resid, float* out, int rows, int H, int group,
+ void* stream) {
+ const int n = rows * H;
+ conv_kernel<<<(n + 255) / 256, 256, 0, (cudaStream_t) stream>>>(x, dyn, dyn_stride, base, resid, out, rows, H, group);
+ check("conv");
+}
+
+void df2_rope_table(const int32_t* pos, int rows, float theta, float* table, void* stream) {
+ rope_table_kernel<<<rows, 64, 0, (cudaStream_t) stream>>>(pos, rows, theta, table);
+ check("rope_table");
+}
+
+void df2_norm_rope(const float* in, int in_stride, const float* gamma, const int32_t* pos, const float* rope_table, float* out, int slot_mod,
+ int rows, int heads, float eps, void* stream) {
+ norm_rope_kernel<<<dim3(rows, heads), HD, 0, (cudaStream_t) stream>>>(in, in_stride, gamma, pos, rope_table, out, slot_mod, heads, eps);
+ check("norm_rope");
+}
+
+void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod, int rows, int cols, void* stream) {
+ const int n = rows * cols;
+ copy_rows_kernel<<<(n + 255) / 256, 256, 0, (cudaStream_t) stream>>>(in, out, pos, slot_mod, rows, cols);
+ check("copy_rows");
+}
+
+void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,
+ int ring_slots, int window, float* part, float* out, void* stream) {
+ attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 256, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part);
+ check("attn_part");
+ attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, out);
+ check("attn_merge");
+}
+
+void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream) {
+ if ((n + kDf2TopBlocks - 1) / kDf2TopBlocks > TK_CHUNK_MAX) throw std::invalid_argument("df2_topk: too many logits per row");
+ float* cval = reinterpret_cast<float*>(scratch);
+ int32_t* cidx = reinterpret_cast<int32_t*>(scratch + (size_t) rows * kDf2TopBlocks * kDf2TopK);
+ topk_part_kernel<<<dim3(rows, kDf2TopBlocks), 256, 0, (cudaStream_t) stream>>>(logits, n, cval, cidx);
+ check("topk_part");
+ topk_merge_kernel<<<rows, 256, 0, (cudaStream_t) stream>>>(cval, cidx, idx_map, val, tok);
+ check("topk_merge");
+}
+
+void df2_select(const float* hp, const float* unary, const int32_t* tok, const uint16_t* pred_cb, const uint16_t* succ_cb, int rank, int rows,
+ const int32_t* anchor_tok, int32_t* out_tok, float* out_prob, void* stream) {
+ if (rank != 256) throw std::invalid_argument("df2_select: the selector rank must be 256");
+ select_kernel<<<1, 256, 0, (cudaStream_t) stream>>>(hp, unary, tok, pred_cb, succ_cb, rank, rows, anchor_tok, out_tok, out_prob);
+ check("select");
+}
+
+void df2_tap_rows(const float* R, float* out, int64_t out_stride, int64_t n_embd, int hc, int rows, void* stream) {
+ tap_rows_kernel<<<dim3((unsigned) ((n_embd + 255) / 256), rows), 256, 0, (cudaStream_t) stream>>>(R, out, out_stride, n_embd, hc);
+ check("tap_rows");
+}
+
+} // namespace strata::kernels
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index c9b453540..6cb888e50 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -1,6 +1,9 @@
// src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.
#include "strata/kernels/fused_gr.hpp"
#include "strata/prefill/prefill.hpp"
+#if defined(STRATA_HAVE_DFLASH2)
+#include "strata/kernels/dflash2.hpp"
+#endif
#include "strata/prefill/warm_policy.hpp"
#include <atomic>
#if defined(__cpp_lib_atomic_wait)
@@ -1051,7 +1054,19 @@ strata::kernels::QsaAttnPools pools_of(const strata::kernels::KvHostPools& h, co
} // namespace
Prefill::Prefill() : impl_(new Impl) {}
-Prefill::~Prefill() { release(); }
+Prefill::~Prefill() {
+ clear_taps();
+ release();
+}
+
+void Prefill::clear_taps() {
+ if (tap_dev_ != nullptr) {
+ cudaFree(tap_dev_);
+ tap_dev_ = nullptr;
+ }
+ tap_layers_.clear();
+ on_taps = nullptr;
+}
void Prefill::reset() {
release();
@@ -2680,6 +2695,20 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
int ple_buf = 0;
if (hand_in_ == nullptr) stopped_at_ = -1;
+#if defined(STRATA_HAVE_DFLASH2)
+ std::vector<int> tap_slot;
+ if (!tap_layers_.empty() && stage_lb_ == 0 && (stage_le_ < 0 || stage_le_ == g.n_layers)) {
+ if (tap_dev_ == nullptr &&
+ cudaMalloc((void**) &tap_dev_, (size_t) tap_cap_ * tap_layers_.size() * (size_t) g.n_embd * 4) != cudaSuccess) {
+ (void) cudaGetLastError();
+ tap_dev_ = nullptr;
+ err = "prefill: the DFlash2 tap staging does not fit in VRAM";
+ return false;
+ }
+ tap_slot.assign((size_t) g.n_layers, -1);
+ for (size_t i = 0; i < tap_layers_.size(); ++i) tap_slot[(size_t) tap_layers_[i]] = (int) i;
+ }
+#endif
for (int64_t c0 = 0; c0 < n; c0 += C) {
if (should_stop && should_stop()) { err = "cancelled"; return false; }
// the first stage: end the run before this chunk (every chunk before it is handed on; the caller's drain
@@ -2691,6 +2720,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (std::getenv("STRATA_TRACE")) { std::fprintf(stderr, "strata trace: prompt chunk %lld of %lld\n", (long long) c0, (long long) n); std::fflush(stderr); }
const int64_t T = std::min(C, n - c0), p0 = pos0 + c0;
core::progress_at("reading the prompt (batched): preparing the chunk from token", p0); // #251
+#if defined(STRATA_HAVE_DFLASH2)
+ // DFlash2 prompt taps: the rows of this chunk at positions >= tap_first_ (the drafter looks back one window)
+ const int64_t tap_t0 = tap_dev_ != nullptr ? std::min<int64_t>(T, std::max<int64_t>(0, tap_first_ - p0)) : T;
+ const int64_t tap_rows = T - tap_t0;
+ if (tap_rows > tap_cap_) { err = "prefill: the DFlash2 tap staging is smaller than the chunk's tapped rows"; return false; }
+#else
+ const int64_t tap_t0 = 0, tap_rows = 0;
+#endif
++stats_.chunks;
pt.mark(kPfStart, cs);
const auto tsetup = Clock::now();
@@ -4836,6 +4873,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
}
if (steered && !cv_fused) // --control-vector-scaled
strata::kernels::cvec_apply(m.R, l, T, D, nullptr, 0, nullptr, 0, false, m.cs);
+#if defined(STRATA_HAVE_DFLASH2)
+ if (half == 1 && tap_rows > 0 && tap_slot[(size_t) l] >= 0) // DFlash2: this layer's completed output, mean of the streams
+ strata::kernels::df2_tap_rows(m.R + (size_t) tap_t0 * D, tap_dev_ + (size_t) tap_slot[(size_t) l] * N, (int64_t) tap_layers_.size() * N, N,
+ (int) HC, (int) tap_rows, m.cs);
+#endif
}
}
if (!ple_land()) return false; // a stage that ends before layer 1: the rows land anyway, the next gather starts
@@ -4933,6 +4975,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
std::fclose(f);
}
}
+#if defined(STRATA_HAVE_DFLASH2)
+ if (tap_rows > 0 && on_taps) {
+ if (cudaStreamSynchronize(m.cs) != cudaSuccess) { err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError()); return false; }
+ if (!on_taps(tap_dev_, tap_rows, p0 + tap_t0, err)) return false;
+ }
+#endif
if (on_chunk || on_stage_chunk) {
const auto toc = Clock::now();
if (cudaStreamSynchronize(m.cs) != cudaSuccess) {
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index d7116d2df..4f75c9fe1 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -66,6 +66,9 @@
#include "strata/core/native_head.hpp"
#include "strata/core/verify.hpp"
#include "strata/core/mtp.hpp"
+#if defined(STRATA_HAVE_DFLASH2)
+#include "strata/core/dflash2.hpp"
+#endif
#include "strata/prefill/prefill.hpp"
#if defined(STRATA_PREFILL_MMQ)
#include "strata/prefill/moe_mmq.hpp"
@@ -781,6 +784,9 @@ struct Options {
std::string mtp_q4; ///< "" (off), all, proj (the projections only) or head (the draft head only)
/// --mtp-draft-vocab FILE (opt-in): the draft head's token subset from FILE instead of <mtp>/draft_vocab.bin
std::string mtp_draft_vocab;
+ /// --draft dflash2 --draft-model PACK (opt-in, CUDA / HIP): the DFlash2 block drafter for Qwen3.8-Flash-Next (tools/dflash2_pack.py) drafts instead
+ /// of the MTP layer, which stays loaded as the fallback. Needs --mtp, --spec and a native pack; a missing pack falls back to the MTP.
+ std::string draft, draft_model;
/// A control vector on the residual stream (strata/kernels/cvec.hpp), with llama.cpp's flags: the
/// `experimental-speed-projection` profile passes `--control-vector-scaled FILE:1.0 --control-vector-layer-range
/// 4 44 --cvec-mode project --cvec-dir per-layer`. None by default; --serve switches a loaded one per request.
@@ -933,6 +939,8 @@ void usage() {
" (default) or one per stream (llama.cpp's MTP graph)\n"
" --mtp-q4 all|proj|head opt-in: the draft layer's projections and/or draft head as 4-bit copies\n"
" --mtp-draft-vocab FILE opt-in: the draft head's token subset (default <mtp>/draft_vocab.bin)\n"
+ " --draft mtp|dflash2 the draft source (default mtp); dflash2 needs --draft-model PACK (tools/dflash2_pack.py)\n"
+ " and keeps the MTP layer loaded as the fallback (opt-in, Flash-Next, CUDA / HIP)\n"
" --control-vector-scaled FILE:SCALE[,...] a control vector GGUF on the residual stream (llama.cpp's\n"
" format; --control-vector FILE = scale 1). --serve: requests switch it (cvec=0|1)\n"
" --control-vector-layer-range A B the layers it follows (inclusive; default 1 .. the last)\n"
@@ -2672,6 +2680,14 @@ int main(int argc, char** argv) {
}
}
else if (a == "--mtp-draft-vocab") o.mtp_draft_vocab = next("--mtp-draft-vocab");
+ else if (a == "--draft") {
+ o.draft = next("--draft");
+ if (o.draft != "dflash2" && o.draft != "mtp") {
+ std::fprintf(stderr, "strata generate: --draft takes mtp or dflash2\n");
+ std::exit(2);
+ }
+ }
+ else if (a == "--draft-model") o.draft_model = next("--draft-model");
else if (a == "--control-vector") o.cvec_files.push_back({next("--control-vector"), 1.0f});
else if (a == "--control-vector-scaled") {
// FILE:SCALE, comma-separated; the LAST colon splits, so a Windows path (C:\...) keeps its drive
@@ -5395,6 +5411,10 @@ int main(int argc, char** argv) {
// Secure MTP's CUDA0 allocations before the large host arena is registered with both CUDA contexts.
// In particular WDDM can refuse the draft weights after mapping tens of GiB of host pages.
strata::core::MtpDrafter mtp;
+#if defined(STRATA_HAVE_DFLASH2)
+ strata::core::Dflash2Drafter df2; // --draft dflash2 (CLI spec loop only)
+ bool use_df2 = false;
+#endif
// the draft layer's geometry (the canonical model's MTP head), `static` because MtpDrafter keeps a reference; the batch
// slots' draft-KV copies use this one too
static const strata::core::ModelGeometry draft_geometry{};
@@ -5418,6 +5438,23 @@ int main(int argc, char** argv) {
if (o.pipeline_windows >= 2) mtp.set_force_capture(true);
if (!o.mtp.empty() && !mtp.load(o.mtp, draft_geometry, last_st && !mtp_first ? last_st->ss : ss, mtp_t, err, o.mtp_window)) { std::fprintf(stderr, "strata generate: %s%s\n", err.c_str(), vram_free_note().c_str()); return 1; }
mtp.set_ple_session(&ss);
+#if defined(STRATA_HAVE_DFLASH2)
+ if (o.draft == "dflash2") {
+ const char* why = o.mtp.empty() ? "it needs --mtp (the fallback) and --spec 2 or more"
+ : !native_pack ? "it needs a native pack"
+ : multi_gpu || o.batch > 0 || o.serve ? "it is for one card and the CLI run (not --serve, --batch or a layer split) in this build"
+ : o.draft_model.empty() ? "it needs --draft-model PACK (tools/dflash2_pack.py)" : nullptr;
+ if (why == nullptr) {
+ if (df2.load(o.draft_model, 0, err)) {
+ use_df2 = true;
+ } else {
+ std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
+ }
+ } else {
+ std::fprintf(stderr, "strata generate: --draft dflash2 is off: %s; the MTP drafts instead\n", why);
+ }
+ }
+#endif
if (batch_mtp) {
// Slot b's drafter lives where the solo drafter does (the guard above: the last stage's GPU), carved from the
// LAST stage's slot-b session (bslot_ss.back(): stage 0's on one GPU): it borrows that session's context
@@ -15636,6 +15673,13 @@ int main(int argc, char** argv) {
return e.empty() && mtp.prefill(R_rows, nxt.data(), T, p0, e);
};
}
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2) { // the drafter's context: the last 2047 prompt rows' taps
+ df2.reset();
+ prefill.set_taps(df2.tap_layers(), std::max<int64_t>(0, n_prompt - 2047), 2048);
+ prefill.on_taps = [&](const float* taps, int64_t rows, int64_t pos0, std::string& e) -> bool { return df2.context(taps, rows, pos0, e); };
+ }
+#endif
if (!kvg_started) { // the elastic K/V: every cell this run can reach (no loan)
kvg_started = true;
kvg_start(xcache.slots());
@@ -15672,6 +15716,9 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "strata generate: %zu lent slots refilled in %.1f ms\n", lent.size(),
std::chrono::duration<double, std::milli>(Clock::now() - tr).count());
}
+#if defined(STRATA_HAVE_DFLASH2)
+ prefill.clear_taps();
+#endif
prefill_batched_ms = std::chrono::duration<double, std::milli>(Clock::now() - tp0).count();
prefill_ms += prefill_batched_ms;
pos_start = n_batched;
@@ -15959,6 +16006,9 @@ int main(int argc, char** argv) {
vh.slot_off = xcache.slot_offsets(); // E-6: the device plan's pointers
vh.n_slots = xcache.slots();
if (pcie_balance()) ver.set_pcie_balance(&drive.d.pcie_model);
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2) ver.set_taps(df2.tap_layers());
+#endif
if (!ver.init(wt, g, ss, vh, native_head.loaded() ? &native_head : nullptr, o.spec, err)) {
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
return 1;
@@ -15968,6 +16018,17 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
return 1;
}
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2 && !ver.tapping()) {
+ std::fprintf(stderr, "strata generate: --draft dflash2: the verifier keeps no taps here; the MTP drafts instead\n");
+ use_df2 = false;
+ }
+ if (use_df2 && !df2.bind(wt, &native_head, o.mtp_draft_vocab.empty() ? o.mtp + "/draft_vocab.bin" : o.mtp_draft_vocab, err)) {
+ std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());
+ use_df2 = false;
+ }
+ if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s), the MTP layer stays as the fallback\n", o.draft_model.c_str());
+#endif
mem_mark("the verifier and the drafter's binding");
ver.set_sampling(sp); // the CLI's own sampling (until 0.1.19 this loop was always greedy); no penalties here
if (use_mtp) mtp.set_draft_sampling(sp); // STRATA_SPEC_COUPLED=1: sampled drafts (a no-op otherwise)
@@ -16263,6 +16324,19 @@ int main(int argc, char** argv) {
}
}).detach();
}
+ // the next round's drafts after a commit: the MTP chain, or the DFlash2 block from the committed rows' taps
+ auto draft_next = [&](int T_, int a_) -> bool {
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2) {
+ int32_t dd[strata::core::Dflash2Drafter::kDrafts];
+ float pp[strata::core::Dflash2Drafter::kDrafts];
+ if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;
+ for (size_t i = 0; i < drafts.size() && i < (size_t) strata::core::Dflash2Drafter::kDrafts; ++i) { drafts[i] = dd[i]; dprob[i] = pp[i]; }
+ return true;
+ }
+#endif
+ return mtp.draft(T_, outv.data(), p, a_, drafts.data(), err, dprob.data(), (float) o.spec_min_p);
+ };
while ((int64_t) produced.size() < max_new) {
const Clock::time_point t0 = Clock::now();
strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());
@@ -16403,8 +16477,7 @@ int main(int argc, char** argv) {
total_ms += std::chrono::duration<double, std::milli>(Clock::now() - t0).count();
break;
}
- const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new ||
- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) o.spec_min_p);
+ const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new || draft_next(T, a);
if (adapt_thr.joinable()) adapt_thr.join();
if (!adapt_ok) return 1;
if (!drafted) {
@@ -16528,6 +16601,12 @@ int main(int argc, char** argv) {
1000 * pm.g0, 1000 * pm.g);
}
(void) pool_ms0;
+#if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2 && df2.rounds > 0)
+ std::printf("%-24s %.3f ms/round drafting (%lld rounds, GPU %.3f), %.0f MiB of VRAM\n", "dflash2", df2.ms_draft / (double) df2.rounds,
+ (long long) df2.rounds, df2.ms_gpu / (double) df2.rounds, (double) df2.vram_bytes() / 1048576.0);
+ else
+#endif
if (use_mtp && rounds > 0)
std::printf("%-24s %.3f ms/round drafting (%lld rounds), MTP prompt %.1f ms, %.0f MiB of VRAM\n", "mtp",
mtp.ms_draft / (double) mtp.rounds, (long long) mtp.rounds, mtp.ms_prefill,
diff --git a/tools/dflash2/mk_parity_case.py b/tools/dflash2/mk_parity_case.py
new file mode 100644
index 000000000..a812270c4
--- /dev/null
+++ b/tools/dflash2/mk_parity_case.py
@@ -0,0 +1,88 @@
+"""Builds the input of src/core/dflash2_parity.cpp and the torch reference answer.
+
+ python mk_parity_case.py <drafter.safetensors> <drafter_config.json> <out.bin> [n_ctx=2300] [vp=20000] [seed=1]
+
+The case: n_ctx random tap rows (positions 0 .. n_ctx-1), an anchor token at position n_ctx, a head of `vp` random rows (Q8_0, the draft vocabulary
+subset path: dvocab maps a row to a token id) and the target embedding rows of the anchor and the mask token. The torch port (dflash2_ref.py,
+equal to z-lab's module) gives the reference: the final normalized block rows, the walk's tokens and their probabilities. A context longer than the
+2048-token window exercises the ring wrap and the window mask.
+"""
+import struct
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+
+sys.path.insert(0, str(Path(__file__).parent))
+import dflash2_ref as R
+
+
+class Embed:
+ def __init__(self, rows):
+ self.rows = rows
+
+ def __getitem__(self, ids):
+ return torch.stack([self.rows[int(i)] for i in ids])
+
+
+def quant_q8_0(w):
+ rows, cols = w.shape
+ b = w.reshape(rows, cols // 32, 32)
+ amax = np.abs(b).max(axis=-1)
+ d = amax / 127.0
+ inv = np.where(d != 0, 1.0 / np.where(d == 0, 1, d), 0.0)
+ q = np.rint(b * inv[..., None]).astype(np.int8)
+ out = np.empty((rows, cols // 32, 34), np.uint8)
+ out[..., :2] = d.astype(np.float16).view(np.uint8).reshape(rows, cols // 32, 2)
+ out[..., 2:] = q.view(np.uint8)
+ return out.tobytes(), (q.astype(np.float32) * d.astype(np.float16).astype(np.float32)[..., None]).reshape(rows, cols)
+
+
+def main():
+ st, cfgp, out = sys.argv[1:4]
+ n_ctx = int(sys.argv[4]) if len(sys.argv) > 4 else 2300
+ vp = int(sys.argv[5]) if len(sys.argv) > 5 else 20000
+ seed = int(sys.argv[6]) if len(sys.argv) > 6 else 1
+ rng = np.random.default_rng(seed)
+ H, V = 2560, 248320
+ mask_id = 248077
+ anchor = int(rng.integers(1000, 200000))
+ taps = (rng.standard_normal((n_ctx, 5 * H)) * 1.0).astype(np.float32)
+ emb = (rng.standard_normal((2, H)) * 1.0).astype(np.float32)
+ dvocab = np.sort(rng.choice(V, size=vp, replace=False)).astype(np.int32)
+ head = (rng.standard_normal((vp, H)) * 0.05).astype(np.float32)
+ head_blob, head_dq = quant_q8_0(head)
+ embed = Embed({anchor: torch.from_numpy(emb[0]), mask_id: torch.from_numpy(emb[1])})
+ d = R.Drafter(st, cfgp, embed, torch.from_numpy(head_dq), threads=6)
+ d.set_context(torch.from_numpy(taps))
+ path, _ = d.draft(n_ctx, anchor, 8) # runs the forward; the tail below redoes the selector with the token map
+ fn = d.last_all.clone() # [8][H]
+ out_rows = fn[1:]
+ logits = out_rows @ torch.from_numpy(head_dq).T
+ unary, idx = torch.topk(logits, 16, dim=-1)
+ cand = torch.from_numpy(dvocab.astype(np.int64))[idx]
+ hp = out_rows @ d.w["candidate_selector.hidden_projection.weight"].T
+ A, B = d.w["candidate_selector.predecessor_codebook"], d.w["candidate_selector.successor_codebook"]
+ pred = anchor
+ toks, probs = [], []
+ for i in range(7):
+ sc = unary[i] + (B[cand[i]].float() @ (A[pred].float() * hp[i]))
+ j = int(torch.argmax(sc))
+ probs.append(float(torch.softmax(sc, dim=-1)[j]))
+ pred = int(cand[i][j])
+ toks.append(pred)
+ with open(out, "wb") as f:
+ f.write(struct.pack("<4i", n_ctx, anchor, vp, 0))
+ f.write(taps.tobytes())
+ f.write(emb.tobytes())
+ f.write(dvocab.tobytes())
+ f.write(head_blob)
+ f.write(struct.pack("<7i", *toks))
+ f.write(struct.pack("<7f", *probs))
+ f.write(fn.numpy().astype(np.float32).tobytes())
+ print("case written:", out, "n_ctx", n_ctx, "anchor", anchor, "tokens", toks, "probs", [round(p, 3) for p in probs])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/dflash2_pack.py b/tools/dflash2_pack.py
new file mode 100644
index 000000000..8949c1489
--- /dev/null
+++ b/tools/dflash2_pack.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Pack the incoai DFlash2 drafter for Qwen3.8-Flash-Next (safetensors) into the engine's runtime file.
+
+ python tools/dflash2_pack.py model.safetensors config.json out.pack [--bf16]
+
+Format (little endian): "DF2P", u32 version (1), u32 index_len, a text index (padded with spaces so the data starts on a 4096 multiple), then
+the tensors, each 256-byte aligned at its offset (from the end of the index). Index lines: "cfg <key> <value>", "t <name> <dtype> <rows>
+<cols> <offset> <bytes>" (a matrix [out][in] is rows x cols; a vector or codebook row set is the leading dims folded into rows), "end".
+dtypes: "q8_0" (ggml Q8_0 blocks, 34 bytes per 32 values, rows of the weight matrix [out][in] one after the other: what the engine's
+multi-column MMVQ reads), "f32" (norms, the two conv base kernels), "bf16" (the selector's two codebooks: only gathered rows are read).
+The matrices' quantization matches ggml's quantize_row_q8_0 (d = amax / 127, q = round(x / d), d stored as fp16); the research harness
+(tools/dflash2/q8test.py) measured no acceptance loss from it.
+"""
+import json
+import struct
+import sys
+
+import numpy as np
+
+try:
+ from safetensors import safe_open
+except ImportError: # pragma: no cover
+ sys.exit("needs the safetensors package")
+
+
+def load_tensor(f, name):
+ """-> float32 ndarray (bf16 tensors are widened exactly)"""
+ return f.get_tensor(name).float().numpy()
+
+
+def raw_bf16(f, name):
+ import torch
+ t = f.get_tensor(name)
+ return np.ascontiguousarray(t.view(torch.int16).numpy().view(np.uint16))
+
+
+def quant_q8_0(w):
+ """w: float32 [rows, cols], cols % 32 == 0 -> bytes of rows * cols/32 blocks (fp16 d, 32 int8)"""
+ rows, cols = w.shape
+ assert cols % 32 == 0
+ b = w.reshape(rows, cols // 32, 32)
+ amax = np.abs(b).max(axis=-1)
+ d = amax / 127.0
+ inv = np.where(d != 0, 1.0 / np.where(d == 0, 1, d), 0.0)
+ q = np.rint(b * inv[..., None]).astype(np.int8) # round half to even; ggml uses roundf (ties are measure-zero for these weights)
+ out = np.empty((rows, cols // 32, 34), np.uint8)
+ out[..., :2] = d.astype(np.float16).view(np.uint8).reshape(rows, cols // 32, 2)
+ out[..., 2:] = q.view(np.uint8)
+ return out.tobytes()
+
+
+def main():
+ if len(sys.argv) < 4:
+ sys.exit(__doc__)
+ st_path, cfg_path, out_path = sys.argv[1:4]
+ cfg = json.load(open(cfg_path))
+ dc = cfg["dflash_config"]
+ f = safe_open(st_path, "pt")
+ tensors = []
+ blobs = []
+
+ def add(name, dtype, shape, data):
+ tensors.append({"name": name, "dtype": dtype, "shape": list(shape), "bytes": len(data)})
+ blobs.append(data)
+
+ def mat(name, new=None):
+ w = load_tensor(f, name)
+ assert w.ndim == 2
+ add(new or name, "q8_0", w.shape, quant_q8_0(w))
+
+ def vec(name, new=None):
+ w = load_tensor(f, name)
+ add(new or name, "f32", w.shape, np.ascontiguousarray(w, np.float32).tobytes())
+
+ L = cfg["num_hidden_layers"]
+ mat("fc.weight")
+ vec("hidden_norm.weight")
+ vec("norm.weight")
+ for l in range(L):
+ p = "layers.%d." % l
+ for n in ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj",
+ "mlp.down_proj", "attention_conv.kernel_projection", "mlp_conv.kernel_projection"):
+ mat(p + n + ".weight")
+ for n in ("input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"):
+ vec(p + n + ".weight")
+ vec(p + "attention_conv.base_kernel")
+ vec(p + "mlp_conv.base_kernel")
+ mat("candidate_selector.hidden_projection.weight")
+ for n in ("predecessor_codebook", "successor_codebook"):
+ a = raw_bf16(f, "candidate_selector." + n)
+ add("candidate_selector." + n, "bf16", a.shape, a.tobytes())
+
+ meta = {
+ "hidden": cfg["hidden_size"], "layers": L, "heads": cfg["num_attention_heads"], "kv_heads": cfg["num_key_value_heads"],
+ "head_dim": cfg["head_dim"], "inter": cfg["intermediate_size"], "window": cfg["sliding_window"], "rope_theta": cfg["rope_parameters"]["rope_theta"],
+ "eps": cfg["rms_norm_eps"], "block": dc["block_size"], "conv_group": dc["conv_group_size"], "conv_kernel": dc["conv_kernel_size"],
+ "mask_token": dc["mask_token_id"], "rank": dc["selector_rank"], "top_k": dc["selector_top_k"], "taps": dc["target_layer_ids"],
+ "vocab": cfg["vocab_size"], "source": "incoai/Qwen3.8-Flash-Next-DFlash2 a22a1744",
+ }
+ # offsets: relative to the end of the (padded) index; 256-byte aligned
+ off = 0
+ for t in tensors:
+ off = (off + 255) & ~255
+ t["offset"] = off
+ off += t["bytes"]
+ lines = []
+ for k, v in meta.items():
+ if k == "taps":
+ v = ",".join(str(x) for x in v)
+ lines.append("cfg %s %s" % (k, str(v).replace(" ", "_")))
+ for t in tensors:
+ sh = t["shape"] + [1] * (2 - len(t["shape"])) if len(t["shape"]) < 2 else t["shape"]
+ lines.append("t %s %s %d %d %d %d" % (t["name"], t["dtype"], int(np.prod(sh[:-1])), sh[-1], t["offset"], t["bytes"]))
+ lines.append("end")
+ js = ("\n".join(lines) + "\n").encode()
+ js += b" " * ((-(len(js) + 12)) % 4096)
+ with open(out_path, "wb") as o:
+ o.write(b"DF2P" + struct.pack("<II", 1, len(js)) + js)
+ base = o.tell()
+ for t, b in zip(tensors, blobs):
+ o.seek(base + t["offset"])
+ o.write(b)
+ print("wrote %s: %d tensors, %.1f MB" % (out_path, len(tensors), (base + off) / 1e6))
+
+
+if __name__ == "__main__":
+ main()