Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DFlash2 drafter for Qwen3.8-Flash-Next: Q8_0 pack tool, GPU forward (conv, K/V ring, windowed attention, top-16 selector), permanent verify/prefill taps, --draft dflash2 --draft-model (opt-in, MTP stays the fallback)
Niko12217f97c35acaf3
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 7c6867668..4ee0af934 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -336,6 +336,8 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_library(strata_core STATIC src/core/device.cu src/core/pinned.cu src/platform/memory.cpp src/core/graph.cpp src/core/weights.cppsrc/core/layout.cpp)target_include_directories(strata_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)+ # the DFlash2 drafter (src/core/dflash2.cpp, src/kernels/cuda/dflash2.cu) exists in the CUDA and HIP builds; the Metal and SYCL builds compile without it+ target_compile_definitions(strata_core PUBLIC STRATA_HAVE_DFLASH2=1)if(STRATA_ENABLE_CUDA AND STRATA_EXPERIMENTAL_SM60)# the runtime device check (src/core/device.cu) admits the cards this build was made for (#236)target_compile_definitions(strata_core PRIVATE STRATA_EXPERIMENTAL_SM60=1)@@ -418,6 +420,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)src/kernels/cuda/dequant_bf16.cusrc/kernels/cuda/qsa_select.cusrc/kernels/cuda/verify_kernels.cu+ src/kernels/cuda/dflash2.cusrc/kernels/cuda/iq_kernels.cusrc/kernels/ngram.cppsrc/ngram/ple_reader.cpp@@ -579,7 +582,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)# about arithmetic, and this one knows the ORDER the arithmetic happens in.add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cppsrc/core/remote_expert_opt.cu- src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp+ src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp src/core/dflash2.cppsrc/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cppsrc/core/arch_defaults.cpp src/core/nv_defaults.cpp src/core/vmm.cpp src/core/conversation_file.cpp)target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)@@ -1081,6 +1084,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(verify_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME verify_parity COMMAND verify_parity --selftest)endif()+ # the DFlash2 drafter against the torch port on a synthetic case: needs the pack and a case file (tools/dflash2/mk_parity_case.py), so not a ctest+ add_executable(dflash2_parity src/core/dflash2_parity.cpp)+ target_include_directories(dflash2_parity PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(dflash2_parity PRIVATE strata_engine strata_kernels ${_strata_gpu_runtime_target})add_executable(fused_gr_bench src/kernels/fused_gr_bench.cpp)target_include_directories(fused_gr_bench PRIVATE ${_strata_gpu_include_directories})diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hppnew file mode 100644index 000000000..75aadb075--- /dev/null+++ b/include/strata/core/dflash2.hpp@@ -0,0 +1,129 @@+// include/strata/core/dflash2.hpp - the DFlash2 drafter for Qwen3.8-Flash-Next on the GPU (CUDA and HIP).+//+// incoai/Qwen3.8-Flash-Next-DFlash2 (Apache-2.0) is a block drafter: 5 transformer layers of hidden 2560 that read the target's hidden+// states (the mean of the 4 hyper-connection streams after layers 5, 15, 25, 35, 45) as context and, from the last accepted token plus+// 7 mask tokens, produce 7 draft tokens in one forward pass. A candidate selector (the head's top-16 per row, a rank-256 bilinear+// score between the previous pick and each candidate) picks the path left to right. tools/dflash2_pack.py packs it as Q8_0.+//+// context taps (5 x 2560 per accepted row) -> fc (12800 -> 2560) -> hidden_norm -> per layer K (norm + rope) and V -> ring slot pos % R+// block 8 rows: embed(anchor), embed(mask) x 7; per layer+// rms -> dynamic 2-tap conv (prepare) -> q, k, v -> q/k norm + rope -> attention over the ring + the block's own keys+// -> o -> conv (finish) + residual -> rms -> conv (prepare) -> gate/up -> silu*up -> down -> conv (finish) + residual+// selector final norm -> head over the draft vocabulary (rows 1..7) -> top-16 -> walk+//+// The ring holds only COMMITTED rows (an accepted window row's taps arrive after the verify window), and the block's keys stay outside+// it, so a rejected draft leaves nothing to roll back. One CUDA graph per context-row count n = 1..8 holds the context update and the+// block; the host stages the positions and the anchor and waits for the 7 drafts.+#pragma once++#include "strata/core/session.hpp"++#include <cuda_runtime.h>++#include <cstdint>+#include <functional>+#include <string>+#include <vector>++namespace strata::core {++class NativeHead;++class Dflash2Drafter {+public:+ static constexpr int kRows = 8; ///< block rows: the anchor + 7 masks+ static constexpr int kDrafts = kRows - 1;+ static constexpr int kRing = 2064; ///< K/V slots per layer: the 2048 window + slack for rows written ahead of a rejection++ Dflash2Drafter() = default;+ ~Dflash2Drafter();+ Dflash2Drafter(const Dflash2Drafter&) = delete;+ Dflash2Drafter& operator=(const Dflash2Drafter&) = delete;++ /// Reads the pack (tools/dflash2_pack.py) to the GPU `device` and allocates the ring and the buffers (about 0.8 GB).+ bool load(const std::string& pack_path, int device, std::string& err);+ /// The target's embedding and head (and the draft vocabulary file, "" or missing: the whole head).+ bool bind(const WeightTable& wt, const NativeHead* head, const std::string& vocab_file, std::string& err);+ /// Tests: an embedding callback and a head given as raw rows of `head_type` (a native_mmvq type).+ using EmbedFn = std::function<void(const int32_t* tokens_dev, int n, float* out, cudaStream_t)>;+ bool bind_raw(EmbedFn embed, const void* head_w, int head_type, int64_t head_rows, const std::vector<int32_t>& vocab, std::string& err);++ /// A new request: the ring is empty again.+ void reset() { have_ctx_ = 0; }+ /// Context rows without a draft: `n` rows of taps (device, [n][5 * n_embd]) for positions pos0 ... pos0 + n - 1.+ bool context(const float* taps_dev, int64_t n, int64_t pos0, std::string& err);+ /// One round: the new context rows (n_new in 1..8: the committed window rows, taps device [n_new][5 * n_embd], positions pos0 ...),+ /// then the block for `anchor` (the last committed token, at pos0 + n_new). `drafts` / `probs` get kDrafts entries (the+ /// selector's pick and its probability). The taps pointer must be the same device buffer on every call (graphs are captured on it).+ bool draft(const float* taps_dev, int n_new, int64_t pos0, int32_t anchor, int32_t* drafts, float* probs, std::string& err);++ uint64_t vram_bytes() const { return vram_; }+ int64_t n_draft_vocab() const { return n_dvocab_; }+ int n_embd() const { return N_; }+ const std::vector<int>& tap_layers() const { return taps_; }+ cudaStream_t stream() const { return cs_; }+ /// Diagnostics (STRATA_DF2_DUMP): the final normalized block rows of the last draft(), host copy.+ bool debug_rows(std::vector<float>& rows, std::string& err);++ double ms_draft = 0, ms_gpu = 0, ms_ctx = 0;+ int64_t rounds = 0;++private:+ struct Tensor { std::string name, dtype; int64_t rows = 0, cols = 0; uint64_t off = 0, bytes = 0; };+ const Tensor& find(const std::string& name) const;+ const void* q8(const std::string& name) const { return blob_ + find(name).off; }+ const float* f32(const std::string& name) const { return reinterpret_cast<const float*>(blob_ + find(name).off); }+ void mm(const std::string& name, const void* xq, float* y, int ncols); ///< Q8_0 matrix x ncols activation columns (quantized in xq)+ void quant(const float* x, int n_in, int ncols);+ bool record_context(const float* taps, int n, cudaStream_t cs, std::string& err);+ bool record_block(cudaStream_t cs, std::string& err);+ bool capture(int n, const float* taps, std::string& err);+ bool alloc(std::string& err);+ struct Prof;+ Prof* prof_ = nullptr;+ void stamp(const char* name);+ void prof_collect();+ void prof_report() const;++ int device_ = -1;+ cudaStream_t cs_ = nullptr;+ std::vector<Tensor> tensors_;+ uint8_t* blob_ = nullptr;+ uint64_t vram_ = 0;+ int N_ = 2560, L_ = 5, NH_ = 32, NKV_ = 8, HD_ = 128, I_ = 7680, window_ = 2048, group_ = 16, rank_ = 256, mask_token_ = 0;+ float theta_ = 10000.0f, eps_ = 1e-6f;+ std::vector<int> taps_;+ int64_t vocab_ = 0;++ // target pieces+ EmbedFn embed_;+ const void* head_w_ = nullptr;+ int head_type_ = -1;+ int64_t n_dvocab_ = 0;+ int32_t* dvocab_ = nullptr;+ uint8_t* dhead_ = nullptr; ///< the draft vocabulary's head rows (owned), or null with head_w_ the whole head+ bool bound_ = false;++ // device buffers+ int32_t* d_meta_ = nullptr; ///< [0..7] context positions, [8..15] block positions, [16] anchor token, [17] anchor position, [24..31] block tokens+ int32_t* h_meta_ = nullptr; ///< host staging of d_meta_ (mapped; m_meta_ is its device alias)+ int32_t* m_meta_ = nullptr;+ int32_t* h_out_ = nullptr; ///< pinned: 7 draft tokens then 7 probabilities+ float* hp_out_ = nullptr;+ float *kring_[8] = {}, *vring_[8] = {};+ float *rt_ctx_ = nullptr, *rt_blk_ = nullptr;+ uint8_t* xq_ = nullptr;+ float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;+ float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;+ float *ao_ = nullptr, *o_ = nullptr, *gate_ = nullptr, *up_ = nullptr, *mlp_ = nullptr, *part_ = nullptr, *fn_ = nullptr;+ float *logits_ = nullptr, *topval_ = nullptr, *hsel_ = nullptr, *out_prob_ = nullptr;+ int32_t *toptok_ = nullptr, *out_tok_ = nullptr;+ uint32_t* topscratch_ = nullptr;+ cudaGraphExec_t exec_[kRows + 1] = {};+ const float* graph_taps_ = nullptr;+ int64_t have_ctx_ = 0;+ cudaEvent_t ev_a_ = nullptr, ev_b_ = nullptr;+ bool timing_ = false;+};++} // namespace strata::corediff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hppindex b2ea81923..db19db664 100644--- a/include/strata/core/verify.hpp+++ b/include/strata/core/verify.hpp@@ -323,6 +323,12 @@ public:/// Token t's residual after the last layer, (hc, n_embd) on the device, valid until the next `run`.const float* final_R(int t) const;const float* final_R_all() const { return next_ ? next_->final_R_all() : R_; }+ /// DFlash2 (before init): keep the completed outputs of these layers (zero-based; the mean of the hc streams) for every row of+ /// each window in a device buffer, tap_dev() = [max_t][n_taps][n_embd] floats, valid until the next window. Single-stage+ /// verifiers only; STRATA_TAP_FILE (research dump) turns the same buffer on without this call.+ void set_taps(const std::vector<int>& layers) { tap_req_ = layers; }+ const float* tap_dev() const { return tap_dev_; }+ bool tapping() const { return tap_on_; }/// The GPU plan the pool writes each layer (VRAM hits + the PCIe share of the misses); give it to the/// dispatch (`ExpertDispatch::plan`) before the first `run`.@@ -552,6 +558,7 @@ private:// (the residual with its pending FFN write applied, the mean of the hc streams) per row; run() appends// (pos, token, taps) records to the file.bool tap_on_ = false;+ std::vector<int> tap_req_; ///< set_tapsstd::vector<int> tap_layers_; ///< the tapped layer ids (zero-based, completed outputs)std::vector<int> tap_slot_; ///< layer -> slot in tap_layers_, or -1float* tap_dev_ = nullptr; ///< [max_t][n_taps][n_embd]diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hppnew file mode 100644index 000000000..8a544f298--- /dev/null+++ b/include/strata/kernels/dflash2.hpp@@ -0,0 +1,66 @@+// include/strata/kernels/dflash2.hpp - the small kernels of the DFlash2 drafter (core/dflash2.hpp) for Qwen3.8-Flash-Next.+//+// The drafter's matrices run through the engine's multi-column MMVQ (native_q8_0_mmvq); what is here is everything between+// them: the grouped dynamic 2-tap convolution, the per-head RMS norm + rotary rotation, the windowed non-causal attention over a+// K/V ring, the top-16 selection over the head's logits and the selector's left-to-right walk. All launches are asynchronous on+// the given stream, allocate nothing and are graph-capturable. Layouts are row-major f32 unless noted.+#pragma once++#include <cstdint>++namespace strata::kernels {++constexpr int kDf2Rows = 8; ///< block rows (the anchor + 7 masks) and the most context rows one update takes+constexpr int kDf2Kv = 8; ///< K/V heads+constexpr int kDf2Gqa = 4; ///< query heads per K/V head+constexpr int kDf2Hd = 128; ///< head dim+constexpr int kDf2Chunk = 256; ///< keys per attention block+constexpr int kDf2MaxChunks = 9; ///< ceil((2047 context keys + 8 block keys) / kDf2Chunk)+constexpr int kDf2TopK = 16;+constexpr int kDf2TopBlocks = 64; ///< partial top-k blocks per row (a block holds at most 4096 logits)++/// out[b][c] = sum_{o < 2, b - o >= 0} (base[o][c] + dyn[b * dyn_stride + o * groups + c / group]) * x[(b - o) * H + c] (+ resid[b][c] when+/// resid != null; out may equal resid). x is [rows][H]; base is [2][H]; `dyn` points at this convolution's [2][groups] slice of row 0.+void df2_conv(const float* x, const float* dyn, int dyn_stride, const float* base, const float* resid, float* out, int rows, int H,+ int group, void* stream);++/// table[r][0][i], table[r][1][i] = cos, sin of pos[r] * theta^(-2 i / 128) for i < 64 (the angle reduced in double first, so a large+/// position keeps its precision under fast math). table: [rows][2][64].+void df2_rope_table(const int32_t* pos, int rows, float theta, float* table, void* stream);++/// Per (row, head): out = rope(rmsnorm(in) * gamma). `in` rows are `in_stride` floats apart (heads * 128 contiguous per row). The+/// rotation is the neox/rotate_half one over the whole head dim with row r's entry of `rope_table` (null: the norm only). The result of row r goes to+/// out + slot(r) * heads * 128 with slot(r) = (slot_mod > 0 ? pos[r] % slot_mod : r). `rope` false: the norm only.+void df2_norm_rope(const float* in, int in_stride, const float* gamma, const int32_t* pos, const float* rope_table, float* out, int slot_mod,+ int rows, int heads, float eps, void* stream);++/// out[slot(r)][c] = in[r][c], slot as above (the value half of the ring write).+void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod, int rows, int cols, void* stream);++/// Attention of the block's 8 query rows (q: [8][32][128], already normed and rotated) over the context ring (kring, vring:+/// [ring_slots][8][128]) and the block's own keys (kblk, vblk: [8][8][128]). Row i at position s + i sees context position j when+/// s + i - j < window (j in [max(0, s - window + 1), s - 1], slot j % ring_slots) and every block key. `s` is read from *anchor_pos+/// on the device. Partials in `part` ([kDf2MaxChunks][8][32][130] floats) are combined into out [8][32*128].+void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,+ int ring_slots, int window, float* part, float* out, void* stream);++/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id.+/// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.+void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);++/// The candidate selector's walk: for row i = 0..rows-1 the 16 candidates' scores are unary[i][j] + dot(succ[tok[i][j]],+/// pred[prev] * hp[i]) over `rank` values (codebooks are bf16 rows of `rank`), the pick is the argmax, prev = the pick. Writes+/// out_tok[i] and out_prob[i] (the softmax probability of the pick among the 16). `prev0` is read from *anchor_tok on the device.+void df2_select(const float* hp, const float* unary, const int32_t* tok, const uint16_t* pred_cb, const uint16_t* succ_cb, int rank, int rows,+ const int32_t* anchor_tok, int32_t* out_tok, float* out_prob, void* stream);++/// The verify window's tap: out[r][d] = mean over the hc streams c of (R[r][c][d] + bo[r][d] * 2 * sigmoid(inject[r][c] / hc)) - the layer output with+/// its pending FFN write (gr_write) applied, averaged over the streams. R rows are `R_stride` floats apart, bo rows `bo_stride`, inject rows+/// `inj_stride`, out rows `out_stride`.+void df2_tap_gr(const float* R, int64_t R_stride, const float* bo, int64_t bo_stride, const float* inject, int64_t inj_stride, float* out, int64_t out_stride,+ int rows, int64_t n_embd, int hc, void* stream);++/// out[r][:n_embd] = the mean of the hc streams of row r of R ([rows][hc][n_embd]) written to out + r * out_stride (a tap slot).+void df2_tap_rows(const float* R, float* out, int64_t out_stride, int64_t n_embd, int hc, int rows, void* stream);++} // namespace strata::kernelsdiff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hppindex 7aadc0726..f9bbca027 100644--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@ -184,6 +184,13 @@ public:/// T x hc*n_embd, valid until the next chunk) and the chunk's first position; the MTP draft layer builds its/// K/V from them. The prefill stream is synchronized before the call.std::function<bool(const float* R_rows, int64_t T, int64_t pos0, std::string& err)> on_chunk;+ /// DFlash2 prompt taps (CUDA / HIP, single stage): before `run`, ask for the completed output of each layer in `layers` (zero-based; the+ /// mean of the hc streams) for the prompt rows at positions >= `first_pos`, at most `cap_rows` rows per chunk (a staging buffer of+ /// cap_rows x layers x n_embd floats is allocated at the first chunk). After each chunk that has such rows, on_taps(taps+ /// [rows][layers][n_embd] device, rows, first position, err) runs with the stream synchronized. clear_taps frees the staging.+ void set_taps(const std::vector<int>& layers, int64_t first_pos, int64_t cap_rows) { tap_layers_ = layers; tap_first_ = first_pos; tap_cap_ = cap_rows; }+ void clear_taps();+ std::function<bool(const float* taps, int64_t rows, int64_t pos0, std::string& err)> on_taps;/// Layer split: called by every stage when it has read a chunk, with the position reached, while its own state/// is still at that chunk's end (its stream synchronized; the last stage calls it just before `on_chunk`). An@@ -256,6 +263,9 @@ private:bool drain_pipeline(std::string& err);int64_t stage_lb_ = 0, stage_le_ = -1;+ std::vector<int> tap_layers_;+ int64_t tap_first_ = 0, tap_cap_ = 0;+ float* tap_dev_ = nullptr;Prefill* next_ = nullptr;Prefill* helper_ = nullptr; ///< set_stage_helperbool single_chunk_ = false; ///< a later stage: the prompt is one chunk (set by the stage before)diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cppnew file mode 100644index 000000000..80687d3fa--- /dev/null+++ b/src/core/dflash2.cpp@@ -0,0 +1,466 @@+// src/core/dflash2.cpp - see include/strata/core/dflash2.hpp.+#include "strata/core/dflash2.hpp"++#include "strata/core/native_head.hpp"+#include "strata/core/on_device.hpp"+#include "strata/core/weights.hpp"+#include "strata/kernels/dflash2.hpp"+#include "strata/kernels/elementwise.hpp"+#include "strata/kernels/native_mmvq.hpp"+#include "strata/kernels/native_qsa.hpp"+#include "strata/kernels/verify_kernels.hpp"++#include <algorithm>+#include <chrono>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <exception>+#include <atomic>+#include <stdexcept>++namespace strata::core {+namespace {++using Clock = std::chrono::steady_clock;+double ms_since(Clock::time_point t) { return std::chrono::duration<double, std::milli>(Clock::now() - t).count(); }++bool mapped(size_t bytes, void** h, void** d) {+ if (cudaHostAlloc(h, bytes, cudaHostAllocMapped) != cudaSuccess) return false;+ std::memset(*h, 0, bytes);+ return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;+}++constexpr int GGML_Q8_0 = 8;++} // namespace++Dflash2Drafter::~Dflash2Drafter() {+ if (device_ < 0) return;+ prof_report();+ delete prof_;+ const OnDevice on_device(device_);+ if (cs_) cudaStreamSynchronize(cs_);+ for (cudaGraphExec_t& e : exec_)+ if (e) cudaGraphExecDestroy(e);+ auto fr = [](void* p) { if (p) cudaFree(p); };+ fr(blob_); fr(d_meta_); fr(xq_); fr(rt_ctx_); fr(rt_blk_);+ for (int l = 0; l < 8; ++l) { fr(kring_[l]); fr(vring_[l]); }+ fr(fcy_); fr(hctx_); fr(kc_); fr(vc_); fr(h_); fr(xn_); fr(xc_); fr(dyn_); fr(q_); fr(qn_); fr(k_); fr(v_); fr(kblk_); fr(ao_); fr(o_);+ fr(gate_); fr(up_); fr(mlp_); fr(part_); fr(fn_); fr(logits_); fr(topval_); fr(hsel_); fr(toptok_); fr(topscratch_);+ fr(dvocab_); fr(dhead_);+ if (h_meta_) cudaFreeHost(h_meta_);+ if (h_out_) cudaFreeHost(h_out_);+ if (cs_) cudaStreamDestroy(cs_);+ if (ev_a_) cudaEventDestroy(ev_a_);+ if (ev_b_) cudaEventDestroy(ev_b_);+}++// STRATA_DF2_PROFILE=1 (measurement only): the round runs without its graph and an event follows every stage; the per-stage averages print at exit+struct Dflash2Drafter::Prof {+ std::vector<cudaEvent_t> ev;+ std::vector<std::string> name;+ std::vector<std::pair<std::string, double>> total;+ int64_t rounds = 0;+};++void Dflash2Drafter::stamp(const char* name) {+ if (!prof_) return;+ cudaEvent_t e;+ cudaEventCreate(&e);+ cudaEventRecord(e, cs_);+ prof_->ev.push_back(e);+ prof_->name.push_back(name);+}++void Dflash2Drafter::prof_collect() {+ if (!prof_ || prof_->ev.size() < 2) return;+ cudaStreamSynchronize(cs_);+ for (size_t i = 1; i < prof_->ev.size(); ++i) {+ float ms = 0;+ cudaEventElapsedTime(&ms, prof_->ev[i - 1], prof_->ev[i]);+ auto it = std::find_if(prof_->total.begin(), prof_->total.end(), [&](const auto& t) { return t.first == prof_->name[i]; });+ if (it == prof_->total.end()) prof_->total.emplace_back(prof_->name[i], ms); else it->second += ms;+ }+ for (cudaEvent_t e : prof_->ev) cudaEventDestroy(e);+ prof_->ev.clear();+ prof_->name.clear();+ ++prof_->rounds;+}++void Dflash2Drafter::prof_report() const {+ if (!prof_ || prof_->rounds == 0) return;+ double sum = 0;+ for (const auto& t : prof_->total) sum += t.second;+ std::fprintf(stderr, "strata dflash2 profile (%lld rounds, ms per round, total %.3f):", (long long) prof_->rounds, sum / (double) prof_->rounds);+ for (const auto& t : prof_->total) std::fprintf(stderr, " %s %.3f", t.first.c_str(), t.second / (double) prof_->rounds);+ std::fputc(10, stderr);+}++const Dflash2Drafter::Tensor& Dflash2Drafter::find(const std::string& name) const {+ for (const Tensor& t : tensors_)+ if (t.name == name) return t;+ throw std::runtime_error("dflash2: tensor " + name + " is missing from the pack");+}++bool Dflash2Drafter::load(const std::string& path, int device, std::string& err) {+ device_ = device;+ const OnDevice on_device(device_);+ std::FILE* f = std::fopen(path.c_str(), "rb");+ if (f == nullptr) { err = "dflash2: cannot open " + path; return false; }+ struct Closer { std::FILE* f; ~Closer() { std::fclose(f); } } closer{f};+ uint32_t hdr[3];+ if (std::fread(hdr, 4, 3, f) != 3 || std::memcmp(hdr, "DF2P", 4) != 0 || hdr[1] != 1) { err = "dflash2: " + path + " is not a DF2P v1 pack"; return false; }+ std::string index(hdr[2], ' ');+ if (std::fread(index.data(), 1, hdr[2], f) != hdr[2]) { err = "dflash2: the pack's index is cut short"; return false; }+ const long data_off = 12 + (long) hdr[2];+ uint64_t total = 0;+ for (size_t i = 0; i < index.size();) {+ const size_t j = index.find('\n', i);+ if (j == std::string::npos) break;+ const std::string line = index.substr(i, j - i);+ i = j + 1;+ char a[256], b[64], c[256];+ long long v1, v2, v3, v4;+ if (std::sscanf(line.c_str(), "cfg %63s %255s", b, c) == 2) {+ const std::string key = b;+ if (key == "hidden") N_ = std::atoi(c);+ else if (key == "layers") L_ = std::atoi(c);+ else if (key == "heads") NH_ = std::atoi(c);+ else if (key == "kv_heads") NKV_ = std::atoi(c);+ else if (key == "head_dim") HD_ = std::atoi(c);+ else if (key == "inter") I_ = std::atoi(c);+ else if (key == "window") window_ = std::atoi(c);+ else if (key == "rope_theta") theta_ = (float) std::atof(c);+ else if (key == "eps") eps_ = (float) std::atof(c);+ else if (key == "conv_group") group_ = std::atoi(c);+ else if (key == "rank") rank_ = std::atoi(c);+ else if (key == "mask_token") mask_token_ = std::atoi(c);+ else if (key == "vocab") vocab_ = std::atoll(c);+ else if (key == "taps") {+ taps_.clear();+ for (const char* q = c; *q;) { taps_.push_back(std::atoi(q)); while (*q && *q != ',') ++q; if (*q == ',') ++q; }+ }+ } else if (std::sscanf(line.c_str(), "t %255s %63s %lld %lld %lld %lld", a, b, &v1, &v2, &v3, &v4) == 6) {+ Tensor t;+ t.name = a; t.dtype = b; t.rows = v1; t.cols = v2; t.off = (uint64_t) v3; t.bytes = (uint64_t) v4;+ tensors_.push_back(t);+ total = std::max(total, t.off + t.bytes);+ }+ }+ if (tensors_.empty() || NH_ != 32 || NKV_ != 8 || HD_ != 128 || rank_ != 256 || N_ != 2560 || L_ > 8 || window_ + 16 > kRing) {+ err = "dflash2: the pack's shape is not the Flash-Next drafter this engine was written for";+ return false;+ }+ if (cudaMalloc((void**) &blob_, total) != cudaSuccess) { err = "dflash2: the pack does not fit in VRAM (" + std::to_string(total >> 20) + " MiB)"; return false; }+ vram_ += total;+ std::vector<uint8_t> stage(64u << 20);+#if defined(_WIN32)+ _fseeki64(f, data_off, SEEK_SET);+#else+ fseeko(f, data_off, SEEK_SET);+#endif+ for (uint64_t o = 0; o < total;) {+ const size_t n = (size_t) std::min<uint64_t>(stage.size(), total - o);+ if (std::fread(stage.data(), 1, n, f) != n) { err = "dflash2: the pack is shorter than its index"; return false; }+ if (cudaMemcpy(blob_ + o, stage.data(), n, cudaMemcpyHostToDevice) != cudaSuccess) { err = "dflash2: weight upload failed"; return false; }+ o += n;+ }+ if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "dflash2: stream"; return false; }+ if (const char* e = std::getenv("STRATA_DF2_PROFILE"); e && e[0] == '1') prof_ = new Prof();+ if (const char* e = std::getenv("STRATA_DF2_TIMING"); e && e[0] == '1') {+ timing_ = true;+ cudaEventCreate(&ev_a_);+ cudaEventCreate(&ev_b_);+ }+ return alloc(err);+}++bool Dflash2Drafter::alloc(std::string& err) {+ auto dev = [&](auto** p, size_t n) {+ using T = std::remove_pointer_t<std::remove_reference_t<decltype(*p)>>;+ if (cudaMalloc((void**) p, n * sizeof(T)) != cudaSuccess) return false;+ vram_ += n * sizeof(T);+ return true;+ };+ const size_t B = kRows;+ bool ok = dev(&d_meta_, 32) && dev(&xq_, kernels::native_q8_1_bytes(std::max(N_ * 5, I_), kRows) + 4096) && dev(&rt_ctx_, B * 128) && dev(&rt_blk_, B * 128) &&+ dev(&fcy_, B * N_) && dev(&hctx_, B * N_) && dev(&kc_, B * NKV_ * HD_) && dev(&vc_, B * NKV_ * HD_) && dev(&h_, B * N_) && dev(&xn_, B * N_) &&+ dev(&xc_, B * N_) && dev(&dyn_, B * 640) && dev(&q_, B * NH_ * HD_) && dev(&qn_, B * NH_ * HD_) && dev(&k_, B * NKV_ * HD_) && dev(&v_, B * NKV_ * HD_) &&+ dev(&kblk_, B * NKV_ * HD_) && dev(&ao_, B * NH_ * HD_) && dev(&o_, B * N_) && dev(&gate_, B * I_) && dev(&up_, B * I_) && dev(&mlp_, B * N_) &&+ dev(&part_, (size_t) kernels::kDf2MaxChunks * 8 * 32 * 130) && dev(&fn_, B * N_) && dev(&topval_, kDrafts * 16) && dev(&toptok_, kDrafts * 16) &&+ dev(&hsel_, kDrafts * rank_) && dev(&topscratch_, (size_t) kDrafts * kernels::kDf2TopBlocks * kernels::kDf2TopK * 2);+ for (int l = 0; ok && l < L_; ++l) ok = dev(&kring_[l], (size_t) kRing * NKV_ * HD_) && dev(&vring_[l], (size_t) kRing * NKV_ * HD_);+ if (!ok) { err = "dflash2: the drafter's buffers do not fit in VRAM"; return false; }+ void* hd = nullptr;+ void* dd = nullptr;+ if (!mapped(32 * 4, (void**) &hd, &dd)) { err = "dflash2: mapped staging"; return false; }+ h_meta_ = (int32_t*) hd;+ m_meta_ = (int32_t*) dd;+ if (!mapped(64 * 4, &hd, &dd)) { err = "dflash2: mapped output"; return false; }+ h_out_ = (int32_t*) hd;+ out_tok_ = (int32_t*) dd; // the select kernel writes the 7 picks here (host-visible) ...+ out_prob_ = (float*) dd + 16; // ... and their probabilities at +16 ints+ cudaMemset(rt_ctx_, 0, B * 128 * 4);+ for (int l = 0; l < L_; ++l) { cudaMemset(kring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); cudaMemset(vring_[l], 0, (size_t) kRing * NKV_ * HD_ * 4); }+ return cudaDeviceSynchronize() == cudaSuccess;+}++bool Dflash2Drafter::bind_raw(EmbedFn embed, const void* head_w, int head_type, int64_t head_rows, const std::vector<int32_t>& vocab, std::string& err) {+ const OnDevice on_device(device_);+ embed_ = std::move(embed);+ head_w_ = head_w;+ head_type_ = head_type;+ n_dvocab_ = head_rows;+ if (!vocab.empty()) {+ n_dvocab_ = (int64_t) vocab.size();+ if (cudaMalloc((void**) &dvocab_, vocab.size() * 4) != cudaSuccess) { err = "dflash2: draft vocabulary"; return false; }+ vram_ += vocab.size() * 4;+ cudaMemcpy(dvocab_, vocab.data(), vocab.size() * 4, cudaMemcpyHostToDevice);+ }+ if ((n_dvocab_ + kernels::kDf2TopBlocks - 1) / kernels::kDf2TopBlocks > 4096) { err = "dflash2: the head has too many rows for the top-k"; return false; }+ if (!logits_ && cudaMalloc((void**) &logits_, (size_t) kDrafts * (size_t) n_dvocab_ * 4) != cudaSuccess) { err = "dflash2: the draft logits do not fit"; return false; }+ vram_ += (uint64_t) kDrafts * (uint64_t) n_dvocab_ * 4;+ bound_ = true;+ return true;+}++bool Dflash2Drafter::bind(const WeightTable& wt, const NativeHead* head, const std::string& vocab_file, std::string& err) {+ const OnDevice on_device(device_);+ if (head == nullptr || !head->loaded()) { err = "dflash2: the drafter needs the native head (--native)"; return false; }+ const WeightRef* we = wt.find("token_embd.weight");+ if (we == nullptr) { err = "dflash2: token_embd.weight is missing"; return false; }+ EmbedFn embed = [we](const int32_t* tok, int n, float* out, cudaStream_t cs) {+ if (const NativeEmbed* ne = native_embed()) {+ ne->gather_dev(tok, n, out, cs);+ } else {+ const auto* codes = (const uint8_t*) we->data;+ const auto* scales = (const float*) (codes + we->codes_bytes);+ const auto* offsets = we->has_offset ? (const float*) (codes + we->codes_bytes + we->scales_bytes) : nullptr;+ kernels::embedding_gather_dev(codes, scales, offsets, tok, n, we->ne0, we->code_bits, we->code_bias, we->group_elems,+ (uint64_t) (we->ne0 / (8 / we->code_bits)), (uint64_t) (we->ne0 / we->group_elems), out, cs);+ }+ };+ std::vector<int32_t> vocab;+ if (!vocab_file.empty()) {+ if (std::FILE* vf = std::fopen(vocab_file.c_str(), "rb")) {+ std::fseek(vf, 0, SEEK_END);+ const long size = std::ftell(vf);+ std::fseek(vf, 0, SEEK_SET);+ if (size >= 4 && size % 4 == 0) {+ vocab.resize((size_t) size / 4);+ if (std::fread(vocab.data(), 4, vocab.size(), vf) != vocab.size()) vocab.clear();+ }+ std::fclose(vf);+ }+ }+ const void* hw = head->weights();+ const int64_t row_bytes = (int64_t) head->row_bytes();+ int64_t head_rows = row_bytes > 0 ? (int64_t) (head->weight_bytes() / (uint64_t) row_bytes) : 0;+ if (!vocab.empty()) {+ int32_t* dv = nullptr;+ if (cudaMalloc((void**) &dv, vocab.size() * 4) != cudaSuccess || cudaMalloc((void**) &dhead_, (size_t) ((int64_t) vocab.size() * row_bytes)) != cudaSuccess) {+ if (dv) cudaFree(dv);+ err = "dflash2: the draft vocabulary's head rows do not fit";+ return false;+ }+ cudaMemcpy(dv, vocab.data(), vocab.size() * 4, cudaMemcpyHostToDevice);+ kernels::gather_rows((const uint8_t*) hw, row_bytes, dv, (int64_t) vocab.size(), dhead_, nullptr);+ cudaDeviceSynchronize();+ cudaFree(dv);+ vram_ += (uint64_t) ((int64_t) vocab.size() * row_bytes);+ hw = dhead_;+ head_rows = (int64_t) vocab.size();+ }+ if (!bind_raw(embed, hw, head->type(), head_rows, vocab, err)) return false;+ std::fprintf(stderr, "strata dflash2: %lld-token draft head (%.1f MiB), drafter %.0f MiB of VRAM\n", (long long) n_dvocab_,+ (double) (n_dvocab_ * row_bytes) / 1048576.0, (double) vram_ / 1048576.0);+ return true;+}++void Dflash2Drafter::quant(const float* x, int n_in, int ncols) { kernels::native_quantize_q8_1(x, xq_, n_in, ncols, cs_); }++void Dflash2Drafter::mm(const std::string& name, const void* xq, float* y, int ncols) {+ const Tensor& t = find(name);+ kernels::native_q8_0_mmvq(blob_ + t.off, xq, y, (int) t.cols, (int) t.rows, ncols, cs_);+}++bool Dflash2Drafter::record_context(const float* taps, int n, cudaStream_t cs, std::string& err) {+ using namespace kernels;+ try {+ const int tap_n = (int) taps_.size() * N_;+ df2_rope_table(d_meta_ + 0, n, theta_, rt_ctx_, cs);+ stamp("ctx-start");+ quant(taps, tap_n, n);+ mm("fc.weight", xq_, fcy_, n);+ stamp("fc");+ native_qsa_rms_norm_weighted(fcy_, f32("hidden_norm.weight"), hctx_, N_, n, eps_, cs);+ quant(hctx_, N_, n);+ for (int l = 0; l < L_; ++l) {+ const std::string p = "layers." + std::to_string(l) + ".";+ mm(p + "self_attn.k_proj.weight", xq_, kc_, n);+ mm(p + "self_attn.v_proj.weight", xq_, vc_, n);+ df2_norm_rope(kc_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 0, rt_ctx_, kring_[l], kRing, n, NKV_, eps_, cs);+ df2_copy_rows(vc_, vring_[l], d_meta_ + 0, kRing, n, NKV_ * HD_, cs);+ }+ stamp("ctx-kv");+ } catch (const std::exception& e) {+ err = e.what();+ return false;+ }+ return true;+}++bool Dflash2Drafter::record_block(cudaStream_t cs, std::string& err) {+ using namespace kernels;+ try {+ const int B = kRows;+ df2_rope_table(d_meta_ + 8, B, theta_, rt_blk_, cs);+ embed_(d_meta_ + 24, B, h_, cs);+ stamp("embed");+ for (int l = 0; l < L_; ++l) {+ const std::string p = "layers." + std::to_string(l) + ".";+ const float* base_a = f32(p + "attention_conv.base_kernel");+ const float* base_m = f32(p + "mlp_conv.base_kernel");+ native_qsa_rms_norm_weighted(h_, f32(p + "input_layernorm.weight"), xn_, N_, B, eps_, cs);+ quant(xn_, N_, B);+ mm(p + "attention_conv.kernel_projection.weight", xq_, dyn_, B);+ df2_conv(xn_, dyn_, 640, base_a, nullptr, xc_, B, N_, group_, cs);+ quant(xc_, N_, B);+ stamp("attn-front");+ mm(p + "self_attn.q_proj.weight", xq_, q_, B);+ mm(p + "self_attn.k_proj.weight", xq_, k_, B);+ mm(p + "self_attn.v_proj.weight", xq_, v_, B);+ stamp("qkv");+ df2_norm_rope(q_, NH_ * HD_, f32(p + "self_attn.q_norm.weight"), d_meta_ + 8, rt_blk_, qn_, 0, B, NH_, eps_, cs);+ df2_norm_rope(k_, NKV_ * HD_, f32(p + "self_attn.k_norm.weight"), d_meta_ + 8, rt_blk_, kblk_, 0, B, NKV_, eps_, cs);+ df2_attention(qn_, kring_[l], vring_[l], kblk_, v_, d_meta_ + 17, kRing, window_, part_, ao_, cs);+ stamp("attention");+ quant(ao_, NH_ * HD_, B);+ mm(p + "self_attn.o_proj.weight", xq_, o_, B);+ stamp("o_proj");+ df2_conv(o_, dyn_ + 320, 640, base_a + 2 * N_, h_, h_, B, N_, group_, cs);+ native_qsa_rms_norm_weighted(h_, f32(p + "post_attention_layernorm.weight"), xn_, N_, B, eps_, cs);+ quant(xn_, N_, B);+ mm(p + "mlp_conv.kernel_projection.weight", xq_, dyn_, B);+ df2_conv(xn_, dyn_, 640, base_m, nullptr, xc_, B, N_, group_, cs);+ quant(xc_, N_, B);+ stamp("mlp-front");+ mm(p + "mlp.gate_proj.weight", xq_, gate_, B);+ mm(p + "mlp.up_proj.weight", xq_, up_, B);+ stamp("gate-up");+ native_swiglu_quantize_q8_1(gate_, up_, xq_, I_, B, cs);+ mm(p + "mlp.down_proj.weight", xq_, mlp_, B);+ stamp("down");+ df2_conv(mlp_, dyn_ + 320, 640, base_m + 2 * N_, h_, h_, B, N_, group_, cs);+ }+ native_qsa_rms_norm_weighted(h_, f32("norm.weight"), fn_, N_, B, eps_, cs);+ quant(fn_ + N_, N_, kDrafts); // rows 1..7+ stamp("final-norm");+ native_mmvq(head_type_, head_w_, xq_, logits_, N_, (int) n_dvocab_, kDrafts, cs);+ stamp("head");+ df2_topk(logits_, (int) n_dvocab_, dvocab_, kDrafts, topscratch_, topval_, toptok_, cs);+ stamp("topk");+ mm("candidate_selector.hidden_projection.weight", xq_, hsel_, kDrafts);+ df2_select(hsel_, topval_, toptok_, reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.predecessor_codebook").off),+ reinterpret_cast<const uint16_t*>(blob_ + find("candidate_selector.successor_codebook").off), rank_, kDrafts, d_meta_ + 16, out_tok_,+ out_prob_, cs);+ stamp("selector");+ } catch (const std::exception& e) {+ err = e.what();+ return false;+ }+ return true;+}++bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {+ if (exec_[n]) return true;+ if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "dflash2: begin capture"; return false; }+ kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);+ bool ok = record_context(taps, n, cs_, err);+ if (ok) ok = record_block(cs_, err);+ cudaGraph_t graph = nullptr;+ const cudaError_t ce = cudaStreamEndCapture(cs_, &graph);+ if (!ok) { if (graph) cudaGraphDestroy(graph); return false; }+ if (ce != cudaSuccess || cudaGraphInstantiate(&exec_[n], graph, 0) != cudaSuccess) {+ if (graph) cudaGraphDestroy(graph);+ err = std::string("dflash2: capture: ") + cudaGetErrorString(ce);+ return false;+ }+ cudaGraphDestroy(graph);+ cudaGraphUpload(exec_[n], cs_);+ cudaStreamSynchronize(cs_);+ return true;+}++bool Dflash2Drafter::context(const float* taps, int64_t n, int64_t pos0, std::string& err) {+ const OnDevice on_device(device_);+ const int tap_n = (int) taps_.size() * N_;+ for (int64_t r0 = 0; r0 < n; r0 += kRows) {+ const int m = (int) std::min<int64_t>(kRows, n - r0);+ for (int i = 0; i < m; ++i) h_meta_[i] = (int32_t) (pos0 + r0 + i);+ std::atomic_thread_fence(std::memory_order_seq_cst);+ kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);+ if (!record_context(taps + (size_t) r0 * (size_t) tap_n, m, cs_, err)) return false;+ if (cudaStreamSynchronize(cs_) != cudaSuccess) { err = std::string("dflash2: ") + cudaGetErrorString(cudaGetLastError()); return false; }+ }+ have_ctx_ = std::max<int64_t>(have_ctx_, pos0 + n);+ return true;+}++bool Dflash2Drafter::draft(const float* taps, int n_new, int64_t pos0, int32_t anchor, int32_t* drafts, float* probs, std::string& err) {+ if (!bound_) { err = "dflash2: not bound"; return false; }+ if (n_new < 1 || n_new > kRows) { err = "dflash2: draft rows out of range"; return false; }+ const OnDevice on_device(device_);+ const Clock::time_point t0 = Clock::now();+ if (graph_taps_ != nullptr && graph_taps_ != taps) { err = "dflash2: the taps buffer moved after capture"; return false; }+ if (!prof_ && !exec_[n_new]) {+ // the staging must hold valid positions while the capture runs the (empty) kernels' parameter checks+ for (int i = 0; i < 8; ++i) h_meta_[i] = (int32_t) (pos0 + i);+ if (!capture(n_new, taps, err)) return false;+ graph_taps_ = taps;+ }+ const int64_t s = pos0 + n_new;+ for (int i = 0; i < 8; ++i) { h_meta_[i] = (int32_t) (pos0 + std::min(i, n_new - 1)); h_meta_[8 + i] = (int32_t) (s + i); }+ h_meta_[16] = anchor;+ h_meta_[17] = (int32_t) s;+ h_meta_[24] = anchor;+ for (int i = 1; i < 8; ++i) h_meta_[24 + i] = mask_token_;+ for (int i = 0; i < kDrafts; ++i) { ((volatile int32_t*) h_out_)[i] = -1; }+ std::atomic_thread_fence(std::memory_order_seq_cst);+ if (timing_) cudaEventRecord(ev_a_, cs_);+ if (prof_) { // the same round without its graph, an event after every stage+ cudaStreamSynchronize(cs_);+ for (cudaEvent_t e : prof_->ev) cudaEventDestroy(e); // the stamps of context() calls are not a round+ prof_->ev.clear();+ prof_->name.clear();+ kernels::copy_i32_from_mapped(d_meta_, m_meta_, 32, cs_);+ stamp("start");+ if (!record_context(taps, n_new, cs_, err) || !record_block(cs_, err)) return false;+ prof_collect();+ } else+ if (cudaGraphLaunch(exec_[n_new], cs_) != cudaSuccess) { err = std::string("dflash2 draft: ") + cudaGetErrorString(cudaGetLastError()); return false; }+ if (timing_) cudaEventRecord(ev_b_, cs_);+ (void) cudaStreamQuery(cs_);+ if (cudaStreamSynchronize(cs_) != cudaSuccess) { err = std::string("dflash2 draft: ") + cudaGetErrorString(cudaGetLastError()); return false; }+ for (int i = 0; i < kDrafts; ++i) {+ drafts[i] = ((volatile int32_t*) h_out_)[i];+ if (probs) probs[i] = ((volatile float*) h_out_)[16 + i];+ }+ if (timing_) {+ float ms = 0;+ if (cudaEventElapsedTime(&ms, ev_a_, ev_b_) == cudaSuccess) ms_gpu += ms;+ }+ have_ctx_ = std::max<int64_t>(have_ctx_, s);+ ms_draft += ms_since(t0);+ ++rounds;+ return true;+}++bool Dflash2Drafter::debug_rows(std::vector<float>& rows, std::string& err) {+ rows.resize((size_t) kRows * (size_t) N_);+ if (cudaMemcpy(rows.data(), fn_, rows.size() * 4, cudaMemcpyDeviceToHost) != cudaSuccess) { err = "dflash2: debug copy"; return false; }+ return true;+}++} // namespace strata::corediff --git a/src/core/dflash2_parity.cpp b/src/core/dflash2_parity.cppnew file mode 100644index 000000000..bf97887e2--- /dev/null+++ b/src/core/dflash2_parity.cpp@@ -0,0 +1,95 @@+// src/core/dflash2_parity.cpp - the engine's DFlash2 drafter against the torch port, on a synthetic case (GPU, no model).+//+// dflash2_parity <pack> <case.bin> [rounds]+//+// case.bin comes from tools/dflash2/mk_parity_case.py: n_ctx random tap rows, an anchor, a random Q8_0 head over a draft-vocabulary subset, the+// target embedding rows of the anchor and the mask token, and the torch answer (tokens, probabilities, the 8 final normalized rows). The engine's+// drafter reads the same pack (Q8_0 weights, Q8_1 activations), so the rows agree to quantization noise, not bitwise: the test checks the rows'+// cosine similarity, the walk's tokens and the time of a round.+#include "strata/core/dflash2.hpp"++#include <cuda_runtime.h>++#include <chrono>+#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <vector>++using strata::core::Dflash2Drafter;++int main(int argc, char** argv) {+ if (argc < 3) { std::fprintf(stderr, "usage: dflash2_parity <pack> <case.bin> [rounds]\n"); return 2; }+ const int rounds = argc > 3 ? std::atoi(argv[3]) : 20;+ std::FILE* f = std::fopen(argv[2], "rb");+ if (!f) { std::fprintf(stderr, "cannot open %s\n", argv[2]); return 2; }+ int32_t hdr[4];+ if (std::fread(hdr, 4, 4, f) != 4) return 2;+ const int n_ctx = hdr[0], anchor = hdr[1], vp = hdr[2];+ const int H = 2560, tapn = 5 * H;+ std::vector<float> taps((size_t) n_ctx * tapn), emb(2 * H), fn_ref(8 * H);+ std::vector<int32_t> dvocab(vp), tok_ref(7);+ std::vector<float> prob_ref(7);+ const size_t row_bytes = (size_t) (H / 32) * 34;+ std::vector<uint8_t> head((size_t) vp * row_bytes);+ bool ok = std::fread(taps.data(), 4, taps.size(), f) == taps.size() && std::fread(emb.data(), 4, emb.size(), f) == emb.size() &&+ std::fread(dvocab.data(), 4, dvocab.size(), f) == dvocab.size() && std::fread(head.data(), 1, head.size(), f) == head.size() &&+ std::fread(tok_ref.data(), 4, 7, f) == 7 && std::fread(prob_ref.data(), 4, 7, f) == 7 && std::fread(fn_ref.data(), 4, fn_ref.size(), f) == fn_ref.size();+ std::fclose(f);+ if (!ok) { std::fprintf(stderr, "case file is cut short\n"); return 2; }++ std::string err;+ Dflash2Drafter d;+ if (!d.load(argv[1], 0, err)) { std::fprintf(stderr, "load: %s\n", err.c_str()); return 1; }+ float *d_taps = nullptr, *d_emb = nullptr;+ uint8_t* d_head = nullptr;+ cudaMalloc((void**) &d_taps, taps.size() * 4);+ cudaMalloc((void**) &d_emb, emb.size() * 4);+ cudaMalloc((void**) &d_head, head.size());+ cudaMemcpy(d_taps, taps.data(), taps.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(d_emb, emb.data(), emb.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(d_head, head.data(), head.size(), cudaMemcpyHostToDevice);+ auto embed = [&](const int32_t*, int n, float* out, cudaStream_t cs) {+ cudaMemcpyAsync(out, d_emb, (size_t) H * 4, cudaMemcpyDeviceToDevice, cs);+ for (int i = 1; i < n; ++i) cudaMemcpyAsync(out + (size_t) i * H, d_emb + H, (size_t) H * 4, cudaMemcpyDeviceToDevice, cs);+ };+ if (!d.bind_raw(embed, d_head, 8, vp, dvocab, err)) { std::fprintf(stderr, "bind: %s\n", err.c_str()); return 1; }+ std::printf("drafter VRAM %.0f MiB\n", (double) d.vram_bytes() / 1048576.0);++ // the context but the last row, then one draft round that adds the last row+ if (!d.context(d_taps, n_ctx - 1, 0, err)) { std::fprintf(stderr, "context: %s\n", err.c_str()); return 1; }+ int32_t drafts[7];+ float probs[7];+ const float* last = d_taps + (size_t) (n_ctx - 1) * tapn;+ if (!d.draft(last, 1, n_ctx - 1, anchor, drafts, probs, err)) { std::fprintf(stderr, "draft: %s\n", err.c_str()); return 1; }+ std::vector<float> rows;+ d.debug_rows(rows, err);+ int bad = 0;+ double min_cos = 1.0;+ for (int r = 0; r < 8; ++r) {+ double dot = 0, na = 0, nb = 0;+ for (int c = 0; c < H; ++c) {+ const double a = rows[(size_t) r * H + c], b = fn_ref[(size_t) r * H + c];+ dot += a * b; na += a * a; nb += b * b;+ }+ const double cs = dot / std::sqrt(na * nb + 1e-30);+ min_cos = std::min(min_cos, cs);+ std::printf("row %d: cos %.5f norm ratio %.4f\n", r, cs, std::sqrt(na / (nb + 1e-30)));+ }+ int same = 0;+ for (int i = 0; i < 7; ++i) {+ same += drafts[i] == tok_ref[i];+ std::printf("draft %d: engine %d (p %.3f) torch %d (p %.3f)\n", i, drafts[i], probs[i], tok_ref[i], prob_ref[i]);+ }+ if (min_cos < 0.98) { std::printf("FAIL: a final row's cosine similarity %.4f is below 0.98\n", min_cos); ++bad; }+ // timing: the same round again (the graph replays; the ring rewrites the same slot)+ cudaDeviceSynchronize();+ const auto t0 = std::chrono::steady_clock::now();+ for (int i = 0; i < rounds; ++i)+ if (!d.draft(last, 1, n_ctx - 1, anchor, drafts, probs, err)) { std::fprintf(stderr, "draft: %s\n", err.c_str()); return 1; }+ const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count() / rounds;+ std::printf("tokens equal %d of 7; %.3f ms per round (1 context row + the block), GPU %.3f ms\n", same, ms, d.rounds > 0 ? d.ms_gpu / (double) d.rounds : 0.0);+ std::printf(bad ? "dflash2_parity: FAIL\n" : "dflash2_parity: ok\n");+ return bad ? 1 : 0;+}diff --git a/src/core/verify.cpp b/src/core/verify.cppindex c9f48a747..5b898f9da 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -56,6 +56,7 @@#include "strata/core/progress.hpp"#include "strata/kernels/shared_expert.hpp"#include "strata/kernels/verify_kernels.hpp"+#include "strata/kernels/dflash2.hpp"#include <algorithm>#include <atomic>@@ -779,35 +780,42 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&mapped(64, (void**) &h_plan_err_, (void**) &m_plan_err_) &&mapped(T * K * N * 4, (void**) &h_ymiss_, (void**) &m_ymiss_);if (!ok) { err = "verify: mapped staging allocation failed " + strata::core::pin_exhausted_hint(); return false; }- // STRATA_TAP_FILE: the DFlash2 experiment's hook (see verify.hpp); off unless the variable is set- if (const char* tap_path = std::getenv("STRATA_TAP_FILE"); tap_path != nullptr && tap_path[0] != '\0' && lb_ == 0 &&- le_ == g.n_layers) {+ // DFlash2 taps: set_taps (the drafter) or STRATA_TAP_FILE (the research dump); off, and nothing is allocated or captured, otherwise+ const char* tap_path = std::getenv("STRATA_TAP_FILE");+ if (tap_path != nullptr && tap_path[0] == '\0') tap_path = nullptr;+ if ((tap_path != nullptr || !tap_req_.empty()) && lb_ == 0 && le_ == g.n_layers) {tap_layers_.clear();- const char* tl = std::getenv("STRATA_TAP_LAYERS");- std::string list = tl != nullptr && tl[0] != '\0' ? tl : "5,15,25,35,45";- for (size_t i = 0; i < list.size();) {- const size_t j = list.find(',', i);- const int l = std::atoi(list.substr(i, j == std::string::npos ? std::string::npos : j - i).c_str());- if (l >= 0 && l < g.n_layers - 1) tap_layers_.push_back(l);- if (j == std::string::npos) break;- i = j + 1;+ if (tap_path == nullptr) {+ for (int l : tap_req_) if (l >= 0 && l < g.n_layers - 1) tap_layers_.push_back(l);+ } else {+ const char* tl = std::getenv("STRATA_TAP_LAYERS");+ std::string list = tl != nullptr && tl[0] != '\0' ? tl : "5,15,25,35,45";+ for (size_t i = 0; i < list.size();) {+ const size_t j = list.find(',', i);+ const int l = std::atoi(list.substr(i, j == std::string::npos ? std::string::npos : j - i).c_str());+ if (l >= 0 && l < g.n_layers - 1) tap_layers_.push_back(l);+ if (j == std::string::npos) break;+ i = j + 1;+ }}tap_slot_.assign((size_t) g.n_layers, -1);for (size_t i = 0; i < tap_layers_.size(); ++i) tap_slot_[(size_t) tap_layers_[i]] = (int) i;const size_t nt = tap_layers_.size();if (nt == 0 || cudaMalloc((void**) &tap_dev_, (size_t) T * nt * N * 4) != cudaSuccess ||- cudaMalloc((void**) &tap_scratch_, (size_t) HC * N * 4) != cudaSuccess ||- !mapped((size_t) T * nt * N * 4, (void**) &h_tap_, (void**) &m_tap_)) {- err = "verify: the STRATA_TAP_FILE buffers could not be allocated";+ (tap_path != nullptr && (cudaMalloc((void**) &tap_scratch_, (size_t) HC * N * 4) != cudaSuccess ||+ !mapped((size_t) T * nt * N * 4, (void**) &h_tap_, (void**) &m_tap_)))) {+ err = "verify: the DFlash2 tap buffers could not be allocated";return false;}- tap_file_ = std::fopen(tap_path, "wb");- if (tap_file_ == nullptr) { err = std::string("verify: cannot write STRATA_TAP_FILE ") + tap_path; return false; }- const int32_t hdr[4] = {0x50415453 /* 'STAP' */, 1, (int32_t) N, (int32_t) nt};- std::fwrite(hdr, sizeof hdr, 1, tap_file_);- for (int l : tap_layers_) { const int32_t v = l; std::fwrite(&v, sizeof v, 1, tap_file_); }+ if (tap_path != nullptr) {+ tap_file_ = std::fopen(tap_path, "wb");+ if (tap_file_ == nullptr) { err = std::string("verify: cannot write STRATA_TAP_FILE ") + tap_path; return false; }+ const int32_t hdr[4] = {0x50415453 /* 'STAP' */, 1, (int32_t) N, (int32_t) nt};+ std::fwrite(hdr, sizeof hdr, 1, tap_file_);+ for (int l : tap_layers_) { const int32_t v = l; std::fwrite(&v, sizeof v, 1, tap_file_); }+ std::fprintf(stderr, "strata: STRATA_TAP_FILE %s: %zu taps of %lld floats per row\n", tap_path, nt, (long long) N);+ }tap_on_ = true;- std::fprintf(stderr, "strata: STRATA_TAP_FILE %s: %zu taps of %lld floats per row\n", tap_path, nt, (long long) N);}if (bal_ != nullptr) { // set_pcie_balance: four stamp words per (layer, group)if (!mapped((size_t) g.n_layers * 2 * 4 * 8, (void**) &h_bal_, (void**) &m_bal_)) {@@ -2179,13 +2187,11 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {device_plan_ ? skip_ + grp : nullptr, ring, cs);stamp(l, 24, grp);if (tap_on_ && !batch_rec_ && l < (int64_t) tap_slot_.size() && tap_slot_[(size_t) l] >= 0) {- // the layer's completed output: R with this FFN half's write applied (the next layer's read would fold it)+ // the layer's completed output: R with this FFN half's write applied (the next layer's read would fold it), the mean of+ // the hc streams, all rows of the group in one launchconst int64_t nt = (int64_t) tap_layers_.size();- for (int t = tb; t < te; ++t) {- cudaMemcpyAsync(tap_scratch_, Rt(t), (size_t) HC * N * 4, cudaMemcpyDeviceToDevice, cs);- gr_write(tap_scratch_, bo_ + t * N, inj2_ + t * HC, gs, tap_scratch_, cs);- hc_mean_streams(tap_scratch_, tap_dev_ + ((size_t) t * nt + (size_t) tap_slot_[(size_t) l]) * N, N, (int) HC, cs);- }+ strata::kernels::df2_tap_gr(Rt(tb), HC * N, bo_ + tb * N, N, inj2_ + tb * HC, HC, tap_dev_ + ((size_t) tb * nt + (size_t) tap_slot_[(size_t) l]) * N,+ nt * N, n, N, (int) HC, cs);}if (l == g.n_layers - 1) {if (!fuse_head_gr) {@@ -2314,7 +2320,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {if (nan_guard_on()) strata::kernels::logits_nonfinite_rows(head_logits_, T, (int) n_vocab_, m_nf_, cs);}stamp(g.n_layers, 1, 0);- if (tap_on_ && !batch_rec_) copy_from_mapped(m_tap_, tap_dev_, (int64_t) T * (int64_t) tap_layers_.size() * N, cs);+ if (tap_on_ && !batch_rec_ && tap_file_ != nullptr) copy_from_mapped(m_tap_, tap_dev_, (int64_t) T * (int64_t) tap_layers_.size() * N, cs);return true;}@@ -3096,7 +3102,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,}}for (int t = 0; t < T; ++t) out[t] = ((volatile int32_t*) h_out_)[t];- if (tap_on_ && !batch_rec_) tap_write(T, tokens, pos0);+ if (tap_file_ != nullptr && !batch_rec_) tap_write(T, tokens, pos0);note_nonfinite(T, nullptr);if (static const bool dbg = std::getenv("STRATA_DBG_NAN") != nullptr; dbg) { // debug: the first non-finite headstatic bool reported = false;diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cunew file mode 100644index 000000000..82c40ad63--- /dev/null+++ b/src/kernels/cuda/dflash2.cu@@ -0,0 +1,412 @@+// src/kernels/cuda/dflash2.cu - the small kernels of the DFlash2 drafter. See include/strata/kernels/dflash2.hpp.+#include "strata/kernels/dflash2.hpp"++#include <cuda_runtime.h>++#include <cfloat>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <stdexcept>+#include <string>++namespace strata::kernels {+namespace {++constexpr int HD = kDf2Hd;++void check(const char* what) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "dflash2 kernel %s: %s\n", what, cudaGetErrorString(e));+ throw std::runtime_error(std::string("dflash2 kernel ") + what + ": " + cudaGetErrorString(e));+ }+}++__device__ __forceinline__ float bf16f(uint16_t h) { return __uint_as_float((uint32_t) h << 16); }++// ---- the grouped dynamic 2-tap convolution over the block's rows+__global__ void conv_kernel(const float* __restrict__ x, const float* __restrict__ dyn, int dyn_stride, const float* __restrict__ base,+ const float* __restrict__ resid, float* __restrict__ out, int rows, int H, int group) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= rows * H) return;+ const int b = i / H, c = i % H;+ const int groups = H / group;+ float acc = 0.0f;+#pragma unroll+ for (int o = 0; o < 2; ++o) {+ if (b - o < 0) continue;+ const float k = base[o * H + c] + dyn[(size_t) b * dyn_stride + o * groups + c / group];+ acc = fmaf(k, x[(size_t) (b - o) * H + c], acc);+ }+ out[i] = resid ? resid[i] + acc : acc;+}++__global__ void rope_table_kernel(const int32_t* __restrict__ pos, int rows, float theta, float* __restrict__ table) {+ const int r = blockIdx.x, i = threadIdx.x; // 64 threads+ if (r >= rows) return;+ const double inv = pow((double) theta, -(double) (2 * i) / (double) HD);+ double ang = (double) pos[r] * inv;+ const double two_pi = 6.283185307179586476925286766559;+ ang -= two_pi * rint(ang / two_pi); // [-pi, pi]+ float s, c;+ sincosf((float) ang, &s, &c);+ table[(r * 2 + 0) * 64 + i] = c;+ table[(r * 2 + 1) * 64 + i] = s;+}++// one block per (row, head), 128 threads: out = rope(rmsnorm(in) * gamma)+__global__ void norm_rope_kernel(const float* __restrict__ in, int in_stride, const float* __restrict__ gamma, const int32_t* __restrict__ pos,+ const float* __restrict__ table, float* __restrict__ out, int slot_mod, int heads, float eps) {+ const int r = blockIdx.x, h = blockIdx.y, d = threadIdx.x;+ __shared__ float red[4];+ __shared__ float y[HD];+ const float x = in[(size_t) r * in_stride + h * HD + d];+ float ss = x * x;+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor_sync(0xffffffffu, ss, o);+ if ((d & 31) == 0) red[d >> 5] = ss;+ __syncthreads();+ const float tot = red[0] + red[1] + red[2] + red[3];+ const float v = x * rsqrtf(tot / (float) HD + eps) * gamma[d];+ float res = v;+ if (table != nullptr) {+ y[d] = v;+ __syncthreads();+ const int i = d & 63;+ const float c = table[(r * 2 + 0) * 64 + i], s = table[(r * 2 + 1) * 64 + i];+ res = d < 64 ? v * c - y[d + 64] * s : v * c + y[d - 64] * s;+ }+ const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;+ out[((size_t) slot * heads + h) * HD + d] = res;+}++__global__ void copy_rows_kernel(const float* __restrict__ in, float* __restrict__ out, const int32_t* __restrict__ pos, int slot_mod, int rows, int cols) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= rows * cols) return;+ const int r = i / cols, c = i % cols;+ const int slot = slot_mod > 0 ? (int) (pos[r] % slot_mod) : r;+ out[(size_t) slot * cols + c] = in[i];+}++// ---- attention. grid (8 K/V heads, kDf2MaxChunks), 256 threads. 32 queries (4 heads x 8 rows) per block share each key tile.+constexpr int KS = 129; // smem row stride of the K / V / Q tiles (conflict-free)+__global__ void __launch_bounds__(256) attn_part_kernel(const float* __restrict__ q, const float* __restrict__ kring, const float* __restrict__ vring,+ const float* __restrict__ kblk, const float* __restrict__ vblk,+ const int32_t* __restrict__ anchor_pos, int ring_slots, int window, float* __restrict__ part) {+ __shared__ float Qs[32 * KS];+ __shared__ float Ks[32 * KS]; // the K tile, then the V tile+ __shared__ float Ss[32 * 33];+ const int kvh = blockIdx.x, chunk = blockIdx.y, tid = threadIdx.x;+ const int s = *anchor_pos;+ const int lo = s - (window - 1) > 0 ? s - (window - 1) : 0;+ const int nctx = s - lo;+ const int nkeys = nctx + kDf2Rows;+ const int qi = tid >> 3, tg = tid & 7;+ const int qrow = qi >> 2, qg = qi & 3;+ float* const pout = part + (((size_t) chunk * kDf2Kv + kvh) * 32 + qi) * 130;+ const int k_begin = chunk * kDf2Chunk;+ if (k_begin >= nkeys) {+ if (tg == 0) { pout[0] = -INFINITY; pout[1] = 0.0f; }+ for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = 0.0f;+ return;+ }+ const float scale = rsqrtf((float) HD);+ for (int i = tid; i < 32 * HD; i += 256) {+ const int a = i / HD, d = i % HD;+ const int row = a >> 2, g = a & 3;+ Qs[a * KS + d] = q[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] * scale;+ }+ float m = -INFINITY, l = 0.0f;+ float O[16];+#pragma unroll+ for (int e = 0; e < 16; ++e) O[e] = 0.0f;+ const int k_end = min(k_begin + kDf2Chunk, nkeys);+ for (int k0 = k_begin; k0 < k_end; k0 += 32) {+ __syncthreads(); // Ks free (previous tile's V reads done), Qs written (first tile)+ for (int i = tid; i < 32 * HD; i += 256) {+ const int t = i / HD, d = i % HD, k = k0 + t;+ float v = 0.0f;+ if (k < nctx) v = kring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];+ else if (k < nkeys) v = kblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];+ Ks[t * KS + d] = v;+ }+ __syncthreads();+ float sc[4] = {0.f, 0.f, 0.f, 0.f};+#pragma unroll 8+ for (int d = 0; d < HD; ++d) {+ const float qv = Qs[qi * KS + d];+#pragma unroll+ for (int u = 0; u < 4; ++u) sc[u] = fmaf(qv, Ks[(tg * 4 + u) * KS + d], sc[u]);+ }+ float tmax = -INFINITY;+#pragma unroll+ for (int u = 0; u < 4; ++u) {+ const int k = k0 + tg * 4 + u;+ bool ok = k < nkeys;+ if (ok && k < nctx) ok = (lo + k) > (s + qrow - window); // s + row - j < window+ sc[u] = ok ? sc[u] : -INFINITY;+ tmax = fmaxf(tmax, sc[u]);+ }+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 1));+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 2));+ tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffffu, tmax, 4));+ const float mn = fmaxf(m, tmax);+ const float corr = (m == -INFINITY) ? 0.0f : __expf(m - mn);+ float psum = 0.0f;+#pragma unroll+ for (int u = 0; u < 4; ++u) {+ const float p = sc[u] == -INFINITY ? 0.0f : __expf(sc[u] - mn);+ Ss[qi * 33 + tg * 4 + u] = p;+ psum += p;+ }+ psum += __shfl_xor_sync(0xffffffffu, psum, 1);+ psum += __shfl_xor_sync(0xffffffffu, psum, 2);+ psum += __shfl_xor_sync(0xffffffffu, psum, 4);+ l = l * corr + psum;+ m = mn;+#pragma unroll+ for (int e = 0; e < 16; ++e) O[e] *= corr;+ __syncthreads(); // Ss complete; every read of Ks done+ for (int i = tid; i < 32 * HD; i += 256) {+ const int t = i / HD, d = i % HD, k = k0 + t;+ float v = 0.0f;+ if (k < nctx) v = vring[((size_t) ((lo + k) % ring_slots) * kDf2Kv + kvh) * HD + d];+ else if (k < nkeys) v = vblk[((size_t) (k - nctx) * kDf2Kv + kvh) * HD + d];+ Ks[t * KS + d] = v;+ }+ __syncthreads();+ for (int t = 0; t < 32; ++t) {+ const float p = Ss[qi * 33 + t];+#pragma unroll+ for (int e = 0; e < 16; ++e) O[e] = fmaf(p, Ks[t * KS + e * 8 + tg], O[e]);+ }+ }+ if (tg == 0) { pout[0] = m; pout[1] = l; }+#pragma unroll+ for (int e = 0; e < 16; ++e) pout[2 + e * 8 + tg] = O[e];+ (void) qg;+}++__global__ void attn_merge_kernel(const float* __restrict__ part, float* __restrict__ out) {+ const int a = blockIdx.x; // kvh * 32 + qi+ const int kvh = a >> 5, qi = a & 31, d = threadIdx.x;+ float M = -INFINITY;+ for (int c = 0; c < kDf2MaxChunks; ++c) M = fmaxf(M, part[(((size_t) c * kDf2Kv + kvh) * 32 + qi) * 130]);+ float L = 0.0f, o = 0.0f;+ for (int c = 0; c < kDf2MaxChunks; ++c) {+ const float* p = part + (((size_t) c * kDf2Kv + kvh) * 32 + qi) * 130;+ const float mc = p[0];+ const float w = (mc == -INFINITY) ? 0.0f : __expf(mc - M);+ L += p[1] * w;+ o = fmaf(p[2 + d], w, o);+ }+ const int row = qi >> 2, g = qi & 3;+ out[((size_t) row * 32 + kvh * kDf2Gqa + g) * HD + d] = L > 0.0f ? o / L : 0.0f;+}++// ---- top-16+constexpr int TK_CHUNK_MAX = 4096;+__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }++__global__ void topk_part_kernel(const float* __restrict__ logits, int n, float* __restrict__ cval, int32_t* __restrict__ cidx) {+ __shared__ float vals[TK_CHUNK_MAX];+ __shared__ float wv[8];+ __shared__ int wi[8];+ __shared__ int win;+ const int row = blockIdx.x, blk = blockIdx.y, tid = threadIdx.x;+ const int cs = (n + kDf2TopBlocks - 1) / kDf2TopBlocks;+ const int c0 = blk * cs, c1 = min(c0 + cs, n);+ for (int i = tid; i < cs; i += 256) vals[i] = (c0 + i < c1) ? logits[(size_t) row * n + c0 + i] : -INFINITY;+ __syncthreads();+ for (int k = 0; k < kDf2TopK; ++k) {+ float bv = -INFINITY;+ int bi = 0x7fffffff;+ for (int i = tid; i < cs; i += 256)+ if (better(vals[i], i, bv, bi)) { bv = vals[i]; bi = i; }+ for (int o = 16; o > 0; o >>= 1) {+ const float ov = __shfl_xor_sync(0xffffffffu, bv, o);+ const int oi = __shfl_xor_sync(0xffffffffu, bi, o);+ if (better(ov, oi, bv, bi)) { bv = ov; bi = oi; }+ }+ if ((tid & 31) == 0) { wv[tid >> 5] = bv; wi[tid >> 5] = bi; }+ __syncthreads();+ if (tid == 0) {+ float v = wv[0];+ int ix = wi[0];+ for (int w = 1; w < 8; ++w)+ if (better(wv[w], wi[w], v, ix)) { v = wv[w]; ix = wi[w]; }+ const size_t o = ((size_t) row * kDf2TopBlocks + blk) * kDf2TopK + k;+ cval[o] = v;+ cidx[o] = v == -INFINITY ? -1 : c0 + ix;+ win = ix;+ if (v != -INFINITY) vals[ix] = -INFINITY;+ }+ __syncthreads();+ }+}++__global__ void topk_merge_kernel(const float* __restrict__ cval, const int32_t* __restrict__ cidx, const int32_t* __restrict__ idx_map,+ float* __restrict__ val, int32_t* __restrict__ tok) {+ constexpr int NC = kDf2TopBlocks * kDf2TopK;+ __shared__ float sv[NC];+ __shared__ int si[NC];+ __shared__ float wv[8];+ __shared__ int wi[8], wp[8];+ const int row = blockIdx.x, tid = threadIdx.x;+ for (int i = tid; i < NC; i += 256) { sv[i] = cval[(size_t) row * NC + i]; si[i] = cidx[(size_t) row * NC + i]; }+ __syncthreads();+ for (int k = 0; k < kDf2TopK; ++k) {+ float bv = -INFINITY;+ int bi = 0x7fffffff, bp = -1;+ for (int i = tid; i < NC; i += 256)+ if (si[i] >= 0 && better(sv[i], si[i], bv, bi)) { bv = sv[i]; bi = si[i]; bp = i; }+ for (int o = 16; o > 0; o >>= 1) {+ const float ov = __shfl_xor_sync(0xffffffffu, bv, o);+ const int oi = __shfl_xor_sync(0xffffffffu, bi, o);+ const int op = __shfl_xor_sync(0xffffffffu, bp, o);+ if (better(ov, oi, bv, bi)) { bv = ov; bi = oi; bp = op; }+ }+ if ((tid & 31) == 0) { wv[tid >> 5] = bv; wi[tid >> 5] = bi; wp[tid >> 5] = bp; }+ __syncthreads();+ if (tid == 0) {+ float v = wv[0];+ int ix = wi[0], p = wp[0];+ for (int w = 1; w < 8; ++w)+ if (better(wv[w], wi[w], v, ix)) { v = wv[w]; ix = wi[w]; p = wp[w]; }+ if (p >= 0) { si[p] = -1; }+ val[row * kDf2TopK + k] = v;+ tok[row * kDf2TopK + k] = p >= 0 ? (idx_map ? idx_map[ix] : ix) : 0;+ }+ __syncthreads();+ }+}++// ---- the selector's walk: one block of 256 threads+__global__ void select_kernel(const float* __restrict__ hp, const float* __restrict__ unary, const int32_t* __restrict__ tok,+ const uint16_t* __restrict__ pred_cb, const uint16_t* __restrict__ succ_cb, int rank, int rows,+ const int32_t* __restrict__ anchor_tok, int32_t* __restrict__ out_tok, float* __restrict__ out_prob) {+ __shared__ float vp[256];+ __shared__ float sc[kDf2TopK];+ __shared__ int prev;+ const int tid = threadIdx.x, w = tid >> 5, lane = tid & 31;+ if (tid == 0) prev = *anchor_tok;+ __syncthreads();+ for (int i = 0; i < rows; ++i) {+ vp[tid] = bf16f(pred_cb[(size_t) prev * rank + tid]) * hp[(size_t) i * rank + tid];+ __syncthreads();+ for (int j = w * 2; j < w * 2 + 2; ++j) {+ const int t = tok[i * kDf2TopK + j];+ const uint4 raw = *reinterpret_cast<const uint4*>(succ_cb + (size_t) t * rank + lane * 8);+ const uint32_t u[4] = {raw.x, raw.y, raw.z, raw.w};+ float acc = 0.0f;+#pragma unroll+ for (int e = 0; e < 4; ++e) {+ acc = fmaf(__uint_as_float(u[e] << 16), vp[lane * 8 + e * 2], acc);+ acc = fmaf(__uint_as_float(u[e] & 0xffff0000u), vp[lane * 8 + e * 2 + 1], acc);+ }+ for (int o = 16; o > 0; o >>= 1) acc += __shfl_xor_sync(0xffffffffu, acc, o);+ if (lane == 0) sc[j] = unary[i * kDf2TopK + j] + acc;+ }+ __syncthreads();+ if (tid == 0) {+ int bj = 0;+ float bv = sc[0];+ for (int j = 1; j < kDf2TopK; ++j)+ if (sc[j] > bv) { bv = sc[j]; bj = j; }+ float z = 0.0f;+ for (int j = 0; j < kDf2TopK; ++j) z += __expf(sc[j] - bv);+ out_tok[i] = tok[i * kDf2TopK + bj];+ out_prob[i] = 1.0f / z;+ prev = out_tok[i];+ }+ __syncthreads();+ }+}++__global__ void tap_rows_kernel(const float* __restrict__ R, float* __restrict__ out, int64_t out_stride, int64_t n, int hc) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ const int r = blockIdx.y;+ if (i >= n) return;+ float s = 0.0f;+ for (int c = 0; c < hc; ++c) s += R[((size_t) r * hc + c) * n + i];+ out[(size_t) r * out_stride + i] = s / (float) hc;+}++__global__ void tap_gr_kernel(const float* __restrict__ R, int64_t R_stride, const float* __restrict__ bo, int64_t bo_stride,+ const float* __restrict__ inject, int64_t inj_stride, float* __restrict__ out, int64_t out_stride, int64_t n, int hc) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ const int r = blockIdx.y;+ if (i >= n) return;+ float wsum = 0.0f, rs = 0.0f;+ for (int c = 0; c < hc; ++c) {+ const float x = inject[(size_t) r * inj_stride + c] / (float) hc;+ wsum += 2.0f / (1.0f + expf(-x));+ rs += R[(size_t) r * R_stride + (size_t) c * n + i];+ }+ out[(size_t) r * out_stride + i] = (rs + bo[(size_t) r * bo_stride + i] * wsum) / (float) hc;+}++} // namespace++void df2_tap_gr(const float* R, int64_t R_stride, const float* bo, int64_t bo_stride, const float* inject, int64_t inj_stride, float* out, int64_t out_stride,+ int rows, int64_t n_embd, int hc, void* stream) {+ tap_gr_kernel<<<dim3((unsigned) ((n_embd + 255) / 256), rows), 256, 0, (cudaStream_t) stream>>>(R, R_stride, bo, bo_stride, inject, inj_stride, out,+ out_stride, n_embd, hc);+ check("tap_gr");+}++void df2_conv(const float* x, const float* dyn, int dyn_stride, const float* base, const float* resid, float* out, int rows, int H, int group,+ void* stream) {+ const int n = rows * H;+ conv_kernel<<<(n + 255) / 256, 256, 0, (cudaStream_t) stream>>>(x, dyn, dyn_stride, base, resid, out, rows, H, group);+ check("conv");+}++void df2_rope_table(const int32_t* pos, int rows, float theta, float* table, void* stream) {+ rope_table_kernel<<<rows, 64, 0, (cudaStream_t) stream>>>(pos, rows, theta, table);+ check("rope_table");+}++void df2_norm_rope(const float* in, int in_stride, const float* gamma, const int32_t* pos, const float* rope_table, float* out, int slot_mod,+ int rows, int heads, float eps, void* stream) {+ norm_rope_kernel<<<dim3(rows, heads), HD, 0, (cudaStream_t) stream>>>(in, in_stride, gamma, pos, rope_table, out, slot_mod, heads, eps);+ check("norm_rope");+}++void df2_copy_rows(const float* in, float* out, const int32_t* pos, int slot_mod, int rows, int cols, void* stream) {+ const int n = rows * cols;+ copy_rows_kernel<<<(n + 255) / 256, 256, 0, (cudaStream_t) stream>>>(in, out, pos, slot_mod, rows, cols);+ check("copy_rows");+}++void df2_attention(const float* q, const float* kring, const float* vring, const float* kblk, const float* vblk, const int32_t* anchor_pos,+ int ring_slots, int window, float* part, float* out, void* stream) {+ attn_part_kernel<<<dim3(kDf2Kv, kDf2MaxChunks), 256, 0, (cudaStream_t) stream>>>(q, kring, vring, kblk, vblk, anchor_pos, ring_slots, window, part);+ check("attn_part");+ attn_merge_kernel<<<kDf2Kv * 32, HD, 0, (cudaStream_t) stream>>>(part, out);+ check("attn_merge");+}++void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream) {+ if ((n + kDf2TopBlocks - 1) / kDf2TopBlocks > TK_CHUNK_MAX) throw std::invalid_argument("df2_topk: too many logits per row");+ float* cval = reinterpret_cast<float*>(scratch);+ int32_t* cidx = reinterpret_cast<int32_t*>(scratch + (size_t) rows * kDf2TopBlocks * kDf2TopK);+ topk_part_kernel<<<dim3(rows, kDf2TopBlocks), 256, 0, (cudaStream_t) stream>>>(logits, n, cval, cidx);+ check("topk_part");+ topk_merge_kernel<<<rows, 256, 0, (cudaStream_t) stream>>>(cval, cidx, idx_map, val, tok);+ check("topk_merge");+}++void df2_select(const float* hp, const float* unary, const int32_t* tok, const uint16_t* pred_cb, const uint16_t* succ_cb, int rank, int rows,+ const int32_t* anchor_tok, int32_t* out_tok, float* out_prob, void* stream) {+ if (rank != 256) throw std::invalid_argument("df2_select: the selector rank must be 256");+ select_kernel<<<1, 256, 0, (cudaStream_t) stream>>>(hp, unary, tok, pred_cb, succ_cb, rank, rows, anchor_tok, out_tok, out_prob);+ check("select");+}++void df2_tap_rows(const float* R, float* out, int64_t out_stride, int64_t n_embd, int hc, int rows, void* stream) {+ tap_rows_kernel<<<dim3((unsigned) ((n_embd + 255) / 256), rows), 256, 0, (cudaStream_t) stream>>>(R, out, out_stride, n_embd, hc);+ check("tap_rows");+}++} // namespace strata::kernelsdiff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex c9b453540..6cb888e50 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1,6 +1,9 @@// src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.#include "strata/kernels/fused_gr.hpp"#include "strata/prefill/prefill.hpp"+#if defined(STRATA_HAVE_DFLASH2)+#include "strata/kernels/dflash2.hpp"+#endif#include "strata/prefill/warm_policy.hpp"#include <atomic>#if defined(__cpp_lib_atomic_wait)@@ -1051,7 +1054,19 @@ strata::kernels::QsaAttnPools pools_of(const strata::kernels::KvHostPools& h, co} // namespacePrefill::Prefill() : impl_(new Impl) {}-Prefill::~Prefill() { release(); }+Prefill::~Prefill() {+ clear_taps();+ release();+}++void Prefill::clear_taps() {+ if (tap_dev_ != nullptr) {+ cudaFree(tap_dev_);+ tap_dev_ = nullptr;+ }+ tap_layers_.clear();+ on_taps = nullptr;+}void Prefill::reset() {release();@@ -2680,6 +2695,20 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striint ple_buf = 0;if (hand_in_ == nullptr) stopped_at_ = -1;+#if defined(STRATA_HAVE_DFLASH2)+ std::vector<int> tap_slot;+ if (!tap_layers_.empty() && stage_lb_ == 0 && (stage_le_ < 0 || stage_le_ == g.n_layers)) {+ if (tap_dev_ == nullptr &&+ cudaMalloc((void**) &tap_dev_, (size_t) tap_cap_ * tap_layers_.size() * (size_t) g.n_embd * 4) != cudaSuccess) {+ (void) cudaGetLastError();+ tap_dev_ = nullptr;+ err = "prefill: the DFlash2 tap staging does not fit in VRAM";+ return false;+ }+ tap_slot.assign((size_t) g.n_layers, -1);+ for (size_t i = 0; i < tap_layers_.size(); ++i) tap_slot[(size_t) tap_layers_[i]] = (int) i;+ }+#endiffor (int64_t c0 = 0; c0 < n; c0 += C) {if (should_stop && should_stop()) { err = "cancelled"; return false; }// the first stage: end the run before this chunk (every chunk before it is handed on; the caller's drain@@ -2691,6 +2720,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striif (std::getenv("STRATA_TRACE")) { std::fprintf(stderr, "strata trace: prompt chunk %lld of %lld\n", (long long) c0, (long long) n); std::fflush(stderr); }const int64_t T = std::min(C, n - c0), p0 = pos0 + c0;core::progress_at("reading the prompt (batched): preparing the chunk from token", p0); // #251+#if defined(STRATA_HAVE_DFLASH2)+ // DFlash2 prompt taps: the rows of this chunk at positions >= tap_first_ (the drafter looks back one window)+ const int64_t tap_t0 = tap_dev_ != nullptr ? std::min<int64_t>(T, std::max<int64_t>(0, tap_first_ - p0)) : T;+ const int64_t tap_rows = T - tap_t0;+ if (tap_rows > tap_cap_) { err = "prefill: the DFlash2 tap staging is smaller than the chunk's tapped rows"; return false; }+#else+ const int64_t tap_t0 = 0, tap_rows = 0;+#endif++stats_.chunks;pt.mark(kPfStart, cs);const auto tsetup = Clock::now();@@ -4836,6 +4873,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri}if (steered && !cv_fused) // --control-vector-scaledstrata::kernels::cvec_apply(m.R, l, T, D, nullptr, 0, nullptr, 0, false, m.cs);+#if defined(STRATA_HAVE_DFLASH2)+ if (half == 1 && tap_rows > 0 && tap_slot[(size_t) l] >= 0) // DFlash2: this layer's completed output, mean of the streams+ strata::kernels::df2_tap_rows(m.R + (size_t) tap_t0 * D, tap_dev_ + (size_t) tap_slot[(size_t) l] * N, (int64_t) tap_layers_.size() * N, N,+ (int) HC, (int) tap_rows, m.cs);+#endif}}if (!ple_land()) return false; // a stage that ends before layer 1: the rows land anyway, the next gather starts@@ -4933,6 +4975,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stristd::fclose(f);}}+#if defined(STRATA_HAVE_DFLASH2)+ if (tap_rows > 0 && on_taps) {+ if (cudaStreamSynchronize(m.cs) != cudaSuccess) { err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError()); return false; }+ if (!on_taps(tap_dev_, tap_rows, p0 + tap_t0, err)) return false;+ }+#endifif (on_chunk || on_stage_chunk) {const auto toc = Clock::now();if (cudaStreamSynchronize(m.cs) != cudaSuccess) {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex d7116d2df..4f75c9fe1 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -66,6 +66,9 @@#include "strata/core/native_head.hpp"#include "strata/core/verify.hpp"#include "strata/core/mtp.hpp"+#if defined(STRATA_HAVE_DFLASH2)+#include "strata/core/dflash2.hpp"+#endif#include "strata/prefill/prefill.hpp"#if defined(STRATA_PREFILL_MMQ)#include "strata/prefill/moe_mmq.hpp"@@ -781,6 +784,9 @@ struct Options {std::string mtp_q4; ///< "" (off), all, proj (the projections only) or head (the draft head only)/// --mtp-draft-vocab FILE (opt-in): the draft head's token subset from FILE instead of <mtp>/draft_vocab.binstd::string mtp_draft_vocab;+ /// --draft dflash2 --draft-model PACK (opt-in, CUDA / HIP): the DFlash2 block drafter for Qwen3.8-Flash-Next (tools/dflash2_pack.py) drafts instead+ /// of the MTP layer, which stays loaded as the fallback. Needs --mtp, --spec and a native pack; a missing pack falls back to the MTP.+ std::string draft, draft_model;/// A control vector on the residual stream (strata/kernels/cvec.hpp), with llama.cpp's flags: the/// `experimental-speed-projection` profile passes `--control-vector-scaled FILE:1.0 --control-vector-layer-range/// 4 44 --cvec-mode project --cvec-dir per-layer`. None by default; --serve switches a loaded one per request.@@ -933,6 +939,8 @@ void usage() {" (default) or one per stream (llama.cpp's MTP graph)\n"" --mtp-q4 all|proj|head opt-in: the draft layer's projections and/or draft head as 4-bit copies\n"" --mtp-draft-vocab FILE opt-in: the draft head's token subset (default <mtp>/draft_vocab.bin)\n"+ " --draft mtp|dflash2 the draft source (default mtp); dflash2 needs --draft-model PACK (tools/dflash2_pack.py)\n"+ " and keeps the MTP layer loaded as the fallback (opt-in, Flash-Next, CUDA / HIP)\n"" --control-vector-scaled FILE:SCALE[,...] a control vector GGUF on the residual stream (llama.cpp's\n"" format; --control-vector FILE = scale 1). --serve: requests switch it (cvec=0|1)\n"" --control-vector-layer-range A B the layers it follows (inclusive; default 1 .. the last)\n"@@ -2672,6 +2680,14 @@ int main(int argc, char** argv) {}}else if (a == "--mtp-draft-vocab") o.mtp_draft_vocab = next("--mtp-draft-vocab");+ else if (a == "--draft") {+ o.draft = next("--draft");+ if (o.draft != "dflash2" && o.draft != "mtp") {+ std::fprintf(stderr, "strata generate: --draft takes mtp or dflash2\n");+ std::exit(2);+ }+ }+ else if (a == "--draft-model") o.draft_model = next("--draft-model");else if (a == "--control-vector") o.cvec_files.push_back({next("--control-vector"), 1.0f});else if (a == "--control-vector-scaled") {// FILE:SCALE, comma-separated; the LAST colon splits, so a Windows path (C:\...) keeps its drive@@ -5395,6 +5411,10 @@ int main(int argc, char** argv) {// Secure MTP's CUDA0 allocations before the large host arena is registered with both CUDA contexts.// In particular WDDM can refuse the draft weights after mapping tens of GiB of host pages.strata::core::MtpDrafter mtp;+#if defined(STRATA_HAVE_DFLASH2)+ strata::core::Dflash2Drafter df2; // --draft dflash2 (CLI spec loop only)+ bool use_df2 = false;+#endif// the draft layer's geometry (the canonical model's MTP head), `static` because MtpDrafter keeps a reference; the batch// slots' draft-KV copies use this one toostatic const strata::core::ModelGeometry draft_geometry{};@@ -5418,6 +5438,23 @@ int main(int argc, char** argv) {if (o.pipeline_windows >= 2) mtp.set_force_capture(true);if (!o.mtp.empty() && !mtp.load(o.mtp, draft_geometry, last_st && !mtp_first ? last_st->ss : ss, mtp_t, err, o.mtp_window)) { std::fprintf(stderr, "strata generate: %s%s\n", err.c_str(), vram_free_note().c_str()); return 1; }mtp.set_ple_session(&ss);+#if defined(STRATA_HAVE_DFLASH2)+ if (o.draft == "dflash2") {+ const char* why = o.mtp.empty() ? "it needs --mtp (the fallback) and --spec 2 or more"+ : !native_pack ? "it needs a native pack"+ : multi_gpu || o.batch > 0 || o.serve ? "it is for one card and the CLI run (not --serve, --batch or a layer split) in this build"+ : o.draft_model.empty() ? "it needs --draft-model PACK (tools/dflash2_pack.py)" : nullptr;+ if (why == nullptr) {+ if (df2.load(o.draft_model, 0, err)) {+ use_df2 = true;+ } else {+ std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());+ }+ } else {+ std::fprintf(stderr, "strata generate: --draft dflash2 is off: %s; the MTP drafts instead\n", why);+ }+ }+#endifif (batch_mtp) {// Slot b's drafter lives where the solo drafter does (the guard above: the last stage's GPU), carved from the// LAST stage's slot-b session (bslot_ss.back(): stage 0's on one GPU): it borrows that session's context@@ -15636,6 +15673,13 @@ int main(int argc, char** argv) {return e.empty() && mtp.prefill(R_rows, nxt.data(), T, p0, e);};}+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2) { // the drafter's context: the last 2047 prompt rows' taps+ df2.reset();+ prefill.set_taps(df2.tap_layers(), std::max<int64_t>(0, n_prompt - 2047), 2048);+ prefill.on_taps = [&](const float* taps, int64_t rows, int64_t pos0, std::string& e) -> bool { return df2.context(taps, rows, pos0, e); };+ }+#endifif (!kvg_started) { // the elastic K/V: every cell this run can reach (no loan)kvg_started = true;kvg_start(xcache.slots());@@ -15672,6 +15716,9 @@ int main(int argc, char** argv) {std::fprintf(stderr, "strata generate: %zu lent slots refilled in %.1f ms\n", lent.size(),std::chrono::duration<double, std::milli>(Clock::now() - tr).count());}+#if defined(STRATA_HAVE_DFLASH2)+ prefill.clear_taps();+#endifprefill_batched_ms = std::chrono::duration<double, std::milli>(Clock::now() - tp0).count();prefill_ms += prefill_batched_ms;pos_start = n_batched;@@ -15959,6 +16006,9 @@ int main(int argc, char** argv) {vh.slot_off = xcache.slot_offsets(); // E-6: the device plan's pointersvh.n_slots = xcache.slots();if (pcie_balance()) ver.set_pcie_balance(&drive.d.pcie_model);+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2) ver.set_taps(df2.tap_layers());+#endifif (!ver.init(wt, g, ss, vh, native_head.loaded() ? &native_head : nullptr, o.spec, err)) {std::fprintf(stderr, "strata generate: %s\n", err.c_str());return 1;@@ -15968,6 +16018,17 @@ int main(int argc, char** argv) {std::fprintf(stderr, "strata generate: %s\n", err.c_str());return 1;}+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2 && !ver.tapping()) {+ std::fprintf(stderr, "strata generate: --draft dflash2: the verifier keeps no taps here; the MTP drafts instead\n");+ use_df2 = false;+ }+ if (use_df2 && !df2.bind(wt, &native_head, o.mtp_draft_vocab.empty() ? o.mtp + "/draft_vocab.bin" : o.mtp_draft_vocab, err)) {+ std::fprintf(stderr, "strata generate: --draft dflash2: %s; the MTP drafts instead\n", err.c_str());+ use_df2 = false;+ }+ if (use_df2) std::fprintf(stderr, "strata generate: DFlash2 drafter on (%s), the MTP layer stays as the fallback\n", o.draft_model.c_str());+#endifmem_mark("the verifier and the drafter's binding");ver.set_sampling(sp); // the CLI's own sampling (until 0.1.19 this loop was always greedy); no penalties hereif (use_mtp) mtp.set_draft_sampling(sp); // STRATA_SPEC_COUPLED=1: sampled drafts (a no-op otherwise)@@ -16263,6 +16324,19 @@ int main(int argc, char** argv) {}}).detach();}+ // the next round's drafts after a commit: the MTP chain, or the DFlash2 block from the committed rows' taps+ auto draft_next = [&](int T_, int a_) -> bool {+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2) {+ int32_t dd[strata::core::Dflash2Drafter::kDrafts];+ float pp[strata::core::Dflash2Drafter::kDrafts];+ if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;+ for (size_t i = 0; i < drafts.size() && i < (size_t) strata::core::Dflash2Drafter::kDrafts; ++i) { drafts[i] = dd[i]; dprob[i] = pp[i]; }+ return true;+ }+#endif+ return mtp.draft(T_, outv.data(), p, a_, drafts.data(), err, dprob.data(), (float) o.spec_min_p);+ };while ((int64_t) produced.size() < max_new) {const Clock::time_point t0 = Clock::now();strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());@@ -16403,8 +16477,7 @@ int main(int argc, char** argv) {total_ms += std::chrono::duration<double, std::milli>(Clock::now() - t0).count();break;}- const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new ||- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) o.spec_min_p);+ const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new || draft_next(T, a);if (adapt_thr.joinable()) adapt_thr.join();if (!adapt_ok) return 1;if (!drafted) {@@ -16528,6 +16601,12 @@ int main(int argc, char** argv) {1000 * pm.g0, 1000 * pm.g);}(void) pool_ms0;+#if defined(STRATA_HAVE_DFLASH2)+ if (use_df2 && df2.rounds > 0)+ std::printf("%-24s %.3f ms/round drafting (%lld rounds, GPU %.3f), %.0f MiB of VRAM\n", "dflash2", df2.ms_draft / (double) df2.rounds,+ (long long) df2.rounds, df2.ms_gpu / (double) df2.rounds, (double) df2.vram_bytes() / 1048576.0);+ else+#endifif (use_mtp && rounds > 0)std::printf("%-24s %.3f ms/round drafting (%lld rounds), MTP prompt %.1f ms, %.0f MiB of VRAM\n", "mtp",mtp.ms_draft / (double) mtp.rounds, (long long) mtp.rounds, mtp.ms_prefill,diff --git a/tools/dflash2/mk_parity_case.py b/tools/dflash2/mk_parity_case.pynew file mode 100644index 000000000..a812270c4--- /dev/null+++ b/tools/dflash2/mk_parity_case.py@@ -0,0 +1,88 @@+"""Builds the input of src/core/dflash2_parity.cpp and the torch reference answer.++ python mk_parity_case.py <drafter.safetensors> <drafter_config.json> <out.bin> [n_ctx=2300] [vp=20000] [seed=1]++The case: n_ctx random tap rows (positions 0 .. n_ctx-1), an anchor token at position n_ctx, a head of `vp` random rows (Q8_0, the draft vocabulary+subset path: dvocab maps a row to a token id) and the target embedding rows of the anchor and the mask token. The torch port (dflash2_ref.py,+equal to z-lab's module) gives the reference: the final normalized block rows, the walk's tokens and their probabilities. A context longer than the+2048-token window exercises the ring wrap and the window mask.+"""+import struct+import sys+from pathlib import Path++import numpy as np+import torch++sys.path.insert(0, str(Path(__file__).parent))+import dflash2_ref as R+++class Embed:+ def __init__(self, rows):+ self.rows = rows++ def __getitem__(self, ids):+ return torch.stack([self.rows[int(i)] for i in ids])+++def quant_q8_0(w):+ rows, cols = w.shape+ b = w.reshape(rows, cols // 32, 32)+ amax = np.abs(b).max(axis=-1)+ d = amax / 127.0+ inv = np.where(d != 0, 1.0 / np.where(d == 0, 1, d), 0.0)+ q = np.rint(b * inv[..., None]).astype(np.int8)+ out = np.empty((rows, cols // 32, 34), np.uint8)+ out[..., :2] = d.astype(np.float16).view(np.uint8).reshape(rows, cols // 32, 2)+ out[..., 2:] = q.view(np.uint8)+ return out.tobytes(), (q.astype(np.float32) * d.astype(np.float16).astype(np.float32)[..., None]).reshape(rows, cols)+++def main():+ st, cfgp, out = sys.argv[1:4]+ n_ctx = int(sys.argv[4]) if len(sys.argv) > 4 else 2300+ vp = int(sys.argv[5]) if len(sys.argv) > 5 else 20000+ seed = int(sys.argv[6]) if len(sys.argv) > 6 else 1+ rng = np.random.default_rng(seed)+ H, V = 2560, 248320+ mask_id = 248077+ anchor = int(rng.integers(1000, 200000))+ taps = (rng.standard_normal((n_ctx, 5 * H)) * 1.0).astype(np.float32)+ emb = (rng.standard_normal((2, H)) * 1.0).astype(np.float32)+ dvocab = np.sort(rng.choice(V, size=vp, replace=False)).astype(np.int32)+ head = (rng.standard_normal((vp, H)) * 0.05).astype(np.float32)+ head_blob, head_dq = quant_q8_0(head)+ embed = Embed({anchor: torch.from_numpy(emb[0]), mask_id: torch.from_numpy(emb[1])})+ d = R.Drafter(st, cfgp, embed, torch.from_numpy(head_dq), threads=6)+ d.set_context(torch.from_numpy(taps))+ path, _ = d.draft(n_ctx, anchor, 8) # runs the forward; the tail below redoes the selector with the token map+ fn = d.last_all.clone() # [8][H]+ out_rows = fn[1:]+ logits = out_rows @ torch.from_numpy(head_dq).T+ unary, idx = torch.topk(logits, 16, dim=-1)+ cand = torch.from_numpy(dvocab.astype(np.int64))[idx]+ hp = out_rows @ d.w["candidate_selector.hidden_projection.weight"].T+ A, B = d.w["candidate_selector.predecessor_codebook"], d.w["candidate_selector.successor_codebook"]+ pred = anchor+ toks, probs = [], []+ for i in range(7):+ sc = unary[i] + (B[cand[i]].float() @ (A[pred].float() * hp[i]))+ j = int(torch.argmax(sc))+ probs.append(float(torch.softmax(sc, dim=-1)[j]))+ pred = int(cand[i][j])+ toks.append(pred)+ with open(out, "wb") as f:+ f.write(struct.pack("<4i", n_ctx, anchor, vp, 0))+ f.write(taps.tobytes())+ f.write(emb.tobytes())+ f.write(dvocab.tobytes())+ f.write(head_blob)+ f.write(struct.pack("<7i", *toks))+ f.write(struct.pack("<7f", *probs))+ f.write(fn.numpy().astype(np.float32).tobytes())+ print("case written:", out, "n_ctx", n_ctx, "anchor", anchor, "tokens", toks, "probs", [round(p, 3) for p in probs])+++if __name__ == "__main__":+ main()diff --git a/tools/dflash2_pack.py b/tools/dflash2_pack.pynew file mode 100644index 000000000..8949c1489--- /dev/null+++ b/tools/dflash2_pack.py@@ -0,0 +1,127 @@+#!/usr/bin/env python3+"""Pack the incoai DFlash2 drafter for Qwen3.8-Flash-Next (safetensors) into the engine's runtime file.++ python tools/dflash2_pack.py model.safetensors config.json out.pack [--bf16]++Format (little endian): "DF2P", u32 version (1), u32 index_len, a text index (padded with spaces so the data starts on a 4096 multiple), then+the tensors, each 256-byte aligned at its offset (from the end of the index). Index lines: "cfg <key> <value>", "t <name> <dtype> <rows>+<cols> <offset> <bytes>" (a matrix [out][in] is rows x cols; a vector or codebook row set is the leading dims folded into rows), "end".+dtypes: "q8_0" (ggml Q8_0 blocks, 34 bytes per 32 values, rows of the weight matrix [out][in] one after the other: what the engine's+multi-column MMVQ reads), "f32" (norms, the two conv base kernels), "bf16" (the selector's two codebooks: only gathered rows are read).+The matrices' quantization matches ggml's quantize_row_q8_0 (d = amax / 127, q = round(x / d), d stored as fp16); the research harness+(tools/dflash2/q8test.py) measured no acceptance loss from it.+"""+import json+import struct+import sys++import numpy as np++try:+ from safetensors import safe_open+except ImportError: # pragma: no cover+ sys.exit("needs the safetensors package")+++def load_tensor(f, name):+ """-> float32 ndarray (bf16 tensors are widened exactly)"""+ return f.get_tensor(name).float().numpy()+++def raw_bf16(f, name):+ import torch+ t = f.get_tensor(name)+ return np.ascontiguousarray(t.view(torch.int16).numpy().view(np.uint16))+++def quant_q8_0(w):+ """w: float32 [rows, cols], cols % 32 == 0 -> bytes of rows * cols/32 blocks (fp16 d, 32 int8)"""+ rows, cols = w.shape+ assert cols % 32 == 0+ b = w.reshape(rows, cols // 32, 32)+ amax = np.abs(b).max(axis=-1)+ d = amax / 127.0+ inv = np.where(d != 0, 1.0 / np.where(d == 0, 1, d), 0.0)+ q = np.rint(b * inv[..., None]).astype(np.int8) # round half to even; ggml uses roundf (ties are measure-zero for these weights)+ out = np.empty((rows, cols // 32, 34), np.uint8)+ out[..., :2] = d.astype(np.float16).view(np.uint8).reshape(rows, cols // 32, 2)+ out[..., 2:] = q.view(np.uint8)+ return out.tobytes()+++def main():+ if len(sys.argv) < 4:+ sys.exit(__doc__)+ st_path, cfg_path, out_path = sys.argv[1:4]+ cfg = json.load(open(cfg_path))+ dc = cfg["dflash_config"]+ f = safe_open(st_path, "pt")+ tensors = []+ blobs = []++ def add(name, dtype, shape, data):+ tensors.append({"name": name, "dtype": dtype, "shape": list(shape), "bytes": len(data)})+ blobs.append(data)++ def mat(name, new=None):+ w = load_tensor(f, name)+ assert w.ndim == 2+ add(new or name, "q8_0", w.shape, quant_q8_0(w))++ def vec(name, new=None):+ w = load_tensor(f, name)+ add(new or name, "f32", w.shape, np.ascontiguousarray(w, np.float32).tobytes())++ L = cfg["num_hidden_layers"]+ mat("fc.weight")+ vec("hidden_norm.weight")+ vec("norm.weight")+ for l in range(L):+ p = "layers.%d." % l+ for n in ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj",+ "mlp.down_proj", "attention_conv.kernel_projection", "mlp_conv.kernel_projection"):+ mat(p + n + ".weight")+ for n in ("input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm"):+ vec(p + n + ".weight")+ vec(p + "attention_conv.base_kernel")+ vec(p + "mlp_conv.base_kernel")+ mat("candidate_selector.hidden_projection.weight")+ for n in ("predecessor_codebook", "successor_codebook"):+ a = raw_bf16(f, "candidate_selector." + n)+ add("candidate_selector." + n, "bf16", a.shape, a.tobytes())++ meta = {+ "hidden": cfg["hidden_size"], "layers": L, "heads": cfg["num_attention_heads"], "kv_heads": cfg["num_key_value_heads"],+ "head_dim": cfg["head_dim"], "inter": cfg["intermediate_size"], "window": cfg["sliding_window"], "rope_theta": cfg["rope_parameters"]["rope_theta"],+ "eps": cfg["rms_norm_eps"], "block": dc["block_size"], "conv_group": dc["conv_group_size"], "conv_kernel": dc["conv_kernel_size"],+ "mask_token": dc["mask_token_id"], "rank": dc["selector_rank"], "top_k": dc["selector_top_k"], "taps": dc["target_layer_ids"],+ "vocab": cfg["vocab_size"], "source": "incoai/Qwen3.8-Flash-Next-DFlash2 a22a1744",+ }+ # offsets: relative to the end of the (padded) index; 256-byte aligned+ off = 0+ for t in tensors:+ off = (off + 255) & ~255+ t["offset"] = off+ off += t["bytes"]+ lines = []+ for k, v in meta.items():+ if k == "taps":+ v = ",".join(str(x) for x in v)+ lines.append("cfg %s %s" % (k, str(v).replace(" ", "_")))+ for t in tensors:+ sh = t["shape"] + [1] * (2 - len(t["shape"])) if len(t["shape"]) < 2 else t["shape"]+ lines.append("t %s %s %d %d %d %d" % (t["name"], t["dtype"], int(np.prod(sh[:-1])), sh[-1], t["offset"], t["bytes"]))+ lines.append("end")+ js = ("\n".join(lines) + "\n").encode()+ js += b" " * ((-(len(js) + 12)) % 4096)+ with open(out_path, "wb") as o:+ o.write(b"DF2P" + struct.pack("<II", 1, len(js)) + js)+ base = o.tell()+ for t, b in zip(tensors, blobs):+ o.seek(base + t["offset"])+ o.write(b)+ print("wrote %s: %d tensors, %.1f MB" % (out_path, len(tensors), (base + off) / 1e6))+++if __name__ == "__main__":+ main()