Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dflash2: drop the cache-touch kernel (no gain), wait-commit measurement switch
Niko12216f83fe2c4283
diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hppindex bad2bf5e9..205bff3d0 100644--- a/include/strata/core/dflash2.hpp+++ b/include/strata/core/dflash2.hpp@@ -124,9 +124,6 @@ private:float* xdc_ = nullptr;cudaStream_t cs2_ = nullptr; ///< the context update's branch of the round graphcudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr, ev_ctx_[8] = {};- const float* touch_p_[48] = {};- int touch_c_[48] = {};- int touch_n_ = 0;bool fork_ = false; ///< recording the forked graph: layer l's attention waits for ev_ctx_[l]float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hppindex 286174981..bdcf0cbe4 100644--- a/include/strata/kernels/dflash2.hpp+++ b/include/strata/kernels/dflash2.hpp@@ -76,9 +76,6 @@ void df2_norm_rope_qk(const float* q, const float* k, const float* gq, const flovoid df2_kv_ring(const float* kin, const float* vin, const float* gk, const int32_t* pos, const float* table, float* kring, float* vring, int slots, int rows,int heads, float eps, void* stream);-/// Reads n small f32 tensors (ptrs[i], counts[i] floats; n <= 48) once to bring them into the caches; sink: one float of scratch.-void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream);-/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id./// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cppindex 3da70987d..5acd76832 100644--- a/src/core/dflash2.cpp+++ b/src/core/dflash2.cpp@@ -208,12 +208,6 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)cudaEventCreate(&ev_a_);cudaEventCreate(&ev_b_);}- // the small f32 tensors the round touches first (see df2_touch)- for (const Tensor& t : tensors_)- if (t.dtype == "f32" && t.rows * t.cols * 4 <= (1 << 20) && touch_n_ < 48) {- touch_p_[touch_n_] = reinterpret_cast<const float*>(blob_ + t.off);- touch_c_[touch_n_++] = (int) (t.rows * t.cols);- }return alloc(err);}@@ -447,7 +441,6 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {cudaEventRecord(ev_fork_, cs_);cudaStreamWaitEvent(cs2_, ev_fork_, 0);}- if (touch_n_ > 0 && prof_ == nullptr && !skip_part("notouch")) kernels::df2_touch(touch_p_, touch_c_, touch_n_, hsel_, cs_);bool ok = record_context(taps, n, fork_ ? cs2_ : cs_, err);if (fork_) cudaEventRecord(ev_join_, cs2_);if (ok) ok = record_block(cs_, err);diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cuindex 43f101457..997a3e353 100644--- a/src/kernels/cuda/dflash2.cu+++ b/src/kernels/cuda/dflash2.cu@@ -510,16 +510,6 @@ __global__ void q8_tile_kernel(const uint8_t* __restrict__ blocks, int8_t* __resfor (int i = 0; i < 32; ++i) o[i] = (int8_t) src[2 + i];}-// Reads a list of small tensors (norm weights, the convolutions' base kernels, ...) once, so they sit in the caches before the first kernel that needs them-// (the verify window before the drafter streams gigabytes through them, and every dependent small kernel would otherwise wait for DRAM).-struct Df2Touch { const float* p[48]; int n[48]; int count; };-__global__ void touch_kernel(Df2Touch t, float* __restrict__ sink) {- float s = 0.0f;- for (int i = 0; i < t.count; ++i)- for (int j = blockIdx.x * blockDim.x + threadIdx.x; j < t.n[i]; j += gridDim.x * blockDim.x) s += t.p[i][j];- if (s == 1.2345e30f) sink[0] = s;-}-// ---- top-16constexpr int TK_CHUNK_MAX = 4096;__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }@@ -722,15 +712,6 @@ void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int3check("kv_ring");}-void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream) {- Df2Touch t;- if (n > 48) n = 48;- for (int i = 0; i < n; ++i) { t.p[i] = ptrs[i]; t.n[i] = counts[i]; }- t.count = n;- touch_kernel<<<40, 256, 0, (cudaStream_t) stream>>>(t, sink);- check("touch");-}-void df2_gemm_variant(int v) { g_gemm_variant = v; }void df2_attention_variant(int v) { g_attn_variant = v; }