Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dflash2: drop the cache-touch kernel (no gain), wait-commit measurement switch

Niko12216f83fe2c4283
diff --git a/include/strata/core/dflash2.hpp b/include/strata/core/dflash2.hpp
index bad2bf5e9..205bff3d0 100644
--- a/include/strata/core/dflash2.hpp
+++ b/include/strata/core/dflash2.hpp
@@ -124,9 +124,6 @@ private:
float* xdc_ = nullptr;
cudaStream_t cs2_ = nullptr; ///< the context update's branch of the round graph
cudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr, ev_ctx_[8] = {};
- const float* touch_p_[48] = {};
- int touch_c_[48] = {};
- int touch_n_ = 0;
bool fork_ = false; ///< recording the forked graph: layer l's attention waits for ev_ctx_[l]
float *fcy_ = nullptr, *hctx_ = nullptr, *kc_ = nullptr, *vc_ = nullptr;
float *h_ = nullptr, *xn_ = nullptr, *xc_ = nullptr, *dyn_ = nullptr, *q_ = nullptr, *qn_ = nullptr, *k_ = nullptr, *v_ = nullptr, *kblk_ = nullptr;
diff --git a/include/strata/kernels/dflash2.hpp b/include/strata/kernels/dflash2.hpp
index 286174981..bdcf0cbe4 100644
--- a/include/strata/kernels/dflash2.hpp
+++ b/include/strata/kernels/dflash2.hpp
@@ -76,9 +76,6 @@ void df2_norm_rope_qk(const float* q, const float* k, const float* gq, const flo
void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int32_t* pos, const float* table, float* kring, float* vring, int slots, int rows,
int heads, float eps, void* stream);
-/// Reads n small f32 tensors (ptrs[i], counts[i] floats; n <= 48) once to bring them into the caches; sink: one float of scratch.
-void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream);
-
/// Top-16 per row of logits [rows][n] (value-descending; ties to the lower index). idx_map (optional) maps a column to a token id.
/// scratch: rows * kDf2TopBlocks * 16 * 2 words. Outputs val[rows][16] and tok[rows][16]. n <= 4096 * kDf2TopBlocks.
void df2_topk(const float* logits, int n, const int32_t* idx_map, int rows, uint32_t* scratch, float* val, int32_t* tok, void* stream);
diff --git a/src/core/dflash2.cpp b/src/core/dflash2.cpp
index 3da70987d..5acd76832 100644
--- a/src/core/dflash2.cpp
+++ b/src/core/dflash2.cpp
@@ -208,12 +208,6 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)
cudaEventCreate(&ev_a_);
cudaEventCreate(&ev_b_);
}
- // the small f32 tensors the round touches first (see df2_touch)
- for (const Tensor& t : tensors_)
- if (t.dtype == "f32" && t.rows * t.cols * 4 <= (1 << 20) && touch_n_ < 48) {
- touch_p_[touch_n_] = reinterpret_cast<const float*>(blob_ + t.off);
- touch_c_[touch_n_++] = (int) (t.rows * t.cols);
- }
return alloc(err);
}
@@ -447,7 +441,6 @@ bool Dflash2Drafter::capture(int n, const float* taps, std::string& err) {
cudaEventRecord(ev_fork_, cs_);
cudaStreamWaitEvent(cs2_, ev_fork_, 0);
}
- if (touch_n_ > 0 && prof_ == nullptr && !skip_part("notouch")) kernels::df2_touch(touch_p_, touch_c_, touch_n_, hsel_, cs_);
bool ok = record_context(taps, n, fork_ ? cs2_ : cs_, err);
if (fork_) cudaEventRecord(ev_join_, cs2_);
if (ok) ok = record_block(cs_, err);
diff --git a/src/kernels/cuda/dflash2.cu b/src/kernels/cuda/dflash2.cu
index 43f101457..997a3e353 100644
--- a/src/kernels/cuda/dflash2.cu
+++ b/src/kernels/cuda/dflash2.cu
@@ -510,16 +510,6 @@ __global__ void q8_tile_kernel(const uint8_t* __restrict__ blocks, int8_t* __res
for (int i = 0; i < 32; ++i) o[i] = (int8_t) src[2 + i];
}
-// Reads a list of small tensors (norm weights, the convolutions' base kernels, ...) once, so they sit in the caches before the first kernel that needs them
-// (the verify window before the drafter streams gigabytes through them, and every dependent small kernel would otherwise wait for DRAM).
-struct Df2Touch { const float* p[48]; int n[48]; int count; };
-__global__ void touch_kernel(Df2Touch t, float* __restrict__ sink) {
- float s = 0.0f;
- for (int i = 0; i < t.count; ++i)
- for (int j = blockIdx.x * blockDim.x + threadIdx.x; j < t.n[i]; j += gridDim.x * blockDim.x) s += t.p[i][j];
- if (s == 1.2345e30f) sink[0] = s;
-}
-
// ---- top-16
constexpr int TK_CHUNK_MAX = 4096;
__device__ __forceinline__ bool better(float va, int ia, float vb, int ib) { return va > vb || (va == vb && ia < ib); }
@@ -722,15 +712,6 @@ void df2_kv_ring(const float* kin, const float* vin, const float* gk, const int3
check("kv_ring");
}
-void df2_touch(const float* const* ptrs, const int* counts, int n, float* sink, void* stream) {
- Df2Touch t;
- if (n > 48) n = 48;
- for (int i = 0; i < n; ++i) { t.p[i] = ptrs[i]; t.n[i] = counts[i]; }
- t.count = n;
- touch_kernel<<<40, 256, 0, (cudaStream_t) stream>>>(t, sink);
- check("touch");
-}
-
void df2_gemm_variant(int v) { g_gemm_variant = v; }
void df2_attention_variant(int v) { g_attn_variant = v; }