Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: the chunked prompt's dense dequantizer writes its 32 values a thread through shared memory, coalesced (a warp covers 1024 contiguous values), instead of each thread writing 2 bytes every 64 - a warp's store touched 32 cache lines. Same arithmetic and the same F16/BF16 values (only the store order changes). R9700 12 GB cell, 2.25 pack, 8K warm: prompt 14474 -> 13840 ms (566 -> 591.9 ppts), the q_b+rope stage 4025 -> 3581 ms, ids identical (5705e1164a); the cold first prompt also uses it. DS4_DQ_OLD=1 keeps the old kernel for A/B.

Niko12218abe826f700e
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index e5f668876..7de79b4ea 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -1278,7 +1278,54 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
// ---------------------------------------------------------------- whole matrices as 16-bit rows (the chunked prompt's GEMMs)
// n rows of k values from `raw` (ld bytes a row): one thread a 32-value sub-block, the sub-block's codes and scales from decode<>
-// (value = d s q - m of its half, ggml's dequantize_row_*), rounded to BF16 (bf16) or F16
+// (value = d s q - m of its half, ggml's dequantize_row_*), rounded to BF16 (bf16) or F16.
+// The 32 values go through shared memory and are written coalesced (a warp's 32 threads cover 1024 contiguous values): a
+// thread writing its own 32 values straight to dst wrote 2 bytes every 64 (a warp's store touches 32 cache lines), which
+// measured 0.5 G weights/s on the R9700 - the 8K prompt's q_b stage 4 s. DS4_DQ_OLD=1 keeps the old kernel for A/B.
+template <int T>
+__global__ void __launch_bounds__(kThreads) dq_rows_kernel2(const uint8_t* __restrict__ raw, size_t ld, int k, int n,
+ uint16_t* __restrict__ dst, bool bf16) {
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ uint16_t sm[kThreads * 32];
+ load_tab<T>(tab);
+ __syncthreads();
+ const int nsb = k / 32;
+ const size_t total = (size_t) n * (size_t) nsb; // 32-value sub-blocks
+ const size_t vals = (size_t) n * (size_t) k;
+ for (size_t base = (size_t) blockIdx.x * kThreads; base < total; base += (size_t) gridDim.x * kThreads) {
+ const size_t i = base + threadIdx.x;
+ uint16_t* o = sm + threadIdx.x * 32;
+ if (i < total) {
+ const size_t r = i / (size_t) nsb;
+ const int kb = (int) (i - r * (size_t) nsb);
+ Sub sb;
+ sb.m0 = sb.m1 = 0.0f;
+ decode<T>(raw + r * ld, kb, tab, sb);
+ const float d0 = __fmul_rn(sb.d, (float) sb.s0), d1 = __fmul_rn(sb.d, (float) sb.s1);
+#pragma unroll
+ for (int j = 0; j < 32; ++j) {
+ const int q = (int) (int8_t) ((uint32_t) sb.q[j >> 2] >> (8 * (j & 3)));
+ const float v = j < 16 ? __fsub_rn(__fmul_rn(d0, (float) q), sb.m0) : __fsub_rn(__fmul_rn(d1, (float) q), sb.m1);
+ o[j] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));
+ }
+ } else {
+#pragma unroll
+ for (int j = 0; j < 32; ++j) o[j] = 0;
+ }
+ __syncthreads();
+ // sm[m] is the block's value m (the sub-blocks are contiguous in the row-major output, rows included): write it
+ // a thread's 32 values apart, so a warp's stores are 512 contiguous bytes each
+ const size_t go = base * 32;
+ uint16_t* d = dst + go;
+#pragma unroll
+ for (int j = 0; j < 32; ++j) {
+ const size_t m = (size_t) j * kThreads + threadIdx.x;
+ if (go + m < vals) d[m] = sm[m];
+ }
+ __syncthreads();
+ }
+}
+
template <int T>
__global__ void __launch_bounds__(kThreads) dq_rows_kernel(const uint8_t* __restrict__ raw, size_t ld, int k, int n,
uint16_t* __restrict__ dst, bool bf16) {
@@ -1309,7 +1356,13 @@ __global__ void __launch_bounds__(kThreads) dq_rows_kernel(const uint8_t* __rest
bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {
if (raw == nullptr || k % 256 != 0 || n <= 0) return false;
const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);
-#define DS4_DQ(T) dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16)
+ static const bool old = std::getenv("DS4_DQ_OLD") != nullptr && std::atoi(std::getenv("DS4_DQ_OLD")) != 0;
+#define DS4_DQ(T) do { \
+ if (old) \
+ dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \
+ else \
+ dq_rows_kernel2<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \
+ } while (0)
switch (type) {
case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;
case kIQ2_XS: DS4_DQ(kIQ2_XS); break;