Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek decode: fewer launches a layer - the int8 roundings of one GEMV call in one launch (identical inputs once) and hc_mix + hc_split in one launch (the token's last mix block runs the split); same arithmetic

Niko122160d856a18edd
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 5bee58804..e7f8a153b 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -201,6 +201,62 @@ void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d,
q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);
}
+// Several activations' rounding in one launch (a GEMV call's jobs: wo_a's eight groups, the shared expert's two, ...): item z is
+// q8_quant's (x, ldx, rows, k, q, d), the same per-element arithmetic, so the bytes are the single launches'. Identical items
+// (the same x, stride, rows and k: two matrices reading one input) are rounded once and share the result.
+constexpr int kQuantItems = x3::kMaxMvJobs;
+struct QuantItems {
+ const float* x[kQuantItems];
+ size_t ldx[kQuantItems];
+ int k[kQuantItems], rows[kQuantItems];
+ int8_t* q[kQuantItems];
+ float* d[kQuantItems];
+ int n;
+};
+__global__ void __launch_bounds__(kThreads) q8_quant_multi_kernel(QuantItems it) {
+ const int z = blockIdx.z;
+ if (z >= it.n) return;
+ const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ const int k = it.k[z];
+ if (r >= it.rows[z] || b * 32 >= k) return;
+ const float v = it.x[z][(size_t) r * it.ldx[z] + (size_t) b * 32 + lane];
+ float am = fabsf(v);
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+ const float sc = am / 127.0f;
+ it.q[z][(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
+ if (lane == 0) it.d[z][(size_t) r * (k / 32) + b] = sc;
+}
+// adds an item unless an identical one is queued; returns the item whose result to read (its q and d)
+int quant_add(QuantItems& it, const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {
+ static const bool nodedup = std::getenv("DS4_QUANT_NODEDUP") != nullptr;
+ for (int i = 0; i < it.n && !nodedup; ++i)
+ if (it.x[i] == x && it.ldx[i] == ldx && it.rows[i] == rows && it.k[i] == k) return i;
+ const int i = it.n++;
+ it.x[i] = x;
+ it.ldx[i] = ldx;
+ it.rows[i] = rows;
+ it.k[i] = k;
+ it.q[i] = q;
+ it.d[i] = d;
+ return i;
+}
+void quant_flush(const QuantItems& it, cudaStream_t s) {
+ if (it.n <= 0) return;
+ static const bool single = std::getenv("DS4_QUANT_SINGLE") != nullptr;
+ if (single) {
+ for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);
+ return;
+ }
+ int kmax = 0, rmax = 0;
+ for (int i = 0; i < it.n; ++i) {
+ kmax = std::max(kmax, it.k[i]);
+ rmax = std::max(rmax, it.rows[i]);
+ }
+ const dim3 grid((unsigned) ((kmax / 32 + kWarps - 1) / kWarps), (unsigned) rmax, (unsigned) it.n);
+ q8_quant_multi_kernel<<<grid, kThreads, 0, s>>>(it);
+}
+
// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),
// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4
__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {
@@ -1077,6 +1133,8 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
bool vec[3] = {true, true, true};
iq::MvJob IJ[x3::kMaxMvJobs]; // the low-bit ones (ds4_iq: a launch a type)
int nij = 0;
+ QuantItems QI; // the int8 roundings of this call's jobs, one launch (identical inputs once)
+ QI.n = 0;
for (int i = 0; i < n; ++i) {
const x3::MvJob& q = jobs[i];
const x3::Mat& m = *q.m;
@@ -1089,7 +1147,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
return false;
}
const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
+ const int qi = quant_add(QI, q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32);
iq::MvJob& b = IJ[nij++];
b.w = (const uint8_t*) m.raw;
b.ld = row_stride(m);
@@ -1097,8 +1155,8 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
b.k = m.k;
b.n = m.n;
b.ncol = q.ncol;
- b.xq = sc->mq + at;
- b.xd = sc->md + at / 32;
+ b.xq = QI.q[qi];
+ b.xd = QI.d[qi];
b.y = q.y;
b.ycs = q.y_cs;
b.bias = q.bias;
@@ -1136,14 +1194,15 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
return false;
}
const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
- b.xq = sc->mq + at;
- b.xd = sc->md + at / 32;
- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
+ const int qi = quant_add(QI, q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32);
+ b.xq = QI.q[qi];
+ b.xd = QI.d[qi];
vec[f] = vec[f] && b.ld % 4 == 0 && ((uintptr_t) b.w & 3u) == 0 && (m.k / 32) % 2 == 0;
} else {
vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));
}
}
+ quant_flush(QI, s);
for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);
if (nij > 0 && !iq::mv(IJ, nij, s)) {
fail("mv: a low-bit job not covered");
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 58df7c19b..f803016b8 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -261,12 +261,11 @@ __global__ void __launch_bounds__(kThreads) hc_mix_kernel(const float* __restric
if (threadIdx.x == 0) mixes[(size_t) t * 32 + m] = acc * rsqrtf(ss / (float) hcdim + eps);
}
-// a warp a token: lane j * hc + k holds comb[j][k] (hc <= 4: lanes 0..15; wider streams fall back to lane 0 alone)
-__global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __restrict__ scale,
- const float* __restrict__ base, int hc, int iters, float eps, float* pre, float* post,
- float* comb) {
- const int t = blockIdx.x, lane = threadIdx.x;
- const float* mx = mixes + (size_t) t * 32;
+// a warp a token: lane j * hc + k holds comb[j][k] (hc <= 4: lanes 0..15; wider streams fall back to lane 0 alone). The body is
+// a device function (hc_mix_kernel's last block runs it too); mx is the token's 32 mixes
+__device__ __forceinline__ void hc_split_body(const float* mx, int t, int lane, const float* __restrict__ scale,
+ const float* __restrict__ base, int hc, int iters, float eps, float* pre,
+ float* post, float* comb) {
pre += (size_t) t * 8;
post += (size_t) t * 8;
comb += (size_t) t * 64;
@@ -334,6 +333,50 @@ __global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __
for (int j = 0; j < hc; ++j)
for (int k = 0; k < hc; ++k) comb[j * hc + k] = c[j][k];
}
+__global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __restrict__ scale,
+ const float* __restrict__ base, int hc, int iters, float eps, float* pre, float* post,
+ float* comb) {
+ const int t = blockIdx.x;
+ hc_split_body(mixes + (size_t) t * 32, t, threadIdx.x, scale, base, hc, iters, eps, pre, post, comb);
+}
+
+// hc_mix_kernel with hc_split_kernel behind it: the block that finishes the token's last mix (an atomic count, reset by it) runs
+// the split on warp 0 - the same arithmetic as the two launches, one launch less and no round trip through the launch gap
+__global__ void __launch_bounds__(kThreads) hc_mix_split_kernel(const float* __restrict__ h, const float* __restrict__ fn,
+ int hcdim, float eps, float* mixes, unsigned* cnt,
+ const float* __restrict__ scale,
+ const float* __restrict__ base, int hc, int iters,
+ float hc_eps, float* pre, float* post, float* comb) {
+ const int m = blockIdx.x, t = blockIdx.y;
+ const float* w = fn + (size_t) m * hcdim;
+ const float* x = h + (size_t) t * hcdim;
+ float acc = 0.0f, ss = 0.0f;
+ for (int k = threadIdx.x * 4; k < hcdim; k += kThreads * 4) {
+ const float4 hv = *(const float4*) (x + k);
+ const float4 wv = *(const float4*) (w + k);
+ acc += hv.x * wv.x + hv.y * wv.y + hv.z * wv.z + hv.w * wv.w;
+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
+ }
+ acc = block_sum(acc);
+ ss = block_sum(ss);
+ __shared__ unsigned last;
+ __shared__ float smx[32];
+ if (threadIdx.x == 0) {
+ mixes[(size_t) t * 32 + m] = acc * rsqrtf(ss / (float) hcdim + eps);
+ __threadfence();
+ last = atomicAdd(cnt + t, 1u) == gridDim.x - 1 ? 1u : 0u;
+ }
+ __syncthreads();
+ if (!last) return;
+ __threadfence();
+ if (threadIdx.x < 32) {
+ const int n = gridDim.x; // the mixes of the token (n_mix <= 32); read past the L1
+ smx[threadIdx.x] = threadIdx.x < n ? *(const volatile float*) (mixes + (size_t) t * 32 + threadIdx.x) : 0.0f;
+ __syncwarp();
+ hc_split_body(smx, t, threadIdx.x, scale, base, hc, iters, hc_eps, pre, post, comb);
+ }
+ if (threadIdx.x == 0) cnt[t] = 0u;
+}
__global__ void __launch_bounds__(1024) hc_pre_norm_kernel(const float* __restrict__ h, const float* __restrict__ mix,
const float* __restrict__ w, int dim, int hc, float eps,
@@ -1884,6 +1927,12 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cud
void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {
hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);
}
+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,
+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
+ float* comb, cudaStream_t s) {
+ hc_mix_split_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes, cnt, scale, base, hc, iters, hc_eps, pre,
+ post, comb);
+}
void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
float* pre, float* post, float* comb, cudaStream_t s) {
hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
index 9bc16233d..ae33e9c83 100644
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@ -61,6 +61,11 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cud
/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination). Strides: mixes
/// 32, pre / post 8, comb 64 floats a row.
void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);
+/// hc_mix then hc_split in one launch (cnt: T zeroed unsigned counters, left zero; n_mix <= 32, T <= kWin): the last block of a token
+/// runs the split - the same arithmetic as the two launches
+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,
+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
+ float* comb, cudaStream_t s);
void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
float* pre, float* post, float* comb, cudaStream_t s);
/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 9b380cbbd..007b55d4d 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -487,6 +487,7 @@ struct Dev {
int grp = 0, mi = 0;
unsigned ar_seq = 0;
unsigned* xcnt = nullptr;
+ unsigned* hccnt = nullptr; // [kWin] hc_mix_split's per-token block counters (zeroed at allocation, left zero)
// the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;
// logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)
x3::Mat head;
@@ -2289,7 +2290,7 @@ struct Model::Impl {
}
if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;
if (!halloc(d.bounce, bounce_bytes(), err)) return false;
- if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
+ if (!dalloc(d, d.xcnt, 4, err) || !dalloc(d, d.hccnt, kWin, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
!dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||
!dalloc(d, d.n_groups, 4, err))
return false;
@@ -8586,8 +8587,8 @@ struct Model::Impl {
engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
}
// attention
- hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
- hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);
+ hc_mix_split(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps,
+ d.a_pre, d.a_post, d.a_comb, s);
hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);
{
x3::MvJob j[4];
@@ -8800,8 +8801,8 @@ struct Model::Impl {
if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)
prof_mark(d, 1);
// the MoE
- hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
- hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);
+ hc_mix_split(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps,
+ d.f_pre, d.f_post, d.f_comb, s);
hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);
if (wp.async) {
// the routing goes to the service thread through the mailbox (a group's lead publishes it, the members