Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek decode: fewer launches a layer - the int8 roundings of one GEMV call in one launch (identical inputs once) and hc_mix + hc_split in one launch (the token's last mix block runs the split); same arithmetic
Niko122160d856a18edd
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 5bee58804..e7f8a153b 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -201,6 +201,62 @@ void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d,q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);}+// Several activations' rounding in one launch (a GEMV call's jobs: wo_a's eight groups, the shared expert's two, ...): item z is+// q8_quant's (x, ldx, rows, k, q, d), the same per-element arithmetic, so the bytes are the single launches'. Identical items+// (the same x, stride, rows and k: two matrices reading one input) are rounded once and share the result.+constexpr int kQuantItems = x3::kMaxMvJobs;+struct QuantItems {+ const float* x[kQuantItems];+ size_t ldx[kQuantItems];+ int k[kQuantItems], rows[kQuantItems];+ int8_t* q[kQuantItems];+ float* d[kQuantItems];+ int n;+};+__global__ void __launch_bounds__(kThreads) q8_quant_multi_kernel(QuantItems it) {+ const int z = blockIdx.z;+ if (z >= it.n) return;+ const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ const int k = it.k[z];+ if (r >= it.rows[z] || b * 32 >= k) return;+ const float v = it.x[z][(size_t) r * it.ldx[z] + (size_t) b * 32 + lane];+ float am = fabsf(v);+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));+ const float sc = am / 127.0f;+ it.q[z][(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);+ if (lane == 0) it.d[z][(size_t) r * (k / 32) + b] = sc;+}+// adds an item unless an identical one is queued; returns the item whose result to read (its q and d)+int quant_add(QuantItems& it, const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {+ static const bool nodedup = std::getenv("DS4_QUANT_NODEDUP") != nullptr;+ for (int i = 0; i < it.n && !nodedup; ++i)+ if (it.x[i] == x && it.ldx[i] == ldx && it.rows[i] == rows && it.k[i] == k) return i;+ const int i = it.n++;+ it.x[i] = x;+ it.ldx[i] = ldx;+ it.rows[i] = rows;+ it.k[i] = k;+ it.q[i] = q;+ it.d[i] = d;+ return i;+}+void quant_flush(const QuantItems& it, cudaStream_t s) {+ if (it.n <= 0) return;+ static const bool single = std::getenv("DS4_QUANT_SINGLE") != nullptr;+ if (single) {+ for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);+ return;+ }+ int kmax = 0, rmax = 0;+ for (int i = 0; i < it.n; ++i) {+ kmax = std::max(kmax, it.k[i]);+ rmax = std::max(rmax, it.rows[i]);+ }+ const dim3 grid((unsigned) ((kmax / 32 + kWarps - 1) / kWarps), (unsigned) rmax, (unsigned) it.n);+ q8_quant_multi_kernel<<<grid, kThreads, 0, s>>>(it);+}+// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {@@ -1077,6 +1133,8 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {bool vec[3] = {true, true, true};iq::MvJob IJ[x3::kMaxMvJobs]; // the low-bit ones (ds4_iq: a launch a type)int nij = 0;+ QuantItems QI; // the int8 roundings of this call's jobs, one launch (identical inputs once)+ QI.n = 0;for (int i = 0; i < n; ++i) {const x3::MvJob& q = jobs[i];const x3::Mat& m = *q.m;@@ -1089,7 +1147,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {return false;}const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ const int qi = quant_add(QI, q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32);iq::MvJob& b = IJ[nij++];b.w = (const uint8_t*) m.raw;b.ld = row_stride(m);@@ -1097,8 +1155,8 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {b.k = m.k;b.n = m.n;b.ncol = q.ncol;- b.xq = sc->mq + at;- b.xd = sc->md + at / 32;+ b.xq = QI.q[qi];+ b.xd = QI.d[qi];b.y = q.y;b.ycs = q.y_cs;b.bias = q.bias;@@ -1136,14 +1194,15 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {return false;}const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;- b.xq = sc->mq + at;- b.xd = sc->md + at / 32;- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ const int qi = quant_add(QI, q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32);+ b.xq = QI.q[qi];+ b.xd = QI.d[qi];vec[f] = vec[f] && b.ld % 4 == 0 && ((uintptr_t) b.w & 3u) == 0 && (m.k / 32) % 2 == 0;} else {vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));}}+ quant_flush(QI, s);for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);if (nij > 0 && !iq::mv(IJ, nij, s)) {fail("mv: a low-bit job not covered");diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 58df7c19b..f803016b8 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -261,12 +261,11 @@ __global__ void __launch_bounds__(kThreads) hc_mix_kernel(const float* __restricif (threadIdx.x == 0) mixes[(size_t) t * 32 + m] = acc * rsqrtf(ss / (float) hcdim + eps);}-// a warp a token: lane j * hc + k holds comb[j][k] (hc <= 4: lanes 0..15; wider streams fall back to lane 0 alone)-__global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __restrict__ scale,- const float* __restrict__ base, int hc, int iters, float eps, float* pre, float* post,- float* comb) {- const int t = blockIdx.x, lane = threadIdx.x;- const float* mx = mixes + (size_t) t * 32;+// a warp a token: lane j * hc + k holds comb[j][k] (hc <= 4: lanes 0..15; wider streams fall back to lane 0 alone). The body is+// a device function (hc_mix_kernel's last block runs it too); mx is the token's 32 mixes+__device__ __forceinline__ void hc_split_body(const float* mx, int t, int lane, const float* __restrict__ scale,+ const float* __restrict__ base, int hc, int iters, float eps, float* pre,+ float* post, float* comb) {pre += (size_t) t * 8;post += (size_t) t * 8;comb += (size_t) t * 64;@@ -334,6 +333,50 @@ __global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __for (int j = 0; j < hc; ++j)for (int k = 0; k < hc; ++k) comb[j * hc + k] = c[j][k];}+__global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __restrict__ scale,+ const float* __restrict__ base, int hc, int iters, float eps, float* pre, float* post,+ float* comb) {+ const int t = blockIdx.x;+ hc_split_body(mixes + (size_t) t * 32, t, threadIdx.x, scale, base, hc, iters, eps, pre, post, comb);+}++// hc_mix_kernel with hc_split_kernel behind it: the block that finishes the token's last mix (an atomic count, reset by it) runs+// the split on warp 0 - the same arithmetic as the two launches, one launch less and no round trip through the launch gap+__global__ void __launch_bounds__(kThreads) hc_mix_split_kernel(const float* __restrict__ h, const float* __restrict__ fn,+ int hcdim, float eps, float* mixes, unsigned* cnt,+ const float* __restrict__ scale,+ const float* __restrict__ base, int hc, int iters,+ float hc_eps, float* pre, float* post, float* comb) {+ const int m = blockIdx.x, t = blockIdx.y;+ const float* w = fn + (size_t) m * hcdim;+ const float* x = h + (size_t) t * hcdim;+ float acc = 0.0f, ss = 0.0f;+ for (int k = threadIdx.x * 4; k < hcdim; k += kThreads * 4) {+ const float4 hv = *(const float4*) (x + k);+ const float4 wv = *(const float4*) (w + k);+ acc += hv.x * wv.x + hv.y * wv.y + hv.z * wv.z + hv.w * wv.w;+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ acc = block_sum(acc);+ ss = block_sum(ss);+ __shared__ unsigned last;+ __shared__ float smx[32];+ if (threadIdx.x == 0) {+ mixes[(size_t) t * 32 + m] = acc * rsqrtf(ss / (float) hcdim + eps);+ __threadfence();+ last = atomicAdd(cnt + t, 1u) == gridDim.x - 1 ? 1u : 0u;+ }+ __syncthreads();+ if (!last) return;+ __threadfence();+ if (threadIdx.x < 32) {+ const int n = gridDim.x; // the mixes of the token (n_mix <= 32); read past the L1+ smx[threadIdx.x] = threadIdx.x < n ? *(const volatile float*) (mixes + (size_t) t * 32 + threadIdx.x) : 0.0f;+ __syncwarp();+ hc_split_body(smx, t, threadIdx.x, scale, base, hc, iters, hc_eps, pre, post, comb);+ }+ if (threadIdx.x == 0) cnt[t] = 0u;+}__global__ void __launch_bounds__(1024) hc_pre_norm_kernel(const float* __restrict__ h, const float* __restrict__ mix,const float* __restrict__ w, int dim, int hc, float eps,@@ -1884,6 +1927,12 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudvoid hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);}+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,+ float* comb, cudaStream_t s) {+ hc_mix_split_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes, cnt, scale, base, hc, iters, hc_eps, pre,+ post, comb);+}void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,float* pre, float* post, float* comb, cudaStream_t s) {hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuhindex 9bc16233d..ae33e9c83 100644--- a/src/dsv41/ds4_kernels.cuh+++ b/src/dsv41/ds4_kernels.cuh@@ -61,6 +61,11 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cud/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination). Strides: mixes/// 32, pre / post 8, comb 64 floats a row.void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);+/// hc_mix then hc_split in one launch (cnt: T zeroed unsigned counters, left zero; n_mix <= 32, T <= kWin): the last block of a token+/// runs the split - the same arithmetic as the two launches+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,+ float* comb, cudaStream_t s);void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,float* pre, float* post, float* comb, cudaStream_t s);/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 9b380cbbd..007b55d4d 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -487,6 +487,7 @@ struct Dev {int grp = 0, mi = 0;unsigned ar_seq = 0;unsigned* xcnt = nullptr;+ unsigned* hccnt = nullptr; // [kWin] hc_mix_split's per-token block counters (zeroed at allocation, left zero)// the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;// logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)x3::Mat head;@@ -2289,7 +2290,7 @@ struct Model::Impl {}if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;if (!halloc(d.bounce, bounce_bytes(), err)) return false;- if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||+ if (!dalloc(d, d.xcnt, 4, err) || !dalloc(d, d.hccnt, kWin, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||!dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||!dalloc(d, d.n_groups, 4, err))return false;@@ -8586,8 +8587,8 @@ struct Model::Impl {engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);}// attention- hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);- hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);+ hc_mix_split(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps,+ d.a_pre, d.a_post, d.a_comb, s);hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);{x3::MvJob j[4];@@ -8800,8 +8801,8 @@ struct Model::Impl {if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)prof_mark(d, 1);// the MoE- hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);- hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);+ hc_mix_split(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps,+ d.f_pre, d.f_post, d.f_comb, s);hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);if (wp.async) {// the routing goes to the service thread through the mailbox (a group's lead publishes it, the members