Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine opt-in: the busy experts of a prompt take FP16 GEMMs, not the tiles (DS4_FAT_ROWS=N, default 0: byte-identical path, the packed tiles run exactly as before). An expert of N rows or more has its weights decoded to FP16 once (iq::f16_rows, the values the tiles multiply), the rows gathered (gather_f16), cublasGemmEx, clamped SwiGLU in FP16 (swiglu_f16); SetArgs skip_ge leaves those experts out of the tiles. The tiles decode an expert's weights again for every 64 of its rows; a popular expert of a long chunk has hundreds. Measured on the RTX 5090 with the IST-DAS 2.25 bpw pack, v0.2.7 base, DS4_FAT_ROWS=64: a 13K-token prompt 1,400 -> 1,950 tok/s, a 5.4K prompt 1,065 -> 1,181, decode unchanged (28-33 short, 47-49 warm). 3x V100, Maya-XS: +6-8% on 8K and 32K prompts. The same products, the sums' order differs: greedy output can diverge after the first rounding (wording equivalent); KL vs the reference 0.276, the stock path 0.285 (the same band, measured on Maya-XS; not yet re-measured on this base).
Lyes SBAHI0c5b157fbd9b
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hppindex f11176de1..afd6def60 100644--- a/include/strata/dsv41/exl3.hpp+++ b/include/strata/dsv41/exl3.hpp@@ -158,6 +158,7 @@ struct SetArgs {// their shared-memory tile anyway, so the same values come out; this only saves the F32 read of the same rows by// every column block of an expert (18-36 reads of 1.8-3.7 MB on a prompt) - MarLin's "small operand cheap".const uint16_t* x16 = nullptr;+ int skip_ge = 0; // (ds4's low-bit GGUF experts) > 0: experts of at least this many rows are the caller's (FP16 GEMMs)#if defined(STRATA_METAL) // (only on Metal: keeps the CUDA / HIP struct as it was)const int* hbounds = nullptr; // the same bounds on the host: [n_exp + 1], absolute rowsint total_rows = 0; // the set's rows in alldiff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 8e1d605be..f537a3ce0 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1563,6 +1563,7 @@ iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (thq.n_ff = sa.n_ff;q.limit = sa.limit;q.xf16 = sa.x16;+ q.skip_ge = sa.skip_ge;#if defined(STRATA_METAL)q.hbounds = sa.hbounds;q.total_rows = sa.total_rows;diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex 186aedb9d..be5d86837 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -742,6 +742,7 @@ __global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;const int r0 = lo + blockIdx.x * RM;if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)+ if (a.skip_ge > 0 && hi - lo >= a.skip_ge) return; // (the caller's FP16 GEMMs')load_tab<T>(tab);const int nr = min(RM, hi - r0);const int c0 = blockIdx.y * BN;@@ -833,6 +834,7 @@ __global__ void __launch_bounds__(kThreads) set_tc_kernel(const SetArgs a) {const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;const int r0 = lo + blockIdx.x * BM;if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)+ if (a.skip_ge > 0 && hi - lo >= a.skip_ge) return; // (the caller's FP16 GEMMs')load_tab<T>(tab);const int nr = min(BM, hi - r0);const int c0 = blockIdx.y * BN;@@ -1618,6 +1620,68 @@ bool set_down(const SetArgs& a, cudaStream_t s) {return true;}+// ---------------------------------------------------------------- the busy experts' FP16 weights (f16_rows)+// 8 values a thread: row r's sub-block kb, piece p - deq8's values rounded as the tensor-core tiles round them+template <int T>+__global__ void __launch_bounds__(kThreads) f16_rows_kernel(const uint8_t* __restrict__ w, int k, int n,+ uint16_t* __restrict__ dst, int ldd) {+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ load_tab<T>(tab);+ __syncthreads();+ const int nq = k / 8;+ const size_t rb = row_bytes_t<T>(k);+ for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < (long long) n * nq;+ i += (long long) gridDim.x * blockDim.x) {+ const int r = (int) (i / nq), q = (int) (i % nq);+ float w8[8];+ deq8<T>(w + (size_t) r * rb, q >> 2, q & 3, tab, w8);+ union {+ uint4 v;+ uint16_t h[8];+ } u;+#pragma unroll+ for (int j = 0; j < 8; ++j) u.h[j] = __half_as_ushort(__float2half_rn(w8[j]));+ *reinterpret_cast<uint4*>(dst + (size_t) r * ldd + (size_t) q * 8) = u.v;+ }+}+__global__ void __launch_bounds__(kThreads) gather_f16_kernel(const uint16_t* __restrict__ x, int ldx,+ const int* __restrict__ idx, int n, int k,+ uint16_t* __restrict__ dst) {+ const int nq = k / 8;+ for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < (long long) n * nq;+ i += (long long) gridDim.x * blockDim.x) {+ const int r = (int) (i / nq), q = (int) (i % nq);+ *reinterpret_cast<uint4*>(dst + (size_t) r * k + (size_t) q * 8) =+ *reinterpret_cast<const uint4*>(x + (size_t) idx[r] * ldx + (size_t) q * 8);+ }+}+__global__ void __launch_bounds__(kThreads) swiglu_f16_kernel(const float* __restrict__ gu, int ld, int n, int ff,+ float limit, uint16_t* __restrict__ h, int ldh) {+ for (long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x; i < (long long) n * ff;+ i += (long long) gridDim.x * blockDim.x) {+ const int r = (int) (i / ff), c = (int) (i % ff);+ const float* g = gu + (size_t) r * ld;+ h[(size_t) r * ldh + c] = __half_as_ushort(__float2half_rn(swiglu(g[c], g[ff + c], limit)));+ }+}+static unsigned grid_for(long long items) {+ return (unsigned) std::max<long long>(1, std::min<long long>(4096, (items + kThreads - 1) / kThreads));+}+bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s) {+ if (k % 256 != 0 || n < 1 || ldd % 8 != 0 || ((uintptr_t) dst & 15) != 0) return false;+ const unsigned blocks = grid_for((long long) n * (k / 8));+#define FR_(T) f16_rows_kernel<T><<<blocks, kThreads, 0, s>>>(w, k, n, dst, ldd)+ DS4_IQ_SWITCH(type, FR_)+#undef FR_+ return cudaGetLastError() == cudaSuccess;+}+void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s) {+ gather_f16_kernel<<<grid_for((long long) n * (k / 8)), kThreads, 0, s>>>(x, ldx, idx, n, k, dst);+}+void swiglu_f16(const float* gu, int ld, int n, int ff, float limit, uint16_t* h, int ldh, cudaStream_t s) {+ swiglu_f16_kernel<<<grid_for((long long) n * ff), kThreads, 0, s>>>(gu, ld, n, ff, limit, h, ldh);+}+bool mv(const MvJob* jobs, int n, cudaStream_t s) {#if defined(STRATA_METAL)if (n > 0 && n <= 64 && ::strata::dsv41::metal_host::iq_mv((const void*) jobs, n)) return true;diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 6a22576ab..8cf48e2c1 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -95,6 +95,7 @@ struct SetArgs {int ldo = 0;int tg = 0, td = 0;int skip_le = 0; // (the tiles) the experts of at most this many rows are left out: the light kernels took them+ int skip_ge = 0; // (the tiles) > 0: the experts of at least this many rows are left out - the caller's GEMMs#if defined(STRATA_METAL) // (only on Metal: the CUDA and HIP kernels take this struct by value, so its size is part of their ABI)const int* hbounds = nullptr; // the bounds on the host: [n_exp + 1], absolute rowsint total_rows = 0; // the set's rows in all@@ -106,6 +107,14 @@ bool set_down(const SetArgs& a, cudaStream_t s);/// count of matrix-core launches so far (both do nothing / return -1 where there are no such tiles; hipk_force returns whether this device has them)bool hipk_force(int mode);long long hipk_launches();+/// The busy experts' path (a prompt's experts of many rows): one expert's matrix - n rows of k values, row_bytes(k)+/// apart - as FP16 rows (ldd apart), the values the tensor-core tiles multiply (their loader's, rounded to FP16 the same+/// way), for the caller's GEMM: the tiles decode an expert's weights again for every 64 of its rows.+bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s);+/// dst[r] = x[idx[r]] for r < n (FP16 rows of k values; x ldx apart, dst k apart)+void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s);+/// h[r][c] = FP16 of the tiles' clamped SwiGLU of gu[r][c] (gate) and gu[r][ff + c] (up), F32 rows ld apart+void swiglu_f16(const float* gu, int ld, int n, int ff, float limit, uint16_t* h, int ldh, cudaStream_t s);/// The light form alone (the set's experts of at most 16 rows; the others untouched), for any of these types and MXFP4/// (tg / td 39: ds4_gguf.cu's experts): false (nothing launched) when it is switched off - the caller's tiles take every/// expert then; true: they take those of more rows (skip_le 16).diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 7ffa7cc4f..8f072397c 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -612,6 +612,12 @@ struct Chunk {size_t stage_stride = 0;std::unique_ptr<Pool> readers;cublasHandle_t blas = nullptr;+ // the busy experts' path (run_set, DS4_FAT_ROWS): one expert's weights as FP16 (gate and up, then down), the rows+ // gathered for its GEMM, the GEMM's gate | up output - fat_r rows a piece; allocated at the first use+ uint16_t *fat_w = nullptr, *fat_x = nullptr;+ float* fat_gu = nullptr;+ int fat_r = 0;+ bool fat_off = false;// host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each// with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)uint8_t* hand[2] = {nullptr, nullptr};@@ -3379,6 +3385,78 @@ struct Model::Impl {}return true;}+ // ---- the busy experts (run_set): an expert of many rows as FP16 GEMMs. The tiles decode its weights again for+ // every 64 of its rows (a prompt's chunk gives a popular expert hundreds); decoded once, to the FP16 values the tiles+ // multiply (iq::f16_rows), the library's GEMM takes them at its own speed. DS4_FAT_ROWS=N (default 0, off; N:+ // the experts of N rows or more) takes the experts of N rows or more; the same products, only the sums' order differ+ // (3x V100, Maya-XS: 64 rows +6-8% on 8K / 32K prompts; KL vs the reference 0.276, the stock path 0.285 - the same;+ // RTX 5090, IST-DAS 2.25 bpw: a 13K-token prompt 1,400 -> 1,625 tok/s at 64 rows, decode unchanged)+ static int fat_rows() {+ static const int n = std::getenv("DS4_FAT_ROWS") ? std::max(0, std::atoi(std::getenv("DS4_FAT_ROWS"))) : 0;+ return n;+ }+ bool fat_ready(Dev& d, Chunk& c, int D, int FF) {+#if defined(STRATA_USE_HIP)+ (void) d, (void) c, (void) D, (void) FF;+ return false;+#else+ if (fat_rows() <= 0 || c.fat_off) return false;+ if (c.fat_w != nullptr) return true;+ static const int R = std::getenv("DS4_FAT_PIECE") ? std::max(64, std::atoi(std::getenv("DS4_FAT_PIECE"))) : 1024;+ if (cudaSetDevice(d.id) != cudaSuccess || cudaMalloc(&c.fat_w, (size_t) 2 * FF * D * 2) != cudaSuccess ||+ cudaMalloc(&c.fat_x, (size_t) R * D * 2) != cudaSuccess ||+ cudaMalloc(&c.fat_gu, (size_t) R * 2 * FF * 4) != cudaSuccess) {+ cudaGetLastError();+ std::fprintf(stderr, "ds4: CUDA%d: no VRAM for the busy experts' path (%.0f MB): the tiles take them\n", d.id,+ ((double) 2 * FF * D * 2 + (double) R * D * 2 + (double) R * 2 * FF * 4) / 1e6);+ c.fat_off = true;+ return false;+ }+ c.fat_r = R;+ std::fprintf(stderr, "ds4: CUDA%d busy experts (%d rows or more) as FP16 GEMMs, %d rows a piece (%.0f MB)\n", d.id,+ fat_rows(), R, ((double) 2 * FF * D * 2 + (double) R * D * 2 + (double) R * 2 * FF * 4) / 1e6);+ return true;+#endif+ }+ // h16[lr0 + r] = FP16 clamped SwiGLU of the expert's gate and up over x16[row_tok[row + r]], r < n+ bool fat_gate_up(Chunk& c, const x3::SetArgs& sa, const uint8_t* blob, int row, int n, int a, int D, int FF,+ cudaStream_t s, std::string& err) {+ const int tg = gg::expert_type(sa.lay, 0);+ if (!gg::iq::f16_rows(tg, blob + sa.lay.trellis[0], D, FF, c.fat_w, D, s) ||+ !gg::iq::f16_rows(tg, blob + sa.lay.trellis[1], D, FF, c.fat_w + (size_t) FF * D, D, s)) {+ err = "the busy experts' path: gate / up not decoded";+ return false;+ }+ const float one = 1.0f, zero = 0.0f;+ for (int p0 = 0; p0 < n; p0 += c.fat_r) {+ const int np = std::min(c.fat_r, n - p0);+ gg::iq::gather_f16(c.x16, D, c.row_tok + row + p0, np, D, c.fat_x, s);+ if (cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, 2 * FF, np, D, &one, c.fat_w, CUDA_R_16F, D, c.fat_x,+ CUDA_R_16F, D, &zero, c.fat_gu, CUDA_R_32F, 2 * FF, CUBLAS_COMPUTE_32F,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP) != CUBLAS_STATUS_SUCCESS) {+ err = "the busy experts' path: cublasGemmEx (gate / up) failed";+ return false;+ }+ gg::iq::swiglu_f16(c.fat_gu, 2 * FF, np, FF, sa.limit, c.h16 + (size_t) (row - a + p0) * FF, FF, s);+ }+ return true;+ }+ // y[lr0 + r] = down(h16[lr0 + r]), r < n+ bool fat_down(Chunk& c, const x3::SetArgs& sa, const uint8_t* blob, int lr0, int n, int D, int FF, cudaStream_t s,+ std::string& err) {+ if (!gg::iq::f16_rows(gg::expert_type(sa.lay, 2), blob + sa.lay.trellis[2], FF, D, c.fat_w, FF, s)) {+ err = "the busy experts' path: down not decoded";+ return false;+ }+ const float one = 1.0f, zero = 0.0f;+ if (cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, D, n, FF, &one, c.fat_w, CUDA_R_16F, FF,+ c.h16 + (size_t) lr0 * FF, CUDA_R_16F, FF, &zero, c.y + (size_t) lr0 * D, CUDA_R_32F, D,+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP) != CUBLAS_STATUS_SUCCESS) {+ err = "the busy experts' path: cublasGemmEx (down) failed";+ return false;+ }+ return true;+ }// y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,float* y, int ldy, std::string& err) {@@ -4045,6 +4123,21 @@ struct Model::Impl {// gate / up read the layer's F16 copy of the activations (the same rounding the tiles make for their// shared tile): with 18-36 column blocks an expert the F32 rows were read that many timessa.x16 = strata::ds4::gate_up_x16_default(defaults_026()) ? c.x16 : nullptr;+ // the busy experts (DS4_FAT_ROWS=N: those of N rows or more): their weights decoded to FP16 once and+ // multiplied by the library's GEMM - the tiles decode an expert's weights again for every 64 of its rows+ std::vector<int> fat;+ if (sa.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(sa.lay) && fat_ready(d, c, D, FF)) {+ int rest = 0;+ for (int i = i0; i < j; ++i) {+ const int r = b[i + 1] - b[i];+ if (r >= fat_rows()) fat.push_back(i);+ else rest = std::max(rest, r);+ }+ if (!fat.empty()) {+ sa.skip_ge = fat_rows();+ sa.max_rows = rest;+ }+ }EkMark ek{};const bool ekp = prof_chunk;if (ekp) {@@ -4063,8 +4156,16 @@ struct Model::Impl {}if (sa.lay.fmt == gg::kFmtMXFP4) {gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);+ for (const int i : fat)+ if (!fat_gate_up(c, sa, sa.wbase + (size_t) (i - i0) * stride, b[i], b[i + 1] - b[i], a, D, FF,+ d.st, err))+ return false;if (ekp) cudaEventRecord(ek.e1, d.st);gg::set_down(sa, c.h16, FF, c.y, D, d.st);+ for (const int i : fat)+ if (!fat_down(c, sa, sa.wbase + (size_t) (i - i0) * stride, b[i] - a, b[i + 1] - b[i], D, FF,+ d.st, err))+ return false;} else {x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);if (ekp) cudaEventRecord(ek.e1, d.st);