Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DS4_FAT_ROWS follow-up to the port: launches through strata_ds4_launch (the Mac build), off on Metal, only for the types f16_rows decodes (no trellis, no Q5_K / Q6_K down: a request would have stopped on 'not decoded'), at least 17 rows (the light kernels own the small experts)
Niko1221d52b73dc6573
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex be5d86837..53bc4052b 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -1667,19 +1667,28 @@ __global__ void __launch_bounds__(kThreads) swiglu_f16_kernel(const float* __resstatic unsigned grid_for(long long items) {return (unsigned) std::max<long long>(1, std::min<long long>(4096, (items + kThreads - 1) / kThreads));}+bool f16_rows_ok(int type) { // (the DS4_IQ_SWITCH list)+ switch (type) {+ case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS: case kIQ1_S: case kIQ1_M:+ case kIQ4_NL: case kQ4_K: case kQ2_K: case kQ3_K:+ return true;+ default:+ return false;+ }+}bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s) {if (k % 256 != 0 || n < 1 || ldd % 8 != 0 || ((uintptr_t) dst & 15) != 0) return false;const unsigned blocks = grid_for((long long) n * (k / 8));-#define FR_(T) f16_rows_kernel<T><<<blocks, kThreads, 0, s>>>(w, k, n, dst, ldd)+#define FR_(T) strata_ds4_launch(f16_rows_kernel<T>, blocks, kThreads, 0, s, w, k, n, dst, ldd)DS4_IQ_SWITCH(type, FR_)#undef FR_return cudaGetLastError() == cudaSuccess;}void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s) {- gather_f16_kernel<<<grid_for((long long) n * (k / 8)), kThreads, 0, s>>>(x, ldx, idx, n, k, dst);+ strata_ds4_launch(gather_f16_kernel, grid_for((long long) n * (k / 8)), kThreads, 0, s, x, ldx, idx, n, k, dst);}void swiglu_f16(const float* gu, int ld, int n, int ff, float limit, uint16_t* h, int ldh, cudaStream_t s) {- swiglu_f16_kernel<<<grid_for((long long) n * ff), kThreads, 0, s>>>(gu, ld, n, ff, limit, h, ldh);+ strata_ds4_launch(swiglu_f16_kernel, grid_for((long long) n * ff), kThreads, 0, s, gu, ld, n, ff, limit, h, ldh);}bool mv(const MvJob* jobs, int n, cudaStream_t s) {diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 8cf48e2c1..922f41852 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -111,6 +111,8 @@ long long hipk_launches();/// apart - as FP16 rows (ldd apart), the values the tensor-core tiles multiply (their loader's, rounded to FP16 the same/// way), for the caller's GEMM: the tiles decode an expert's weights again for every 64 of its rows.bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s);+/// Whether f16_rows decodes this type (the IQ and K-quant types of the tiles; not the trellis types, not Q5_K / Q6_K).+bool f16_rows_ok(int type);/// dst[r] = x[idx[r]] for r < n (FP16 rows of k values; x ldx apart, dst k apart)void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s);/// h[r][c] = FP16 of the tiles' clamped SwiGLU of gu[r][c] (gate) and gu[r][ff + c] (up), F32 rows ld apartdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 8f072397c..b9ac202d5 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -3392,11 +3392,14 @@ struct Model::Impl {// (3x V100, Maya-XS: 64 rows +6-8% on 8K / 32K prompts; KL vs the reference 0.276, the stock path 0.285 - the same;// RTX 5090, IST-DAS 2.25 bpw: a 13K-token prompt 1,400 -> 1,625 tok/s at 64 rows, decode unchanged)static int fat_rows() {- static const int n = std::getenv("DS4_FAT_ROWS") ? std::max(0, std::atoi(std::getenv("DS4_FAT_ROWS"))) : 0;+ static const int n = [] {+ const int v = std::getenv("DS4_FAT_ROWS") ? std::max(0, std::atoi(std::getenv("DS4_FAT_ROWS"))) : 0;+ return v > 0 ? std::max(v, 17) : 0; // (the light kernels take the experts of 16 rows or fewer)+ }();return n;}bool fat_ready(Dev& d, Chunk& c, int D, int FF) {-#if defined(STRATA_USE_HIP)+#if defined(STRATA_USE_HIP) || defined(STRATA_METAL) // (CUDA only: the Mac has no dispatch for these kernels, HIP is unmeasured)(void) d, (void) c, (void) D, (void) FF;return false;#else@@ -4126,7 +4129,10 @@ struct Model::Impl {// the busy experts (DS4_FAT_ROWS=N: those of N rows or more): their weights decoded to FP16 once and// multiplied by the library's GEMM - the tiles decode an expert's weights again for every 64 of its rowsstd::vector<int> fat;- if (sa.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(sa.lay) && fat_ready(d, c, D, FF)) {+ if (sa.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(sa.lay) && D % 256 == 0 && FF % 256 == 0 &&+ gg::expert_type(sa.lay, 1) == gg::expert_type(sa.lay, 0) &&+ gg::iq::f16_rows_ok(gg::expert_type(sa.lay, 0)) && gg::iq::f16_rows_ok(gg::expert_type(sa.lay, 2)) &&+ fat_ready(d, c, D, FF)) {int rest = 0;for (int i = i0; i < j; ++i) {const int r = b[i + 1] - b[i];