Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DS4_FAT_ROWS follow-up to the port: launches through strata_ds4_launch (the Mac build), off on Metal, only for the types f16_rows decodes (no trellis, no Q5_K / Q6_K down: a request would have stopped on 'not decoded'), at least 17 rows (the light kernels own the small experts)

Niko1221d52b73dc6573
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index be5d86837..53bc4052b 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -1667,19 +1667,28 @@ __global__ void __launch_bounds__(kThreads) swiglu_f16_kernel(const float* __res
static unsigned grid_for(long long items) {
return (unsigned) std::max<long long>(1, std::min<long long>(4096, (items + kThreads - 1) / kThreads));
}
+bool f16_rows_ok(int type) { // (the DS4_IQ_SWITCH list)
+ switch (type) {
+ case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS: case kIQ1_S: case kIQ1_M:
+ case kIQ4_NL: case kQ4_K: case kQ2_K: case kQ3_K:
+ return true;
+ default:
+ return false;
+ }
+}
bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s) {
if (k % 256 != 0 || n < 1 || ldd % 8 != 0 || ((uintptr_t) dst & 15) != 0) return false;
const unsigned blocks = grid_for((long long) n * (k / 8));
-#define FR_(T) f16_rows_kernel<T><<<blocks, kThreads, 0, s>>>(w, k, n, dst, ldd)
+#define FR_(T) strata_ds4_launch(f16_rows_kernel<T>, blocks, kThreads, 0, s, w, k, n, dst, ldd)
DS4_IQ_SWITCH(type, FR_)
#undef FR_
return cudaGetLastError() == cudaSuccess;
}
void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s) {
- gather_f16_kernel<<<grid_for((long long) n * (k / 8)), kThreads, 0, s>>>(x, ldx, idx, n, k, dst);
+ strata_ds4_launch(gather_f16_kernel, grid_for((long long) n * (k / 8)), kThreads, 0, s, x, ldx, idx, n, k, dst);
}
void swiglu_f16(const float* gu, int ld, int n, int ff, float limit, uint16_t* h, int ldh, cudaStream_t s) {
- swiglu_f16_kernel<<<grid_for((long long) n * ff), kThreads, 0, s>>>(gu, ld, n, ff, limit, h, ldh);
+ strata_ds4_launch(swiglu_f16_kernel, grid_for((long long) n * ff), kThreads, 0, s, gu, ld, n, ff, limit, h, ldh);
}
bool mv(const MvJob* jobs, int n, cudaStream_t s) {
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 8cf48e2c1..922f41852 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -111,6 +111,8 @@ long long hipk_launches();
/// apart - as FP16 rows (ldd apart), the values the tensor-core tiles multiply (their loader's, rounded to FP16 the same
/// way), for the caller's GEMM: the tiles decode an expert's weights again for every 64 of its rows.
bool f16_rows(int type, const uint8_t* w, int k, int n, uint16_t* dst, int ldd, cudaStream_t s);
+/// Whether f16_rows decodes this type (the IQ and K-quant types of the tiles; not the trellis types, not Q5_K / Q6_K).
+bool f16_rows_ok(int type);
/// dst[r] = x[idx[r]] for r < n (FP16 rows of k values; x ldx apart, dst k apart)
void gather_f16(const uint16_t* x, int ldx, const int* idx, int n, int k, uint16_t* dst, cudaStream_t s);
/// h[r][c] = FP16 of the tiles' clamped SwiGLU of gu[r][c] (gate) and gu[r][ff + c] (up), F32 rows ld apart
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 8f072397c..b9ac202d5 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -3392,11 +3392,14 @@ struct Model::Impl {
// (3x V100, Maya-XS: 64 rows +6-8% on 8K / 32K prompts; KL vs the reference 0.276, the stock path 0.285 - the same;
// RTX 5090, IST-DAS 2.25 bpw: a 13K-token prompt 1,400 -> 1,625 tok/s at 64 rows, decode unchanged)
static int fat_rows() {
- static const int n = std::getenv("DS4_FAT_ROWS") ? std::max(0, std::atoi(std::getenv("DS4_FAT_ROWS"))) : 0;
+ static const int n = [] {
+ const int v = std::getenv("DS4_FAT_ROWS") ? std::max(0, std::atoi(std::getenv("DS4_FAT_ROWS"))) : 0;
+ return v > 0 ? std::max(v, 17) : 0; // (the light kernels take the experts of 16 rows or fewer)
+ }();
return n;
}
bool fat_ready(Dev& d, Chunk& c, int D, int FF) {
-#if defined(STRATA_USE_HIP)
+#if defined(STRATA_USE_HIP) || defined(STRATA_METAL) // (CUDA only: the Mac has no dispatch for these kernels, HIP is unmeasured)
(void) d, (void) c, (void) D, (void) FF;
return false;
#else
@@ -4126,7 +4129,10 @@ struct Model::Impl {
// the busy experts (DS4_FAT_ROWS=N: those of N rows or more): their weights decoded to FP16 once and
// multiplied by the library's GEMM - the tiles decode an expert's weights again for every 64 of its rows
std::vector<int> fat;
- if (sa.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(sa.lay) && fat_ready(d, c, D, FF)) {
+ if (sa.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(sa.lay) && D % 256 == 0 && FF % 256 == 0 &&
+ gg::expert_type(sa.lay, 1) == gg::expert_type(sa.lay, 0) &&
+ gg::iq::f16_rows_ok(gg::expert_type(sa.lay, 0)) && gg::iq::f16_rows_ok(gg::expert_type(sa.lay, 2)) &&
+ fat_ready(d, c, D, FF)) {
int rest = 0;
for (int i = i0; i < j; ++i) {
const int r = b[i + 1] - b[i];