Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine (AMD): a staging ring sized from the VRAM room, and expert-tile knobs for the MarLin work
Niko12211e72cdc4b99c
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hppindex b080a5970..a5455b75a 100644--- a/include/strata/dsv41/exl3.hpp+++ b/include/strata/dsv41/exl3.hpp@@ -154,6 +154,10 @@ struct SetArgs {int max_rows = 0; // the most rows one expert of the set hasint n_embd = 4096, n_ff = 2048;float limit = 0.0f;+ // gate / up: the caller's F16 copy of the activations (row_idx picks the rows). The tiles round F32 to F16 for+ // their shared-memory tile anyway, so the same values come out; this only saves the F32 read of the same rows by+ // every column block of an expert (18-36 reads of 1.8-3.7 MB on a prompt) - MarLin's "small operand cheap".+ const uint16_t* x16 = nullptr;};/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex e7f8a153b..55231c8d3 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1402,6 +1402,7 @@ iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (thq.n_embd = sa.n_embd;q.n_ff = sa.n_ff;q.limit = sa.limit;+ q.xf16 = sa.x16;return q;}} // namespacediff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex c2b74ff11..5cfb4f7b3 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -86,6 +86,9 @@ struct SetArgs {const float* xf = nullptr;const int* idx = nullptr;const uint16_t* xh = nullptr;+ // gate / up from the caller's F16 copy of the activations (the same rounding the tiles make themselves: the same+ // bits), so the F32 rows are not read 18+ times a prompt (MarLin's "keep the small operand in its cheap format")+ const uint16_t* xf16 = nullptr;int ldx = 0;uint16_t* h = nullptr;float* y = nullptr;diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.incindex 4e62dffd8..d4e117965 100644--- a/src/dsv41/ds4_iq_hip.inc+++ b/src/dsv41/ds4_iq_hip.inc@@ -311,6 +311,19 @@ inline bool launch(const SetArgs& a, cudaStream_t s) {if (cfg == 3 && a.max_rows <= 32) return runr<T, GU, 32, 2, 2>(a, s);if (cfg == 4 && a.max_rows <= 32) return runr<T, GU, 32, 8, 2>(a, s);if (cfg == 5 && a.max_rows <= 32) return runr<T, GU, 32, 2, 1>(a, s);+ // (MarLin tile sweep on long sets: a shorter block for the tail rows - one kernel, same arithmetic)+ if (cfg == 6) return runr<T, GU, 64, 8, 1>(a, s);+ if (cfg == 7) return runr<T, GU, 32, 8, 1>(a, s);+ if (cfg == 8) return runr<T, GU, 128, 8, 2>(a, s);+ if (cfg == 9) return runr<T, GU, 64, 4, 2>(a, s);+ // (a wider column block for the down pass only: its K is short and its F16 activation tile is re-read by+ // every column block - 40 of them at BN 128, 20 at BN 256, the weight tile unchanged)+ if (cfg == 10) {+ if constexpr (GU) return runr<T, GU, 128, 8, 1>(a, s);+ else return runr<T, GU, 128, 8, 2>(a, s);+ }+ // (64 rows x 256 columns a block: half the activation re-reads of 128 x 128 at the same accumulator count)+ if (cfg == 11) return runr<T, GU, 64, 8, 2>(a, s);if (a.max_rows <= 32) return runr<T, GU, 32, 4, 2>(a, s);if (a.max_rows <= 64) return runr<T, GU, 64, 4, 2>(a, s);return runr<T, GU, 128, 8, 1>(a, s);diff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.incindex 0844177be..85f1a0f4b 100644--- a/src/dsv41/ds4_iq_hip_r.inc+++ b/src/dsv41/ds4_iq_hip_r.inc@@ -10,7 +10,7 @@ __device__ __forceinline__ float hf(uint32_t h) { return __half2float(__ushort_aconstexpr int kKC = 64; // K a step: two sub-blocks// BM rows (MT tiles of 16) x BN = NW 16 NTW weight rows per block; NW waves, each NTW column tiles of 16 and every row tile-template <int T, bool GU, int BM, int NW, int NTW>+template <int T, bool GU, int BM, int NW, int NTW, bool X16 = false>__global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg) {constexpr int NM = GU ? 2 : 1, MT = BM / 16, BN = NW * 16 * NTW, NTH = NW * 32;static_assert(BM % 16 == 0, "row tiles");@@ -39,16 +39,17 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgfor (int j = 0; j < NTW; ++j)wrow[m][j] = blob + a.tr[GU ? m : 2] + (size_t) ((c0 + warp * (16 * NTW) + j * 16 + l16)) * rb;- constexpr int NX = GU ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;+ constexpr bool XS = GU && X16; // gate / up take the caller's F16 rows (the same values the F32 path rounds to)+ constexpr int NX = (GU && !XS) ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;struct XR {- float4 f[GU ? NX : 1];- uint4 u[GU ? 1 : NX];+ float4 f[(GU && !XS) ? NX : 1];+ uint4 u[(GU && !XS) ? 1 : NX];};auto fetchx = [&](int s, XR& R) {#pragma unrollfor (int xi = 0; xi < NX; ++xi) {const int c = tid + xi * NTH;- if constexpr (GU) {+ if constexpr (GU && !XS) {if (c < BM * 16) {const int row = c >> 4, ch = c & 15;R.f[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 4);@@ -56,7 +57,8 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg} else {if (c < BM * 8) {const int row = c >> 3, ch = c & 7;- R.u[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);+ const uint16_t* px = GU ? a.xf16 : a.xh;+ R.u[xi] = *(const uint4*) (px + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);}}}@@ -65,7 +67,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg#pragma unrollfor (int xi = 0; xi < NX; ++xi) {const int c = tid + xi * NTH;- if constexpr (GU) {+ if constexpr (GU && !XS) {if (c < BM * 16) {const int row = c >> 4, ch = c & 15;const float4 v = R.f[xi];@@ -88,6 +90,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgfor (int i = 0; i < MT; ++i)#pragma unrollfor (int j = 0; j < NTW; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};+ uint32_t sink = 0; // (HIPK_DBG=1: keeps the decode's registers alive when the MMA is switched off){XR R0;@@ -123,7 +126,13 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgfor (int j = 0; j < NTW; ++j) {int c0_, c1_;float sc;- step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);+ if (!(dbg & 2)) { // HIPK_DBG=2: no decode (the MMA path alone), =1: no MMA (the decode alone)+ step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);+ } else {+ c0_ = (int) cur[m][j].w[0];+ c1_ = (int) cur[m][j].w[1];+ sc = 1.0f;+ }const hv2 h = {(_Float16) sc, (_Float16) sc};const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);#if defined(DS4_HIPK_G11)@@ -139,6 +148,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg#elsebf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));#endif+ if (dbg & 1) sink ^= u0.x ^ u0.y ^ u1.x ^ u1.y;}#if defined(DS4_HIPK_G11)const int kcol = sub * 32 + ks * 16;@@ -147,7 +157,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg#endif#pragma unrollfor (int i = 0; i < MT; ++i) {- if (i * 16 < nr) { // (a tile past the rows: skipped, uniform)+ if (i * 16 < nr && !(dbg & 1)) { // (a tile past the rows: skipped, uniform; HIPK_DBG=1: no MMA)#if defined(DS4_HIPK_G11)const hv16 af = *(const hv16*) &xs[b][i * 16 + l16][kcol];#pragma unroll@@ -195,6 +205,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgelse a.y[lr * a.ldo + col] = acc[0][i][j][q];}}+ if (sink == 0xdeadbeefu && tid == 0) a.y[0] = 0.0f; // (HIPK_DBG=1 only: never true)}template <int T, bool GU, int BM, int NW, int NTW>@@ -204,6 +215,17 @@ inline bool runr(const SetArgs& a, cudaStream_t s) {if (N % BN != 0) return false;const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;+ static const bool x16 = [] { // STRATA_DS4_X16=0: gate / up read the F32 rows as before (A/B)+ const char* v = std::getenv("STRATA_DS4_X16");+ return !(v != nullptr && v[0] == '0');+ }();+ if constexpr (GU) {+ if (x16 && a.xf16 != nullptr) {+ set_kernel_r<T, GU, BM, NW, NTW, true><<<grid, NW * 32, 0, s>>>(a, dbg);+ launches().fetch_add(1, std::memory_order_relaxed);+ return cudaPeekAtLastError() == cudaSuccess;+ }+ }set_kernel_r<T, GU, BM, NW, NTW><<<grid, NW * 32, 0, s>>>(a, dbg);launches().fetch_add(1, std::memory_order_relaxed);return cudaPeekAtLastError() == cudaSuccess;diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex f7e29e0ec..e81458ede 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2908,11 +2908,18 @@ struct Model::Impl {size_t ring_mb_auto() const {double exp_b = 0, other_b = 0, room = 0;const double share = gpu_expert_share(&exp_b, &room, &other_b);- const bool fit = exp_b > 0 && share >= 1.0 / 1.3; // (R9700 x4: 0.86 -> ring 16: decode -1.8% after a prompt, 64: -4.4%, 400: -4.6%)+ // The prompt path streams the experts the cards cannot hold. When every expert fits, the ring is worth+ // nothing (16 MB): the VRAM is better as slots for the decode after the prompt (measured: decode -1.8 to -4.6%+ // at 64-400 MB, unchanged at 16). When they do not fit, the stream IS the prompt's expert stage, and its depth+ // is what decides how long the tiles wait: on the 4x R9700 8K prompt a 16 MB ring left 1.7 s of DMA waits in a+ // 5.2 s expert stage, and 1.2-3 GB of ring measured 632 -> 678-684 ppts (+7%) for about -2% decode (off a+ // fifth more CPU-lane experts). So: a share of the room, bounded, rather than the fixed 400 MB of 0.2.6.+ const size_t deep = (size_t) std::min(2000.0, std::max(400.0, room / 1e9 * 0.05 * 1000.0)); // (MB)+ const size_t mb = share >= 0.98 ? 16 : deep;if (o.verbose)std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,- room / 1e9, other_b / 1e9, fit ? 16 : 400);- return fit ? 16 : 400;+ room / 1e9, other_b / 1e9, (int) mb);+ return mb;}bool alloc_chunks(std::string& err) {chunks.resize(devs.size());@@ -3629,6 +3636,9 @@ struct Model::Impl {sa.n_embd = D;sa.n_ff = FF;sa.limit = g.lim;+ // gate / up read the layer's F16 copy of the activations (the same rounding the tiles make for their+ // shared tile): with 18-36 column blocks an expert the F32 rows were read that many times+ sa.x16 = c.x16;EkMark ek{};const bool ekp = prof_chunk;if (ekp) {