Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine (AMD): a staging ring sized from the VRAM room, and expert-tile knobs for the MarLin work

Niko12211e72cdc4b99c
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hpp
index b080a5970..a5455b75a 100644
--- a/include/strata/dsv41/exl3.hpp
+++ b/include/strata/dsv41/exl3.hpp
@@ -154,6 +154,10 @@ struct SetArgs {
int max_rows = 0; // the most rows one expert of the set has
int n_embd = 4096, n_ff = 2048;
float limit = 0.0f;
+ // gate / up: the caller's F16 copy of the activations (row_idx picks the rows). The tiles round F32 to F16 for
+ // their shared-memory tile anyway, so the same values come out; this only saves the F32 read of the same rows by
+ // every column block of an expert (18-36 reads of 1.8-3.7 MB on a prompt) - MarLin's "small operand cheap".
+ const uint16_t* x16 = nullptr;
};
/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).
void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index e7f8a153b..55231c8d3 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1402,6 +1402,7 @@ iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (th
q.n_embd = sa.n_embd;
q.n_ff = sa.n_ff;
q.limit = sa.limit;
+ q.xf16 = sa.x16;
return q;
}
} // namespace
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index c2b74ff11..5cfb4f7b3 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -86,6 +86,9 @@ struct SetArgs {
const float* xf = nullptr;
const int* idx = nullptr;
const uint16_t* xh = nullptr;
+ // gate / up from the caller's F16 copy of the activations (the same rounding the tiles make themselves: the same
+ // bits), so the F32 rows are not read 18+ times a prompt (MarLin's "keep the small operand in its cheap format")
+ const uint16_t* xf16 = nullptr;
int ldx = 0;
uint16_t* h = nullptr;
float* y = nullptr;
diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.inc
index 4e62dffd8..d4e117965 100644
--- a/src/dsv41/ds4_iq_hip.inc
+++ b/src/dsv41/ds4_iq_hip.inc
@@ -311,6 +311,19 @@ inline bool launch(const SetArgs& a, cudaStream_t s) {
if (cfg == 3 && a.max_rows <= 32) return runr<T, GU, 32, 2, 2>(a, s);
if (cfg == 4 && a.max_rows <= 32) return runr<T, GU, 32, 8, 2>(a, s);
if (cfg == 5 && a.max_rows <= 32) return runr<T, GU, 32, 2, 1>(a, s);
+ // (MarLin tile sweep on long sets: a shorter block for the tail rows - one kernel, same arithmetic)
+ if (cfg == 6) return runr<T, GU, 64, 8, 1>(a, s);
+ if (cfg == 7) return runr<T, GU, 32, 8, 1>(a, s);
+ if (cfg == 8) return runr<T, GU, 128, 8, 2>(a, s);
+ if (cfg == 9) return runr<T, GU, 64, 4, 2>(a, s);
+ // (a wider column block for the down pass only: its K is short and its F16 activation tile is re-read by
+ // every column block - 40 of them at BN 128, 20 at BN 256, the weight tile unchanged)
+ if (cfg == 10) {
+ if constexpr (GU) return runr<T, GU, 128, 8, 1>(a, s);
+ else return runr<T, GU, 128, 8, 2>(a, s);
+ }
+ // (64 rows x 256 columns a block: half the activation re-reads of 128 x 128 at the same accumulator count)
+ if (cfg == 11) return runr<T, GU, 64, 8, 2>(a, s);
if (a.max_rows <= 32) return runr<T, GU, 32, 4, 2>(a, s);
if (a.max_rows <= 64) return runr<T, GU, 64, 4, 2>(a, s);
return runr<T, GU, 128, 8, 1>(a, s);
diff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.inc
index 0844177be..85f1a0f4b 100644
--- a/src/dsv41/ds4_iq_hip_r.inc
+++ b/src/dsv41/ds4_iq_hip_r.inc
@@ -10,7 +10,7 @@ __device__ __forceinline__ float hf(uint32_t h) { return __half2float(__ushort_a
constexpr int kKC = 64; // K a step: two sub-blocks
// BM rows (MT tiles of 16) x BN = NW 16 NTW weight rows per block; NW waves, each NTW column tiles of 16 and every row tile
-template <int T, bool GU, int BM, int NW, int NTW>
+template <int T, bool GU, int BM, int NW, int NTW, bool X16 = false>
__global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg) {
constexpr int NM = GU ? 2 : 1, MT = BM / 16, BN = NW * 16 * NTW, NTH = NW * 32;
static_assert(BM % 16 == 0, "row tiles");
@@ -39,16 +39,17 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
for (int j = 0; j < NTW; ++j)
wrow[m][j] = blob + a.tr[GU ? m : 2] + (size_t) ((c0 + warp * (16 * NTW) + j * 16 + l16)) * rb;
- constexpr int NX = GU ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;
+ constexpr bool XS = GU && X16; // gate / up take the caller's F16 rows (the same values the F32 path rounds to)
+ constexpr int NX = (GU && !XS) ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;
struct XR {
- float4 f[GU ? NX : 1];
- uint4 u[GU ? 1 : NX];
+ float4 f[(GU && !XS) ? NX : 1];
+ uint4 u[(GU && !XS) ? 1 : NX];
};
auto fetchx = [&](int s, XR& R) {
#pragma unroll
for (int xi = 0; xi < NX; ++xi) {
const int c = tid + xi * NTH;
- if constexpr (GU) {
+ if constexpr (GU && !XS) {
if (c < BM * 16) {
const int row = c >> 4, ch = c & 15;
R.f[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 4);
@@ -56,7 +57,8 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
} else {
if (c < BM * 8) {
const int row = c >> 3, ch = c & 7;
- R.u[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);
+ const uint16_t* px = GU ? a.xf16 : a.xh;
+ R.u[xi] = *(const uint4*) (px + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);
}
}
}
@@ -65,7 +67,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
#pragma unroll
for (int xi = 0; xi < NX; ++xi) {
const int c = tid + xi * NTH;
- if constexpr (GU) {
+ if constexpr (GU && !XS) {
if (c < BM * 16) {
const int row = c >> 4, ch = c & 15;
const float4 v = R.f[xi];
@@ -88,6 +90,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
for (int i = 0; i < MT; ++i)
#pragma unroll
for (int j = 0; j < NTW; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};
+ uint32_t sink = 0; // (HIPK_DBG=1: keeps the decode's registers alive when the MMA is switched off)
{
XR R0;
@@ -123,7 +126,13 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
for (int j = 0; j < NTW; ++j) {
int c0_, c1_;
float sc;
- step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);
+ if (!(dbg & 2)) { // HIPK_DBG=2: no decode (the MMA path alone), =1: no MMA (the decode alone)
+ step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);
+ } else {
+ c0_ = (int) cur[m][j].w[0];
+ c1_ = (int) cur[m][j].w[1];
+ sc = 1.0f;
+ }
const hv2 h = {(_Float16) sc, (_Float16) sc};
const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);
#if defined(DS4_HIPK_G11)
@@ -139,6 +148,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
#else
bf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));
#endif
+ if (dbg & 1) sink ^= u0.x ^ u0.y ^ u1.x ^ u1.y;
}
#if defined(DS4_HIPK_G11)
const int kcol = sub * 32 + ks * 16;
@@ -147,7 +157,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
#endif
#pragma unroll
for (int i = 0; i < MT; ++i) {
- if (i * 16 < nr) { // (a tile past the rows: skipped, uniform)
+ if (i * 16 < nr && !(dbg & 1)) { // (a tile past the rows: skipped, uniform; HIPK_DBG=1: no MMA)
#if defined(DS4_HIPK_G11)
const hv16 af = *(const hv16*) &xs[b][i * 16 + l16][kcol];
#pragma unroll
@@ -195,6 +205,7 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
else a.y[lr * a.ldo + col] = acc[0][i][j][q];
}
}
+ if (sink == 0xdeadbeefu && tid == 0) a.y[0] = 0.0f; // (HIPK_DBG=1 only: never true)
}
template <int T, bool GU, int BM, int NW, int NTW>
@@ -204,6 +215,17 @@ inline bool runr(const SetArgs& a, cudaStream_t s) {
if (N % BN != 0) return false;
const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);
static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;
+ static const bool x16 = [] { // STRATA_DS4_X16=0: gate / up read the F32 rows as before (A/B)
+ const char* v = std::getenv("STRATA_DS4_X16");
+ return !(v != nullptr && v[0] == '0');
+ }();
+ if constexpr (GU) {
+ if (x16 && a.xf16 != nullptr) {
+ set_kernel_r<T, GU, BM, NW, NTW, true><<<grid, NW * 32, 0, s>>>(a, dbg);
+ launches().fetch_add(1, std::memory_order_relaxed);
+ return cudaPeekAtLastError() == cudaSuccess;
+ }
+ }
set_kernel_r<T, GU, BM, NW, NTW><<<grid, NW * 32, 0, s>>>(a, dbg);
launches().fetch_add(1, std::memory_order_relaxed);
return cudaPeekAtLastError() == cudaSuccess;
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index f7e29e0ec..e81458ede 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2908,11 +2908,18 @@ struct Model::Impl {
size_t ring_mb_auto() const {
double exp_b = 0, other_b = 0, room = 0;
const double share = gpu_expert_share(&exp_b, &room, &other_b);
- const bool fit = exp_b > 0 && share >= 1.0 / 1.3; // (R9700 x4: 0.86 -> ring 16: decode -1.8% after a prompt, 64: -4.4%, 400: -4.6%)
+ // The prompt path streams the experts the cards cannot hold. When every expert fits, the ring is worth
+ // nothing (16 MB): the VRAM is better as slots for the decode after the prompt (measured: decode -1.8 to -4.6%
+ // at 64-400 MB, unchanged at 16). When they do not fit, the stream IS the prompt's expert stage, and its depth
+ // is what decides how long the tiles wait: on the 4x R9700 8K prompt a 16 MB ring left 1.7 s of DMA waits in a
+ // 5.2 s expert stage, and 1.2-3 GB of ring measured 632 -> 678-684 ppts (+7%) for about -2% decode (off a
+ // fifth more CPU-lane experts). So: a share of the room, bounded, rather than the fixed 400 MB of 0.2.6.
+ const size_t deep = (size_t) std::min(2000.0, std::max(400.0, room / 1e9 * 0.05 * 1000.0)); // (MB)
+ const size_t mb = share >= 0.98 ? 16 : deep;
if (o.verbose)
std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,
- room / 1e9, other_b / 1e9, fit ? 16 : 400);
- return fit ? 16 : 400;
+ room / 1e9, other_b / 1e9, (int) mb);
+ return mb;
}
bool alloc_chunks(std::string& err) {
chunks.resize(devs.size());
@@ -3629,6 +3636,9 @@ struct Model::Impl {
sa.n_embd = D;
sa.n_ff = FF;
sa.limit = g.lim;
+ // gate / up read the layer's F16 copy of the activations (the same rounding the tiles make for their
+ // shared tile): with 18-36 column blocks an expert the F32 rows were read that many times
+ sa.x16 = c.x16;
EkMark ek{};
const bool ekp = prof_chunk;
if (ekp) {