Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge main 2d07d1c0b (Strix Halo DeepSeek work: fd33cb6ee, 0eea4a3d5, 3133160ed, 2d07d1c0b) into f-027-win. The attention slab: the base is STRATA_DS4_A_MB megabytes of q, 32 MB on a unified-memory GPU (where main measured it) and 8 MB on a discrete card (the 0.2.6 slab); the automatic growth (attn_slab_rows) stays on top for discrete cards with spare VRAM, DS4_ATTN_SLAB still overrides, both env names work. The dense dequantizer: dq_rows_kernel2 stays the default; a gfx11 device (gcnArchName) defaults to main's improved row-per-lane kernel, DS4_DQ_OLD=1/0 forces either.

Niko1221afc45665fc54
diff --cc src/dsv41/ds4_iq.cu
index 7de79b4ea,e6674b4b8..b0539d1dc
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@@ -1353,16 -1341,10 +1388,40 @@@ __global__ void __launch_bounds__(kThre
} // namespace
++// Which dense dequantizer: dq_rows_kernel2 (the coalesced transpose, 0.2.7 DS Support: 566 -> 592 ppts on the R9700, ids identical) or the
++// row-per-lane dq_rows_kernel, which main's fd33cb6ee gave 16-byte stores and a staged input: the Strix Halo (gfx11) 4K
++// prompt 79.5 -> 88.5 tok/s with it, so a gfx11 device defaults to it. DS4_DQ_OLD=1 forces it anywhere, DS4_DQ_OLD=0 forces kernel2
++// anywhere. The detection is the device's gcnArchName (HIP only; a CUDA build has no gfx11). Decided once per device.
++static bool dq_use_old_kernel() {
++ if (const char* e = std::getenv("DS4_DQ_OLD"); e != nullptr && *e != '
++#if defined(STRATA_USE_HIP)
++ static std::atomic<int> cache[64]; // 0 unknown, 1 no, 2 yes
++ int dev = 0;
++ cudaGetDevice(&dev);
++ if (dev < 0 || dev >= 64) return false;
++ int c = cache[dev].load();
++ if (c == 0) {
++ cudaDeviceProp prop{};
++ const bool yes = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0;
++ c = yes ? 2 : 1;
++ cache[dev].store(c);
++ }
++ return c == 2;
++#else
++ return false;
++#endif
++}
++
bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {
if (raw == nullptr || k % 256 != 0 || n <= 0) return false;
const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);
- static const bool old = std::getenv("DS4_DQ_OLD") != nullptr && std::atoi(std::getenv("DS4_DQ_OLD")) != 0;
-#define DS4_DQ(T) dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16)
++ const bool old = dq_use_old_kernel();
+#define DS4_DQ(T) do { \
+ if (old) \
+ dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \
+ else \
+ dq_rows_kernel2<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \
+ } while (0)
switch (type) {
case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;
case kIQ2_XS: DS4_DQ(kIQ2_XS); break;
diff --cc src/dsv41/ds4_model.cu
index 07b88f48e,48d6d54f9..e224ad6a5
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@@ -2962,14 -2953,18 +2963,20 @@@ struct Model::Impl
double exp_b = 0, other_b = 0, room = 0;
const double share = gpu_expert_share(&exp_b, &room, &other_b);
const bool fit = exp_b > 0 && share >= 1.0 / 1.3; // (R9700 x4: 0.86 -> ring 16: decode -1.8% after a prompt, 64: -4.4%, 400: -4.6%)
+ // A box whose experts do not fit (a unified-memory APU) streams them from the disk, and the ring is what lets the
+ // prompt queue its likely ones early: 400 -> 1200 MB on Strix Halo took the queue 3,200 -> 8,087 experts, the disk
+ // 1.49 -> 2.44 GB/s and the compute's exposed read wait 11.4 -> 3.5 s (4K prompt 27.3 -> 21.3 s, 148.98 -> 190.98
+ // tok/s) with decode unchanged (17.54/16.39 against 17.61 tok/s interleaved). A sixteenth of the VRAM room, so a
+ // small pool still loads. DS4_RING_MB overrides.
+ const size_t want = (size_t) std::max<double>(400.0, std::min<double>(1200.0, room / 1e9 * 64.0));
if (o.verbose)
std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,
- room / 1e9, other_b / 1e9, fit ? 16 : 400);
- return fit ? 16 : 400;
+ room / 1e9, other_b / 1e9, fit ? 16 : (int) want);
+ return fit ? 16 : want;
}
+ bool slab_auto_grew_ = false, slab_force_base_ = false; // the automatic attention slab: grew this try / must not grow (it did not fit)
bool alloc_chunks(std::string& err) {
+ slab_auto_grew_ = false;
chunks.resize(devs.size());
const int D = g.dim, H = g.hc, HD = g.hd;
const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
@@@ -2984,33 -2979,15 +2991,48 @@@
split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
const size_t C = (size_t) chunk;
c.C = chunk;
- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);
- // the attention slab's rows: 32 MB of q. Every dense GEMM of a layer is issued once a slab, so the slab width
- // divides the prompt's dense weight traffic (each matrix is re-read once a slab): 8 -> 32 MB took the 32K prompt
- // 325.89 -> 357.78 tok/s, 16K 315.01 -> 328.04, 8K +1.6%, 4K flat, with decode unmoved (16.81 against 17.60 tok/s
- // interleaved, noise-limited). STRATA_DS4_A_MB overrides (8 restores the old slab).
- static const size_t a_mb = [] {
++ // the base slab: STRATA_DS4_A_MB megabytes of q. Every dense GEMM of a layer is issued once a slab, so the slab width divides
++ // the prompt's dense weight traffic. A unified-memory GPU (Strix Halo, main 3133160ed) takes 32 MB: measured 8 -> 32 MB took
++ // its 32K prompt 325.89 -> 357.78 tok/s, 16K 315.01 -> 328.04, 8K +1.6%, 4K flat, decode unmoved. A discrete card keeps 8 MB
++ // (the 0.2.6 slab): its VRAM is the expert tier's, and on the RTX 5070 a bigger slab bought +2% prompt for -3% decode.
++ // STRATA_DS4_A_MB overrides the base on any GPU (8 restores the old slab).
++ bool unified = false;
++ {
++ cudaDeviceProp ip{};
++ unified = cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0;
++ }
++ static const size_t a_mb_env = [] {
+ const char* v = std::getenv("STRATA_DS4_A_MB");
- return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 32);
++ return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 0);
+ }();
++ const size_t a_mb = a_mb_env != 0 ? a_mb_env : (unified ? 32 : 8);
+ c.A = std::max(64, (int) ((a_mb << 20) / (size_t) HQ) / 64 * 64);
+ // DS4_ATTN_SLAB=N: the attention slab's rows. The dense projections (q_b, wo_a/b) are dequantized and GEMM'd
+ // once a slab, so a slab of 256 rows dequantizes the same weights 32 times for an 8K prompt (measured: the
+ // dense dequant 916 ms of the 8K prompt, rocBLAS 867 ms); a larger slab costs VRAM for c.q/c.o and the oa
+ // buffers (A * HQ x 4 bytes each). 0.2.7: unset = automatic (attn_slab_rows: up to 2048 rows where the buffers
+ // take VRAM no expert would have used), 0 = the 0.2.6 rule above, N = that many rows.
+ {
+ int env = -1;
+ if (const char* v = std::getenv("DS4_ATTN_SLAB")) env = std::max(0, std::atoi(v));
- if (slab_force_base_ && env < 0) env = 0;
++ if ((slab_force_base_ || unified || a_mb_env != 0) && env < 0) env = 0; // (no automatic growth on a unified-memory GPU, or when STRATA_DS4_A_MB chose the base)
+ // per row: c.q + c.o (float), c.o16 (half), c.oa (float), c.oa16 (half)
+ const double bpr = (double) HQ * 10.0 + (double) OG * (double) g.o_lora * 6.0;
+ double spare = 1e18; // the expert tier off (--vram-experts 0): the buffers cost no expert
+ if (o.vram_experts > 0.0) {
+ double exp_b = 0, room = 0, other_b = 0;
+ gpu_expert_share(&exp_b, &room, &other_b);
+ spare = (room - other_b - exp_b) / (double) std::max<size_t>(1, devs.size());
+ }
+ const int base = c.A;
+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr);
+ if (env < 0 && c.A > base) {
+ slab_auto_grew_ = true;
+ if (di == 0 && o.verbose)
+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: %.1f GB of VRAM to spare beyond the experts; DS4_ATTN_SLAB=0 keeps %d)\n",
+ c.A, spare / 1e9, base);
+ }
+ }
// (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
c.w16_elems = (size_t) 8 << 20;
@@@ -6383,11 -6282,11 +6417,14 @@@
if (sw_inflight == 0) break; // (nothing queued: every layer's best CPU-lane expert is not hotter than its coldest VRAM one)
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
- if (store && env_f("DS4_SETTLE_STORE", 1.0f) != 0.0f) { // the RAM store too: a window prompt fills it through the CPU
+ // The RAM store's preload after a chunked prompt is off by default: measured on Strix Halo (code fixture, prompt +
+ // 256 tokens, 3 runs a side) it costs the prompt 4.7 s and the decode that follows gains nothing (13.9/14.4 s with it
+ // against 14.2 s without). The swaps above are what the decode actually uses; DS4_SETTLE_STORE=1 restores it.
+ if (store && env_f("DS4_SETTLE_STORE", 0.0f) != 0.0f) { // the RAM store too: a window prompt fills it through the CPU
// lane's picks, a chunked one streams past it - the hottest CPU-lane experts of the prompt into its free slots
+ // (copy_mode: an expert already in the CPU lane's RAM copy gains nothing from a second copy of the same bytes -
+ // measured on the R9700 12 GB cell: 4152 such experts, 40 GB read at 6.4 GB/s, 6.2 s of an 8K prompt, 0.0 store
+ // hits in the decode after it; the VRAM swaps above already read them from the copy)
std::vector<std::pair<float, std::pair<int, int>>> hot;
for (const Layer& l : L) {
if (l.mtp) continue;