Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge main 2d07d1c0b (Strix Halo DeepSeek work: fd33cb6ee, 0eea4a3d5, 3133160ed, 2d07d1c0b) into f-027-win. The attention slab: the base is STRATA_DS4_A_MB megabytes of q, 32 MB on a unified-memory GPU (where main measured it) and 8 MB on a discrete card (the 0.2.6 slab); the automatic growth (attn_slab_rows) stays on top for discrete cards with spare VRAM, DS4_ATTN_SLAB still overrides, both env names work. The dense dequantizer: dq_rows_kernel2 stays the default; a gfx11 device (gcnArchName) defaults to main's improved row-per-lane kernel, DS4_DQ_OLD=1/0 forces either.
Niko1221afc45665fc54
diff --cc src/dsv41/ds4_iq.cuindex 7de79b4ea,e6674b4b8..b0539d1dc--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@@ -1353,16 -1341,10 +1388,40 @@@ __global__ void __launch_bounds__(kThre} // namespace++// Which dense dequantizer: dq_rows_kernel2 (the coalesced transpose, 0.2.7 DS Support: 566 -> 592 ppts on the R9700, ids identical) or the++// row-per-lane dq_rows_kernel, which main's fd33cb6ee gave 16-byte stores and a staged input: the Strix Halo (gfx11) 4K++// prompt 79.5 -> 88.5 tok/s with it, so a gfx11 device defaults to it. DS4_DQ_OLD=1 forces it anywhere, DS4_DQ_OLD=0 forces kernel2++// anywhere. The detection is the device's gcnArchName (HIP only; a CUDA build has no gfx11). Decided once per device.++static bool dq_use_old_kernel() {++ if (const char* e = std::getenv("DS4_DQ_OLD"); e != nullptr && *e != '++#if defined(STRATA_USE_HIP)++ static std::atomic<int> cache[64]; // 0 unknown, 1 no, 2 yes++ int dev = 0;++ cudaGetDevice(&dev);++ if (dev < 0 || dev >= 64) return false;++ int c = cache[dev].load();++ if (c == 0) {++ cudaDeviceProp prop{};++ const bool yes = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0;++ c = yes ? 2 : 1;++ cache[dev].store(c);++ }++ return c == 2;++#else++ return false;++#endif++}++bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {if (raw == nullptr || k % 256 != 0 || n <= 0) return false;const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);- static const bool old = std::getenv("DS4_DQ_OLD") != nullptr && std::atoi(std::getenv("DS4_DQ_OLD")) != 0;-#define DS4_DQ(T) dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16)++ const bool old = dq_use_old_kernel();+#define DS4_DQ(T) do { \+ if (old) \+ dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \+ else \+ dq_rows_kernel2<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \+ } while (0)switch (type) {case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;case kIQ2_XS: DS4_DQ(kIQ2_XS); break;diff --cc src/dsv41/ds4_model.cuindex 07b88f48e,48d6d54f9..e224ad6a5--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@@ -2962,14 -2953,18 +2963,20 @@@ struct Model::Impldouble exp_b = 0, other_b = 0, room = 0;const double share = gpu_expert_share(&exp_b, &room, &other_b);const bool fit = exp_b > 0 && share >= 1.0 / 1.3; // (R9700 x4: 0.86 -> ring 16: decode -1.8% after a prompt, 64: -4.4%, 400: -4.6%)+ // A box whose experts do not fit (a unified-memory APU) streams them from the disk, and the ring is what lets the+ // prompt queue its likely ones early: 400 -> 1200 MB on Strix Halo took the queue 3,200 -> 8,087 experts, the disk+ // 1.49 -> 2.44 GB/s and the compute's exposed read wait 11.4 -> 3.5 s (4K prompt 27.3 -> 21.3 s, 148.98 -> 190.98+ // tok/s) with decode unchanged (17.54/16.39 against 17.61 tok/s interleaved). A sixteenth of the VRAM room, so a+ // small pool still loads. DS4_RING_MB overrides.+ const size_t want = (size_t) std::max<double>(400.0, std::min<double>(1200.0, room / 1e9 * 64.0));if (o.verbose)std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,- room / 1e9, other_b / 1e9, fit ? 16 : 400);- return fit ? 16 : 400;+ room / 1e9, other_b / 1e9, fit ? 16 : (int) want);+ return fit ? 16 : want;}+ bool slab_auto_grew_ = false, slab_force_base_ = false; // the automatic attention slab: grew this try / must not grow (it did not fit)bool alloc_chunks(std::string& err) {+ slab_auto_grew_ = false;chunks.resize(devs.size());const int D = g.dim, H = g.hc, HD = g.hd;const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;@@@ -2984,33 -2979,15 +2991,48 @@@split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)const size_t C = (size_t) chunk;c.C = chunk;- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);- // the attention slab's rows: 32 MB of q. Every dense GEMM of a layer is issued once a slab, so the slab width- // divides the prompt's dense weight traffic (each matrix is re-read once a slab): 8 -> 32 MB took the 32K prompt- // 325.89 -> 357.78 tok/s, 16K 315.01 -> 328.04, 8K +1.6%, 4K flat, with decode unmoved (16.81 against 17.60 tok/s- // interleaved, noise-limited). STRATA_DS4_A_MB overrides (8 restores the old slab).- static const size_t a_mb = [] {++ // the base slab: STRATA_DS4_A_MB megabytes of q. Every dense GEMM of a layer is issued once a slab, so the slab width divides++ // the prompt's dense weight traffic. A unified-memory GPU (Strix Halo, main 3133160ed) takes 32 MB: measured 8 -> 32 MB took++ // its 32K prompt 325.89 -> 357.78 tok/s, 16K 315.01 -> 328.04, 8K +1.6%, 4K flat, decode unmoved. A discrete card keeps 8 MB++ // (the 0.2.6 slab): its VRAM is the expert tier's, and on the RTX 5070 a bigger slab bought +2% prompt for -3% decode.++ // STRATA_DS4_A_MB overrides the base on any GPU (8 restores the old slab).++ bool unified = false;++ {++ cudaDeviceProp ip{};++ unified = cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0;++ }++ static const size_t a_mb_env = [] {+ const char* v = std::getenv("STRATA_DS4_A_MB");- return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 32);++ return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 0);+ }();++ const size_t a_mb = a_mb_env != 0 ? a_mb_env : (unified ? 32 : 8);+ c.A = std::max(64, (int) ((a_mb << 20) / (size_t) HQ) / 64 * 64);+ // DS4_ATTN_SLAB=N: the attention slab's rows. The dense projections (q_b, wo_a/b) are dequantized and GEMM'd+ // once a slab, so a slab of 256 rows dequantizes the same weights 32 times for an 8K prompt (measured: the+ // dense dequant 916 ms of the 8K prompt, rocBLAS 867 ms); a larger slab costs VRAM for c.q/c.o and the oa+ // buffers (A * HQ x 4 bytes each). 0.2.7: unset = automatic (attn_slab_rows: up to 2048 rows where the buffers+ // take VRAM no expert would have used), 0 = the 0.2.6 rule above, N = that many rows.+ {+ int env = -1;+ if (const char* v = std::getenv("DS4_ATTN_SLAB")) env = std::max(0, std::atoi(v));- if (slab_force_base_ && env < 0) env = 0;++ if ((slab_force_base_ || unified || a_mb_env != 0) && env < 0) env = 0; // (no automatic growth on a unified-memory GPU, or when STRATA_DS4_A_MB chose the base)+ // per row: c.q + c.o (float), c.o16 (half), c.oa (float), c.oa16 (half)+ const double bpr = (double) HQ * 10.0 + (double) OG * (double) g.o_lora * 6.0;+ double spare = 1e18; // the expert tier off (--vram-experts 0): the buffers cost no expert+ if (o.vram_experts > 0.0) {+ double exp_b = 0, room = 0, other_b = 0;+ gpu_expert_share(&exp_b, &room, &other_b);+ spare = (room - other_b - exp_b) / (double) std::max<size_t>(1, devs.size());+ }+ const int base = c.A;+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr);+ if (env < 0 && c.A > base) {+ slab_auto_grew_ = true;+ if (di == 0 && o.verbose)+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: %.1f GB of VRAM to spare beyond the experts; DS4_ATTN_SLAB=0 keeps %d)\n",+ c.A, spare / 1e9, base);+ }+ }// (the indexer's slab: its scores rows are the context wide - about 64 MB of them)c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));c.w16_elems = (size_t) 8 << 20;@@@ -6383,11 -6282,11 +6417,14 @@@if (sw_inflight == 0) break; // (nothing queued: every layer's best CPU-lane expert is not hotter than its coldest VRAM one)std::this_thread::sleep_for(std::chrono::milliseconds(1));}- if (store && env_f("DS4_SETTLE_STORE", 1.0f) != 0.0f) { // the RAM store too: a window prompt fills it through the CPU+ // The RAM store's preload after a chunked prompt is off by default: measured on Strix Halo (code fixture, prompt ++ // 256 tokens, 3 runs a side) it costs the prompt 4.7 s and the decode that follows gains nothing (13.9/14.4 s with it+ // against 14.2 s without). The swaps above are what the decode actually uses; DS4_SETTLE_STORE=1 restores it.+ if (store && env_f("DS4_SETTLE_STORE", 0.0f) != 0.0f) { // the RAM store too: a window prompt fills it through the CPU// lane's picks, a chunked one streams past it - the hottest CPU-lane experts of the prompt into its free slots+ // (copy_mode: an expert already in the CPU lane's RAM copy gains nothing from a second copy of the same bytes -+ // measured on the R9700 12 GB cell: 4152 such experts, 40 GB read at 6.4 GB/s, 6.2 s of an 8K prompt, 0.0 store+ // hits in the decode after it; the VRAM swaps above already read them from the copy)std::vector<std::pair<float, std::pair<int, int>>> hot;for (const Layer& l : L) {if (l.mtp) continue;