Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-024-hipk (cdf8b98f5) into f-024-rel: gfx12 WMMA i-quant prompt tiles for DeepSeek V4.1 (HIP only, default on for gfx12, STRATA_DS4_HIPK=0 off)
Niko1221aebdc298f9c2
diff --cc src/dsv41/ds4_model.cuindex 29c2f48dd,2126ab33c..d9e75cb8e--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@@ -6275,37 -6300,37 +6323,39 @@@ struct Model::Implhc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);+ prof_mark(d, 5);- if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||- !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))+ if (!kv_only && (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err)))return false;if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;+ prof_mark(d, 6);- Finish fq;- fq.w = l.q_norm;- fq.eps = g.eps;- finish(c.qr, T, g.q_lora, g.q_lora, fq, s);- x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);- // the window rows: the ring's (positions p0 - win ..) then the chunk's- ring_to_ext(l.win, c.ext, p0, g.win, HD, s);- Finish fk;- fk.w = l.kv_norm;- fk.eps = g.eps;- fk.rd = g.rd;- fk.pos = (int) p0;- fk.freqs = freqs;- fk.quant = 1;- fk.dst = c.ext + (size_t) g.win * HD;- fk.dst_stride = HD;- finish(c.kvr, T, HD, HD, fk, s);- // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it- const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;- if (snap) {- CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));- if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);- }- ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);+ const bool snap = !kv_only && wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;+ if (!kv_only) {+ Finish fq;+ fq.w = l.q_norm;+ fq.eps = g.eps;+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);+ // the window rows: the ring's (positions p0 - win ..) then the chunk's+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) p0;+ fk.freqs = freqs;+ fk.quant = 1;+ fk.dst = c.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(c.kvr, T, HD, HD, fk, s);+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it+ if (snap) {+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);+ }+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);+ }const uint8_t* ckv = nullptr;const long long ss = (long long) o.max_context + 64;if (l.ratio > 0) {@@@ -6434,7 -6459,7 +6484,8 @@@}ckv = sr.ckv[(size_t) l.dev];}+ if (kv_only) return true;+ prof_mark(d, 7);// the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into accif (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {