Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-024-hipk (cdf8b98f5) into f-024-rel: gfx12 WMMA i-quant prompt tiles for DeepSeek V4.1 (HIP only, default on for gfx12, STRATA_DS4_HIPK=0 off)

Niko1221aebdc298f9c2
diff --cc src/dsv41/ds4_model.cu
index 29c2f48dd,2126ab33c..d9e75cb8e
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@@ -6275,37 -6300,37 +6323,39 @@@ struct Model::Impl
hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ prof_mark(d, 5);
- if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
- !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
+ if (!kv_only && (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err)))
return false;
if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
+ prof_mark(d, 6);
- Finish fq;
- fq.w = l.q_norm;
- fq.eps = g.eps;
- finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
- x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
- // the window rows: the ring's (positions p0 - win ..) then the chunk's
- ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) p0;
- fk.freqs = freqs;
- fk.quant = 1;
- fk.dst = c.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(c.kvr, T, HD, HD, fk, s);
- // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
- const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
- if (snap) {
- CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
- if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
- }
- ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ const bool snap = !kv_only && wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
+ if (!kv_only) {
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
+ // the window rows: the ring's (positions p0 - win ..) then the chunk's
+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = c.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(c.kvr, T, HD, HD, fk, s);
+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
+ if (snap) {
+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
+ }
+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ }
const uint8_t* ckv = nullptr;
const long long ss = (long long) o.max_context + 64;
if (l.ratio > 0) {
@@@ -6434,7 -6459,7 +6484,8 @@@
}
ckv = sr.ckv[(size_t) l.dev];
}
+ if (kv_only) return true;
+ prof_mark(d, 7);
// the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {