Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge catch-up: f-dsv41-win fd19f9d6b, f-dsv41-h100 c0f944e12, f-dsv41-cuk

Niko1221428950b96ce0
diff --cc src/dsv41/ds4_model.cu
index 154a8af48,df20a70da,b64d3934b,154a8af48..73a518bcb
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@@@@ -6336,18 -6325,6 -6267,49 -6336,18 +6370,49 @@@@@ struct Model::Impl
std::fwrite(gb.data(), 4, gb.size(), dump_r);
std::fflush(dump_r);
}
++ + if (mi == 0 && !l.mtp && (pred_f != nullptr || xin_f != nullptr)) {
++ + std::lock_guard<std::mutex> lk(dump_mu);
++ + if (pred_f != nullptr) {
++ + float* dp = nullptr;
++ + CK(cudaMalloc(&dp, (size_t) T * 256 * 4));
++ + pred_proj_k<<<T, 256, 0, s>>>(c.x, (int) D, dp);
++ + std::vector<float> hp((size_t) T * 256), hl((size_t) T * l.n_exp);
++ + CK(cudaMemcpyAsync(hp.data(), dp, hp.size() * 4, cudaMemcpyDeviceToHost, s));
++ + CK(cudaMemcpy2DAsync(hl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
++ + cudaMemcpyDeviceToHost, s));
++ + CK(cudaStreamSynchronize(s));
++ + CK(cudaFree(dp));
++ + std::vector<uint16_t> o(hp.size() + hl.size());
++ + for (size_t i = 0; i < hl.size(); ++i) o[i] = strata::kernels::f16_from_f32(hl[i]);
++ + for (size_t i = 0; i < hp.size(); ++i) o[hl.size() + i] = strata::kernels::f16_from_f32(hp[i]);
++ + const uint8_t hd[3] = {(uint8_t) l.il, (uint8_t) (T & 255), (uint8_t) (T >> 8)};
++ + std::fwrite(hd, 1, 3, pred_f);
++ + std::fwrite(o.data(), 2, o.size(), pred_f);
++ + }
++ + if (xin_f != nullptr && xin_got[(size_t) l.il] < xin_n) { // up to 32 evenly spaced tokens of the chunk
++ + const int tk = std::min(std::min(T, 32), xin_n - xin_got[(size_t) l.il]);
++ + const int stride = std::max(1, T / tk);
++ + std::vector<float> hx((size_t) T * D);
++ + CK(cudaMemcpyAsync(hx.data(), c.x, hx.size() * 4, cudaMemcpyDeviceToHost, s));
++ + CK(cudaStreamSynchronize(s));
++ + const uint8_t hd[3] = {(uint8_t) l.il, (uint8_t) (tk & 255), (uint8_t) (tk >> 8)};
++ + std::fwrite(hd, 1, 3, xin_f);
++ + for (int i = 0; i < tk; ++i) std::fwrite(hx.data() + (size_t) std::min(T - 1, i * stride) * D, 4, (size_t) D, xin_f);
++ + xin_got[(size_t) l.il] += tk;
++ + }
++ + }
+ if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids
+ std::vector<int> ri((size_t) T * kPicks);
+ CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};
+ std::fwrite(hd, 1, 3, trace_f);
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ }
if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)
uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
for (int e = 0; e < l.n_exp; ++e) {