Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-dflash2 (the DFlash2 drafter, opt-in) into f-023-int

Niko1221b96f174775a5
diff --cc src/program/generate.cpp
index bd8a398b6,3ec650534..d38eecd33
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -16348,19 -16374,61 +16459,65 @@@ int main(int argc, char** argv)
}
}).detach();
}
+ // the next round's drafts after a commit: the MTP chain, or the DFlash2 block from the committed rows' taps
+ auto draft_next = [&](int T_, int a_) -> bool {
+ #if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2) {
+ static const bool wait_commit_first = std::getenv("STRATA_DF2_WAITCOMMIT") != nullptr; // measurement: no overlap with the async commit
+ if (wait_commit_first && !ver.wait_commit(err)) return false;
+ int32_t dd[strata::core::Dflash2Drafter::kDrafts];
+ float pp[strata::core::Dflash2Drafter::kDrafts];
+ if (!df2.draft(ver.tap_dev(), a_ + 1, p, outv[(size_t) a_], dd, pp, err)) return false;
+ for (size_t i = 0; i < drafts.size() && i < (size_t) strata::core::Dflash2Drafter::kDrafts; ++i) { drafts[i] = dd[i]; dprob[i] = pp[i]; }
+ // what the MTP does while it drafts: the PLE rows of the tokens the next window will hold are read ahead
+ // (the window's gather would otherwise wait for them: 1.8 ms a round on strix)
+ if (ss.ple.ready() && ss.ple.table != nullptr) {
+ int n_pf = 0;
+ const int cap = std::min<int>(S_mtp, (int) drafts.size() + 1);
+ double chain = 1.0;
+ for (int t = 1; t < cap; ++t) {
+ if (o.draft_conf > 0.0) { if ((chain *= (double) dprob[(size_t) t - 1]) < o.draft_conf) break; }
+ else if (o.spec_min_p > 0.0 && dprob[(size_t) t - 1] < (float) o.spec_min_p) break;
+ ++n_pf;
+ }
+ int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};
+ auto pf = [&](int32_t tok) {
+ uint32_t rows16[strata::kernels::PLE_N_HEADS];
+ strata::kernels::ngram_rows(&tok, prev, 1, ss.ple.consts, rows16);
+ prev[0] = prev[1];
+ prev[1] = tok;
+ ss.ple.table->prefetch_rows(rows16);
+ };
+ pf(outv[(size_t) a_]);
+ for (int i = 0; i < n_pf; ++i) pf(drafts[(size_t) i]);
+ }
+ return true;
+ }
+ #endif
+ return mtp.draft(T_, outv.data(), p, a_, drafts.data(), err, dprob.data(), (float) o.spec_min_p);
+ };
+ int win_cap = o.spec; // the widest window the VRAM allows (lowered on an out-of-memory capture, see below)
while ((int64_t) produced.size() < max_new) {
const Clock::time_point t0 = Clock::now();
+ retry_round:
strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());
int T = S_mtp;
- if (use_mtp && o.spec_min_p > 0.0) {
+ policy.set_context(p);
+ if (use_mtp && spec_cost && !first_window) {
+ T = std::clamp(chain_rows, 1, S_mtp);
+ } else if (use_mtp && o.spec_min_p > 0.0) {
T = 1;
while (T < S_mtp && dprob[(size_t) T - 1] >= (float) o.spec_min_p) ++T;
}
+ #if defined(STRATA_HAVE_DFLASH2)
+ if (use_df2 && o.draft_conf > 0.0) { // DFlash2's own confidence: the chance that every draft so far holds, from the selector's probabilities
+ T = 1;
+ double chain = 1.0;
+ while (T < S_mtp && (chain *= (double) dprob[(size_t) T - 1]) >= o.draft_conf) ++T;
+ }
+ #endif
if (first_window) T = 1;
+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) o.spec, dwin.begin());
if (!t_cycle.empty() && !first_window) T = t_cycle[(size_t) (rounds % (int64_t) t_cycle.size())];
bool from_sfx = false;
int sfx_match = 0;