Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
spec: cost-aware chain rule keeps the best width seen and looks one or two drafts ahead (the row cost is not convex); unmeasured widths are probed a few times
Niko1221d232721db616
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hppindex d91543d83..ed83abf0c 100644--- a/include/strata/spec/draft_policy.hpp+++ b/include/strata/spec/draft_policy.hpp@@ -46,17 +46,20 @@ public:double lookup_rate(int match) const; // current q for a match lengthdouble cost_ms(int t) const; // measured or scaled round time of a window of t tokens- /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1): the MTP chain is drafted one step at a time and keeps going only while the- /// next row is expected to pay: keep draft j (a window of j + 2 rows instead of j + 1) when- /// (E + q_j) / cost(j + 2) >= (1 + margin) * E / cost(j + 1)- /// with E = 1 + q_0 + .. + q_{j-1} the expected tokens of the window so far, q_j = prod_{i<=j} a(p_i), a() the acceptance measured for- /// MTP drafts of that probability (draft_rate), cost() the measured round time of that window size (cost_ms: it includes the draft).+ /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1): the MTP chain is drafted one step at a time; after draft j (probability p_j)+ /// E_j = 1 + q_0 + .. + q_j, q_i = prod_{k<=i} a(p_k) a() = the acceptance measured for MTP drafts of that probability+ /// is the expected tokens of the window of j + 2 rows, worth E_j / cost(j + 2) tokens per ms (cost() = the measured round time of+ /// that window size, which includes the draft). The window is the best of the widths seen; the chain goes on only while one or two+ /// more drafts could still beat it (the cost is not convex: on a card with experts in RAM a window of 3 costs about a window of 2)./// It only chooses which rows are verified; the output is unchanged.struct ChainState {double q = 1.0, e = 1.0;- int kept = 0; ///< drafts kept so far = the window's rows - 1+ double best_v = 0.0;+ int best_rows = 1; ///< the window to verify: 1 + the drafts kept+ int kept = 0; ///< best_rows - 1};- /// Draft `idx` (0-based, probability p) was just computed: does it join the window? idx == 0 starts a new chain.+ /// Draft `idx` (0-based, probability p) was just computed: update `st` and say whether to compute draft idx + 1.+ /// idx == 0 starts a new chain. `cap` = the widest window (rows).bool keep_draft(ChainState& st, int idx, float p, int cap) const;/// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches draft_rate().void observe_probs(const float* p, int n_window_drafts, int accepted);diff --git a/src/program/generate.cpp b/src/program/generate.cppindex d62fd406d..653b9aad5 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -11037,7 +11037,7 @@ int main(int argc, char** argv) {if (spec_cost && use_mtp)mtp.set_chain_rule([&](int idx, float pj) {const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);- chain_rows = 1 + chain_st.kept;+ chain_rows = chain_st.best_rows;return k;});// The vision path (--vision): GENI <max_new> <embeddings file> <id,id,...> carries images. The file is one@@ -16250,7 +16250,7 @@ int main(int argc, char** argv) {if (spec_cost && use_mtp)mtp.set_chain_rule([&](int idx, float pj) {const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);- chain_rows = 1 + chain_st.kept;+ chain_rows = chain_st.best_rows;return k;});std::vector<int32_t> sbuf((size_t) o.spec, 0);diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cppindex 3359b5fa3..124e3fcaf 100644--- a/src/spec/draft_policy.cpp+++ b/src/spec/draft_policy.cpp@@ -133,16 +133,30 @@ void DraftPolicy::observe_probs(const float* p, int n, int accepted) {}bool DraftPolicy::keep_draft(ChainState& st, int idx, float p, int cap) const {- if (idx == 0) st = ChainState{};- st.q *= draft_rate(p);- if (idx + 2 > std::clamp(cap, 1, max_t_)) return false; // no row left for it- const double c0 = cost_ms(idx + 1), c1 = std::max(c0, cost_ms(idx + 2)); // a wider window never costs less- const bool keep = (st.e + st.q) / c1 >= (1.0 + margin_ / 3.0) * st.e / c0;- if (keep) {- st.e += st.q;- st.kept = idx + 1;+ cap = std::clamp(cap, 1, max_t_);+ if (idx == 0) {+ st = ChainState{};+ st.best_v = 1.0 / cost_ms(1);}- return keep;+ const double a = draft_rate(p);+ st.q *= a;+ st.e += st.q;+ const int rows = idx + 2;+ if (rows <= cap) {+ const double v = st.e / cost_ms(rows);+ if (v > st.best_v * (1.0 + margin_ / 3.0)) {+ st.best_v = v;+ st.best_rows = rows;+ }+ // a width not measured yet is tried a few times when the chain is likely to hold that far+ if (cost_n_[rows] < kProbes && st.q >= 0.15) st.best_rows = rows;+ }+ st.kept = st.best_rows - 1;+ if (rows + 1 > cap) return false; // no row left for another draft+ // one or two more drafts, each as likely to hold as this one did, could they beat the best width so far?+ const double v1 = (st.e + st.q * a) / cost_ms(rows + 1);+ const double v2 = rows + 2 <= cap ? (st.e + st.q * a + st.q * a * a) / cost_ms(rows + 2) : 0.0;+ return std::max(v1, v2) > st.best_v * (1.0 + margin_ / 3.0) || (cost_n_[rows + 1] < kProbes && st.q * a >= 0.15);}double DraftPolicy::chain_rate(int match) const {