Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

spec: cost-aware chain rule keeps the best width seen and looks one or two drafts ahead (the row cost is not convex); unmeasured widths are probed a few times

Niko1221d232721db616
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hpp
index d91543d83..ed83abf0c 100644
--- a/include/strata/spec/draft_policy.hpp
+++ b/include/strata/spec/draft_policy.hpp
@@ -46,17 +46,20 @@ public:
double lookup_rate(int match) const; // current q for a match length
double cost_ms(int t) const; // measured or scaled round time of a window of t tokens
- /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1): the MTP chain is drafted one step at a time and keeps going only while the
- /// next row is expected to pay: keep draft j (a window of j + 2 rows instead of j + 1) when
- /// (E + q_j) / cost(j + 2) >= (1 + margin) * E / cost(j + 1)
- /// with E = 1 + q_0 + .. + q_{j-1} the expected tokens of the window so far, q_j = prod_{i<=j} a(p_i), a() the acceptance measured for
- /// MTP drafts of that probability (draft_rate), cost() the measured round time of that window size (cost_ms: it includes the draft).
+ /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1): the MTP chain is drafted one step at a time; after draft j (probability p_j)
+ /// E_j = 1 + q_0 + .. + q_j, q_i = prod_{k<=i} a(p_k) a() = the acceptance measured for MTP drafts of that probability
+ /// is the expected tokens of the window of j + 2 rows, worth E_j / cost(j + 2) tokens per ms (cost() = the measured round time of
+ /// that window size, which includes the draft). The window is the best of the widths seen; the chain goes on only while one or two
+ /// more drafts could still beat it (the cost is not convex: on a card with experts in RAM a window of 3 costs about a window of 2).
/// It only chooses which rows are verified; the output is unchanged.
struct ChainState {
double q = 1.0, e = 1.0;
- int kept = 0; ///< drafts kept so far = the window's rows - 1
+ double best_v = 0.0;
+ int best_rows = 1; ///< the window to verify: 1 + the drafts kept
+ int kept = 0; ///< best_rows - 1
};
- /// Draft `idx` (0-based, probability p) was just computed: does it join the window? idx == 0 starts a new chain.
+ /// Draft `idx` (0-based, probability p) was just computed: update `st` and say whether to compute draft idx + 1.
+ /// idx == 0 starts a new chain. `cap` = the widest window (rows).
bool keep_draft(ChainState& st, int idx, float p, int cap) const;
/// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches draft_rate().
void observe_probs(const float* p, int n_window_drafts, int accepted);
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index d62fd406d..653b9aad5 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -11037,7 +11037,7 @@ int main(int argc, char** argv) {
if (spec_cost && use_mtp)
mtp.set_chain_rule([&](int idx, float pj) {
const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);
- chain_rows = 1 + chain_st.kept;
+ chain_rows = chain_st.best_rows;
return k;
});
// The vision path (--vision): GENI <max_new> <embeddings file> <id,id,...> carries images. The file is one
@@ -16250,7 +16250,7 @@ int main(int argc, char** argv) {
if (spec_cost && use_mtp)
mtp.set_chain_rule([&](int idx, float pj) {
const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);
- chain_rows = 1 + chain_st.kept;
+ chain_rows = chain_st.best_rows;
return k;
});
std::vector<int32_t> sbuf((size_t) o.spec, 0);
diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cpp
index 3359b5fa3..124e3fcaf 100644
--- a/src/spec/draft_policy.cpp
+++ b/src/spec/draft_policy.cpp
@@ -133,16 +133,30 @@ void DraftPolicy::observe_probs(const float* p, int n, int accepted) {
}
bool DraftPolicy::keep_draft(ChainState& st, int idx, float p, int cap) const {
- if (idx == 0) st = ChainState{};
- st.q *= draft_rate(p);
- if (idx + 2 > std::clamp(cap, 1, max_t_)) return false; // no row left for it
- const double c0 = cost_ms(idx + 1), c1 = std::max(c0, cost_ms(idx + 2)); // a wider window never costs less
- const bool keep = (st.e + st.q) / c1 >= (1.0 + margin_ / 3.0) * st.e / c0;
- if (keep) {
- st.e += st.q;
- st.kept = idx + 1;
+ cap = std::clamp(cap, 1, max_t_);
+ if (idx == 0) {
+ st = ChainState{};
+ st.best_v = 1.0 / cost_ms(1);
}
- return keep;
+ const double a = draft_rate(p);
+ st.q *= a;
+ st.e += st.q;
+ const int rows = idx + 2;
+ if (rows <= cap) {
+ const double v = st.e / cost_ms(rows);
+ if (v > st.best_v * (1.0 + margin_ / 3.0)) {
+ st.best_v = v;
+ st.best_rows = rows;
+ }
+ // a width not measured yet is tried a few times when the chain is likely to hold that far
+ if (cost_n_[rows] < kProbes && st.q >= 0.15) st.best_rows = rows;
+ }
+ st.kept = st.best_rows - 1;
+ if (rows + 1 > cap) return false; // no row left for another draft
+ // one or two more drafts, each as likely to hold as this one did, could they beat the best width so far?
+ const double v1 = (st.e + st.q * a) / cost_ms(rows + 1);
+ const double v2 = rows + 2 <= cap ? (st.e + st.q * a + st.q * a * a) / cost_ms(rows + 2) : 0.0;
+ return std::max(v1, v2) > st.best_v * (1.0 + margin_ / 3.0) || (cost_n_[rows + 1] < kProbes && st.q * a >= 0.15);
}
double DraftPolicy::chain_rate(int match) const {