Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
spec: the cost-aware window keeps one row-cost table per context length (< 6K, < 24K, longer), seedable per bucket
Niko1221b1ebc3d7ca40
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hppindex 0254fb719..88970c8bd 100644--- a/include/strata/spec/draft_policy.hpp+++ b/include/strata/spec/draft_policy.hpp@@ -24,6 +24,7 @@ class DraftPolicy {public:static constexpr int kMaxT = 8;static constexpr int kBuckets = 4;+ static constexpr int kCtxBuckets = 3; // STRATA_SPEC_COST: row costs per context length (< 6K, < 24K, longer)explicit DraftPolicy(int max_t, double margin = 0.03);@@ -48,7 +49,10 @@ public:double cost_ms(int t) const; // measured or scaled round time of a window of t tokens/// A per-card row-cost table (round ms of the windows of 1..n rows, the draft included) counted as `weight` earlier rounds each:/// the start of a process that already knows the card (STRATA_SPEC_COST_TABLE; tuning time), replaced by what it then measures.- void seed_costs(const double* ms, int n, double weight);+ void seed_costs(const double* ms, int n, double weight, int bucket = 0);+ /// STRATA_SPEC_COST: the context position of the next round; selects the row-cost table of its length (a row of attention costs+ /// more at 16K than at 1K). A bucket with no measurement yet starts from the nearest one that has. A no-op without set_robust(true).+ void set_context(long long pos);/// The measured round cost of every window size seen: "T1:39.8(n=120) T2:..." for the log / the next process's table.std::string cost_table_text() const;/// STRATA_SPEC_COST: a round time far above what is known for that width (the first use of a width captures its graph; a stalled@@ -81,7 +85,8 @@ private:int max_t_;double margin_;- std::array<double, kMaxT + 1> cost_{}, cost_n_{}; // round ms by window size+ std::array<std::array<double, kMaxT + 1>, kCtxBuckets> cost_{}, cost_n_{}; // round ms by context bucket and window size+ int bk_ = 0; // the current context bucketstd::array<double, kMaxT + 1> mtp_tok_{}, mtp_n_{}; // tokens committed by MTP windows of that sizestd::array<double, kBuckets> ok_{}, bad_{}; // lookup drafts accepted / windows cut short, decayedstd::array<double, kBuckets> cok_{}, cbad_{}; // chained lookup drafts, the same (reached rounds only)@@ -90,7 +95,7 @@ private:double clamp_sample(int t, double ms) const;static constexpr int kPBuckets = 8;std::array<double, kPBuckets> pa_ok_{}, pa_n_{}; // MTP drafts reached / accepted by probability bucket (decayed)- std::array<double, kMaxT + 1> last_{}; // the round at which each size was last measured+ std::array<std::array<double, kMaxT + 1>, kCtxBuckets> last_{}; // the round at which each size was last measured};} // namespace strata::specdiff --git a/src/program/generate.cpp b/src/program/generate.cppindex fc83195bb..c099b1bdb 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -14734,6 +14734,7 @@ int main(int argc, char** argv) {}while (!pl_ran && !cancelled && produced_n < max_new) {int T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft fires+ policy.set_context(p);if (use_mtp && spec_cost && !first_window) {T = std::clamp(chain_rows, 1, S_mtp);} else if (use_mtp && req_spec_min_p > 0.0) {@@ -16249,14 +16250,19 @@ int main(int argc, char** argv) {strata::spec::DraftPolicy::ChainState chain_st;int chain_rows = 1; // STRATA_SPEC_COST: the window the cost-aware rule kept in the MTP chain just draftedif (const char* tab = std::getenv("STRATA_SPEC_COST_TABLE"); spec_cost && tab != nullptr && tab[0] != '\0') {+ // "T1,T2,..;T1,T2,..;T1,..": the row costs of the 3 context buckets (< 6K, < 24K, longer); one list seeds the firstdouble ms[strata::spec::DraftPolicy::kMaxT] = {};- int n = 0;- for (const char* q = tab; *q != '\0' && n < strata::spec::DraftPolicy::kMaxT;) {- ms[n++] = std::atof(q);- while (*q != '\0' && *q != ',') ++q;- if (*q == ',') ++q;+ int n = 0, bucket = 0;+ for (const char* q = tab; *q != '\0' && bucket < strata::spec::DraftPolicy::kCtxBuckets;) {+ if (n < strata::spec::DraftPolicy::kMaxT) ms[n++] = std::atof(q);+ while (*q != '\0' && *q != ',' && *q != ';') ++q;+ const bool end_list = *q == ';' || *q == '\0';+ if (*q != '\0') ++q;+ if (end_list) {+ policy.seed_costs(ms, n, 6.0, bucket++);+ n = 0;+ }}- policy.seed_costs(ms, n, 6.0);}policy.set_robust(spec_cost);if (spec_cost && use_mtp)@@ -16309,6 +16315,7 @@ int main(int argc, char** argv) {const Clock::time_point t0 = Clock::now();strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());int T = S_mtp;+ policy.set_context(p);if (use_mtp && spec_cost && !first_window) {T = std::clamp(chain_rows, 1, S_mtp);} else if (use_mtp && o.spec_min_p > 0.0) {diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cppindex 4651e9c22..ffd9bbbf2 100644--- a/src/spec/draft_policy.cpp+++ b/src/spec/draft_policy.cpp@@ -40,13 +40,13 @@ double DraftPolicy::lookup_rate(int match) const {double DraftPolicy::cost_ms(int t) const {t = std::clamp(t, 1, kMaxT);- if (cost_n_[t] > 0) return cost_[t];+ if (cost_n_[bk_][t] > 0) return cost_[bk_][t];// scale from the measured sizes, weighting each by how often it was seendouble num = 0.0, den = 0.0;for (int u = 1; u <= kMaxT; ++u)- if (cost_n_[u] > 0) {- const double w = std::min(cost_n_[u], 20.0);- num += w * cost_[u] * kShape[t] / kShape[u];+ if (cost_n_[bk_][u] > 0) {+ const double w = std::min(cost_n_[bk_][u], 20.0);+ num += w * cost_[bk_][u] * kShape[t] / kShape[u];den += w;}return den > 0 ? num / den : kShape[t];@@ -55,35 +55,65 @@ double DraftPolicy::cost_ms(int t) const {double DraftPolicy::clamp_sample(int t, double ms) const {if (!robust_) return ms;const double known = cost_ms(t); // the measured value, or the scaled guess from the other widths- if (cost_n_[t] == 0) {+ if (cost_n_[bk_][t] == 0) {// the first use of a width captures its graph: that round says nothing about its cost, the guess from the other widths stands infor (int u = 1; u <= kMaxT; ++u)- if (cost_n_[u] > 0) return known;+ if (cost_n_[bk_][u] > 0) return known;return ms;}return std::min(ms, 2.0 * known);}-void DraftPolicy::seed_costs(const double* ms, int n, double weight) {+void DraftPolicy::seed_costs(const double* ms, int n, double weight, int bucket) {+ bucket = std::clamp(bucket, 0, kCtxBuckets - 1);for (int t = 1; t <= std::min(n, kMaxT); ++t)- if (ms[t - 1] > 0.0 && cost_n_[t] == 0) {- cost_[t] = ms[t - 1];- cost_n_[t] = weight;+ if (ms[t - 1] > 0.0 && cost_n_[bucket][t] == 0) {+ cost_[bucket][t] = ms[t - 1];+ cost_n_[bucket][t] = weight;}}+void DraftPolicy::set_context(long long pos) {+ if (!robust_) return;+ const int nb = pos < 6000 ? 0 : pos < 24000 ? 1 : 2;+ if (nb == bk_) return;+ bool has = false;+ for (int t = 1; t <= kMaxT; ++t) has = has || cost_n_[nb][t] > 0;+ if (!has) { // nothing measured at this length yet: the nearest bucket that has data is the prior (one sample's worth)+ for (int d = 1; d < kCtxBuckets && !has; ++d)+ for (const int src : {nb - d, nb + d}) {+ if (src < 0 || src >= kCtxBuckets || has) continue;+ for (int t = 1; t <= kMaxT; ++t)+ if (cost_n_[src][t] > 0) {+ cost_[nb][t] = cost_[src][t];+ cost_n_[nb][t] = 1.0;+ last_[nb][t] = rounds_;+ has = true;+ }+ }+ }+ bk_ = nb;+}+std::string DraftPolicy::cost_table_text() const {std::string out;char b[64];- for (int t = 1; t <= kMaxT; ++t)- if (cost_n_[t] > 0) {- std::snprintf(b, sizeof b, "%sT%d:%.2f(n=%.0f)", out.empty() ? "" : " ", t, cost_[t], cost_n_[t]);- out += b;- }+ for (int k = 0; k < kCtxBuckets; ++k) {+ bool any = false;+ for (int t = 1; t <= kMaxT; ++t) any = any || cost_n_[k][t] > 0;+ if (!any) continue;+ out += (out.empty() ? "" : " | ");+ out += k == 0 ? "ctx<6K:" : k == 1 ? "ctx<24K:" : "ctx>=24K:";+ for (int t = 1; t <= kMaxT; ++t)+ if (cost_n_[k][t] > 0) {+ std::snprintf(b, sizeof b, " T%d:%.2f(n=%.0f)", t, cost_[k][t], cost_n_[k][t]);+ out += b;+ }+ }return out;}-bool DraftPolicy::stale(int t) const { return cost_n_[t] > 0 && rounds_ - last_[t] > kStaleRounds; }+bool DraftPolicy::stale(int t) const { return cost_n_[bk_][t] > 0 && rounds_ - last_[bk_][t] > kStaleRounds; }double DraftPolicy::mtp_tokens(int t) const {if (mtp_n_[t] > 0) return mtp_tok_[t];@@ -114,7 +144,7 @@ DraftPolicy::Pick DraftPolicy::choose(int t_mtp, int lookup_k, int match) const}// a guessed cost can keep the policy from ever measuring a size: the first few times a confident lookup would// need a size not measured yet, it is tried (verification keeps the output; only the one round's speed is at stake)- if (t_full > p.t && (cost_n_[t_full] < kProbes || stale(t_full)) && q >= 0.85) {+ if (t_full > p.t && (cost_n_[bk_][t_full] < kProbes || stale(t_full)) && q >= 0.85) {p.lookup = true;p.t = t_full;}@@ -125,10 +155,10 @@ void DraftPolicy::observe(bool lookup, int t, int accepted, int match, double rot = std::clamp(t, 1, kMaxT);if (round_ms > 0) {round_ms = clamp_sample(t, round_ms);- const double alpha = robust_ ? std::max(kCostAlpha, 1.0 / (cost_n_[t] + 1.0)) : kCostAlpha; // a running mean for the first samples- cost_[t] = cost_n_[t] > 0 ? (1.0 - alpha) * cost_[t] + alpha * round_ms : round_ms;- cost_n_[t] += 1.0;- last_[t] = rounds_;+ const double alpha = robust_ ? std::max(kCostAlpha, 1.0 / (cost_n_[bk_][t] + 1.0)) : kCostAlpha; // a running mean for the first samples+ cost_[bk_][t] = cost_n_[bk_][t] > 0 ? (1.0 - alpha) * cost_[bk_][t] + alpha * round_ms : round_ms;+ cost_n_[bk_][t] += 1.0;+ last_[bk_][t] = rounds_;}rounds_ += 1.0;if (lookup) {@@ -183,14 +213,14 @@ bool DraftPolicy::keep_draft(ChainState& st, int idx, float p, int cap) const {st.best_rows = rows;}// a width not measured yet is tried a few times when the chain is likely to hold that far- if (cost_n_[rows] < kProbes && st.q >= 0.15) st.best_rows = rows;+ if (cost_n_[bk_][rows] < kProbes && st.q >= 0.15) st.best_rows = rows;}st.kept = st.best_rows - 1;if (rows + 1 > cap) return false; // no row left for another draft// one or two more drafts, each as likely to hold as this one did, could they beat the best width so far?const double v1 = (st.e + st.q * a) / cost_ms(rows + 1);const double v2 = rows + 2 <= cap ? (st.e + st.q * a + st.q * a * a) / cost_ms(rows + 2) : 0.0;- return std::max(v1, v2) > st.best_v * (1.0 + margin_ / 3.0) || (cost_n_[rows + 1] < kProbes && st.q * a >= 0.15);+ return std::max(v1, v2) > st.best_v * (1.0 + margin_ / 3.0) || (cost_n_[bk_][rows + 1] < kProbes && st.q * a >= 0.15);}double DraftPolicy::chain_rate(int match) const {@@ -221,7 +251,7 @@ int DraftPolicy::chain(int t_mtp, double p_mtp, int k_avail, int match) const {return best_k;}// as choose(): a size whose cost is only guessed is tried a few times when the continuation looks likely- if ((cost_n_[t_full] < kProbes || stale(t_full)) && p_mtp * c >= 0.6) return kmax;+ if ((cost_n_[bk_][t_full] < kProbes || stale(t_full)) && p_mtp * c >= 0.6) return kmax;return 0;}@@ -229,10 +259,10 @@ void DraftPolicy::observe_chain(int t_mtp, int k, int accepted, int match, doublconst int t = std::clamp(t_mtp + k, 1, kMaxT);if (round_ms > 0) {round_ms = clamp_sample(t, round_ms);- const double alpha = robust_ ? std::max(kCostAlpha, 1.0 / (cost_n_[t] + 1.0)) : kCostAlpha; // a running mean for the first samples- cost_[t] = cost_n_[t] > 0 ? (1.0 - alpha) * cost_[t] + alpha * round_ms : round_ms;- cost_n_[t] += 1.0;- last_[t] = rounds_;+ const double alpha = robust_ ? std::max(kCostAlpha, 1.0 / (cost_n_[bk_][t] + 1.0)) : kCostAlpha; // a running mean for the first samples+ cost_[bk_][t] = cost_n_[bk_][t] > 0 ? (1.0 - alpha) * cost_[bk_][t] + alpha * round_ms : round_ms;+ cost_n_[bk_][t] += 1.0;+ last_[bk_][t] = rounds_;}rounds_ += 1.0;// the MTP part is an ordinary MTP window as far as its own drafts go