Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
spec: STRATA_SPEC_COST, a cost-aware MTP window (DraftPolicy::best_rows: the window with the best expected tokens per ms from the chain's probabilities, a measured probability-to-acceptance table and the measured row costs); the trace now logs the probabilities the window was chosen from (A/B arm, off by default)
Niko1221687810556040
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hppindex 09c06d74d..245fa4b15 100644--- a/include/strata/spec/draft_policy.hpp+++ b/include/strata/spec/draft_policy.hpp@@ -46,6 +46,16 @@ public:double lookup_rate(int match) const; // current q for a match lengthdouble cost_ms(int t) const; // measured or scaled round time of a window of t tokens+ /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1 / --spec-cost): the window of 1..cap rows with the best expected committed+ /// tokens per ms, from the MTP chain's draft probabilities p[0..n) (n = drafts computed; the chain is drafted in full):+ /// E(T) = 1 + q_1 + .. + q_{T-1}, q_j = prod_{i<=j} a(p_i), a() = the acceptance measured for drafts of that probability+ /// cost(T) = the measured round time of that window size (cost_ms, kept non-decreasing)+ /// A wider window is taken only if its E/cost beats the narrower best by `margin`. Only chooses which rows are verified.+ int best_rows(const float* p, int n, int cap) const;+ /// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches a().+ void observe_probs(const float* p, int n_window_drafts, int accepted);+ double draft_rate(double p) const; // acceptance of an MTP draft that came with probability p+private:static int bucket(int match);double mtp_tokens(int t) const;@@ -58,6 +68,8 @@ private:std::array<double, kBuckets> ok_{}, bad_{}; // lookup drafts accepted / windows cut short, decayedstd::array<double, kBuckets> cok_{}, cbad_{}; // chained lookup drafts, the same (reached rounds only)double rounds_ = 0; // rounds observed so far+ static constexpr int kPBuckets = 8;+ std::array<double, kPBuckets> pa_ok_{}, pa_n_{}; // MTP drafts reached / accepted by probability bucket (decayed)std::array<double, kMaxT + 1> last_{}; // the round at which each size was last measured};diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 494b46068..19bfa2c53 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -11031,6 +11031,7 @@ int main(int argc, char** argv) {else if (use_mtp && mtp.max_t() > S_mtp) mtp.set_max_drafts(S_mtp - 1);strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);strata::spec::DraftPolicy policy(S); // MTP or lookup window, learned over the whole process+ const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }(); // cost-aware MTP window (A/B arm)// The vision path (--vision): GENI <max_new> <embeddings file> <id,id,...> carries images. The file is one// or more strata-vision records (int32 'SVE1', n, nx, ny, n_embd, then n x n_embd floats) in prompt order;// each image's rows go to its run of <|image_pad|> tokens, whose M-RoPE positions are mtmd's: t = p,@@ -14015,6 +14016,7 @@ int main(int argc, char** argv) {const size_t WS = (size_t) std::max(S, merge_T);std::vector<int32_t> drafts(DS, 0), window(WS), outv(WS);std::vector<float> dprob(DS, 0.0f);+ std::array<float, 8> dwin_s{}; // the probabilities the current window was chosen from (STRATA_SPEC_COST)std::vector<int32_t> sbuf((size_t) S, 0);const bool sfx_on = o.suffix_draft > 0 || o.lookup_chain > 0;if (sfx_on) {@@ -14723,11 +14725,14 @@ int main(int argc, char** argv) {}while (!pl_ran && !cancelled && produced_n < max_new) {int T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft fires- if (use_mtp && req_spec_min_p > 0.0) {+ if (use_mtp && spec_cost && !first_window) {+ T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);+ } else if (use_mtp && req_spec_min_p > 0.0) {T = 1;while (T < S_mtp && dprob[(size_t) T - 1] >= (float) req_spec_min_p) ++T;}if (first_window) T = merge_T > 0 ? merge_T : 1;+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) std::min<size_t>(dprob.size(), 8), dwin_s.begin());const bool merged = first_window && merge_T > 0; // the prompt's last tokens and the first token// a repeat of earlier context (prompt lookup) where the MTP's own first guess agrees: the policy takes it// when its expected tokens per ms, from the measured acceptance and window costs, beat the MTP window's@@ -15006,7 +15011,7 @@ int main(int argc, char** argv) {if (use_mtp && hist_n > 0 && mtp.coupled() && !eos && produced_n < max_new)mtp.set_draft_history(consumed.data(), (int64_t) consumed.size(), outv[(size_t) a]);const bool drafted = !use_mtp || eos || produced_n >= max_new ||- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p);+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) req_spec_min_p);{const Clock::time_point tw3 = Clock::now();auto msd = [](Clock::time_point a0, Clock::time_point b0) { return std::chrono::duration<double, std::milli>(b0 - a0).count(); };@@ -15027,6 +15032,7 @@ int main(int argc, char** argv) {std::printf("ERR %s\n", err.c_str());return 1;}+ if (spec_cost && use_mtp && timed_round && !from_sfx) policy.observe_probs(dwin_s.data(), T_mtp - 1, a);if (timed_round && !eos && chain_n == 0)policy.observe(from_sfx, T, a, sfx_match,std::chrono::duration<double, std::milli>(Clock::now() - round0).count());@@ -16201,6 +16207,10 @@ int main(int argc, char** argv) {}// STRATA_SPEC_TRACE=<file> (bench only, off by default): one line per round, "T a sfx ms p1 p2 ..." (window rows, drafts accepted,// lookup window, round ms, the MTP draft probabilities), for offline window-length policy studies+ // STRATA_SPEC_COST=1 (A/B arm, bench): the MTP window by expected tokens per ms at this machine's measured row costs+ // (DraftPolicy::best_rows) instead of "draft probability >= --spec-min-p"; the chain is drafted in full+ const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }();+ std::vector<float> dwin((size_t) o.spec, 0.0f); // the draft probabilities this round's window was chosen fromstd::FILE* spec_trace = nullptr;if (const char* tf = std::getenv("STRATA_SPEC_TRACE"); tf != nullptr && tf[0] != '\0') spec_trace = std::fopen(tf, "w");std::vector<double> t_cycle_ms((size_t) o.spec + 1, 0.0);@@ -16209,7 +16219,7 @@ int main(int argc, char** argv) {// generated token and the MTP's first cell); otherwise the token loop already did that.bool first_window = native_pack;if (use_mtp && !first_window &&- !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), (float) o.spec_min_p)) {+ !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p)) {std::fprintf(stderr, "strata generate: %s\n", err.c_str());return 1;}@@ -16271,11 +16281,14 @@ int main(int argc, char** argv) {const Clock::time_point t0 = Clock::now();strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());int T = S_mtp;- if (use_mtp && o.spec_min_p > 0.0) {+ if (use_mtp && spec_cost && !first_window) {+ T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);+ } else if (use_mtp && o.spec_min_p > 0.0) {T = 1;while (T < S_mtp && dprob[(size_t) T - 1] >= (float) o.spec_min_p) ++T;}if (first_window) T = 1;+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) o.spec, dwin.begin());if (!t_cycle.empty() && !first_window) T = t_cycle[(size_t) (rounds % (int64_t) t_cycle.size())];bool from_sfx = false;int sfx_match = 0;@@ -16408,7 +16421,7 @@ int main(int argc, char** argv) {break;}const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new ||- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) o.spec_min_p);+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p);if (adapt_thr.joinable()) adapt_thr.join();if (!adapt_ok) return 1;if (!drafted) {@@ -16421,13 +16434,14 @@ int main(int argc, char** argv) {total_ms += round_ms;if (spec_trace != nullptr && timed_round) {std::fprintf(spec_trace, "%d %d %d %.3f", T, a, from_sfx ? 1 : 0, round_ms);- for (int i = 0; i < T_mtp - 1; ++i) std::fprintf(spec_trace, " %.4f", (double) dprob[(size_t) i]);+ for (int i = 0; i < o.spec - 1; ++i) std::fprintf(spec_trace, " %.4f", (double) dwin[(size_t) i]);std::fprintf(spec_trace, "\n");}if (!t_cycle.empty() && timed_round && T <= o.spec && t_cycle_seen[(size_t) T]++ > 0) {t_cycle_ms[(size_t) T] += round_ms;++t_cycle_n[(size_t) T];}+ if (spec_cost && use_mtp && timed_round && !from_sfx) policy.observe_probs(dwin.data(), T_mtp - 1, a);if (timed_round && chain_n == 0) policy.observe(from_sfx, T, a, sfx_match, round_ms);else if (timed_round) policy.observe_chain(T_mtp, chain_n, a, cm, round_ms);if (rounds % 64 == 0)diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cppindex 8d422789b..8ec78ea56 100644--- a/src/spec/draft_policy.cpp+++ b/src/spec/draft_policy.cpp@@ -12,6 +12,7 @@ namespace {constexpr double kShape[DraftPolicy::kMaxT + 1] = {0.0, 1.0, 1.35, 1.7, 2.05, 2.45, 2.85, 3.25, 3.6};constexpr double kCostAlpha = 0.1; // EMA weight of a new round timeconstexpr double kTokAlpha = 0.05; // EMA weight of a new MTP window outcome+constexpr double kCalDecay = 0.995; // the MTP probability -> acceptance table fades slowlyconstexpr double kDecay = 0.97; // lookup counts: older windows fade// Before a bucket has data: the longer the match, the likelier its continuation (llama.cpp's lookup decoding gates// on the same thing); worth 4 observations, so a few real windows override it@@ -107,6 +108,50 @@ void DraftPolicy::observe(bool lookup, int t, int accepted, int match, double ro}}+namespace {+int p_bucket(double p) {+ const int b = (int) (p * 8.0);+ return b < 0 ? 0 : b > 7 ? 7 : b;+}+} // namespace++// Until a bucket has data a draft is as likely to hold as the MTP says (worth kPriorN observations); measured+// acceptance of the MTP chain is not the same as its own softmax (code and thinking text are over-confident).+double DraftPolicy::draft_rate(double p) const {+ const int b = p_bucket(p);+ const double centre = std::clamp(p, 0.0, 1.0);+ return (pa_ok_[b] + kPriorN * centre) / (pa_n_[b] + kPriorN);+}++void DraftPolicy::observe_probs(const float* p, int n, int accepted) {+ // draft j was reached when the drafts before it held; it held when j < accepted+ for (int j = 0; j < n && j <= accepted; ++j) {+ const int b = p_bucket(p[j]);+ pa_ok_[b] = kCalDecay * pa_ok_[b] + (j < accepted ? 1.0 : 0.0);+ pa_n_[b] = kCalDecay * pa_n_[b] + 1.0;+ }+}++int DraftPolicy::best_rows(const float* p, int n, int cap) const {+ cap = std::clamp(cap, 1, max_t_);+ const int m = std::min(n, cap - 1);+ double q = 1.0, e = 1.0, floor_cost = cost_ms(1);+ double best_v = e / floor_cost;+ int best_t = 1;+ for (int j = 0; j < m; ++j) {+ q *= draft_rate(p[j]);+ e += q;+ const int t = j + 2;+ floor_cost = std::max(floor_cost, cost_ms(t));+ const double v = e / floor_cost;+ if (v > best_v * (1.0 + margin_)) {+ best_v = v;+ best_t = t;+ }+ }+ return best_t;+}+double DraftPolicy::chain_rate(int match) const {// prior: a chained draft is a lookup continuing the MTP's drafts - weaker evidence than a plain match of that lengthconstexpr double kChainPrior[kBuckets] = {0.45, 0.65, 0.8, 0.9};