Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
spec: STRATA_SPEC_COST keeps an MTP draft only while the next row pays at the measured row cost (step-by-step chain rule in MtpDrafter, DraftPolicy::keep_draft)
Niko12215f49ec459972
diff --git a/include/strata/core/mtp.hpp b/include/strata/core/mtp.hppindex 7216f015a..d97d26bdb 100644--- a/include/strata/core/mtp.hpp+++ b/include/strata/core/mtp.hpp@@ -27,6 +27,7 @@#include <cuda_runtime.h>#include <cstdint>+#include <functional>#include <string>#include <vector>@@ -105,6 +106,10 @@ public:float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);/// The first round: one cell (`cell`) from `R_row` (device) and `token` -> T-1 drafts.+ /// STRATA_SPEC_COST: called after each draft step with (index, probability) instead of the `min_p` test; false ends the chain.+ /// Also called once for the last step of a chain that reached its length (so the rule sees every draft). Empty: min_p decides.+ void set_chain_rule(std::function<bool(int, float)> f) { chain_rule_ = std::move(f); }+bool draft_first(int T, const float* R_row, int32_t token, int64_t cell, int32_t* drafts, std::string& err,float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);@@ -117,6 +122,7 @@ public:/// window's commit (`tail`) and the window's pick at its last accepted row (`next`, the next window's row 0).void set_draft_history(const int32_t* tail, int64_t n_tail, int32_t next);void set_ple_session(SessionState* s) { ple_ss_ = s; }+ std::function<bool(int, float)> chain_rule_;bool coupled() const { return coupled_active_; }/// PROBABILISTIC DRAFT ACCEPTANCE (core/spec_prob.hpp, STRATA_SPEC_PROB=1): this request's drafts are SAMPLED from/// q and each draft j's distribution is in spec_q() + j * kSpecQStride (ids, -1 terminated; then float bits),diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hppindex 245fa4b15..d91543d83 100644--- a/include/strata/spec/draft_policy.hpp+++ b/include/strata/spec/draft_policy.hpp@@ -46,13 +46,19 @@ public:double lookup_rate(int match) const; // current q for a match lengthdouble cost_ms(int t) const; // measured or scaled round time of a window of t tokens- /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1 / --spec-cost): the window of 1..cap rows with the best expected committed- /// tokens per ms, from the MTP chain's draft probabilities p[0..n) (n = drafts computed; the chain is drafted in full):- /// E(T) = 1 + q_1 + .. + q_{T-1}, q_j = prod_{i<=j} a(p_i), a() = the acceptance measured for drafts of that probability- /// cost(T) = the measured round time of that window size (cost_ms, kept non-decreasing)- /// A wider window is taken only if its E/cost beats the narrower best by `margin`. Only chooses which rows are verified.- int best_rows(const float* p, int n, int cap) const;- /// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches a().+ /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1): the MTP chain is drafted one step at a time and keeps going only while the+ /// next row is expected to pay: keep draft j (a window of j + 2 rows instead of j + 1) when+ /// (E + q_j) / cost(j + 2) >= (1 + margin) * E / cost(j + 1)+ /// with E = 1 + q_0 + .. + q_{j-1} the expected tokens of the window so far, q_j = prod_{i<=j} a(p_i), a() the acceptance measured for+ /// MTP drafts of that probability (draft_rate), cost() the measured round time of that window size (cost_ms: it includes the draft).+ /// It only chooses which rows are verified; the output is unchanged.+ struct ChainState {+ double q = 1.0, e = 1.0;+ int kept = 0; ///< drafts kept so far = the window's rows - 1+ };+ /// Draft `idx` (0-based, probability p) was just computed: does it join the window? idx == 0 starts a new chain.+ bool keep_draft(ChainState& st, int idx, float p, int cap) const;+ /// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches draft_rate().void observe_probs(const float* p, int n_window_drafts, int accepted);double draft_rate(double p) const; // acceptance of an MTP draft that came with probability pdiff --git a/src/core/mtp.cpp b/src/core/mtp.cppindex 58627029d..f4b82b334 100644--- a/src/core/mtp.cpp+++ b/src/core/mtp.cpp@@ -1620,7 +1620,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*};int n = 0;- if (min_p <= 0.0f && max_steps > 0) {+ if (min_p <= 0.0f && !chain_rule_ && max_steps > 0) {if (cudaGraphLaunch(cp ? round_exec_c_[T] : round_exec_[T], cs_) != cudaSuccess) {err = std::string("mtp draft: ") + cudaGetErrorString(cudaGetLastError());return false;@@ -1686,7 +1686,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*float pj = ((volatile float*) h_prob_)[0];if (probs) probs[0] = pj;n = 1;- for (int j = 1; j < max_steps && pj >= min_p; ++j) {+ for (int j = 1; j < max_steps && (chain_rule_ ? chain_rule_(j - 1, pj) : pj >= min_p); ++j) {if (dt) cudaEventRecord(g_dt.eb[j], cs_);tl = Clock::now();if (cudaGraphLaunch(cp ? step_exec_c_[j] : step_exec_[j], cs_) != cudaSuccess) {@@ -1705,6 +1705,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*if (probs) probs[j] = pj;++n;}+ if (chain_rule_ && n >= max_steps) (void) chain_rule_(n - 1, pj); // the last step of a full chainprefetch_ple(drafts[n - 1]);const Clock::time_point t_pre_sync = Clock::now();if (cudaStreamSynchronize(cs_) != cudaSuccess) {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 19bfa2c53..d62fd406d 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -11032,6 +11032,14 @@ int main(int argc, char** argv) {strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);strata::spec::DraftPolicy policy(S); // MTP or lookup window, learned over the whole processconst bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }(); // cost-aware MTP window (A/B arm)+ strata::spec::DraftPolicy::ChainState chain_st;+ int chain_rows = 1; // the window the cost-aware rule kept in the MTP chain just drafted+ if (spec_cost && use_mtp)+ mtp.set_chain_rule([&](int idx, float pj) {+ const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);+ chain_rows = 1 + chain_st.kept;+ return k;+ });// The vision path (--vision): GENI <max_new> <embeddings file> <id,id,...> carries images. The file is one// or more strata-vision records (int32 'SVE1', n, nx, ny, n_embd, then n x n_embd floats) in prompt order;// each image's rows go to its run of <|image_pad|> tokens, whose M-RoPE positions are mtmd's: t = p,@@ -14726,7 +14734,7 @@ int main(int argc, char** argv) {while (!pl_ran && !cancelled && produced_n < max_new) {int T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft firesif (use_mtp && spec_cost && !first_window) {- T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);+ T = std::clamp(chain_rows, 1, S_mtp);} else if (use_mtp && req_spec_min_p > 0.0) {T = 1;while (T < S_mtp && dprob[(size_t) T - 1] >= (float) req_spec_min_p) ++T;@@ -15011,7 +15019,7 @@ int main(int argc, char** argv) {if (use_mtp && hist_n > 0 && mtp.coupled() && !eos && produced_n < max_new)mtp.set_draft_history(consumed.data(), (int64_t) consumed.size(), outv[(size_t) a]);const bool drafted = !use_mtp || eos || produced_n >= max_new ||- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) req_spec_min_p);+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p);{const Clock::time_point tw3 = Clock::now();auto msd = [](Clock::time_point a0, Clock::time_point b0) { return std::chrono::duration<double, std::milli>(b0 - a0).count(); };@@ -16208,7 +16216,7 @@ int main(int argc, char** argv) {// STRATA_SPEC_TRACE=<file> (bench only, off by default): one line per round, "T a sfx ms p1 p2 ..." (window rows, drafts accepted,// lookup window, round ms, the MTP draft probabilities), for offline window-length policy studies// STRATA_SPEC_COST=1 (A/B arm, bench): the MTP window by expected tokens per ms at this machine's measured row costs- // (DraftPolicy::best_rows) instead of "draft probability >= --spec-min-p"; the chain is drafted in full+ // (DraftPolicy::keep_draft, step by step) instead of "draft probability >= --spec-min-p"const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }();std::vector<float> dwin((size_t) o.spec, 0.0f); // the draft probabilities this round's window was chosen fromstd::FILE* spec_trace = nullptr;@@ -16219,7 +16227,7 @@ int main(int argc, char** argv) {// generated token and the MTP's first cell); otherwise the token loop already did that.bool first_window = native_pack;if (use_mtp && !first_window &&- !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p)) {+ !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), (float) o.spec_min_p)) {std::fprintf(stderr, "strata generate: %s\n", err.c_str());return 1;}@@ -16237,6 +16245,14 @@ int main(int argc, char** argv) {if (use_mtp && S_mtp < o.spec) mtp.set_max_drafts(S_mtp - 1);strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);strata::spec::DraftPolicy policy(o.spec); // MTP or lookup window (see draft_policy.hpp)+ strata::spec::DraftPolicy::ChainState chain_st;+ int chain_rows = 1; // STRATA_SPEC_COST: the window the cost-aware rule kept in the MTP chain just drafted+ if (spec_cost && use_mtp)+ mtp.set_chain_rule([&](int idx, float pj) {+ const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);+ chain_rows = 1 + chain_st.kept;+ return k;+ });std::vector<int32_t> sbuf((size_t) o.spec, 0);int64_t sfx_windows = 0, sfx_drafts = 0, sfx_ok = 0;int64_t chain_windows = 0, chain_drafts = 0, chain_ok = 0; // --lookup-chain's own counts@@ -16282,7 +16298,7 @@ int main(int argc, char** argv) {strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());int T = S_mtp;if (use_mtp && spec_cost && !first_window) {- T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);+ T = std::clamp(chain_rows, 1, S_mtp);} else if (use_mtp && o.spec_min_p > 0.0) {T = 1;while (T < S_mtp && dprob[(size_t) T - 1] >= (float) o.spec_min_p) ++T;@@ -16421,7 +16437,7 @@ int main(int argc, char** argv) {break;}const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new ||- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p);+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) o.spec_min_p);if (adapt_thr.joinable()) adapt_thr.join();if (!adapt_ok) return 1;if (!drafted) {diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cppindex 8ec78ea56..3359b5fa3 100644--- a/src/spec/draft_policy.cpp+++ b/src/spec/draft_policy.cpp@@ -132,24 +132,17 @@ void DraftPolicy::observe_probs(const float* p, int n, int accepted) {}}-int DraftPolicy::best_rows(const float* p, int n, int cap) const {- cap = std::clamp(cap, 1, max_t_);- const int m = std::min(n, cap - 1);- double q = 1.0, e = 1.0, floor_cost = cost_ms(1);- double best_v = e / floor_cost;- int best_t = 1;- for (int j = 0; j < m; ++j) {- q *= draft_rate(p[j]);- e += q;- const int t = j + 2;- floor_cost = std::max(floor_cost, cost_ms(t));- const double v = e / floor_cost;- if (v > best_v * (1.0 + margin_)) {- best_v = v;- best_t = t;- }+bool DraftPolicy::keep_draft(ChainState& st, int idx, float p, int cap) const {+ if (idx == 0) st = ChainState{};+ st.q *= draft_rate(p);+ if (idx + 2 > std::clamp(cap, 1, max_t_)) return false; // no row left for it+ const double c0 = cost_ms(idx + 1), c1 = std::max(c0, cost_ms(idx + 2)); // a wider window never costs less+ const bool keep = (st.e + st.q) / c1 >= (1.0 + margin_ / 3.0) * st.e / c0;+ if (keep) {+ st.e += st.q;+ st.kept = idx + 1;}- return best_t;+ return keep;}double DraftPolicy::chain_rate(int match) const {