Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

spec: STRATA_SPEC_COST keeps an MTP draft only while the next row pays at the measured row cost (step-by-step chain rule in MtpDrafter, DraftPolicy::keep_draft)

Niko12215f49ec459972
diff --git a/include/strata/core/mtp.hpp b/include/strata/core/mtp.hpp
index 7216f015a..d97d26bdb 100644
--- a/include/strata/core/mtp.hpp
+++ b/include/strata/core/mtp.hpp
@@ -27,6 +27,7 @@
#include <cuda_runtime.h>
#include <cstdint>
+#include <functional>
#include <string>
#include <vector>
@@ -105,6 +106,10 @@ public:
float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);
/// The first round: one cell (`cell`) from `R_row` (device) and `token` -> T-1 drafts.
+ /// STRATA_SPEC_COST: called after each draft step with (index, probability) instead of the `min_p` test; false ends the chain.
+ /// Also called once for the last step of a chain that reached its length (so the rule sees every draft). Empty: min_p decides.
+ void set_chain_rule(std::function<bool(int, float)> f) { chain_rule_ = std::move(f); }
+
bool draft_first(int T, const float* R_row, int32_t token, int64_t cell, int32_t* drafts, std::string& err,
float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);
@@ -117,6 +122,7 @@ public:
/// window's commit (`tail`) and the window's pick at its last accepted row (`next`, the next window's row 0).
void set_draft_history(const int32_t* tail, int64_t n_tail, int32_t next);
void set_ple_session(SessionState* s) { ple_ss_ = s; }
+ std::function<bool(int, float)> chain_rule_;
bool coupled() const { return coupled_active_; }
/// PROBABILISTIC DRAFT ACCEPTANCE (core/spec_prob.hpp, STRATA_SPEC_PROB=1): this request's drafts are SAMPLED from
/// q and each draft j's distribution is in spec_q() + j * kSpecQStride (ids, -1 terminated; then float bits),
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hpp
index 245fa4b15..d91543d83 100644
--- a/include/strata/spec/draft_policy.hpp
+++ b/include/strata/spec/draft_policy.hpp
@@ -46,13 +46,19 @@ public:
double lookup_rate(int match) const; // current q for a match length
double cost_ms(int t) const; // measured or scaled round time of a window of t tokens
- /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1 / --spec-cost): the window of 1..cap rows with the best expected committed
- /// tokens per ms, from the MTP chain's draft probabilities p[0..n) (n = drafts computed; the chain is drafted in full):
- /// E(T) = 1 + q_1 + .. + q_{T-1}, q_j = prod_{i<=j} a(p_i), a() = the acceptance measured for drafts of that probability
- /// cost(T) = the measured round time of that window size (cost_ms, kept non-decreasing)
- /// A wider window is taken only if its E/cost beats the narrower best by `margin`. Only chooses which rows are verified.
- int best_rows(const float* p, int n, int cap) const;
- /// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches a().
+ /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1): the MTP chain is drafted one step at a time and keeps going only while the
+ /// next row is expected to pay: keep draft j (a window of j + 2 rows instead of j + 1) when
+ /// (E + q_j) / cost(j + 2) >= (1 + margin) * E / cost(j + 1)
+ /// with E = 1 + q_0 + .. + q_{j-1} the expected tokens of the window so far, q_j = prod_{i<=j} a(p_i), a() the acceptance measured for
+ /// MTP drafts of that probability (draft_rate), cost() the measured round time of that window size (cost_ms: it includes the draft).
+ /// It only chooses which rows are verified; the output is unchanged.
+ struct ChainState {
+ double q = 1.0, e = 1.0;
+ int kept = 0; ///< drafts kept so far = the window's rows - 1
+ };
+ /// Draft `idx` (0-based, probability p) was just computed: does it join the window? idx == 0 starts a new chain.
+ bool keep_draft(ChainState& st, int idx, float p, int cap) const;
+ /// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches draft_rate().
void observe_probs(const float* p, int n_window_drafts, int accepted);
double draft_rate(double p) const; // acceptance of an MTP draft that came with probability p
diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp
index 58627029d..f4b82b334 100644
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@ -1620,7 +1620,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
};
int n = 0;
- if (min_p <= 0.0f && max_steps > 0) {
+ if (min_p <= 0.0f && !chain_rule_ && max_steps > 0) {
if (cudaGraphLaunch(cp ? round_exec_c_[T] : round_exec_[T], cs_) != cudaSuccess) {
err = std::string("mtp draft: ") + cudaGetErrorString(cudaGetLastError());
return false;
@@ -1686,7 +1686,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
float pj = ((volatile float*) h_prob_)[0];
if (probs) probs[0] = pj;
n = 1;
- for (int j = 1; j < max_steps && pj >= min_p; ++j) {
+ for (int j = 1; j < max_steps && (chain_rule_ ? chain_rule_(j - 1, pj) : pj >= min_p); ++j) {
if (dt) cudaEventRecord(g_dt.eb[j], cs_);
tl = Clock::now();
if (cudaGraphLaunch(cp ? step_exec_c_[j] : step_exec_[j], cs_) != cudaSuccess) {
@@ -1705,6 +1705,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
if (probs) probs[j] = pj;
++n;
}
+ if (chain_rule_ && n >= max_steps) (void) chain_rule_(n - 1, pj); // the last step of a full chain
prefetch_ple(drafts[n - 1]);
const Clock::time_point t_pre_sync = Clock::now();
if (cudaStreamSynchronize(cs_) != cudaSuccess) {
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 19bfa2c53..d62fd406d 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -11032,6 +11032,14 @@ int main(int argc, char** argv) {
strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);
strata::spec::DraftPolicy policy(S); // MTP or lookup window, learned over the whole process
const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }(); // cost-aware MTP window (A/B arm)
+ strata::spec::DraftPolicy::ChainState chain_st;
+ int chain_rows = 1; // the window the cost-aware rule kept in the MTP chain just drafted
+ if (spec_cost && use_mtp)
+ mtp.set_chain_rule([&](int idx, float pj) {
+ const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);
+ chain_rows = 1 + chain_st.kept;
+ return k;
+ });
// The vision path (--vision): GENI <max_new> <embeddings file> <id,id,...> carries images. The file is one
// or more strata-vision records (int32 'SVE1', n, nx, ny, n_embd, then n x n_embd floats) in prompt order;
// each image's rows go to its run of <|image_pad|> tokens, whose M-RoPE positions are mtmd's: t = p,
@@ -14726,7 +14734,7 @@ int main(int argc, char** argv) {
while (!pl_ran && !cancelled && produced_n < max_new) {
int T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft fires
if (use_mtp && spec_cost && !first_window) {
- T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);
+ T = std::clamp(chain_rows, 1, S_mtp);
} else if (use_mtp && req_spec_min_p > 0.0) {
T = 1;
while (T < S_mtp && dprob[(size_t) T - 1] >= (float) req_spec_min_p) ++T;
@@ -15011,7 +15019,7 @@ int main(int argc, char** argv) {
if (use_mtp && hist_n > 0 && mtp.coupled() && !eos && produced_n < max_new)
mtp.set_draft_history(consumed.data(), (int64_t) consumed.size(), outv[(size_t) a]);
const bool drafted = !use_mtp || eos || produced_n >= max_new ||
- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) req_spec_min_p);
+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p);
{
const Clock::time_point tw3 = Clock::now();
auto msd = [](Clock::time_point a0, Clock::time_point b0) { return std::chrono::duration<double, std::milli>(b0 - a0).count(); };
@@ -16208,7 +16216,7 @@ int main(int argc, char** argv) {
// STRATA_SPEC_TRACE=<file> (bench only, off by default): one line per round, "T a sfx ms p1 p2 ..." (window rows, drafts accepted,
// lookup window, round ms, the MTP draft probabilities), for offline window-length policy studies
// STRATA_SPEC_COST=1 (A/B arm, bench): the MTP window by expected tokens per ms at this machine's measured row costs
- // (DraftPolicy::best_rows) instead of "draft probability >= --spec-min-p"; the chain is drafted in full
+ // (DraftPolicy::keep_draft, step by step) instead of "draft probability >= --spec-min-p"
const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }();
std::vector<float> dwin((size_t) o.spec, 0.0f); // the draft probabilities this round's window was chosen from
std::FILE* spec_trace = nullptr;
@@ -16219,7 +16227,7 @@ int main(int argc, char** argv) {
// generated token and the MTP's first cell); otherwise the token loop already did that.
bool first_window = native_pack;
if (use_mtp && !first_window &&
- !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p)) {
+ !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), (float) o.spec_min_p)) {
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
return 1;
}
@@ -16237,6 +16245,14 @@ int main(int argc, char** argv) {
if (use_mtp && S_mtp < o.spec) mtp.set_max_drafts(S_mtp - 1);
strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);
strata::spec::DraftPolicy policy(o.spec); // MTP or lookup window (see draft_policy.hpp)
+ strata::spec::DraftPolicy::ChainState chain_st;
+ int chain_rows = 1; // STRATA_SPEC_COST: the window the cost-aware rule kept in the MTP chain just drafted
+ if (spec_cost && use_mtp)
+ mtp.set_chain_rule([&](int idx, float pj) {
+ const bool k = policy.keep_draft(chain_st, idx, pj, S_mtp);
+ chain_rows = 1 + chain_st.kept;
+ return k;
+ });
std::vector<int32_t> sbuf((size_t) o.spec, 0);
int64_t sfx_windows = 0, sfx_drafts = 0, sfx_ok = 0;
int64_t chain_windows = 0, chain_drafts = 0, chain_ok = 0; // --lookup-chain's own counts
@@ -16282,7 +16298,7 @@ int main(int argc, char** argv) {
strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());
int T = S_mtp;
if (use_mtp && spec_cost && !first_window) {
- T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);
+ T = std::clamp(chain_rows, 1, S_mtp);
} else if (use_mtp && o.spec_min_p > 0.0) {
T = 1;
while (T < S_mtp && dprob[(size_t) T - 1] >= (float) o.spec_min_p) ++T;
@@ -16421,7 +16437,7 @@ int main(int argc, char** argv) {
break;
}
const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new ||
- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p);
+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) o.spec_min_p);
if (adapt_thr.joinable()) adapt_thr.join();
if (!adapt_ok) return 1;
if (!drafted) {
diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cpp
index 8ec78ea56..3359b5fa3 100644
--- a/src/spec/draft_policy.cpp
+++ b/src/spec/draft_policy.cpp
@@ -132,24 +132,17 @@ void DraftPolicy::observe_probs(const float* p, int n, int accepted) {
}
}
-int DraftPolicy::best_rows(const float* p, int n, int cap) const {
- cap = std::clamp(cap, 1, max_t_);
- const int m = std::min(n, cap - 1);
- double q = 1.0, e = 1.0, floor_cost = cost_ms(1);
- double best_v = e / floor_cost;
- int best_t = 1;
- for (int j = 0; j < m; ++j) {
- q *= draft_rate(p[j]);
- e += q;
- const int t = j + 2;
- floor_cost = std::max(floor_cost, cost_ms(t));
- const double v = e / floor_cost;
- if (v > best_v * (1.0 + margin_)) {
- best_v = v;
- best_t = t;
- }
+bool DraftPolicy::keep_draft(ChainState& st, int idx, float p, int cap) const {
+ if (idx == 0) st = ChainState{};
+ st.q *= draft_rate(p);
+ if (idx + 2 > std::clamp(cap, 1, max_t_)) return false; // no row left for it
+ const double c0 = cost_ms(idx + 1), c1 = std::max(c0, cost_ms(idx + 2)); // a wider window never costs less
+ const bool keep = (st.e + st.q) / c1 >= (1.0 + margin_ / 3.0) * st.e / c0;
+ if (keep) {
+ st.e += st.q;
+ st.kept = idx + 1;
}
- return best_t;
+ return keep;
}
double DraftPolicy::chain_rate(int match) const {