Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

spec: STRATA_SPEC_COST, a cost-aware MTP window (DraftPolicy::best_rows: the window with the best expected tokens per ms from the chain's probabilities, a measured probability-to-acceptance table and the measured row costs); the trace now logs the probabilities the window was chosen from (A/B arm, off by default)

Niko1221687810556040
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hpp
index 09c06d74d..245fa4b15 100644
--- a/include/strata/spec/draft_policy.hpp
+++ b/include/strata/spec/draft_policy.hpp
@@ -46,6 +46,16 @@ public:
double lookup_rate(int match) const; // current q for a match length
double cost_ms(int t) const; // measured or scaled round time of a window of t tokens
+ /// COST-AWARE MTP WINDOW (STRATA_SPEC_COST=1 / --spec-cost): the window of 1..cap rows with the best expected committed
+ /// tokens per ms, from the MTP chain's draft probabilities p[0..n) (n = drafts computed; the chain is drafted in full):
+ /// E(T) = 1 + q_1 + .. + q_{T-1}, q_j = prod_{i<=j} a(p_i), a() = the acceptance measured for drafts of that probability
+ /// cost(T) = the measured round time of that window size (cost_ms, kept non-decreasing)
+ /// A wider window is taken only if its E/cost beats the narrower best by `margin`. Only chooses which rows are verified.
+ int best_rows(const float* p, int n, int cap) const;
+ /// After a round of the MTP window T whose drafts were p[0..T-1): `accepted` of them held. Teaches a().
+ void observe_probs(const float* p, int n_window_drafts, int accepted);
+ double draft_rate(double p) const; // acceptance of an MTP draft that came with probability p
+
private:
static int bucket(int match);
double mtp_tokens(int t) const;
@@ -58,6 +68,8 @@ private:
std::array<double, kBuckets> ok_{}, bad_{}; // lookup drafts accepted / windows cut short, decayed
std::array<double, kBuckets> cok_{}, cbad_{}; // chained lookup drafts, the same (reached rounds only)
double rounds_ = 0; // rounds observed so far
+ static constexpr int kPBuckets = 8;
+ std::array<double, kPBuckets> pa_ok_{}, pa_n_{}; // MTP drafts reached / accepted by probability bucket (decayed)
std::array<double, kMaxT + 1> last_{}; // the round at which each size was last measured
};
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 494b46068..19bfa2c53 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -11031,6 +11031,7 @@ int main(int argc, char** argv) {
else if (use_mtp && mtp.max_t() > S_mtp) mtp.set_max_drafts(S_mtp - 1);
strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);
strata::spec::DraftPolicy policy(S); // MTP or lookup window, learned over the whole process
+ const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }(); // cost-aware MTP window (A/B arm)
// The vision path (--vision): GENI <max_new> <embeddings file> <id,id,...> carries images. The file is one
// or more strata-vision records (int32 'SVE1', n, nx, ny, n_embd, then n x n_embd floats) in prompt order;
// each image's rows go to its run of <|image_pad|> tokens, whose M-RoPE positions are mtmd's: t = p,
@@ -14015,6 +14016,7 @@ int main(int argc, char** argv) {
const size_t WS = (size_t) std::max(S, merge_T);
std::vector<int32_t> drafts(DS, 0), window(WS), outv(WS);
std::vector<float> dprob(DS, 0.0f);
+ std::array<float, 8> dwin_s{}; // the probabilities the current window was chosen from (STRATA_SPEC_COST)
std::vector<int32_t> sbuf((size_t) S, 0);
const bool sfx_on = o.suffix_draft > 0 || o.lookup_chain > 0;
if (sfx_on) {
@@ -14723,11 +14725,14 @@ int main(int argc, char** argv) {
}
while (!pl_ran && !cancelled && produced_n < max_new) {
int T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft fires
- if (use_mtp && req_spec_min_p > 0.0) {
+ if (use_mtp && spec_cost && !first_window) {
+ T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);
+ } else if (use_mtp && req_spec_min_p > 0.0) {
T = 1;
while (T < S_mtp && dprob[(size_t) T - 1] >= (float) req_spec_min_p) ++T;
}
if (first_window) T = merge_T > 0 ? merge_T : 1;
+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) std::min<size_t>(dprob.size(), 8), dwin_s.begin());
const bool merged = first_window && merge_T > 0; // the prompt's last tokens and the first token
// a repeat of earlier context (prompt lookup) where the MTP's own first guess agrees: the policy takes it
// when its expected tokens per ms, from the measured acceptance and window costs, beat the MTP window's
@@ -15006,7 +15011,7 @@ int main(int argc, char** argv) {
if (use_mtp && hist_n > 0 && mtp.coupled() && !eos && produced_n < max_new)
mtp.set_draft_history(consumed.data(), (int64_t) consumed.size(), outv[(size_t) a]);
const bool drafted = !use_mtp || eos || produced_n >= max_new ||
- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p);
+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) req_spec_min_p);
{
const Clock::time_point tw3 = Clock::now();
auto msd = [](Clock::time_point a0, Clock::time_point b0) { return std::chrono::duration<double, std::milli>(b0 - a0).count(); };
@@ -15027,6 +15032,7 @@ int main(int argc, char** argv) {
std::printf("ERR %s\n", err.c_str());
return 1;
}
+ if (spec_cost && use_mtp && timed_round && !from_sfx) policy.observe_probs(dwin_s.data(), T_mtp - 1, a);
if (timed_round && !eos && chain_n == 0)
policy.observe(from_sfx, T, a, sfx_match,
std::chrono::duration<double, std::milli>(Clock::now() - round0).count());
@@ -16201,6 +16207,10 @@ int main(int argc, char** argv) {
}
// STRATA_SPEC_TRACE=<file> (bench only, off by default): one line per round, "T a sfx ms p1 p2 ..." (window rows, drafts accepted,
// lookup window, round ms, the MTP draft probabilities), for offline window-length policy studies
+ // STRATA_SPEC_COST=1 (A/B arm, bench): the MTP window by expected tokens per ms at this machine's measured row costs
+ // (DraftPolicy::best_rows) instead of "draft probability >= --spec-min-p"; the chain is drafted in full
+ const bool spec_cost = [] { const char* v = std::getenv("STRATA_SPEC_COST"); return v != nullptr && v[0] == '1'; }();
+ std::vector<float> dwin((size_t) o.spec, 0.0f); // the draft probabilities this round's window was chosen from
std::FILE* spec_trace = nullptr;
if (const char* tf = std::getenv("STRATA_SPEC_TRACE"); tf != nullptr && tf[0] != '\0') spec_trace = std::fopen(tf, "w");
std::vector<double> t_cycle_ms((size_t) o.spec + 1, 0.0);
@@ -16209,7 +16219,7 @@ int main(int argc, char** argv) {
// generated token and the MTP's first cell); otherwise the token loop already did that.
bool first_window = native_pack;
if (use_mtp && !first_window &&
- !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), (float) o.spec_min_p)) {
+ !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p)) {
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
return 1;
}
@@ -16271,11 +16281,14 @@ int main(int argc, char** argv) {
const Clock::time_point t0 = Clock::now();
strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());
int T = S_mtp;
- if (use_mtp && o.spec_min_p > 0.0) {
+ if (use_mtp && spec_cost && !first_window) {
+ T = policy.best_rows(dprob.data(), S_mtp - 1, S_mtp);
+ } else if (use_mtp && o.spec_min_p > 0.0) {
T = 1;
while (T < S_mtp && dprob[(size_t) T - 1] >= (float) o.spec_min_p) ++T;
}
if (first_window) T = 1;
+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) o.spec, dwin.begin());
if (!t_cycle.empty() && !first_window) T = t_cycle[(size_t) (rounds % (int64_t) t_cycle.size())];
bool from_sfx = false;
int sfx_match = 0;
@@ -16408,7 +16421,7 @@ int main(int argc, char** argv) {
break;
}
const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new ||
- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) o.spec_min_p);
+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), spec_cost ? 0.0f : (float) o.spec_min_p);
if (adapt_thr.joinable()) adapt_thr.join();
if (!adapt_ok) return 1;
if (!drafted) {
@@ -16421,13 +16434,14 @@ int main(int argc, char** argv) {
total_ms += round_ms;
if (spec_trace != nullptr && timed_round) {
std::fprintf(spec_trace, "%d %d %d %.3f", T, a, from_sfx ? 1 : 0, round_ms);
- for (int i = 0; i < T_mtp - 1; ++i) std::fprintf(spec_trace, " %.4f", (double) dprob[(size_t) i]);
+ for (int i = 0; i < o.spec - 1; ++i) std::fprintf(spec_trace, " %.4f", (double) dwin[(size_t) i]);
std::fprintf(spec_trace, "\n");
}
if (!t_cycle.empty() && timed_round && T <= o.spec && t_cycle_seen[(size_t) T]++ > 0) {
t_cycle_ms[(size_t) T] += round_ms;
++t_cycle_n[(size_t) T];
}
+ if (spec_cost && use_mtp && timed_round && !from_sfx) policy.observe_probs(dwin.data(), T_mtp - 1, a);
if (timed_round && chain_n == 0) policy.observe(from_sfx, T, a, sfx_match, round_ms);
else if (timed_round) policy.observe_chain(T_mtp, chain_n, a, cm, round_ms);
if (rounds % 64 == 0)
diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cpp
index 8d422789b..8ec78ea56 100644
--- a/src/spec/draft_policy.cpp
+++ b/src/spec/draft_policy.cpp
@@ -12,6 +12,7 @@ namespace {
constexpr double kShape[DraftPolicy::kMaxT + 1] = {0.0, 1.0, 1.35, 1.7, 2.05, 2.45, 2.85, 3.25, 3.6};
constexpr double kCostAlpha = 0.1; // EMA weight of a new round time
constexpr double kTokAlpha = 0.05; // EMA weight of a new MTP window outcome
+constexpr double kCalDecay = 0.995; // the MTP probability -> acceptance table fades slowly
constexpr double kDecay = 0.97; // lookup counts: older windows fade
// Before a bucket has data: the longer the match, the likelier its continuation (llama.cpp's lookup decoding gates
// on the same thing); worth 4 observations, so a few real windows override it
@@ -107,6 +108,50 @@ void DraftPolicy::observe(bool lookup, int t, int accepted, int match, double ro
}
}
+namespace {
+int p_bucket(double p) {
+ const int b = (int) (p * 8.0);
+ return b < 0 ? 0 : b > 7 ? 7 : b;
+}
+} // namespace
+
+// Until a bucket has data a draft is as likely to hold as the MTP says (worth kPriorN observations); measured
+// acceptance of the MTP chain is not the same as its own softmax (code and thinking text are over-confident).
+double DraftPolicy::draft_rate(double p) const {
+ const int b = p_bucket(p);
+ const double centre = std::clamp(p, 0.0, 1.0);
+ return (pa_ok_[b] + kPriorN * centre) / (pa_n_[b] + kPriorN);
+}
+
+void DraftPolicy::observe_probs(const float* p, int n, int accepted) {
+ // draft j was reached when the drafts before it held; it held when j < accepted
+ for (int j = 0; j < n && j <= accepted; ++j) {
+ const int b = p_bucket(p[j]);
+ pa_ok_[b] = kCalDecay * pa_ok_[b] + (j < accepted ? 1.0 : 0.0);
+ pa_n_[b] = kCalDecay * pa_n_[b] + 1.0;
+ }
+}
+
+int DraftPolicy::best_rows(const float* p, int n, int cap) const {
+ cap = std::clamp(cap, 1, max_t_);
+ const int m = std::min(n, cap - 1);
+ double q = 1.0, e = 1.0, floor_cost = cost_ms(1);
+ double best_v = e / floor_cost;
+ int best_t = 1;
+ for (int j = 0; j < m; ++j) {
+ q *= draft_rate(p[j]);
+ e += q;
+ const int t = j + 2;
+ floor_cost = std::max(floor_cost, cost_ms(t));
+ const double v = e / floor_cost;
+ if (v > best_v * (1.0 + margin_)) {
+ best_v = v;
+ best_t = t;
+ }
+ }
+ return best_t;
+}
+
double DraftPolicy::chain_rate(int match) const {
// prior: a chained draft is a lookup continuing the MTP's drafts - weaker evidence than a plain match of that length
constexpr double kChainPrior[kBuckets] = {0.45, 0.65, 0.8, 0.9};