Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
spec: a gated request's lookup policy learns from committed rounds only, DraftPolicy choose() unchanged otherwise (#3416)
Keyan Huca17d0f5cdb7
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hppindex 88970c8bd..6bf82d7ae 100644--- a/include/strata/spec/draft_policy.hpp+++ b/include/strata/spec/draft_policy.hpp@@ -16,7 +16,9 @@#pragma once#include <array>+#include <cstdint>#include <string>+#include <vector>namespace strata::spec {@@ -36,6 +38,20 @@ public:Pick choose(int t_mtp, int lookup_k, int match) const;/// After the round: the window it used, the drafts accepted, and the round's time (verify + commit + draft).void observe(bool lookup, int t, int accepted, int match, double round_ms);+ /// Gated proposals, inside one generation: a round's committed outcome only, never its time. `t` is the+ /// lookup window chosen before the draws, or the configured MTP window; both are fixed by the history. A+ /// lookup window counts as a probe of its size, so a size whose cost is still guessed is tried kProbes times,+ /// not every round. Round times include the next window's drafts and the rows past the accepted ones, whose+ /// Philox counters the next rounds draw again, so they wait (defer_cost) or go to the live policy (observe).+ void observe_committed(bool lookup, int t, int accepted, int match);+ /// Gated proposals, inside one generation: a round's time, held until the committed position passes `end`,+ /// the last Philox counter of any draw that time depends on (the window's drafts and rows, the next window's+ /// drafts when the time includes them). Until then those counters can be drawn again by the next rounds and the+ /// time (rejected rows' routing, the next draft's gated length) would steer a window by its own draws.+ void defer_cost(int64_t end, int t, double round_ms);+ /// Apply, in order, the held times whose `end` is below `committed` (the next window's first counter), exactly+ /// as observe() prices a round, in the context bucket the round ran in.+ void apply_costs(int64_t committed);/// --lookup-chain: how many of `k_avail` lookup tokens to append after an MTP window of `t_mtp` tokens whose/// drafts all hold with probability `p_mtp` (estimated from the draft layer's own probabilities): the k with the@@ -96,6 +112,11 @@ private:static constexpr int kPBuckets = 8;std::array<double, kPBuckets> pa_ok_{}, pa_n_{}; // MTP drafts reached / accepted by probability bucket (decayed)std::array<std::array<double, kMaxT + 1>, kCtxBuckets> last_{}; // the round at which each size was last measured+ // Gated proposals (observe_committed / defer_cost / apply_costs); all zero or empty on any other path.+ std::array<std::array<double, kMaxT + 1>, kCtxBuckets> tried_{}; // lookup windows of that size run with no time yet+ std::array<std::array<double, kMaxT + 1>, kCtxBuckets> delayed_n_{}; // of cost_n_: times applied by apply_costs (not new probes)+ struct HeldCost { int64_t end; int bucket; int t; double ms; };+ std::vector<HeldCost> held_; // defer_cost, oldest first};} // namespace strata::specdiff --git a/src/program/generate.cpp b/src/program/generate.cppindex 71ce02a18..39e196647 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -14470,9 +14470,9 @@ int main(int argc, char** argv) {extra_sources_reset();for (int64_t t : ids) { const int32_t t32 = (int32_t) t; extra_sources_append(&t32, 1); }}- // This request's source planner: the live policy as it stands at the start (the choice must not depend on the- // request's own draws, so it is not updated from them).- const auto source_policy = policy;+ // This request's source planner: the live policy at its start, then only committed outcomes (no round+ // times, no rows past the accepted ones: their Philox counters are drawn again by the next rounds).+ auto source_policy = policy;strata::core::MtpDrafter::DraftResult generation;strata::core::gated::Packet proposal; // immutable through the synchronous verifier callbool first_window = true;@@ -15179,6 +15179,7 @@ int main(int argc, char** argv) {int sfx_match = 0;if (gated_proposal) {const int room = (int) std::min<int64_t>(S, o.max_context - p);+ source_policy.apply_costs(p); // held round times whose draws are all committed nowif (!prepare_gated_window(proposal, generation, sfx, source_policy, o.suffix_draft > 0,first_window, room, S_mtp, (int) ver.vocab(), (float) req_spec_min_p,mtp.spec_q(), drafts.data(), dprob.data(), sbuf.data(), sfx_match, err)) {@@ -15462,6 +15463,7 @@ int main(int argc, char** argv) {draft_offered += T - 1;draft_accepted += a;}+ const bool initial_round = first_window; // a single-row window: no proposal to learn fromfirst_window = false;bool eos = false;for (int i = merged ? a : 0; i <= a && produced_n < max_new && !eos; ++i) { // (merged: row T-1 only)@@ -15529,6 +15531,12 @@ int main(int argc, char** argv) {std::printf("ERR %s\n", err.c_str());return 1;}+ if (gated_proposal && !eos && !initial_round)+ source_policy.observe_committed(from_sfx, from_sfx ? T : std::min<int>(S_mtp, (int) std::min<int64_t>(S, o.max_context - p)),+ a, sfx_match);+ if (gated_proposal && !eos && !initial_round && timed_round) // after the next draft: its counters too+ source_policy.defer_cost(std::max<int64_t>(p + T - 1, p + a + 1 + S_mtp), T,+ std::chrono::duration<double, std::milli>(Clock::now() - round0).count());if (spec_cost && use_mtp && timed_round && !from_sfx) policy.observe_probs(dwin_s.data(), T_mtp - 1, a);if (timed_round && !eos && chain_n == 0)policy.observe(from_sfx, T, a, sfx_match,@@ -16782,7 +16790,7 @@ int main(int argc, char** argv) {if (use_mtp && S_mtp < o.spec) mtp.set_max_drafts(S_mtp - 1);strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);strata::spec::DraftPolicy policy(o.spec); // MTP or lookup window (see draft_policy.hpp)- const auto source_policy = policy; // the lookup choice of a probabilistic window: fixed at the start, never fed by this run's draws+ auto source_policy = policy; // this generation's: committed outcomes only (observe_committed)strata::spec::DraftPolicy::ChainState chain_st;int chain_rows = 1; // STRATA_SPEC_COST: the window the cost-aware rule kept in the MTP chain just draftedif (const char* tab = std::getenv("STRATA_SPEC_COST_TABLE"); spec_cost && tab != nullptr && tab[0] != '\0') {@@ -16895,6 +16903,7 @@ int main(int argc, char** argv) {bool from_sfx = false;int sfx_match = 0;if (gated_proposal) {+ source_policy.apply_costs(p); // held round times whose draws are all committed nowif (!prepare_gated_window(proposal, generation, sfx, source_policy, o.suffix_draft > 0, first_window,(int) std::min<int64_t>(win_cap, o.max_context - p), S_mtp, (int) ver.vocab(),(float) o.spec_min_p, mtp.spec_q(), drafts.data(), dprob.data(), sbuf.data(),@@ -17019,6 +17028,7 @@ int main(int argc, char** argv) {}int a = 0;while (a < T - 1 && window[(size_t) a + 1] == outv[(size_t) a]) ++a;+ const bool initial_round = first_window; // a single-row window: no proposal to learn fromif (first_window) {first_window = false;ttft_ms = std::chrono::duration<double, std::milli>(Clock::now() - t_start).count();@@ -17077,6 +17087,9 @@ int main(int argc, char** argv) {p += a + 1;const double round_ms = std::chrono::duration<double, std::milli>(Clock::now() - t0).count();total_ms += round_ms;+ if (gated_proposal && !initial_round) source_policy.observe_committed(from_sfx, from_sfx ? T : std::min(S_mtp, room), a, sfx_match);+ if (gated_proposal && !initial_round && timed_round) // p has advanced: the window was p - a - 1 ..+ source_policy.defer_cost(std::max<int64_t>(p - a - 2 + T, p + S_mtp), T, round_ms);if (spec_trace != nullptr && timed_round) {std::fprintf(spec_trace, "%d %d %d %.3f", T, a, from_sfx ? 1 : 0, round_ms);for (int i = 0; i < o.spec - 1; ++i) std::fprintf(spec_trace, " %.4f", (double) dwin[(size_t) i]);diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cppindex ffd9bbbf2..bbcd883ad 100644--- a/src/spec/draft_policy.cpp+++ b/src/spec/draft_policy.cpp@@ -144,7 +144,7 @@ DraftPolicy::Pick DraftPolicy::choose(int t_mtp, int lookup_k, int match) const}// a guessed cost can keep the policy from ever measuring a size: the first few times a confident lookup would// need a size not measured yet, it is tried (verification keeps the output; only the one round's speed is at stake)- if (t_full > p.t && (cost_n_[bk_][t_full] < kProbes || stale(t_full)) && q >= 0.85) {+ if (t_full > p.t && (cost_n_[bk_][t_full] - delayed_n_[bk_][t_full] + tried_[bk_][t_full] < kProbes || stale(t_full)) && q >= 0.85) {p.lookup = true;p.t = t_full;}@@ -172,6 +172,43 @@ void DraftPolicy::observe(bool lookup, int t, int accepted, int match, double ro}}+void DraftPolicy::observe_committed(bool lookup, int t, int accepted, int match) {+ t = std::clamp(t, 1, kMaxT);+ rounds_ += 1.0;+ if (lookup) {+ tried_[bk_][t] += 1.0;+ last_[bk_][t] = rounds_; // tried now: not stale within this generation+ const int b = bucket(match);+ ok_[b] = kDecay * ok_[b] + accepted;+ bad_[b] = kDecay * bad_[b] + (accepted < t - 1 ? 1.0 : 0.0);+ } else {+ const double got = accepted + 1.0;+ mtp_tok_[t] = mtp_n_[t] > 0 ? (1.0 - kTokAlpha) * mtp_tok_[t] + kTokAlpha * got : got;+ mtp_n_[t] += 1.0;+ }+}++void DraftPolicy::defer_cost(int64_t end, int t, double round_ms) {+ if (round_ms > 0) held_.push_back({end, bk_, std::clamp(t, 1, kMaxT), round_ms});+}++void DraftPolicy::apply_costs(int64_t committed) {+ size_t n = 0;+ const int bucket_now = bk_;+ for (; n < held_.size() && held_[n].end < committed; ++n) {+ const HeldCost& h = held_[n];+ bk_ = h.bucket; // observe()'s update, in the bucket the round ran in+ const double ms = clamp_sample(h.t, h.ms);+ const double alpha = robust_ ? std::max(kCostAlpha, 1.0 / (cost_n_[bk_][h.t] + 1.0)) : kCostAlpha;+ cost_[bk_][h.t] = cost_n_[bk_][h.t] > 0 ? (1.0 - alpha) * cost_[bk_][h.t] + alpha * ms : ms;+ cost_n_[bk_][h.t] += 1.0;+ delayed_n_[bk_][h.t] += 1.0; // a round already counted by observe_committed+ last_[bk_][h.t] = rounds_;+ }+ bk_ = bucket_now;+ held_.erase(held_.begin(), held_.begin() + (std::ptrdiff_t) n);+}+namespace {int p_bucket(double p) {const int b = (int) (p * 8.0);diff --git a/src/spec/draft_policy_test.cpp b/src/spec/draft_policy_test.cppindex 48a4f7952..d9c64bd4b 100644--- a/src/spec/draft_policy_test.cpp+++ b/src/spec/draft_policy_test.cpp@@ -114,6 +114,54 @@ int main() {"chain: stale full size is re-probed despite a profitable shorter chain");check(probes > 0 && confident, "chain: stale re-probes retain the MTP confidence gate");}+ {+ // Gated proposals: within one generation the policy copy learns from committed outcomes only. A size+ // whose cost is still guessed is probed until kProbes rounds of it have run, timed or not, not every round.+ DraftPolicy live(6);+ for (int i = 0; i < 50; ++i) live.observe(false, 4, 3, 0, cost(4)); // a strong MTP+ live.observe(true, 6, 5, 8, 3.0 * cost(6)); // size 6 timed once, dear+ DraftPolicy frozen = live, copy = live;+ int frozen_probes = 0, copy_probes = 0;+ for (int i = 0; i < 20; ++i) {+ const DraftPolicy::Pick f = frozen.choose(4, 5, 8);+ frozen_probes += f.lookup && f.t == 6;+ const DraftPolicy::Pick c = copy.choose(4, 5, 8);+ copy_probes += c.lookup && c.t == 6;+ copy.observe_committed(c.lookup, c.lookup ? c.t : 4, c.lookup ? 5 : 3, 8);+ }+ check(frozen_probes == 20, "gated: a frozen copy probes the guessed size every round");+ check(copy_probes == 2, "gated: committed outcomes end the probes after kProbes rounds of the size");+ check(copy.cost_ms(6) == live.cost_ms(6), "gated: committed outcomes leave the measured costs alone");+ }+ {+ // Gated proposals: a round's time reaches the generation's copy only once the committed position has passed+ // every counter it depends on. Cheap costs from earlier text stop steering long lookups once the dear+ // repetitive-text times are applied; a held time is not a new probe.+ DraftPolicy live(6);+ for (int t = 2; t <= 6; ++t) for (int i = 0; i < 5; ++i) live.observe(false, t, 1, 0, cost(t) * 0.2); // cheap stretch+ DraftPolicy copy = live;+ check(copy.choose(4, 5, 40).lookup && copy.choose(4, 5, 40).t == 6, "delayed: cheap inherited costs take the full lookup");+ const double before = copy.cost_ms(6);+ copy.observe_committed(true, 6, 5, 40);+ copy.defer_cost(110, 6, 40.0 * cost(6));+ copy.apply_costs(110);+ check(copy.cost_ms(6) == before, "delayed: a time is held while its counters can still be drawn");+ copy.apply_costs(111);+ check(copy.cost_ms(6) > before, "delayed: applied once the committed position passes its last counter");+ for (int i = 0; i < 40; ++i) {+ copy.observe_committed(true, 6, 5, 40);+ copy.defer_cost(120 + i, 6, 40.0 * cost(6));+ copy.apply_costs(121 + i);+ }+ check(copy.choose(4, 5, 40).t < 6, "delayed: dear measured times shorten the lookup window");+ DraftPolicy fresh(6);+ fresh.observe_committed(true, 6, 5, 40);+ fresh.defer_cost(5, 6, 30.0);+ fresh.apply_costs(6);+ fresh.observe_committed(true, 6, 5, 40);+ check(fresh.choose(4, 5, 40).lookup && fresh.choose(4, 5, 40).t == 6,+ "delayed: a held time does not count as another probe");+ }std::printf(g_fail ? "FAIL\n" : "PASS\n");return g_fail ? 1 : 0;}