Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

spec: a gated request's lookup policy learns from committed rounds only, DraftPolicy choose() unchanged otherwise (#3416)

Keyan Huca17d0f5cdb7
diff --git a/include/strata/spec/draft_policy.hpp b/include/strata/spec/draft_policy.hpp
index 88970c8bd..6bf82d7ae 100644
--- a/include/strata/spec/draft_policy.hpp
+++ b/include/strata/spec/draft_policy.hpp
@@ -16,7 +16,9 @@
#pragma once
#include <array>
+#include <cstdint>
#include <string>
+#include <vector>
namespace strata::spec {
@@ -36,6 +38,20 @@ public:
Pick choose(int t_mtp, int lookup_k, int match) const;
/// After the round: the window it used, the drafts accepted, and the round's time (verify + commit + draft).
void observe(bool lookup, int t, int accepted, int match, double round_ms);
+ /// Gated proposals, inside one generation: a round's committed outcome only, never its time. `t` is the
+ /// lookup window chosen before the draws, or the configured MTP window; both are fixed by the history. A
+ /// lookup window counts as a probe of its size, so a size whose cost is still guessed is tried kProbes times,
+ /// not every round. Round times include the next window's drafts and the rows past the accepted ones, whose
+ /// Philox counters the next rounds draw again, so they wait (defer_cost) or go to the live policy (observe).
+ void observe_committed(bool lookup, int t, int accepted, int match);
+ /// Gated proposals, inside one generation: a round's time, held until the committed position passes `end`,
+ /// the last Philox counter of any draw that time depends on (the window's drafts and rows, the next window's
+ /// drafts when the time includes them). Until then those counters can be drawn again by the next rounds and the
+ /// time (rejected rows' routing, the next draft's gated length) would steer a window by its own draws.
+ void defer_cost(int64_t end, int t, double round_ms);
+ /// Apply, in order, the held times whose `end` is below `committed` (the next window's first counter), exactly
+ /// as observe() prices a round, in the context bucket the round ran in.
+ void apply_costs(int64_t committed);
/// --lookup-chain: how many of `k_avail` lookup tokens to append after an MTP window of `t_mtp` tokens whose
/// drafts all hold with probability `p_mtp` (estimated from the draft layer's own probabilities): the k with the
@@ -96,6 +112,11 @@ private:
static constexpr int kPBuckets = 8;
std::array<double, kPBuckets> pa_ok_{}, pa_n_{}; // MTP drafts reached / accepted by probability bucket (decayed)
std::array<std::array<double, kMaxT + 1>, kCtxBuckets> last_{}; // the round at which each size was last measured
+ // Gated proposals (observe_committed / defer_cost / apply_costs); all zero or empty on any other path.
+ std::array<std::array<double, kMaxT + 1>, kCtxBuckets> tried_{}; // lookup windows of that size run with no time yet
+ std::array<std::array<double, kMaxT + 1>, kCtxBuckets> delayed_n_{}; // of cost_n_: times applied by apply_costs (not new probes)
+ struct HeldCost { int64_t end; int bucket; int t; double ms; };
+ std::vector<HeldCost> held_; // defer_cost, oldest first
};
} // namespace strata::spec
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 71ce02a18..39e196647 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -14470,9 +14470,9 @@ int main(int argc, char** argv) {
extra_sources_reset();
for (int64_t t : ids) { const int32_t t32 = (int32_t) t; extra_sources_append(&t32, 1); }
}
- // This request's source planner: the live policy as it stands at the start (the choice must not depend on the
- // request's own draws, so it is not updated from them).
- const auto source_policy = policy;
+ // This request's source planner: the live policy at its start, then only committed outcomes (no round
+ // times, no rows past the accepted ones: their Philox counters are drawn again by the next rounds).
+ auto source_policy = policy;
strata::core::MtpDrafter::DraftResult generation;
strata::core::gated::Packet proposal; // immutable through the synchronous verifier call
bool first_window = true;
@@ -15179,6 +15179,7 @@ int main(int argc, char** argv) {
int sfx_match = 0;
if (gated_proposal) {
const int room = (int) std::min<int64_t>(S, o.max_context - p);
+ source_policy.apply_costs(p); // held round times whose draws are all committed now
if (!prepare_gated_window(proposal, generation, sfx, source_policy, o.suffix_draft > 0,
first_window, room, S_mtp, (int) ver.vocab(), (float) req_spec_min_p,
mtp.spec_q(), drafts.data(), dprob.data(), sbuf.data(), sfx_match, err)) {
@@ -15462,6 +15463,7 @@ int main(int argc, char** argv) {
draft_offered += T - 1;
draft_accepted += a;
}
+ const bool initial_round = first_window; // a single-row window: no proposal to learn from
first_window = false;
bool eos = false;
for (int i = merged ? a : 0; i <= a && produced_n < max_new && !eos; ++i) { // (merged: row T-1 only)
@@ -15529,6 +15531,12 @@ int main(int argc, char** argv) {
std::printf("ERR %s\n", err.c_str());
return 1;
}
+ if (gated_proposal && !eos && !initial_round)
+ source_policy.observe_committed(from_sfx, from_sfx ? T : std::min<int>(S_mtp, (int) std::min<int64_t>(S, o.max_context - p)),
+ a, sfx_match);
+ if (gated_proposal && !eos && !initial_round && timed_round) // after the next draft: its counters too
+ source_policy.defer_cost(std::max<int64_t>(p + T - 1, p + a + 1 + S_mtp), T,
+ std::chrono::duration<double, std::milli>(Clock::now() - round0).count());
if (spec_cost && use_mtp && timed_round && !from_sfx) policy.observe_probs(dwin_s.data(), T_mtp - 1, a);
if (timed_round && !eos && chain_n == 0)
policy.observe(from_sfx, T, a, sfx_match,
@@ -16782,7 +16790,7 @@ int main(int argc, char** argv) {
if (use_mtp && S_mtp < o.spec) mtp.set_max_drafts(S_mtp - 1);
strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);
strata::spec::DraftPolicy policy(o.spec); // MTP or lookup window (see draft_policy.hpp)
- const auto source_policy = policy; // the lookup choice of a probabilistic window: fixed at the start, never fed by this run's draws
+ auto source_policy = policy; // this generation's: committed outcomes only (observe_committed)
strata::spec::DraftPolicy::ChainState chain_st;
int chain_rows = 1; // STRATA_SPEC_COST: the window the cost-aware rule kept in the MTP chain just drafted
if (const char* tab = std::getenv("STRATA_SPEC_COST_TABLE"); spec_cost && tab != nullptr && tab[0] != '\0') {
@@ -16895,6 +16903,7 @@ int main(int argc, char** argv) {
bool from_sfx = false;
int sfx_match = 0;
if (gated_proposal) {
+ source_policy.apply_costs(p); // held round times whose draws are all committed now
if (!prepare_gated_window(proposal, generation, sfx, source_policy, o.suffix_draft > 0, first_window,
(int) std::min<int64_t>(win_cap, o.max_context - p), S_mtp, (int) ver.vocab(),
(float) o.spec_min_p, mtp.spec_q(), drafts.data(), dprob.data(), sbuf.data(),
@@ -17019,6 +17028,7 @@ int main(int argc, char** argv) {
}
int a = 0;
while (a < T - 1 && window[(size_t) a + 1] == outv[(size_t) a]) ++a;
+ const bool initial_round = first_window; // a single-row window: no proposal to learn from
if (first_window) {
first_window = false;
ttft_ms = std::chrono::duration<double, std::milli>(Clock::now() - t_start).count();
@@ -17077,6 +17087,9 @@ int main(int argc, char** argv) {
p += a + 1;
const double round_ms = std::chrono::duration<double, std::milli>(Clock::now() - t0).count();
total_ms += round_ms;
+ if (gated_proposal && !initial_round) source_policy.observe_committed(from_sfx, from_sfx ? T : std::min(S_mtp, room), a, sfx_match);
+ if (gated_proposal && !initial_round && timed_round) // p has advanced: the window was p - a - 1 ..
+ source_policy.defer_cost(std::max<int64_t>(p - a - 2 + T, p + S_mtp), T, round_ms);
if (spec_trace != nullptr && timed_round) {
std::fprintf(spec_trace, "%d %d %d %.3f", T, a, from_sfx ? 1 : 0, round_ms);
for (int i = 0; i < o.spec - 1; ++i) std::fprintf(spec_trace, " %.4f", (double) dwin[(size_t) i]);
diff --git a/src/spec/draft_policy.cpp b/src/spec/draft_policy.cpp
index ffd9bbbf2..bbcd883ad 100644
--- a/src/spec/draft_policy.cpp
+++ b/src/spec/draft_policy.cpp
@@ -144,7 +144,7 @@ DraftPolicy::Pick DraftPolicy::choose(int t_mtp, int lookup_k, int match) const
}
// a guessed cost can keep the policy from ever measuring a size: the first few times a confident lookup would
// need a size not measured yet, it is tried (verification keeps the output; only the one round's speed is at stake)
- if (t_full > p.t && (cost_n_[bk_][t_full] < kProbes || stale(t_full)) && q >= 0.85) {
+ if (t_full > p.t && (cost_n_[bk_][t_full] - delayed_n_[bk_][t_full] + tried_[bk_][t_full] < kProbes || stale(t_full)) && q >= 0.85) {
p.lookup = true;
p.t = t_full;
}
@@ -172,6 +172,43 @@ void DraftPolicy::observe(bool lookup, int t, int accepted, int match, double ro
}
}
+void DraftPolicy::observe_committed(bool lookup, int t, int accepted, int match) {
+ t = std::clamp(t, 1, kMaxT);
+ rounds_ += 1.0;
+ if (lookup) {
+ tried_[bk_][t] += 1.0;
+ last_[bk_][t] = rounds_; // tried now: not stale within this generation
+ const int b = bucket(match);
+ ok_[b] = kDecay * ok_[b] + accepted;
+ bad_[b] = kDecay * bad_[b] + (accepted < t - 1 ? 1.0 : 0.0);
+ } else {
+ const double got = accepted + 1.0;
+ mtp_tok_[t] = mtp_n_[t] > 0 ? (1.0 - kTokAlpha) * mtp_tok_[t] + kTokAlpha * got : got;
+ mtp_n_[t] += 1.0;
+ }
+}
+
+void DraftPolicy::defer_cost(int64_t end, int t, double round_ms) {
+ if (round_ms > 0) held_.push_back({end, bk_, std::clamp(t, 1, kMaxT), round_ms});
+}
+
+void DraftPolicy::apply_costs(int64_t committed) {
+ size_t n = 0;
+ const int bucket_now = bk_;
+ for (; n < held_.size() && held_[n].end < committed; ++n) {
+ const HeldCost& h = held_[n];
+ bk_ = h.bucket; // observe()'s update, in the bucket the round ran in
+ const double ms = clamp_sample(h.t, h.ms);
+ const double alpha = robust_ ? std::max(kCostAlpha, 1.0 / (cost_n_[bk_][h.t] + 1.0)) : kCostAlpha;
+ cost_[bk_][h.t] = cost_n_[bk_][h.t] > 0 ? (1.0 - alpha) * cost_[bk_][h.t] + alpha * ms : ms;
+ cost_n_[bk_][h.t] += 1.0;
+ delayed_n_[bk_][h.t] += 1.0; // a round already counted by observe_committed
+ last_[bk_][h.t] = rounds_;
+ }
+ bk_ = bucket_now;
+ held_.erase(held_.begin(), held_.begin() + (std::ptrdiff_t) n);
+}
+
namespace {
int p_bucket(double p) {
const int b = (int) (p * 8.0);
diff --git a/src/spec/draft_policy_test.cpp b/src/spec/draft_policy_test.cpp
index 48a4f7952..d9c64bd4b 100644
--- a/src/spec/draft_policy_test.cpp
+++ b/src/spec/draft_policy_test.cpp
@@ -114,6 +114,54 @@ int main() {
"chain: stale full size is re-probed despite a profitable shorter chain");
check(probes > 0 && confident, "chain: stale re-probes retain the MTP confidence gate");
}
+ {
+ // Gated proposals: within one generation the policy copy learns from committed outcomes only. A size
+ // whose cost is still guessed is probed until kProbes rounds of it have run, timed or not, not every round.
+ DraftPolicy live(6);
+ for (int i = 0; i < 50; ++i) live.observe(false, 4, 3, 0, cost(4)); // a strong MTP
+ live.observe(true, 6, 5, 8, 3.0 * cost(6)); // size 6 timed once, dear
+ DraftPolicy frozen = live, copy = live;
+ int frozen_probes = 0, copy_probes = 0;
+ for (int i = 0; i < 20; ++i) {
+ const DraftPolicy::Pick f = frozen.choose(4, 5, 8);
+ frozen_probes += f.lookup && f.t == 6;
+ const DraftPolicy::Pick c = copy.choose(4, 5, 8);
+ copy_probes += c.lookup && c.t == 6;
+ copy.observe_committed(c.lookup, c.lookup ? c.t : 4, c.lookup ? 5 : 3, 8);
+ }
+ check(frozen_probes == 20, "gated: a frozen copy probes the guessed size every round");
+ check(copy_probes == 2, "gated: committed outcomes end the probes after kProbes rounds of the size");
+ check(copy.cost_ms(6) == live.cost_ms(6), "gated: committed outcomes leave the measured costs alone");
+ }
+ {
+ // Gated proposals: a round's time reaches the generation's copy only once the committed position has passed
+ // every counter it depends on. Cheap costs from earlier text stop steering long lookups once the dear
+ // repetitive-text times are applied; a held time is not a new probe.
+ DraftPolicy live(6);
+ for (int t = 2; t <= 6; ++t) for (int i = 0; i < 5; ++i) live.observe(false, t, 1, 0, cost(t) * 0.2); // cheap stretch
+ DraftPolicy copy = live;
+ check(copy.choose(4, 5, 40).lookup && copy.choose(4, 5, 40).t == 6, "delayed: cheap inherited costs take the full lookup");
+ const double before = copy.cost_ms(6);
+ copy.observe_committed(true, 6, 5, 40);
+ copy.defer_cost(110, 6, 40.0 * cost(6));
+ copy.apply_costs(110);
+ check(copy.cost_ms(6) == before, "delayed: a time is held while its counters can still be drawn");
+ copy.apply_costs(111);
+ check(copy.cost_ms(6) > before, "delayed: applied once the committed position passes its last counter");
+ for (int i = 0; i < 40; ++i) {
+ copy.observe_committed(true, 6, 5, 40);
+ copy.defer_cost(120 + i, 6, 40.0 * cost(6));
+ copy.apply_costs(121 + i);
+ }
+ check(copy.choose(4, 5, 40).t < 6, "delayed: dear measured times shorten the lookup window");
+ DraftPolicy fresh(6);
+ fresh.observe_committed(true, 6, 5, 40);
+ fresh.defer_cost(5, 6, 30.0);
+ fresh.apply_costs(6);
+ fresh.observe_committed(true, 6, 5, 40);
+ check(fresh.choose(4, 5, 40).lookup && fresh.choose(4, 5, 40).t == 6,
+ "delayed: a held time does not count as another probe");
+ }
std::printf(g_fail ? "FAIL\n" : "PASS\n");
return g_fail ? 1 : 0;
}