Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

spec: STRATA_SPEC_PROB=1 keeps the confidence gate in the proposal law, opt-in (#3416)

Keyan Huf4612b1addd9
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9c6f28790..a5fb2b6f8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1278,6 +1278,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(spec_verify_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_test(NAME spec_verify_parity COMMAND spec_verify_parity)
set_tests_properties(spec_verify_parity PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=0")
+ # the explicit mu / NULL verifier of the gated proposal (STRATA_SPEC_PROB=1, pick gate) against a CPU oracle
+ add_executable(spec_proposal_parity src/kernels/spec_proposal_parity.cpp)
+ target_include_directories(spec_proposal_parity PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(spec_proposal_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+ add_test(NAME spec_proposal_parity COMMAND spec_proposal_parity)
+ set_tests_properties(spec_proposal_parity PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=0" TIMEOUT 120)
# the same fixtures on the other two sampled paths: the one-block kernel and the 0.1.20 kernel
add_test(NAME sampler_parity_one_block COMMAND sampler_parity --selftest)
set_tests_properties(sampler_parity_one_block PROPERTIES ENVIRONMENT "STRATA_OLD_SAMPLER=0;STRATA_SAMPLER_ONE_BLOCK=1")
@@ -1626,6 +1632,17 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
add_executable(spec_prob_test src/core/spec_prob_test.cpp)
target_include_directories(spec_prob_test PRIVATE include)
add_test(NAME spec_prob_test COMMAND spec_prob_test)
+ # gated proposal packets (STRATA_SPEC_PROB=1, pick gate): masked measure, NULL tail, root rescue (header-only, CPU)
+ add_executable(gated_proposal_test src/core/gated_proposal_test.cpp)
+ target_include_directories(gated_proposal_test PRIVATE include)
+ add_test(NAME gated_proposal_test COMMAND gated_proposal_test)
+ # the admission law of that packet, enumerated exactly (no Monte Carlo), in the three gate modes (CPU)
+ add_executable(spec_admission_test src/core/spec_admission_test.cpp src/spec/draft_policy.cpp)
+ target_include_directories(spec_admission_test PRIVATE include)
+ add_test(NAME spec_admission_default COMMAND ${CMAKE_COMMAND} -E env --unset=STRATA_SPEC_PROB_GATE $<TARGET_FILE:spec_admission_test>)
+ add_test(NAME spec_admission_pick COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=pick $<TARGET_FILE:spec_admission_test> pick)
+ add_test(NAME spec_admission_top COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=top $<TARGET_FILE:spec_admission_test> top)
+ add_test(NAME spec_admission_exact_top_only COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=TOP $<TARGET_FILE:spec_admission_test> pick)
# the per-card window cap of --draft dflash2 (header-only, CPU)
add_executable(draft_cap_test src/core/draft_cap_test.cpp)
target_include_directories(draft_cap_test PRIVATE include)
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index f60429bf2..32c37e30f 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -507,6 +507,17 @@ applies at the next start. Network, key, MCP and program keys are not editable t
`STRATA_SPEC_GUMBEL=1` raise the accepted-draft rate on some machines; `STRATA_SPEC_PROB=1` is speculative rejection
sampling. `STRATA_SPEC_DEPTH=1` prints acceptance by draft depth and `STRATA_SPEC_MIN_TEMP=0.9` limits sampled drafting
to hotter requests. Try them on your own card before relying on them.
+- `STRATA_SPEC_PROB=1` with the default `--spec-min-p` gate: the gate tests the token the draft head drew from its distribution
+ q. A draft that fails the gate still counts as a proposal. The verifier keeps q's mass on the tokens that pass the gate
+ (not renormalised; call it mu), accepts an offered token D with probability min(1, p(D)/mu(D)), and draws the replacement
+ for a rejection or a failed gate from the positive part of p - mu. The older rule checked against the unfiltered q and
+ treated a failed gate as a plain sample, which shifts the output: with p = (0.5, 0.5), q = (0.875, 0.125) and a threshold
+ of 0.5 it gave (0.5625, 0.4375). `STRATA_SPEC_PROB_GATE=top` gates on q's largest probability before the draw instead and
+ keeps the normalised-q rule. The gated rule needs serial decode on one device with `--lookup-chain 0 --adapt-swaps 0
+ --pcie-frac 0` and without `STRATA_SPEC_COST`; otherwise the engine says so once and keeps the older rule. This is a
+ correctness change, not a speed change. `spec_admission_test` enumerates the rule exactly for a two-token vocabulary,
+ `gated_proposal_test` checks the proposal packet and `spec_proposal_parity` checks the GPU verifier against a CPU
+ reference. A model-level distribution run is not published yet.
- **Non-finite logits end the request and drop the cached state (on by default).** If a verify window's logits are NaN, the
window emits no tokens, the request ends, and the next request drops the live session, checkpoints and parked
conversations and reads its prompt from token 0. The server retries a request that had sent nothing once, transparently.
diff --git a/include/strata/core/gated_proposal.hpp b/include/strata/core/gated_proposal.hpp
new file mode 100644
index 000000000..bfd5cb005
--- /dev/null
+++ b/include/strata/core/gated_proposal.hpp
@@ -0,0 +1,128 @@
+#pragma once
+// Host packet builder for gated speculative proposals.
+// Draft tokens, scores and raw q remain read-only. Retained float probabilities
+// keep their original bits; masked probabilities become +0 without renormalization.
+#include "strata/core/spec_prob.hpp"
+
+#include <array>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+namespace strata::core::gated {
+constexpr int kRows = 8;
+constexpr int kEntries = strata::core::kSpecQEntries;
+constexpr int kStride = strata::core::kSpecQStride;
+struct RootChoice {
+ bool eligible = false;
+ int rescue_id = -1;
+ int window = 1;
+};
+struct Packet {
+ int T = 1, n_q = 0, offers = 0, attempts = 0, failed_index = -1;
+ bool from_sfx = false, null_tail = false;
+ std::array<int32_t, kRows * kStride> mu{};
+ const char* error = nullptr;
+};
+
+inline float probability(const int32_t* row, int entry) {
+ float p;
+ std::memcpy(&p, row + kEntries + entry, sizeof(p));
+ return p;
+}
+
+inline float mass_at(const int32_t* row, int id) {
+ for (int i = 0; i < kEntries && row[i] >= 0; ++i)
+ if (row[i] == id) return probability(row, i);
+ return 0.0f;
+}
+
+// Preserve IDs, padding and retained probability bits. A negative rescue_id keeps
+// only entries at or above theta; otherwise retain the rescue token as well.
+inline bool mask_row(const int32_t* raw, int32_t* masked, float theta, int rescue_id, int vocab) {
+ if (!raw || !masked || !std::isfinite(theta) || theta < 0.0f || vocab <= 0) return false;
+ std::memcpy(masked, raw, kStride * sizeof(int32_t));
+ int n = 0;
+ for (; n < kEntries && raw[n] >= 0; ++n) {
+ const float q = probability(raw, n);
+ if (raw[n] >= vocab || !std::isfinite(q) || q < 0.0f || q > 1.0f) return false;
+ for (int j = 0; j < n; ++j)
+ if (raw[j] == raw[n]) return false;
+ if (!(q >= theta || raw[n] == rescue_id)) {
+ const float zero = 0.0f;
+ std::memcpy(masked + kEntries + n, &zero, sizeof(zero));
+ }
+ }
+ return n > 0;
+}
+
+inline bool draw_matches(const int32_t* raw, int d, float score, int vocab) {
+ if (d < 0 || d >= vocab || !std::isfinite(score)) return false;
+ const float q = mass_at(raw, d);
+ return q > 0.0f && std::memcmp(&q, &score, sizeof(q)) == 0;
+}
+
+// budget is the actual producer max_steps for the latest successful draft call,
+// not guessed from generated count. attempts counts rows inspected for this
+// packet; n_generated separately counts all materialized rows, including discarded
+// MTP lookahead when suffix wins. RootChoice must be computed without D/Tactual.
+inline bool make_packet(Packet& out, const int32_t* raw_q, const int32_t* drafts, const float* scores,
+ int n_generated, int budget, int capacity, int vocab, float theta,
+ bool initial, const RootChoice& root) {
+ out = Packet{};
+ auto fail = [&](const char* message) {
+ out.error = message;
+ return false;
+ };
+ if (capacity < 1 || capacity > kRows || budget < 0 || budget >= capacity || n_generated < 0 ||
+ n_generated > budget || !std::isfinite(theta) || theta < 0.0f) return fail("invalid attempt budget/count");
+ if (initial) {
+ if (n_generated != 0) return fail("initial window has stale attempts");
+ return true; // No proposal attempt: ordinary single-row sampling.
+ }
+ if (n_generated == 0) {
+ if (budget != 0) return fail("missing materialized attempt");
+ return true; // Zero proposal budget: ordinary single-row sampling.
+ }
+ if (!raw_q || !drafts || !scores) return fail("missing raw proposal buffers");
+ if (root.eligible && (root.rescue_id < 0 || root.rescue_id >= vocab || root.window < 2 || root.window > capacity))
+ return fail("invalid prefix-only root choice");
+ // Build the common root measure before consulting D0: every root branch sees
+ // H union {s0} when E, including MTP and NULL branches where D0 != s0.
+ if (!mask_row(raw_q, out.mu.data(), theta, root.eligible ? root.rescue_id : -1, vocab)) return fail("invalid root q");
+ if (!draw_matches(raw_q, drafts[0], scores[0], vocab)) return fail("raw q(D0) differs from own-score dprob");
+ if (root.eligible && drafts[0] == root.rescue_id) {
+ if (!(mass_at(out.mu.data(), drafts[0]) > 0.0f)) return fail("rescued offer outside root mu");
+ out.T = root.window;
+ out.n_q = 1;
+ out.offers = out.T - 1;
+ out.attempts = 1;
+ out.from_sfx = true;
+ return true; // real root mu, then suffix deltas, last row plain bonus
+ }
+ for (int j = 0; j < n_generated; ++j) {
+ const int32_t* raw = raw_q + (size_t) j * kStride;
+ int32_t* mu = out.mu.data() + (size_t) j * kStride;
+ if (j > 0 && !mask_row(raw, mu, theta, -1, vocab)) return fail("invalid later q");
+ if (!draw_matches(raw, drafts[j], scores[j], vocab)) return fail("raw q(D) differs from own-score dprob");
+ out.attempts = j + 1;
+ const bool offer = scores[j] >= theta;
+ if (!offer) {
+ if (mass_at(mu, drafts[j]) != 0.0f) return fail("failed draw retained outside rescue branch");
+ out.offers = j;
+ out.T = j + 1;
+ out.n_q = out.T;
+ out.null_tail = true;
+ out.failed_index = j;
+ return true; // The failed draft is not a window input.
+ }
+ if (!(mass_at(mu, drafts[j]) > 0.0f)) return fail("offered draw outside positive mu support");
+ }
+ if (n_generated != budget) return fail("incomplete attempts are not a budget bonus");
+ out.offers = n_generated;
+ out.T = n_generated + 1;
+ out.n_q = n_generated;
+ return true;
+}
+} // namespace strata::core::gated
diff --git a/include/strata/core/mtp.hpp b/include/strata/core/mtp.hpp
index 38fe8f011..0cd247e48 100644
--- a/include/strata/core/mtp.hpp
+++ b/include/strata/core/mtp.hpp
@@ -100,11 +100,16 @@ public:
bool reserve_prefill(int64_t n, std::string& err);
cudaStream_t stream() const { return cs_; }
+ /// Successful producer result, including a generated row that failed admission.
+ /// budget is the actual producer max_steps; failure/entry resets both fields.
+ /// With result supplied, n_drafts (if supplied too) is also reset on entry.
+ struct DraftResult { int generated = 0; int budget = 0; };
+
/// One round: catch-up over T cells from `p` (rows = the window's final residuals, `tokens` = the window's
/// argmaxes: row t pairs R_{p+t} with the token at p+t+1), then the draft chain from row `a` (the last
/// accepted row) for T-1 drafts at cells p+a+1 ... `drafts` gets T-1 tokens.
bool draft(int T, const int32_t* tokens, int64_t p, int a, int32_t* drafts, std::string& err,
- float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);
+ float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr, DraftResult* result = nullptr);
/// The first round: one cell (`cell`) from `R_row` (device) and `token` -> T-1 drafts.
/// STRATA_SPEC_COST: called after each draft step with (index, probability) instead of the `min_p` test; false ends the chain.
@@ -112,7 +117,7 @@ public:
void set_chain_rule(std::function<bool(int, float)> f) { chain_rule_ = std::move(f); }
bool draft_first(int T, const float* R_row, int32_t token, int64_t cell, int32_t* drafts, std::string& err,
- float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr);
+ float* probs = nullptr, float min_p = 0.0f, int* n_drafts = nullptr, DraftResult* result = nullptr);
/// COUPLED DRAFT SAMPLING (core/coupled_draft.hpp; STRATA_SPEC_COUPLED=1, set up by bind()): the request's
/// sampling. A sampled request (temperature > 0, not greedy) then drafts by SAMPLING with the target's chain and
diff --git a/include/strata/core/spec_prob.hpp b/include/strata/core/spec_prob.hpp
index ab45e5552..47089ef8f 100644
--- a/include/strata/core/spec_prob.hpp
+++ b/include/strata/core/spec_prob.hpp
@@ -39,9 +39,9 @@
namespace strata::core {
-/// --spec-min-p gates a further draft on the drawn token's probability under q (as the coupled drafter does).
-/// STRATA_SPEC_PROB_GATE=top gates on q's top probability instead: measured on the RTX 3060 it drafts longer windows
-/// whose later guesses are kept less often, and decodes slower.
+/// The default confidence gate tests the drawn token. gated_proposal.hpp carries its admitted
+/// mass and a NULL outcome to the verifier; a failed gate is still a proposal attempt.
+/// Explicit `top` tests q's largest probability before the draw and uses the normalized-q path.
inline bool spec_gate_pick() {
static const bool on = [] {
const char* e = std::getenv("STRATA_SPEC_PROB_GATE");
diff --git a/include/strata/core/spec_proposal.hpp b/include/strata/core/spec_proposal.hpp
new file mode 100644
index 000000000..f53e40776
--- /dev/null
+++ b/include/strata/core/spec_proposal.hpp
@@ -0,0 +1,28 @@
+#pragma once
+#include <cstdint>
+
+namespace strata::core::proposal {
+// Per-call proposal row contract; no process-wide mode.
+// Never a vocabulary ID or the existing no-draft marker (-1).
+inline constexpr int32_t kProposalSamplerErrorToken = -2147483647;
+enum class RowKind { MuOffer, PointOffer, NullTail, PlainBonus };
+#if defined(__CUDACC__) || defined(__HIPCC__)
+#define STRATA_PROPOSAL_HD __host__ __device__
+#else
+#define STRATA_PROPOSAL_HD
+#endif
+// Preconditions: 0 <= t < T, 0 <= nq <= T. The proposal API permits nq == T
+// for a NULL tail, which has no candidate token even when T == 1.
+STRATA_PROPOSAL_HD constexpr RowKind rowkind(int t, int T, int nq) {
+ return t == T - 1 ? (nq == T ? RowKind::NullTail : RowKind::PlainBonus)
+ : (t < nq ? RowKind::MuOffer : RowKind::PointOffer);
+}
+#undef STRATA_PROPOSAL_HD
+// qbuf owns immutable masked mu, not a renormalized conditional proposal.
+// MuOffer: require mu(D)>0; accept min(1,p(D)/mu(D)), else residual (p-mu)+.
+// NullTail: no dtok read and no accept draw; sample residual (p-mu)+ directly.
+// PointOffer and PlainBonus retain existing semantics. Target ex is untouched.
+// Missing/nonpositive/nonfinite mu(D), or zero NULL residual, emits the unique
+// error token above. The verifier rejects it after synchronization.
+// This is a packet/interface check, not a replacement for caller law validation.
+} // namespace strata::core::proposal
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index 2b2d9a0ec..dfa9085b3 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -153,8 +153,20 @@ public:
void set_spec_q(const int32_t* q, int n_q) {
spec_q_ = q;
spec_nq_ = n_q;
+ spec_measure_ = false;
if (next_) next_->set_spec_q(q, n_q);
}
+ /// Borrow immutable gated measure rows for the next run only. nq==T denotes
+ /// a NULL tail, nq<T a plain bonus. Errors never fall back to ordinary p.
+ /// nq == 0 accepts a null pointer: point-only rows and a plain bonus (T1: plain p).
+ /// Keep the host rows alive and unchanged until run returns.
+ void set_spec_proposal(const int32_t* q, int n_q) {
+ spec_q_ = q;
+ spec_nq_ = n_q;
+ spec_measure_ = true;
+ if (next_) next_->set_spec_proposal(q, n_q);
+ }
+
/// Counters of the rejection path (windows judged, drafts kept / offered over its rows with a q list).
int64_t spec_windows = 0;
@@ -458,6 +470,7 @@ private:
const int32_t* hist_d_ = nullptr; ///< penalty-history row (set_history); null = no penalties apply
const int32_t* spec_q_ = nullptr; ///< set_spec_q: the drafter's q rows for the next window (host)
int spec_nq_ = 0;
+ bool spec_measure_ = false; ///< explicit per-call measure borrow
int32_t* d_spec_ = nullptr; ///< device: kVerifyMaxT draft ids, then kVerifyMaxT q rows
int hist_len_ = 0;
bool head_sampling_ = true; ///< set_head_sampling
diff --git a/include/strata/kernels/sampler.hpp b/include/strata/kernels/sampler.hpp
index a3121ba1f..662b277b5 100644
--- a/include/strata/kernels/sampler.hpp
+++ b/include/strata/kernels/sampler.hpp
@@ -83,6 +83,13 @@ void spec_draft_sample(float* logits, int nv, const int32_t* sub_to_id, const in
bool sample_tokens_spec(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
const SamplerParams& p, const int32_t* dtok, const int32_t* qbuf, int n_q, int* out,
void* stream);
+// Explicit gated measure: rows<n_q use mu; n_q==n_tokens means a NULL tail.
+// n_q == 0 permits null qbuf: point-only rows, or plain sampling for T1.
+// Same device-buffer layout as sample_tokens_spec. False is a hard failure:
+// work may have been queued; synchronize before releasing buffers. No plain-p fallback.
+bool sample_tokens_proposal(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+ const SamplerParams& p, const int32_t* dtok, const int32_t* qbuf, int n_q, int* out,
+ void* stream);
// The penalty-history rows of a verify window, on the host: row t of `out` (T rows of `h` slots) is the last `h`
// tokens of `tail[0..n_tail)` followed by `window[0..t]`, most recent LAST, -1 in the unused front slots.
diff --git a/src/core/gated_proposal_test.cpp b/src/core/gated_proposal_test.cpp
new file mode 100644
index 000000000..5c7bf22a8
--- /dev/null
+++ b/src/core/gated_proposal_test.cpp
@@ -0,0 +1,184 @@
+// Actual gated proposal packet contract, CPU-only; hand-specified expectations.
+// Includes immutable q bits, root rescue on all branches, NULL versus budget, and invalid packets.
+// Does not validate a GPU execution or main's final window copy/commit loop.
+#include "strata/core/gated_proposal.hpp"
+#include "strata/core/spec_proposal.hpp"
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <string>
+
+namespace {
+namespace ap = strata::core::gated;
+namespace ac = strata::core::proposal;
+constexpr int A=7, B=4097, C=8190, V=8192;
+constexpr float theta=.5f;
+using Row=std::array<int32_t,ap::kStride>;
+using Rows=std::array<int32_t,ap::kRows*ap::kStride>;
+using Floats=std::array<float,ap::kRows>;
+using Drafts=std::array<int32_t,ap::kRows>;
+int checks=0;
+void need(bool ok,const char* msg) {++checks;if(!ok)throw std::runtime_error(msg);}
+uint32_t bits(float f) {uint32_t n;std::memcpy(&n,&f,4);return n;}
+Row row(float a=.625f,float b=.25f,float c=.125f) {
+ Row r; r.fill(-1);
+ // IDs/padding deliberately unlike target order. Padding must survive masking.
+ for(int i=0;i<ap::kEntries;++i)r[ap::kEntries+i]=(int32_t)(0x3e800000u+(uint32_t)i);
+ const int ids[]={C,A,B};const float probs[]={c,a,b};
+ for(int i=0;i<3;++i){r[i]=ids[i];std::memcpy(r.data()+ap::kEntries+i,&probs[i],4);}
+ return r;
+}
+void put(Rows& rows,int n,const Row& r) {std::memcpy(rows.data()+n*ap::kStride,r.data(),sizeof(r));}
+void expect_mask(const int32_t* actual,const Row& original,float a,float b,float c) {
+ Row expected=original;const float p[]={c,a,b};
+ for(int i=0;i<3;++i)std::memcpy(expected.data()+ap::kEntries+i,&p[i],4);
+ need(std::memcmp(actual,expected.data(),sizeof(expected))==0,"mask differs in retained bits / IDs / padding / +0");
+}
+struct Expected {int T,nq,offers,attempts,failed;bool suffix,nulltail;};
+void packet(const ap::Packet& p,Expected x) {
+ need(!p.error,"unexpected packet error");
+ need(p.T==x.T && p.n_q==x.nq && p.offers==x.offers && p.attempts==x.attempts && p.failed_index==x.failed && p.from_sfx==x.suffix && p.null_tail==x.nulltail,"packet fields differ from hand-written expectation");
+ need(p.T==p.offers+1,"one terminal output after offered prefix");
+ need(ac::rowkind(p.T-1,p.T,p.n_q)==(p.null_tail?ac::RowKind::NullTail:ac::RowKind::PlainBonus),"AA packet / Hubble terminal rowkind mismatch");
+}
+struct Fixture {
+ Rows raw{};Drafts d{};Floats scores{};
+ Fixture(){raw.fill(-1);d.fill(-777);scores.fill(std::numeric_limits<float>::quiet_NaN());}
+ void set(int j,const Row& r,int id,float score){put(raw,j,r);d[j]=id;scores[j]=score;}
+ bool call(ap::Packet& out,int generated,int budget,const ap::RootChoice& root={},bool initial=false,int capacity=6) const {
+ const Rows saved=raw;const Drafts ds=d;const Floats ss=scores;
+ const bool ok=ap::make_packet(out,raw.data(),d.data(),scores.data(),generated,budget,capacity,V,theta,initial,root);
+ need(std::memcmp(raw.data(),saved.data(),sizeof raw)==0,"raw q mutated");
+ need(std::memcmp(d.data(),ds.data(),sizeof d)==0,"raw drafts mutated");
+ need(std::memcmp(scores.data(),ss.data(),sizeof scores)==0,"own-score mutated");
+ need(ok==(out.error==nullptr),"return/error inconsistent");
+ return ok;
+ }
+};
+void positive(const char* name,const Fixture& f,int n,int budget,ap::RootChoice root,Expected x) {
+ ap::Packet p;need(f.call(p,n,budget,root),name);packet(p,x);
+ std::printf("{\"case\":\"%s\",\"passed\":true}\n",name);
+}
+void tests() {
+ const Row raw=row();Row masked{};const Row saved=raw;
+ need(ap::mask_row(raw.data(),masked.data(),theta,-1,V),"mask H");
+ expect_mask(masked.data(),raw,.625f,0,0); // exact GPU T1_NULL mu
+ need(ap::mask_row(raw.data(),masked.data(),theta,B,V),"mask UNION");
+ expect_mask(masked.data(),raw,.625f,.25f,0); // GPU UNION mu, sum=.875 !=1
+ need(std::memcmp(raw.data(),saved.data(),sizeof raw)==0,"mask modified raw");
+ const Row edge=row(.5f,std::nextafter(.5f,0.0f),0x1p-25f);
+ need(ap::mask_row(edge.data(),masked.data(),theta,-1,V),"mask inclusive edge");
+ expect_mask(masked.data(),edge,.5f,0,0);
+ const Row above=row(std::nextafter(.5f,1.0f),.25f,.25f-0x1p-24f);
+ need(ap::mask_row(above.data(),masked.data(),theta,-1,V),"mask above edge");
+ expect_mask(masked.data(),above,std::nextafter(.5f,1.0f),0,0);
+ const Row none=row(.375f,.375f,.25f);
+ need(ap::mask_row(none.data(),masked.data(),theta,-1,V),"mask zero measure");
+ expect_mask(masked.data(),none,0,0,0);
+ const Row negzero=row(1.0f,-0.0f,0.0f);
+ need(ap::mask_row(negzero.data(),masked.data(),0.0f,-1,V),"theta zero mask");
+ expect_mask(masked.data(),negzero,1.0f,-0.0f,0.0f);
+ need(bits(ap::mass_at(masked.data(),B))==bits(-0.0f),"retained negative zero bits changed");
+
+ ap::Packet p;
+ need(ap::make_packet(p,nullptr,nullptr,nullptr,0,3,6,V,theta,true,{}),"initial no buffers");
+ packet(p,{1,0,0,0,-1,false,false});
+ need(ap::make_packet(p,nullptr,nullptr,nullptr,0,0,6,V,theta,false,{}),"zero producer budget");
+ packet(p,{1,0,0,0,-1,false,false});
+
+ Fixture firstnull;firstnull.set(0,raw,B,.25f);
+ positive("root_NULL",firstnull,1,3,{}, {1,1,0,1,0,false,true});
+ Fixture allzero;allzero.set(0,none,B,.375f);
+ positive("all_zero_mu_NULL",allzero,1,3,{}, {1,1,0,1,0,false,true});
+ Fixture one;one.set(0,raw,A,.625f);
+ positive("actual_max_steps_1_is_bonus",one,1,1,{}, {2,1,1,1,-1,false,false});
+ need(!one.call(p,1,3),"incomplete generated1/budget3 must not become bonus");
+ Fixture later;later.set(0,raw,A,.625f);later.set(1,raw,C,.125f);
+ positive("second_attempt_NULL",later,2,3,{}, {2,2,1,2,1,false,true});
+ need(later.call(p,2,3),"NULL packet");
+ need(p.offers==1 && p.failed_index==1 && p.T-1==1,"failed D excluded from offered window input span");
+ // Packet contains no token array. These counts exclude failed D from the
+ // intended copy span; actual main window construction remains integration work.
+ expect_mask(p.mu.data()+ap::kStride,raw,.625f,0,0);
+ Fixture full;full.set(0,raw,A,.625f);full.set(1,raw,A,.625f);full.set(2,raw,A,.625f);
+ positive("actual_max_steps_3_bonus",full,3,3,{}, {4,3,3,3,-1,false,false});
+ need(!full.call(p,3,4),"generated3/budget4 must not guess exhausted budget");
+
+ const ap::RootChoice E{true,B,6};
+ positive("UNION_low_D_suffix",firstnull,1,3,E, {6,1,5,1,-1,true,false});
+ need(firstnull.call(p,1,3,E),"UNION rescue");expect_mask(p.mu.data(),raw,.625f,.25f,0);
+ const Row union_root_snapshot=[](){Row r=row();Row out;need(ap::mask_row(r.data(),out.data(),theta,B,V),"snapshot union");return out;}();
+ Fixture high;high.set(0,raw,A,.625f);
+ need(high.call(p,1,1,E),"UNION D!=s offer");packet(p,{2,1,1,1,-1,false,false});
+ need(std::memcmp(p.mu.data(),union_root_snapshot.data(),sizeof(Row))==0,"UNION must exist on MTP branch");
+ Fixture otherlow;otherlow.set(0,raw,C,.125f);
+ need(otherlow.call(p,1,3,E),"UNION D!=s NULL");packet(p,{1,1,0,1,0,false,true});
+ need(std::memcmp(p.mu.data(),union_root_snapshot.data(),sizeof(Row))==0,"UNION must exist on NULL branch");
+ // Later row's B is NOT rescued by the root set.
+ Fixture next;next.set(0,raw,A,.625f);next.set(1,raw,B,.25f);
+ need(next.call(p,2,3,E),"later H only");packet(p,{2,2,1,2,1,false,true});
+ expect_mask(p.mu.data(),raw,.625f,.25f,0);
+ expect_mask(p.mu.data()+ap::kStride,raw,.625f,0,0);
+ // Eligibility false despite populated rescue fields must mean plain H.
+ positive("ineligible_rescue_ignored",firstnull,1,3,{false,B,6}, {1,1,0,1,0,false,true});
+
+ // Frozen source choice must not inspect invalid discarded later materialization.
+ // A future row's poison may be present, but only root is consumed if suffix wins.
+ Fixture eager=firstnull;
+ positive("suffix_ignores_later_D_score_q",eager,3,3,E, {6,1,5,1,-1,true,false});
+ Fixture stopped=firstnull;
+ positive("NULL_ignores_later_materialization",stopped,3,3,{}, {1,1,0,1,0,false,true});
+ Fixture edgeoffer;edgeoffer.set(0,edge,A,.5f);
+ positive("inclusive_D_at_theta",edgeoffer,1,1,{}, {2,1,1,1,-1,false,false});
+ Fixture edgenull;edgenull.set(0,edge,B,std::nextafter(.5f,0.0f));
+ positive("D_just_below_theta_NULL",edgenull,1,3,{}, {1,1,0,1,0,false,true});
+
+ Fixture bad=one;bad.scores[0]=std::nextafter(.625f,1.0f);
+ need(!bad.call(p,1,1),"own-score mismatch must fail");
+ bad=one;bad.d[0]=123;
+ need(!bad.call(p,1,1),"D outside q support must fail");
+ bad=one;bad.raw[1]=bad.raw[0];
+ need(!bad.call(p,1,1),"duplicate support must fail");
+ bad=one;float nan=std::numeric_limits<float>::quiet_NaN();std::memcpy(bad.raw.data()+ap::kEntries+1,&nan,4);
+ need(!bad.call(p,1,1),"nonfinite q must fail");
+ need(!one.call(p,1,0),"n_generated exceeds actual budget");
+ need(!one.call(p,1,1,{},true),"initial stale attempts");
+ need(!one.call(p,1,1,{true,V,6}),"out-of-vocab rescue");
+ need(!one.call(p,1,1,{true,B,7}),"suffix width exceeds capacity");
+ need(!one.call(p,1,6),"budget cannot fill terminal slot");
+ need(!ap::make_packet(p,nullptr,nullptr,nullptr,0,1,6,V,theta,false,{}),"missing expected attempt");
+ need(!ap::make_packet(p,nullptr,nullptr,nullptr,1,1,6,V,theta,false,{}),"null buffers");
+ need(!ap::mask_row(raw.data(),masked.data(),nan,-1,V),"NaN theta");
+ need(!ap::mask_row(raw.data(),masked.data(),-.1f,-1,V),"negative theta");
+ // Concrete producer/context boundaries: budget is not inferred from count.
+ // The failed row may be the LAST physically generated row at actual budget.
+ Fixture last_attempt;last_attempt.set(0,raw,A,.625f);last_attempt.set(1,raw,B,.25f);
+ need(last_attempt.call(p,2,2,{},false,3),"last producer attempt failed at exact budget");
+ packet(p,{2,2,1,2,1,false,true});
+ expect_mask(p.mu.data()+ap::kStride,raw,.625f,0,0);
+ // Zero remaining proposal budget has NO attempt, regardless of stale bytes.
+ Fixture poison;
+ need(poison.call(p,0,0,{},false,1),"capacity1 zero producer budget");
+ packet(p,{1,0,0,0,-1,false,false});
+ need(std::all_of(p.mu.begin(),p.mu.end(),[](int32_t word){return word==0;}),"zero-budget packet retained old mu");
+ need(!last_attempt.call(p,1,0,{},false,1),"actual generated row cannot be relabeled zero budget");
+ // If a caller has already materialized a failed row at the final target
+ // capacity, it must preserve NULL or explicitly reject this packet. Passing
+ // min(generated,room-1) here would erase evidence and evade this contract.
+ Fixture edge_failed;edge_failed.set(0,raw,B,.25f);
+ const bool edge_ok=edge_failed.call(p,1,1,{},false,1);
+ if(edge_ok)packet(p,{1,1,0,1,0,false,true});
+ else need(p.error!=nullptr,"capacity failure must be explicit, never bonus");
+ // A subsequent successful call must clear every stale error/count/mode field.
+ need(one.call(p,1,1),"packet reset after errors");packet(p,{2,1,1,1,-1,false,false});
+}
+} // namespace
+int main() {
+ try {tests();std::printf("{\"summary\":\"gated_proposal_test\",\"checks\":%d,\"passed\":true,\"GPU_executed\":false,\"actual_main_window_copy_tested\":false}\n",checks);return 0;}
+ catch(const std::exception& e){std::fprintf(stderr,"gated_proposal_test FAILED after %d checks: %s\n",checks,e.what());return 1;}
+}
diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp
index 292a5d2b4..8721624a6 100644
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@ -599,8 +599,10 @@ bool MtpDrafter::setup_coupled(std::string& err) {
coupled_ok_ = true;
if (spec_prob_env())
std::fprintf(stderr, "strata mtp: probabilistic draft acceptance on (STRATA_SPEC_PROB): sampled requests draft by "
- "sampling the draft head's distribution over %lld tokens; the verifier accepts with "
- "min(1, p/q) and resamples the residual\n", (long long) nv);
+ "sampling the draft head's distribution over %lld tokens; %s\n", (long long) nv,
+ spec_gate_pick()
+ ? "pick gate: masked mu offers, min(1, p/mu) acceptance and (p-mu)+ rejection/NULL residuals"
+ : "top gate (compatibility): q offers, min(1, p/q) acceptance and (p-q)+ rejection residuals");
else
std::fprintf(stderr, "strata mtp: coupled draft sampling on (STRATA_SPEC_COUPLED): sampled requests draft with the "
"target's chain and Philox draw over %lld tokens\n", (long long) nv);
@@ -1555,7 +1557,8 @@ DraftTiming g_dt;
} // namespace
bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t* drafts, std::string& err,
- float* probs, float min_p, int* n_drafts) {
+ float* probs, float min_p, int* n_drafts, DraftResult* result) {
+ if (result) { *result = {}; if (n_drafts) *n_drafts = 0; }
#if defined(STRATA_GPU_METAL)
{
if (T < 1 || T > max_t_ || a < 0 || a >= T) { err = "mtp: draft arguments out of range"; return false; }
@@ -1565,6 +1568,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
if (!strata::metal_engine::mtp_draft(T, tokens, p, a, max_steps, drafts, probs, min_p, &n, err)) return false;
for (int j = n; j < max_t_ - 1; ++j) { drafts[j] = 0; if (probs) probs[j] = 0.0f; }
if (n_drafts) *n_drafts = n;
+ if (result) *result = {n, max_steps};
ms_draft += ms_since(t0);
++rounds;
return true;
@@ -1736,6 +1740,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
}
for (int j = n; j < max_t_ - 1; ++j) { drafts[j] = 0; if (probs) probs[j] = 0.0f; }
if (n_drafts) *n_drafts = n;
+ if (result) *result = {n, max_steps};
ms_draft += ms_since(t0);
++rounds;
return true;
@@ -1855,12 +1860,13 @@ int MtpDrafter::chain_poll(std::string& err) {
}
bool MtpDrafter::draft_first(int T, const float* R_row, int32_t token, int64_t cell, int32_t* drafts, std::string& err,
- float* probs, float min_p, int* n_drafts) {
+ float* probs, float min_p, int* n_drafts, DraftResult* result) {
+ if (result) { *result = {}; if (n_drafts) *n_drafts = 0; }
#if defined(STRATA_GPU_METAL)
{ // the decode window's row 0 is the residual (the window that just ran)
(void) R_row;
std::vector<int32_t> toks((size_t) T, token);
- return draft(1, toks.data(), cell, 0, drafts, err, probs, min_p, n_drafts);
+ return draft(1, toks.data(), cell, 0, drafts, err, probs, min_p, n_drafts, result);
}
#endif
const OnDevice on_device(device_);
@@ -1873,7 +1879,7 @@ bool MtpDrafter::draft_first(int T, const float* R_row, int32_t token, int64_t c
return false;
}
std::vector<int32_t> toks((size_t) T, token);
- return draft(T, toks.data(), cell, 0, drafts, err, probs, min_p, n_drafts);
+ return draft(T, toks.data(), cell, 0, drafts, err, probs, min_p, n_drafts, result);
}
} // namespace strata::core
diff --git a/src/core/spec_admission_test.cpp b/src/core/spec_admission_test.cpp
new file mode 100644
index 000000000..3c8c2ca56
--- /dev/null
+++ b/src/core/spec_admission_test.cpp
@@ -0,0 +1,101 @@
+// Enumerate proposal and acceptance branches exactly; no Monte Carlo or GPU.
+#include "strata/core/spec_prob.hpp"
+#include "strata/core/gated_proposal.hpp"
+#include "strata/spec/draft_policy.hpp"
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+using namespace strata::core;
+using Dist = std::array<double, 2>;
+int failures = 0;
+void check(bool ok, const char* message) {
+ if (!ok) { ++failures; std::fprintf(stderr, "FAIL: %s\n", message); }
+}
+Dist judged(const Dist& p, const std::array<float,2>& q, int d) {
+ const int32_t ids[2] = {0,1};
+ const SpecP P{ids,p.data(),2}; const SpecQ Q{ids,q.data(),2};
+ const double a=std::min(1.0,p[d]/q[d]); Dist result{};
+ if (a>0) result[spec_verify_row(P,d,Q,(float)(a/2),0.5f).token]+=a;
+ if (a<1) result[spec_verify_row(P,d,Q,(float)((1+a)/2),0.5f).token]+=1-a;
+ return result;
+}
+Dist legacy_raw_q_plain_p(const Dist& p, const std::array<float,2>& q, bool own_draw, float threshold) {
+ Dist result{};
+ for (int d=0;d<2;++d) {
+ const float gate=own_draw?q[d]:std::max(q[0],q[1]);
+ const auto row=gate>=threshold?judged(p,q,d):p;
+ for (int i=0;i<2;++i) result[i]+=q[d]*row[i];
+ }
+ return result;
+}
+bool equal(const Dist& a,const Dist& b) {
+ return std::abs(a[0]-b[0])<1e-12 && std::abs(a[1]-b[1])<1e-12;
+}
+// Enumerate the repaired first-row mixture using the ACTUAL packet builder.
+// No candidate-conditioned renormalization: every draw receives the same root mu.
+Dist proposal_contract(const Dist& p,const std::array<float,2>& q,float threshold,
+ strata::core::gated::RootChoice root={}) {
+ std::array<int32_t,strata::core::gated::kStride> raw;
+ raw.fill(-1);raw[0]=0;raw[1]=1;
+ std::memcpy(raw.data()+strata::core::gated::kEntries,q.data(),sizeof(q));
+ Dist result{};
+ for (int d=0;d<2;++d) {
+ const int32_t draft=d;const float score=q[d];strata::core::gated::Packet packet;
+ const bool ok=strata::core::gated::make_packet(packet,raw.data(),&draft,&score,1,1,2,2,threshold,false,root);
+ check(ok,"valid gated packet");if(!ok)continue;
+ Dist mu{strata::core::gated::mass_at(packet.mu.data(),0),strata::core::gated::mass_at(packet.mu.data(),1)};
+ double acceptance=0;
+ if (!packet.null_tail) {
+ check(mu[d]>0,"offer has positive mu");
+ acceptance=std::min(1.0,p[d]/mu[d]);
+ result[d]+=q[d]*acceptance;
+ }
+ if (acceptance<1) {
+ Dist residual{std::max(0.0,p[0]-mu[0]),std::max(0.0,p[1]-mu[1])};
+ const double sum=residual[0]+residual[1];check(sum>0,"reachable failure has residual");
+ if(sum>0)for(int y=0;y<2;++y)result[y]+=q[d]*(1-acceptance)*residual[y]/sum;
+ }
+ }
+ return result;
+}
+int main(int argc,char** argv) {
+ const bool expect_pick=!(argc>1 && std::strcmp(argv[1],"top")==0);
+ check(spec_gate_pick()==expect_pick,"pick is default; only explicit lowercase top selects Top compatibility");
+ const Dist p{0.5,0.5};const std::array<float,2> q{0.875f,0.125f};
+ check(equal(legacy_raw_q_plain_p(p,q,true,0.5f),Dist{0.5625,0.4375}),
+ "LEGACY negative control: own-draw gate with full-q verifier/plain-p failure is biased");
+ check(equal(proposal_contract(p,q,0.5f),p),"new pick mu/NULL mixture preserves p");
+ check(equal(spec_gate_pick()?proposal_contract(p,q,0.5f):legacy_raw_q_plain_p(p,q,false,0.5f),p),
+ "configured repaired pick or Top compatibility preserves p");
+ int top_cases=0,proposal_cases=0;
+ for(int a=1;a<16;++a)for(int b=1;b<16;++b) {
+ const Dist P{a/16.0,1-a/16.0};const std::array<float,2> Q{b/16.0f,1-b/16.0f};
+ for(float threshold:{0.0f,0.5f,0.9f,1.1f}) {
+ check(equal(legacy_raw_q_plain_p(P,Q,false,threshold),P),"Top compatibility distribution law");++top_cases;
+ check(equal(proposal_contract(P,Q,threshold),P),"H-only proposal mixture");++proposal_cases;
+ for(int rescue:{0,1}) {
+ check(equal(proposal_contract(P,Q,threshold,{true,rescue,2}),P),"fixed root UNION on every source branch");++proposal_cases;
+ }
+ }
+ }
+ // A planner must not learn from draws it may select, including discarded position-keyed lookahead.
+ strata::spec::DraftPolicy live(6);
+ for(int i=0;i<5;++i) { live.observe(false,4,3,24,1.0);live.observe(false,6,5,24,1.5); }
+ const auto request_plan=live;
+ auto short_chain=live,long_chain=live;
+ short_chain.observe(false,4,3,24,0.2);long_chain.observe(false,4,3,24,8.0);
+ const bool short_lookup=short_chain.choose(4,5,24).lookup,long_lookup=long_chain.choose(4,5,24).lookup;
+ check(!short_lookup && long_lookup,"draw-dependent chain time can change the actual policy choice");
+ Dist conditional{},planned{};
+ for(int d=0;d<2;++d) {
+ const auto selected=(d==0?long_lookup:short_lookup)?p:judged(p,q,d);
+ const auto fixed=request_plan.choose(4,5,24).lookup?p:judged(p,q,d);
+ for(int i=0;i<2;++i) { conditional[i]+=q[d]*selected[i];planned[i]+=q[d]*fixed[i]; }
+ }
+ check(equal(conditional,Dist{0.4375,0.5625}),"LEGACY negative: D-conditioned source without common first measure changes proposal law");
+ check(equal(planned,p),"request plan preserves p despite subsequent timing observations");
+ std::printf("spec_admission_test: %d Top compatibility cases, %d actual mu-packet cases, legacy negatives; %d failures\n",top_cases,proposal_cases,failures);
+ return failures?1:0;
+}
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index 5e97b8f52..f82c846f4 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -12,6 +12,7 @@
#include "strata/core/remote_expert_opt.hpp"
#include "strata/core/dma_batch.hpp"
#include "strata/core/spec_prob.hpp"
+#include "strata/core/spec_proposal.hpp"
#if defined(_WIN32)
#ifndef NOMINMAX
#define NOMINMAX
@@ -2865,7 +2866,22 @@ bool Verifier::prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string
bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, void* user, int32_t* out,
std::string& err) {
+ const bool measure = spec_measure_;
+ // Clear this borrow and propagated stage borrows on every exit.
+ struct ProposalBorrow {
+ Verifier* verifier;
+ bool active;
+ ~ProposalBorrow() { if (active) verifier->set_spec_q(nullptr, 0); }
+ } proposal_borrow{this, measure};
+ if (measure && (T < 1 || T > strata::kernels::kVerifyMaxT ||
+ spec_nq_ < 0 || spec_nq_ > T || (spec_nq_ > 0 && spec_q_ == nullptr))) {
+ err = "verify: proposal row count/pointer is invalid"; return false;
+ }
+ if (measure && (!head_sampling_ || sampling_.greedy || sampling_.temperature <= 0.0f)) {
+ err = "verify: proposal requires sampled head"; return false;
+ }
#if defined(STRATA_GPU_METAL)
+ if (measure) { err = "verify: proposal unsupported on Metal"; return false; }
(void) pool;
(void) user;
return strata::metal_engine::window_run(metal_, T, tokens, pos0, head_sampling_, out, err);
@@ -3169,7 +3185,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
sp.counter = (uint64_t) pos0;
// STRATA_SPEC_PROB: the drafter's q lists for this window, judged by rejection sampling (spec_prob.hpp)
bool judged = false;
- if (sampled && spec_q_ != nullptr && spec_nq_ > 0 && T > 1 && T <= kVerifyMaxT) {
+ if (sampled && (measure || (spec_q_ != nullptr && spec_nq_ > 0 && T > 1)) && T <= kVerifyMaxT) {
if (d_spec_ == nullptr &&
cudaMalloc((void**) &d_spec_, (size_t) kVerifyMaxT * (1 + strata::core::kSpecQStride) * sizeof(int32_t)) !=
cudaSuccess) {
@@ -3177,18 +3193,26 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
d_spec_ = nullptr;
}
if (d_spec_ != nullptr) {
- const int nq = std::min(spec_nq_, T - 1);
+ const int nq = measure ? spec_nq_ : std::min(spec_nq_, T - 1);
int32_t* d_q = d_spec_ + kVerifyMaxT;
- if (cudaMemcpyAsync(d_spec_, tokens + 1, (size_t) (T - 1) * sizeof(int32_t), cudaMemcpyHostToDevice, cs_) ==
- cudaSuccess &&
- cudaMemcpyAsync(d_q, spec_q_, (size_t) nq * strata::core::kSpecQStride * sizeof(int32_t),
- cudaMemcpyHostToDevice, cs_) == cudaSuccess)
- judged = sample_tokens_spec(head_logits_, T, (int) n_vocab_, hist_d_, hist_len_, sp, d_spec_, d_q, nq,
- m_out_, cs_);
+ if ((T == 1 || cudaMemcpyAsync(d_spec_, tokens + 1, (size_t) (T - 1) * sizeof(int32_t), cudaMemcpyHostToDevice, cs_) ==
+ cudaSuccess) &&
+ (nq == 0 || cudaMemcpyAsync(d_q, spec_q_, (size_t) nq * strata::core::kSpecQStride * sizeof(int32_t),
+ cudaMemcpyHostToDevice, cs_) == cudaSuccess))
+ judged = measure
+ ? sample_tokens_proposal(head_logits_, T, (int) n_vocab_, hist_d_, hist_len_, sp,
+ d_spec_, d_q, nq, m_out_, cs_)
+ : sample_tokens_spec(head_logits_, T, (int) n_vocab_, hist_d_, hist_len_, sp,
+ d_spec_, d_q, nq, m_out_, cs_);
}
}
spec_q_ = nullptr;
spec_nq_ = 0;
+ if (measure && !judged) {
+ (void) cudaStreamSynchronize(cs_);
+ err = "verify: proposal sampler unavailable or allocation/copy/launch failed";
+ return false;
+ }
if (judged) ++spec_windows;
else sample_tokens(head_logits_, T, (int) n_vocab_, hist_d_, hist_len_, sp, m_out_, cs_);
if (cudaStreamSynchronize(cs_) != cudaSuccess) { // m_out_ is the mapped h_out_: synced, it is readable
@@ -3196,6 +3220,13 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
return false;
}
}
+ if (measure) {
+ for (int t = 0; t < T; ++t)
+ if (((volatile int32_t*) h_out_)[t] == strata::core::proposal::kProposalSamplerErrorToken) {
+ err = "verify: invalid proposal candidate or zero NULL residual";
+ return false;
+ }
+ }
for (int t = 0; t < T; ++t) out[t] = ((volatile int32_t*) h_out_)[t];
if (tap_file_ != nullptr && !batch_rec_) tap_write(T, tokens, pos0);
note_nonfinite(T, nullptr);
diff --git a/src/kernels/cuda/sampler.cu b/src/kernels/cuda/sampler.cu
index b9e04318f..6661a6244 100644
--- a/src/kernels/cuda/sampler.cu
+++ b/src/kernels/cuda/sampler.cu
@@ -22,6 +22,7 @@
#include "strata/kernels/sampler.hpp"
#include "strata/core/coupled_draft.hpp"
#include "strata/core/spec_prob.hpp"
+#include "strata/core/spec_proposal.hpp"
#include "strata/core/emulate.hpp"
#include <cuda_runtime.h>
@@ -1007,6 +1008,7 @@ spec_draft_merge_kernel(const int2* __restrict__ cand, int n_blocks, int nv, int
/// < n_q, a point mass otherwise), and write out[t] = d on an accept, the residual sample on a reject (never d), the
/// plain sample on the window's last row. The host's `while (window[a+1] == out[a]) ++a` is unchanged. Rows are
/// independent: each draws its own uniforms from its position (p.counter + t). `dtok` holds the T-1 drafts.
+template<bool Augmented>
__global__ void __launch_bounds__(32)
spec_verify_merge_kernel(const int2* __restrict__ cand, int n_blocks, int n_vocab, const SamplerParams p, int k, int T,
const int32_t* __restrict__ dtok, const int32_t* __restrict__ qbuf, int n_q,
@@ -1028,9 +1030,10 @@ spec_verify_merge_kernel(const int2* __restrict__ cand, int n_blocks, int n_voca
const int n_keep = sampled_probs_warp(sel_logit, k, p, ex);
const uint64_t pos = p.counter + (uint64_t) t;
const int d = t < T - 1 ? dtok[t] : -1;
- const int32_t* qid = qbuf + (size_t) t * strata::core::kSpecQStride;
- const float* qpr = (const float*) (qid + strata::core::kSpecQEntries);
const bool has_q = t < n_q;
+ const int32_t* qid = has_q ? qbuf + (size_t) t * strata::core::kSpecQStride : nullptr;
+ const float* qpr = has_q ? (const float*) (qid + strata::core::kSpecQEntries) : nullptr;
+ const auto kind = strata::core::proposal::rowkind(t, T, n_q);
// lane 0: p(d), q(d) and the accept decision. dec: 0 plain sample, 1 accept, 2 reject; point: no usable q(d)
int dec = 0, point = 1;
if (lane == 0 && d >= 0) {
@@ -1044,6 +1047,17 @@ spec_verify_merge_kernel(const int2* __restrict__ cand, int n_blocks, int n_voca
point = qd > 0.0 ? 0 : 1;
const float u = philox_uniform(p.seed, pos | strata::core::kSpecCtrAccept);
dec = (pd > 0.0 && (point ? (double) u < pd : (double) u * qd < pd)) ? 1 : 2;
+ if constexpr (Augmented) {
+ // An explicit mu candidate cannot silently become a point proposal.
+ if (has_q && (!(qd > 0.0) || !isfinite(qd))) dec = 3;
+ }
+ }
+ if constexpr (Augmented) {
+ if (lane == 0 && kind == strata::core::proposal::RowKind::MuOffer && (d < 0 || d >= n_vocab)) dec = 3;
+ if (lane == 0 && kind == strata::core::proposal::RowKind::NullTail) {
+ // Last row (including T=1) has d=-1, no dtok read or acceptance draw.
+ dec = 2; point = 0;
+ }
}
dec = __shfl_sync(kFullMask, dec, 0);
point = __shfl_sync(kFullMask, point, 0);
@@ -1065,7 +1079,9 @@ spec_verify_merge_kernel(const int2* __restrict__ cand, int n_blocks, int n_voca
}
if (lane == 0) {
int tok;
- if (dec == 0) { // no draft on this row: the plain sample, the exact-match sampler's own draw
+ if (dec == 3) {
+ tok = strata::core::proposal::kProposalSamplerErrorToken;
+ } else if (dec == 0) { // no draft on this row: the plain sample, the exact-match sampler's own draw
const float u = philox_uniform(p.seed, pos);
double cum = 0.0;
tok = sel_ids[n_keep > 0 ? n_keep - 1 : 0];
@@ -1079,7 +1095,12 @@ spec_verify_merge_kernel(const int2* __restrict__ cand, int n_blocks, int n_voca
double sum = 0.0;
for (int i = 0; i < n_keep; ++i) sum += rw[i];
if (!(sum > 0.0)) {
- tok = d; // p and q agree everywhere (rounding): the draft is as likely as it can be
+ if constexpr (Augmented) {
+ tok = kind == strata::core::proposal::RowKind::NullTail
+ ? strata::core::proposal::kProposalSamplerErrorToken : d;
+ } else {
+ tok = d; // preserve the legacy non-NULL rounding behavior
+ }
} else {
const float u = philox_uniform(p.seed, pos | strata::core::kSpecCtrResid);
double cum = 0.0;
@@ -1344,9 +1365,14 @@ void coupled_draft_sample(float* logits, int nv, const int32_t* sub_to_id, const
}
-bool sample_tokens_spec(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+template<bool Augmented>
+static bool sample_tokens_spec_impl(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
const SamplerParams& p, const int32_t* dtok, const int32_t* qbuf, int n_q, int* out,
void* stream) {
+ if constexpr (Augmented) {
+ if (n_tokens < 1 || n_q < 0 || n_q > n_tokens || (n_q > 0 && qbuf == nullptr) ||
+ (n_tokens > 1 && dtok == nullptr)) return false;
+ }
if (n_tokens <= 0 || n_vocab <= 0 || p.greedy || p.temperature <= 0.0f) return false;
if (p.penalty_last_n > 0 && (history == nullptr || history_len <= 0)) return false;
const int k = sampled_k(p.top_k, n_vocab);
@@ -1358,17 +1384,33 @@ bool sample_tokens_spec(const float* logits, int n_tokens, int n_vocab, const in
if (scratch == nullptr) return false;
sampler_split_part_kernel<<<dim3((unsigned) n_blocks, (unsigned) n_tokens), kSplitWarps * 32, 0,
(cudaStream_t) stream>>>(logits, n_vocab, history, history_len, p, k, n_blocks, scratch);
- spec_verify_merge_kernel<<<(unsigned) n_tokens, 32, 0, (cudaStream_t) stream>>>(scratch, n_blocks, n_vocab, p, k,
+ spec_verify_merge_kernel<Augmented><<<(unsigned) n_tokens, 32, 0, (cudaStream_t) stream>>>(scratch, n_blocks, n_vocab, p, k,
n_tokens, dtok, qbuf, n_q, out);
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) {
+ if constexpr (Augmented) return false;
std::fprintf(stderr, "sample_tokens_spec launch: %s\n", cudaGetErrorString(e));
std::exit(1);
}
- if (stream == nullptr) cudaDeviceSynchronize();
+ if (stream == nullptr) {
+ const cudaError_t sync = cudaDeviceSynchronize();
+ if constexpr (Augmented) { if (sync != cudaSuccess) return false; }
+ }
return true;
}
+bool sample_tokens_spec(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+ const SamplerParams& p, const int32_t* dtok, const int32_t* qbuf, int n_q, int* out,
+ void* stream) {
+ return sample_tokens_spec_impl<false>(logits, n_tokens, n_vocab, history, history_len, p, dtok, qbuf, n_q, out, stream);
+}
+
+bool sample_tokens_proposal(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+ const SamplerParams& p, const int32_t* dtok, const int32_t* qbuf, int n_q, int* out,
+ void* stream) {
+ return sample_tokens_spec_impl<true>(logits, n_tokens, n_vocab, history, history_len, p, dtok, qbuf, n_q, out, stream);
+}
+
void spec_draft_sample(float* logits, int nv, const int32_t* sub_to_id, const int32_t* id_to_sub, int id_vocab,
const SamplerParams* params, int32_t* ring, int cap, int j, const int32_t* step_rec,
void* scratch, int32_t* out_id, float* out_prob, int32_t* qrows, int gate_pick, float tscale, void* stream) {
diff --git a/src/kernels/spec_proposal_parity.cpp b/src/kernels/spec_proposal_parity.cpp
new file mode 100644
index 000000000..4161ec98e
--- /dev/null
+++ b/src/kernels/spec_proposal_parity.cpp
@@ -0,0 +1,167 @@
+// Explicit mu/NULL verifier against a CPU oracle; CUDA/HIP, no model or tracing.
+// Equal finite logits at three known IDs, all other logits -inf, give p=1/3
+// at each ID independently of an exp implementation. A top-k1 fixture gives
+// exact p=delta. Thus no test-only inspection API or SamplerTraceRow is needed.
+// These finite per-token checks do not claim arbitrary-logit bitwise equality.
+#include "strata/core/spec_prob.hpp"
+#include "strata/core/spec_proposal.hpp"
+#include "strata/kernels/sampler.hpp"
+#include <cuda_runtime.h>
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+constexpr int V=8192, MaxT=3, A=7, B=4097, C=8190;
+constexpr int Entries=strata::core::kSpecQEntries, Stride=strata::core::kSpecQStride;
+constexpr int Error=strata::core::proposal::kProposalSamplerErrorToken;
+using Mass=std::array<float,3>;
+constexpr std::array<int,3> IDs{{A,B,C}};
+struct Result {int token;bool accepted;};
+struct Case {
+ const char* name;int T,nq;std::vector<int32_t> drafts;
+ std::vector<Mass> mu,raw;int discarded=B;
+ bool error=false,legacy_equal=false;
+};
+void require(bool ok,const char* message) {if(!ok)throw std::runtime_error(message);}
+void cu(cudaError_t code,const char* message) {
+ if(code!=cudaSuccess)throw std::runtime_error(std::string(message)+": "+cudaGetErrorString(code));
+}
+float uniform(uint64_t seed,uint64_t counter) {
+ uint32_t c0=(uint32_t)counter,c1=(uint32_t)(counter>>32),c2=(uint32_t)seed,c3=(uint32_t)(seed>>32);
+ for(int i=0;i<10;++i) {
+ const uint64_t x=0x9E3779B9u*(uint64_t)c0,y=0xBB67AE85u*(uint64_t)c2;
+ const uint32_t n0=(uint32_t)(y>>32)^c1^(uint32_t)i,n2=(uint32_t)(x>>32)^c3;
+ c0=n0;c1=(uint32_t)y;c2=n2;c3=(uint32_t)x;
+ }
+ return (float)(c0>>8)*(1.0f/16777216.0f);
+}
+double mass(const Mass& q,int id) {
+ for(int i=0;i<3;++i)if(IDs[i]==id)return q[i];
+ return 0;
+}
+int choose(const std::array<double,3>& weights,float u,bool normalize) {
+ const double sum=weights[0]+weights[1]+weights[2];require(sum>0,"empty reachable law");
+ double cum=0;int last=-1;
+ for(int i=0;i<3;++i) {
+ if(normalize && weights[i]<=0)continue;
+ last=IDs[i];cum+=normalize?weights[i]/sum:weights[i];
+ if((double)u<cum)return IDs[i];
+ }
+ require(last>=0,"empty support");return last;
+}
+Result reference(const Case& c,int row,uint64_t seed,const Mass* alternate=nullptr) {
+ const uint64_t pos=365+(uint64_t)row;
+ const std::array<double,3> p=c.error?std::array<double,3>{{1,0,0}}:std::array<double,3>{{1.0/3,1.0/3,1.0/3}};
+ const bool last=row==c.T-1,null=last && c.nq==c.T;
+ if(last && !null)return {choose(p,uniform(seed,pos),false),false};
+ const int d=null?-1:c.drafts[row];const bool point=!null && row>=c.nq;
+ const Mass q=alternate?*alternate:(row<c.nq?c.mu[row]:Mass{{0,0,0}});
+ double pd=0;for(int i=0;i<3;++i)if(IDs[i]==d)pd=p[i];
+ if(!null) {
+ const double qd=point?1:mass(q,d);
+ if(!(qd>0) || !std::isfinite(qd))return {Error,false};
+ if(pd>0 && (double)uniform(seed,pos|strata::core::kSpecCtrAccept)*qd<pd)return {d,true};
+ }
+ std::array<double,3> weights{};
+ for(int i=0;i<3;++i)weights[i]=point?(IDs[i]==d?0:p[i]):std::max(0.0,p[i]-mass(q,IDs[i]));
+ if(weights[0]+weights[1]+weights[2]<=0)return {Error,false};
+ return {choose(weights,uniform(seed,pos|strata::core::kSpecCtrResid),true),false};
+}
+struct Device {
+ float* logits=nullptr;int* out=nullptr;int32_t* draft=nullptr;int32_t* mu=nullptr;
+ Device() {
+ cu(cudaMalloc((void**)&logits,MaxT*V*sizeof(float)),"logits");
+ cu(cudaMalloc((void**)&out,MaxT*sizeof(int)),"out");
+ cu(cudaMalloc((void**)&draft,(MaxT-1)*sizeof(int32_t)),"draft");
+ cu(cudaMalloc((void**)&mu,MaxT*Stride*sizeof(int32_t)),"mu");
+ }
+ ~Device(){cudaDeviceSynchronize();cudaFree(logits);cudaFree(out);cudaFree(draft);cudaFree(mu);}
+};
+std::vector<Case> cases() {
+ const Mass raw{{.625f,.25f,.125f}},m{{.625f,0,0}},root{{.625f,.25f,0}};
+ return {
+ {"T1 NULL",1,1,{}, {m},{raw}},
+ {"zero mu NULL",1,1,{}, {Mass{{0,0,0}}},{Mass{{.375f,.375f,.25f}}}},
+ {"offer then NULL",2,2,{A},{m,m},{raw,raw}},
+ {"two offers then NULL",3,3,{A,A},{m,m,m},{raw,raw,raw}},
+ {"plain budget",1,0,{}, {},{}},
+ {"offer reject and bonus",2,1,{A},{m},{raw}},
+ {"mu then delta then bonus",3,1,{A,B},{m},{raw}},
+ {"point then bonus",2,0,{A},{},{}},
+ {"UNION low-q rescue",2,1,{B},{root},{raw}},
+ {"UNION other draw rejection",2,1,{A},{root},{raw}},
+ {"UNION NULL",1,1,{}, {root},{raw},C},
+ {"inclusive threshold NULL",1,1,{}, {Mass{{.5f,0,0}}},{Mass{{.5f,.25f,.25f}}}},
+ {"zero residual sentinel",1,1,{}, {Mass{{1,0,0}}},{Mass{{1,0,0}}},-1,true},
+ {"legacy full-q parity",3,2,{A,B},{raw,raw},{raw,raw},-1,false,true},
+ {"missing offered mu sentinel",2,1,{B},{m},{raw}}
+ };
+}
+int run() {
+ Device d;long long mismatches=0,rows=0,accepts=0,rejects=0,null_same=0,plain_witness=0,fullq_witness=0,union_witness=0;
+ const uint64_t seeds[]={0,1,2,3,7,11,17,29,41,67,101,777,2026101600ULL,2026101700ULL,0x123456789abcdef0ULL,3728620368707436339ULL};
+ for(const Case& c:cases()) {
+ require((int)c.drafts.size()==c.T-1 && (int)c.mu.size()==c.nq,"bad fixture");
+ std::vector<float> logits(c.T*V,-std::numeric_limits<float>::infinity());
+ for(int t=0;t<c.T;++t){logits[t*V+A]=0;if(!c.error){logits[t*V+B]=0;logits[t*V+C]=0;}}
+ std::vector<int32_t> packed(c.T*Stride,-1);const int permutation[]={2,0,1};
+ for(int t=0;t<c.nq;++t)for(int i=0;i<Entries;++i) {
+ const float q=i<3?c.mu[t][permutation[i]]:0;
+ if(i<3)packed[t*Stride+i]=IDs[permutation[i]];
+ std::memcpy(&packed[t*Stride+Entries+i],&q,sizeof(q));
+ }
+ cu(cudaMemcpy(d.mu,packed.data(),packed.size()*sizeof(int32_t),cudaMemcpyHostToDevice),"copy mu");
+ if(!c.drafts.empty())cu(cudaMemcpy(d.draft,c.drafts.data(),c.drafts.size()*sizeof(int32_t),cudaMemcpyHostToDevice),"copy drafts");
+ for(uint64_t seed:seeds) {
+ strata::kernels::SamplerParams p;p.top_k=c.error?1:3;p.top_p=1;p.min_p=0;p.temperature=1;p.seed=seed;p.counter=365;
+ const int32_t* drafts=c.drafts.empty()?nullptr:d.draft;
+ cu(cudaMemcpy(d.logits,logits.data(),logits.size()*sizeof(float),cudaMemcpyHostToDevice),"copy logits");
+ require(strata::kernels::sample_tokens_proposal(d.logits,c.T,V,nullptr,0,p,drafts,d.mu,c.nq,d.out,nullptr),"proposal Split refused");
+ cu(cudaDeviceSynchronize(),"proposal sync");std::vector<int> out(c.T);
+ cu(cudaMemcpy(out.data(),d.out,c.T*sizeof(int),cudaMemcpyDeviceToHost),"copy out");
+ for(int t=0;t<c.T;++t) {
+ const auto ref=reference(c,t,seed);++rows;
+ if(out[t]!=ref.token){++mismatches;std::fprintf(stderr,"%s seed=%llu row=%d: got=%d expected=%d\n",c.name,(unsigned long long)seed,t,out[t],ref.token);}
+ const bool null=t==c.T-1 && c.nq==c.T;
+ if(t<c.T-1 && ref.token!=Error)(ref.accepted?accepts:rejects)++;
+ if(null && !c.error) {
+ if(out[t]==c.discarded)++null_same;
+ const std::array<double,3> target{{1.0/3,1.0/3,1.0/3}};
+ if(ref.token!=choose(target,uniform(seed,365+t),false))++plain_witness;
+ if(ref.token!=reference(c,t,seed,&c.raw[t]).token)++fullq_witness;
+ if(std::strcmp(c.name,"UNION NULL")==0) {
+ const Mass old{{.625f,0,0}};
+ if(ref.token!=reference(c,t,seed,&old).token)++union_witness;
+ }
+ // NULL is terminal even when residual==discarded D. T1 has
+ // zero offered inputs; no dtok slot exists for its raw D.
+ if(c.T==1 && out[t]==c.discarded)require(c.drafts.empty(),"NULL candidate leaked into offered inputs");
+ }
+ }
+ if(c.legacy_equal) {
+ cu(cudaMemcpy(d.logits,logits.data(),logits.size()*sizeof(float),cudaMemcpyHostToDevice),"legacy logits");
+ require(strata::kernels::sample_tokens_spec(d.logits,c.T,V,nullptr,0,p,drafts,d.mu,c.nq,d.out,nullptr),"legacy Split refused");
+ cu(cudaDeviceSynchronize(),"legacy sync");std::vector<int> old(c.T);
+ cu(cudaMemcpy(old.data(),d.out,c.T*sizeof(int),cudaMemcpyDeviceToHost),"legacy output");
+ require(old==out,"explicit proposal call changed legacy full-q result");
+ }
+ }
+ }
+ require(accepts>0 && rejects>0 && null_same>0,"missing accept/reject/NULL-equality coverage");
+ require(plain_witness>0 && fullq_witness>0 && union_witness>0,"fixed seeds did not discriminate incorrect laws");
+ std::printf("spec_proposal_parity: %lld rows, %lld mismatches; accepts=%lld rejects=%lld NULL==rawD=%lld; ordinary/fullq/omitted-rescue witnesses=%lld/%lld/%lld\n",rows,mismatches,accepts,rejects,null_same,plain_witness,fullq_witness,union_witness);
+ return mismatches?1:0;
+}
+} // namespace
+int main() {
+ try{return run();}catch(const std::exception& e){std::fprintf(stderr,"spec_proposal_parity: %s\n",e.what());return 1;}
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 75d404ac9..cd3cb12e3 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -68,6 +68,7 @@
#include "strata/core/native_head.hpp"
#include "strata/core/verify.hpp"
#include "strata/core/mtp.hpp"
+#include "strata/core/gated_proposal.hpp"
#if defined(STRATA_HAVE_DFLASH2)
#include "strata/core/dflash2.hpp"
#include "strata/core/draft_cap.hpp"
@@ -2340,6 +2341,89 @@ static void prime_hw_queues() {
}
#endif
+// STRATA_SPEC_PROB=1 with the default (pick) gate: a draft that fails the confidence gate is still a proposal attempt, and the
+// verifier gets the gated measure (core/gated_proposal.hpp). That is exact only where the window choice cannot see a draw or a
+// timing that the next rounds draw again. Where it cannot be kept, say why and return false: the caller keeps the older rule
+// (a warning, not a refusal).
+static bool gated_proposal_usable(bool serial_single_device, int lookup_chain, int adapt_swaps, bool pcie_share, bool own_window_rule,
+ int window, double min_p) {
+ const char* why = nullptr;
+#if defined(STRATA_GPU_METAL)
+ why = "the Metal engine has no gated verifier";
+#endif
+ if (why == nullptr && !serial_single_device) why = "it needs serial decode on one device";
+ if (why == nullptr && (lookup_chain != 0 || adapt_swaps != 0 || pcie_share))
+ why = "it needs --lookup-chain 0, --adapt-swaps 0 and --pcie-frac 0";
+ if (why == nullptr && own_window_rule) why = "STRATA_SPEC_COST, STRATA_SPEC_T_CYCLE, DFlash2 and --spec-corrupt change the window by their own rule";
+ if (why == nullptr && window > strata::core::gated::kRows) why = "the window is wider than 8 rows";
+ if (why == nullptr && (!std::isfinite(min_p) || min_p < 0.0)) why = "--spec-min-p must be a finite number >= 0";
+ if (why == nullptr) return true;
+ static bool warned = false;
+ if (!warned) {
+ warned = true;
+ std::fprintf(stderr, "strata: STRATA_SPEC_PROB=1 keeps the older acceptance rule in this run (%s). That rule's output "
+ "distribution is off when the confidence gate fails; STRATA_SPEC_PROB_GATE=top is exact here.\n", why);
+ }
+ return false;
+}
+
+// A draft at MTP cell c is sampled at counter c+1. The next target starts
+// there and needs one final cell for its bonus. Cap BEFORE any proposal draw;
+// a zero budget therefore means no attempt, never a discarded failed NULL.
+class GatedDraftLimit {
+public:
+ GatedDraftLimit(strata::core::MtpDrafter& mtp, bool active, int configured_mtp,
+ int64_t next_target_pos, int64_t context_end)
+ : mtp_(mtp), active_(active), configured_(configured_mtp - 1) {
+ if (active_) {
+ const int64_t room = std::max<int64_t>(0, context_end - next_target_pos);
+ mtp_.set_max_drafts((int) std::min<int64_t>(configured_, std::max<int64_t>(0, room - 1)));
+ }
+ }
+ ~GatedDraftLimit() { if (active_) mtp_.set_max_drafts(configured_); }
+ GatedDraftLimit(const GatedDraftLimit&) = delete;
+ GatedDraftLimit& operator=(const GatedDraftLimit&) = delete;
+private:
+ strata::core::MtpDrafter& mtp_;
+ bool active_;
+ int configured_;
+};
+
+// Prepare one gated window from the last successful producer result. Context room
+// is fixed before consulting D; the producer was already capped before drawing.
+// No max_new/output-cap shortening: the ordinary emission loop owns that limit.
+static bool prepare_gated_window(strata::core::gated::Packet& packet,
+ const strata::core::MtpDrafter::DraftResult& generation,
+ strata::spec::SuffixDrafter& suffix, const strata::spec::DraftPolicy& policy,
+ bool suffix_enabled, bool initial, int room, int configured_mtp,
+ int vocab, float threshold, const int32_t* q, const int32_t* drafts,
+ const float* scores, int32_t* suffix_tokens, int& match, std::string& err) {
+ if (room < 1 || generation.budget < 0 || generation.generated < 0 ||
+ generation.generated > generation.budget || generation.budget >= room) {
+ err = "gated proposal: invalid context room or producer result";
+ return false;
+ }
+ strata::core::gated::RootChoice root;
+ match = 0;
+ if (suffix_enabled && !initial && room > 1) {
+ const int k = suffix.propose(room - 1, suffix_tokens);
+ match = suffix.last_match();
+ if (k > 0) {
+ // Prefix/configuration only: never inspect a draw or its gated width.
+ const auto pick = policy.choose(std::min(configured_mtp, room), k, match);
+ root.eligible = pick.lookup;
+ root.rescue_id = suffix_tokens[0];
+ root.window = pick.t;
+ }
+ }
+ if (!strata::core::gated::make_packet(packet, q, drafts, scores, generation.generated, generation.budget,
+ room, vocab, threshold, initial, root)) {
+ err = std::string("gated proposal: ") + packet.error;
+ return false;
+ }
+ return true;
+}
+
int main(int argc, char** argv) {
// **UNBUFFERED, BECAUSE THE INTERESTING OUTPUT IS THE OUTPUT BEFORE A CRASH.** `stdout` redirected to a
// pipe or a file is block-buffered, so a program that dies loses every line it had already printed - which
@@ -14132,6 +14216,10 @@ int main(int argc, char** argv) {
ver.set_sampling(req_sp);
if (pipe) ver_b.set_sampling(req_sp); // (reaches the later stage's odd verifier)
if (use_mtp) mtp.set_draft_sampling(req_sp); // STRATA_SPEC_COUPLED=1: sampled drafts (a no-op otherwise)
+ const bool gated_proposal = use_mtp && mtp.prob() && strata::core::spec_gate_pick() &&
+ gated_proposal_usable(o.pipeline_windows < 2 && o.batch == 0 && n_stages == 1 && !multi_gpu && drive.d.remote_count == 0,
+ o.lookup_chain, o.adapt_swaps, req_pcie_frac != 0.0 || o.pcie_frac_auto, spec_cost, S,
+ req_spec_min_p);
{ // aux_cpus=0|1: where the non-pool, non-host threads run for this request (-1 = the start-up --aux-cpus); moves
// threads only. An enabled placement is swept at every request: it catches threads created since.
static const bool aux_start = strata::aux_cpus::enabled();
@@ -14366,6 +14454,11 @@ int main(int argc, char** argv) {
extra_sources_reset();
for (int64_t t : ids) { const int32_t t32 = (int32_t) t; extra_sources_append(&t32, 1); }
}
+ // This request's source planner: the live policy as it stands at the start (the choice must not depend on the
+ // request's own draws, so it is not updated from them).
+ const auto source_policy = policy;
+ strata::core::MtpDrafter::DraftResult generation;
+ strata::core::gated::Packet proposal; // immutable through the synchronous verifier call
bool first_window = true;
int64_t produced_n = 0, sfx_windows = 0, sfx_drafts = 0, sfx_ok = 0;
int64_t chain_windows = 0, chain_drafts = 0, chain_ok = 0; // --lookup-chain's own counts
@@ -15063,29 +15156,45 @@ int main(int argc, char** argv) {
}
}
while (!pl_ran && !cancelled && produced_n < max_new) {
- int T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft fires
+ int T = 1;
policy.set_context(p);
- if (use_mtp && spec_cost && !first_window) {
- T = std::clamp(chain_rows, 1, S_mtp);
- } else if (use_mtp && req_spec_min_p > 0.0) {
- T = 1;
- while (T < S_mtp && dprob[(size_t) T - 1] >= (float) req_spec_min_p) ++T;
- }
- if (first_window) T = merge_T > 0 ? merge_T : 1;
- std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) std::min<size_t>(dprob.size(), 8), dwin_s.begin());
const bool merged = first_window && merge_T > 0; // the prompt's last tokens and the first token
- // a repeat of earlier context (prompt lookup) where the MTP's own first guess agrees: the policy takes it
- // when its expected tokens per ms, from the measured acceptance and window costs, beat the MTP window's
bool from_sfx = false;
int sfx_match = 0;
- if (o.suffix_draft > 0 && !first_window) {
- const int k = sfx.propose(S - 1, sbuf.data());
- sfx_match = sfx.last_match();
- if (k > 0 && (!use_mtp || sbuf[0] == drafts[0])) {
- const strata::spec::DraftPolicy::Pick pk = policy.choose(T, k, sfx_match);
- if (pk.lookup) { T = pk.t; from_sfx = true; }
+ if (gated_proposal) {
+ const int room = (int) std::min<int64_t>(S, o.max_context - p);
+ if (!prepare_gated_window(proposal, generation, sfx, source_policy, o.suffix_draft > 0,
+ first_window, room, S_mtp, (int) ver.vocab(), (float) req_spec_min_p,
+ mtp.spec_q(), drafts.data(), dprob.data(), sbuf.data(), sfx_match, err)) {
+ std::printf("ERR %s\n", err.c_str()); return 1;
+ }
+ T = proposal.T; from_sfx = proposal.from_sfx;
+ if (first_window && merge_T > 0) T = merge_T; // STRATA_ADMIT_MERGE_FIRST: the prompt's last tokens ride in this window
+ } else {
+ T = use_mtp ? S_mtp : 1; // no --mtp: one token a round unless a lookup draft fires
+ if (use_mtp && spec_cost && !first_window) {
+ T = std::clamp(chain_rows, 1, S_mtp);
+ } else if (use_mtp && req_spec_min_p > 0.0) {
+ T = 1;
+ while (T < S_mtp && dprob[(size_t) T - 1] >= (float) req_spec_min_p) ++T;
+ }
+ if (first_window) T = merge_T > 0 ? merge_T : 1;
+ // a repeat of earlier context (prompt lookup) where the MTP's own first guess agrees: the policy takes it
+ // when its expected tokens per ms, from the measured acceptance and window costs, beat the MTP window's
+ if (o.suffix_draft > 0 && !first_window) {
+ const int k = sfx.propose(S - 1, sbuf.data());
+ sfx_match = sfx.last_match();
+ // a probabilistic window must not condition the source on the draw: the policy copy fixed at the request's start
+ // chooses, and the lookup is taken without asking whether the MTP's first guess agrees
+ const bool probabilistic = use_mtp && mtp.prob();
+ if (k > 0 && (probabilistic || !use_mtp || sbuf[0] == drafts[0])) {
+ const strata::spec::DraftPolicy::Pick pk = probabilistic
+ ? source_policy.choose(S_mtp, k, sfx_match) : policy.choose(T, k, sfx_match);
+ if (pk.lookup) { T = pk.t; from_sfx = true; }
+ }
}
}
+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) std::min<size_t>(dprob.size(), 8), dwin_s.begin());
// --lookup-chain: what followed an earlier occurrence of the context + the MTP's drafts, after them
int chain_n = 0, cm = 0;
if (o.lookup_chain > 0 && !first_window && !from_sfx && T < S) {
@@ -15273,7 +15382,8 @@ int main(int argc, char** argv) {
// STRATA_SPEC_PROB: the MTP drafts' distributions q, judged by rejection sampling (core/spec_prob.hpp);
// a suffix window and the lookup chain's tail are point masses and keep the exact-match rule
// (never on a first window: a merged one holds prompt tokens, not drafts)
- if (use_mtp && mtp.prob() && !from_sfx && !first_window && T_mtp > 1) ver.set_spec_q(mtp.spec_q(), T_mtp - 1);
+ if (gated_proposal) ver.set_spec_proposal(proposal.n_q ? proposal.mu.data() : nullptr, proposal.n_q);
+ else if (use_mtp && mtp.prob() && !from_sfx && !first_window && T_mtp > 1) ver.set_spec_q(mtp.spec_q(), T_mtp - 1);
if (!ver.run(T, window.data(), p, win_pool_fn, win_pool_user, outv.data(), err) || drive.d.failed) {
std::printf("ERR %s\n", drive.d.failed && drive.d.fail ? drive.d.fail : err.c_str());
return 1;
@@ -15353,8 +15463,11 @@ int main(int argc, char** argv) {
// commit, outv[a] is its row 0) - the drafts extend it on the device as the verify rows will
if (use_mtp && hist_n > 0 && mtp.coupled() && !eos && produced_n < max_new)
mtp.set_draft_history(consumed.data(), (int64_t) consumed.size(), outv[(size_t) a]);
+ generation = {}; // a skipped or failed production must not expose an older q view
+ const GatedDraftLimit draft_limit(mtp, gated_proposal, S_mtp, p + a + 1, o.max_context);
const bool drafted = !use_mtp || eos || produced_n >= max_new ||
- mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p);
+ mtp.draft(T, outv.data(), p, a, drafts.data(), err, dprob.data(), (float) req_spec_min_p,
+ nullptr, gated_proposal ? &generation : nullptr);
{
const Clock::time_point tw3 = Clock::now();
auto msd = [](Clock::time_point a0, Clock::time_point b0) { return std::chrono::duration<double, std::milli>(b0 - a0).count(); };
@@ -16621,10 +16734,23 @@ int main(int argc, char** argv) {
// plan v0.3 P6: with a native pack the first window is the last prompt token alone (it produces the first
// generated token and the MTP's first cell); otherwise the token loop already did that.
bool first_window = native_pack;
- if (use_mtp && !first_window &&
- !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), (float) o.spec_min_p)) {
- std::fprintf(stderr, "strata generate: %s\n", err.c_str());
- return 1;
+ const int S_mtp = o.mtp_max_t > 0 ? std::min(o.mtp_max_t, o.spec) : o.spec;
+ const bool gated_proposal = use_mtp && mtp.prob() && strata::core::spec_gate_pick() &&
+ gated_proposal_usable(o.pipeline_windows < 2 && o.batch == 0 && !multi_gpu && drive.d.remote_count == 0,
+ o.lookup_chain, o.adapt_swaps, o.pcie_frac != 0.0 || o.pcie_frac_auto,
+ spec_cost || use_df2 || !t_cycle.empty() || o.spec_corrupt > 0, o.spec, o.spec_min_p);
+ if (gated_proposal && !follow.empty())
+ std::fprintf(stderr, "strata generate: --spec-follow replaces outputs before acceptance (teacher-forced diagnostic, not sampling-law evidence)\n");
+ strata::core::MtpDrafter::DraftResult generation;
+ strata::core::gated::Packet proposal;
+ { // Non-native prefill already emitted x; draft_first(cell=p-1) samples at p.
+ const GatedDraftLimit draft_limit(mtp, gated_proposal, S_mtp, p, o.max_context);
+ if (use_mtp && !first_window &&
+ !mtp.draft_first(o.spec, ss.R, x, p - 1, drafts.data(), err, dprob.data(), (float) o.spec_min_p,
+ nullptr, gated_proposal ? &generation : nullptr)) {
+ std::fprintf(stderr, "strata generate: %s\n", err.c_str());
+ return 1;
+ }
}
std::vector<int32_t> window((size_t) o.spec), outv((size_t) o.spec);
std::vector<uint8_t> fdiff((size_t) o.spec, 0);
@@ -16636,10 +16762,10 @@ int main(int argc, char** argv) {
std::vector<float> hash_rows;
std::vector<int64_t> accepted_hist((size_t) o.spec, 0);
int64_t rounds = 0, drafts_total = 0, drafts_ok = 0, corrupt_counter = 0;
- const int S_mtp = o.mtp_max_t > 0 ? std::min(o.mtp_max_t, o.spec) : o.spec;
if (use_mtp && S_mtp < o.spec) mtp.set_max_drafts(S_mtp - 1);
strata::spec::SuffixDrafter sfx(std::max(1, o.suffix_draft), 64, (size_t) o.max_context + 4096);
strata::spec::DraftPolicy policy(o.spec); // MTP or lookup window (see draft_policy.hpp)
+ const auto source_policy = policy; // the lookup choice of a probabilistic window: fixed at the start, never fed by this run's draws
strata::spec::DraftPolicy::ChainState chain_st;
int chain_rows = 1; // STRATA_SPEC_COST: the window the cost-aware rule kept in the MTP chain just drafted
if (const char* tab = std::getenv("STRATA_SPEC_COST_TABLE"); spec_cost && tab != nullptr && tab[0] != '\0') {
@@ -16739,7 +16865,8 @@ int main(int argc, char** argv) {
return true;
}
#endif
- return mtp.draft(T_, outv.data(), p, a_, drafts.data(), err, dprob.data(), (float) o.spec_min_p);
+ return mtp.draft(T_, outv.data(), p, a_, drafts.data(), err, dprob.data(), (float) o.spec_min_p,
+ nullptr, gated_proposal ? &generation : nullptr);
};
int win_cap = o.spec; // the widest window the VRAM allows (lowered on an out-of-memory capture, see below)
while ((int64_t) produced.size() < max_new) {
@@ -16748,6 +16875,18 @@ int main(int argc, char** argv) {
strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());
int T = S_mtp;
policy.set_context(p);
+ bool from_sfx = false;
+ int sfx_match = 0;
+ if (gated_proposal) {
+ if (!prepare_gated_window(proposal, generation, sfx, source_policy, o.suffix_draft > 0, first_window,
+ (int) std::min<int64_t>(win_cap, o.max_context - p), S_mtp, (int) ver.vocab(),
+ (float) o.spec_min_p, mtp.spec_q(), drafts.data(), dprob.data(), sbuf.data(),
+ sfx_match, err)) {
+ std::fprintf(stderr, "strata generate: %s\n", err.c_str()); return 1;
+ }
+ T = proposal.T; from_sfx = proposal.from_sfx;
+ std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) o.spec, dwin.begin());
+ } else {
if (use_mtp && spec_cost && !first_window) {
T = std::clamp(chain_rows, 1, S_mtp);
} else if (use_mtp && o.spec_min_p > 0.0) {
@@ -16764,16 +16903,19 @@ int main(int argc, char** argv) {
if (first_window) T = 1;
std::copy(dprob.begin(), dprob.begin() + (std::ptrdiff_t) o.spec, dwin.begin());
if (!t_cycle.empty() && !first_window) T = t_cycle[(size_t) (rounds % (int64_t) t_cycle.size())];
- bool from_sfx = false;
- int sfx_match = 0;
if (o.suffix_draft > 0 && !first_window) {
const int k = sfx.propose(o.spec - 1, sbuf.data());
sfx_match = sfx.last_match();
- if (k > 0 && (!use_mtp || sbuf[0] == drafts[0])) {
- const strata::spec::DraftPolicy::Pick pk = policy.choose(T, k, sfx_match);
+ // a probabilistic window must not condition the source on the draw: the policy copy fixed at the start chooses, and the
+ // lookup is taken without asking whether the MTP's first guess agrees
+ const bool probabilistic = use_mtp && mtp.prob();
+ if (k > 0 && (probabilistic || !use_mtp || sbuf[0] == drafts[0])) {
+ const strata::spec::DraftPolicy::Pick pk = probabilistic
+ ? source_policy.choose(S_mtp, k, sfx_match) : policy.choose(T, k, sfx_match);
if (pk.lookup) { T = pk.t; from_sfx = true; }
}
}
+ }
// The window's rows sit at p .. p + T - 1, and past the context there is no cell to verify them in: near
// its end the window takes only the rows that fit (the admission checked the prompt and max_new, which a
// full window can overrun)
@@ -16818,11 +16960,12 @@ int main(int argc, char** argv) {
// (STRATA_ADAPT_NOWAIT=1: 0.1.37's non-blocking query, the A/B)
if (adapt_nowait()) apply_pending(false);
else if (pending.empty() || ++pending_age >= adapt_lag()) apply_pending(true);
- if (use_mtp && mtp.prob() && !from_sfx && T_mtp > 1 && o.spec_corrupt <= 0) ver.set_spec_q(mtp.spec_q(), T_mtp - 1);
+ if (gated_proposal) ver.set_spec_proposal(proposal.n_q ? proposal.mu.data() : nullptr, proposal.n_q);
+ else if (use_mtp && mtp.prob() && !from_sfx && T_mtp > 1 && o.spec_corrupt <= 0) ver.set_spec_q(mtp.spec_q(), T_mtp - 1);
if (!ver.run(T, window.data(), p, &drive_pool_multi, &drive, outv.data(), err)) {
// The graph of a window size is captured the first time it is used, and the draft source took VRAM: when the capture runs
// out of memory the window gets one row narrower (the widths below it are already captured) instead of the run failing.
- if (T > 2 && !first_window && err.find("out of memory") != std::string::npos) {
+ if (T > 2 && !first_window && !gated_proposal && err.find("out of memory") != std::string::npos) {
win_cap = T - 1;
--window_hist[(size_t) T];
std::fprintf(stderr, "strata generate: not enough VRAM left for a %d-token verify window: windows are %d tokens at most from here\n", T, win_cap);
@@ -16903,6 +17046,8 @@ int main(int argc, char** argv) {
total_ms += std::chrono::duration<double, std::milli>(Clock::now() - t0).count();
break;
}
+ generation = {};
+ const GatedDraftLimit draft_limit(mtp, gated_proposal, S_mtp, p + a + 1, o.max_context);
const bool drafted = !use_mtp || (int64_t) produced.size() >= max_new || draft_next(T, a);
if (adapt_thr.joinable()) adapt_thr.join();
if (!adapt_ok) return 1;
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt
index e20b1c7cc..3e070cc20 100644
--- a/sycl/CMakeLists.txt
+++ b/sycl/CMakeLists.txt
@@ -279,6 +279,16 @@ target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec)
option(STRATA_SYCL_PARITY "Build the kernel parity tests" ON)
if(STRATA_SYCL_PARITY)
enable_testing()
+ # Host-only proposal contracts of the gated MTP proposal; no SYCL runtime or GPU library needed.
+ add_executable(gated_proposal_test ${STRATA_ROOT}/src/core/gated_proposal_test.cpp)
+ target_include_directories(gated_proposal_test PRIVATE ${STRATA_ROOT}/include)
+ add_test(NAME gated_proposal_test COMMAND gated_proposal_test)
+ add_executable(spec_admission_test ${STRATA_ROOT}/src/core/spec_admission_test.cpp ${STRATA_ROOT}/src/spec/draft_policy.cpp)
+ target_include_directories(spec_admission_test PRIVATE ${STRATA_ROOT}/include)
+ add_test(NAME spec_admission_default COMMAND ${CMAKE_COMMAND} -E env --unset=STRATA_SPEC_PROB_GATE $<TARGET_FILE:spec_admission_test>)
+ add_test(NAME spec_admission_pick COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=pick $<TARGET_FILE:spec_admission_test> pick)
+ add_test(NAME spec_admission_top COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=top $<TARGET_FILE:spec_admission_test> top)
+ add_test(NAME spec_admission_exact_top_only COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=TOP $<TARGET_FILE:spec_admission_test> pick)
foreach(p IN ITEMS dequant_s2 s2_gemv shared_expert gr gdn s2_gemv_q8 sampler rope quantize_act router_top10
s_gemv elementwise bf16_gemv s_gemv_q8k qsa kv_q8 kv_stream kv_q4 cvec)
strata_resolve(_src src/kernels/${p}_parity.cpp)
diff --git a/sycl/include/strata/kernels/sampler.hpp b/sycl/include/strata/kernels/sampler.hpp
index ed6eae1eb..aa64c80ab 100644
--- a/sycl/include/strata/kernels/sampler.hpp
+++ b/sycl/include/strata/kernels/sampler.hpp
@@ -84,4 +84,9 @@ inline void penalty_rows(const int32_t* tail, int64_t n_tail, const int32_t* win
}
}
+// Gated proposal verification is not implemented by the SYCL backend.
+bool sample_tokens_proposal(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+ const SamplerParams& p, const int32_t* dtok, const int32_t* qbuf, int n_q, int* out,
+ void* stream);
+
} // namespace strata::kernels
diff --git a/sycl/src/kernels/cuda/sampler.dp.cpp b/sycl/src/kernels/cuda/sampler.dp.cpp
index be751a4fb..d60041048 100644
--- a/sycl/src/kernels/cuda/sampler.dp.cpp
+++ b/sycl/src/kernels/cuda/sampler.dp.cpp
@@ -1774,4 +1774,10 @@ void coupled_draft_sample(float* logits, int nv, const int32_t* sub_to_id, const
coupled_check("coupled_draft merge");
}
+// No ordinary-p fallback for an explicit gated measure.
+bool sample_tokens_proposal(const float*, int, int, const int*, int,
+ const SamplerParams&, const int32_t*, const int32_t*, int, int*, void*) {
+ return false;
+}
+
} // namespace strata::kernels