Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
51.2 KB
Raw Blame
// include/strata/core/verify.hpp - plan v0.3 P6: the speculative VERIFY window.
//
// T tokens at consecutive positions p0 .. p0+T-1 - the last accepted token and T-1 drafts - go through all 48
// layers in ONE captured graph, and the head's argmax is produced for every one of them.  Token t's argmax is
// what plain greedy decode would produce after token t, BIT FOR BIT: every kernel here is either the single-token
// kernel applied per token, or a multi-token kernel whose per-token arithmetic is the single-token kernel's
// (multi-column MMVQ in exact mode, the T-token GDN kernels, the per-token hit activation, the multi-token CPU
// expert rows).  So a draft is accepted exactly when greedy decode would have produced it.
//
// What the window costs is the dense weights read ONCE for T tokens and the union of the T tokens' missed
// experts on the CPU (measured on decode traces: 1.75x one token's misses for T=2, 2.4x for 3, 3.05x for 4).
//
// STATE.  The window appends K/V and indexer keys for all T positions and leaves the GDN state untouched.
// `commit(n_keep)` then makes the first `n_keep` tokens permanent: the GDN conv history and recurrent state are
// advanced by replaying those tokens from inputs the window stored, the indexer's key tail is restored from a
// snapshot and the accepted keys re-appended (a rejected key can land in a slot the current block still needs),
// and the PLE history is set to its snapshot after token n_keep-1.  K/V cells past the accepted prefix are simply
// overwritten when those positions are processed again, before any query can read them.
//
// Requires the default native decode configuration (native projections, fused GR, fused GDN, fast attention and
// selection, native indexer) and a profile-filled VRAM expert tier with its residency table on the device.
#pragma once

#include <cstdio>

#include "strata/core/expert_source.hpp"
#include "strata/core/layer.hpp"
#include "strata/core/session.hpp"
#include "strata/kernels/sampler.hpp"
#include "strata/kernels/window_limits.hpp"

#include <cuda_runtime.h>

#include <atomic>
#include <chrono>
#include <functional>
#include <map>
#include <cstdint>
#include <string>
#include <vector>

namespace strata::core {

/// 0.1.43 (STRATA_ADAPT_OVERLAP): the adaptive tier's copies still landing while a decode window runs.  The window's
/// residency tables are the baseline's (the swapped-in experts are resident from the window's first layer), but their
/// bytes arrive layer by layer: `run` holds a layer's plan (the first thing that lets the GPU read the layer's expert
/// slots) back until `landed >= layer` for a layer with a swap (`has[layer]`).  Set `active` only around one window.
struct AdaptGate {
    struct Tick { AdaptGate* gate; int32_t layer; };
    std::atomic<int32_t> landed{-1};      ///< the highest layer whose swap copies have landed (n_layers: all of them)
    std::atomic<bool> active{false};
    std::vector<uint8_t> has;             ///< per layer: a swap copies into this layer's slots
    std::vector<Tick> ticks;              ///< host-function contexts, one per layer and the final one (stable storage)
    const int32_t* dvals = nullptr;       ///< device [0..n_layers]: the source of those copies
    volatile int32_t* hland = nullptr;   ///< pinned host int the copy stream writes (a D2H copy of the layer number), instead of a host function; null: host functions
    int32_t reached() const {
        const int32_t a = landed.load(std::memory_order_acquire);
        if (hland == nullptr) return a;
        const int32_t b = *hland;
        return a > b ? a : b;
    }
};

class NativeHead;
class RemoteExpertOpt;

/// The CPU pool for a window: x_f (n_tok, n_embd), ids (n_tok, k) -> out (n_tok * k, n_embd), hit rows zeroed.
using PoolMultiFn = void (*)(void* user, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k, float* out,
                             int64_t layer);

struct VerifyHits {
    const int32_t* d_res = nullptr;      ///< device [n_layers * n_expert] slot or -1
    const int32_t* h_res = nullptr;      ///< host [n_layers * n_expert] slot or -1 (when all >= 0 in stage: zero-doorbell)
    const uint8_t* cache_base = nullptr; ///< slot 0 of the VRAM expert arena
    const uint64_t* slot_off = nullptr;   ///< E-6: host per-slot offsets when slots differ in size (null: slot * blob)
    int64_t n_slots = 0;                  ///< E-6: how many (for the device copy)
    int64_t blob = 0;
};

class Verifier {
public:
    Verifier() = default;
    ~Verifier();
    Verifier(const Verifier&) = delete;
    Verifier& operator=(const Verifier&) = delete;
    void set_remote_expert_opt(RemoteExpertOpt* opt) { remote_opt_ = opt; } // before init/capture

    /// The watchdog's view of the window in flight (issue #31): the layer, the GPU's sequence, the flags.
    void diag(std::FILE* f) const;
    /// #649 (STRATA_VERIFY_TRACE=1, opt-in): the last windows' host-side handshake events and the GPU's breadcrumbs
    /// (in mapped memory, so they read while the GPU hangs).  Nothing when the trace is off.
    void trace_dump(std::FILE* f) const;

    /// #267: raise every flag the window's spin kernels wait on past any ring (UINT32_MAX), so a window the GPU
    /// cannot finish drains instead of staying resident, then wait up to `timeout_ms` for its streams.  For the
    /// paths that give up on the engine (a timed-out window, the serve watchdog): the window then ran on whatever
    /// the flags guarded, so this verifier refuses every later window.  True when the streams finished.
    bool release_gpu_waits(int timeout_ms);

    /// `max_t` <= kVerifyMaxT.  `head` may be null (the canonical head is then run per token).
    bool init(const WeightTable& wt, const ModelGeometry& g, SessionState& ss, const VerifyHits& hits,
              const NativeHead* head, int max_t, std::string& err);

    /// One window: `tokens[0..T)` at positions pos0.., the pool served per layer; `out[t]` = argmax after token t.
    /// The PLE rows are gathered here from `ss.ple_prev` and the tokens.  Captures the T-token graph on first use.
    bool run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, void* user, int32_t* out, std::string& err);
    /// Diagnostics: row `t` of the last window's head logits (n_vocab floats) to the host. Valid after run().
    bool copy_logits(int t, float* host) const;
    int64_t vocab() const { return next_ ? next_->vocab() : n_vocab_; }
    /// #879 GUARD: the first row of the last finished window (run, run_slot_rows, batch_poll, pl_finish) whose head
    /// logits held a NaN or an infinity, else -1; nonfinite_pos() is that row's position.  Every window graph
    /// checks its T rows on the device beside the greedy pick (logits_nonfinite_rows into mapped flags, read after
    /// the sync the window already does: no extra sync).  On the stage with the head (a layer split asks the last
    /// stage).  STRATA_NAN_GUARD=0: off (always -1).  STRATA_NAN_INJECT=<n>[,<n>...] (debug): the n-th window the
    /// guard checks in this process (counted from 1 over every stage-with-head verifier: prompt-tail windows, decode
    /// windows and batch windows alike) reports row 0 as non-finite, so the whole response can be tested.
    int nonfinite_row() const { return next_ != nullptr && !last_stage() ? next_->nonfinite_row() : nf_row_; }
    int64_t nonfinite_pos() const { return next_ != nullptr && !last_stage() ? next_->nonfinite_pos() : nf_pos_; }
    /// Row t of the last window (a batch window: each slot's own rows) held a non-finite logit.
    bool nonfinite_in(int t) const {
        if (next_ != nullptr && !last_stage()) return next_->nonfinite_in(t);
        return t >= 0 && t < 32 && ((nf_mask_ >> t) & 1u) != 0;
    }
    static bool nan_guard_on();
    /// The most tokens one window holds (`init`'s `max_t`; 0 before it).
    int max_t() const { return max_t_; }
    /// The sampling the verify window's head applies (temperature / top_p / top_k / seed).  Set per
    /// request; greedy by default.  The sampling itself runs OUTSIDE the captured graph - its
    /// parameters would otherwise be baked forever - so this can change between requests freely.
    void set_sampling(const strata::kernels::SamplerParams& sp) {
        sampling_ = sp;   // row t of a window at pos0 draws Philox(seed, pos0 + t): see run()
        sampling_.logit_bias = logit_bias_host_.empty() ? nullptr : logit_bias_device_;
        if (next_) next_->set_sampling(sp);
    }
    // One request's fixed vocabulary bias. Empty clears it; copied only when changed, on the final stage's device.
    bool set_logit_bias(const std::vector<float>& bias, std::string& err);

    /// The penalty histories for `sampling_.penalty_last_n`: ONE ROW PER WINDOW ROW, T rows of `history_len`
    /// int32 slots at that stride (`strata::kernels::penalty_rows` builds them), most recent token LAST, unused
    /// front slots -1 (the kernel reads only the tail window).  Row t follows the window's drafts 1..t - staging
    /// row 0 alone (before 0.1.19) left the drafted rows with unwritten histories.  Null disables the penalties
    /// entirely - the neutral run's sampling call is byte-for-byte what it was.  The engine re-uploads the rows
    /// before every window; the buffer must hold kVerifyMaxT rows and stay alive across the request.
    void set_history(const int32_t* history, int history_len) {
        hist_d_ = history;
        hist_len_ = history_len;
        if (next_) next_->set_history(history, history_len);
    }
    /// PROBABILISTIC DRAFT ACCEPTANCE (core/spec_prob.hpp, STRATA_SPEC_PROB=1): for the NEXT run() only, judge the
    /// window's first `n_q` drafts against the drafter's distributions `q` (host memory, n_q rows of kSpecQStride
    /// int32: ids, -1 terminated, then probabilities as float bits) with rejection sampling instead of exact match; the
    /// drafts past `n_q` (a lookup chain's tail) are point masses.  Has no effect on a greedy request, a window of one
    /// token, or where the split sampler cannot run - those take the exact-match path.  Cleared by run().
    void set_spec_q(const int32_t* q, int n_q) {
        spec_q_ = q;
        spec_nq_ = n_q;
        spec_measure_ = false;
        if (next_) next_->set_spec_q(q, n_q);
    }
    /// Borrow immutable gated measure rows for the next run only. nq==T denotes
    /// a NULL tail, nq<T a plain bonus. Errors never fall back to ordinary p.
    /// nq == 0 accepts a null pointer: point-only rows and a plain bonus (T1: plain p).
    /// Keep the host rows alive and unchanged until run returns.
    void set_spec_proposal(const int32_t* q, int n_q) {
        spec_q_ = q;
        spec_nq_ = n_q;
        spec_measure_ = true;
        if (next_) next_->set_spec_proposal(q, n_q);
    }

    /// Counters of the rejection path (windows judged, drafts kept / offered over its rows with a q list).
    int64_t spec_windows = 0;

    /// Off: `run` skips the request's head sampling and `out` is the recorded greedy pick.  For windows whose
    /// picks are discarded - a prompt read through windows commits every token - so they cost no sampler launch
    /// or sync and never read a history staged for another position.
    void set_head_sampling(bool on) { head_sampling_ = on; if (next_) next_->set_head_sampling(on); }

    /// LAYER SPLIT (multi-GPU): this verifier runs layers [layer_begin, layer_end) of every window.  A stage that
    /// does not start at layer 0 takes its residual from `handoff_in` instead of embedding the tokens; a stage that
    /// does not end at the last layer writes its residual to `handoff_out` and has no head.  The hand-off holds,
    /// per token, the residual R (hc x n_embd), the last layer's pending write bo (n_embd) and inject (hc): the next
    /// stage folds that write into its first read exactly as the unsplit window does, so the split is bit-exact.
    /// Both pointers must be device-visible (mapped pinned memory, portable when the stages are on two devices).
    /// Set before `init`.  Default: the whole model, no hand-off.
    void set_stage(int64_t layer_begin, int64_t layer_end, const float* handoff_in, float* handoff_out) {
        lb_ = layer_begin; le_ = layer_end; hand_in_ = handoff_in; hand_out_ = handoff_out;
    }
    /// The rows the hand-off buffers of `set_stage` hold (0: kVerifyMaxT, the default).  More than --batch's 8 slots on a
    /// layer split: the pipelined slot groups write their rows at hbase = group * rows-per-group, one row per slot.
    void set_handoff_rows(int rows) { handoff_rows_ = rows; }
    /// Is the zero-doorbell graph the one a batch window would run now (every expert of the stage in VRAM by the host
    /// table)?  Looks at the table, as each window does.  (STRATA_ADMIT_MERGE_FIRST's gate.)
    bool ar_live() { refresh_ar(); return ar_on(); }
    /// STRATA_SPLIT_OVERLAP=1 (opt-in; stages on DIFFERENT devices only): the hand-off carries a ready flag (one
    /// mapped word per hand-off buffer: host and device addresses).  The writing stage raises it from the GPU when
    /// its hand-off is out (handoff_publish); the reading stage's window graph waits on it (wait_flag_ge) - so the
    /// next stage's graph is launched while this stage is still running, and the host no longer syncs between the
    /// stages.  Null (the default): the old order - sync, then launch the next stage.  Set before `init`.
    void set_handoff_flags(uint32_t* in_host, uint32_t* in_dev, uint32_t* out_host, uint32_t* out_dev) {
        hflag_in_h_ = in_host; hflag_in_d_ = in_dev; hflag_out_h_ = out_host; hflag_out_d_ = out_dev;
    }
    /// The next stage: `run` and `commit` continue into it (its pool calls get `next_user`); sampling settings
    /// and `final_R` are the last stage's.
    void set_next(Verifier* next, void* next_user) { next_ = next; next_user_ = next_user; }
    /// floats per token in a hand-off buffer
    static int64_t handoff_floats(const ModelGeometry& g) { return (int64_t) g.hc * g.n_embd + g.n_embd + g.hc; }

    /// Keep the first `n_keep` (1..T) tokens of the last window; advances `ss.ple_prev` by them.
    bool commit(int n_keep, std::string& err);

    // ================================ SEVERAL SEQUENCES IN ONE WINDOW ================================
    //
    // A batch window holds S INDEPENDENT sequences, one token each: row s is slot s, at slot s's own position,
    // reading and writing slot s's own state (GDN recurrence and conv history, QSA K/V and indexer, PLE history),
    // which lives in `slots[s]` - a session carved like this verifier's own (same layer range, same max_cells).
    // Everything that is per row already (hyper-connections, dense projections, router, shared expert, the
    // routed experts on the GPU and the CPU, the head) runs once over the S rows, so the weights are read once
    // per window for all the sequences.  Row s's arithmetic is the single-token window's, so a slot's greedy
    // tokens are its solo greedy tokens (modulo the multi-token CPU kernel choice: STRATA_IQ_MT_MIN=1).
    //
    // Greedy only, no drafts (MTP) in a batch window.  `init_slots` once after `init` (S <= max_t); a layer
    // split's stages each get their own sessions, and run_slots/commit_slots continue into the next stage.
    bool init_slots(const std::vector<SessionState*>& slots, std::string& err);
    int n_slots() const { return (int) slots_.size(); }
    /// One batch window over slots [0, S): tokens[s] at positions pos[s]; out[s] = the greedy pick after it.
    bool run_slots(int S, const int32_t* tokens, const int64_t* pos, PoolMultiFn pool, void* user, int32_t* out,
                   std::string& err);
    /// The same over the S slots `rows` (row t is slot rows[t], any distinct slots in any order): the slots not
    /// listed are not touched, so an idle slot keeps its state (a finished conversation it may continue later).
    bool run_slot_rows(const int* rows, int S, const int32_t* tokens, const int64_t* pos, PoolMultiFn pool, void* user,
                       int32_t* out, std::string& err);
    /// Keep every row of the last batch window.
    bool commit_slots(std::string& err);
    /// Commit an accepted prefix in each contiguous slot group of the last window. `keep` has
    /// one entry per slot (indexed by slot ID), each in 1..that slot's group length.
    bool commit_slot_prefixes(const int* keep, std::string& err);
    /// --batch-mtp: slot rotation makes many row layouts, so bound the captured batch graph pairs (LRU). 0 = unbounded.
    void set_batch_graph_limit(size_t n) { batch_graph_limit_ = n; }

    // ---- The stages of a layer split as a PIPELINE.  A batch window over the slot GROUP
    // [base, base + S) is launched on ONE stage with its commit right behind it on the stage's stream (a batch window
    // keeps every row, so the commit needs no host decision), and the host serves the rings of every stage that has
    // a window in flight from one thread (batch_poll does not block).  Stage k can then run group g while stage k+1
    // runs group g-1.  Rows of group `base` use hand-off rows [base, base + S), so groups never share a hand-off row.
    bool batch_launch(int base, int S, const int32_t* tokens, const int64_t* pos, std::string& err);
    /// 1 = this stage's window and commit are done (the last stage's picks are in batch_out), 0 = still running,
    /// -1 = an error (err).  Serves every layer that has rung so far.
    int batch_poll(PoolMultiFn pool, void* user, std::string& err);
    bool batch_busy() const { return b_running_.load(std::memory_order_acquire); }
    // ---- STRATA_PIPED_MTP: the pipelined groups with --batch-mtp's drafts.  A group's window holds its slot's token
    // and its drafts, so how many rows to keep is known only once the LAST stage has picked - and by then this stage
    // runs another group's window, which reuses the scratch the commit replays from.  So right behind the window a
    // save graph copies those rows (each GDN layer's conv and recurrence inputs, each QSA layer's raw indexer keys, the
    // PLE history snapshot) to the group's own rows [hbase, hbase + S) of the saved buffers, and the commit, queued
    // after the verdict, replays from there.  Everything else the commit reads is per slot already.
    /// The saved buffers for `rows` hand-off rows (after init_slots).
    bool init_deferred_commit(int rows, std::string& err);
    bool deferred_commit_on() const { return dc_rows_ > 0; }
    /// The window over the slots `rows` (a slot's rows contiguous) at hand-off rows [hbase, hbase + S), its save right
    /// behind it, and no commit (batch_commit_rows, after the verdict).  The PLE history advances at the commit.
    bool batch_launch_rows(const int* rows, int S, int hbase, const int32_t* tokens, const int64_t* pos,
                           std::string& err);
    /// Queue the commit of that window's accepted prefixes on this stage's stream (no host wait): `keep` per slot ID,
    /// 1..the slot's rows.  `rows`, `S`, `hbase`, `tokens` and `pos` are the window's, as batch_launch_rows got them.
    bool batch_commit_rows(const int* rows, int S, int hbase, const int* keep, const int32_t* tokens,
                           const int64_t* pos, std::string& err);
    /// Serves the layers of this stage's window that have rung (the CPU pool and the flags), with no CUDA call: 1 once
    /// every layer is served (or the window raises no host doorbells), 0 while one is still to ring.  batch_poll
    /// calls it, unless set_external_serve(true) moved it to a thread of its own (STRATA_PIPED_CONCURRENT: a CUDA
    /// call can wait for the driver's lock while another thread holds it waiting for the GPU, whose window waits
    /// for its doorbell, #31); batch_poll then makes no CUDA call before every layer is served.
    int batch_serve(PoolMultiFn pool, void* user);
    void set_external_serve(bool on) { ext_serve_.store(on, std::memory_order_release); }
    /// A slot's sampling (temperature / top_p / top_k / min_p / seed; penalties are not applied in batch windows):
    /// its row is drawn again on the last stage with Philox(seed, position), as a solo window draws it.  Greedy by
    /// default.  Set on the first stage, it reaches the last.
    void set_slot_sampling(int slot, const strata::kernels::SamplerParams& sp) {
        if (slot >= 0 && slot < (int) slot_sp_.size()) slot_sp_[(size_t) slot] = sp;
        if (next_) next_->set_slot_sampling(slot, sp);
    }
    const int32_t* batch_out() const { return b_out_; }
    bool last_stage() const { return g_ != nullptr && le_ == g_->n_layers; }
    /// commit() returns without waiting for its graph (a single-GPU session sets it): the next window follows it on
    /// the same stream and the drafter reads nothing it writes, so it overlaps the draft. Whoever reads or writes
    /// the session from another stream or the host afterwards (a new request, a checkpoint, a snapshot, the prompt
    /// path, the end of a run) calls wait_commit() first.  STRATA_COMMIT_SYNC=1 keeps the wait.
    static void set_commit_async(bool on);
    /// Waits for the last commit graph when commit() did not (an event recorded after it, not the whole device);
    /// false with `err` when it failed.  Free when nothing is pending.
    bool wait_commit(std::string& err);

    // ---- PIPELINED WINDOWS (--pipeline-windows, a layer split on two GPUs).  One conversation's windows with the
    // stages overlapped: stage 0 runs window K+1 while stage 1 still runs window K.  The same window as `run`, driven
    // without blocking the host, so one host thread keeps a window in flight on each stage (the batch pipeline's
    // pattern, batch_launch / batch_poll, for one sequence with drafts).  Two verifiers per stage (one per window
    // parity) share the stage's stream and each has its own hand-off.  `pl_launch` stages and launches (it never
    // captures: `capture_all` first, with nothing in flight), `service` serves the layers whose doorbells have rung
    // and returns at once, `done` polls the window's completion, `pl_finish` reads the picks, `pl_commit_async`
    // queues the commit.  Nothing chains to `next_`: the caller drives every stage.
    /// The compute stream to use instead of a private one (the two verifiers of one stage share it).  Before `init`.
    void set_stream(cudaStream_t s) { ext_stream_ = s; }
    /// Every layer copies the token rows to the host (doorbell_publish), not only the layers with a routed expert
    /// outside the VRAM tier by the device's residency table: with windows in flight the adaptive tier marks an
    /// expert evicted on the host (the pool then computes it on the CPU) before the device table follows.  Also turns
    /// the device-planned layers (E-6) off, which read the device table this mode lets lag.  Before the first window
    /// (its graphs are captured there).
    void set_always_publish(bool on) {
        always_publish_ = on;
        if (on) device_plan_ = false;
    }
    /// 0.1.43: the adaptive tier's overlap gate (see AdaptGate).  Refused (false) with the device-planned layers (E-6),
    /// which never ask the host.  The pointer must outlive the verifier's windows.
    bool set_adapt_gate(AdaptGate* g) {
        if (g != nullptr && device_plan_) return false;
        adapt_gate_ = g;
        return true;
    }
    double ms_gate = 0;   ///< host time spent holding a layer's plan for the adaptive copies (decode timing prints it)
    cudaStream_t stream() const { return cs_; }
    int device() const { return device_; }
    /// Capture every window size and the commit graph now (a capture syncs the stream: never with a window in flight).
    bool capture_all(std::string& err);
    bool pl_launch(int T, const int32_t* tokens, int64_t pos0, std::string& err);
    /// Stage a window ahead of its launch (positions, and the PLE rows from `ple_prev` = the two tokens before the
    /// window as they WILL be, their pages prefetched).  A later `pl_launch` of the same window (T, pos0, tokens, and
    /// `ss.ple_prev` equal to `ple_prev` by then) skips the staging; anything else stages again.
    bool prestage(int T, const int32_t* tokens, int64_t pos0, const int32_t ple_prev[2], std::string& err);
    /// 1: every layer served; 0: the GPU has not reached the next layer yet; -1: an error (`err`).
    int service(PoolMultiFn pool, void* user, std::string& err);
    bool in_flight() const { return fl_active_; }
    /// The window's graph (and its profile copy) completed; false while it runs.  An error sets `err`.
    bool done(std::string& err);
    /// After `done`: the profile, the last stage's host sampling and picks (`out` may be null on an earlier stage).
    bool pl_finish(int32_t* out, std::string& err);
    /// The commit without a host sync; `ss.ple_prev` advances now (host side).  A second call for the same window
    /// (after its state was restored) replays it with another count.
    bool pl_commit_async(int n_keep, std::string& err);
    /// Fold another verifier's counters and GPU profile into this one's (the two verifiers of one stage report once).
    void absorb_stats(Verifier& o);
    /// The watchdog's line for a pipelined verifier: in flight, layers served, the GPU's ring and flags, its events.
    void diag_pipelined(std::FILE* f, const char* name) const;

    /// Measurement hook (STRATA_LOGPOS): after run(), write one line per row t of the last window's head -
    /// "pos target logprob top top_logprob hit extra_logprob target_logprob_without_extra" - where row t is the
    /// distribution at pos0 + t, targets[t] is the token at pos0 + t + 1, extra_logprob is the log-probability of
    /// `extra_id` in the same row, and the last column is the target's log-probability in that row renormalized
    /// over every token but `extra_id` (both nan when they do not apply).  A layer split's earlier stage forwards
    /// to the stage that holds the head.
    bool window_logprobs(const int32_t* targets, int T, int64_t pos0, int32_t extra_id, std::FILE* out,
                         std::string& err);

    /// Token t's residual after the last layer, (hc, n_embd) on the device, valid until the next `run`.
    const float* final_R(int t) const;
    const float* final_R_all() const { return next_ ? next_->final_R_all() : R_; }
    /// DFlash2 (before init): keep the completed outputs of these layers (zero-based; the mean of the hc streams) for every row of
    /// each window in a device buffer, tap_dev() = [max_t][n_taps][n_embd] floats, valid until the next window.  Single-stage
    /// verifiers only; STRATA_TAP_FILE (research dump) turns the same buffer on without this call.
    void set_taps(const std::vector<int>& layers) { tap_req_ = layers; }
    const float* tap_dev() const { return tap_dev_; }
    bool tapping() const { return tap_on_; }

    /// The GPU plan the pool writes each layer (VRAM hits + the PCIe share of the misses); give it to the
    /// dispatch (`ExpertDispatch::plan`) before the first `run`.
    GpuPlanSink* plan_sink() { return &sink_; }
    /// Plan v0.3 P6: split the window into two token groups and pipeline the CPU experts of one with the GPU work
    /// of the other (default on).  Set before the first `run`.
    void set_split(bool on) { split_ = on; }
    /// Plan v0.3 P6: how the PCIe share of the misses reaches the GPU: 0 = DMA into staging (the copy engine works
    /// beside the CPU; best when the CPU is compute-bound, the i-quants), 1 = the grouped kernel reads the mapped
    /// arena directly, 2 = a copy kernel stages it inside the graph (no API calls on the pool's thread; best when
    /// the CPU is RAM-bound, Q2_0).  Set before the first `run`.
    void set_pcie_mode(int mode) { sink_.pcie_mode = mode; }
    /// Called before every window this verifier launches (run, the batch and pipelined launches): the device's
    /// residency table catches up with the host's (the adaptive tier's evictions while its copies are in flight).
    /// It gets this verifier's stream; false (with the message in `err`) fails the launch.
    void set_pre_launch(std::function<bool(cudaStream_t cs, std::string& err)> f) { pre_launch_ = std::move(f); }
    /// the pool never plans a PCIe share (--pcie-frac 0): the window skips that path.  Before the first run.

    /// The PCIe count of each layer from measured costs (PcieModel) instead of a fixed share: the spin kernels stamp
    /// each step's GPU work (the plan's arrival, the PCIe part's start, the wait for the CPU's rows; GPU clock), the host
    /// times its pool, and after each window *model (the dispatch's) takes the costs. One GPU, no batch slots. Set
    /// before `init` (the stamps are captured). `paused`: a request's own --pcie-frac holds meanwhile.
    void set_pcie_balance(PcieModel* model) { bal_ = model; }
    void pause_pcie_balance(bool paused) {
        bal_paused_ = paused;
        if (bal_ != nullptr && m_bal_ != nullptr) bal_->on = !paused;
    }
    bool pcie_balance_on() const { return bal_ != nullptr && m_bal_ != nullptr; }

    double ms_wait = 0, ms_pool = 0, ms_host = 0, ms_commit = 0;
    int64_t windows = 0;
    /// STRATA_VERIFY_PROFILE=1 - GPU stage times of the windows since the last call (ms per
    /// window), as one line; empty when off.
    std::string profile_report();
    /// STRATA_VERIFY_PROFILE: the same stage split, one line per window size T (bench: with STRATA_SPEC_T_CYCLE). Resets.
    std::string profile_by_t_report();
    /// STRATA_ROUTE_STATS=1: per window size, the routed experts per layer: distinct, and new per row (not in the rows before it). Resets.
    std::string route_stats_report();

private:
    RemoteExpertOpt* remote_opt_ = nullptr;
    bool capture(int T, std::string& err);
    // batch windows (see init_slots)
    std::vector<SessionState*> slots_;
    bool batch_rec_ = false;               ///< record_window is capturing a batch window
    int row_base_ = 0;                     ///< ... its hand-off rows start here (a pipeline group's own rows)
    int brow_[strata::kernels::kWindowMaxT] = {};                     ///< ... and row t is slot brow_[t]
    bool last_batch_ = false;              ///< the last run was a batch window (set_plan_slot: one group)
    std::map<std::vector<int>, cudaGraphExec_t> exec_bm_, commit_bm_;   ///< full row layout avoids slot-ID collisions
    std::map<std::vector<int>, uint64_t> bm_used_;   ///< last use of each captured layout (LRU)
    uint64_t bm_tick_ = 0;
    size_t batch_graph_limit_ = 0;         ///< 0: keep every captured batch graph (0.1.39); N: LRU-evict beyond N layouts
    int last_rows_[strata::kernels::kWindowMaxT] = {};                ///< the slots of the last batch window's rows
    std::vector<int> bkey(const int* rows, int S, int hbase) const {   // #871: the doorbell variant has its own graphs
        std::vector<int> k = batch_key(rows, S, hbase);
        k.push_back(ar_off_ ? 1 : 0);
        return k;
    }
    static std::vector<int> batch_key(const int* rows, int S, int hbase) {
        std::vector<int> k;
        k.reserve((size_t) S + 1);
        k.push_back(hbase);
        for (int t = 0; t < S; ++t) k.push_back(rows[t]);
        return k;
    }
    // batch_launch / batch_poll (with external serving, batch_serve runs on another thread: running and the served
    // layer count are atomics, set by batch_launch before running)
    std::atomic<bool> b_running_{false};
    std::atomic<int64_t> b_k_{0};
    int64_t b_steps_ = 0;
    std::atomic<bool> ext_serve_{false};
    // when a layer was last served (steady clock ticks): the serving thread writes it, batch_poll's checks read it
    std::atomic<int64_t> b_last_{0};
    void ring_now() {
        b_last_.store((int64_t) std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release);
    }
    std::chrono::steady_clock::time_point last_ring() const {
        return std::chrono::steady_clock::time_point(
            std::chrono::steady_clock::duration(b_last_.load(std::memory_order_acquire)));
    }
    std::chrono::steady_clock::time_point ext_checked_{};   ///< batch_poll's last check with external serving
    int32_t b_out_[strata::kernels::kWindowMaxT] = {};
    std::vector<strata::kernels::SamplerParams> slot_sp_;
    bool sample_rows(int S, std::string& err);   ///< the sampled slots' rows of the last batch window
    int32_t* h_commitb_ = nullptr; int32_t* m_commitb_ = nullptr;   // per slot [1, 0, pos, -1 ..], stride 2 + max_t
    int32_t* commitb_ = nullptr;
    float* tail_snap_b_ = nullptr;         ///< per (slot, QSA layer) indexer tail snapshot
    void* arena_b_ = nullptr;
    int64_t last_pos_b_[strata::kernels::kWindowMaxT] = {};
    bool capture_batch(const int* rows, int S, int hbase, std::string& err);
    // STRATA_PIPED_MTP (init_deferred_commit): the saved rows, laid out as the window scratch (global layer index) with
    // dc_rows_ rows per layer instead of max_t_; the slots' accepted prefixes (mapped and device, 2 + max_t_ per slot)
    int dc_rows_ = 0;
    bool staging_deferred_ = false;   ///< stage_batch for batch_launch_rows: no capture_commit_batch
    float *dc_qkv_ = nullptr, *dc_h_ = nullptr, *dc_gate_ = nullptr, *dc_beta_ = nullptr, *dc_idx_ = nullptr;
    float* dc_hist_ = nullptr;
    void* dc_arena_ = nullptr;
    int32_t *h_commitd_ = nullptr, *m_commitd_ = nullptr, *commitd_ = nullptr;
    std::map<std::vector<int>, cudaGraphExec_t> save_bm_, dcommit_bm_;
    bool capture_deferred(const int* rows, int S, int hbase, std::string& err);
    /// Free the graph pair of the least recently used batch layout other than `keep` (`evicted`: there was one).
    bool evict_batch_graph(const std::vector<int>& keep, bool& evicted, std::string& err);
    /// cudaGraphInstantiate; out of VRAM, batch layouts other than `key` are evicted (LRU) until it fits, and then (a
    /// single-request window, `keep_t` > 0) the graphs of the other window sizes, largest first (#3702): a window size
    /// whose graph does not fit used to end the engine ("verify: instantiate: out of memory"); a freed size is captured
    /// again when it is next used.
    bool instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key, const char* what,
                              std::string& err, int keep_t = -1, unsigned long long flags = 0);
    /// Free the graph of the largest window size other than `keep_t` (and the prelaunched one); `evicted`: there was one.
    bool evict_window_graph(int keep_t, bool& evicted, std::string& err);
    bool no_window_eviction_ = false;   ///< capture_all wants every size at once: it must not free one to capture another
    // STRATA_GRAPH_PRIO (record_window / capture): the captured tails of the priority streams at their joins and the
    // main stream's tails at their forks, so that a capture that dropped the stream priorities can be repaired per
    // node (apply_node_priorities walks each tail back to its fork)
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)   // (the three methods are no-ops there)
    std::vector<cudaGraphNode_t> prio_forks_;
    std::vector<std::pair<cudaGraphNode_t, int>> prio_tails_;
#endif
    void note_prio_fork(cudaStream_t from);
    void note_prio_tail(cudaStream_t on, int prio);
    void apply_node_priorities(cudaGraph_t graph, int T);
    void collect_profile();   ///< STRATA_VERIFY_PROFILE: add the last window's stamps to prof_sum_
    void accumulate_profile(const unsigned long long* stamps);
    void accumulate_profile_inner(const unsigned long long* stamps);   ///< one window's stamps (host copy) into prof_sum_
    // pipelined windows (pl_launch ...)
    cudaStream_t ext_stream_ = nullptr;   ///< set_stream: the stage's shared stream (not destroyed here)
    bool always_publish_ = false;
    bool adapt_gate_wait(int32_t layer, std::string& err);
    AdaptGate* adapt_gate_ = nullptr;     ///< 0.1.43: STRATA_ADAPT_OVERLAP's gate (null: off)
    void pl_stage(int T, const int32_t* tokens, int64_t pos0, const int32_t ple_prev[2]);
    cudaEvent_t ev_done_ = nullptr, ev_commit_ = nullptr;
    unsigned long long* prof_pin_ = nullptr;   ///< pinned host copy of the stamps (pipelined windows)
    bool fl_active_ = false, fl_prof_ = false, fl_ple_ = false, commit_live_ = false, pl_prestaged_ = false;
    int fl_T_ = 0;
    int64_t fl_k_ = 0, fl_total_ = 0;
    double fl_since_ms_ = 0, fl_flush_ms_ = 0, fl_launch_ms_ = 0;
    int32_t pl_prev_[2] = {-1, -1};
    std::vector<uint32_t> pl_ple_rows_;   ///< the window's PLE rows (T x PLE_N_HEADS), gathered when layer 0 is served
    bool capture_commit_batch(const int* rows, int S, int hbase, std::string& err);
    bool stage_batch(const int* rows, int S, int hbase, const int32_t* tokens, const int64_t* pos, std::string& err);
    strata::kernels::SamplerParams sampling_ = [] {
        strata::kernels::SamplerParams s;
        s.greedy = true;
        s.temperature = 0.0f;
        return s;
    }();   ///< greedy by default; per-request via set_sampling
    const int32_t* hist_d_ = nullptr;   ///< penalty-history row (set_history); null = no penalties apply
    const int32_t* spec_q_ = nullptr;   ///< set_spec_q: the drafter's q rows for the next window (host)
    int spec_nq_ = 0;
    bool spec_measure_ = false;   ///< explicit per-call measure borrow
    int32_t* d_spec_ = nullptr;          ///< device: kVerifyMaxT draft ids, then kVerifyMaxT q rows
    int hist_len_ = 0;
    bool head_sampling_ = true;          ///< set_head_sampling
#if defined(STRATA_GPU_METAL)
    void* metal_ = nullptr;              ///< the Metal decode window (metal/src/engine/window.cpp): init/run/commit go there
#endif
    std::vector<float> logit_bias_host_;
    float* logit_bias_device_ = nullptr;
    int device_ = -1;                    ///< the device `init` ran on: run/commit switch to it (layer split)
    unsigned long long* rr_stats_ = nullptr;   ///< STRATA_ROUTE_RESIDENT's counters on device_ (#1578); null: none
    std::atomic<bool> released_{false};  ///< #267: release_gpu_waits ran (maybe on the watchdog thread): no more windows
    bool all_resident_ = false;           ///< 100% of experts in [lb_, le_) resident in VRAM: zero-doorbell graph
    /// #871: the zero-doorbell graph plans from the device residency table alone, so it is only right while every
    /// expert of the stage is in VRAM.  A prompt loan, a VRAM shrink or an adaptive swap marks some -1 for a while:
    /// every window then runs the doorbell graph (exec_nr_ / the batch key's top bit), the same as a stage that never
    /// was all-resident.  refresh_ar() looks at the host table before each window.
    bool ar_off_ = false;
    bool ar_on() const { return all_resident_ && !ar_off_; }
    void refresh_ar();
    uint32_t* h_plan_err_ = nullptr; uint32_t* m_plan_err_ = nullptr;   // set by resident_plan on a -1 (all-resident graph)
    const int32_t* h_res_ = nullptr;      ///< the host residency table (VerifyHits::h_res)
    bool device_plan_ = false;            ///< E-6: resident-only layers planned on the device (STRATA_VERIFY_DEVICE_PLAN)
    uint32_t* skip_ = nullptr;            ///< E-6: per group, the ring whose plan the device built (0: the host's)
    unsigned* qcnt_ = nullptr;             ///< S26 STRATA_QFUSE: the HC read's q8_1 group counters (n_embd / 32)
    unsigned long long* slot_off_d_ = nullptr;   ///< E-6: the slot offsets on the device
    int64_t lb_ = 0, le_ = -1;           ///< set_stage: the layers this verifier runs (-1: to the last)
    int handoff_rows_ = 0;               ///< set_handoff_rows (0: kVerifyNarrowT)
    const float* hand_in_ = nullptr;
    float* hand_out_ = nullptr;
    uint32_t* hflag_in_h_ = nullptr;     ///< set_handoff_flags: the incoming hand-off's ready word (host / device)
    uint32_t* hflag_in_d_ = nullptr;
    uint32_t* hflag_out_h_ = nullptr;    ///< ... and the outgoing one's
    uint32_t* hflag_out_d_ = nullptr;
    uint32_t* hcount_d_ = nullptr;       ///< handoff_publish's block counter (device memory)
    uint32_t* herr_h_ = nullptr;         ///< the bounded hand-off wait gave up (mapped; the reading stage's)
    uint32_t* herr_d_ = nullptr;
    uint32_t* hdrop_h_ = nullptr;        ///< STRATA_TEST_HANDOFF_DROP: the writing stage withholds its flag (mapped)
    uint32_t* hdrop_d_ = nullptr;
    bool handoff_failed(std::string& err) const;   ///< the bounded hand-off wait timed out in the last window
    int prelaunched_ = 0;                ///< prelaunch ran for a window of this size: run skips staging + launch
    /// The overlapped split: stage this window's inputs and launch its graph now (the graph waits on the hand-off
    /// flag); `run` then only serves the host side.  Called on a helper thread while the previous stage runs.
    bool prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err);
    bool chain_overlap() const;          ///< this stage hands off through a flag to a next stage that waits on it
    Verifier* next_ = nullptr;
    void* next_user_ = nullptr;
    bool ple_stage() const { return lb_ <= 1 && 1 < le_; }   ///< holds layer 1, where the PLE block runs
    void stage_inputs(int T, const int32_t* tokens, int64_t pos0);
    bool staged_ = false;
    bool copy_used_ = false;
    bool capture_commit(std::string& err);
    bool record_window(int T, cudaStream_t cs, std::string& err);
    // #649: STRATA_VERIFY_TRACE=1 - a host event ring (trace_ev) and GPU breadcrumbs: the profiler's stamp points,
    // per layer and token group, written to mapped memory (null when the trace is off)
    void trace_ev(const char* what, int64_t step, int64_t layer, int64_t aux) const;
    unsigned long long* trace_h_ = nullptr;
    unsigned long long* trace_m_ = nullptr;
    size_t trace_n_ = 0;
    static constexpr int kProfPer = 33;              // stamps per layer (32 left the hc-read second
                                      // half's up-stamp at slot 32 = the next layer's slot 0: D8)
    bool prof_on_ = false;
    unsigned long long* prof_ = nullptr;              // device: n_layers * kProfPer + 4 stamps
    std::vector<unsigned long long> prof_h_;
    double prof_sum_[2][kProfPer] = {};   // [GDN / QSA layers][stage]
    int64_t prof_windows_ = 0;
    double prof_t_sum_[9][2][kProfPer] = {};   // the same per window size T (1..8: wider windows are not profiled), for profile_by_t_report
    int64_t prof_t_win_[9] = {};
    bool rs_on_ = false;                      // STRATA_ROUTE_STATS
    unsigned long long* rs_dev_ = nullptr;    // [9 T][n_layers][8 rows]: experts of row r that no earlier row of the window routed to
    int64_t rs_win_[9] = {};

    const WeightTable* wt_ = nullptr;
    const ModelGeometry* g_ = nullptr;
    SessionState* ss_ = nullptr;
    VerifyHits hits_;
    const NativeHead* head_ = nullptr;
    int max_t_ = 0;
    float* ple_key_ = nullptr;   ///< STRATA_PLE_BATCH: the window rows' PLE key / value projections
    float* ple_val_ = nullptr;
    int last_t_ = 0;
    int64_t last_pos0_ = 0;
    int32_t last_tokens_[strata::kernels::kWindowMaxT] = {};
    int64_t n_vocab_ = 0;
    cudaStream_t cs_ = nullptr;
    cudaStream_t sh_cs_ = nullptr;
    cudaEvent_t ev_fork_ = nullptr, ev_join_ = nullptr;
    // STRATA_DF_BRANCH: a layer's mixer work that reads only the layer's input, captured as parallel graph branches
    // on these side streams (record_window); the same kernels on the same inputs, only their order is freer
    bool df_branch_ = false;
    cudaStream_t df_side_[2] = {};
    cudaEvent_t df_fork_ = nullptr;
    cudaEvent_t df_join_[2] = {};
    // [sel_long_][T]: 1 is the long-context QSA select (STRATA_SELECT_LONG=1: multi-CTA top-k, shuffle-free decode
    // scores; see run()); without it only [0] is ever captured
    // (per window size T = 1..kWindowMaxT: a solo window is at most 8 rows, but a merge window of a wide batch is not)
    cudaGraphExec_t exec_[2][strata::kernels::kWindowMaxT + 1] = {};
    cudaGraphExec_t exec_nr_[2][strata::kernels::kWindowMaxT + 1] = {};   // #871: the doorbell variant of a stage that is all-resident otherwise
    bool launched_[2][strata::kernels::kWindowMaxT + 1] = {}, launched_nr_[2][strata::kernels::kWindowMaxT + 1] = {};   ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)
    int sel_long_ = 0;
    // fork 34a35db (STRATA_SIDE_MIXER=1): the GDN gates and the QSA indexer on a branch of the window's graph beside the
    // mixer's projections, joined before the recurrence / the attention
    cudaStream_t side_ = nullptr;
    cudaEvent_t mfork_ = nullptr, mjoin_ = nullptr;
    // fork b063c13 (STRATA_SIDE_MIXER=2): a second branch for the GDN gate projection and the QSA queries' chain
    cudaStream_t side2_ = nullptr;
    cudaEvent_t pfork_ = nullptr, pjoin_ = nullptr;
    cudaGraphExec_t commit_exec_ = nullptr;

    // mapped staging (host pointer, device alias)
    int32_t* h_tok_ = nullptr;   int32_t* m_tok_ = nullptr;     // T
    int32_t* h_step_ = nullptr;  int32_t* m_step_ = nullptr;    // T * kStepCount
    int32_t* h_pos_ = nullptr;   int32_t* m_pos_ = nullptr;     // T * n_head
    int32_t* h_commit_ = nullptr; int32_t* m_commit_ = nullptr; // [n_keep, n_keep-1, pos_0 .. pos_{T-1}]
    float* h_ple_ = nullptr;     float* m_ple_ = nullptr;       // T * n_embd
    int32_t* h_out_ = nullptr;   int32_t* m_out_ = nullptr;     // T argmax ids
    // STRATA_TAP_FILE (off by default; nothing is allocated or captured without it): the DFlash2 experiment's hook.
    // After each layer in STRATA_TAP_LAYERS (default 5,15,25,35,45) the window copies the layer's completed output
    // (the residual with its pending FFN write applied, the mean of the hc streams) per row; run() appends
    // (pos, token, taps) records to the file.
    bool tap_on_ = false;
    std::vector<int> tap_req_;                    ///< set_taps
    std::vector<int> tap_layers_;                 ///< the tapped layer ids (zero-based, completed outputs)
    std::vector<int> tap_slot_;                   ///< layer -> slot in tap_layers_, or -1
    float* tap_dev_ = nullptr;                    ///< [max_t][n_taps][n_embd]
    float* tap_scratch_ = nullptr;                ///< one row's streams, [hc][n_embd]
    float* h_tap_ = nullptr;     float* m_tap_ = nullptr;       ///< the mapped copy of tap_dev_
    std::FILE* tap_file_ = nullptr;
    void tap_write(int T, const int32_t* tokens, int64_t pos0);
    uint32_t* h_nf_ = nullptr;   uint32_t* m_nf_ = nullptr;     // #879: kVerifyMaxT * kNonfiniteBlocks flags
    int nf_row_ = -1;                     ///< nonfinite_row()
    int64_t nf_pos_ = -1;                 ///< nonfinite_pos()
    uint32_t nf_mask_ = 0;                ///< nonfinite_in(): bit t = row t
    void note_nonfinite(int T, const int64_t* pos_b);   ///< the window's flags (+ STRATA_NAN_INJECT) -> nf_row_
    float* h_x_ = nullptr;       float* m_x_ = nullptr;         // doorbell payload: T * n_embd
    int32_t* h_ids_ = nullptr;   int32_t* m_ids_ = nullptr;     // T * k
    float* h_w_ = nullptr;       float* m_w_ = nullptr;         // T * k
    uint32_t* h_seq_ = nullptr;  uint32_t* m_seq_ = nullptr;
    uint32_t* h_flag_ = nullptr; uint32_t* m_flag_ = nullptr;
    uint32_t* h_flagA_ = nullptr; uint32_t* m_flagA_ = nullptr;  // the GPU plan is in place
    uint32_t* h_flagB_ = nullptr; uint32_t* m_flagB_ = nullptr;  // the PCIe share's DMA copies have landed
    // set_pcie_balance: per step (layer x group, stride 2 groups) the GPU's clock as the plan arrives, as the PCIe
    // part starts and as it waits for the CPU rows (4 words a step); the pool's ms, its experts, the PCIe ones, and
    // whether the step had a plan
    unsigned long long* h_bal_ = nullptr; unsigned long long* m_bal_ = nullptr;
    std::vector<double> bal_p_;
    std::vector<int32_t> bal_nc_, bal_np_;
    std::vector<char> bal_ok_;
    PcieModel* bal_ = nullptr;
    bool bal_paused_ = false;
    void pcie_balance_window(int64_t steps, int G);
    cudaEvent_t commit_done_ = nullptr;   // recorded after an async commit (set_commit_async); see wait_commit
    bool commit_pending_ = false;
    cudaStream_t copy_ = nullptr;                                 // the copy engine's stream (DMA of missed experts)
    struct FlagSet { uint32_t* flag; uint32_t value; };
    FlagSet flag_sets_[2 * 64 * 2] = {};                          // host-function arguments, one per (layer, group)
    static void fetch_dma(void* ctx, const uint8_t* const* src, int n, size_t bytes);
    static void raise_flag(uint32_t* flag, uint32_t value);
    int32_t* h_plan_ = nullptr;  int32_t* m_plan_ = nullptr;     // counts | start | dst | tok | ptr (as int32 pairs)
    int64_t plan_i32_ = 0;                                        // int32 words in the plan block
    GpuPlanSink sink_;
    uint32_t cur_layer_ = 0;
    static void publish_plan(void* ctx);
    void set_plan_slot(int grp);
    bool split_ = false;   // opt-in (--spec-split): exact but slower, see the overlap study
    std::function<bool(cudaStream_t, std::string&)> pre_launch_;   // set_pre_launch
    int groups_[strata::kernels::kWindowMaxT + 1] = {};
    float* h_ymiss_ = nullptr;   float* m_ymiss_ = nullptr;     // T * k * n_embd

    // device
    void* arena_ = nullptr;
    int32_t *tok_ = nullptr, *step_ = nullptr, *pos_ = nullptr, *commit_ = nullptr;
    float *ple_ = nullptr, *emb_ = nullptr, *R_ = nullptr, *mixed_ = nullptr, *bo_ = nullptr;
    float *inj_ = nullptr, *inj2_ = nullptr, *lo_ = nullptr, *rs_ = nullptr, *xn_ = nullptr;
    uint8_t* xq_ = nullptr;                                   // T columns of q8_1
    uint8_t* xil_ = nullptr;                                  // fork F4: the interleaved copy of xq_'s 2-4 columns
    uint8_t* sh_xq_ = nullptr;                                // T columns of q8_1 for shared expert branch
    float *qkv_L_ = nullptr, *h_L_ = nullptr, *gate_L_ = nullptr, *beta_L_ = nullptr;   // per GDN layer
    float *z_ = nullptr, *y_ = nullptr, *y_dummy_ = nullptr;
    float *qfull_ = nullptr, *qcur_ = nullptr, *kcur_ = nullptr, *vcur_ = nullptr, *idx_raw_L_ = nullptr;
    float *qidx_ = nullptr, *scores_ = nullptr, *attn_ = nullptr, *attn32_ = nullptr, *attn_scratch_ = nullptr;
    float* tail_snap_ = nullptr;                              // per QSA layer
    int32_t* sel_ = nullptr;
    int32_t* topk_scratch_ = nullptr;   // qsa_block_topk_long's multi-CTA exchange (STRATA_SELECT_LONG=1 only)
    float *logits_ = nullptr, *w_ = nullptr, *shared_ = nullptr, *parts_ = nullptr, *hit_out_ = nullptr;
    int32_t *ids_ = nullptr, *hit_slot_ = nullptr, *hit_dst_ = nullptr, *hit_count_ = nullptr;
    int32_t* plan_ = nullptr;                                     // device copy of the plan block
    uint8_t* staging_ = nullptr;                                  // VRAM slots for the PCIe share of the misses
    static constexpr int64_t kStagingBlobs = 16;
    static constexpr int64_t kPcieGroupRows = 4;                  // the PCIe call's groups side by side (of <= 16)
    uint8_t* hit_xq_ = nullptr;
    uint8_t* nat_xq_ = nullptr;   // plan v0.3 P6: q8_1 activations for a native pack's grouped experts
    float* nat_xr_ = nullptr;     // APR: the Hadamard-rotated expert input (the shared expert and the CPU keep xm); nullptr
                                  // unless some layer's gate/up is folded
    const float* had_sx_ = nullptr;   // APR: device sign vectors for x (n_embd) and h (n_ff); nullptr = identity
    const float* had_sh_ = nullptr;
    float* hit_xs_ = nullptr;
    void* hit_scratch_ = nullptr;
    float *head_mixed_ = nullptr, *head_inj_ = nullptr, *head_logits_ = nullptr;
    uint16_t* sh_bf16_ = nullptr;
    uint8_t* arg_scratch_ = nullptr;   ///< argmax_rows' partials and counters
    int32_t* one_ = nullptr;           ///< device {1}: the n_keep of a one-token window, which commits itself
    float *sh_gate_ = nullptr, *sh_up_ = nullptr, *sh_g_ = nullptr;
    float* hist_snap_ = nullptr;                              // T * NG_HIST * NG_HC_DIM
    int64_t cap_ = 0, max_blocks_ = 0, attn_scratch_floats_ = 0;
};

}  // namespace strata::core