Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Decode window: opt-in STRATA_GRAPH_PRIO=1 gives the window graph's kernel nodes their stream's priority (#3779, #3712)

linxuhao0049fa73fff7
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index 2b2d9a0ec..60707b979 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -426,10 +426,20 @@ private:
/// whose graph does not fit used to end the engine ("verify: instantiate: out of memory"); a freed size is captured
/// again when it is next used.
bool instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key, const char* what,
- std::string& err, int keep_t = -1);
+ std::string& err, int keep_t = -1, unsigned long long flags = 0);
/// Free the graph of the largest window size other than `keep_t` (and the prelaunched one); `evicted`: there was one.
bool evict_window_graph(int keep_t, bool& evicted, std::string& err);
bool no_window_eviction_ = false; ///< capture_all wants every size at once: it must not free one to capture another
+ // STRATA_GRAPH_PRIO (record_window / capture): the captured tails of the priority streams at their joins and the
+ // main stream's tails at their forks, so that a capture that dropped the stream priorities can be repaired per
+ // node (apply_node_priorities walks each tail back to its fork)
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL) // (the three methods are no-ops there)
+ std::vector<cudaGraphNode_t> prio_forks_;
+ std::vector<std::pair<cudaGraphNode_t, int>> prio_tails_;
+#endif
+ void note_prio_fork(cudaStream_t from);
+ void note_prio_tail(cudaStream_t on, int prio);
+ void apply_node_priorities(cudaGraph_t graph, int T);
void collect_profile(); ///< STRATA_VERIFY_PROFILE: add the last window's stamps to prof_sum_
void accumulate_profile(const unsigned long long* stamps);
void accumulate_profile_inner(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index a87e995d2..8a12c185b 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -64,6 +64,7 @@
#include <algorithm>
#include <atomic>
#include <map>
+#include <set>
#include <mutex>
#include <string>
#include <optional>
@@ -183,6 +184,38 @@ inline bool g_qfuse_hc() {
return false;
#endif
}
+// STRATA_GRAPH_PRIO=1 (opt-in): the window graph's kernel nodes keep their capturing stream's
+// priority (side_ high, sh_cs_ low). Unset / 0: every node at the launch stream's priority, as before.
+[[maybe_unused]] inline bool graph_prio_on() { // (every HIP caller is guarded off)
+#if defined(STRATA_USE_HIP) || defined(STRATA_HIP_GFX906) || defined(STRATA_GPU_METAL)
+ return false;
+#else
+ static const bool on = [] { const char* v = std::getenv("STRATA_GRAPH_PRIO"); return v != nullptr && std::atoi(v) != 0; }();
+ return on;
+#endif
+}
+// STRATA_GRAPH_PRIO: the capture and graph queries by CUDA version (13 renamed the edge-data forms: the same split as
+// verify_kernels.cu's and the PDL edge check above). Nothing is returned for the edge data.
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+inline cudaError_t prio_capture_deps(cudaStream_t s, cudaStreamCaptureStatus* st, const cudaGraphNode_t** deps, size_t* n) {
+#if CUDART_VERSION >= 13000
+ return cudaStreamGetCaptureInfo(s, st, nullptr, nullptr, deps, nullptr, n);
+#elif CUDART_VERSION >= 12030
+ return cudaStreamGetCaptureInfo_v3(s, st, nullptr, nullptr, deps, nullptr, n);
+#else
+ return cudaStreamGetCaptureInfo_v2(s, st, nullptr, nullptr, deps, n);
+#endif
+}
+inline cudaError_t prio_node_deps(cudaGraphNode_t node, cudaGraphNode_t* deps, size_t* n) {
+#if CUDART_VERSION >= 13000
+ return cudaGraphNodeGetDependencies(node, deps, nullptr, n);
+#elif CUDART_VERSION >= 12030
+ return cudaGraphNodeGetDependencies_v2(node, deps, nullptr, n);
+#else
+ return cudaGraphNodeGetDependencies(node, deps, n);
+#endif
+}
+#endif
// fork 34a35db + b063c13 (Eddoursul), STRATA_SIDE_MIXER=1 / =2: the mixer's work that reads only the layer's input on
// branches of the window's graph (bitwise the same outputs); 0 = in line, and no branch stream is created. Unset: 2 on a
// compute capability 12.0 card (sm120_series.hpp; STRATA_SM120_PERF=0 turns it off), 0 elsewhere.
@@ -966,9 +999,24 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
sink_.staging = (unsigned long long) staging_;
sink_.staging_cap = kStagingBlobs;
(void) TS;
+ // STRATA_GRAPH_PRIO=1 (opt-in): the window's streams carry priorities that their captured kernel
+ // nodes keep (the window graph is instantiated with cudaGraphInstantiateFlagUseNodePriority): the GDN gates / QSA
+ // indexer branch (side_) at the device's highest, cs_ and side2_ in the middle, the shared expert (sh_cs_) at the
+ // lowest (= the default). The indexer chain's 1-block kernels are then dispatched ahead of the q GEMV's queued
+ // blocks, and the shared expert behind the experts. Unset: every stream at the default priority, as before.
+ int gp_lo = 0, gp_hi = 0;
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
+ const bool gprio = graph_prio_on() && cudaDeviceGetStreamPriorityRange(&gp_lo, &gp_hi) == cudaSuccess && gp_lo != gp_hi;
+ if (graph_prio_on() && !gprio) (void) cudaGetLastError(); // (only a failed range query's error: unset clears nothing)
+#else
+ const bool gprio = false; // (HIP: the default priority, as make_stream's below)
+#endif
+ const int gp_mid = gprio ? (gp_lo + gp_hi) / 2 : 0;
+ if (gprio) std::fprintf(stderr, "strata verify: STRATA_GRAPH_PRIO: stream priorities side %d, cs / side2 %d, shared expert %d\n",
+ gp_hi, gp_mid, gp_lo);
// STRATA_PIPED_CONCURRENT=1 (opt-in): a prompt can run on the same GPU while the batch windows run, so the windows'
// streams get the device's highest priority (HIP: the default priority, as the drafter's)
- auto make_stream = [](cudaStream_t* s) -> cudaError_t {
+ auto make_stream = [&](cudaStream_t* s, int gp = 0) -> cudaError_t {
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
static const bool hi = [] { // (STRATA_PIPED_CONCURRENT_PRIO=0: the default priority)
const char* v = std::getenv("STRATA_PIPED_CONCURRENT");
@@ -982,6 +1030,12 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
return cudaSuccess;
cudaGetLastError(); // (only the failed priority call's error: the default path clears nothing)
}
+ if (gprio) { // STRATA_GRAPH_PRIO: this stream's own priority (gp)
+ if (cudaStreamCreateWithPriority(s, cudaStreamNonBlocking, gp) == cudaSuccess) return cudaSuccess;
+ cudaGetLastError();
+ }
+#else
+ (void) gp;
#endif
return cudaStreamCreateWithFlags(s, cudaStreamNonBlocking);
};
@@ -1003,8 +1057,8 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
return false;
}
if (ext_stream_ != nullptr) cs_ = ext_stream_; // set_stream (pipelined windows): the stage's shared stream
- else if (make_stream(&cs_) != cudaSuccess) cs_ = nullptr;
- if (cs_ == nullptr || make_stream(&sh_cs_) != cudaSuccess) {
+ else if (make_stream(&cs_, gp_mid) != cudaSuccess) cs_ = nullptr;
+ if (cs_ == nullptr || make_stream(&sh_cs_, gp_lo) != cudaSuccess) {
err = "verify: stream create failed";
return false;
}
@@ -1050,10 +1104,10 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
}
}
if (g_side_mixer() > 0 &&
- (cudaStreamCreateWithFlags(&side_, cudaStreamNonBlocking) != cudaSuccess ||
+ ((gprio ? make_stream(&side_, gp_hi) : cudaStreamCreateWithFlags(&side_, cudaStreamNonBlocking)) != cudaSuccess ||
cudaEventCreateWithFlags(&mfork_, cudaEventDisableTiming) != cudaSuccess ||
cudaEventCreateWithFlags(&mjoin_, cudaEventDisableTiming) != cudaSuccess ||
- cudaStreamCreateWithFlags(&side2_, cudaStreamNonBlocking) != cudaSuccess ||
+ (gprio ? make_stream(&side2_, gp_mid) : cudaStreamCreateWithFlags(&side2_, cudaStreamNonBlocking)) != cudaSuccess ||
cudaEventCreateWithFlags(&pfork_, cudaEventDisableTiming) != cudaSuccess ||
cudaEventCreateWithFlags(&pjoin_, cudaEventDisableTiming) != cudaSuccess)) {
err = "verify: the mixer branch's stream / events could not be created";
@@ -1129,6 +1183,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
const bool side_mixer = side_mixer_mode >= 1;
const bool side_proj = side_mixer_mode >= 2 && !batch_rec_; // fork b063c13: the GDN gate projection, the QSA queries
auto branch = [&](cudaEvent_t e, cudaStream_t from, cudaStream_t to, const char* what) {
+ if (to == side_) note_prio_fork(from); // STRATA_GRAPH_PRIO: where the high-priority branch leaves cs
if (cudaEventRecord(e, from) == cudaSuccess && cudaStreamWaitEvent(to, e, 0) == cudaSuccess) return true;
err = std::string("verify: the mixer branch could not ") + what;
return false;
@@ -1432,6 +1487,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,
(const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N,
(int) HV, n, side_);
+ note_prio_tail(side_, 1);
if (cudaEventRecord(mjoin_, side_) != cudaSuccess) { err = "verify: the GDN gates' branch could not join"; return false; }
}
if (!q8_attn) quant_in(xm, (int) N);
@@ -1715,6 +1771,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
idx_append(side_);
idx_query(side_);
idx_select(side_);
+ note_prio_tail(side_, 1);
if (cudaEventRecord(mjoin_, side_) != cudaSuccess) { err = "verify: the indexer's branch could not join"; return false; }
}
if (side_proj) { // the queries beside the keys and values (fork b063c13)
@@ -1931,6 +1988,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
const bool sh_fork_late = sh_fork && sh_fork_late_env && !ar_on();
cudaStream_t sh_stream = sh_fork ? sh_cs_ : cs;
if (sh_fork && !sh_fork_late) {
+ note_prio_fork(cs);
cudaEventRecord(ev_fork_, cs);
cudaStreamWaitEvent(sh_cs_, ev_fork_, 0);
}
@@ -2009,6 +2067,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
m_x_ + tb * N, m_ids_ + tb * K, m_seq_, cs);
}
if (sh_fork_late) {
+ note_prio_fork(cs);
cudaEventRecord(ev_fork_, cs);
cudaStreamWaitEvent(sh_cs_, ev_fork_, 0);
}
@@ -2046,7 +2105,10 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
err = std::string("verify shared expert: ") + e.what();
return false;
}
- if (sh_fork) cudaEventRecord(ev_join_, sh_cs_);
+ if (sh_fork) {
+ note_prio_tail(sh_cs_, -1);
+ cudaEventRecord(ev_join_, sh_cs_);
+ }
}
if (strata::kernels::cpu::expert_layout().native) {
// APR: Hadamard-folded gate/up take H (s * x); xm itself stays as it is (the shared expert above and the
@@ -2466,6 +2528,10 @@ bool Verifier::capture(int T, std::string& err) {
err = "verify: begin capture failed";
return false;
}
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+ prio_forks_.clear();
+ prio_tails_.clear();
+#endif
std::string rerr;
const bool ok = record_window(T, cs_, rerr);
cudaGraph_t graph = nullptr;
@@ -2576,7 +2642,14 @@ bool Verifier::capture(int T, std::string& err) {
#endif
}
#endif
- const bool made = instantiate_evicting(exec_t, graph, {}, "verify: instantiate: ", err, T);
+ unsigned long long iflags = 0;
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+ if (graph_prio_on()) { // STRATA_GRAPH_PRIO: the nodes run at their own (captured or repaired) priorities
+ apply_node_priorities(graph, T);
+ iflags = cudaGraphInstantiateFlagUseNodePriority;
+ }
+#endif
+ const bool made = instantiate_evicting(exec_t, graph, {}, "verify: instantiate: ", err, T, iflags);
cudaGraphDestroy(graph);
if (!made) return false;
const cudaError_t ue = cudaGraphUpload(exec_t, cs_);
@@ -3733,19 +3806,115 @@ bool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {
return true;
}
+// STRATA_GRAPH_PRIO: the capture-time notes. A fork note keeps cs's current tail (the node the branch's first kernel
+// will depend on); a tail note keeps a priority stream's current tail and the priority its nodes should run at
+// (1 = the device's highest, -1 = its lowest). No-ops with the switch off, outside a capture, and on HIP / Metal.
+void Verifier::note_prio_fork(cudaStream_t from) {
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+ if (!graph_prio_on()) return;
+ cudaStreamCaptureStatus st = cudaStreamCaptureStatusNone;
+ const cudaGraphNode_t* deps = nullptr;
+ size_t nd = 0;
+ if (prio_capture_deps(from, &st, &deps, &nd) != cudaSuccess || st != cudaStreamCaptureStatusActive) {
+ (void) cudaGetLastError();
+ return;
+ }
+ for (size_t i = 0; i < nd; ++i) prio_forks_.push_back(deps[i]);
+#else
+ (void) from;
+#endif
+}
+
+void Verifier::note_prio_tail(cudaStream_t on, int prio) {
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+ if (!graph_prio_on()) return;
+ cudaStreamCaptureStatus st = cudaStreamCaptureStatusNone;
+ const cudaGraphNode_t* deps = nullptr;
+ size_t nd = 0;
+ if (prio_capture_deps(on, &st, &deps, &nd) != cudaSuccess || st != cudaStreamCaptureStatusActive) {
+ (void) cudaGetLastError();
+ return;
+ }
+ for (size_t i = 0; i < nd; ++i) prio_tails_.push_back({deps[i], prio});
+#else
+ (void) on; (void) prio;
+#endif
+}
+
+// STRATA_GRAPH_PRIO: what priorities the captured kernel nodes carry (a histogram, logged once per window graph). If the
+// capture copied none from the streams (design 1's R1: some drivers read back 0 everywhere), each priority stream's
+// nodes are found by walking its noted tails back to the noted forks and the attribute is set per node.
+void Verifier::apply_node_priorities(cudaGraph_t graph, int T) {
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+ int lo = 0, hi = 0;
+ if (cudaDeviceGetStreamPriorityRange(&lo, &hi) != cudaSuccess) { (void) cudaGetLastError(); return; }
+ auto histogram = [&](std::map<int, int>& h) {
+ size_t nn = 0;
+ cudaGraphGetNodes(graph, nullptr, &nn);
+ std::vector<cudaGraphNode_t> nodes(nn);
+ cudaGraphGetNodes(graph, nodes.data(), &nn);
+ h.clear();
+ for (cudaGraphNode_t nd : nodes) {
+ cudaGraphNodeType ty;
+ if (cudaGraphNodeGetType(nd, &ty) != cudaSuccess || ty != cudaGraphNodeTypeKernel) continue;
+ cudaKernelNodeAttrValue v{};
+ if (cudaGraphKernelNodeGetAttribute(nd, cudaKernelNodeAttributePriority, &v) == cudaSuccess) ++h[v.priority];
+ else { (void) cudaGetLastError(); ++h[999]; }
+ }
+ };
+ std::map<int, int> h;
+ histogram(h);
+ bool captured = false;
+ for (auto& [p, c] : h) captured = captured || (p != 0 && p != 999 && c > 0);
+ int repaired = 0, walked_forks = (int) prio_forks_.size(), walked_tails = (int) prio_tails_.size();
+ if (!captured && !prio_tails_.empty()) { // the fallback: per node from the capture-time notes
+ std::set<cudaGraphNode_t> stop(prio_forks_.begin(), prio_forks_.end()), seen;
+ for (auto& [tail, pr] : prio_tails_) {
+ std::vector<cudaGraphNode_t> stack{tail};
+ while (!stack.empty()) {
+ cudaGraphNode_t nd = stack.back();
+ stack.pop_back();
+ if (stop.count(nd) || !seen.insert(nd).second) continue;
+ cudaGraphNodeType ty;
+ if (cudaGraphNodeGetType(nd, &ty) == cudaSuccess && ty == cudaGraphNodeTypeKernel) {
+ cudaKernelNodeAttrValue v{};
+ v.priority = pr > 0 ? hi : lo;
+ if (cudaGraphKernelNodeSetAttribute(nd, cudaKernelNodeAttributePriority, &v) == cudaSuccess) ++repaired;
+ else (void) cudaGetLastError();
+ }
+ size_t nd_n = 0;
+ prio_node_deps(nd, nullptr, &nd_n);
+ std::vector<cudaGraphNode_t> deps(nd_n);
+ if (nd_n > 0 && prio_node_deps(nd, deps.data(), &nd_n) == cudaSuccess)
+ for (size_t i = 0; i < nd_n; ++i) stack.push_back(deps[i]);
+ else (void) cudaGetLastError();
+ }
+ }
+ histogram(h);
+ }
+ std::string s;
+ for (auto& [p, c] : h) s += " " + std::to_string(c) + " x prio " + (p == 999 ? std::string("?") : std::to_string(p)) + ";";
+ std::fprintf(stderr, "strata verify: the %d-token window's kernel node priorities (range %d..%d):%s %s (%d forks, %d tails noted%s)\n",
+ T, lo, hi, s.c_str(), captured ? "captured from the streams" : "NOT captured from the streams",
+ walked_forks, walked_tails, repaired > 0 ? (", " + std::to_string(repaired) + " nodes set from the notes").c_str() : "");
+#else
+ (void) graph; (void) T;
+#endif
+}
+
bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key,
- const char* what, std::string& err, int keep_t) {
+ const char* what, std::string& err, int keep_t, unsigned long long flags) {
// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache
// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can
// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)
- cudaError_t ie = cudaGraphInstantiate(&ex, graph, 0);
+ cudaError_t ie = cudaGraphInstantiate(&ex, graph, flags);
int freed = 0;
for (bool evicted = true; ie == cudaErrorMemoryAllocation && evicted;) {
(void) cudaGetLastError();
if (!evict_batch_graph(key, evicted, err)) return false;
if (evicted) {
++freed;
- ie = cudaGraphInstantiate(&ex, graph, 0);
+ ie = cudaGraphInstantiate(&ex, graph, flags);
}
}
if (freed > 0)
@@ -3758,7 +3927,7 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons
if (!evict_window_graph(keep_t, evicted, err)) return false;
if (evicted) {
++freed_sizes;
- ie = cudaGraphInstantiate(&ex, graph, 0);
+ ie = cudaGraphInstantiate(&ex, graph, flags);
}
}
if (freed_sizes > 0)