Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

verify: a window graph that does not fit frees the graphs of other window sizes instead of ending the engine (#3702)

Niko1221a8c1d4b7f868
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 695f94b40..9c6f28790 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1610,6 +1610,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
add_executable(adapt_decay_test src/program/adapt_decay_test.cpp)
target_include_directories(adapt_decay_test PRIVATE include)
add_test(NAME adapt_decay_test COMMAND adapt_decay_test)
+ # which window-size graph is freed when the next one does not fit (#3702; header-only, CPU)
+ add_executable(window_evict_test src/core/window_evict_test.cpp)
+ target_include_directories(window_evict_test PRIVATE include)
+ add_test(NAME window_evict_test COMMAND window_evict_test)
# which path computes the draft layer's prompt K/V, and STRATA_SPLIT_MTP_BATCH's parsing (header-only, CPU)
add_executable(draft_kv_plan_test src/program/draft_kv_plan_test.cpp)
target_include_directories(draft_kv_plan_test PRIVATE include)
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index dc6613da7..2b2d9a0ec 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -421,9 +421,15 @@ private:
bool capture_batch(const int* rows, int S, int hbase, std::string& err);
/// Free the graph pair of the least recently used batch layout other than `keep` (`evicted`: there was one).
bool evict_batch_graph(const std::vector<int>& keep, bool& evicted, std::string& err);
- /// cudaGraphInstantiate; out of VRAM, batch layouts other than `key` are evicted (LRU) until it fits.
+ /// cudaGraphInstantiate; out of VRAM, batch layouts other than `key` are evicted (LRU) until it fits, and then (a
+ /// single-request window, `keep_t` > 0) the graphs of the other window sizes, largest first (#3702): a window size
+ /// whose graph does not fit used to end the engine ("verify: instantiate: out of memory"); a freed size is captured
+ /// again when it is next used.
bool instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key, const char* what,
- std::string& err);
+ std::string& err, int keep_t = -1);
+ /// Free the graph of the largest window size other than `keep_t` (and the prelaunched one); `evicted`: there was one.
+ bool evict_window_graph(int keep_t, bool& evicted, std::string& err);
+ bool no_window_eviction_ = false; ///< capture_all wants every size at once: it must not free one to capture another
void collect_profile(); ///< STRATA_VERIFY_PROFILE: add the last window's stamps to prof_sum_
void accumulate_profile(const unsigned long long* stamps);
void accumulate_profile_inner(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_
diff --git a/include/strata/core/window_evict.hpp b/include/strata/core/window_evict.hpp
new file mode 100644
index 000000000..129647761
--- /dev/null
+++ b/include/strata/core/window_evict.hpp
@@ -0,0 +1,19 @@
+#pragma once
+// Which single-request window graph to free when the next one cannot be instantiated for lack of VRAM (#3702).
+// Header-only so a CPU test can run it (src/core/window_evict_test.cpp).
+
+namespace strata::core {
+
+/// The size T (1..max_t) whose captured graph is freed first: the largest size other than `keep` (the one being
+/// captured) and `pinned` (a window of that size was prelaunched and must stay), among those with a graph (`have[t]`).
+/// The larger sizes cost the most VRAM and are asked for least often, and a freed size is captured again when it is
+/// next needed. 0: nothing to free.
+inline int pick_window_victim(const bool* have, int max_t, int keep, int pinned) {
+ for (int t = max_t; t >= 1; --t) {
+ if (t == keep || t == pinned || !have[t]) continue;
+ return t;
+ }
+ return 0;
+}
+
+} // namespace strata::core
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index b82181c25..5e97b8f52 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -4,6 +4,7 @@
#include "strata/core/sm120_series.hpp"
#include "strata/kernels/gfx_arch.hpp"
#include "strata/core/verify.hpp"
+#include "strata/core/window_evict.hpp"
#include "strata/core/pin_budget.hpp"
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
#include "strata/core/host_graph_launch.hpp"
@@ -2575,7 +2576,7 @@ bool Verifier::capture(int T, std::string& err) {
#endif
}
#endif
- const bool made = instantiate_evicting(exec_t, graph, {}, "verify: instantiate: ", err);
+ const bool made = instantiate_evicting(exec_t, graph, {}, "verify: instantiate: ", err, T);
cudaGraphDestroy(graph);
if (!made) return false;
const cudaError_t ue = cudaGraphUpload(exec_t, cs_);
@@ -3695,8 +3696,27 @@ bool Verifier::evict_batch_graph(const std::vector<int>& keep, bool& evicted, st
return true;
}
+bool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {
+ evicted = false;
+ if (no_window_eviction_ || keep_t < 1) return true;
+ bool have[9] = {};
+ for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[t] : exec_[t]) != nullptr;
+ const int victim = strata::core::pick_window_victim(have, std::min(max_t_, 8), keep_t, prelaunched_);
+ if (victim == 0) return true;
+ if (cudaStreamSynchronize(cs_) != cudaSuccess) {
+ err = "verify: synchronizing before window graph eviction failed";
+ return false;
+ }
+ cudaGraphExec_t& e = ar_off_ ? exec_nr_[victim] : exec_[victim];
+ cudaGraphExecDestroy(e);
+ e = nullptr;
+ (ar_off_ ? launched_nr_ : launched_)[victim] = false; // the next graph of that size is a cold first launch again
+ evicted = true;
+ return true;
+}
+
bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key,
- const char* what, std::string& err) {
+ const char* what, std::string& err, int keep_t) {
// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache
// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can
// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)
@@ -3713,6 +3733,20 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons
if (freed > 0)
std::fprintf(stderr, "strata verify: no VRAM for a new window graph: freed the graphs of %d older batch slot "
"layouts, %zu kept (a larger --vram-reserve-mib keeps more)\n", freed, exec_bm_.size());
+ // #3702: a single-request window with nothing of a batch left to free: free the graphs of other window sizes
+ int freed_sizes = 0;
+ for (bool evicted = true; ie == cudaErrorMemoryAllocation && evicted && keep_t > 0;) {
+ (void) cudaGetLastError();
+ if (!evict_window_graph(keep_t, evicted, err)) return false;
+ if (evicted) {
+ ++freed_sizes;
+ ie = cudaGraphInstantiate(&ex, graph, 0);
+ }
+ }
+ if (freed_sizes > 0)
+ std::fprintf(stderr, "strata verify: no VRAM for the %d-token window graph: freed the graphs of %d other window "
+ "sizes (they are captured again when used; a larger --vram-reserve-mib or "
+ "--vram-reserve-later-mib keeps them)\n", keep_t, freed_sizes);
if (ie != cudaSuccess) {
err = std::string(what) + cudaGetErrorString(ie);
return false;
@@ -4204,6 +4238,11 @@ bool Verifier::capture_all(std::string& err) {
if (g_ == nullptr) { err = "verify: capture_all before init"; return false; }
if (remote_opt_ != nullptr) { err = "verify: pipelined windows do not serve --remote-expert-opt"; return false; }
if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }
+ struct NoEvict { // every size is wanted at once here: a capture must not free another size's graph
+ bool& f;
+ explicit NoEvict(bool& flag) : f(flag) { f = true; }
+ ~NoEvict() { f = false; }
+ } no_evict(no_window_eviction_);
for (int T = 1; T <= max_t_; ++T)
if (!capture(T, err)) return false;
if (!capture_commit(err)) return false;
diff --git a/src/core/window_evict_test.cpp b/src/core/window_evict_test.cpp
new file mode 100644
index 000000000..05e1b8be7
--- /dev/null
+++ b/src/core/window_evict_test.cpp
@@ -0,0 +1,33 @@
+// pick_window_victim (include/strata/core/window_evict.hpp): CPU only (#3702).
+#include "strata/core/window_evict.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+
+using strata::core::pick_window_victim;
+
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+ ++checks;
+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+} // namespace
+
+int main() {
+ bool have[9] = {};
+ check(pick_window_victim(have, 8, 5, 0) == 0, "nothing captured: nothing to free");
+ have[2] = have[3] = have[5] = true;
+ check(pick_window_victim(have, 8, 6, 0) == 5, "the largest captured size goes first");
+ check(pick_window_victim(have, 8, 5, 0) == 3, "never the size being captured");
+ check(pick_window_victim(have, 8, 5, 3) == 2, "never the prelaunched size");
+ check(pick_window_victim(have, 4, 1, 0) == 3, "sizes above max_t are not considered");
+ have[5] = false;
+ check(pick_window_victim(have, 8, 3, 2) == 0, "only the kept and the pinned sizes are left: nothing");
+ have[2] = have[3] = false;
+ have[1] = true;
+ check(pick_window_victim(have, 8, 7, 0) == 1, "size 1 can be freed");
+ check(pick_window_victim(have, 8, 1, 0) == 0, "size 1 kept");
+ std::printf("window_evict_test: %d checks passed\n", checks);
+ return 0;
+}