Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
verify: a window graph that does not fit frees the graphs of other window sizes instead of ending the engine (#3702)
Niko1221a8c1d4b7f868
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 695f94b40..9c6f28790 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1610,6 +1610,10 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)add_executable(adapt_decay_test src/program/adapt_decay_test.cpp)target_include_directories(adapt_decay_test PRIVATE include)add_test(NAME adapt_decay_test COMMAND adapt_decay_test)+ # which window-size graph is freed when the next one does not fit (#3702; header-only, CPU)+ add_executable(window_evict_test src/core/window_evict_test.cpp)+ target_include_directories(window_evict_test PRIVATE include)+ add_test(NAME window_evict_test COMMAND window_evict_test)# which path computes the draft layer's prompt K/V, and STRATA_SPLIT_MTP_BATCH's parsing (header-only, CPU)add_executable(draft_kv_plan_test src/program/draft_kv_plan_test.cpp)target_include_directories(draft_kv_plan_test PRIVATE include)diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hppindex dc6613da7..2b2d9a0ec 100644--- a/include/strata/core/verify.hpp+++ b/include/strata/core/verify.hpp@@ -421,9 +421,15 @@ private:bool capture_batch(const int* rows, int S, int hbase, std::string& err);/// Free the graph pair of the least recently used batch layout other than `keep` (`evicted`: there was one).bool evict_batch_graph(const std::vector<int>& keep, bool& evicted, std::string& err);- /// cudaGraphInstantiate; out of VRAM, batch layouts other than `key` are evicted (LRU) until it fits.+ /// cudaGraphInstantiate; out of VRAM, batch layouts other than `key` are evicted (LRU) until it fits, and then (a+ /// single-request window, `keep_t` > 0) the graphs of the other window sizes, largest first (#3702): a window size+ /// whose graph does not fit used to end the engine ("verify: instantiate: out of memory"); a freed size is captured+ /// again when it is next used.bool instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key, const char* what,- std::string& err);+ std::string& err, int keep_t = -1);+ /// Free the graph of the largest window size other than `keep_t` (and the prelaunched one); `evicted`: there was one.+ bool evict_window_graph(int keep_t, bool& evicted, std::string& err);+ bool no_window_eviction_ = false; ///< capture_all wants every size at once: it must not free one to capture anothervoid collect_profile(); ///< STRATA_VERIFY_PROFILE: add the last window's stamps to prof_sum_void accumulate_profile(const unsigned long long* stamps);void accumulate_profile_inner(const unsigned long long* stamps); ///< one window's stamps (host copy) into prof_sum_diff --git a/include/strata/core/window_evict.hpp b/include/strata/core/window_evict.hppnew file mode 100644index 000000000..129647761--- /dev/null+++ b/include/strata/core/window_evict.hpp@@ -0,0 +1,19 @@+#pragma once+// Which single-request window graph to free when the next one cannot be instantiated for lack of VRAM (#3702).+// Header-only so a CPU test can run it (src/core/window_evict_test.cpp).++namespace strata::core {++/// The size T (1..max_t) whose captured graph is freed first: the largest size other than `keep` (the one being+/// captured) and `pinned` (a window of that size was prelaunched and must stay), among those with a graph (`have[t]`).+/// The larger sizes cost the most VRAM and are asked for least often, and a freed size is captured again when it is+/// next needed. 0: nothing to free.+inline int pick_window_victim(const bool* have, int max_t, int keep, int pinned) {+ for (int t = max_t; t >= 1; --t) {+ if (t == keep || t == pinned || !have[t]) continue;+ return t;+ }+ return 0;+}++} // namespace strata::corediff --git a/src/core/verify.cpp b/src/core/verify.cppindex b82181c25..5e97b8f52 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -4,6 +4,7 @@#include "strata/core/sm120_series.hpp"#include "strata/kernels/gfx_arch.hpp"#include "strata/core/verify.hpp"+#include "strata/core/window_evict.hpp"#include "strata/core/pin_budget.hpp"#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)#include "strata/core/host_graph_launch.hpp"@@ -2575,7 +2576,7 @@ bool Verifier::capture(int T, std::string& err) {#endif}#endif- const bool made = instantiate_evicting(exec_t, graph, {}, "verify: instantiate: ", err);+ const bool made = instantiate_evicting(exec_t, graph, {}, "verify: instantiate: ", err, T);cudaGraphDestroy(graph);if (!made) return false;const cudaError_t ue = cudaGraphUpload(exec_t, cs_);@@ -3695,8 +3696,27 @@ bool Verifier::evict_batch_graph(const std::vector<int>& keep, bool& evicted, streturn true;}+bool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {+ evicted = false;+ if (no_window_eviction_ || keep_t < 1) return true;+ bool have[9] = {};+ for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[t] : exec_[t]) != nullptr;+ const int victim = strata::core::pick_window_victim(have, std::min(max_t_, 8), keep_t, prelaunched_);+ if (victim == 0) return true;+ if (cudaStreamSynchronize(cs_) != cudaSuccess) {+ err = "verify: synchronizing before window graph eviction failed";+ return false;+ }+ cudaGraphExec_t& e = ar_off_ ? exec_nr_[victim] : exec_[victim];+ cudaGraphExecDestroy(e);+ e = nullptr;+ (ar_off_ ? launched_nr_ : launched_)[victim] = false; // the next graph of that size is a cold first launch again+ evicted = true;+ return true;+}+bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key,- const char* what, std::string& err) {+ const char* what, std::string& err, int keep_t) {// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)@@ -3713,6 +3733,20 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, consif (freed > 0)std::fprintf(stderr, "strata verify: no VRAM for a new window graph: freed the graphs of %d older batch slot ""layouts, %zu kept (a larger --vram-reserve-mib keeps more)\n", freed, exec_bm_.size());+ // #3702: a single-request window with nothing of a batch left to free: free the graphs of other window sizes+ int freed_sizes = 0;+ for (bool evicted = true; ie == cudaErrorMemoryAllocation && evicted && keep_t > 0;) {+ (void) cudaGetLastError();+ if (!evict_window_graph(keep_t, evicted, err)) return false;+ if (evicted) {+ ++freed_sizes;+ ie = cudaGraphInstantiate(&ex, graph, 0);+ }+ }+ if (freed_sizes > 0)+ std::fprintf(stderr, "strata verify: no VRAM for the %d-token window graph: freed the graphs of %d other window "+ "sizes (they are captured again when used; a larger --vram-reserve-mib or "+ "--vram-reserve-later-mib keeps them)\n", keep_t, freed_sizes);if (ie != cudaSuccess) {err = std::string(what) + cudaGetErrorString(ie);return false;@@ -4204,6 +4238,11 @@ bool Verifier::capture_all(std::string& err) {if (g_ == nullptr) { err = "verify: capture_all before init"; return false; }if (remote_opt_ != nullptr) { err = "verify: pipelined windows do not serve --remote-expert-opt"; return false; }if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }+ struct NoEvict { // every size is wanted at once here: a capture must not free another size's graph+ bool& f;+ explicit NoEvict(bool& flag) : f(flag) { f = true; }+ ~NoEvict() { f = false; }+ } no_evict(no_window_eviction_);for (int T = 1; T <= max_t_; ++T)if (!capture(T, err)) return false;if (!capture_commit(err)) return false;diff --git a/src/core/window_evict_test.cpp b/src/core/window_evict_test.cppnew file mode 100644index 000000000..05e1b8be7--- /dev/null+++ b/src/core/window_evict_test.cpp@@ -0,0 +1,33 @@+// pick_window_victim (include/strata/core/window_evict.hpp): CPU only (#3702).+#include "strata/core/window_evict.hpp"++#include <cstdio>+#include <cstdlib>++using strata::core::pick_window_victim;++namespace {+int checks = 0;+void check(bool ok, const char* label) {+ ++checks;+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }+}+} // namespace++int main() {+ bool have[9] = {};+ check(pick_window_victim(have, 8, 5, 0) == 0, "nothing captured: nothing to free");+ have[2] = have[3] = have[5] = true;+ check(pick_window_victim(have, 8, 6, 0) == 5, "the largest captured size goes first");+ check(pick_window_victim(have, 8, 5, 0) == 3, "never the size being captured");+ check(pick_window_victim(have, 8, 5, 3) == 2, "never the prelaunched size");+ check(pick_window_victim(have, 4, 1, 0) == 3, "sizes above max_t are not considered");+ have[5] = false;+ check(pick_window_victim(have, 8, 3, 2) == 0, "only the kept and the pinned sizes are left: nothing");+ have[2] = have[3] = false;+ have[1] = true;+ check(pick_window_victim(have, 8, 7, 0) == 1, "size 1 can be freed");+ check(pick_window_victim(have, 8, 1, 0) == 0, "size 1 kept");+ std::printf("window_evict_test: %d checks passed\n", checks);+ return 0;+}