Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Conversation cache: opt-in STRATA_PARK_VRAM_MIB=N keeps parked conversations' K/V in VRAM in front of the host copy (#3779, #3712)
linxuhao85e3dde522cb
diff --git a/include/strata/core/conversation_cache.hpp b/include/strata/core/conversation_cache.hppindex c35aef88c..348983453 100644--- a/include/strata/core/conversation_cache.hpp+++ b/include/strata/core/conversation_cache.hpp@@ -62,6 +62,14 @@ struct ConversationKv {int format = 0;int64_t cells = 0, heads = 0, head_dim = 0, page_size = 0, pooled_rows = 0, idx_dim = 0;ConversationBuffer k, v, k_scale, v_scale, pooled;+ // STRATA_PARK_VRAM_MIB: the tag this layer's parts were recorded under in the park's VRAM slot (0: none). Set by+ // conversation_kv_save; a restore that finds the slot copies from it instead of from the buffers above.+ uint64_t dev_tag = 0;+ // STRATA_PARK_VRAM_MIB: set while a background copy fills the buffers above from the slot; its deleter waits for+ // that copy and releases the slot. This layer's own token (not shared): the layers outlive their image when they+ // are retained for the next park (ConversationCache::retain), and go in any order. Last member: it goes before+ // any buffer.+ std::shared_ptr<void> pending;size_t bytes() const {return k.bytes() + v.bytes() + k_scale.bytes() + v_scale.bytes() + pooled.bytes();}diff --git a/include/strata/core/conversation_snapshot.hpp b/include/strata/core/conversation_snapshot.hppindex be4412e1d..28071715e 100644--- a/include/strata/core/conversation_snapshot.hpp+++ b/include/strata/core/conversation_snapshot.hpp@@ -90,6 +90,21 @@ bool conversation_park_async_begin(size_t need);bool conversation_park_async_launch(std::string& error);/// Abandons a begun park that was not launched (the error paths): nothing was queued to the background.void conversation_park_async_abandon();+/// STRATA_PARK_VRAM_MIB=N (with STRATA_PARK_ASYNC): a park also keeps the whole K/V of its image in VRAM, in a slot+/// of its own taken at begin while the slots' total stays within N MiB (and 1.5 GB of VRAM stays free), so a restore+/// of that image copies device to device from the slot (conversation_kv_restore / _prefix) instead of host to+/// device, and does not wait for the background copy. The slot lives as long as the image and its K/V layers+/// (SavedConversation::pending, ConversationKv::pending). Default 0: off.+/// The tag of the begun park's slot (0: none); kept until the next begin, for the tokens.+uint64_t conversation_park_slot_tag();+/// Frees the slot under `tag` (no-op for 0 or a slot already gone); the caller joined the copy that reads it.+void conversation_park_slot_release(uint64_t tag);+/// VRAM pressure: joins the background copy, then frees every slot and the stage. Returns the bytes freed (0, and+/// nothing is touched, while STRATA_PARK_VRAM_MIB is unset).+size_t conversation_park_vram_release_all();+/// conversation_park_join, unless every K/V layer of `image` has its slot (the restore then reads no host buffer+/// the background copy could still be filling; a layer that falls back to the host joins for itself).+void conversation_park_join_for(const SavedConversation& image);bool conversation_checkpoint_restore(const ConversationCheckpoint& checkpoint, SessionState& session,const ModelGeometry& g, std::string& error);// conversation_checkpoint_save(from) + conversation_checkpoint_restore(to) of `ids` without a host image, for twodiff --git a/src/core/conversation_snapshot.cpp b/src/core/conversation_snapshot.cppindex 77d5703ad..0e4c440b0 100644--- a/src/core/conversation_snapshot.cpp+++ b/src/core/conversation_snapshot.cpp@@ -115,6 +115,103 @@ bool valid(const QsaState& st, const Layout& l, int64_t upto, std::string& errorreturn true;}+// STRATA_PARK_VRAM_MIB=N (with STRATA_PARK_ASYNC): a park also keeps the whole K/V of its image in VRAM, in a slot+// of its own, while the slots' total stays within N MiB; a restore of that image copies from the slot device to+// device (~1 ms) instead of from the host buffers over PCIe, and needs no wait for the background copy. A slot is+// released with the image or its K/V layers (their tokens), or all at once under VRAM pressure+// (conversation_park_vram_release_all). Host thread only; the background copy only reads a slot's bytes, and every+// reuse or free of one comes after a join (conversation_park_async_begin, release_all).+struct VramKvSlot {+ void* p = nullptr;+ size_t cap = 0, used = 0;+ uint64_t tag = 0; // 0: free (kept for reuse by size)+ int dev = -1;+ struct Part { uint64_t tag; size_t part, offset, bytes; }; // one layer's part (k, v, k_scale, v_scale, pooled)+ std::vector<Part> parts;+};+struct VramKv {+ std::vector<std::unique_ptr<VramKvSlot>> slots;+ uint64_t next_tag = 1; // slot and layer tags, never reused+};+VramKv& vram_kv() { static VramKv v; return v; }+size_t park_vram_budget() {+ static const size_t b = [] {+ const char* v = std::getenv("STRATA_PARK_VRAM_MIB");+ const long long mib = v != nullptr ? std::atoll(v) : 0;+ return mib > 0 ? (size_t) mib << 20 : (size_t) 0;+ }();+ return b;+}+bool park_trace() { static const bool on = std::getenv("STRATA_PARK_TRACE") != nullptr; return on; }+void vram_kv_free(VramKvSlot& s) {+ int cur = s.dev;+ (void) cudaGetDevice(&cur);+ if (cur != s.dev) (void) cudaSetDevice(s.dev);+ (void) cudaFree(s.p);+ if (cur != s.dev) (void) cudaSetDevice(cur);+ (void) cudaGetLastError();+ s.p = nullptr; s.cap = 0;+}+void vram_kv_release(VramKvSlot& s) { s.tag = 0; s.used = 0; s.parts.clear(); } // free, kept for reuse+// A slot of `need` bytes on `dev`: the best-fitting free one, else (the free ones freed first) a new one, rounded up+// to 64 MiB so a growing conversation's next parks reuse it, while the slots' total stays within the budget and+// 1.5 GB of VRAM stays free beside it. nullptr, with `why`, otherwise.+VramKvSlot* vram_kv_take(size_t need, int dev, const char*& why) {+ VramKv& v = vram_kv();+ const size_t budget = park_vram_budget();+ if (budget == 0) { why = "off"; return nullptr; }+ VramKvSlot* best = nullptr;+ for (auto& s : v.slots)+ if (s->tag == 0 && s->dev == dev && s->cap >= need && (best == nullptr || s->cap < best->cap)) best = s.get();+ if (best == nullptr) {+ size_t held = 0;+ for (size_t i = 0; i < v.slots.size();) {+ if (v.slots[i]->tag != 0) { held += v.slots[i]->cap; ++i; continue; }+ vram_kv_free(*v.slots[i]);+ v.slots.erase(v.slots.begin() + (std::ptrdiff_t) i);+ }+ const size_t want = (need + ((size_t) 64 << 20) - 1) & ~(((size_t) 64 << 20) - 1);+ if (held > budget || want > budget - held) { why = "budget"; return nullptr; }+ size_t fr = 0, tot = 0;+ if (cudaMemGetInfo(&fr, &tot) != cudaSuccess || fr < want + ((size_t) 3 << 29)) {+ (void) cudaGetLastError();+ why = "vram";+ return nullptr;+ }+ void* p = nullptr;+ if (cudaMalloc(&p, want) != cudaSuccess) { (void) cudaGetLastError(); why = "cudaMalloc"; return nullptr; }+ auto s = std::make_unique<VramKvSlot>();+ s->p = p; s->cap = want; s->dev = dev;+ best = s.get();+ v.slots.push_back(std::move(s));+ }+ best->tag = v.next_tag++;+ best->used = 0;+ best->parts.clear();+ return best;+}+// The live slot on the current device holding parts under `tag` (nullptr: none, or tag 0)+const VramKvSlot* vram_kv_find(uint64_t tag) {+ if (tag == 0) return nullptr;+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess) { (void) cudaGetLastError(); return nullptr; }+ for (const auto& s : vram_kv().slots) {+ if (s->tag == 0 || s->dev != dev) continue;+ for (const auto& p : s->parts)+ if (p.tag == tag) return s.get();+ }+ return nullptr;+}+// Whether the slot holds, under `tag`, at least sizes[i] bytes of every part i that has any; `at` gets the offsets+bool vram_kv_locate(const VramKvSlot& s, uint64_t tag, const std::array<size_t,5>& sizes, std::array<size_t,5>& at) {+ std::array<bool,5> have{};+ for (const auto& p : s.parts)+ if (p.tag == tag && p.part < at.size() && p.bytes >= sizes[p.part]) { have[p.part] = true; at[p.part] = p.offset; }+ for (size_t i = 0; i < sizes.size(); ++i)+ if (sizes[i] > 0 && !have[i]) return false;+ return true;+}+struct ParkAsync {std::mutex mu;std::future<cudaError_t> pending;@@ -126,6 +223,8 @@ struct ParkAsync {cudaStream_t stream = nullptr;int dev = -1;bool active = false;+ VramKvSlot* slot = nullptr; // STRATA_PARK_VRAM_MIB: the begun park's slot (the stage is not used then)+ uint64_t slot_tag = 0; // its tag, kept past launch for the image's and its layers' tokens};ParkAsync& park_async() { static ParkAsync a; return a; }constexpr size_t kParkPiece = (size_t) 4 << 20;@@ -211,12 +310,44 @@ bool conversation_kv_save(ConversationKv& image, const QsaState& st, const Modelconst auto src = pools(st);const std::array<size_t,5> sizes = {l.data, l.value_data, l.scales, l.value_scales, l.pooled};const std::array<ConversationBuffer*,5> dst = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};+ // STRATA_PARK_VRAM_MIB: this layer's parts go into the park's slot under a fresh tag (0: no slot this park)+ image.dev_tag = park_async().slot != nullptr ? vram_kv().next_tag++ : 0;for (size_t i = 0; i < dst.size(); ++i) {// Recopy the partial page and the indexer's moving spare row. Completed// pages/rows strictly before the first rewritten token remain identical.const size_t keep = i == 4 ? (index ? size_t(unchanged_tokens / strata::kernels::qsa_real_shapes().idx_block) * g.idx_key_dim * 4 : 0): l.cells ? (sizes[i] / size_t(l.cells)) * size_t(whole_cells) : 0;if (keep > dst[i]->size()) { error = "conversation snapshot: missing reusable prefix"; return false; }+ { // STRATA_PARK_VRAM_MIB: with a slot, the whole part goes device to device into it (a restore of this image+ // reads it from there) and the host buffer gets only the bytes past the kept prefix, read from the slot on+ // the background thread; a part the slot cannot take is copied synchronously - never through the stage,+ // whose offsets launch would read from the slot+ ParkAsync& a = park_async();+ if (a.slot != nullptr) {+ VramKvSlot& s = *a.slot;+ const bool fits = src[i] != nullptr && sizes[i] > 0 && s.used + sizes[i] <= s.cap &&+ (st.kv_mode == 0 || on_device(src[i])); // (kv_mode 0: the pools are device memory)+ if (fits) {+ const size_t at0 = s.used;+ const cudaError_t e = conversation_detail::copy_nonblocking(static_cast<uint8_t*>(s.p) + at0, src[i], sizes[i], true);+ if (e != cudaSuccess) { error = std::string("conversation snapshot slot copy: ") + cudaGetErrorString(e); return false; }+ s.parts.push_back({image.dev_tag, i, at0, sizes[i]});+ s.used += sizes[i];+ dst[i]->resize_for_overwrite(sizes[i]);+ if (!dst[i]->visit(keep, sizes[i] - keep, [&](uint8_t* p, size_t n, size_t at) {+ a.items.push_back({p, n, at0 + at});+ return true;+ })) return false;+ } else {+ dst[i]->resize(sizes[i]);+ if (!dst[i]->visit(keep, sizes[i] - keep, [&](uint8_t* p, size_t n, size_t at) {+ return transfer(p, src[i] ? static_cast<const uint8_t*>(src[i]) + at : nullptr, n, error);+ })) return false;+ }+ if (reused_bytes) *reused_bytes += keep;+ continue;+ }+ }// STRATA_PARK_ASYNC: device bytes go to the VRAM stage now and into the buffer on the background threadParkAsync& a = park_async();const bool trace = park_trace_on();@@ -323,10 +454,20 @@ bool conversation_kv_copy(const QsaState& from, const QsaState& to, const ModelGbool conversation_kv_restore(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,int64_t upto, bool index, std::string& error) {- conversation_park_join();+ // STRATA_PARK_VRAM_MIB: from the image's VRAM slot when it holds every part (no host buffer is read: no join)+ const VramKvSlot* slot = vram_kv_find(image.dev_tag);+ if (slot == nullptr) conversation_park_join();if (!conversation_kv_validate(image, st, g, upto, index, error)) return false;const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};const auto dst = pools(st);+ std::array<size_t,5> sizes{}, off{};+ for (size_t i = 0; i < src.size(); ++i) sizes[i] = src[i]->size();+ if (slot != nullptr && !vram_kv_locate(*slot, image.dev_tag, sizes, off)) { slot = nullptr; conversation_park_join(); }+ if (slot != nullptr) {+ for (size_t i = 0; i < src.size(); ++i)+ if (!transfer(dst[i], static_cast<const uint8_t*>(slot->p) + off[i], sizes[i], error)) return false;+ return refresh_residency(st, g, upto, error);+ }for (size_t i = 0; i < src.size(); ++i)if (!src[i]->visit(0, src[i]->size(), [&](const uint8_t* p, size_t n, size_t at) {return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);@@ -336,7 +477,8 @@ bool conversation_kv_restore(const ConversationKv& image, const QsaState& st, cobool conversation_kv_restore_prefix(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,int64_t captured, int64_t upto, bool index, std::string& error) {- conversation_park_join();+ const VramKvSlot* slot = vram_kv_find(image.dev_tag); // (STRATA_PARK_VRAM_MIB, as in conversation_kv_restore)+ if (slot == nullptr) conversation_park_join();if (upto < 0 || upto > captured) {error = "conversation snapshot: prefix beyond the captured K/V";return false;@@ -349,6 +491,14 @@ bool conversation_kv_restore_prefix(const ConversationKv& image, const QsaState&for (size_t i = 0; i < src.size(); ++i)if (sizes[i] > src[i]->size()) { error = "conversation snapshot: prefix beyond the K/V payload"; return false; }const auto dst = pools(st);+ // STRATA_PARK_VRAM_MIB: the prefix is the leading bytes of each part in the slot too+ std::array<size_t,5> off{};+ if (slot != nullptr && !vram_kv_locate(*slot, image.dev_tag, sizes, off)) { slot = nullptr; conversation_park_join(); }+ if (slot != nullptr) {+ for (size_t i = 0; i < src.size(); ++i)+ if (!transfer(dst[i], static_cast<const uint8_t*>(slot->p) + off[i], sizes[i], error)) return false;+ return refresh_residency(st, g, upto, error);+ }for (size_t i = 0; i < src.size(); ++i)if (!src[i]->visit(0, sizes[i], [&](const uint8_t* p, size_t n, size_t at) {return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);@@ -449,10 +599,23 @@ bool conversation_park_async_begin(size_t need) {a.active = false;a.items.clear();a.used = 0;+ a.slot = nullptr;+ a.slot_tag = 0;if (!park_async_on() || need == 0) return false;int dev = 0;if (cudaGetDevice(&dev) != cudaSuccess) { (void) cudaGetLastError(); return false; }- if (a.stage != nullptr && (a.cap < need || a.dev != dev)) {+ // STRATA_PARK_VRAM_MIB: a slot of the whole K/V instead of the stage, while the budget allows - taken before the+ // stage would be re-allocated, so a growing conversation's park pays no cudaFree (a device sync) when it gets one+ const char* why = "";+ a.slot = vram_kv_take(need, dev, why);+ a.slot_tag = a.slot != nullptr ? a.slot->tag : 0;+ if (park_trace() && park_vram_budget() != 0) {+ if (a.slot != nullptr)+ std::fprintf(stderr, "strata parktrace: kv slot %zu MB tag %llu\n", need >> 20, (unsigned long long) a.slot_tag);+ else+ std::fprintf(stderr, "strata parktrace: kv slot: none (%s)\n", why);+ }+ if (a.stage != nullptr && ((a.slot == nullptr && a.cap < need) || a.dev != dev)) {const int cur = dev;(void) cudaSetDevice(a.dev); (void) cudaFree(a.stage); (void) cudaSetDevice(cur);a.stage = nullptr; a.cap = 0;@@ -472,8 +635,13 @@ bool conversation_park_async_begin(size_t need) {if (a.stream) (void) cudaStreamDestroy(a.stream);a.stream = nullptr; a.host[0] = a.host[1] = nullptr;(void) cudaGetLastError();+ if (a.slot != nullptr) { vram_kv_release(*a.slot); a.slot = nullptr; a.slot_tag = 0; } // (STRATA_PARK_VRAM_MIB)return false;}+ if (a.slot != nullptr) { // STRATA_PARK_VRAM_MIB: the slot stands in for the stage this park+ a.active = true;+ return true;+ }if (a.stage == nullptr) {// rounded up to 64 MiB: the next parks of a growing conversation reuse it instead of re-allocating (a cudaFree// synchronizes the device, and the checkpoint DMA is in flight at this point)@@ -499,6 +667,7 @@ void conversation_park_async_abandon() {cudaStream_t s = nullptr;if (conversation_detail::private_stream(s) == cudaSuccess) (void) cudaStreamSynchronize(s); // nothing reads the stage after(void) cudaGetLastError();+ if (a.slot != nullptr) { vram_kv_release(*a.slot); a.slot = nullptr; a.slot_tag = 0; } // (STRATA_PARK_VRAM_MIB)}bool conversation_park_async_launch(std::string& error) {@@ -510,13 +679,16 @@ bool conversation_park_async_launch(std::string& error) {if (e == cudaSuccess) e = cudaStreamSynchronize(s); // the device-to-device copies into the stageif (e != cudaSuccess) {a.items.clear();+ if (a.slot != nullptr) { vram_kv_release(*a.slot); a.slot = nullptr; a.slot_tag = 0; } // (STRATA_PARK_VRAM_MIB)error = std::string("conversation snapshot stage: ") + cudaGetErrorString(e);return false;}+ // STRATA_PARK_VRAM_MIB: the host copies read the slot; it stays until the image and its K/V layers go (their tokens)+ const uint8_t* stage = a.slot != nullptr ? static_cast<const uint8_t*>(a.slot->p) : a.stage;+ a.slot = nullptr;if (a.items.empty()) return true;std::vector<ParkAsync::Item> items;items.swap(a.items);- const uint8_t* stage = a.stage;void* host[2] = {a.host[0], a.host[1]};cudaStream_t st = a.stream;const int dev = a.dev;@@ -554,4 +726,45 @@ bool conversation_park_async_launch(std::string& error) {return true;}+uint64_t conversation_park_slot_tag() { return park_async().slot_tag; }++void conversation_park_slot_release(uint64_t tag) {+ if (tag == 0) return;+ for (auto& s : vram_kv().slots)+ if (s->tag == tag) { vram_kv_release(*s); return; }+}++size_t conversation_park_vram_release_all() {+ if (park_vram_budget() == 0) return 0; // off: nothing was taken, and the stage stays as it was+ conversation_park_join(); // the background copy may be reading a slot or the stage+ ParkAsync& a = park_async();+ VramKv& v = vram_kv();+ size_t freed = 0;+ for (size_t i = 0; i < v.slots.size();) {+ if (a.active && v.slots[i].get() == a.slot) { ++i; continue; } // a park in progress keeps its own+ freed += v.slots[i]->cap;+ vram_kv_free(*v.slots[i]);+ v.slots.erase(v.slots.begin() + (std::ptrdiff_t) i);+ }+ if (a.stage != nullptr && !a.active) {+ int cur = a.dev;+ (void) cudaGetDevice(&cur);+ if (cur != a.dev) (void) cudaSetDevice(a.dev);+ (void) cudaFree(a.stage);+ if (cur != a.dev) (void) cudaSetDevice(cur);+ (void) cudaGetLastError();+ freed += a.cap;+ a.stage = nullptr; a.cap = 0;+ }+ return freed;+}++void conversation_park_join_for(const SavedConversation& image) {+ size_t hits = 0;+ for (const auto& k : image.kv) hits += vram_kv_find(k.dev_tag) != nullptr ? 1 : 0;+ if (park_trace() && park_vram_budget() != 0)+ std::fprintf(stderr, "strata parktrace: kv restore: %zu of %zu layers from the slot\n", hits, image.kv.size());+ if (hits < image.kv.size()) conversation_park_join();+}+} // namespace strata::corediff --git a/src/core/conversation_state.cpp b/src/core/conversation_state.cppindex 594a2d8a1..4bfd47d6f 100644--- a/src/core/conversation_state.cpp+++ b/src/core/conversation_state.cpp@@ -552,6 +552,9 @@ bool conversation_snapshot_save(SavedConversation& image, const ConversationViewcaptured.kv = std::move(reuse.kv);const size_t layers = owned_qsa(ss);captured.kv.resize(layers + (draft ? 1 : 0));+ // STRATA_PARK_VRAM_MIB: the retained layers' tokens go now (joined above, so no wait): held past the launch below,+ // their replacement would join the copy just launched+ for (auto& k : captured.kv) k.pending.reset();bool live_staged = false;const bool live_saved = checkpoint_save(captured.live, ss, g, error, live_staged);// STRATA_CKPT_ASYNC: the running state lands in live.gdn after the save returns; the image waits for that copy@@ -596,7 +599,22 @@ bool conversation_snapshot_save(SavedConversation& image, const ConversationViewstd::max<int64_t>(0, unchanged - 1), reused_bytes)) return false;abandon.on = false;if (staged && !conversation_park_async_launch(error)) return false;- if (staged) captured.pending = std::shared_ptr<void>(new char, [](void* p) { conversation_park_join(); delete (char*) p; });+ if (staged) {+ // STRATA_PARK_VRAM_MIB: the image's slot goes with the image, after the copy that reads it; each K/V layer+ // carries a token of its own too (the layers outlive the image when they are retained for the next park, and+ // the copy fills their buffers). Without a slot: the image's join as before, no layer tokens.+ const uint64_t slot = conversation_park_slot_tag();+ auto token = [slot] {+ return std::shared_ptr<void>(new char, [slot](void* p) {+ conversation_park_join();+ conversation_park_slot_release(slot);+ delete (char*) p;+ });+ };+ captured.pending = token();+ if (slot != 0)+ for (auto& k : captured.kv) k.pending = token();+ }image = std::move(captured);return true;}@@ -677,7 +695,7 @@ ConversationRestore conversation_snapshot_restore(const SavedConversation& imageconst ModelGeometry& g, const QsaState* draft, std::string& error) {conversation_checkpoint_join(image.live.gdn.data());for (const auto& c : image.checkpoints) conversation_checkpoint_join(c.gdn.data());- conversation_park_join();+ conversation_park_join_for(image); // (STRATA_PARK_VRAM_MIB: no wait when the K/V comes from its slot)if (!conversation_snapshot_validate(image, ss, g, draft, error)) return ConversationRestore::invalid;if (!sync(error)) return ConversationRestore::transfer_failed;const int64_t upto = (int64_t) image.live.ids.size();@@ -714,7 +732,7 @@ ConversationRestore conversation_snapshot_restore(const SavedConversation& imageconst ModelGeometry& g, const QsaState& draft, std::string& error) {conversation_checkpoint_join(image.live.gdn.data());for (const auto& c : image.checkpoints) conversation_checkpoint_join(c.gdn.data());- conversation_park_join();+ conversation_park_join_for(image); // (STRATA_PARK_VRAM_MIB: no wait when the K/V comes from its slot)return conversation_snapshot_restore(image, ss, g, &draft, error);}ConversationRestore conversation_snapshot_restore_prefix(const SavedConversation& image, int64_t upto,@@ -722,7 +740,7 @@ ConversationRestore conversation_snapshot_restore_prefix(const SavedConversationconst QsaState& draft, std::string& error) {conversation_checkpoint_join(image.live.gdn.data());for (const auto& c : image.checkpoints) conversation_checkpoint_join(c.gdn.data());- conversation_park_join();+ conversation_park_join_for(image); // (STRATA_PARK_VRAM_MIB: no wait when the K/V comes from its slot)return conversation_snapshot_restore_prefix(image, upto, ss, g, &draft, error);}ConversationRestore conversation_snapshot_restore_prefix(const SavedConversation& image, int64_t upto,@@ -730,7 +748,7 @@ ConversationRestore conversation_snapshot_restore_prefix(const SavedConversationconst QsaState* draft, std::string& error) {conversation_checkpoint_join(image.live.gdn.data());for (const auto& c : image.checkpoints) conversation_checkpoint_join(c.gdn.data());- conversation_park_join();+ conversation_park_join_for(image); // (STRATA_PARK_VRAM_MIB: no wait when the K/V comes from its slot)// a layer split's image (stage images) is restored stage by stage, never in partif (!image.stage_images.empty()) {fail(error, "a layer split's image restored as a single session");diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 4833ed748..318c507c4 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -8146,12 +8146,18 @@ int main(int argc, char** argv) {return false;}int64_t fresh = 0;- const bool ok = strata::core::qsa_kv_elastic_grow(target, [&]() -> strata::core::VmmChunk {+ auto grow_take = [&]() -> strata::core::VmmChunk {if (kvg.spare.empty()) { ++fresh; return 0; } // the cache is at its floor: new memoryconst strata::core::VmmChunk h = kvg.spare.back();kvg.spare.pop_back();return h;- });+ };+ bool ok = strata::core::qsa_kv_elastic_grow(target, grow_take);+ // STRATA_PARK_VRAM_MIB: the parked conversations' VRAM copies of their K/V give way to the K/V itself+ if (!ok && strata::core::conversation_park_vram_release_all() > 0) {+ (void) cudaGetLastError();+ ok = strata::core::qsa_kv_elastic_grow(target, grow_take);+ }kvg.fresh += fresh;kvg.cells = strata::core::qsa_kv_elastic_cells();++kvg.grows;@@ -13328,7 +13334,10 @@ int main(int argc, char** argv) {}}// the elastic K/V: a parked conversation is restored whole, and it may be longer than this prompt- if (incoming && !kvg_ensure((int64_t) incoming->live.ids.size() + 256, kv_quiesce)) {+ // (STRATA_PARK_VRAM_MIB: the parked K/V copies in VRAM go first, then one more try)+ if (incoming && !kvg_ensure((int64_t) incoming->live.ids.size() + 256, kv_quiesce) &&+ (strata::core::conversation_park_vram_release_all() == 0 ||+ !kvg_ensure((int64_t) incoming->live.ids.size() + 256, kv_quiesce))) {std::printf("ERR the K/V cannot grow to the parked conversation: no VRAM is left\n");return 1;}