Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
conversation cache: never park one conversation with another's retained K/V (#1503)
Stefanos Anastasiouf8ff3420d997
diff --git a/include/strata/core/conversation_cache.hpp b/include/strata/core/conversation_cache.hppindex b2e6a74f7..973b25938 100644--- a/include/strata/core/conversation_cache.hpp+++ b/include/strata/core/conversation_cache.hpp@@ -69,8 +69,12 @@ struct ConversationKvReuse {int64_t captured_tokens = 0, unchanged_tokens = 0;// With a layer split: the later stages' own retained K/V, one per stage, same extents (empty: none)std::vector<ConversationKvReuse> stages;+ // The conversation the image was captured from (the first stage's only): a park reuses it only for the prefix it+ // shares with the conversation then in the main session (a batch slot may have replaced that session's K/V since).+ std::vector<int32_t> ids;size_t bytes() const {- size_t n = kv.capacity() * sizeof(ConversationKv) + stages.capacity() * sizeof(ConversationKvReuse);+ size_t n = kv.capacity() * sizeof(ConversationKv) + stages.capacity() * sizeof(ConversationKvReuse) ++ ids.capacity() * sizeof(int32_t);for (const auto& layer : kv) n += layer.bytes();for (const auto& s : stages) n += s.bytes();return n;@@ -228,11 +232,12 @@ public:// Retain only the restored K/V buffers, not duplicate running checkpoints.// This optimization never evicts a parked conversation to make itself fit.// `stage_kv`: with a layer split, the later stages' restored K/V (one per stage), retained with the first's.+ // `ids`: the conversation the K/V belongs to (take_reuse_for refuses it for any other; none: never reused).void retain(std::vector<ConversationKv>&& kv, int64_t tokens,- std::vector<std::vector<ConversationKv>>&& stage_kv = {}) {+ std::vector<std::vector<ConversationKv>>&& stage_kv = {}, std::vector<int32_t> ids = {}) {reuse_ = {};- ConversationKvReuse candidate{std::move(kv), tokens, tokens, {}};- for (auto& k : stage_kv) candidate.stages.push_back(ConversationKvReuse{std::move(k), tokens, tokens, {}});+ ConversationKvReuse candidate{std::move(kv), tokens, tokens, {}, std::move(ids)};+ for (auto& k : stage_kv) candidate.stages.push_back(ConversationKvReuse{std::move(k), tokens, tokens, {}, {}});if (enabled() && candidate.bytes() <= budget_ - bytes_) reuse_ = std::move(candidate);}void limit_reuse(int64_t first_dirty) {@@ -241,7 +246,22 @@ public:if (reuse_.unchanged_tokens <= 0) reuse_ = {};}ConversationKvReuse take_reuse() { return std::exchange(reuse_, {}); }+ // The retained K/V for parking `live`: limited to the prefix `live` shares with the conversation it was captured+ // from, and nothing when they share none. Without this a park after a batch slot gave its conversation back to the+ // main session wrote the RESTORED conversation's K/V into the new one's image (#1503, a cross-conversation leak).+ template<class Token>+ ConversationKvReuse take_reuse_for(const std::vector<Token>& live) {+ ConversationKvReuse r = std::exchange(reuse_, {});+ const int64_t n = std::min<int64_t>((int64_t) r.ids.size(), (int64_t) live.size());+ int64_t same = 0;+ while (same < n && (int64_t) r.ids[(size_t) same] == (int64_t) live[(size_t) same]) ++same;+ r.unchanged_tokens = std::min(r.unchanged_tokens, same);+ for (auto& st : r.stages) st.unchanged_tokens = std::min(st.unchanged_tokens, same);+ if (r.unchanged_tokens <= 0) return {};+ return r;+ }size_t retained_bytes() const { return reuse_.bytes(); }+ int64_t retained_tokens() const { return reuse_.kv.empty() ? 0 : reuse_.unchanged_tokens; }bool can_fit(size_t incoming, size_t held = 0) const {return enabled() && held <= budget_ && incoming <= budget_ - held &&entries_.size() < slots_ && bytes() <= budget_ - held - incoming;diff --git a/src/core/conversation_cache_test.cpp b/src/core/conversation_cache_test.cppindex a21c0d7fe..8c1e61c37 100644--- a/src/core/conversation_cache_test.cpp+++ b/src/core/conversation_cache_test.cpp@@ -502,6 +502,64 @@ int main() {auto none = conversation_checkpoints_split({}, 3);check(conversation_checkpoints_merge(std::move(none), back) && back.empty(), "no checkpoints: nothing to do");}+ {+ // #1503, the cross-conversation leak: the retained K/V of a restored conversation must never be reused to+ // park ANOTHER conversation the main session later holds (a batch slot gave its conversation back)+ auto kv = [](size_t n) {+ std::vector<ConversationKv> v(2);+ for (auto& l : v) l.k.resize(n, 1);+ return v;+ };+ ConversationCache cache(1 << 20, 4);+ const std::vector<int32_t> a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};+ std::vector<std::vector<ConversationKv>> st;+ st.push_back(kv(100));+ cache.retain(kv(500), 16, std::move(st), a);+ std::vector<int32_t> b(16, 99);+ auto other = cache.take_reuse_for(b);+ check(other.kv.empty() && other.stages.empty() && other.unchanged_tokens == 0,+ "another conversation never gets the retained K/V");+ check(cache.retained_bytes() == 0, "the retained K/V is dropped once refused");+ cache.retain(kv(500), 16, {}, a);+ check(cache.retained_tokens() == 16, "the retained extent is reported");+ std::vector<int32_t> cont(a.begin(), a.begin() + 10);+ cont.push_back(42);+ cont.push_back(43);+ auto partly = cache.take_reuse_for(cont);+ check(partly.unchanged_tokens == 10 && !partly.kv.empty(), "a conversation that diverges reuses only the shared prefix");+ std::vector<std::vector<ConversationKv>> st2;+ st2.push_back(kv(100));+ cache.retain(kv(500), 16, std::move(st2), a);+ cache.limit_reuse(9);+ auto same = cache.take_reuse_for(a);+ check(same.unchanged_tokens == 9 && same.stages.size() == 1 && same.stages[0].unchanged_tokens == 9,+ "the same conversation keeps the first rewrite's limit, on every stage");+ cache.retain(kv(500), 16);+ check(cache.take_reuse_for(a).kv.empty(), "retained K/V of unknown identity is never reused");++ // the sequence of the report: P restored (retained), a slot hands Q to the main session, Q is parked. Q shares+ // only the system prompt with P (4 tokens): with the slot's drop, nothing is reused; without it (the ids check+ // alone), at most the shared system prompt+ const std::vector<int32_t> p = {7, 7, 7, 7, 100, 101, 102, 103, 104, 105};+ const std::vector<int32_t> q = {7, 7, 7, 7, 200, 201, 202, 203, 204, 205, 206, 207};+ std::vector<std::vector<ConversationKv>> stp;+ stp.push_back(kv(100));+ cache.retain(kv(500), 10, std::move(stp), p);+ (void) cache.take_reuse(); // copy_from_slot+ check(cache.take_reuse_for(q).kv.empty(), "a slot that gave its conversation back drops the retained K/V");+ std::vector<std::vector<ConversationKv>> stp2;+ stp2.push_back(kv(100));+ cache.retain(kv(500), 10, std::move(stp2), p);+ auto sys_only = cache.take_reuse_for(q);+ check(sys_only.unchanged_tokens == 4 && sys_only.stages.size() == 1 && sys_only.stages[0].unchanged_tokens == 4,+ "without the drop, only the shared system prompt is reused, on every stage");+ check(cache.retained_bytes() == 0, "taken once");+ // a conversation that is a strict extension of the retained one keeps the whole extent+ cache.retain(kv(500), 10, {}, p);+ std::vector<int32_t> p_more = p;+ p_more.push_back(300);+ check(cache.take_reuse_for(p_more).unchanged_tokens == 10, "the conversation's own continuation reuses all of it");+ }{// the later stages' retained K/V, kept with the first stage's and limited togetherauto kv = [](size_t n) {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex a4732a1c6..53a6085c6 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -9632,7 +9632,12 @@ int main(int argc, char** argv) {~MergeBack() { if (on && !strata::core::conversation_checkpoints_merge(std::move(cs), checks)) checks.clear(); }} merge_back{cs, checks, n_st > 0};const strata::core::ConversationView view{live, live_imgs, n_st > 0 ? cs.stage0 : checks, cvec_cached};- auto reuse = conversations.take_reuse();+ const int64_t retained_n = conversations.retained_tokens();+ auto reuse = conversations.take_reuse_for(live); // never another conversation's K/V+ if (retained_n > 0 && reuse.unchanged_tokens < retained_n)+ std::fprintf(stderr, "strata serve: conversation cache: retained K/V reused for %lld of its %lld tokens "+ "(the rest is not this conversation's)\n", (long long) reuse.unchanged_tokens,+ (long long) retained_n);std::vector<strata::core::ConversationKvReuse> stage_reuse = std::move(reuse.stages);stage_reuse.resize(n_st);reuse.stages.clear();@@ -11607,6 +11612,8 @@ int main(int argc, char** argv) {// conversation an idle slot holds reads on from there. Every stage, the same calls.// `at`: one of the slot's checkpoints - only the K/V up to it is copied and its state restored insteadauto copy_from_slot = [&](int b, const ConvCheckpoint* at, std::string& e) -> bool {+ // the main session's K/V is replaced by the slot's: K/V retained from a restore described the old one+ conversations.take_reuse();const int64_t upto = at != nullptr ? (int64_t) at->ids.size() : (int64_t) bs[(size_t) b].ids.size();if (at != nullptr && at->stage_parts.size() != stages.size()) { e = "a checkpoint without its stage parts"; return false; }if (slot_copy_direct && at == nullptr) {@@ -13048,7 +13055,7 @@ int main(int argc, char** argv) {if (std::getenv("STRATA_SNAPSHOT_FULL_CAPTURE") == nullptr) {std::vector<std::vector<strata::core::ConversationKv>> stage_kv; // every stage's, for its next parkfor (auto& si : incoming->stage_images) stage_kv.push_back(std::move(si.kv));- conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv));+ conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv), live);}incoming.reset(); // Running-state/checkpoint copies are no longer needed.lent_slot = -1; // (the main session holds the parked conversation now)diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cppindex 2a4db4d16..5a111ea47 100644--- a/sycl/src/program/generate.cpp+++ b/sycl/src/program/generate.cpp@@ -7694,7 +7694,12 @@ int main(int argc, char **argv) try {~MergeBack() { if (on && !strata::core::conversation_checkpoints_merge(std::move(cs), checks)) checks.clear(); }} merge_back{cs, checks, n_st > 0};const strata::core::ConversationView view{live, live_imgs, n_st > 0 ? cs.stage0 : checks, cvec_cached};- auto reuse = conversations.take_reuse();+ const int64_t retained_n = conversations.retained_tokens();+ auto reuse = conversations.take_reuse_for(live); // never another conversation's K/V+ if (retained_n > 0 && reuse.unchanged_tokens < retained_n)+ std::fprintf(stderr, "strata serve: conversation cache: retained K/V reused for %lld of its %lld tokens "+ "(the rest is not this conversation's)\n", (long long) reuse.unchanged_tokens,+ (long long) retained_n);std::vector<strata::core::ConversationKvReuse> stage_reuse = std::move(reuse.stages);stage_reuse.resize(n_st);reuse.stages.clear();@@ -9051,6 +9056,8 @@ int main(int argc, char **argv) try {// `at`: one of the slot's checkpoints - only the K/V up to it is copied and its state restored insteadauto copy_from_slot = [&](int b, const ConvCheckpoint *at,std::string &e) -> bool {+ // the main session's K/V is replaced by the slot's: K/V retained from a restore described the old one+ conversations.take_reuse();try {const int64_t upto = at != nullptr ? (int64_t)at->ids.size(): (int64_t)bs[(size_t)b].ids.size();@@ -10063,7 +10070,7 @@ int main(int argc, char **argv) try {if (std::getenv("STRATA_SNAPSHOT_FULL_CAPTURE") == nullptr) {std::vector<std::vector<strata::core::ConversationKv>> stage_kv; // every stage's, for its next parkfor (auto& si : incoming->stage_images) stage_kv.push_back(std::move(si.kv));- conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv));+ conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv), live);}incoming.reset(); // Running-state/checkpoint copies are no longer needed.std::fprintf(stderr, "strata serve: conversation cache: restored %lld tokens (%s) in %.1f ms; parked=%zu bytes=%zu\n",