Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

conversation cache: never park one conversation with another's retained K/V (#1503)

Stefanos Anastasiouf8ff3420d997
diff --git a/include/strata/core/conversation_cache.hpp b/include/strata/core/conversation_cache.hpp
index b2e6a74f7..973b25938 100644
--- a/include/strata/core/conversation_cache.hpp
+++ b/include/strata/core/conversation_cache.hpp
@@ -69,8 +69,12 @@ struct ConversationKvReuse {
int64_t captured_tokens = 0, unchanged_tokens = 0;
// With a layer split: the later stages' own retained K/V, one per stage, same extents (empty: none)
std::vector<ConversationKvReuse> stages;
+ // The conversation the image was captured from (the first stage's only): a park reuses it only for the prefix it
+ // shares with the conversation then in the main session (a batch slot may have replaced that session's K/V since).
+ std::vector<int32_t> ids;
size_t bytes() const {
- size_t n = kv.capacity() * sizeof(ConversationKv) + stages.capacity() * sizeof(ConversationKvReuse);
+ size_t n = kv.capacity() * sizeof(ConversationKv) + stages.capacity() * sizeof(ConversationKvReuse) +
+ ids.capacity() * sizeof(int32_t);
for (const auto& layer : kv) n += layer.bytes();
for (const auto& s : stages) n += s.bytes();
return n;
@@ -228,11 +232,12 @@ public:
// Retain only the restored K/V buffers, not duplicate running checkpoints.
// This optimization never evicts a parked conversation to make itself fit.
// `stage_kv`: with a layer split, the later stages' restored K/V (one per stage), retained with the first's.
+ // `ids`: the conversation the K/V belongs to (take_reuse_for refuses it for any other; none: never reused).
void retain(std::vector<ConversationKv>&& kv, int64_t tokens,
- std::vector<std::vector<ConversationKv>>&& stage_kv = {}) {
+ std::vector<std::vector<ConversationKv>>&& stage_kv = {}, std::vector<int32_t> ids = {}) {
reuse_ = {};
- ConversationKvReuse candidate{std::move(kv), tokens, tokens, {}};
- for (auto& k : stage_kv) candidate.stages.push_back(ConversationKvReuse{std::move(k), tokens, tokens, {}});
+ ConversationKvReuse candidate{std::move(kv), tokens, tokens, {}, std::move(ids)};
+ for (auto& k : stage_kv) candidate.stages.push_back(ConversationKvReuse{std::move(k), tokens, tokens, {}, {}});
if (enabled() && candidate.bytes() <= budget_ - bytes_) reuse_ = std::move(candidate);
}
void limit_reuse(int64_t first_dirty) {
@@ -241,7 +246,22 @@ public:
if (reuse_.unchanged_tokens <= 0) reuse_ = {};
}
ConversationKvReuse take_reuse() { return std::exchange(reuse_, {}); }
+ // The retained K/V for parking `live`: limited to the prefix `live` shares with the conversation it was captured
+ // from, and nothing when they share none. Without this a park after a batch slot gave its conversation back to the
+ // main session wrote the RESTORED conversation's K/V into the new one's image (#1503, a cross-conversation leak).
+ template<class Token>
+ ConversationKvReuse take_reuse_for(const std::vector<Token>& live) {
+ ConversationKvReuse r = std::exchange(reuse_, {});
+ const int64_t n = std::min<int64_t>((int64_t) r.ids.size(), (int64_t) live.size());
+ int64_t same = 0;
+ while (same < n && (int64_t) r.ids[(size_t) same] == (int64_t) live[(size_t) same]) ++same;
+ r.unchanged_tokens = std::min(r.unchanged_tokens, same);
+ for (auto& st : r.stages) st.unchanged_tokens = std::min(st.unchanged_tokens, same);
+ if (r.unchanged_tokens <= 0) return {};
+ return r;
+ }
size_t retained_bytes() const { return reuse_.bytes(); }
+ int64_t retained_tokens() const { return reuse_.kv.empty() ? 0 : reuse_.unchanged_tokens; }
bool can_fit(size_t incoming, size_t held = 0) const {
return enabled() && held <= budget_ && incoming <= budget_ - held &&
entries_.size() < slots_ && bytes() <= budget_ - held - incoming;
diff --git a/src/core/conversation_cache_test.cpp b/src/core/conversation_cache_test.cpp
index a21c0d7fe..8c1e61c37 100644
--- a/src/core/conversation_cache_test.cpp
+++ b/src/core/conversation_cache_test.cpp
@@ -502,6 +502,64 @@ int main() {
auto none = conversation_checkpoints_split({}, 3);
check(conversation_checkpoints_merge(std::move(none), back) && back.empty(), "no checkpoints: nothing to do");
}
+ {
+ // #1503, the cross-conversation leak: the retained K/V of a restored conversation must never be reused to
+ // park ANOTHER conversation the main session later holds (a batch slot gave its conversation back)
+ auto kv = [](size_t n) {
+ std::vector<ConversationKv> v(2);
+ for (auto& l : v) l.k.resize(n, 1);
+ return v;
+ };
+ ConversationCache cache(1 << 20, 4);
+ const std::vector<int32_t> a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+ std::vector<std::vector<ConversationKv>> st;
+ st.push_back(kv(100));
+ cache.retain(kv(500), 16, std::move(st), a);
+ std::vector<int32_t> b(16, 99);
+ auto other = cache.take_reuse_for(b);
+ check(other.kv.empty() && other.stages.empty() && other.unchanged_tokens == 0,
+ "another conversation never gets the retained K/V");
+ check(cache.retained_bytes() == 0, "the retained K/V is dropped once refused");
+ cache.retain(kv(500), 16, {}, a);
+ check(cache.retained_tokens() == 16, "the retained extent is reported");
+ std::vector<int32_t> cont(a.begin(), a.begin() + 10);
+ cont.push_back(42);
+ cont.push_back(43);
+ auto partly = cache.take_reuse_for(cont);
+ check(partly.unchanged_tokens == 10 && !partly.kv.empty(), "a conversation that diverges reuses only the shared prefix");
+ std::vector<std::vector<ConversationKv>> st2;
+ st2.push_back(kv(100));
+ cache.retain(kv(500), 16, std::move(st2), a);
+ cache.limit_reuse(9);
+ auto same = cache.take_reuse_for(a);
+ check(same.unchanged_tokens == 9 && same.stages.size() == 1 && same.stages[0].unchanged_tokens == 9,
+ "the same conversation keeps the first rewrite's limit, on every stage");
+ cache.retain(kv(500), 16);
+ check(cache.take_reuse_for(a).kv.empty(), "retained K/V of unknown identity is never reused");
+
+ // the sequence of the report: P restored (retained), a slot hands Q to the main session, Q is parked. Q shares
+ // only the system prompt with P (4 tokens): with the slot's drop, nothing is reused; without it (the ids check
+ // alone), at most the shared system prompt
+ const std::vector<int32_t> p = {7, 7, 7, 7, 100, 101, 102, 103, 104, 105};
+ const std::vector<int32_t> q = {7, 7, 7, 7, 200, 201, 202, 203, 204, 205, 206, 207};
+ std::vector<std::vector<ConversationKv>> stp;
+ stp.push_back(kv(100));
+ cache.retain(kv(500), 10, std::move(stp), p);
+ (void) cache.take_reuse(); // copy_from_slot
+ check(cache.take_reuse_for(q).kv.empty(), "a slot that gave its conversation back drops the retained K/V");
+ std::vector<std::vector<ConversationKv>> stp2;
+ stp2.push_back(kv(100));
+ cache.retain(kv(500), 10, std::move(stp2), p);
+ auto sys_only = cache.take_reuse_for(q);
+ check(sys_only.unchanged_tokens == 4 && sys_only.stages.size() == 1 && sys_only.stages[0].unchanged_tokens == 4,
+ "without the drop, only the shared system prompt is reused, on every stage");
+ check(cache.retained_bytes() == 0, "taken once");
+ // a conversation that is a strict extension of the retained one keeps the whole extent
+ cache.retain(kv(500), 10, {}, p);
+ std::vector<int32_t> p_more = p;
+ p_more.push_back(300);
+ check(cache.take_reuse_for(p_more).unchanged_tokens == 10, "the conversation's own continuation reuses all of it");
+ }
{
// the later stages' retained K/V, kept with the first stage's and limited together
auto kv = [](size_t n) {
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index a4732a1c6..53a6085c6 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -9632,7 +9632,12 @@ int main(int argc, char** argv) {
~MergeBack() { if (on && !strata::core::conversation_checkpoints_merge(std::move(cs), checks)) checks.clear(); }
} merge_back{cs, checks, n_st > 0};
const strata::core::ConversationView view{live, live_imgs, n_st > 0 ? cs.stage0 : checks, cvec_cached};
- auto reuse = conversations.take_reuse();
+ const int64_t retained_n = conversations.retained_tokens();
+ auto reuse = conversations.take_reuse_for(live); // never another conversation's K/V
+ if (retained_n > 0 && reuse.unchanged_tokens < retained_n)
+ std::fprintf(stderr, "strata serve: conversation cache: retained K/V reused for %lld of its %lld tokens "
+ "(the rest is not this conversation's)\n", (long long) reuse.unchanged_tokens,
+ (long long) retained_n);
std::vector<strata::core::ConversationKvReuse> stage_reuse = std::move(reuse.stages);
stage_reuse.resize(n_st);
reuse.stages.clear();
@@ -11607,6 +11612,8 @@ int main(int argc, char** argv) {
// conversation an idle slot holds reads on from there. Every stage, the same calls.
// `at`: one of the slot's checkpoints - only the K/V up to it is copied and its state restored instead
auto copy_from_slot = [&](int b, const ConvCheckpoint* at, std::string& e) -> bool {
+ // the main session's K/V is replaced by the slot's: K/V retained from a restore described the old one
+ conversations.take_reuse();
const int64_t upto = at != nullptr ? (int64_t) at->ids.size() : (int64_t) bs[(size_t) b].ids.size();
if (at != nullptr && at->stage_parts.size() != stages.size()) { e = "a checkpoint without its stage parts"; return false; }
if (slot_copy_direct && at == nullptr) {
@@ -13048,7 +13055,7 @@ int main(int argc, char** argv) {
if (std::getenv("STRATA_SNAPSHOT_FULL_CAPTURE") == nullptr) {
std::vector<std::vector<strata::core::ConversationKv>> stage_kv; // every stage's, for its next park
for (auto& si : incoming->stage_images) stage_kv.push_back(std::move(si.kv));
- conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv));
+ conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv), live);
}
incoming.reset(); // Running-state/checkpoint copies are no longer needed.
lent_slot = -1; // (the main session holds the parked conversation now)
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cpp
index 2a4db4d16..5a111ea47 100644
--- a/sycl/src/program/generate.cpp
+++ b/sycl/src/program/generate.cpp
@@ -7694,7 +7694,12 @@ int main(int argc, char **argv) try {
~MergeBack() { if (on && !strata::core::conversation_checkpoints_merge(std::move(cs), checks)) checks.clear(); }
} merge_back{cs, checks, n_st > 0};
const strata::core::ConversationView view{live, live_imgs, n_st > 0 ? cs.stage0 : checks, cvec_cached};
- auto reuse = conversations.take_reuse();
+ const int64_t retained_n = conversations.retained_tokens();
+ auto reuse = conversations.take_reuse_for(live); // never another conversation's K/V
+ if (retained_n > 0 && reuse.unchanged_tokens < retained_n)
+ std::fprintf(stderr, "strata serve: conversation cache: retained K/V reused for %lld of its %lld tokens "
+ "(the rest is not this conversation's)\n", (long long) reuse.unchanged_tokens,
+ (long long) retained_n);
std::vector<strata::core::ConversationKvReuse> stage_reuse = std::move(reuse.stages);
stage_reuse.resize(n_st);
reuse.stages.clear();
@@ -9051,6 +9056,8 @@ int main(int argc, char **argv) try {
// `at`: one of the slot's checkpoints - only the K/V up to it is copied and its state restored instead
auto copy_from_slot = [&](int b, const ConvCheckpoint *at,
std::string &e) -> bool {
+ // the main session's K/V is replaced by the slot's: K/V retained from a restore described the old one
+ conversations.take_reuse();
try {
const int64_t upto = at != nullptr ? (int64_t)at->ids.size()
: (int64_t)bs[(size_t)b].ids.size();
@@ -10063,7 +10070,7 @@ int main(int argc, char **argv) try {
if (std::getenv("STRATA_SNAPSHOT_FULL_CAPTURE") == nullptr) {
std::vector<std::vector<strata::core::ConversationKv>> stage_kv; // every stage's, for its next park
for (auto& si : incoming->stage_images) stage_kv.push_back(std::move(si.kv));
- conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv));
+ conversations.retain(std::move(incoming->kv), int64_t(live.size()), std::move(stage_kv), live);
}
incoming.reset(); // Running-state/checkpoint copies are no longer needed.
std::fprintf(stderr, "strata serve: conversation cache: restored %lld tokens (%s) in %.1f ms; parked=%zu bytes=%zu\n",