Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Conversation cache: parking an idle --batch slot no longer drops the main session's copy with the shared-prefix checkpoint (#4420)

Niko1221c7a0cef13a7b
diff --git a/include/strata/core/conversation_cache.hpp b/include/strata/core/conversation_cache.hpp
index 348983453..78ff9f189 100644
--- a/include/strata/core/conversation_cache.hpp
+++ b/include/strata/core/conversation_cache.hpp
@@ -465,6 +465,33 @@ public:
superseded_ += dropped;
return dropped;
}
+ // A batch slot's chain holds one checkpoint (the one its admission copied in), while the main session's parked copy of the same
+ // conversation keeps the whole chain - the root checkpoint at the end of an agent's system prompt and tools among them, the one its
+ // next conversations borrow (borrows()). Parking the slot (#2526) would drop that copy as superseded (#342: the slot's chain holds its
+ // deepest checkpoint) and the root with it: every sibling then reads the shared prefix again (#4420, 0.2.6 -> 0.2.7: 323 ms -> 4130 ms
+ // for a 5 K-token prefix). True when drop_superseded(ids, images, checkpoints, cvec) would drop an entry that has a checkpoint the
+ // given chain does not hold. Nothing is changed.
+ bool supersedes_richer(const std::vector<int32_t>& ids, const std::vector<ConversationImageKey>& images,
+ const std::vector<ConversationCheckpoint>& checkpoints, bool cvec) const {
+ auto held = [&](const ConversationCheckpoint& c) {
+ if (c.ids == ids && c.imgs == images) return true;
+ for (const auto& k : checkpoints)
+ if (k.ids == c.ids && k.imgs == c.imgs) return true;
+ return false;
+ };
+ for (size_t i = 0; i < entries_.size(); ++i) {
+ const auto& e = entries_[i];
+ const ConversationCheckpoint* deepest = nullptr;
+ for (const auto& c : e.checkpoints)
+ if (!deepest || c.ids.size() > deepest->ids.size()) deepest = &c;
+ if (e.cvec != cvec || !deepest || deepest->ids.empty() || !held(*deepest) ||
+ (guarded_ != 0 && serials_[i] == guarded_))
+ continue;
+ for (const auto& c : e.checkpoints)
+ if (!held(c)) return true;
+ }
+ return false;
+ }
size_t superseded() const { return superseded_; }
/// --conversation-cache-park-gain N: the longest parked entry this conversation's outgoing state supersedes, in
/// tokens, 0 when it has none. It is the entry drop_superseded() would remove, so the caller can tell whether a
diff --git a/src/core/conversation_cache_test.cpp b/src/core/conversation_cache_test.cpp
index 242e3760f..7c4dba46d 100644
--- a/src/core/conversation_cache_test.cpp
+++ b/src/core/conversation_cache_test.cpp
@@ -720,5 +720,30 @@ int main() {
check(tiny.put(std::move(p2)), "a pinned one fits an empty cache");
check(!tiny.put(image({7,8,9})) && tiny.size() == 1, "nothing else fits beside it: parking is refused, the pin stays");
}
+ {
+ // #4420: a batch slot's chain holds one checkpoint (its admission's): parking it must not drop (#342) the main session's copy of the
+ // same conversation that also holds the root a sibling borrows from (an agent's system prompt and tools) - supersedes_richer()
+ // names that case, so the slot is not parked
+ auto cp = [](std::vector<int32_t> ids) { ConversationCheckpoint c; c.ids = std::move(ids); return c; };
+ SavedConversation main_copy = image({1, 2, 3, 4, 10, 11, 12});
+ main_copy.checkpoints = {cp({1, 2, 3, 4}), cp({1, 2, 3, 4, 10, 11})};
+ ConversationCache cache(1 << 20, 4);
+ check(cache.put(std::move(main_copy)), "park the main session's copy with its root and turn checkpoints");
+ const std::vector<int32_t> slot_ids{1, 2, 3, 4, 10, 11, 12, 20, 21}; // the slot decoded the reply
+ const std::vector<ConversationCheckpoint> slot_chain{cp({1, 2, 3, 4, 10, 11})}; // the admission's only one
+ check(cache.supersedes_richer(slot_ids, {}, slot_chain, true), "the slot's park would drop the root checkpoint");
+ const std::vector<ConversationCheckpoint> full_chain{cp({1, 2, 3, 4}), cp({1, 2, 3, 4, 10, 11})};
+ check(!cache.supersedes_richer(slot_ids, {}, full_chain, true), "a chain that holds both loses nothing");
+ check(!cache.supersedes_richer(slot_ids, {}, slot_chain, false), "steering mode is isolated");
+ check(!cache.supersedes_richer(std::vector<int32_t>{1, 2, 3, 4, 30, 31}, {},
+ std::vector<ConversationCheckpoint>{cp({1, 2, 3, 4, 30})}, true),
+ "another conversation on the same root supersedes nothing");
+ check(cache.size() == 1 && cache.superseded() == 0, "the predicate changes nothing");
+ const auto sib = cache.best(std::vector<int64_t>{1, 2, 3, 4, 40}, {}, true);
+ check(sib.tokens == 4 && cache.borrows(sib), "a sibling still borrows the root");
+ // what drop_superseded does with the slot's chain (the park this rule prevents): the root goes
+ check(cache.drop_superseded(slot_ids, {}, slot_chain, true) == 1 && cache.size() == 0,
+ "the slot's chain supersedes the richer copy");
+ }
std::printf("conversation_cache_test: %d checks passed\n", checks);
}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 824bd19db..59f384285 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -12609,6 +12609,13 @@ int main(int argc, char** argv) {
const bool in_main = live_ok && cvec_cached == sl.cvec && prefix_of(live) &&
imgs_below(live_imgs, (int64_t) sl.ids.size()).empty();
if (in_main || conversations.holds(sl.ids, {}, sl.cvec)) return;
+ if (conversations.supersedes_richer(sl.ids, {}, sl.checks, sl.cvec)) {
+ // its park would drop (as superseded) the main session's copy of this conversation, and with it the checkpoints the slot
+ // does not hold - an agent's system-prompt root that the next requests borrow (#4420, as 0.2.6 did: the slot is overwritten)
+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (a parked copy of its "
+ "conversation holds checkpoints the slot does not; it stays)\n", b);
+ return;
+ }
if (use_mtp && !batch_mtp) { // no draft K/V of its own: an image without it would not restore beside the drafter
std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (its %zu tokens "
"have no per-slot draft K/V without --batch-mtp)\n", b, sl.ids.size());