Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: a slot's shorter checkpoints come along at a give-back only when they are prefixes of the tokens copied back (#2526)

Niko1221cea3ed40db98
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 77efc0709..4ce6a3dc0 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -13061,6 +13061,13 @@ int main(int argc, char** argv) {
const bool move_cks = slot_ckpt_move_env() && admit_slot == slot_source;
const int64_t ck_len = slot_ck != nullptr ? (int64_t) slot_ck->ids.size() : -1;
std::vector<ConvCheckpoint>& sc = bs[(size_t) slot_source].checks;
+ // a shorter checkpoint comes along only when it is a prefix of the tokens copied back (text only):
+ // its cells are then the ones the main session holds
+ const std::vector<int32_t> copied_back = slot_ck != nullptr ? slot_ck->ids : bs[(size_t) slot_source].ids;
+ auto prefix_of_copied = [&](const ConvCheckpoint& c) {
+ return c.imgs.empty() && c.ids.size() <= copied_back.size() &&
+ std::equal(c.ids.begin(), c.ids.end(), copied_back.begin());
+ };
// the one continued from first, exactly as before this change (its copy is the one the request needs)
const ConvCheckpoint* first = slot_ck;
if (first != nullptr) {
@@ -13075,7 +13082,7 @@ int main(int argc, char** argv) {
// then the shorter ones: an optimization, so a host allocation that fails skips them
try {
for (auto it = sc.begin(); it != sc.end();) {
- if ((int64_t) it->ids.size() > slot_tokens || &*it == first) {
+ if ((int64_t) it->ids.size() > slot_tokens || &*it == first || !prefix_of_copied(*it)) {
++it;
} else if (move_cks) {
checks.push_back(std::move(*it));
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cpp
index 893e7d6bf..3e257771b 100644
--- a/sycl/src/program/generate.cpp
+++ b/sycl/src/program/generate.cpp
@@ -10116,8 +10116,12 @@ int main(int argc, char **argv) try {
// copies of this conversation a turn back - whose deepest checkpoint the chain holds again - go
// as superseded (#342) instead of taking a parked entry each until the cache evicts another chat
try { // an optimization: a host allocation that fails skips the rest
+ // (a shorter checkpoint only when it is a prefix of the tokens copied back, text only)
+ const std::vector<int32_t>& copied_back = slot_ck != nullptr ? slot_ck->ids : bs[(size_t) slot_source].ids;
for (const ConvCheckpoint& c : bs[(size_t) slot_source].checks)
- if ((int64_t) c.ids.size() <= slot_tokens) checks.push_back(c);
+ if ((int64_t) c.ids.size() <= slot_tokens && c.imgs.empty() && c.ids.size() <= copied_back.size() &&
+ std::equal(c.ids.begin(), c.ids.end(), copied_back.begin()))
+ checks.push_back(c);
} catch (const std::bad_alloc&) {
std::fprintf(stderr, "strata batch: slot %d: its checkpoints were not all copied (out of memory); the rest are skipped\n", slot_source);
}