Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: a slot's shorter checkpoints come along at a give-back only when they are prefixes of the tokens copied back (#2526)
Niko1221cea3ed40db98
diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 77efc0709..4ce6a3dc0 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -13061,6 +13061,13 @@ int main(int argc, char** argv) {const bool move_cks = slot_ckpt_move_env() && admit_slot == slot_source;const int64_t ck_len = slot_ck != nullptr ? (int64_t) slot_ck->ids.size() : -1;std::vector<ConvCheckpoint>& sc = bs[(size_t) slot_source].checks;+ // a shorter checkpoint comes along only when it is a prefix of the tokens copied back (text only):+ // its cells are then the ones the main session holds+ const std::vector<int32_t> copied_back = slot_ck != nullptr ? slot_ck->ids : bs[(size_t) slot_source].ids;+ auto prefix_of_copied = [&](const ConvCheckpoint& c) {+ return c.imgs.empty() && c.ids.size() <= copied_back.size() &&+ std::equal(c.ids.begin(), c.ids.end(), copied_back.begin());+ };// the one continued from first, exactly as before this change (its copy is the one the request needs)const ConvCheckpoint* first = slot_ck;if (first != nullptr) {@@ -13075,7 +13082,7 @@ int main(int argc, char** argv) {// then the shorter ones: an optimization, so a host allocation that fails skips themtry {for (auto it = sc.begin(); it != sc.end();) {- if ((int64_t) it->ids.size() > slot_tokens || &*it == first) {+ if ((int64_t) it->ids.size() > slot_tokens || &*it == first || !prefix_of_copied(*it)) {++it;} else if (move_cks) {checks.push_back(std::move(*it));diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cppindex 893e7d6bf..3e257771b 100644--- a/sycl/src/program/generate.cpp+++ b/sycl/src/program/generate.cpp@@ -10116,8 +10116,12 @@ int main(int argc, char **argv) try {// copies of this conversation a turn back - whose deepest checkpoint the chain holds again - go// as superseded (#342) instead of taking a parked entry each until the cache evicts another chattry { // an optimization: a host allocation that fails skips the rest+ // (a shorter checkpoint only when it is a prefix of the tokens copied back, text only)+ const std::vector<int32_t>& copied_back = slot_ck != nullptr ? slot_ck->ids : bs[(size_t) slot_source].ids;for (const ConvCheckpoint& c : bs[(size_t) slot_source].checks)- if ((int64_t) c.ids.size() <= slot_tokens) checks.push_back(c);+ if ((int64_t) c.ids.size() <= slot_tokens && c.imgs.empty() && c.ids.size() <= copied_back.size() &&+ std::equal(c.ids.begin(), c.ids.end(), copied_back.begin()))+ checks.push_back(c);} catch (const std::bad_alloc&) {std::fprintf(stderr, "strata batch: slot %d: its checkpoints were not all copied (out of memory); the rest are skipped\n", slot_source);}