Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: an idle batch slot's decoded conversation is parked before an admission overwrites it (#2526)
hotschmoe01e404c14023
diff --git a/include/strata/core/conversation_cache.hpp b/include/strata/core/conversation_cache.hppindex 973b25938..b977ac492 100644--- a/include/strata/core/conversation_cache.hpp+++ b/include/strata/core/conversation_cache.hpp@@ -187,6 +187,24 @@ struct SavedConversation {}};+/// `c` holds the state `ids` (with `images`) describes, or a continuation of it: its tokens start with `ids` and+/// its pictures below ids.size() are exactly `images`. What an idle batch slot holds is then parked already (or+/// a longer continuation is), so the slot need not park it again before an admission overwrites it.+template<class Token>+bool conversation_covers(const ConversationCheckpoint& c, const std::vector<Token>& ids,+ const std::vector<ConversationImageKey>& images) {+ const size_t n = ids.size();+ if (n == 0 || c.ids.size() < n) return false;+ for (size_t i = 0; i < n; ++i)+ if ((int64_t) c.ids[i] != (int64_t) ids[i]) return false;+ size_t j = 0;+ for (const auto& image : c.imgs) {+ if (image.start >= (int64_t) n) continue;+ if (j == images.size() || !(images[j++] == image)) return false;+ }+ return j == images.size();+}+template<class Token>int64_t conversation_prefix(const ConversationCheckpoint& c, const std::vector<Token>& prompt,const std::vector<ConversationImageKey>& images) {@@ -285,6 +303,19 @@ public:return best;}+ /// A parked conversation holds this state already (its live tokens or one of its checkpoints start with `ids`,+ /// same pictures, same steering): see conversation_covers. Read-only; no LRU effect.+ template<class Token>+ bool holds(const std::vector<Token>& ids, const std::vector<ConversationImageKey>& images, bool cvec) const {+ for (const auto& e : entries_) {+ if (e.cvec != cvec) continue;+ if (conversation_covers(e.live, ids, images)) return true;+ for (const auto& c : e.checkpoints)+ if (conversation_covers(c, ids, images)) return true;+ }+ return false;+ }+SavedConversation take(size_t index) {SavedConversation out = std::move(entries_.at(index));bytes_ -= out.bytes();diff --git a/src/core/conversation_cache_test.cpp b/src/core/conversation_cache_test.cppindex 8c1e61c37..242e3760f 100644--- a/src/core/conversation_cache_test.cpp+++ b/src/core/conversation_cache_test.cpp@@ -253,6 +253,110 @@ int main() {const auto s = cache.best(with(history, {40}), {}, true);check(s.tokens == (int64_t) history.size() - 2, "SUB resumes from its last turn boundary");}+ {+ // holds(): what an idle batch slot holds is parked already (or a continuation of it is)+ auto s = image({1, 2, 3, 4, 5});+ ConversationCheckpoint cp;+ cp.ids = {1, 2, 3};+ s.checkpoints.push_back(cp);+ ConversationCache cache(4096, 4);+ cache.put(std::move(s));+ check(cache.holds(std::vector<int32_t>{1, 2, 3, 4, 5}, {}, true), "the parked live state itself");+ check(cache.holds(std::vector<int32_t>{1, 2, 3, 4}, {}, true), "a prefix of the parked live state");+ check(cache.holds(std::vector<int32_t>{1, 2}, {}, true), "a prefix of a parked checkpoint");+ check(!cache.holds(std::vector<int32_t>{1, 2, 3, 4, 5, 6}, {}, true), "a longer state is not held");+ check(!cache.holds(std::vector<int32_t>{1, 2, 9}, {}, true), "a diverging state is not held");+ check(!cache.holds(std::vector<int32_t>{}, {}, true), "an empty state is never held");+ check(!cache.holds(std::vector<int32_t>{1, 2, 3}, {}, false), "steering mode is isolated");+ check(!cache.holds(std::vector<int32_t>{1, 2, 3}, {{1, 77}}, true), "a slot with pictures is not held by a text image");+ auto p = image({1, 2, 3, 4});+ p.live.imgs = {{1, 77}, {3, 78}};+ cache.put(std::move(p));+ check(cache.holds(std::vector<int32_t>{1, 2, 3}, {{1, 77}}, true), "pictures below the state must match");+ check(!cache.holds(std::vector<int32_t>{1, 2, 3, 4}, {{1, 77}}, true), "a picture inside the state that the slot lacks");+ }+ {+ // The 2026-10-14 four-conversation measurement replayed at policy level: two batch slots, four parked+ // entries, conversations A..D. Each has a prompt P (8 tokens; the read's periodic checkpoint at 4), a first+ // reply R1 (3 tokens; the slot holds P + 2 of them), the next turn as the client renders it (P + R1 + end ++ // header: 14 tokens, the read's turn checkpoint at 12), a second reply R2 (4 tokens; the slot holds 17) and+ // a third turn (21 tokens). The engine parks the MAIN session (the prompt as read, plus the admission's+ // token) whenever another conversation displaces it; the replies are decoded in the slots. Measured:+ // A's third turn found 199584 of 199839 held tokens (its prompt-read state), B found nothing.+ // slot_parks: an idle slot parks what it holds before an admission overwrites it+ // checks_back: a slot that gives "all it holds" back to the main session gives its checkpoints with it,+ // so the chain still supersedes (#342) the parked copies a turn back+ auto replay = [&](bool slot_parks, bool checks_back) {+ std::vector<std::vector<int32_t>> full(4);+ for (int c = 0; c < 4; ++c) {+ for (int32_t t = 1; t <= 8; ++t) full[(size_t) c].push_back(c * 100 + t);+ for (int32_t t : {21, 22, 23, 9, 31, 32, 41, 42, 43, 44, 9, 51, 52}) full[(size_t) c].push_back(c * 100 + t);+ }+ auto st = [&](int c, size_t n) { return std::vector<int32_t>(full[(size_t) c].begin(), full[(size_t) c].begin() + (std::ptrdiff_t) n); };+ auto cp = [&](int c, size_t n) { ConversationCheckpoint k; k.ids = st(c, n); return k; };+ auto park = [&](ConversationCache& cache, int c, size_t n, const std::vector<size_t>& cks) {+ SavedConversation s = image({});+ s.live.ids = st(c, n);+ for (size_t k : cks) s.checkpoints.push_back(cp(c, k));+ check(cache.put(std::move(s)), "the replay's parks all fit the budget");+ };+ auto restore = [&](ConversationCache& cache, int c, size_t prompt) -> int64_t {+ const auto m = cache.best(st(c, prompt), {}, true);+ if (m.tokens > 0) cache.take(m.index);+ return m.tokens;+ };+ // the main chain after a slot gave "all it holds" back: its checkpoints, or none+ auto back = [&](const std::vector<size_t>& slot_cks) { return checks_back ? slot_cks : std::vector<size_t>{}; };+ auto slot_park = [&](ConversationCache& cache, int c, size_t n, const std::vector<size_t>& cks) {+ if (slot_parks && !cache.holds(st(c, n), {}, true)) park(cache, c, n, cks);+ };+ enum { A, B, C, D };+ ConversationCache cache(1 << 20, 4);+ // initial-AB: B solo then admitted (its prompt + the first token parked when A's request came), A+ // admitted directly, then alone: solo again from its slot (A's prompt-read state parked first)+ park(cache, B, 9, {4});+ park(cache, A, 8, {4});+ std::vector<size_t> chain_a = back({4}); // slot 1 gave back A's 10 tokens+ // resident1-AB: B's request displaces A's solo state; B's slot gives back 10; B admitted with the turn+ // checkpoint; A's request restores A's solo state from RAM and is admitted beside B+ park(cache, A, 12, chain_a);+ std::vector<size_t> chain_b = back({4});+ chain_b.push_back(12);+ // (a request takes its parked conversation before it parks the one it displaces)+ check(restore(cache, A, 14) == 12, "A's second turn resumes from its parked solo state");+ park(cache, B, 15, chain_b);+ chain_a.push_back(12);+ // both decode: slot 0 holds B's 17, slot 1 holds A's 17+ // initial-CD: C's request displaces A's main state; D admitted into slot 0 (B's reply), C into slot 1+ park(cache, A, 14, chain_a);+ slot_park(cache, B, 17, {12});+ park(cache, D, 9, {4});+ slot_park(cache, A, 17, {12});+ park(cache, C, 8, {4}); // C alone: solo again from its slot+ std::vector<size_t> chain_c = back({4});+ // resident1-CD: C continues from the main session (no park), D's request displaces it (slot 1 held C's+ // own earlier state: covered, no slot park), D's slot gives back 10 (covered as well)+ chain_c.push_back(12);+ park(cache, C, 15, chain_c);+ std::vector<size_t> chain_d = back({4});+ chain_d.push_back(12);+ // both decode: slot 0 holds D's 17, slot 1 holds C's 17+ // exchange1-AB: A's request takes A's parked state, displaces D's main state, is admitted into slot 1+ // (C's reply); B's request displaces A's state+ const int64_t a_found = restore(cache, A, 21);+ park(cache, D, 14, chain_d);+ slot_park(cache, C, 17, {12});+ park(cache, A, 22, {12, 20});+ const int64_t b_found = restore(cache, B, 21);+ return std::pair<int64_t, int64_t>(a_found, b_found);+ };+ const auto baseline = replay(false, false);+ check(baseline.first == 14 && baseline.second == 0, "baseline: A returns to its prompt-read state, B to nothing (as measured)");+ const auto parks_only = replay(true, false);+ check(parks_only.first == 17 && parks_only.second == 0, "slot parks alone: A whole, B's copy evicted by a stale initial image");+ const auto both = replay(true, true);+ check(both.first == 17 && both.second == 17, "slot parks + checkpoints back: both return whole");+ }{// what is NOT superseded: another conversation sharing only the root, an entry without checkpoints, the// other steering mode, a branch whose deepest checkpoint the new chain does not holddiff --git a/src/program/generate.cpp b/src/program/generate.cppindex 75d404ac9..29172c9bc 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -9600,10 +9600,23 @@ int main(int argc, char** argv) {id.config = *config_fp;return true;};+ // What a park captures: the main session's conversation (park_current) or an idle batch slot's (park_slot:+ // the reply it decoded, before an admission overwrites it). One budget / admission / snapshot path for both.+ struct ParkSource {+ const std::vector<int32_t>& ids;+ const std::vector<ImgKey>& imgs;+ std::vector<ConvCheckpoint>& checks; ///< moved apart into stage parts for the capture, put back after+ bool cvec;+ strata::core::SessionState& session; ///< stage 0's+ std::vector<strata::core::SessionState*> stage_sessions; ///< the later stages', in stage order (stages[k]->dev)+ const strata::core::QsaState* draft0; ///< the draft layer's K/V saved with stage 0's image (or none)+ const strata::core::QsaState* draft_last; ///< ... with the last stage's image (a split: the drafter's GPU)+ bool with_reuse; ///< the retained K/V describes this source (the main session's)+ std::string what; ///< the log's words after the token count ("", " from slot 1")+ };// Save only on a switch/rewind, not on each continuing request. No graph// addresses change: all parked images live in ordinary host vectors.- auto park_current_body = [&](size_t held) -> bool {- if (!conversations.enabled() || !live_ok || live.empty()) return true;+ auto park_source = [&](const ParkSource& src, size_t held) -> bool {// #2049: parking copies the outgoing conversation's running checkpoints and K/V into host memory, which// is a silent stretch of the "request" stage; when the OS has paged that memory out it takes minutes.// Named, so a stall report says "parking"; its copies beat (conversation_copy.hpp), so the watchdog@@ -9616,15 +9629,15 @@ int main(int argc, char** argv) {// switch-time decision, so a client's small side requests would otherwise park themselves and, once the// cache is full, evict the long conversation they were interleaved with (oldest first) - the reuse it// wanted. Returning true is the same non-error skip the byte-budget and RAM-floor paths below take.- if (!conversations.wants((int64_t) live.size())) {+ if (!conversations.wants((int64_t) src.ids.size())) {std::fprintf(stderr, "strata serve: conversation cache: skip parking (%zu tokens, below the %lld ""minimum; parked=%zu)\n",- live.size(), (long long) o.conversation_cache_min_tokens, conversations.size());+ src.ids.size(), (long long) o.conversation_cache_min_tokens, conversations.size());return true;}// #342: before make_room evicts oldest-first, the copies of this conversation a turn back go (they hold// nothing the outgoing chain does not, apart from the tail this conversation rewrote)- if (const size_t dropped = conversations.drop_superseded(live, live_imgs, checks, cvec_cached))+ if (const size_t dropped = conversations.drop_superseded(src.ids, src.imgs, src.checks, src.cvec))std::fprintf(stderr, "strata serve: conversation cache: dropped %zu superseded cop%s of this ""conversation; parked=%zu\n", dropped, dropped == 1 ? "y" : "ies", conversations.size());// A layer split parks one image per stage: the first stage's (with the draft layer's K/V) and one per@@ -9633,17 +9646,21 @@ int main(int argc, char** argv) {const size_t n_st = stages.size();// the draft layer's K/V lives on the last stage's GPU (the drafter is loaded there): with a split it is// saved with that stage's image, under its device; stage 0's image holds none- const strata::core::QsaState* draft0 = (n_st > 0 && !mtp_first) || !use_mtp ? nullptr : &mtp.kv_state(); // no --mtp: no draft K/V (snapshots accept null)- auto draft_of = [&](size_t k) -> const strata::core::QsaState* { return use_mtp && !mtp_first && k + 1 == n_st ? &mtp.kv_state() : nullptr; };+ const strata::core::QsaState* draft0 = src.draft0; // no --mtp: no draft K/V (snapshots accept null)+ auto draft_of = [&](size_t k) -> const strata::core::QsaState* { return k + 1 == n_st ? src.draft_last : nullptr; };+ std::vector<ConvCheckpoint>& checks = src.checks;strata::core::ConversationCheckpointSplit cs;if (n_st > 0) cs = strata::core::conversation_checkpoints_split(std::move(checks), n_st); // #752: two-phase; a bad_alloc leaves `checks` wholestruct MergeBack {strata::core::ConversationCheckpointSplit& cs; std::vector<ConvCheckpoint>& checks; bool on;~MergeBack() { if (on && !strata::core::conversation_checkpoints_merge(std::move(cs), checks)) checks.clear(); }} merge_back{cs, checks, n_st > 0};- const strata::core::ConversationView view{live, live_imgs, n_st > 0 ? cs.stage0 : checks, cvec_cached};- const int64_t retained_n = conversations.retained_tokens();- auto reuse = conversations.take_reuse_for(live); // never another conversation's K/V+ const strata::core::ConversationView view{src.ids, src.imgs, n_st > 0 ? cs.stage0 : checks, src.cvec};+ const int64_t retained_n = src.with_reuse ? conversations.retained_tokens() : 0;+ // a batch slot's image carries no retained K/V (it describes the main session's conversation), and the+ // main session's stays retained for its own park+ auto reuse = src.with_reuse ? conversations.take_reuse_for(src.ids) // never another conversation's K/V+ : strata::core::ConversationKvReuse{};if (retained_n > 0 && reuse.unchanged_tokens < retained_n)std::fprintf(stderr, "strata serve: conversation cache: retained K/V reused for %lld of its %lld tokens ""(the rest is not this conversation's)\n", (long long) reuse.unchanged_tokens,@@ -9656,9 +9673,9 @@ int main(int argc, char** argv) {for (size_t k = 0; k < n_st; ++k) {auto& st = stages[k];const strata::core::OnDevice on(st->dev);- const strata::core::ConversationView view_k{live, live_imgs, cs.parts[k], cvec_cached};+ const strata::core::ConversationView view_k{src.ids, src.imgs, cs.parts[k], src.cvec};size_t b = 0;- if (!strata::core::conversation_snapshot_bytes(view_k, st->ss, g, draft_of(k), b, err)) {+ if (!strata::core::conversation_snapshot_bytes(view_k, *src.stage_sessions[k], g, draft_of(k), b, err)) {std::fprintf(stderr, "strata serve: conversation cache: skip parking (stage CUDA%d: %s)\n",st->dev, err.c_str());err.clear();@@ -9666,8 +9683,8 @@ int main(int argc, char** argv) {}if (!stage_reuse[k].kv.empty()) {size_t r = 0;- if (strata::core::conversation_snapshot_capture_bytes(stage_reuse[k], view_k, st->ss, g, draft_of(k),- r, err)) {+ if (strata::core::conversation_snapshot_capture_bytes(stage_reuse[k], view_k, *src.stage_sessions[k], g,+ draft_of(k), r, err)) {b = r;stage_retained += stage_reuse[k].bytes();} else {@@ -9678,14 +9695,14 @@ int main(int argc, char** argv) {stage_estimate += b;}size_t estimate = 0;- if (!strata::core::conversation_snapshot_bytes(view, ss, g, draft0, estimate, err)) {+ if (!strata::core::conversation_snapshot_bytes(view, src.session, g, draft0, estimate, err)) {std::fprintf(stderr, "strata serve: conversation cache: skip parking (%s)\n", err.c_str());err.clear(); // A recoverable miss must not poison the batched draft prefill's error channel.return true;}const size_t fresh_estimate = estimate;if (!reuse.kv.empty() && !strata::core::conversation_snapshot_capture_bytes(- reuse, view, ss, g, draft0, estimate, err)) {+ reuse, view, src.session, g, draft0, estimate, err)) {reuse = {};estimate = fresh_estimate;err.clear();@@ -9739,15 +9756,15 @@ int main(int argc, char** argv) {(long long) o.conversation_cache_min_free_mib);strata::core::SavedConversation image;size_t reused_bytes = 0;- if (!strata::core::conversation_snapshot_save(image, view, ss, g, draft0, err,+ if (!strata::core::conversation_snapshot_save(image, view, src.session, g, draft0, err,std::move(reuse), &reused_bytes)) return false;for (size_t k = 0; k < n_st; ++k) { // the later stages (the last one with the draft layer's K/V)auto& st = stages[k];const strata::core::OnDevice on(st->dev);if (cudaDeviceSynchronize() != cudaSuccess) { err = "stage sync failed"; return false; }- const strata::core::ConversationView view_k{live, live_imgs, cs.parts[k], cvec_cached};+ const strata::core::ConversationView view_k{src.ids, src.imgs, cs.parts[k], src.cvec};strata::core::SavedConversation part;- if (!strata::core::conversation_snapshot_save(part, view_k, st->ss, g, draft_of(k), err,+ if (!strata::core::conversation_snapshot_save(part, view_k, *src.stage_sessions[k], g, draft_of(k), err,std::move(stage_reuse[k]), &reused_bytes))return false;image.stage_images.push_back(std::move(part));@@ -9761,8 +9778,8 @@ int main(int argc, char** argv) {}const size_t snapshot_bytes = image.bytes();const bool stored = conversations.put(std::move(image), held);- std::fprintf(stderr, "strata serve: conversation cache: %s %zu tokens in %.1f ms; parked=%zu bytes=%zu evictions=%zu snapshot_bytes=%zu reused_kv_bytes=%zu\n",- stored ? "parked" : "skipped", live.size(),+ std::fprintf(stderr, "strata serve: conversation cache: %s %zu tokens%s in %.1f ms; parked=%zu bytes=%zu evictions=%zu snapshot_bytes=%zu reused_kv_bytes=%zu\n",+ stored ? "parked" : "skipped", src.ids.size(), src.what.c_str(),std::chrono::duration<double, std::milli>(Clock::now() - t0).count(),conversations.size(), conversations.bytes(), conversations.evictions(), snapshot_bytes, reused_bytes);} catch (const std::bad_alloc&) {@@ -9772,6 +9789,15 @@ int main(int argc, char** argv) {}return true;};+ auto park_current_body = [&](size_t held) -> bool {+ if (!conversations.enabled() || !live_ok || live.empty()) return true;+ const size_t n_st = stages.size();+ ParkSource src{live, live_imgs, checks, cvec_cached, ss, {},+ (n_st > 0 && !mtp_first) || !use_mtp ? nullptr : &mtp.kv_state(),+ use_mtp && !mtp_first && n_st > 0 ? &mtp.kv_state() : nullptr, true, ""};+ for (auto& st : stages) src.stage_sessions.push_back(&st->ss);+ return park_source(src, held);+ };// #752: the whole park (the checkpoint split included) is under one try, so a host OOM skips parking instead of// ending the serverauto park_current = [&](size_t held) -> bool {@@ -11686,6 +11712,60 @@ int main(int argc, char** argv) {return true;};auto stage_verifier = [&](int k) -> strata::core::Verifier& { return k == 0 ? ver : stages[(size_t) k - 1]->ver; };+ // What an idle slot holds (the conversation it decoded last: its sessions, its draft K/V and its checkpoints)+ // goes to the parked conversations before an admission overwrites it. The main session was parked with that+ // conversation's prompt only (the admission's copy), so the reply the slot decoded lives nowhere else: without+ // this, the conversation's next turn finds its prompt-read state and reads the whole reply again (measured on+ // four 200K chats over two slots: 255 of a 256-token reply re-read, and a chat whose copies the cache had+ // evicted meanwhile read from token 0). Nothing to do when the live main session or a parked conversation+ // already holds that state (or a continuation of it).+ // Parking is an optimization like a checkpoint (#3350): a skip and a failed snapshot (a host or device+ // allocation that ran out) are logged and the slot is overwritten as before. Nothing here ends the request+ // or the engine.+ auto park_slot = [&](int b) {+ BSlot& sl = bs[(size_t) b];+ if (!conversations.enabled() || sl.active || !sl.cached || sl.img || sl.ids.empty()) return;+ auto prefix_of = [&](const std::vector<int32_t>& whole) {+ return whole.size() >= sl.ids.size() && std::equal(sl.ids.begin(), sl.ids.end(), whole.begin());+ };+ const bool in_main = live_ok && cvec_cached == sl.cvec && prefix_of(live) &&+ imgs_below(live_imgs, (int64_t) sl.ids.size()).empty();+ if (in_main || conversations.holds(sl.ids, {}, sl.cvec)) return;+ if (use_mtp && !batch_mtp) { // no draft K/V of its own: an image without it would not restore beside the drafter+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (its %zu tokens "+ "have no per-slot draft K/V without --batch-mtp)\n", b, sl.ids.size());+ return;+ }+ if (cudaDeviceSynchronize() != cudaSuccess) {+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (device sync "+ "failed)\n", b);+ return;+ }+ std::string why;+ if (batch_mtp && !slot_mtp[(size_t) b]->idle(why)) {+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (%s)\n",+ b, why.c_str());+ return;+ }+ const size_t n_st = stages.size();+ static const std::vector<ImgKey> no_imgs;+ ParkSource src{sl.ids, no_imgs, sl.checks, sl.cvec, *bslot_ss[0][(size_t) b], {},+ (n_st > 0 && !mtp_first) || !batch_mtp ? nullptr : &slot_mtp[(size_t) b]->kv_state(),+ batch_mtp && !mtp_first && n_st > 0 ? &slot_mtp[(size_t) b]->kv_state() : nullptr, false,+ " from slot " + std::to_string(b)};+ for (size_t k = 1; k < bslot_ss.size(); ++k) src.stage_sessions.push_back(bslot_ss[k][(size_t) b].get());+ try {+ if (park_source(src, 0)) return;+ // the snapshot failed (the copy's own memory, as in #3350): the slot is not touched, so go on without+ // the image; a non-sticky allocation error is cleared, a sticky one comes back from the next call+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (%s)\n",+ b, err.empty() ? "the snapshot failed" : err.c_str());+ err.clear();+ (void) checkpoint_failure_is_survivable();+ } catch (const std::bad_alloc&) {+ std::fprintf(stderr, "strata serve: conversation cache: allocation failed; skip parking slot %d\n", b);+ }+ };// one batch window over the active slots only (row t is the t-th active slot): an idle slot is not touched,// so it keeps the conversation it holdsauto batch_step = [&]() -> bool {@@ -12967,25 +13047,54 @@ int main(int argc, char** argv) {resume_plan = bs[(size_t) slot_source].partial_plan;resume_plan_at = slot_tokens;resume_plan_end = resume_plan.to;- for (const ConvCheckpoint& c : bs[(size_t) slot_source].checks) checks.push_back(c);bs[(size_t) slot_source].partial = false;bs[(size_t) slot_source].partial_plan = {};}- if (slot_ck != nullptr) {- live = slot_ck->ids;- if (slot_ckpt_move_env() && admit_slot == slot_source) {- // STRATA_SLOT_CKPT_MOVE: the slot this request continues in drops its checkpoints at the- // admission, so this one moves (and moves on into the slot then: moved_in)- std::vector<ConvCheckpoint>& sc = bs[(size_t) slot_source].checks;- const auto it = sc.begin() + (slot_ck - sc.data());- checks.push_back(std::move(*it));- sc.erase(it);- slot_ck = &checks.back();- moved_in = (int64_t) slot_ck->ids.size();+ // the slot's checkpoints come along (the one continued from, and any shorter one: prefixes of what+ // was copied back, their K/V cells with it), so the chain keeps its branch points and the parked+ // copies of this conversation a turn back - whose deepest checkpoint the chain holds again - go+ // as superseded (#342) instead of taking a parked entry each until the cache evicts another chat.+ // STRATA_SLOT_CKPT_MOVE: the slot this request continues in drops its checkpoints at the+ // admission, so they move (the one continued from moves on into the slot then: moved_in)+ const bool move_cks = slot_ckpt_move_env() && admit_slot == slot_source;+ const int64_t ck_len = slot_ck != nullptr ? (int64_t) slot_ck->ids.size() : -1;+ std::vector<ConvCheckpoint>& sc = bs[(size_t) slot_source].checks;+ // the one continued from first, exactly as before this change (its copy is the one the request needs)+ const ConvCheckpoint* first = slot_ck;+ if (first != nullptr) {+ if (move_cks) {+ checks.push_back(std::move(sc[(size_t) (first - sc.data())]));+ sc.erase(sc.begin() + (first - sc.data()));+ first = nullptr; // gone from the slot} else {- checks.push_back(*slot_ck); // the main session's chain has it now+ checks.push_back(*first);}- checks.back().used = ++check_clock;+ }+ // then the shorter ones: an optimization, so a host allocation that fails skips them+ try {+ for (auto it = sc.begin(); it != sc.end();) {+ if ((int64_t) it->ids.size() > slot_tokens || &*it == first) {+ ++it;+ } else if (move_cks) {+ checks.push_back(std::move(*it));+ it = sc.erase(it);+ } else {+ checks.push_back(*it);+ ++it;+ }+ }+ } catch (const std::bad_alloc&) {+ std::fprintf(stderr, "strata batch: slot %d: its shorter checkpoints were not all copied (out of "+ "memory); the rest are skipped\n", slot_source);+ }+ if (slot_ck != nullptr) {+ for (ConvCheckpoint& c : checks) // the main session's chain has it now+ if ((int64_t) c.ids.size() == ck_len) {+ live = c.ids;+ c.used = ++check_clock;+ slot_ck = &c;+ }+ if (move_cks) moved_in = ck_len;} else {live = bs[(size_t) slot_source].ids;}@@ -13684,6 +13793,7 @@ int main(int argc, char** argv) {std::string ye;const auto ty = Clock::now();std::vector<int32_t> pre(ids.begin(), ids.begin() + q);+ if (can) park_slot(ys); // the conversation the slot holds, before this read overwrites itif (can && copy_to_slot(ys, pre, ye)) {BSlot& sl = bs[(size_t) ys];sl = BSlot{};@@ -15603,6 +15713,7 @@ int main(int argc, char** argv) {bool cont = !cancelled && produced_n == 1 && admit_max_new > 1 && std::strcmp(finish, "length") == 0 &&(int64_t) live.size() == p;const auto tc0 = Clock::now();+ if (cont) park_slot(admit_slot); // the conversation the slot holds, before this one overwrites itif (cont && !copy_to_slot(admit_slot, live, err)) {std::fprintf(stderr, "strata serve: batch admission failed: %s\n", err.c_str());bs[(size_t) admit_slot].cached = false; // its sessions may be half writtendiff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cppindex a38529303..893e7d6bf 100644--- a/sycl/src/program/generate.cpp+++ b/sycl/src/program/generate.cpp@@ -7685,13 +7685,26 @@ int main(int argc, char **argv) try {id.config = *config_fp;return true;};+ // What a park captures: the main session's conversation (park_current) or an idle batch slot's (park_slot:+ // the reply it decoded, before an admission overwrites it). One budget / admission / snapshot path for both.+ struct ParkSource {+ const std::vector<int32_t>& ids;+ const std::vector<ImgKey>& imgs;+ std::vector<ConvCheckpoint>& checks; ///< moved apart into stage parts for the capture, put back after+ bool cvec;+ strata::core::SessionState& session; ///< stage 0's+ std::vector<strata::core::SessionState*> stage_sessions; ///< the later stages', in stage order (stages[k]->dev)+ const strata::core::QsaState* draft0; ///< the draft layer's K/V saved with stage 0's image (or none)+ const strata::core::QsaState* draft_last; ///< ... with the last stage's image (a split: the drafter's GPU)+ bool with_reuse; ///< the retained K/V describes this source (the main session's)+ std::string what; ///< the log's words after the token count ("", " from slot 1")+ };// Save only on a switch/rewind, not on each continuing request. No graph// addresses change: all parked images live in ordinary host vectors.- auto park_current_body = [&](size_t held) -> bool {- if (!conversations.enabled() || !live_ok || live.empty()) return true;+ auto park_source = [&](const ParkSource& src, size_t held) -> bool {// #342: before make_room evicts oldest-first, the copies of this conversation a turn back go (they hold// nothing the outgoing chain does not, apart from the tail this conversation rewrote)- if (const size_t dropped = conversations.drop_superseded(live, live_imgs, checks, cvec_cached))+ if (const size_t dropped = conversations.drop_superseded(src.ids, src.imgs, src.checks, src.cvec))std::fprintf(stderr, "strata serve: conversation cache: dropped %zu superseded cop%s of this ""conversation; parked=%zu\n", dropped, dropped == 1 ? "y" : "ies", conversations.size());// A layer split parks one image per stage: the first stage's (with the draft layer's K/V) and one per@@ -7700,17 +7713,21 @@ int main(int argc, char **argv) try {const size_t n_st = stages.size();// the draft layer's K/V lives on the last stage's GPU (the drafter is loaded there): with a split it is// saved with that stage's image, under its device; stage 0's image holds none- const strata::core::QsaState* draft0 = n_st > 0 || !use_mtp ? nullptr : &mtp.kv_state(); // no --mtp: no draft K/V (snapshots accept null)- auto draft_of = [&](size_t k) -> const strata::core::QsaState* { return use_mtp && k + 1 == n_st ? &mtp.kv_state() : nullptr; };+ const strata::core::QsaState* draft0 = src.draft0; // no --mtp: no draft K/V (snapshots accept null)+ auto draft_of = [&](size_t k) -> const strata::core::QsaState* { return k + 1 == n_st ? src.draft_last : nullptr; };+ std::vector<ConvCheckpoint>& checks = src.checks;strata::core::ConversationCheckpointSplit cs;if (n_st > 0) cs = strata::core::conversation_checkpoints_split(std::move(checks), n_st); // #752: two-phase; a bad_alloc leaves `checks` wholestruct MergeBack {strata::core::ConversationCheckpointSplit& cs; std::vector<ConvCheckpoint>& checks; bool on;~MergeBack() { if (on && !strata::core::conversation_checkpoints_merge(std::move(cs), checks)) checks.clear(); }} merge_back{cs, checks, n_st > 0};- const strata::core::ConversationView view{live, live_imgs, n_st > 0 ? cs.stage0 : checks, cvec_cached};- const int64_t retained_n = conversations.retained_tokens();- auto reuse = conversations.take_reuse_for(live); // never another conversation's K/V+ const strata::core::ConversationView view{src.ids, src.imgs, n_st > 0 ? cs.stage0 : checks, src.cvec};+ const int64_t retained_n = src.with_reuse ? conversations.retained_tokens() : 0;+ // a batch slot's image carries no retained K/V (it describes the main session's conversation), and the+ // main session's stays retained for its own park+ auto reuse = src.with_reuse ? conversations.take_reuse_for(src.ids) // never another conversation's K/V+ : strata::core::ConversationKvReuse{};if (retained_n > 0 && reuse.unchanged_tokens < retained_n)std::fprintf(stderr, "strata serve: conversation cache: retained K/V reused for %lld of its %lld tokens ""(the rest is not this conversation's)\n", (long long) reuse.unchanged_tokens,@@ -7723,9 +7740,9 @@ int main(int argc, char **argv) try {for (size_t k = 0; k < n_st; ++k) {auto& st = stages[k];const strata::core::OnDevice on(st->dev);- const strata::core::ConversationView view_k{live, live_imgs, cs.parts[k], cvec_cached};+ const strata::core::ConversationView view_k{src.ids, src.imgs, cs.parts[k], src.cvec};size_t b = 0;- if (!strata::core::conversation_snapshot_bytes(view_k, st->ss, g, draft_of(k), b, err)) {+ if (!strata::core::conversation_snapshot_bytes(view_k, *src.stage_sessions[k], g, draft_of(k), b, err)) {std::fprintf(stderr, "strata serve: conversation cache: skip parking (stage CUDA%d: %s)\n",st->dev, err.c_str());err.clear();@@ -7733,8 +7750,8 @@ int main(int argc, char **argv) try {}if (!stage_reuse[k].kv.empty()) {size_t r = 0;- if (strata::core::conversation_snapshot_capture_bytes(stage_reuse[k], view_k, st->ss, g, draft_of(k),- r, err)) {+ if (strata::core::conversation_snapshot_capture_bytes(stage_reuse[k], view_k, *src.stage_sessions[k], g,+ draft_of(k), r, err)) {b = r;stage_retained += stage_reuse[k].bytes();} else {@@ -7745,14 +7762,14 @@ int main(int argc, char **argv) try {stage_estimate += b;}size_t estimate = 0;- if (!strata::core::conversation_snapshot_bytes(view, ss, g, draft0, estimate, err)) {+ if (!strata::core::conversation_snapshot_bytes(view, src.session, g, draft0, estimate, err)) {std::fprintf(stderr, "strata serve: conversation cache: skip parking (%s)\n", err.c_str());err.clear(); // A recoverable miss must not poison the batched draft prefill's error channel.return true;}const size_t fresh_estimate = estimate;if (!reuse.kv.empty() && !strata::core::conversation_snapshot_capture_bytes(- reuse, view, ss, g, draft0, estimate, err)) {+ reuse, view, src.session, g, draft0, estimate, err)) {reuse = {};estimate = fresh_estimate;err.clear();@@ -7782,7 +7799,7 @@ int main(int argc, char **argv) try {}strata::core::SavedConversation image;size_t reused_bytes = 0;- if (!strata::core::conversation_snapshot_save(image, view, ss, g, draft0, err,+ if (!strata::core::conversation_snapshot_save(image, view, src.session, g, draft0, err,std::move(reuse), &reused_bytes)) return false;for (size_t k = 0; k < n_st; ++k) { // the later stages (the last one with the draft layer's K/V)auto& st = stages[k];@@ -7791,9 +7808,9 @@ int main(int argc, char **argv) try {.queues_wait_and_throw()) != 0) {err = "stage sync failed"; return false;}- const strata::core::ConversationView view_k{live, live_imgs, cs.parts[k], cvec_cached};+ const strata::core::ConversationView view_k{src.ids, src.imgs, cs.parts[k], src.cvec};strata::core::SavedConversation part;- if (!strata::core::conversation_snapshot_save(part, view_k, st->ss, g, draft_of(k), err,+ if (!strata::core::conversation_snapshot_save(part, view_k, *src.stage_sessions[k], g, draft_of(k), err,std::move(stage_reuse[k]), &reused_bytes))return false;image.stage_images.push_back(std::move(part));@@ -7804,8 +7821,8 @@ int main(int argc, char **argv) try {}const size_t snapshot_bytes = image.bytes();const bool stored = conversations.put(std::move(image), held);- std::fprintf(stderr, "strata serve: conversation cache: %s %zu tokens in %.1f ms; parked=%zu bytes=%zu evictions=%zu snapshot_bytes=%zu reused_kv_bytes=%zu\n",- stored ? "parked" : "skipped", live.size(),+ std::fprintf(stderr, "strata serve: conversation cache: %s %zu tokens%s in %.1f ms; parked=%zu bytes=%zu evictions=%zu snapshot_bytes=%zu reused_kv_bytes=%zu\n",+ stored ? "parked" : "skipped", src.ids.size(), src.what.c_str(),std::chrono::duration<double, std::milli>(Clock::now() - t0).count(),conversations.size(), conversations.bytes(), conversations.evictions(), snapshot_bytes, reused_bytes);} catch (const std::bad_alloc&) {@@ -7815,6 +7832,15 @@ int main(int argc, char **argv) try {}return true;};+ auto park_current_body = [&](size_t held) -> bool {+ if (!conversations.enabled() || !live_ok || live.empty()) return true;+ const size_t n_st = stages.size();+ ParkSource src{live, live_imgs, checks, cvec_cached, ss, {},+ n_st > 0 || !use_mtp ? nullptr : &mtp.kv_state(),+ use_mtp && n_st > 0 ? &mtp.kv_state() : nullptr, true, ""};+ for (auto& st : stages) src.stage_sessions.push_back(&st->ss);+ return park_source(src, held);+ };// #752: the whole park (the checkpoint split included) is under one try, so a host OOM skips parking instead of// ending the serverauto park_current = [&](size_t held) -> bool {@@ -9155,6 +9181,65 @@ int main(int argc, char **argv) try {std::exit(1);}};+ // What an idle slot holds (the conversation it decoded last: its sessions, its draft K/V and its checkpoints)+ // goes to the parked conversations before an admission overwrites it. The main session was parked with that+ // conversation's prompt only (the admission's copy), so the reply the slot decoded lives nowhere else: without+ // this, the conversation's next turn finds its prompt-read state and reads the whole reply again (measured on+ // four 200K chats over two slots: 255 of a 256-token reply re-read, and a chat whose copies the cache had+ // evicted meanwhile read from token 0). Nothing to do when the live main session or a parked conversation+ // already holds that state (or a continuation of it).+ // Parking is an optimization like a checkpoint (#3350): a skip and a failed snapshot (a host or device+ // allocation that ran out) are logged and the slot is overwritten as before. Nothing here ends the request+ // or the engine.+ auto park_slot = [&](int b) {+ BSlot& sl = bs[(size_t) b];+ if (!conversations.enabled() || sl.active || !sl.cached || sl.img || sl.ids.empty()) return;+ auto prefix_of = [&](const std::vector<int32_t>& whole) {+ return whole.size() >= sl.ids.size() && std::equal(sl.ids.begin(), sl.ids.end(), whole.begin());+ };+ const bool in_main = live_ok && cvec_cached == sl.cvec && prefix_of(live) &&+ imgs_below(live_imgs, (int64_t) sl.ids.size()).empty();+ if (in_main || conversations.holds(sl.ids, {}, sl.cvec)) return;+ if (use_mtp && !batch_mtp) { // no draft K/V of its own: an image without it would not restore beside the drafter+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (its %zu tokens "+ "have no per-slot draft K/V without --batch-mtp)\n", b, sl.ids.size());+ return;+ }+ try {+ if (DPCT_CHECK_ERROR(dpct::get_current_device().queues_wait_and_throw()) != 0) {+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (device "+ "sync failed)\n", b);+ return;+ }+ } catch (sycl::exception const& exc) {+ std::cerr << exc.what() << "Exception caught at file:" << __FILE__ << ", line:" << __LINE__ << std::endl;+ std::exit(1);+ }+ std::string why;+ if (batch_mtp && !slot_mtp[(size_t) b]->idle(why)) {+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (%s)\n",+ b, why.c_str());+ return;+ }+ const size_t n_st = stages.size();+ static const std::vector<ImgKey> no_imgs;+ ParkSource src{sl.ids, no_imgs, sl.checks, sl.cvec, *bslot_ss[0][(size_t) b], {},+ n_st > 0 || !batch_mtp ? nullptr : &slot_mtp[(size_t) b]->kv_state(),+ batch_mtp && n_st > 0 ? &slot_mtp[(size_t) b]->kv_state() : nullptr, false,+ " from slot " + std::to_string(b)};+ for (size_t k = 1; k < bslot_ss.size(); ++k) src.stage_sessions.push_back(bslot_ss[k][(size_t) b].get());+ try {+ if (park_source(src, 0)) return;+ // the snapshot failed (the copy's own memory, as in #3350): the slot is not touched, so go on without+ // the image+ std::fprintf(stderr, "strata serve: conversation cache: slot %d not parked before its overwrite (%s)\n",+ b, err.empty() ? "the snapshot failed" : err.c_str());+ err.clear();+ (void) checkpoint_failure_is_survivable();+ } catch (const std::bad_alloc&) {+ std::fprintf(stderr, "strata serve: conversation cache: allocation failed; skip parking slot %d\n", b);+ }+ };// one batch window over the active slots only (row t is the t-th active slot): an idle slot is not touched,// so it keeps the conversation it holdsauto batch_step = [&]() -> bool {@@ -10024,13 +10109,22 @@ int main(int argc, char **argv) try {// checkpoints as if it had not stoppedif (slot_ck == nullptr && bs[(size_t) slot_source].partial && admit_slot == slot_source) {resumed_from0 = bs[(size_t) slot_source].partial_from0;- for (const ConvCheckpoint& c : bs[(size_t) slot_source].checks) checks.push_back(c);bs[(size_t) slot_source].partial = false;}+ // the slot's checkpoints come along (the one continued from, and any shorter one: prefixes of what+ // was copied back, their K/V cells with it), so the chain keeps its branch points and the parked+ // copies of this conversation a turn back - whose deepest checkpoint the chain holds again - go+ // as superseded (#342) instead of taking a parked entry each until the cache evicts another chat+ try { // an optimization: a host allocation that fails skips the rest+ for (const ConvCheckpoint& c : bs[(size_t) slot_source].checks)+ if ((int64_t) c.ids.size() <= slot_tokens) checks.push_back(c);+ } catch (const std::bad_alloc&) {+ std::fprintf(stderr, "strata batch: slot %d: its checkpoints were not all copied (out of memory); the rest are skipped\n", slot_source);+ }if (slot_ck != nullptr) {live = slot_ck->ids;- checks.push_back(*slot_ck); // the main session's chain has it now- checks.back().used = ++check_clock;+ for (ConvCheckpoint& c : checks) // the main session's chain has it now+ if (c.ids.size() == slot_ck->ids.size()) c.used = ++check_clock;} else {live = bs[(size_t) slot_source].ids;}@@ -10554,6 +10648,7 @@ int main(int argc, char **argv) try {std::string ye;const auto ty = Clock::now();std::vector<int32_t> pre(ids.begin(), ids.begin() + q);+ if (can) park_slot(ys); // the conversation the slot holds, before this read overwrites itif (can && copy_to_slot(ys, pre, ye)) {BSlot& sl = bs[(size_t) ys];sl = BSlot{};@@ -11865,6 +11960,7 @@ int main(int argc, char **argv) try {bool cont = !cancelled && produced_n == 1 && admit_max_new > 1 && std::strcmp(finish, "length") == 0 &&(int64_t) live.size() == p;const auto tc0 = Clock::now();+ if (cont) park_slot(admit_slot); // the conversation the slot holds, before this one overwrites itif (cont && !copy_to_slot(admit_slot, live, err)) {std::fprintf(stderr, "strata serve: batch admission failed: %s\n", err.c_str());bs[(size_t) admit_slot].cached = false; // its sessions may be half written