Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge perf-PR group serve (f-145-pp-serve)
Niko12212f3a5d901397
diff --cc include/strata/core/conversation_snapshot.hppindex 449fc8316,d3ded85e7..dcee29fa8--- a/include/strata/core/conversation_snapshot.hpp+++ b/include/strata/core/conversation_snapshot.hpp@@@ -25,12 -25,12 +25,18 @@@ bool conversation_kv_validate(const Conint64_t upto, bool include_index, std::string& error);bool conversation_kv_restore(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,int64_t upto, bool include_index, std::string& error);+// conversation_kv_save(from) + conversation_kv_restore(to) without a host image, for two states of the same layout.+// The copies are complete on return; the residency refresh is queued on the legacy stream like the restore's.+// `queued`: the copies are only queued on the private copy stream, the refresh is ordered after them on the device,+// and the caller synchronizes the device before it reads or writes either state.+bool conversation_kv_copy(const QsaState& from, const QsaState& to, const ModelGeometry& g,+ int64_t upto, bool include_index, std::string& error, bool queued = false);+ // The first `upto` tokens of an image captured at `captured` tokens: the leading bytes of each buffer (cells come+ // first in the identity layout, as conversation_kv_save takes them), the image untouched. Cells and indexer rows+ // past `upto` in the restored pages hold the image's later tokens until a request writes them, as after a whole+ // restore followed by a rewind to a checkpoint.+ bool conversation_kv_restore_prefix(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,+ int64_t captured, int64_t upto, bool include_index, std::string& error);// Diagnostic read-back after a synchronized restore. Uses 64 KiB of stack// workspace, compares authoritative bytes and resident draft-ring pages, and// fingerprints the authoritative payload only. Never changes model state.diff --cc src/core/conversation_snapshot.cppindex d0dd80a11,ac29b3fa4..76e69b5fb--- a/src/core/conversation_snapshot.cpp+++ b/src/core/conversation_snapshot.cpp@@@ -201,68 -201,45 +201,89 @@@ bool conversation_kv_validate(const Conreturn true;}+namespace {+// After the authoritative pools were rewritten: the VRAM slots of a streamed state still hold the outgoing+// conversation, so resolve must refill them before any attention reads (shared by restore and direct copy).+bool refresh_residency(const QsaState& st, const ModelGeometry& g, int64_t upto, std::string& error) {+ if (st.kv_mode == 1) strata::kernels::kv_stream_reset(st.map, nullptr);+ if (st.kv_mode == 2 && upto > 0) {+ auto shapes = strata::kernels::qsa_real_shapes();+ shapes.n_head_kv = g.n_head_kv; shapes.head_dim = g.head_dim;+ const int64_t end = (upto + shapes.page_size - 1) / shapes.page_size;+ strata::kernels::kv_ring_restore(qsa_attn_pools(st), st.host, qsa_kv_format(st),+ std::max<int64_t>(0, end - st.n_slots), end, st.n_slots, shapes, nullptr);+ }+ const auto status = cudaGetLastError();+ if (status == cudaSuccess) return true;+ error = std::string("conversation snapshot residency restore: ") + cudaGetErrorString(status);+ return false;+}+} // namespace++bool conversation_kv_copy(const QsaState& from, const QsaState& to, const ModelGeometry& g,+ int64_t upto, bool index, std::string& error, bool queued) {+ Layout lf{}, lt{};+ if (!layout(from, g, upto, index, lf, error) || !valid(from, lf, upto, error) ||+ !layout(to, g, upto, index, lt, error) || !valid(to, lt, upto, error)) return false;+ if (lf.format != lt.format || lf.cells != lt.cells || lf.page_size != lt.page_size ||+ lf.pooled_rows != lt.pooled_rows || lf.data != lt.data || lf.value_data != lt.value_data ||+ lf.scales != lt.scales || lf.value_scales != lt.value_scales || lf.pooled != lt.pooled) {+ error = "conversation snapshot: direct K/V copy between different layouts";+ return false;+ }+ // The same bytes conversation_kv_save + conversation_kv_restore move, without the host image: each part+ // straight from the source's authoritative pool into the target's (device or mapped host memory), through the+ // private non-blocking stream the other snapshot copies use (complete on return, before the residency refresh).+ // `queued`: the copies are only queued, and a streamed state's refresh waits for them on the device (an event).+ const auto src = pools(from);+ const auto dst = pools(to);+ const std::array<size_t,5> sizes = {lf.data, lf.value_data, lf.scales, lf.value_scales, lf.pooled};+ for (size_t i = 0; i < sizes.size(); ++i) {+ if (!sizes[i]) continue;+ if (!src[i] || !dst[i]) { error = "conversation snapshot: missing state buffer"; return false; }+ const cudaError_t e = conversation_detail::copy_nonblocking(dst[i], src[i], sizes[i], queued);+ if (e != cudaSuccess) { error = std::string("conversation snapshot direct copy: ") + cudaGetErrorString(e); return false; }+ }+ if (queued && to.kv_mode != 0) {+ const cudaError_t e = conversation_detail::order_after_queued();+ if (e != cudaSuccess) { error = std::string("conversation snapshot direct copy: ") + cudaGetErrorString(e); return false; }+ }+ return refresh_residency(to, g, upto, error);+}+bool conversation_kv_restore(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,int64_t upto, bool index, std::string& error) {- return conversation_kv_restore_prefix(image, st, g, upto, upto, index, error);+ if (!conversation_kv_validate(image, st, g, upto, index, error)) return false;+ const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};+ const auto dst = pools(st);+ for (size_t i = 0; i < src.size(); ++i)+ if (!src[i]->visit(0, src[i]->size(), [&](const uint8_t* p, size_t n, size_t at) {+ return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);+ })) return false;+ return refresh_residency(st, g, upto, error);}+ bool conversation_kv_restore_prefix(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,+ int64_t captured, int64_t upto, bool index, std::string& error) {+ if (upto < 0 || upto > captured) {+ error = "conversation snapshot: prefix beyond the captured K/V";+ return false;+ }+ if (!conversation_kv_validate(image, st, g, captured, index, error)) return false;+ Layout l{}; // the prefix's extent; validation above checked the image against the captured one+ if (!layout(st, g, upto, index, l, error) || !valid(st, l, upto, error)) return false;+ const std::array<size_t,5> sizes = {l.data, l.value_data, l.scales, l.value_scales, l.pooled};+ const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};+ for (size_t i = 0; i < src.size(); ++i)+ if (sizes[i] > src[i]->size()) { error = "conversation snapshot: prefix beyond the K/V payload"; return false; }+ const auto dst = pools(st);+ for (size_t i = 0; i < src.size(); ++i)+ if (!src[i]->visit(0, sizes[i], [&](const uint8_t* p, size_t n, size_t at) {+ return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);+ })) return false;- // VRAM slots still contain the outgoing conversation. Resolve must refill- // them from the restored authoritative pools before any attention reads.- if (st.kv_mode == 1) strata::kernels::kv_stream_reset(st.map, nullptr);- if (st.kv_mode == 2 && upto > 0) {- auto shapes = strata::kernels::qsa_real_shapes();- shapes.n_head_kv = g.n_head_kv; shapes.head_dim = g.head_dim;- const int64_t end = (upto + shapes.page_size - 1) / shapes.page_size;- strata::kernels::kv_ring_restore(qsa_attn_pools(st), st.host, qsa_kv_format(st),- std::max<int64_t>(0, end - st.n_slots), end, st.n_slots, shapes, nullptr);- }- const auto status = cudaGetLastError();- if (status == cudaSuccess) return true;- error = std::string("conversation snapshot residency restore: ") + cudaGetErrorString(status);- return false;++ return refresh_residency(st, g, upto, error);+ }+bool conversation_kv_part_sizes(const QsaState& st, const ModelGeometry& g, int64_t upto, bool index,std::array<uint64_t, 5>& sizes, std::string& error) {Layout l{};diff --cc src/program/generate.cppindex d4ea3891c,9f2b0b53f..2c7f94923--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@@ -9076,17 -8511,15 +9092,22 @@@ int main(int argc, char** argv)for (size_t i = 0; i < checks.size(); ++i) pin_flags[i] = checks[i].pinned;break;}- const size_t victim = strata::program::conv_cache::eviction_victim(- stamps.data(), stamps.size(), o.prompt_cache, tail_flags.get(), pin_flags.get());+ std::vector<int64_t> lens;+ for (const ConvCheckpoint& k : checks) lens.push_back((int64_t) k.ids.size());+ const size_t victim = o.prompt_cache_spread+ ? strata::program::conv_cache::spread_victim(lens.data(), stamps.data(), stamps.size(),+ o.prompt_cache, tail_flags.get(), pin_flags.get())+ : strata::program::conv_cache::eviction_victim(stamps.data(), stamps.size(), o.prompt_cache,+ tail_flags.get(), pin_flags.get());+ ckpt_pool.give(std::move(checks[victim])); // (STRATA_CKPT_POOL; else nothing)checks.erase(checks.begin() + (std::ptrdiff_t) victim);}+ if (ckpt_trace && checks.size() < before_evict) {+ const double ms = std::chrono::duration<double, std::milli>(Clock::now() - te0).count();+ req_times.evict_ms += ms;+ std::fprintf(stderr, "strata trace: checkpoint at %lld: %zu evicted in %.1f ms\n", (long long) L,+ before_evict - checks.size(), ms);+ }return true;};// STRATA_SPLIT_MTP_BATCH=1 (default off): the layer split's drafter is on the last stage, so E-9 (Prefill::draft_kv)