Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge perf-PR group serve (f-145-pp-serve)

Niko12212f3a5d901397
diff --cc include/strata/core/conversation_snapshot.hpp
index 449fc8316,d3ded85e7..dcee29fa8
--- a/include/strata/core/conversation_snapshot.hpp
+++ b/include/strata/core/conversation_snapshot.hpp
@@@ -25,12 -25,12 +25,18 @@@ bool conversation_kv_validate(const Con
int64_t upto, bool include_index, std::string& error);
bool conversation_kv_restore(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,
int64_t upto, bool include_index, std::string& error);
+// conversation_kv_save(from) + conversation_kv_restore(to) without a host image, for two states of the same layout.
+// The copies are complete on return; the residency refresh is queued on the legacy stream like the restore's.
+// `queued`: the copies are only queued on the private copy stream, the refresh is ordered after them on the device,
+// and the caller synchronizes the device before it reads or writes either state.
+bool conversation_kv_copy(const QsaState& from, const QsaState& to, const ModelGeometry& g,
+ int64_t upto, bool include_index, std::string& error, bool queued = false);
+ // The first `upto` tokens of an image captured at `captured` tokens: the leading bytes of each buffer (cells come
+ // first in the identity layout, as conversation_kv_save takes them), the image untouched. Cells and indexer rows
+ // past `upto` in the restored pages hold the image's later tokens until a request writes them, as after a whole
+ // restore followed by a rewind to a checkpoint.
+ bool conversation_kv_restore_prefix(const ConversationKv& image, const QsaState& state, const ModelGeometry& g,
+ int64_t captured, int64_t upto, bool include_index, std::string& error);
// Diagnostic read-back after a synchronized restore. Uses 64 KiB of stack
// workspace, compares authoritative bytes and resident draft-ring pages, and
// fingerprints the authoritative payload only. Never changes model state.
diff --cc src/core/conversation_snapshot.cpp
index d0dd80a11,ac29b3fa4..76e69b5fb
--- a/src/core/conversation_snapshot.cpp
+++ b/src/core/conversation_snapshot.cpp
@@@ -201,68 -201,45 +201,89 @@@ bool conversation_kv_validate(const Con
return true;
}
+namespace {
+// After the authoritative pools were rewritten: the VRAM slots of a streamed state still hold the outgoing
+// conversation, so resolve must refill them before any attention reads (shared by restore and direct copy).
+bool refresh_residency(const QsaState& st, const ModelGeometry& g, int64_t upto, std::string& error) {
+ if (st.kv_mode == 1) strata::kernels::kv_stream_reset(st.map, nullptr);
+ if (st.kv_mode == 2 && upto > 0) {
+ auto shapes = strata::kernels::qsa_real_shapes();
+ shapes.n_head_kv = g.n_head_kv; shapes.head_dim = g.head_dim;
+ const int64_t end = (upto + shapes.page_size - 1) / shapes.page_size;
+ strata::kernels::kv_ring_restore(qsa_attn_pools(st), st.host, qsa_kv_format(st),
+ std::max<int64_t>(0, end - st.n_slots), end, st.n_slots, shapes, nullptr);
+ }
+ const auto status = cudaGetLastError();
+ if (status == cudaSuccess) return true;
+ error = std::string("conversation snapshot residency restore: ") + cudaGetErrorString(status);
+ return false;
+}
+} // namespace
+
+bool conversation_kv_copy(const QsaState& from, const QsaState& to, const ModelGeometry& g,
+ int64_t upto, bool index, std::string& error, bool queued) {
+ Layout lf{}, lt{};
+ if (!layout(from, g, upto, index, lf, error) || !valid(from, lf, upto, error) ||
+ !layout(to, g, upto, index, lt, error) || !valid(to, lt, upto, error)) return false;
+ if (lf.format != lt.format || lf.cells != lt.cells || lf.page_size != lt.page_size ||
+ lf.pooled_rows != lt.pooled_rows || lf.data != lt.data || lf.value_data != lt.value_data ||
+ lf.scales != lt.scales || lf.value_scales != lt.value_scales || lf.pooled != lt.pooled) {
+ error = "conversation snapshot: direct K/V copy between different layouts";
+ return false;
+ }
+ // The same bytes conversation_kv_save + conversation_kv_restore move, without the host image: each part
+ // straight from the source's authoritative pool into the target's (device or mapped host memory), through the
+ // private non-blocking stream the other snapshot copies use (complete on return, before the residency refresh).
+ // `queued`: the copies are only queued, and a streamed state's refresh waits for them on the device (an event).
+ const auto src = pools(from);
+ const auto dst = pools(to);
+ const std::array<size_t,5> sizes = {lf.data, lf.value_data, lf.scales, lf.value_scales, lf.pooled};
+ for (size_t i = 0; i < sizes.size(); ++i) {
+ if (!sizes[i]) continue;
+ if (!src[i] || !dst[i]) { error = "conversation snapshot: missing state buffer"; return false; }
+ const cudaError_t e = conversation_detail::copy_nonblocking(dst[i], src[i], sizes[i], queued);
+ if (e != cudaSuccess) { error = std::string("conversation snapshot direct copy: ") + cudaGetErrorString(e); return false; }
+ }
+ if (queued && to.kv_mode != 0) {
+ const cudaError_t e = conversation_detail::order_after_queued();
+ if (e != cudaSuccess) { error = std::string("conversation snapshot direct copy: ") + cudaGetErrorString(e); return false; }
+ }
+ return refresh_residency(to, g, upto, error);
+}
+
bool conversation_kv_restore(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,
int64_t upto, bool index, std::string& error) {
- return conversation_kv_restore_prefix(image, st, g, upto, upto, index, error);
+ if (!conversation_kv_validate(image, st, g, upto, index, error)) return false;
+ const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+ const auto dst = pools(st);
+ for (size_t i = 0; i < src.size(); ++i)
+ if (!src[i]->visit(0, src[i]->size(), [&](const uint8_t* p, size_t n, size_t at) {
+ return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);
+ })) return false;
+ return refresh_residency(st, g, upto, error);
}
+ bool conversation_kv_restore_prefix(const ConversationKv& image, const QsaState& st, const ModelGeometry& g,
+ int64_t captured, int64_t upto, bool index, std::string& error) {
+ if (upto < 0 || upto > captured) {
+ error = "conversation snapshot: prefix beyond the captured K/V";
+ return false;
+ }
+ if (!conversation_kv_validate(image, st, g, captured, index, error)) return false;
+ Layout l{}; // the prefix's extent; validation above checked the image against the captured one
+ if (!layout(st, g, upto, index, l, error) || !valid(st, l, upto, error)) return false;
+ const std::array<size_t,5> sizes = {l.data, l.value_data, l.scales, l.value_scales, l.pooled};
+ const std::array<const ConversationBuffer*,5> src = {&image.k, &image.v, &image.k_scale, &image.v_scale, &image.pooled};
+ for (size_t i = 0; i < src.size(); ++i)
+ if (sizes[i] > src[i]->size()) { error = "conversation snapshot: prefix beyond the K/V payload"; return false; }
+ const auto dst = pools(st);
+ for (size_t i = 0; i < src.size(); ++i)
+ if (!src[i]->visit(0, sizes[i], [&](const uint8_t* p, size_t n, size_t at) {
+ return transfer(static_cast<uint8_t*>(dst[i]) + at, p, n, error);
+ })) return false;
- // VRAM slots still contain the outgoing conversation. Resolve must refill
- // them from the restored authoritative pools before any attention reads.
- if (st.kv_mode == 1) strata::kernels::kv_stream_reset(st.map, nullptr);
- if (st.kv_mode == 2 && upto > 0) {
- auto shapes = strata::kernels::qsa_real_shapes();
- shapes.n_head_kv = g.n_head_kv; shapes.head_dim = g.head_dim;
- const int64_t end = (upto + shapes.page_size - 1) / shapes.page_size;
- strata::kernels::kv_ring_restore(qsa_attn_pools(st), st.host, qsa_kv_format(st),
- std::max<int64_t>(0, end - st.n_slots), end, st.n_slots, shapes, nullptr);
- }
- const auto status = cudaGetLastError();
- if (status == cudaSuccess) return true;
- error = std::string("conversation snapshot residency restore: ") + cudaGetErrorString(status);
- return false;
++ return refresh_residency(st, g, upto, error);
+ }
+
bool conversation_kv_part_sizes(const QsaState& st, const ModelGeometry& g, int64_t upto, bool index,
std::array<uint64_t, 5>& sizes, std::string& error) {
Layout l{};
diff --cc src/program/generate.cpp
index d4ea3891c,9f2b0b53f..2c7f94923
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -9076,17 -8511,15 +9092,22 @@@ int main(int argc, char** argv)
for (size_t i = 0; i < checks.size(); ++i) pin_flags[i] = checks[i].pinned;
break;
}
- const size_t victim = strata::program::conv_cache::eviction_victim(
- stamps.data(), stamps.size(), o.prompt_cache, tail_flags.get(), pin_flags.get());
+ std::vector<int64_t> lens;
+ for (const ConvCheckpoint& k : checks) lens.push_back((int64_t) k.ids.size());
+ const size_t victim = o.prompt_cache_spread
+ ? strata::program::conv_cache::spread_victim(lens.data(), stamps.data(), stamps.size(),
+ o.prompt_cache, tail_flags.get(), pin_flags.get())
+ : strata::program::conv_cache::eviction_victim(stamps.data(), stamps.size(), o.prompt_cache,
+ tail_flags.get(), pin_flags.get());
+ ckpt_pool.give(std::move(checks[victim])); // (STRATA_CKPT_POOL; else nothing)
checks.erase(checks.begin() + (std::ptrdiff_t) victim);
}
+ if (ckpt_trace && checks.size() < before_evict) {
+ const double ms = std::chrono::duration<double, std::milli>(Clock::now() - te0).count();
+ req_times.evict_ms += ms;
+ std::fprintf(stderr, "strata trace: checkpoint at %lld: %zu evicted in %.1f ms\n", (long long) L,
+ before_evict - checks.size(), ms);
+ }
return true;
};
// STRATA_SPLIT_MTP_BATCH=1 (default off): the layer split's drafter is on the last stage, so E-9 (Prefill::draft_kv)