Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: keep a conversation another request displaces and take it back at its checkpoints (STRATA_DS4_SLOT_KEEP), shared-start checkpoint (pin=N)
Niko1221cdc3a265773f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex c46b305c7..bbe296b20 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -884,6 +884,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)if(STRATA_ENABLE_HIP)target_compile_options(ds4_iq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")endif()+ # the kept conversations' .meta file and prefix matching (ds4_slotmeta.hpp): no GPU, no model+ add_executable(ds4_slot_test tests/dsv41/slot_test.cpp)+ target_include_directories(ds4_slot_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)+ target_compile_features(ds4_slot_test PRIVATE cxx_std_17)endif()if(STRATA_ENABLE_HIP)foreach(_t strata_dsv41_kernels strata-dsv41)diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex e614e6cb1..0ea0c1517 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -11,16 +11,27 @@// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures-// (the file: see read_pics); the state is shared only where the same picture sits at the same place.+// (the file: see read_pics); the state is shared only where the same picture sits at the same place. With+// STRATA_DS4_SLOT_KEEP=1 the conversations are kept when another one takes the model (a slot each: the state, the+// compressed KV included), and across runs in STRATA_DS4_SLOT_DIR - so a request that does not continue the state but+// does continue a kept conversation reads only its new tokens.+// `pin=N` on a GEN marks the first N prompt tokens as a start other requests share (a system prompt and its tools): the+// read stops there for a checkpoint, so a request behind the same start - another agent - goes back to it instead of+// reading it again (also without slots). STRATA_DS4_REUSE=0 turns every reuse off (each request reads its whole prompt).#include "ds4_model.hpp"#include "ds4_packinfo.hpp"+#include "ds4_slotmeta.hpp"#include <algorithm>#include <cmath>#include <numeric>#include <atomic>#include <chrono>+#include <climits>#include <condition_variable>+#include <csignal>+#include <filesystem>+#include <memory>#include <cstdio>#include <fstream>#include <sstream>@@ -182,6 +193,9 @@ struct Req {unsigned long long seed = 0;int spec = -1; // spec=0|1: this request drafts or not (-1: as the engine was started)float conf = -1.0f; // dspark_conf=F: the drafts' confidence floor for this request (-1: the engine's)+ long long pin = 0; // pin=N: the first N prompt tokens are a start other requests share (a system prompt and its+ // tools): the state at N is kept as a checkpoint, so another agent behind the same start+ // goes back to it instead of reading it againstd::string file; // GENI: the picturesstd::vector<int> ids;};@@ -277,11 +291,12 @@ bool parse_gen(const std::string& line, Req& r, bool geni) {else if (k == "seed") r.seed = std::strtoull(v.c_str(), nullptr, 10);else if (k == "spec") r.spec = std::atoi(v.c_str()) != 0;else if (k == "dspark_conf") r.conf = std::strtof(v.c_str(), nullptr);+ else if (k == "pin") r.pin = std::atoll(v.c_str());}return !r.ids.empty() && (!geni || !r.file.empty());}-int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float conf_min) {+int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float conf_min, const std::string& pack) {std::vector<int> eos = parse_ids(eos_list);for (const std::string& l : m.info_lines()) std::printf("%s\n", l.c_str());std::printf("READY %lld stop\n", (long long) m.max_context());@@ -294,6 +309,321 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float cstd::vector<Pic> held; // the pictures the state holds (start, hash, span)SpanCache spans;const int img_tok = m.image_token();+ // CONVERSATION SLOTS: a request that does not continue the conversation in the model first sets that one aside+ // (Model::slot_save: its window rings, the compressors' partial groups and every source's compressed KV and index+ // keys), then takes back the kept conversation this prompt does continue (slot_load) - chats, and an agent's side+ // requests, that interleave no longer re-read each other's prompts (70k and 185k tokens alternating: 75-200 s a+ // switch at ~930 tok/s). STRATA_DS4_SLOT_KEEP=1 turns them on and keeps them across runs, in STRATA_DS4_SLOT_DIR+ // (default <pack>/slots): a slot is a .bin (the state) and a .meta (this model's identity, the conversation's+ // tokens and its size), both written under a temporary name first; a slot whose .meta is missing, foreign or does+ // not match its file, or unused for STRATA_DS4_SLOT_DAYS (7) days, is removed at start. STRATA_DS4_SLOT_CAP (4) is+ // the conversations kept in RAM; past it the least recently used is written to the folder and read back from there+ // (RAM first, the folder past the cap), and the engine's end writes the ones in RAM to the folder, so the next run+ // reads on from them. STRATA_DS4_SLOT_GB (16) caps the folder's .bin files: at start, and after every write, the+ // folder's least recently used conversation is removed while they are over it (never the one just written, so a+ // conversation bigger than the cap stays). STRATA_DS4_SLOT_MIN (1024) is the shortest conversation set aside (a+ // shorter one is read again in about the time its slot would take). Text only: a conversation that holds pictures+ // is not set aside, and a prompt with pictures takes none back.+ const auto env_num = [](const char* k, double d) { const char* v = std::getenv(k); return v ? std::atof(v) : d; };+ const bool slot_keep =+ std::getenv("STRATA_DS4_SLOT_KEEP") != nullptr && std::atoi(std::getenv("STRATA_DS4_SLOT_KEEP")) != 0;+ const bool reuse = std::getenv("STRATA_DS4_REUSE") == nullptr || std::atoi(std::getenv("STRATA_DS4_REUSE")) != 0;+ int slot_max = slot_keep ? std::max(0, (int) env_num("STRATA_DS4_SLOT_CAP", 4)) : 0;+ const size_t slot_min = (size_t) std::max(1.0, env_num("STRATA_DS4_SLOT_MIN", 1024));+ const double slot_days = env_num("STRATA_DS4_SLOT_DAYS", 7);+ const uint64_t slot_budget = (uint64_t) (env_num("STRATA_DS4_SLOT_GB", 16) * 1073741824.0);+ namespace fs = std::filesystem;+ std::filesystem::path slot_dir;+ struct Slot {+ std::vector<int32_t> tokens;+ std::string path; // empty: the state is in data (RAM)+ uint64_t bytes = 0, used = 0;+ std::shared_ptr<std::vector<uint8_t>> data; // the RAM slot's bytes (null: a file slot)+ std::vector<int64_t> cks; // the positions short of its end it can be taken back at+ };+ std::vector<Slot> slots;+ uint64_t slot_clock = 0, slot_seq = 0;+ const std::string slot_identity = m.slot_identity();+ const auto note = [&](const std::string& msg) {+ std::fprintf(stderr, "ds4 slots: %s\n", msg.c_str());+ std::printf("NOTE sessions: %s\n", msg.c_str());+ std::fflush(stdout);+ };+ const auto gb = [](uint64_t b) {+ char s[32];+ std::snprintf(s, sizeof s, "%.2f GB", (double) b / 1e9);+ return std::string(s);+ };+ const auto meta_of = [](const std::string& bin) { return bin.substr(0, bin.size() - 4) + ".meta"; };+ const auto meta_write = [&](const std::string& path, const std::vector<int32_t>& toks, uint64_t bytes,+ const std::vector<int64_t>& cks) -> bool {+ return strata::ds4::slotmeta::write(path, slot_identity, toks, bytes, cks);+ };+ // a .meta of this model -> its tokens and the slot's size (false: foreign, damaged or not this model's)+ const auto meta_read = [&](const std::string& path, std::vector<int32_t>& toks, uint64_t& bytes,+ std::vector<int64_t>& cks) -> bool {+ return strata::ds4::slotmeta::read(path, slot_identity, m.max_context(), toks, bytes, cks);+ };+ const auto slot_age_days = [](const std::string& p) {+ std::error_code ec;+ const auto t = fs::last_write_time(p, ec);+ if (ec) return 1e9;+ return std::chrono::duration<double>(fs::file_time_type::clock::now() - t).count() / 86400.0;+ };+ const auto slot_drop = [&](size_t i) {+ if (slots[i].data) // a RAM slot: its buffer simply dies+ slots[i].data.reset();+ else {+ std::error_code ec;+ std::filesystem::remove(slots[i].path, ec);+ std::filesystem::remove(meta_of(slots[i].path), ec);+ }+ slots.erase(slots.begin() + (long) i);+ };+ const auto slot_count = [&](int& ram, int& disk) {+ ram = disk = 0;+ for (const Slot& s : slots) (s.data ? ram : disk) += 1;+ };+ // the folder's conversations (their .bin; a .meta beside each is a few KB: identity and tokens)+ const auto slot_folder_bytes = [&]() {+ uint64_t total = 0;+ std::error_code ec;+ for (const auto& de : std::filesystem::directory_iterator(slot_dir, ec)) {+ const std::string fn = de.path().filename().string();+ if (fn.rfind("slot-", 0) == 0 && fn.size() > 9 && fn.compare(fn.size() - 4, 4, ".bin") == 0) {+ const uint64_t bytes = (uint64_t) std::filesystem::file_size(de.path(), ec);+ if (!ec) total += bytes;+ }+ }+ return total;+ };+ // the folder over STRATA_DS4_SLOT_GB: its least recently used conversation goes first, and never `keep` (the one just+ // written), so a conversation bigger than the cap stays+ const auto slot_trim = [&](const std::string& keep) {+ const uint64_t before = slot_folder_bytes();+ size_t dropped = 0;+ while (slot_folder_bytes() > slot_budget) {+ size_t lru = SIZE_MAX;+ for (size_t i = 0; i < slots.size(); ++i)+ if (!slots[i].data && slots[i].path != keep && (lru == SIZE_MAX || slots[i].used < slots[lru].used))+ lru = i;+ if (lru == SIZE_MAX) break;+ slot_drop(lru);+ ++dropped;+ }+ if (dropped > 0) {+ char msg[256];+ std::snprintf(msg, sizeof msg, "the folder held %s (over STRATA_DS4_SLOT_GB, %s): %zu least recently used "+ "conversation(s) removed, %s left", gb(before).c_str(), gb(slot_budget).c_str(), dropped,+ gb(slot_folder_bytes()).c_str());+ note(msg);+ }+ };+ // a RAM slot into the folder (its bytes, then its .meta); false: not written (its bytes go, then)+ const auto slot_write_file = [&](Slot& s) -> bool {+ std::error_code ec;+ const auto space = std::filesystem::space(slot_dir, ec);+ if (!ec && space.available < ((uint64_t) 8 << 30)) { // (8 GB free, the GLM engine's guard)+ std::fprintf(stderr, "ds4 slots: under 8 GB free in %s - the conversation is not written\n",+ slot_dir.string().c_str());+ return false;+ }+ const auto stamp = std::chrono::duration_cast<std::chrono::milliseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ const std::string path =+ (slot_dir / ("slot-" + std::to_string(stamp) + "-" + std::to_string(++slot_seq) + ".bin")).string();+ {+ std::ofstream f(path + ".tmp", std::ios::binary | std::ios::trunc);+ f.write((const char*) s.data->data(), (std::streamsize) s.data->size());+ f.flush();+ if (!f) {+ std::filesystem::remove(path + ".tmp", ec);+ std::fprintf(stderr, "ds4 slots: writing %s.tmp failed (disk full?)\n", path.c_str());+ return false;+ }+ }+ fs::rename(path + ".tmp", path, ec);+ if (ec || !meta_write(meta_of(path), s.tokens, s.data->size(), s.cks)) {+ std::filesystem::remove(path + ".tmp", ec);+ std::filesystem::remove(path, ec);+ std::fprintf(stderr, "ds4 slots: writing %s failed\n", path.c_str());+ return false;+ }+ s.path = path;+ s.bytes = s.data->size();+ s.data.reset();+ slot_trim(path); // (the folder over its cap: the least recently used conversation goes, not this one)+ return true;+ };+ // the conversation in the model into a slot (the state is lost otherwise: another one takes the model)+ const auto slot_put = [&](const char* why) {+ if (slot_max <= 0) return;+ const std::vector<int32_t>& t = m.tokens();+ if (t.size() < slot_min) return; // (a fresh engine's 0-token state is no conversation: nothing to set aside)+ const auto t0 = std::chrono::steady_clock::now();+ auto buf = std::make_shared<std::vector<uint8_t>>();+ std::string e;+ const uint64_t bytes = m.slot_save(*buf, e); // (a state unchanged since it was taken refuses here: nothing+ if (bytes == 0) { // changes, its slot stays)+ if (e != "it is already set aside")+ std::fprintf(stderr, "ds4 slots: setting the conversation aside failed (%s) - it is read from token 0 "+ "when it comes back\n", e.c_str());+ return;+ }+ for (size_t i = slots.size(); i-- > 0;) // an earlier state of this conversation is superseded+ if (slots[i].tokens.size() <= t.size() && std::equal(slots[i].tokens.begin(), slots[i].tokens.end(), t.begin()))+ slot_drop(i);+ slots.push_back({t, "", bytes, ++slot_clock, nullptr, m.check_positions()});+ slots.back().data = buf;+ while (true) { // past the cap: the least recently used conversation in RAM to the folder, or gone+ int ram = 0, disk = 0;+ slot_count(ram, disk);+ if (ram <= slot_max) break;+ size_t lru = SIZE_MAX;+ for (size_t i = 0; i < slots.size(); ++i)+ if (slots[i].data && (lru == SIZE_MAX || slots[i].used < slots[lru].used)) lru = i;+ if (lru == SIZE_MAX) break;+ if (!slot_write_file(slots[lru])) slot_drop(lru);+ }+ int ram = 0, disk = 0;+ slot_count(ram, disk);+ char msg[256];+ std::snprintf(msg, sizeof msg, "saved a %zu-token conversation%s (%s in RAM in %.1f s); %d in RAM, %d in %s",+ t.size(), why, gb(bytes).c_str(),+ std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(), ram, disk,+ slot_dir.string().c_str());+ note(msg);+ };+ // the kept conversation this prompt shares the most of - past `floor` tokens, what the model itself can go back+ // to - back in the model: the tokens it covers (0: none). A conversation the prompt continues whole comes back+ // whole (`whole`); one it leaves part way - a retried prompt, a history the client renders again without the+ // earlier reasoning - comes back at its latest checkpoint the prompt still agrees with (the end of a prompt, the+ // position before it, the anchor where its run of continued prompts began).+ const auto slot_take = [&](const std::vector<int>& prompt, size_t floor, bool& whole) -> size_t {+ whole = false;+ if (slot_max <= 0 || prompt.size() < 2) return 0;+ std::vector<strata::ds4::slotmeta::Cand> cands;+ for (const Slot& sl : slots) cands.push_back({&sl.tokens, &sl.cks});+ size_t len = 0;+ const size_t best = strata::ds4::slotmeta::pick(cands, prompt, floor, &len);+ if (best == SIZE_MAX) return 0;+ const size_t whole_len = slots[best].tokens.size();+ const auto t0 = std::chrono::steady_clock::now();+ const bool ram = slots[best].data != nullptr;+ std::string e;+ bool back = false;+ if (ram) back = m.slot_load(slots[best].data->data(), slots[best].data->size(), (int64_t) whole_len, e);+ else {+ std::ifstream f(slots[best].path, std::ios::binary);+ std::vector<uint8_t> file;+ if (f) {+ f.seekg(0, std::ios::end);+ const std::streamoff n = f.tellg();+ f.seekg(0, std::ios::beg);+ if (n > 0) {+ file.resize((size_t) n);+ if (f.read((char*) file.data(), n))+ back = m.slot_load(file.data(), file.size(), (int64_t) whole_len, e);+ else e = "reading " + slots[best].path + " failed";+ } else e = slots[best].path + " is empty";+ } else e = "cannot read " + slots[best].path;+ }+ if (!back) {+ note("taking a conversation back failed (" + (e.empty() ? std::string("it did not load") : e) ++ ") - reading its prompt again");+ slot_drop(best);+ return 0;+ }+ if (len < whole_len && !(m.restore((int64_t) len) && (size_t) m.pos() == len)) {+ note("going back to token " + std::to_string(len) + " of a kept conversation failed - reading its prompt again");+ m.reset();+ return 0;+ }+ whole = len == whole_len;+ slots[best].used = ++slot_clock;+ if (!ram) { // its age counts from its last use (a file slot's only)+ std::error_code ec;+ fs::last_write_time(meta_of(slots[best].path), fs::file_time_type::clock::now(), ec);+ }+ char msg[256];+ std::string part;+ if (!whole) part = " at token " + std::to_string(len) + " of " + std::to_string(whole_len);+ std::snprintf(msg, sizeof msg, "took back a %zu-token conversation%s from %s (%.1f s); %zu new tokens to read",+ len, part.c_str(), ram ? "RAM" : "the folder",+ std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),+ prompt.size() - len);+ note(msg);+ return len;+ };+ if (slot_max > 0) {+ slot_dir = std::getenv("STRATA_DS4_SLOT_DIR") ? std::filesystem::path(std::getenv("STRATA_DS4_SLOT_DIR"))+ : std::filesystem::path(pack) / "slots";+ std::error_code ec;+ std::filesystem::create_directories(slot_dir, ec);+ if (!std::filesystem::is_directory(slot_dir, ec)) {+ std::fprintf(stderr, "ds4 slots: %s is not writable - conversations are not set aside\n",+ slot_dir.string().c_str());+ slot_max = 0;+ } else {+ struct Found {+ std::string bin;+ std::vector<int32_t> toks;+ uint64_t bytes;+ std::vector<int64_t> cks;+ fs::file_time_type t;+ };+ std::vector<Found> found;+ size_t refused = 0;+ for (const auto& de : std::filesystem::directory_iterator(slot_dir, ec)) { // a previous run's slots+ const std::string fn = de.path().filename().string(), p = de.path().string();+ if (fn.rfind("slot-", 0) != 0) continue;+ const bool is_bin = fn.size() > 9 && fn.compare(fn.size() - 4, 4, ".bin") == 0;+ const bool is_meta = fn.size() > 10 && fn.compare(fn.size() - 5, 5, ".meta") == 0;+ if (!is_bin && !is_meta) { // a .tmp an interrupted write left+ std::filesystem::remove(de.path(), ec);+ continue;+ }+ if (!is_meta) continue; // a .bin is judged by its .meta+ const std::string bin = p.substr(0, p.size() - 5) + ".bin";+ Found fd{bin, {}, 0, {}, fs::last_write_time(de.path(), ec)};+ const uint64_t have = fs::exists(bin, ec) ? (uint64_t) fs::file_size(bin, ec) : 0;+ if (meta_read(p, fd.toks, fd.bytes, fd.cks) && have == fd.bytes && fd.bytes > 0 && slot_age_days(p) <= slot_days)+ found.push_back(std::move(fd));+ else {+ ++refused;+ std::filesystem::remove(p, ec);+ std::filesystem::remove(bin, ec);+ }+ }+ for (const auto& de : std::filesystem::directory_iterator(slot_dir, ec)) { // a .bin without its .meta+ const std::string p = de.path().string(), fn = de.path().filename().string();+ if (fn.rfind("slot-", 0) == 0 && fn.size() > 9 && fn.compare(fn.size() - 4, 4, ".bin") == 0 &&+ !fs::exists(meta_of(p), ec))+ std::filesystem::remove(de.path(), ec);+ }+ std::sort(found.begin(), found.end(), [](const Found& a, const Found& b) { return a.t < b.t; });+ uint64_t total = 0;+ for (auto& fd : found) {+ total += fd.bytes;+ slots.push_back({std::move(fd.toks), fd.bin, fd.bytes, ++slot_clock, nullptr, std::move(fd.cks)});+ }+ std::string msg = "folder " + slot_dir.string() + ": " + std::to_string(slots.size()) ++ " conversation(s) kept from earlier runs (" + gb(total) + ")";+ if (refused) msg += ", " + std::to_string(refused) + " not this model's or incomplete - removed";+ note(msg);+ slot_trim(""); // (the folder over STRATA_DS4_SLOT_GB: the least recently used conversation goes first)+ }+ }+ if (slot_max > 0) {+#if !defined(_WIN32)+ // (the server ends the engine with QUIT and waits for it; a SIGTERM to the whole process group - systemd+ // stopping a service, a supervisor killing the server's group - must not end it mid-write, and a server that+ // is already gone must not end it with SIGPIPE when it writes its last lines)+ std::signal(SIGTERM, SIG_IGN);+ std::signal(SIGINT, SIG_IGN);+ std::signal(SIGPIPE, SIG_IGN);+#endif+ }while (in.next(line)) {if (line == "QUIT") break;const bool geni = line.rfind("GENI ", 0) == 0;@@ -325,6 +655,10 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float ccontinue;}}+ if (!reuse) { // (STRATA_DS4_REUSE=0, for A/Bs and tests: every request is a fresh read of its whole prompt)+ m.reset();+ held.clear();+ }const auto& st = m.tokens();const size_t n_state = st.size();size_t c = 0;@@ -340,12 +674,32 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float cfor (const Pic& q : held)if ((size_t) q.start < c && !same(pics, q)) c = (size_t) q.start;size_t from = 0;- if (c == n_state && n_state < r.ids.size()) from = n_state;- else if (m.restorable((int64_t) std::min(c, r.ids.size() - 1)) >= 0 &&- m.restore((int64_t) std::min(c, r.ids.size() - 1)))- from = (size_t) m.pos();- else m.reset();- if (from == 0) held.clear();+ bool took = false, continued = false; // (continued: the conversation goes on - its anchor stays)+ // the conversation continues, reused in place - a conversation has to be in the model first: a fresh engine+ // (the start of a run, so every restart) holds 0 tokens, and every prompt "continues" that to token 0; taken+ // as a continuation, the take below would never run for the folder's conversations, the case they are for+ if (n_state > 0 && c == n_state && n_state < r.ids.size()) {+ from = n_state;+ continued = true;+ }+ else {+ // CONVERSATION SLOTS: this prompt does not continue the conversation in the model - that one is set aside+ // first (another one takes the model, and its state is lost otherwise), then the kept conversation this+ // prompt does continue is taken back - if it goes further back than the checkpoints the state itself holds+ if (held.empty()) slot_put("");+ const int64_t limit = (int64_t) std::min(c, r.ids.size() - 1);+ const int64_t rs = m.restorable(limit);+ bool whole = false;+ const size_t back = pics.empty() ? slot_take(r.ids, rs > 0 ? (size_t) rs : 0, whole) : 0;+ if (back > 0) {+ from = back;+ took = true;+ continued = whole;+ } else if (rs >= 0 && m.restore(limit))+ from = (size_t) m.pos();+ else m.reset();+ }+ if (took || from == 0) held.clear(); // (a slot is text only: the state that comes back holds no pictures)if ((int64_t) r.ids.size() + std::max(0LL, r.max_new) > m.max_context()) {std::printf("ERR the prompt (%zu tokens) and max_new %lld are more than the context %lld\n", r.ids.size(),r.max_new, (long long) m.max_context());@@ -399,8 +753,12 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c});// (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;+ // (pin=N: the read stops at N for a checkpoint of the shared start; only when this read crosses N, and N leaves+ // the prompt's last token to the usual end)+ const size_t pin_at = (r.pin >= 256 && (size_t) r.pin + 1 < total) ? (size_t) r.pin : 0;while (ok && done < total) {- const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);+ size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);+ if (pin_at > done && pin_at < done + n) n = pin_at - done;// (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)const bool fin = done + n == total;if (fin && last_window) {@@ -411,6 +769,7 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float cok = fin ? m.feed_mark(r.ids.data() + done, (int) n, 1, 1, err): m.feed(r.ids.data() + done, (int) n, 0, err);done += n;+ if (ok && pin_at != 0 && done == pin_at) m.checkpoint(3);if (ok && (done >= next_pp || done == total)) {const double ms = now_ms() - t0;std::printf("PP %zu %zu %.1f %.1f %zu\n", done, r.ids.size(), ms,@@ -431,6 +790,7 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c}held = std::move(pics);m.checkpoint(0);+ if (!continued) m.pin(); // (a new run of prompts: where it may be rendered again from)const double t1 = now_ms();long long produced = 0;const char* finish = "length";@@ -527,6 +887,31 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float cper_position(s_dec, s).c_str());m.save_profile();}+ // the engine ends (QUIT or the end of its input): the conversation in the model is set aside, and every one in+ // RAM is written to the folder, so the next run reads on from them+ if (slot_max > 0) {+ if (held.empty()) slot_put(" as the engine ended");+ size_t written = 0;+ while (true) { // (a write can take the folder's least recently used conversation away: the list is asked again)+ size_t i = SIZE_MAX;+ for (size_t k = 0; k < slots.size(); ++k)+ if (slots[k].data) {+ i = k;+ break;+ }+ if (i == SIZE_MAX) break;+ if (slot_write_file(slots[i])) ++written;+ else slot_drop(i); // (its state could not be written: there is nothing more to do at the engine's end)+ }+ if (written > 0) {+ int ram = 0, disk = 0;+ slot_count(ram, disk);+ char msg[256];+ std::snprintf(msg, sizeof msg, "wrote %zu conversation(s) to %s; %d there in all", written,+ slot_dir.string().c_str(), disk);+ note(msg);+ }+ }m.save_profile();return 0;}@@ -847,7 +1232,7 @@ int main(int argc, char** argv) {return 1;}if (!o.vision_only) std::fprintf(stderr, "%s", m.describe().c_str());- if (serve_mode) return serve(m, eos, spec, o.dspark_conf);+ if (serve_mode) return serve(m, eos, spec, o.dspark_conf, o.pack);const int window_given = window;if (window == 0) window = 16; // (the scoring / profiling runs; the plain run below takes the model's feed unit)if (!prof_ids.empty()) return profile_run(m, prof_ids, window);diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex a9e370441..7d17fc565 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -41,6 +41,7 @@#include <cstdio>#include <cstdlib>#include <cstring>+#include <filesystem>#include <fstream>#include <functional>#include <future>@@ -920,7 +921,14 @@ struct Model::Impl {std::vector<std::vector<uint8_t>> win;std::vector<std::vector<float>> comp;};- Check ck[2]; // the end of the last prompt, and the position before it+ static constexpr int kChecks = 4;+ Check ck[kChecks]; // the end of the last prompt, the position before it, the anchor (the position+ // before the end of the first prompt of the run of prompts that each continued+ // the last in place: a chat template that drops earlier turns' reasoning at a new+ // user message re-renders the history from about there) and the prefix (where the+ // server said the prompt's shared start - system prompt and tools - ends: another+ // agent behind the same start goes back to it)+ bool slot_held = false; // slot_save took this state and nothing has changed it sincestd::vector<uint32_t> prof; // [nl][n_exp] routed-expert hitsStats st;int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query@@ -4528,7 +4536,8 @@ struct Model::Impl {pos = 0;toks.clear();ctoks.clear();- ck[0].pos = ck[1].pos = -1;+ slot_held = false;+ for (Check& c0 : ck) c0.pos = -1;for (Layer& l0 : L)for (int m = 0; m < l0.nm; ++m) {Layer& l = member(l0, m);@@ -4547,7 +4556,7 @@ struct Model::Impl {// the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the// state is indexed by position, and what lies past a restored position is written again)void checkpoint(int slot) {- Check& c = ck[slot & 1];+ Check& c = ck[slot & (kChecks - 1)];c.pos = pos;c.win.resize(L.size());c.comp.resize(L.size() * 2);@@ -4574,7 +4583,7 @@ struct Model::Impl {// the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1int best_check(int64_t limit) const {int b = -1;- for (int i = 0; i < 2; ++i)+ for (int i = 0; i < kChecks; ++i)if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;return b;}@@ -4583,6 +4592,33 @@ struct Model::Impl {return b < 0 ? -1 : ck[b].pos;}+ // checkpoint 1 (the position before the last prompt's end) becomes the anchor too+ void pin() {+ if (ck[1].pos > 0) ck[2] = ck[1];+ }+ // the checkpoints the state can go back to (what a slot of it keeps), latest first+ std::vector<int64_t> check_positions() const {+ std::vector<int64_t> v;+ for (const Check& c : ck)+ if (c.pos > 0 && c.pos <= pos && c.win.size() == L.size() &&+ std::find(v.begin(), v.end(), c.pos) == v.end())+ v.push_back(c.pos);+ std::sort(v.rbegin(), v.rend());+ return v;+ }+ // the checkpoints a slot keeps (each with its role: 0, 1 or the anchor)+ std::vector<int> slot_checks() const {+ std::vector<int> v;+ for (int i = 0; i < kChecks; ++i)+ if (ck[i].pos > 0 && ck[i].pos <= pos && ck[i].win.size() == L.size()) v.push_back(i);+ return v;+ }+ size_t check_bytes() const {+ size_t n = 16;+ for (const Layer& l : L) n += (size_t) g.win * win_rb() + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);+ return n;+ }+bool restore(int64_t limit) {const int b = best_check(limit);if (b < 0) return false;@@ -4606,11 +4642,362 @@ struct Model::Impl {pos = c.pos;toks.resize((size_t) pos);ctoks.resize((size_t) pos);+ slot_held = false;for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer hasif (o2.pos > pos) o2.pos = -1;return true;}+ // ---- the kept conversations (a slot): the state out of the model and back, through a file or a host buffer.+ // A prompt that continues the state is reused in place (checkpoint / restore, the compressed KV left where it is);+ // a conversation that leaves the model for another one is a slot - the whole state: the position, its tokens,+ // every window ring (the DSpark blocks' included), the compressors' partial groups and each KV source's compressed+ // KV and index keys up to it (rows 0 .. pos / ratio, the groups its position completed; one the prompt has not+ // completed yet lives in the compressor's partial group). A tensor parallel group's members hold the same state,+ // so one member's ring is saved and written back into all of them. Text only: a conversation that holds pictures+ // is not a slot (the Engram history's rows are the tokens' again, no -1 among them).+ static constexpr uint64_t kSlotMagic = 0x31544f4c53345344ull; // "DS4SLOT1" (little endian)++ int64_t src_rows(const Src& s) const { return pos / std::max(1, s.ratio); }++ // the slot's bytes: the header, the tokens, every layer's ring and partial group, every source's rows, the end+ size_t slot_bytes() const {+ size_t n = 8 + 8 + slot_identity().size() + 8 + (size_t) pos * 4;+ for (const Layer& l : L) n += 8 + (size_t) g.win * win_rb() + 8 + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);+ for (const Src& s : srcs) n += 8 + (size_t) src_rows(s) * (ckv_rb() + ik_rb());+ n += 8 + slot_checks().size() * check_bytes(); // (the checkpoints: what the slot can be taken back at)+ return n + 8;+ }++ uint64_t slot_save(std::vector<uint8_t>& out, std::string& err) {+ if (slot_held) {+ err = "it is already set aside";+ return 0;+ }+ if (pos <= 0) {+ err = "there is no conversation";+ return 0;+ }+ if ((int64_t) toks.size() != pos) {+ err = "the state's tokens do not match its position";+ return 0;+ }+ if (!imgs.empty()) {+ err = "it holds pictures";+ return 0;+ }+ const std::string id = slot_identity();+ out.assign(slot_bytes(), 0);+ uint8_t* p = out.data();+ size_t off = 0;+ const auto put = [&](const void* s, size_t b) {+ std::memcpy(p + off, s, b);+ off += b;+ };+ const uint64_t il = id.size(), n = (uint64_t) pos;+ put(&kSlotMagic, 8);+ put(&il, 8);+ put(id.data(), (size_t) il);+ put(&n, 8);+ put(toks.data(), (size_t) n * 4);+ for (Dev& d : devs) { // (the devices' work ends before their state is read)+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ for (size_t i = 0; i < L.size(); ++i) {+ const Layer& l = L[i];+ const Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ const uint64_t wb = (uint64_t) g.win * win_rb();+ put(&wb, 8);+ if (cudaMemcpy(p + off, l.win, (size_t) wb, cudaMemcpyDeviceToHost) != cudaSuccess) {+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());+ return 0;+ }+ off += (size_t) wb;+ const uint64_t cn = l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0;+ put(&cn, 8);+ if (cn > 0) {+ const size_t cb = (size_t) cn * 4;+ if (cudaMemcpy(p + off, l.c_kv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||+ cudaMemcpy(p + off + cb, l.c_score, cb, cudaMemcpyDeviceToHost) != cudaSuccess) {+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());+ return 0;+ }+ off += 2 * cb;+ }+ }+ for (const Src& s : srcs) {+ const uint64_t rows = (uint64_t) src_rows(s);+ put(&rows, 8);+ if (rows == 0) continue;+ const int di = L[(size_t) s.layer].dev;+ const uint8_t* ckv = s.ckv[(size_t) di];+ const uint8_t* ik = s.ik[(size_t) di]; // (a pack whose sources are not index sources: none, zeros)+ if (ckv == nullptr) {+ err = "source layer " + std::to_string(s.layer) + "'s rows are not on its device";+ return 0;+ }+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();+ cudaSetDevice(devs[(size_t) di].id);+ if (cudaMemcpy(p + off, ckv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||+ (ik != nullptr && cudaMemcpy(p + off + cb, ik, ib, cudaMemcpyDeviceToHost) != cudaSuccess)) {+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());+ return 0;+ }+ off += cb + ib;+ }+ {+ const std::vector<int> cs = slot_checks();+ const uint64_t nc = cs.size();+ put(&nc, 8);+ for (int k : cs) {+ const Check& c = ck[k];+ const uint64_t role = (uint64_t) k, cp = (uint64_t) c.pos;+ put(&role, 8);+ put(&cp, 8);+ for (size_t i = 0; i < L.size(); ++i) {+ put(c.win[i].data(), (size_t) g.win * win_rb());+ if (L[i].c_kv) {+ put(c.comp[2 * i].data(), (size_t) L[i].ratio * g.hd * 4);+ put(c.comp[2 * i + 1].data(), (size_t) L[i].ratio * g.hd * 4);+ }+ }+ }+ }+ if (off + 8 != out.size()) {+ err = "the state's size does not add up";+ return 0;+ }+ put(&kSlotMagic, 8);+ slot_held = true;+ return (uint64_t) off;+ }++ bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {+ size_t off = 0;+ bool wrote = false, ok = false;+ // a load that stops after some of another conversation's state went into the layers leaves a state no+ // checkpoint can repair: it is reset (the prompt is read from token 0), never served from the mix+ struct FailClosed {+ Impl& p;+ const bool& wrote;+ const bool& ok;+ ~FailClosed() {+ if (!ok && wrote) p.reset();+ }+ } guard{*this, wrote, ok};+ const auto room = [&](size_t b) { return b <= bytes && off <= bytes - b; };+ uint64_t magic = 0, il = 0, n = 0;+ if (!room(16)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&magic, data + off, 8);+ std::memcpy(&il, data + off + 8, 8);+ off += 16;+ if (magic != kSlotMagic || il == 0 || il > (1u << 20)) {+ err = "not a saved conversation";+ return false;+ }+ if (!room((size_t) il + 8)) {+ err = "the slot is cut short";+ return false;+ }+ {+ std::string id((size_t) il, '\0');+ std::memcpy(id.data(), data + off, (size_t) il);+ off += (size_t) il;+ std::memcpy(&n, data + off, 8);+ off += 8;+ if (id != slot_identity()) {+ err = "another model's, or another engine version's, conversation";+ return false;+ }+ }+ if (n != (uint64_t) n_pos || n_pos <= 0 || n_pos > o.max_context || !room((size_t) n * 4)) {+ err = "the conversation's position is not this prompt's";+ return false;+ }+ std::vector<int32_t> toks_in((size_t) n);+ std::memcpy(toks_in.data(), data + off, (size_t) n * 4);+ off += (size_t) n * 4;+ for (Dev& d : devs) { // (their work ends before the state is written over)+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ for (size_t i = 0; i < L.size(); ++i) {+ Layer& l = L[i];+ const Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ uint64_t wb = 0, cn = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&wb, data + off, 8);+ off += 8;+ if (wb != (uint64_t) g.win * win_rb() || !room((size_t) wb + 8)) {+ err = l.name + "window ring of another size";+ return false;+ }+ wrote = true;+ for (int m = 0; m < l.nm; ++m) {+ Layer& lm = member(l, m);+ if (cudaMemcpy(lm.win, data + off, (size_t) wb, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ }+ off += (size_t) wb;+ std::memcpy(&cn, data + off, 8);+ off += 8;+ if (cn != (l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0)) {+ err = l.name + "compressor's group of another size";+ return false;+ }+ if (cn > 0) {+ const size_t cb = (size_t) cn * 4;+ if (!room(2 * cb)) {+ err = "the slot is cut short";+ return false;+ }+ for (int m = 0; m < l.nm; ++m) {+ Layer& lm = member(l, m);+ if (cudaMemcpy(lm.c_kv, data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess ||+ cudaMemcpy(lm.c_score, data + off + cb, cb, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ }+ off += 2 * cb;+ }+ }+ for (const Src& s : srcs) {+ uint64_t rows = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&rows, data + off, 8);+ off += 8;+ if (rows != (uint64_t) (n_pos / std::max(1, s.ratio))) {+ err = "source layer " + std::to_string(s.layer) + "'s rows are not this conversation's";+ return false;+ }+ if (rows == 0) continue;+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();+ if (!room(cb + ib)) {+ err = "the slot is cut short";+ return false;+ }+ for (size_t di = 0; di < devs.size(); ++di) {+ if (s.ckv[di] == nullptr && s.ik[di] == nullptr) continue;+ cudaSetDevice(devs[di].id);+ if (s.ckv[di] != nullptr && cudaMemcpy(s.ckv[di], data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ if (s.ik[di] != nullptr &&+ cudaMemcpy(s.ik[di], data + off + cb, ib, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = "source layer " + std::to_string(s.layer) + "'s index keys: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ }+ off += cb + ib;+ }+ Check got[kChecks]; // the slot's checkpoints: the positions it can be taken back at+ {+ uint64_t nc = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&nc, data + off, 8);+ off += 8;+ if (nc > (uint64_t) kChecks) {+ err = "the slot's checkpoints are damaged";+ return false;+ }+ for (uint64_t k = 0; k < nc; ++k) {+ uint64_t role = 0, cp = 0;+ if (!room(16)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&role, data + off, 8);+ std::memcpy(&cp, data + off + 8, 8);+ off += 16;+ if (role >= (uint64_t) kChecks || got[role].pos >= 0 || cp == 0 || cp > (uint64_t) n_pos) {+ err = "the slot's checkpoints are damaged";+ return false;+ }+ Check& c = got[role];+ c.pos = (int64_t) cp;+ c.win.resize(L.size());+ c.comp.resize(L.size() * 2);+ for (size_t i = 0; i < L.size(); ++i) {+ const size_t wb = (size_t) g.win * win_rb(), cb = L[i].c_kv ? (size_t) L[i].ratio * g.hd * 4 : 0;+ if (!room(wb + 2 * cb)) {+ err = "the slot is cut short";+ return false;+ }+ c.win[i].assign(data + off, data + off + wb);+ off += wb;+ if (cb > 0) {+ c.comp[2 * i].resize(cb / 4);+ c.comp[2 * i + 1].resize(cb / 4);+ std::memcpy(c.comp[2 * i].data(), data + off, cb);+ std::memcpy(c.comp[2 * i + 1].data(), data + off + cb, cb);+ off += 2 * cb;+ }+ }+ }+ }+ uint64_t end = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&end, data + off, 8);+ off += 8;+ if (end != kSlotMagic) {+ err = "the slot does not end where it should";+ return false;+ }+ pos = n_pos;+ toks.swap(toks_in);+ ctoks.resize((size_t) pos);+ for (size_t i = 0; i < (size_t) pos; ++i)+ ctoks[i] = toks[i] < pk.eg_vocab ? pk.eg_map[(size_t) toks[i]] : 0;+ for (int k = 0; k < kChecks; ++k) ck[k] = std::move(got[k]); // (the slot's own; the old ones held the state+ // this one replaced)+ slot_held = false;+ ok = true;+ return true;+ }++ std::string slot_identity() const {+ const auto size_of = [](const std::string& p) {+ std::error_code ec;+ const auto s = std::filesystem::file_size(p, ec);+ return ec ? 0ull : (unsigned long long) s;+ };+ std::ostringstream id;+ id << "strata-ds4-session-3 engine=" << STRATA_VERSION << " mtp=" << g.n_mtp << " dim=" << g.dim << " hd=" << g.hd+ << " win=" << g.win << " ratios=";+ for (int r : g.ratios) id << r << ",";+ id << " kv=";+ for (int v : g.kv_src) id << v << ",";+ id << " ix=";+ for (int v : g.ix_src) id << v << ",";+ id << " ds4.txt:" << size_of(pk.dir + "/ds4.txt") << " engram.bin:" << size_of(pk.dir + "/engram.bin")+ << " tensors.txt:" << size_of(pk.dir + "/tensors.txt");+ for (size_t i = 0; i < pk.files.size(); ++i)+ id << " " << pk.files[i] << ":" << os::file_size(pk.fds[i]);+ return id.str();+ }+size_t bounce_bytes() const {return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);}@@ -6211,6 +6598,7 @@ struct Model::Impl {pos = p0 + keep;toks.resize((size_t) pos);ctoks.resize((size_t) pos);+ slot_held = false;}// ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows@@ -7583,6 +7971,7 @@ struct Model::Impl {return false;}const int64_t p0 = pos;+ slot_held = false; // (the state grows now: what slot_save took is no longer the conversation)for (int t = 0; t < T; ++t) {toks.push_back(tk[t]);ctoks.push_back(ctok(pos + t, tk[t]));@@ -8404,7 +8793,14 @@ const std::vector<int32_t>& Model::tokens() const { return p_->toks; }void Model::reset() { p_->reset(); }void Model::checkpoint(int slot) { p_->checkpoint(slot); }bool Model::restore(int64_t limit) { return p_->restore(limit); }+void Model::pin() { p_->pin(); }+std::vector<int64_t> Model::check_positions() const { return p_->check_positions(); }int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }+uint64_t Model::slot_save(std::vector<uint8_t>& out, std::string& err) { return p_->slot_save(out, err); }+bool Model::slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {+ return p_->slot_load(data, bytes, n_pos, err);+}+std::string Model::slot_identity() const { return p_->slot_identity(); }void Model::save_profile() const { p_->save_profile(); }const Stats& Model::stats() const {Stats& st = p_->st;diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex 929636d53..b4c35f0b2 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -8,7 +8,9 @@// its own 128-token window, the compressed groups completed by its position. The state a token leaves is the// 128-token window ring of every layer, the compressors' partial groups, the compressed KV and index keys of the KV// source layers (append-only), and the Engram n-gram history; checkpoint() / restore() keep the first two (the rest is-// truncated by position) so a conversation can go back to the end of its last prompt.+// truncated by position) so a conversation can go back to the end of its last prompt. A conversation that leaves the+// model altogether (another chat answered in between) is a slot: slot_save / slot_load take the same state - the+// compressed KV and index keys included, which a checkpoint leaves in place - through a file or a host buffer.#pragma once#include <cstddef>@@ -146,6 +148,23 @@ public:void checkpoint(int slot = 0);bool restore(int64_t limit);int64_t restorable(int64_t limit) const;+ /// Checkpoint 1 also becomes the anchor (kept until a restore goes before it): the position before the end of the+ /// first prompt of a run that continued in place, where a re-rendered history (earlier reasoning dropped) splits.+ void pin();+ /// The positions restore() can go back to, latest first (a slot keeps them: it can be taken back at any of them).+ std::vector<int64_t> check_positions() const;++ /// The conversation in the model into `out` (slot_save): its position, its tokens, every window ring (the DSpark+ /// blocks' included), the compressors' partial groups and each KV source's compressed KV and index keys up to it,+ /// the pieces the state a prompt continues is made of. 0 and err: nothing was set aside (it holds pictures, there+ /// is no conversation, or the conversation is already set aside - unchanged since).+ uint64_t slot_save(std::vector<uint8_t>& out, std::string& err);+ /// `data` (a slot's bytes) back into the model: the state at the slot's position n_pos, as slot_save took it -+ /// what a prompt that continues the slot's tokens reads on from (the caller checked the tokens are a prefix).+ bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err);+ /// What a saved conversation must match: this engine's version, this model's geometry and files and the drafter's+ /// blocks - a slot naming anything else is not this model's state (a file's .meta says so; slot_load refuses).+ std::string slot_identity() const;/// The vision tower: the token every position of an image's span carries (-1: no tower).int image_token() const;diff --git a/src/dsv41/ds4_slotmeta.hpp b/src/dsv41/ds4_slotmeta.hppnew file mode 100644index 000000000..dab9fea3e--- /dev/null+++ b/src/dsv41/ds4_slotmeta.hpp@@ -0,0 +1,98 @@+// The kept-conversation (slot) bookkeeping that needs no GPU: a slot's .meta file (identity, size, tokens, checkpoint+// positions) and the choice of which kept conversation a prompt continues, and from where. The engine's serve loop+// (ds4_main.cpp) uses these; tests/dsv41/slot_test.cpp checks them alone.+//+// Slot .meta layout (little endian), "DS4META2":+// u64 magic, u64 identity length, identity bytes, u64 .bin bytes, u64 token count, i32 tokens[],+// u64 checkpoint count (0..4), i64 checkpoints[], u64 magic+// The magic at both ends and the .bin size make a half-written or foreign file refused, never read.+#pragma once++#include <algorithm>+#include <cstddef>+#include <cstdint>+#include <filesystem>+#include <fstream>+#include <string>+#include <vector>++namespace strata::ds4::slotmeta {++constexpr uint64_t kMagic = 0x324154454d345344ull; // "DS4META2"++// Written under a temporary name and renamed over `path` (an interrupted write leaves no half file).+inline bool write(const std::string& path, const std::string& identity, const std::vector<int32_t>& toks,+ uint64_t bytes, const std::vector<int64_t>& cks) {+ const std::string tmp = path + ".tmp";+ {+ std::ofstream f(tmp, std::ios::binary | std::ios::trunc);+ const uint64_t il = identity.size(), nt = toks.size(), nc = cks.size();+ f.write((const char*) &kMagic, 8);+ f.write((const char*) &il, 8);+ f.write(identity.data(), (std::streamsize) il);+ f.write((const char*) &bytes, 8);+ f.write((const char*) &nt, 8);+ f.write((const char*) toks.data(), (std::streamsize) (nt * sizeof(int32_t)));+ f.write((const char*) &nc, 8);+ f.write((const char*) cks.data(), (std::streamsize) (nc * sizeof(int64_t)));+ f.write((const char*) &kMagic, 8);+ f.flush();+ if (!f) return false;+ }+ std::error_code ec;+ std::filesystem::rename(tmp, path, ec);+ return !ec;+}++// false: foreign (another identity), damaged, cut short, or not plausible (no tokens, more than max_ctx, a checkpoint+// outside the tokens).+inline bool read(const std::string& path, const std::string& identity, int64_t max_ctx, std::vector<int32_t>& toks,+ uint64_t& bytes, std::vector<int64_t>& cks) {+ std::ifstream f(path, std::ios::binary);+ uint64_t m0 = 0, il = 0, nt = 0, nc = 0, m1 = 0;+ if (!f.read((char*) &m0, 8) || m0 != kMagic || !f.read((char*) &il, 8) || il != identity.size()) return false;+ std::string id(il, '\0');+ if (!f.read(id.data(), (std::streamsize) il) || id != identity) return false;+ if (!f.read((char*) &bytes, 8) || !f.read((char*) &nt, 8) || nt == 0 || nt > (uint64_t) max_ctx) return false;+ toks.resize(nt);+ if (!f.read((char*) toks.data(), (std::streamsize) (nt * sizeof(int32_t))) || !f.read((char*) &nc, 8) || nc > 4)+ return false;+ cks.resize(nc);+ if (!f.read((char*) cks.data(), (std::streamsize) (nc * sizeof(int64_t))) || !f.read((char*) &m1, 8) || m1 != kMagic)+ return false;+ for (int64_t c : cks)+ if (c <= 0 || (uint64_t) c > nt) return false;+ return true;+}++// The kept conversation `prompt` continues the furthest, and where: *len is the number of the slot's tokens that are+// the prompt's (0: none). A conversation the prompt continues whole comes back whole (len == its size); one the prompt+// leaves part way - a retried prompt, a history re-rendered without the earlier reasoning - comes back at its latest+// checkpoint the prompt still agrees with. Only a match longer than `floor` (what the model itself can go back to)+// counts. The last prompt token is never covered (a prompt always reads its last token). Ties keep the earlier slot.+struct Cand {+ const std::vector<int32_t>* tokens;+ const std::vector<int64_t>* cks;+};+inline size_t pick(const std::vector<Cand>& slots, const std::vector<int>& prompt, size_t floor, size_t* len) {+ *len = 0;+ if (prompt.size() < 2) return SIZE_MAX;+ size_t best = SIZE_MAX;+ for (size_t i = 0; i < slots.size(); ++i) {+ const std::vector<int32_t>& t = *slots[i].tokens;+ const size_t lim = std::min(t.size(), prompt.size() - 1);+ size_t c = 0;+ while (c < lim && t[c] == prompt[c]) ++c;+ size_t u = c == t.size() ? c : 0;+ if (u == 0)+ for (int64_t k : *slots[i].cks)+ if ((size_t) k <= c && (size_t) k > u) u = (size_t) k;+ if (u > *len && u > floor) {+ best = i;+ *len = u;+ }+ }+ return best;+}++} // namespace strata::ds4::slotmetadiff --git a/tests/dsv41/slot_test.cpp b/tests/dsv41/slot_test.cppnew file mode 100644index 000000000..0235b05c1--- /dev/null+++ b/tests/dsv41/slot_test.cpp@@ -0,0 +1,153 @@+// The kept conversations' bookkeeping without a GPU (src/dsv41/ds4_slotmeta.hpp): the .meta file's round trip and+// every way it is refused, and which kept conversation a prompt continues, and from where.+// ds4_slot_test exit 0: all passed+#include "ds4_slotmeta.hpp"++#include <cstdio>+#include <cstring>+#include <filesystem>+#include <string>+#include <vector>++namespace sm = strata::ds4::slotmeta;+namespace fs = std::filesystem;++static int fails = 0;+#define CHECK(c) \+ do { \+ if (!(c)) { \+ std::printf("FAIL line %d: %s\n", __LINE__, #c); \+ ++fails; \+ } \+ } while (0)++static std::vector<int32_t> seq(int a, int n) {+ std::vector<int32_t> v((size_t) n);+ for (int i = 0; i < n; ++i) v[(size_t) i] = a + i;+ return v;+}+static std::vector<int> as_int(const std::vector<int32_t>& v) { return std::vector<int>(v.begin(), v.end()); }++static void patch(const std::string& p, size_t off, uint8_t b) {+ std::FILE* f = std::fopen(p.c_str(), "r+b");+ std::fseek(f, (long) off, SEEK_SET);+ std::fwrite(&b, 1, 1, f);+ std::fclose(f);+}++static void meta_tests(const fs::path& dir) {+ const std::string id = "strata-ds4-session-2 engine=test", path = (dir / "slot-1.meta").string();+ const std::vector<int32_t> toks = seq(100, 50);+ const std::vector<int64_t> cks = {40, 33, 7};+ CHECK(sm::write(path, id, toks, 123456789ull, cks));+ CHECK(!fs::exists(path + ".tmp")); // (renamed over, no half file left)+ std::vector<int32_t> t;+ std::vector<int64_t> c;+ uint64_t bytes = 0;+ CHECK(sm::read(path, id, 1000, t, bytes, c));+ CHECK(t == toks && c == cks && bytes == 123456789ull);+ // no checkpoints is valid (a slot of a state that had none)+ CHECK(sm::write(path, id, toks, 5, {}));+ CHECK(sm::read(path, id, 1000, t, bytes, c) && c.empty() && bytes == 5);+ // another model's / engine's identity: refused, never read+ CHECK(!sm::read(path, id + "x", 1000, t, bytes, c));+ CHECK(!sm::read(path, "other", 1000, t, bytes, c));+ // more tokens than the context: refused+ CHECK(!sm::read(path, id, 49, t, bytes, c));+ CHECK(sm::read(path, id, 50, t, bytes, c));+ // a missing file+ CHECK(!sm::read((dir / "none.meta").string(), id, 1000, t, bytes, c));+ // every cut length, a damaged magic at either end+ CHECK(sm::write(path, id, toks, 9, cks));+ const size_t full = (size_t) fs::file_size(path);+ for (size_t keep = 0; keep < full; ++keep) {+ fs::copy_file(path, dir / "cut.meta", fs::copy_options::overwrite_existing);+ fs::resize_file(dir / "cut.meta", keep);+ if (sm::read((dir / "cut.meta").string(), id, 1000, t, bytes, c)) {+ std::printf("FAIL: a .meta cut to %zu of %zu bytes was read\n", keep, full);+ ++fails;+ break;+ }+ }+ fs::copy_file(path, dir / "bad0.meta", fs::copy_options::overwrite_existing);+ patch((dir / "bad0.meta").string(), 0, 0x55);+ CHECK(!sm::read((dir / "bad0.meta").string(), id, 1000, t, bytes, c));+ fs::copy_file(path, dir / "bad1.meta", fs::copy_options::overwrite_existing);+ patch((dir / "bad1.meta").string(), full - 1, 0x55);+ CHECK(!sm::read((dir / "bad1.meta").string(), id, 1000, t, bytes, c));+ // a checkpoint outside the conversation, a zero one, more than four+ CHECK(sm::write(path, id, toks, 9, {51}));+ CHECK(!sm::read(path, id, 1000, t, bytes, c));+ CHECK(sm::write(path, id, toks, 9, {0}));+ CHECK(!sm::read(path, id, 1000, t, bytes, c));+ CHECK(sm::write(path, id, toks, 9, {4, 3, 2, 1}));+ CHECK(sm::read(path, id, 1000, t, bytes, c) && c.size() == 4);+ CHECK(sm::write(path, id, toks, 9, {1, 2, 3, 4, 5}));+ CHECK(!sm::read(path, id, 1000, t, bytes, c));+ // no tokens at all+ CHECK(sm::write(path, id, {}, 9, {}));+ CHECK(!sm::read(path, id, 1000, t, bytes, c));+ // an unwritable place: false, not a crash+ CHECK(!sm::write((dir / "nodir" / "x.meta").string(), id, toks, 1, {}));+}++static size_t pick(const std::vector<std::vector<int32_t>>& toks, const std::vector<std::vector<int64_t>>& cks,+ const std::vector<int>& prompt, size_t floor, size_t* len) {+ std::vector<sm::Cand> c;+ for (size_t i = 0; i < toks.size(); ++i) c.push_back({&toks[i], &cks[i]});+ return sm::pick(c, prompt, floor, len);+}++static void pick_tests() {+ size_t len = 99;+ const std::vector<int32_t> a = seq(1000, 3000), b = seq(5000, 2000);+ // the next turn of A: A whole, then new tokens+ std::vector<int32_t> a_turn = a;+ for (int32_t x : seq(9000, 40)) a_turn.push_back(x);+ CHECK(pick({a, b}, {{}, {}}, as_int(a_turn), 0, &len) == 0 && len == a.size());+ CHECK(pick({b, a}, {{}, {}}, as_int(a_turn), 0, &len) == 1 && len == a.size());+ // nothing shared: none, len 0+ CHECK(pick({a, b}, {{}, {}}, as_int(seq(7777, 100)), 0, &len) == SIZE_MAX && len == 0);+ // a prompt equal to the slot's tokens: the last token is still read, so the slot is not taken whole+ CHECK(pick({a}, {{}}, as_int(a), 0, &len) == SIZE_MAX && len == 0);+ CHECK(pick({a}, {{2900}}, as_int(a), 0, &len) == 0 && len == 2900);+ // a shorter prompt than the slot (a prefix of it): taken at a checkpoint within it, never past it+ std::vector<int32_t> head(a.begin(), a.begin() + 2000);+ CHECK(pick({a}, {{2900, 1500}}, as_int(head), 0, &len) == 0 && len == 1500);+ // a retried prompt: A's prompt end (2950) is a checkpoint; the new prompt agrees up to 2960 then differs+ std::vector<int32_t> retry(a.begin(), a.begin() + 2960);+ for (int32_t x : seq(8000, 30)) retry.push_back(x);+ CHECK(pick({a}, {{2950, 2000, 1000}}, as_int(retry), 0, &len) == 0 && len == 2950);+ // diverging before the latest checkpoint: the latest one it still agrees with (the anchor)+ std::vector<int32_t> early(a.begin(), a.begin() + 1200);+ for (int32_t x : seq(8000, 30)) early.push_back(x);+ CHECK(pick({a}, {{2950, 2000, 1000}}, as_int(early), 0, &len) == 0 && len == 1000);+ // diverging before every checkpoint: none+ std::vector<int32_t> first(a.begin(), a.begin() + 500);+ for (int32_t x : seq(8000, 30)) first.push_back(x);+ CHECK(pick({a}, {{2950, 2000, 1000}}, as_int(first), 0, &len) == SIZE_MAX && len == 0);+ // the floor: what the model itself can go back to wins unless the slot goes further+ CHECK(pick({a}, {{}}, as_int(a_turn), 3000, &len) == SIZE_MAX && len == 0);+ CHECK(pick({a}, {{}}, as_int(a_turn), 2999, &len) == 0 && len == 3000);+ // two slots, the longer match wins; a tie keeps the earlier one+ std::vector<int32_t> a_short(a.begin(), a.begin() + 1000);+ CHECK(pick({a_short, a}, {{}, {}}, as_int(a_turn), 0, &len) == 1 && len == 3000);+ CHECK(pick({a, a}, {{}, {}}, as_int(a_turn), 0, &len) == 0 && len == 3000);+ // too short a prompt+ CHECK(pick({a}, {{}}, {1000}, 0, &len) == SIZE_MAX && len == 0);+ CHECK(pick({a}, {{}}, {}, 0, &len) == SIZE_MAX && len == 0);+ // an empty slot list+ CHECK(pick({}, {}, as_int(a_turn), 0, &len) == SIZE_MAX && len == 0);+}++int main() {+ const fs::path dir = fs::temp_directory_path() / "ds4-slot-test";+ std::error_code ec;+ fs::remove_all(dir, ec);+ fs::create_directories(dir);+ meta_tests(dir);+ pick_tests();+ fs::remove_all(dir, ec);+ std::printf(fails ? "FAILED: %d\n" : "slot_test: all passed\n", fails);+ return fails ? 1 : 0;+}