Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: keep a conversation another request displaces and take it back at its checkpoints (STRATA_DS4_SLOT_KEEP), shared-start checkpoint (pin=N)

Niko1221cdc3a265773f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c46b305c7..bbe296b20 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -884,6 +884,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
if(STRATA_ENABLE_HIP)
target_compile_options(ds4_iq_test PRIVATE "$<$<COMPILE_LANGUAGE:HIP,CXX>:SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/include/strata/dsv41/hip_extra.h>")
endif()
+ # the kept conversations' .meta file and prefix matching (ds4_slotmeta.hpp): no GPU, no model
+ add_executable(ds4_slot_test tests/dsv41/slot_test.cpp)
+ target_include_directories(ds4_slot_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)
+ target_compile_features(ds4_slot_test PRIVATE cxx_std_17)
endif()
if(STRATA_ENABLE_HIP)
foreach(_t strata_dsv41_kernels strata-dsv41)
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index e614e6cb1..0ea0c1517 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -11,16 +11,27 @@
// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that
// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to
// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures
-// (the file: see read_pics); the state is shared only where the same picture sits at the same place.
+// (the file: see read_pics); the state is shared only where the same picture sits at the same place. With
+// STRATA_DS4_SLOT_KEEP=1 the conversations are kept when another one takes the model (a slot each: the state, the
+// compressed KV included), and across runs in STRATA_DS4_SLOT_DIR - so a request that does not continue the state but
+// does continue a kept conversation reads only its new tokens.
+// `pin=N` on a GEN marks the first N prompt tokens as a start other requests share (a system prompt and its tools): the
+// read stops there for a checkpoint, so a request behind the same start - another agent - goes back to it instead of
+// reading it again (also without slots). STRATA_DS4_REUSE=0 turns every reuse off (each request reads its whole prompt).
#include "ds4_model.hpp"
#include "ds4_packinfo.hpp"
+#include "ds4_slotmeta.hpp"
#include <algorithm>
#include <cmath>
#include <numeric>
#include <atomic>
#include <chrono>
+#include <climits>
#include <condition_variable>
+#include <csignal>
+#include <filesystem>
+#include <memory>
#include <cstdio>
#include <fstream>
#include <sstream>
@@ -182,6 +193,9 @@ struct Req {
unsigned long long seed = 0;
int spec = -1; // spec=0|1: this request drafts or not (-1: as the engine was started)
float conf = -1.0f; // dspark_conf=F: the drafts' confidence floor for this request (-1: the engine's)
+ long long pin = 0; // pin=N: the first N prompt tokens are a start other requests share (a system prompt and its
+ // tools): the state at N is kept as a checkpoint, so another agent behind the same start
+ // goes back to it instead of reading it again
std::string file; // GENI: the pictures
std::vector<int> ids;
};
@@ -277,11 +291,12 @@ bool parse_gen(const std::string& line, Req& r, bool geni) {
else if (k == "seed") r.seed = std::strtoull(v.c_str(), nullptr, 10);
else if (k == "spec") r.spec = std::atoi(v.c_str()) != 0;
else if (k == "dspark_conf") r.conf = std::strtof(v.c_str(), nullptr);
+ else if (k == "pin") r.pin = std::atoll(v.c_str());
}
return !r.ids.empty() && (!geni || !r.file.empty());
}
-int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float conf_min) {
+int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float conf_min, const std::string& pack) {
std::vector<int> eos = parse_ids(eos_list);
for (const std::string& l : m.info_lines()) std::printf("%s\n", l.c_str());
std::printf("READY %lld stop\n", (long long) m.max_context());
@@ -294,6 +309,321 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
std::vector<Pic> held; // the pictures the state holds (start, hash, span)
SpanCache spans;
const int img_tok = m.image_token();
+ // CONVERSATION SLOTS: a request that does not continue the conversation in the model first sets that one aside
+ // (Model::slot_save: its window rings, the compressors' partial groups and every source's compressed KV and index
+ // keys), then takes back the kept conversation this prompt does continue (slot_load) - chats, and an agent's side
+ // requests, that interleave no longer re-read each other's prompts (70k and 185k tokens alternating: 75-200 s a
+ // switch at ~930 tok/s). STRATA_DS4_SLOT_KEEP=1 turns them on and keeps them across runs, in STRATA_DS4_SLOT_DIR
+ // (default <pack>/slots): a slot is a .bin (the state) and a .meta (this model's identity, the conversation's
+ // tokens and its size), both written under a temporary name first; a slot whose .meta is missing, foreign or does
+ // not match its file, or unused for STRATA_DS4_SLOT_DAYS (7) days, is removed at start. STRATA_DS4_SLOT_CAP (4) is
+ // the conversations kept in RAM; past it the least recently used is written to the folder and read back from there
+ // (RAM first, the folder past the cap), and the engine's end writes the ones in RAM to the folder, so the next run
+ // reads on from them. STRATA_DS4_SLOT_GB (16) caps the folder's .bin files: at start, and after every write, the
+ // folder's least recently used conversation is removed while they are over it (never the one just written, so a
+ // conversation bigger than the cap stays). STRATA_DS4_SLOT_MIN (1024) is the shortest conversation set aside (a
+ // shorter one is read again in about the time its slot would take). Text only: a conversation that holds pictures
+ // is not set aside, and a prompt with pictures takes none back.
+ const auto env_num = [](const char* k, double d) { const char* v = std::getenv(k); return v ? std::atof(v) : d; };
+ const bool slot_keep =
+ std::getenv("STRATA_DS4_SLOT_KEEP") != nullptr && std::atoi(std::getenv("STRATA_DS4_SLOT_KEEP")) != 0;
+ const bool reuse = std::getenv("STRATA_DS4_REUSE") == nullptr || std::atoi(std::getenv("STRATA_DS4_REUSE")) != 0;
+ int slot_max = slot_keep ? std::max(0, (int) env_num("STRATA_DS4_SLOT_CAP", 4)) : 0;
+ const size_t slot_min = (size_t) std::max(1.0, env_num("STRATA_DS4_SLOT_MIN", 1024));
+ const double slot_days = env_num("STRATA_DS4_SLOT_DAYS", 7);
+ const uint64_t slot_budget = (uint64_t) (env_num("STRATA_DS4_SLOT_GB", 16) * 1073741824.0);
+ namespace fs = std::filesystem;
+ std::filesystem::path slot_dir;
+ struct Slot {
+ std::vector<int32_t> tokens;
+ std::string path; // empty: the state is in data (RAM)
+ uint64_t bytes = 0, used = 0;
+ std::shared_ptr<std::vector<uint8_t>> data; // the RAM slot's bytes (null: a file slot)
+ std::vector<int64_t> cks; // the positions short of its end it can be taken back at
+ };
+ std::vector<Slot> slots;
+ uint64_t slot_clock = 0, slot_seq = 0;
+ const std::string slot_identity = m.slot_identity();
+ const auto note = [&](const std::string& msg) {
+ std::fprintf(stderr, "ds4 slots: %s\n", msg.c_str());
+ std::printf("NOTE sessions: %s\n", msg.c_str());
+ std::fflush(stdout);
+ };
+ const auto gb = [](uint64_t b) {
+ char s[32];
+ std::snprintf(s, sizeof s, "%.2f GB", (double) b / 1e9);
+ return std::string(s);
+ };
+ const auto meta_of = [](const std::string& bin) { return bin.substr(0, bin.size() - 4) + ".meta"; };
+ const auto meta_write = [&](const std::string& path, const std::vector<int32_t>& toks, uint64_t bytes,
+ const std::vector<int64_t>& cks) -> bool {
+ return strata::ds4::slotmeta::write(path, slot_identity, toks, bytes, cks);
+ };
+ // a .meta of this model -> its tokens and the slot's size (false: foreign, damaged or not this model's)
+ const auto meta_read = [&](const std::string& path, std::vector<int32_t>& toks, uint64_t& bytes,
+ std::vector<int64_t>& cks) -> bool {
+ return strata::ds4::slotmeta::read(path, slot_identity, m.max_context(), toks, bytes, cks);
+ };
+ const auto slot_age_days = [](const std::string& p) {
+ std::error_code ec;
+ const auto t = fs::last_write_time(p, ec);
+ if (ec) return 1e9;
+ return std::chrono::duration<double>(fs::file_time_type::clock::now() - t).count() / 86400.0;
+ };
+ const auto slot_drop = [&](size_t i) {
+ if (slots[i].data) // a RAM slot: its buffer simply dies
+ slots[i].data.reset();
+ else {
+ std::error_code ec;
+ std::filesystem::remove(slots[i].path, ec);
+ std::filesystem::remove(meta_of(slots[i].path), ec);
+ }
+ slots.erase(slots.begin() + (long) i);
+ };
+ const auto slot_count = [&](int& ram, int& disk) {
+ ram = disk = 0;
+ for (const Slot& s : slots) (s.data ? ram : disk) += 1;
+ };
+ // the folder's conversations (their .bin; a .meta beside each is a few KB: identity and tokens)
+ const auto slot_folder_bytes = [&]() {
+ uint64_t total = 0;
+ std::error_code ec;
+ for (const auto& de : std::filesystem::directory_iterator(slot_dir, ec)) {
+ const std::string fn = de.path().filename().string();
+ if (fn.rfind("slot-", 0) == 0 && fn.size() > 9 && fn.compare(fn.size() - 4, 4, ".bin") == 0) {
+ const uint64_t bytes = (uint64_t) std::filesystem::file_size(de.path(), ec);
+ if (!ec) total += bytes;
+ }
+ }
+ return total;
+ };
+ // the folder over STRATA_DS4_SLOT_GB: its least recently used conversation goes first, and never `keep` (the one just
+ // written), so a conversation bigger than the cap stays
+ const auto slot_trim = [&](const std::string& keep) {
+ const uint64_t before = slot_folder_bytes();
+ size_t dropped = 0;
+ while (slot_folder_bytes() > slot_budget) {
+ size_t lru = SIZE_MAX;
+ for (size_t i = 0; i < slots.size(); ++i)
+ if (!slots[i].data && slots[i].path != keep && (lru == SIZE_MAX || slots[i].used < slots[lru].used))
+ lru = i;
+ if (lru == SIZE_MAX) break;
+ slot_drop(lru);
+ ++dropped;
+ }
+ if (dropped > 0) {
+ char msg[256];
+ std::snprintf(msg, sizeof msg, "the folder held %s (over STRATA_DS4_SLOT_GB, %s): %zu least recently used "
+ "conversation(s) removed, %s left", gb(before).c_str(), gb(slot_budget).c_str(), dropped,
+ gb(slot_folder_bytes()).c_str());
+ note(msg);
+ }
+ };
+ // a RAM slot into the folder (its bytes, then its .meta); false: not written (its bytes go, then)
+ const auto slot_write_file = [&](Slot& s) -> bool {
+ std::error_code ec;
+ const auto space = std::filesystem::space(slot_dir, ec);
+ if (!ec && space.available < ((uint64_t) 8 << 30)) { // (8 GB free, the GLM engine's guard)
+ std::fprintf(stderr, "ds4 slots: under 8 GB free in %s - the conversation is not written\n",
+ slot_dir.string().c_str());
+ return false;
+ }
+ const auto stamp = std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+ const std::string path =
+ (slot_dir / ("slot-" + std::to_string(stamp) + "-" + std::to_string(++slot_seq) + ".bin")).string();
+ {
+ std::ofstream f(path + ".tmp", std::ios::binary | std::ios::trunc);
+ f.write((const char*) s.data->data(), (std::streamsize) s.data->size());
+ f.flush();
+ if (!f) {
+ std::filesystem::remove(path + ".tmp", ec);
+ std::fprintf(stderr, "ds4 slots: writing %s.tmp failed (disk full?)\n", path.c_str());
+ return false;
+ }
+ }
+ fs::rename(path + ".tmp", path, ec);
+ if (ec || !meta_write(meta_of(path), s.tokens, s.data->size(), s.cks)) {
+ std::filesystem::remove(path + ".tmp", ec);
+ std::filesystem::remove(path, ec);
+ std::fprintf(stderr, "ds4 slots: writing %s failed\n", path.c_str());
+ return false;
+ }
+ s.path = path;
+ s.bytes = s.data->size();
+ s.data.reset();
+ slot_trim(path); // (the folder over its cap: the least recently used conversation goes, not this one)
+ return true;
+ };
+ // the conversation in the model into a slot (the state is lost otherwise: another one takes the model)
+ const auto slot_put = [&](const char* why) {
+ if (slot_max <= 0) return;
+ const std::vector<int32_t>& t = m.tokens();
+ if (t.size() < slot_min) return; // (a fresh engine's 0-token state is no conversation: nothing to set aside)
+ const auto t0 = std::chrono::steady_clock::now();
+ auto buf = std::make_shared<std::vector<uint8_t>>();
+ std::string e;
+ const uint64_t bytes = m.slot_save(*buf, e); // (a state unchanged since it was taken refuses here: nothing
+ if (bytes == 0) { // changes, its slot stays)
+ if (e != "it is already set aside")
+ std::fprintf(stderr, "ds4 slots: setting the conversation aside failed (%s) - it is read from token 0 "
+ "when it comes back\n", e.c_str());
+ return;
+ }
+ for (size_t i = slots.size(); i-- > 0;) // an earlier state of this conversation is superseded
+ if (slots[i].tokens.size() <= t.size() && std::equal(slots[i].tokens.begin(), slots[i].tokens.end(), t.begin()))
+ slot_drop(i);
+ slots.push_back({t, "", bytes, ++slot_clock, nullptr, m.check_positions()});
+ slots.back().data = buf;
+ while (true) { // past the cap: the least recently used conversation in RAM to the folder, or gone
+ int ram = 0, disk = 0;
+ slot_count(ram, disk);
+ if (ram <= slot_max) break;
+ size_t lru = SIZE_MAX;
+ for (size_t i = 0; i < slots.size(); ++i)
+ if (slots[i].data && (lru == SIZE_MAX || slots[i].used < slots[lru].used)) lru = i;
+ if (lru == SIZE_MAX) break;
+ if (!slot_write_file(slots[lru])) slot_drop(lru);
+ }
+ int ram = 0, disk = 0;
+ slot_count(ram, disk);
+ char msg[256];
+ std::snprintf(msg, sizeof msg, "saved a %zu-token conversation%s (%s in RAM in %.1f s); %d in RAM, %d in %s",
+ t.size(), why, gb(bytes).c_str(),
+ std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(), ram, disk,
+ slot_dir.string().c_str());
+ note(msg);
+ };
+ // the kept conversation this prompt shares the most of - past `floor` tokens, what the model itself can go back
+ // to - back in the model: the tokens it covers (0: none). A conversation the prompt continues whole comes back
+ // whole (`whole`); one it leaves part way - a retried prompt, a history the client renders again without the
+ // earlier reasoning - comes back at its latest checkpoint the prompt still agrees with (the end of a prompt, the
+ // position before it, the anchor where its run of continued prompts began).
+ const auto slot_take = [&](const std::vector<int>& prompt, size_t floor, bool& whole) -> size_t {
+ whole = false;
+ if (slot_max <= 0 || prompt.size() < 2) return 0;
+ std::vector<strata::ds4::slotmeta::Cand> cands;
+ for (const Slot& sl : slots) cands.push_back({&sl.tokens, &sl.cks});
+ size_t len = 0;
+ const size_t best = strata::ds4::slotmeta::pick(cands, prompt, floor, &len);
+ if (best == SIZE_MAX) return 0;
+ const size_t whole_len = slots[best].tokens.size();
+ const auto t0 = std::chrono::steady_clock::now();
+ const bool ram = slots[best].data != nullptr;
+ std::string e;
+ bool back = false;
+ if (ram) back = m.slot_load(slots[best].data->data(), slots[best].data->size(), (int64_t) whole_len, e);
+ else {
+ std::ifstream f(slots[best].path, std::ios::binary);
+ std::vector<uint8_t> file;
+ if (f) {
+ f.seekg(0, std::ios::end);
+ const std::streamoff n = f.tellg();
+ f.seekg(0, std::ios::beg);
+ if (n > 0) {
+ file.resize((size_t) n);
+ if (f.read((char*) file.data(), n))
+ back = m.slot_load(file.data(), file.size(), (int64_t) whole_len, e);
+ else e = "reading " + slots[best].path + " failed";
+ } else e = slots[best].path + " is empty";
+ } else e = "cannot read " + slots[best].path;
+ }
+ if (!back) {
+ note("taking a conversation back failed (" + (e.empty() ? std::string("it did not load") : e) +
+ ") - reading its prompt again");
+ slot_drop(best);
+ return 0;
+ }
+ if (len < whole_len && !(m.restore((int64_t) len) && (size_t) m.pos() == len)) {
+ note("going back to token " + std::to_string(len) + " of a kept conversation failed - reading its prompt again");
+ m.reset();
+ return 0;
+ }
+ whole = len == whole_len;
+ slots[best].used = ++slot_clock;
+ if (!ram) { // its age counts from its last use (a file slot's only)
+ std::error_code ec;
+ fs::last_write_time(meta_of(slots[best].path), fs::file_time_type::clock::now(), ec);
+ }
+ char msg[256];
+ std::string part;
+ if (!whole) part = " at token " + std::to_string(len) + " of " + std::to_string(whole_len);
+ std::snprintf(msg, sizeof msg, "took back a %zu-token conversation%s from %s (%.1f s); %zu new tokens to read",
+ len, part.c_str(), ram ? "RAM" : "the folder",
+ std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),
+ prompt.size() - len);
+ note(msg);
+ return len;
+ };
+ if (slot_max > 0) {
+ slot_dir = std::getenv("STRATA_DS4_SLOT_DIR") ? std::filesystem::path(std::getenv("STRATA_DS4_SLOT_DIR"))
+ : std::filesystem::path(pack) / "slots";
+ std::error_code ec;
+ std::filesystem::create_directories(slot_dir, ec);
+ if (!std::filesystem::is_directory(slot_dir, ec)) {
+ std::fprintf(stderr, "ds4 slots: %s is not writable - conversations are not set aside\n",
+ slot_dir.string().c_str());
+ slot_max = 0;
+ } else {
+ struct Found {
+ std::string bin;
+ std::vector<int32_t> toks;
+ uint64_t bytes;
+ std::vector<int64_t> cks;
+ fs::file_time_type t;
+ };
+ std::vector<Found> found;
+ size_t refused = 0;
+ for (const auto& de : std::filesystem::directory_iterator(slot_dir, ec)) { // a previous run's slots
+ const std::string fn = de.path().filename().string(), p = de.path().string();
+ if (fn.rfind("slot-", 0) != 0) continue;
+ const bool is_bin = fn.size() > 9 && fn.compare(fn.size() - 4, 4, ".bin") == 0;
+ const bool is_meta = fn.size() > 10 && fn.compare(fn.size() - 5, 5, ".meta") == 0;
+ if (!is_bin && !is_meta) { // a .tmp an interrupted write left
+ std::filesystem::remove(de.path(), ec);
+ continue;
+ }
+ if (!is_meta) continue; // a .bin is judged by its .meta
+ const std::string bin = p.substr(0, p.size() - 5) + ".bin";
+ Found fd{bin, {}, 0, {}, fs::last_write_time(de.path(), ec)};
+ const uint64_t have = fs::exists(bin, ec) ? (uint64_t) fs::file_size(bin, ec) : 0;
+ if (meta_read(p, fd.toks, fd.bytes, fd.cks) && have == fd.bytes && fd.bytes > 0 && slot_age_days(p) <= slot_days)
+ found.push_back(std::move(fd));
+ else {
+ ++refused;
+ std::filesystem::remove(p, ec);
+ std::filesystem::remove(bin, ec);
+ }
+ }
+ for (const auto& de : std::filesystem::directory_iterator(slot_dir, ec)) { // a .bin without its .meta
+ const std::string p = de.path().string(), fn = de.path().filename().string();
+ if (fn.rfind("slot-", 0) == 0 && fn.size() > 9 && fn.compare(fn.size() - 4, 4, ".bin") == 0 &&
+ !fs::exists(meta_of(p), ec))
+ std::filesystem::remove(de.path(), ec);
+ }
+ std::sort(found.begin(), found.end(), [](const Found& a, const Found& b) { return a.t < b.t; });
+ uint64_t total = 0;
+ for (auto& fd : found) {
+ total += fd.bytes;
+ slots.push_back({std::move(fd.toks), fd.bin, fd.bytes, ++slot_clock, nullptr, std::move(fd.cks)});
+ }
+ std::string msg = "folder " + slot_dir.string() + ": " + std::to_string(slots.size()) +
+ " conversation(s) kept from earlier runs (" + gb(total) + ")";
+ if (refused) msg += ", " + std::to_string(refused) + " not this model's or incomplete - removed";
+ note(msg);
+ slot_trim(""); // (the folder over STRATA_DS4_SLOT_GB: the least recently used conversation goes first)
+ }
+ }
+ if (slot_max > 0) {
+#if !defined(_WIN32)
+ // (the server ends the engine with QUIT and waits for it; a SIGTERM to the whole process group - systemd
+ // stopping a service, a supervisor killing the server's group - must not end it mid-write, and a server that
+ // is already gone must not end it with SIGPIPE when it writes its last lines)
+ std::signal(SIGTERM, SIG_IGN);
+ std::signal(SIGINT, SIG_IGN);
+ std::signal(SIGPIPE, SIG_IGN);
+#endif
+ }
while (in.next(line)) {
if (line == "QUIT") break;
const bool geni = line.rfind("GENI ", 0) == 0;
@@ -325,6 +655,10 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
continue;
}
}
+ if (!reuse) { // (STRATA_DS4_REUSE=0, for A/Bs and tests: every request is a fresh read of its whole prompt)
+ m.reset();
+ held.clear();
+ }
const auto& st = m.tokens();
const size_t n_state = st.size();
size_t c = 0;
@@ -340,12 +674,32 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
for (const Pic& q : held)
if ((size_t) q.start < c && !same(pics, q)) c = (size_t) q.start;
size_t from = 0;
- if (c == n_state && n_state < r.ids.size()) from = n_state;
- else if (m.restorable((int64_t) std::min(c, r.ids.size() - 1)) >= 0 &&
- m.restore((int64_t) std::min(c, r.ids.size() - 1)))
- from = (size_t) m.pos();
- else m.reset();
- if (from == 0) held.clear();
+ bool took = false, continued = false; // (continued: the conversation goes on - its anchor stays)
+ // the conversation continues, reused in place - a conversation has to be in the model first: a fresh engine
+ // (the start of a run, so every restart) holds 0 tokens, and every prompt "continues" that to token 0; taken
+ // as a continuation, the take below would never run for the folder's conversations, the case they are for
+ if (n_state > 0 && c == n_state && n_state < r.ids.size()) {
+ from = n_state;
+ continued = true;
+ }
+ else {
+ // CONVERSATION SLOTS: this prompt does not continue the conversation in the model - that one is set aside
+ // first (another one takes the model, and its state is lost otherwise), then the kept conversation this
+ // prompt does continue is taken back - if it goes further back than the checkpoints the state itself holds
+ if (held.empty()) slot_put("");
+ const int64_t limit = (int64_t) std::min(c, r.ids.size() - 1);
+ const int64_t rs = m.restorable(limit);
+ bool whole = false;
+ const size_t back = pics.empty() ? slot_take(r.ids, rs > 0 ? (size_t) rs : 0, whole) : 0;
+ if (back > 0) {
+ from = back;
+ took = true;
+ continued = whole;
+ } else if (rs >= 0 && m.restore(limit))
+ from = (size_t) m.pos();
+ else m.reset();
+ }
+ if (took || from == 0) held.clear(); // (a slot is text only: the state that comes back holds no pictures)
if ((int64_t) r.ids.size() + std::max(0LL, r.max_new) > m.max_context()) {
std::printf("ERR the prompt (%zu tokens) and max_new %lld are more than the context %lld\n", r.ids.size(),
r.max_new, (long long) m.max_context());
@@ -399,8 +753,12 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
});
// (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)
static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;
+ // (pin=N: the read stops at N for a checkpoint of the shared start; only when this read crosses N, and N leaves
+ // the prompt's last token to the usual end)
+ const size_t pin_at = (r.pin >= 256 && (size_t) r.pin + 1 < total) ? (size_t) r.pin : 0;
while (ok && done < total) {
- const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);
+ size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);
+ if (pin_at > done && pin_at < done + n) n = pin_at - done;
// (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)
const bool fin = done + n == total;
if (fin && last_window) {
@@ -411,6 +769,7 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
ok = fin ? m.feed_mark(r.ids.data() + done, (int) n, 1, 1, err)
: m.feed(r.ids.data() + done, (int) n, 0, err);
done += n;
+ if (ok && pin_at != 0 && done == pin_at) m.checkpoint(3);
if (ok && (done >= next_pp || done == total)) {
const double ms = now_ms() - t0;
std::printf("PP %zu %zu %.1f %.1f %zu\n", done, r.ids.size(), ms,
@@ -431,6 +790,7 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
}
held = std::move(pics);
m.checkpoint(0);
+ if (!continued) m.pin(); // (a new run of prompts: where it may be rendered again from)
const double t1 = now_ms();
long long produced = 0;
const char* finish = "length";
@@ -527,6 +887,31 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
per_position(s_dec, s).c_str());
m.save_profile();
}
+ // the engine ends (QUIT or the end of its input): the conversation in the model is set aside, and every one in
+ // RAM is written to the folder, so the next run reads on from them
+ if (slot_max > 0) {
+ if (held.empty()) slot_put(" as the engine ended");
+ size_t written = 0;
+ while (true) { // (a write can take the folder's least recently used conversation away: the list is asked again)
+ size_t i = SIZE_MAX;
+ for (size_t k = 0; k < slots.size(); ++k)
+ if (slots[k].data) {
+ i = k;
+ break;
+ }
+ if (i == SIZE_MAX) break;
+ if (slot_write_file(slots[i])) ++written;
+ else slot_drop(i); // (its state could not be written: there is nothing more to do at the engine's end)
+ }
+ if (written > 0) {
+ int ram = 0, disk = 0;
+ slot_count(ram, disk);
+ char msg[256];
+ std::snprintf(msg, sizeof msg, "wrote %zu conversation(s) to %s; %d there in all", written,
+ slot_dir.string().c_str(), disk);
+ note(msg);
+ }
+ }
m.save_profile();
return 0;
}
@@ -847,7 +1232,7 @@ int main(int argc, char** argv) {
return 1;
}
if (!o.vision_only) std::fprintf(stderr, "%s", m.describe().c_str());
- if (serve_mode) return serve(m, eos, spec, o.dspark_conf);
+ if (serve_mode) return serve(m, eos, spec, o.dspark_conf, o.pack);
const int window_given = window;
if (window == 0) window = 16; // (the scoring / profiling runs; the plain run below takes the model's feed unit)
if (!prof_ids.empty()) return profile_run(m, prof_ids, window);
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index a9e370441..7d17fc565 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -41,6 +41,7 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
+#include <filesystem>
#include <fstream>
#include <functional>
#include <future>
@@ -920,7 +921,14 @@ struct Model::Impl {
std::vector<std::vector<uint8_t>> win;
std::vector<std::vector<float>> comp;
};
- Check ck[2]; // the end of the last prompt, and the position before it
+ static constexpr int kChecks = 4;
+ Check ck[kChecks]; // the end of the last prompt, the position before it, the anchor (the position
+ // before the end of the first prompt of the run of prompts that each continued
+ // the last in place: a chat template that drops earlier turns' reasoning at a new
+ // user message re-renders the history from about there) and the prefix (where the
+ // server said the prompt's shared start - system prompt and tools - ends: another
+ // agent behind the same start goes back to it)
+ bool slot_held = false; // slot_save took this state and nothing has changed it since
std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits
Stats st;
int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query
@@ -4528,7 +4536,8 @@ struct Model::Impl {
pos = 0;
toks.clear();
ctoks.clear();
- ck[0].pos = ck[1].pos = -1;
+ slot_held = false;
+ for (Check& c0 : ck) c0.pos = -1;
for (Layer& l0 : L)
for (int m = 0; m < l0.nm; ++m) {
Layer& l = member(l0, m);
@@ -4547,7 +4556,7 @@ struct Model::Impl {
// the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the
// state is indexed by position, and what lies past a restored position is written again)
void checkpoint(int slot) {
- Check& c = ck[slot & 1];
+ Check& c = ck[slot & (kChecks - 1)];
c.pos = pos;
c.win.resize(L.size());
c.comp.resize(L.size() * 2);
@@ -4574,7 +4583,7 @@ struct Model::Impl {
// the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1
int best_check(int64_t limit) const {
int b = -1;
- for (int i = 0; i < 2; ++i)
+ for (int i = 0; i < kChecks; ++i)
if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;
return b;
}
@@ -4583,6 +4592,33 @@ struct Model::Impl {
return b < 0 ? -1 : ck[b].pos;
}
+ // checkpoint 1 (the position before the last prompt's end) becomes the anchor too
+ void pin() {
+ if (ck[1].pos > 0) ck[2] = ck[1];
+ }
+ // the checkpoints the state can go back to (what a slot of it keeps), latest first
+ std::vector<int64_t> check_positions() const {
+ std::vector<int64_t> v;
+ for (const Check& c : ck)
+ if (c.pos > 0 && c.pos <= pos && c.win.size() == L.size() &&
+ std::find(v.begin(), v.end(), c.pos) == v.end())
+ v.push_back(c.pos);
+ std::sort(v.rbegin(), v.rend());
+ return v;
+ }
+ // the checkpoints a slot keeps (each with its role: 0, 1 or the anchor)
+ std::vector<int> slot_checks() const {
+ std::vector<int> v;
+ for (int i = 0; i < kChecks; ++i)
+ if (ck[i].pos > 0 && ck[i].pos <= pos && ck[i].win.size() == L.size()) v.push_back(i);
+ return v;
+ }
+ size_t check_bytes() const {
+ size_t n = 16;
+ for (const Layer& l : L) n += (size_t) g.win * win_rb() + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);
+ return n;
+ }
+
bool restore(int64_t limit) {
const int b = best_check(limit);
if (b < 0) return false;
@@ -4606,11 +4642,362 @@ struct Model::Impl {
pos = c.pos;
toks.resize((size_t) pos);
ctoks.resize((size_t) pos);
+ slot_held = false;
for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has
if (o2.pos > pos) o2.pos = -1;
return true;
}
+ // ---- the kept conversations (a slot): the state out of the model and back, through a file or a host buffer.
+ // A prompt that continues the state is reused in place (checkpoint / restore, the compressed KV left where it is);
+ // a conversation that leaves the model for another one is a slot - the whole state: the position, its tokens,
+ // every window ring (the DSpark blocks' included), the compressors' partial groups and each KV source's compressed
+ // KV and index keys up to it (rows 0 .. pos / ratio, the groups its position completed; one the prompt has not
+ // completed yet lives in the compressor's partial group). A tensor parallel group's members hold the same state,
+ // so one member's ring is saved and written back into all of them. Text only: a conversation that holds pictures
+ // is not a slot (the Engram history's rows are the tokens' again, no -1 among them).
+ static constexpr uint64_t kSlotMagic = 0x31544f4c53345344ull; // "DS4SLOT1" (little endian)
+
+ int64_t src_rows(const Src& s) const { return pos / std::max(1, s.ratio); }
+
+ // the slot's bytes: the header, the tokens, every layer's ring and partial group, every source's rows, the end
+ size_t slot_bytes() const {
+ size_t n = 8 + 8 + slot_identity().size() + 8 + (size_t) pos * 4;
+ for (const Layer& l : L) n += 8 + (size_t) g.win * win_rb() + 8 + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);
+ for (const Src& s : srcs) n += 8 + (size_t) src_rows(s) * (ckv_rb() + ik_rb());
+ n += 8 + slot_checks().size() * check_bytes(); // (the checkpoints: what the slot can be taken back at)
+ return n + 8;
+ }
+
+ uint64_t slot_save(std::vector<uint8_t>& out, std::string& err) {
+ if (slot_held) {
+ err = "it is already set aside";
+ return 0;
+ }
+ if (pos <= 0) {
+ err = "there is no conversation";
+ return 0;
+ }
+ if ((int64_t) toks.size() != pos) {
+ err = "the state's tokens do not match its position";
+ return 0;
+ }
+ if (!imgs.empty()) {
+ err = "it holds pictures";
+ return 0;
+ }
+ const std::string id = slot_identity();
+ out.assign(slot_bytes(), 0);
+ uint8_t* p = out.data();
+ size_t off = 0;
+ const auto put = [&](const void* s, size_t b) {
+ std::memcpy(p + off, s, b);
+ off += b;
+ };
+ const uint64_t il = id.size(), n = (uint64_t) pos;
+ put(&kSlotMagic, 8);
+ put(&il, 8);
+ put(id.data(), (size_t) il);
+ put(&n, 8);
+ put(toks.data(), (size_t) n * 4);
+ for (Dev& d : devs) { // (the devices' work ends before their state is read)
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ for (size_t i = 0; i < L.size(); ++i) {
+ const Layer& l = L[i];
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ const uint64_t wb = (uint64_t) g.win * win_rb();
+ put(&wb, 8);
+ if (cudaMemcpy(p + off, l.win, (size_t) wb, cudaMemcpyDeviceToHost) != cudaSuccess) {
+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());
+ return 0;
+ }
+ off += (size_t) wb;
+ const uint64_t cn = l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0;
+ put(&cn, 8);
+ if (cn > 0) {
+ const size_t cb = (size_t) cn * 4;
+ if (cudaMemcpy(p + off, l.c_kv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||
+ cudaMemcpy(p + off + cb, l.c_score, cb, cudaMemcpyDeviceToHost) != cudaSuccess) {
+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());
+ return 0;
+ }
+ off += 2 * cb;
+ }
+ }
+ for (const Src& s : srcs) {
+ const uint64_t rows = (uint64_t) src_rows(s);
+ put(&rows, 8);
+ if (rows == 0) continue;
+ const int di = L[(size_t) s.layer].dev;
+ const uint8_t* ckv = s.ckv[(size_t) di];
+ const uint8_t* ik = s.ik[(size_t) di]; // (a pack whose sources are not index sources: none, zeros)
+ if (ckv == nullptr) {
+ err = "source layer " + std::to_string(s.layer) + "'s rows are not on its device";
+ return 0;
+ }
+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();
+ cudaSetDevice(devs[(size_t) di].id);
+ if (cudaMemcpy(p + off, ckv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||
+ (ik != nullptr && cudaMemcpy(p + off + cb, ik, ib, cudaMemcpyDeviceToHost) != cudaSuccess)) {
+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());
+ return 0;
+ }
+ off += cb + ib;
+ }
+ {
+ const std::vector<int> cs = slot_checks();
+ const uint64_t nc = cs.size();
+ put(&nc, 8);
+ for (int k : cs) {
+ const Check& c = ck[k];
+ const uint64_t role = (uint64_t) k, cp = (uint64_t) c.pos;
+ put(&role, 8);
+ put(&cp, 8);
+ for (size_t i = 0; i < L.size(); ++i) {
+ put(c.win[i].data(), (size_t) g.win * win_rb());
+ if (L[i].c_kv) {
+ put(c.comp[2 * i].data(), (size_t) L[i].ratio * g.hd * 4);
+ put(c.comp[2 * i + 1].data(), (size_t) L[i].ratio * g.hd * 4);
+ }
+ }
+ }
+ }
+ if (off + 8 != out.size()) {
+ err = "the state's size does not add up";
+ return 0;
+ }
+ put(&kSlotMagic, 8);
+ slot_held = true;
+ return (uint64_t) off;
+ }
+
+ bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {
+ size_t off = 0;
+ bool wrote = false, ok = false;
+ // a load that stops after some of another conversation's state went into the layers leaves a state no
+ // checkpoint can repair: it is reset (the prompt is read from token 0), never served from the mix
+ struct FailClosed {
+ Impl& p;
+ const bool& wrote;
+ const bool& ok;
+ ~FailClosed() {
+ if (!ok && wrote) p.reset();
+ }
+ } guard{*this, wrote, ok};
+ const auto room = [&](size_t b) { return b <= bytes && off <= bytes - b; };
+ uint64_t magic = 0, il = 0, n = 0;
+ if (!room(16)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&magic, data + off, 8);
+ std::memcpy(&il, data + off + 8, 8);
+ off += 16;
+ if (magic != kSlotMagic || il == 0 || il > (1u << 20)) {
+ err = "not a saved conversation";
+ return false;
+ }
+ if (!room((size_t) il + 8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ {
+ std::string id((size_t) il, '\0');
+ std::memcpy(id.data(), data + off, (size_t) il);
+ off += (size_t) il;
+ std::memcpy(&n, data + off, 8);
+ off += 8;
+ if (id != slot_identity()) {
+ err = "another model's, or another engine version's, conversation";
+ return false;
+ }
+ }
+ if (n != (uint64_t) n_pos || n_pos <= 0 || n_pos > o.max_context || !room((size_t) n * 4)) {
+ err = "the conversation's position is not this prompt's";
+ return false;
+ }
+ std::vector<int32_t> toks_in((size_t) n);
+ std::memcpy(toks_in.data(), data + off, (size_t) n * 4);
+ off += (size_t) n * 4;
+ for (Dev& d : devs) { // (their work ends before the state is written over)
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ uint64_t wb = 0, cn = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&wb, data + off, 8);
+ off += 8;
+ if (wb != (uint64_t) g.win * win_rb() || !room((size_t) wb + 8)) {
+ err = l.name + "window ring of another size";
+ return false;
+ }
+ wrote = true;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& lm = member(l, m);
+ if (cudaMemcpy(lm.win, data + off, (size_t) wb, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ }
+ off += (size_t) wb;
+ std::memcpy(&cn, data + off, 8);
+ off += 8;
+ if (cn != (l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0)) {
+ err = l.name + "compressor's group of another size";
+ return false;
+ }
+ if (cn > 0) {
+ const size_t cb = (size_t) cn * 4;
+ if (!room(2 * cb)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& lm = member(l, m);
+ if (cudaMemcpy(lm.c_kv, data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess ||
+ cudaMemcpy(lm.c_score, data + off + cb, cb, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ }
+ off += 2 * cb;
+ }
+ }
+ for (const Src& s : srcs) {
+ uint64_t rows = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&rows, data + off, 8);
+ off += 8;
+ if (rows != (uint64_t) (n_pos / std::max(1, s.ratio))) {
+ err = "source layer " + std::to_string(s.layer) + "'s rows are not this conversation's";
+ return false;
+ }
+ if (rows == 0) continue;
+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();
+ if (!room(cb + ib)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ for (size_t di = 0; di < devs.size(); ++di) {
+ if (s.ckv[di] == nullptr && s.ik[di] == nullptr) continue;
+ cudaSetDevice(devs[di].id);
+ if (s.ckv[di] != nullptr && cudaMemcpy(s.ckv[di], data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ if (s.ik[di] != nullptr &&
+ cudaMemcpy(s.ik[di], data + off + cb, ib, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = "source layer " + std::to_string(s.layer) + "'s index keys: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ }
+ off += cb + ib;
+ }
+ Check got[kChecks]; // the slot's checkpoints: the positions it can be taken back at
+ {
+ uint64_t nc = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&nc, data + off, 8);
+ off += 8;
+ if (nc > (uint64_t) kChecks) {
+ err = "the slot's checkpoints are damaged";
+ return false;
+ }
+ for (uint64_t k = 0; k < nc; ++k) {
+ uint64_t role = 0, cp = 0;
+ if (!room(16)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&role, data + off, 8);
+ std::memcpy(&cp, data + off + 8, 8);
+ off += 16;
+ if (role >= (uint64_t) kChecks || got[role].pos >= 0 || cp == 0 || cp > (uint64_t) n_pos) {
+ err = "the slot's checkpoints are damaged";
+ return false;
+ }
+ Check& c = got[role];
+ c.pos = (int64_t) cp;
+ c.win.resize(L.size());
+ c.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ const size_t wb = (size_t) g.win * win_rb(), cb = L[i].c_kv ? (size_t) L[i].ratio * g.hd * 4 : 0;
+ if (!room(wb + 2 * cb)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ c.win[i].assign(data + off, data + off + wb);
+ off += wb;
+ if (cb > 0) {
+ c.comp[2 * i].resize(cb / 4);
+ c.comp[2 * i + 1].resize(cb / 4);
+ std::memcpy(c.comp[2 * i].data(), data + off, cb);
+ std::memcpy(c.comp[2 * i + 1].data(), data + off + cb, cb);
+ off += 2 * cb;
+ }
+ }
+ }
+ }
+ uint64_t end = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&end, data + off, 8);
+ off += 8;
+ if (end != kSlotMagic) {
+ err = "the slot does not end where it should";
+ return false;
+ }
+ pos = n_pos;
+ toks.swap(toks_in);
+ ctoks.resize((size_t) pos);
+ for (size_t i = 0; i < (size_t) pos; ++i)
+ ctoks[i] = toks[i] < pk.eg_vocab ? pk.eg_map[(size_t) toks[i]] : 0;
+ for (int k = 0; k < kChecks; ++k) ck[k] = std::move(got[k]); // (the slot's own; the old ones held the state
+ // this one replaced)
+ slot_held = false;
+ ok = true;
+ return true;
+ }
+
+ std::string slot_identity() const {
+ const auto size_of = [](const std::string& p) {
+ std::error_code ec;
+ const auto s = std::filesystem::file_size(p, ec);
+ return ec ? 0ull : (unsigned long long) s;
+ };
+ std::ostringstream id;
+ id << "strata-ds4-session-3 engine=" << STRATA_VERSION << " mtp=" << g.n_mtp << " dim=" << g.dim << " hd=" << g.hd
+ << " win=" << g.win << " ratios=";
+ for (int r : g.ratios) id << r << ",";
+ id << " kv=";
+ for (int v : g.kv_src) id << v << ",";
+ id << " ix=";
+ for (int v : g.ix_src) id << v << ",";
+ id << " ds4.txt:" << size_of(pk.dir + "/ds4.txt") << " engram.bin:" << size_of(pk.dir + "/engram.bin")
+ << " tensors.txt:" << size_of(pk.dir + "/tensors.txt");
+ for (size_t i = 0; i < pk.files.size(); ++i)
+ id << " " << pk.files[i] << ":" << os::file_size(pk.fds[i]);
+ return id.str();
+ }
+
size_t bounce_bytes() const {
return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
}
@@ -6211,6 +6598,7 @@ struct Model::Impl {
pos = p0 + keep;
toks.resize((size_t) pos);
ctoks.resize((size_t) pos);
+ slot_held = false;
}
// ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows
@@ -7583,6 +7971,7 @@ struct Model::Impl {
return false;
}
const int64_t p0 = pos;
+ slot_held = false; // (the state grows now: what slot_save took is no longer the conversation)
for (int t = 0; t < T; ++t) {
toks.push_back(tk[t]);
ctoks.push_back(ctok(pos + t, tk[t]));
@@ -8404,7 +8793,14 @@ const std::vector<int32_t>& Model::tokens() const { return p_->toks; }
void Model::reset() { p_->reset(); }
void Model::checkpoint(int slot) { p_->checkpoint(slot); }
bool Model::restore(int64_t limit) { return p_->restore(limit); }
+void Model::pin() { p_->pin(); }
+std::vector<int64_t> Model::check_positions() const { return p_->check_positions(); }
int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
+uint64_t Model::slot_save(std::vector<uint8_t>& out, std::string& err) { return p_->slot_save(out, err); }
+bool Model::slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {
+ return p_->slot_load(data, bytes, n_pos, err);
+}
+std::string Model::slot_identity() const { return p_->slot_identity(); }
void Model::save_profile() const { p_->save_profile(); }
const Stats& Model::stats() const {
Stats& st = p_->st;
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index 929636d53..b4c35f0b2 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -8,7 +8,9 @@
// its own 128-token window, the compressed groups completed by its position. The state a token leaves is the
// 128-token window ring of every layer, the compressors' partial groups, the compressed KV and index keys of the KV
// source layers (append-only), and the Engram n-gram history; checkpoint() / restore() keep the first two (the rest is
-// truncated by position) so a conversation can go back to the end of its last prompt.
+// truncated by position) so a conversation can go back to the end of its last prompt. A conversation that leaves the
+// model altogether (another chat answered in between) is a slot: slot_save / slot_load take the same state - the
+// compressed KV and index keys included, which a checkpoint leaves in place - through a file or a host buffer.
#pragma once
#include <cstddef>
@@ -146,6 +148,23 @@ public:
void checkpoint(int slot = 0);
bool restore(int64_t limit);
int64_t restorable(int64_t limit) const;
+ /// Checkpoint 1 also becomes the anchor (kept until a restore goes before it): the position before the end of the
+ /// first prompt of a run that continued in place, where a re-rendered history (earlier reasoning dropped) splits.
+ void pin();
+ /// The positions restore() can go back to, latest first (a slot keeps them: it can be taken back at any of them).
+ std::vector<int64_t> check_positions() const;
+
+ /// The conversation in the model into `out` (slot_save): its position, its tokens, every window ring (the DSpark
+ /// blocks' included), the compressors' partial groups and each KV source's compressed KV and index keys up to it,
+ /// the pieces the state a prompt continues is made of. 0 and err: nothing was set aside (it holds pictures, there
+ /// is no conversation, or the conversation is already set aside - unchanged since).
+ uint64_t slot_save(std::vector<uint8_t>& out, std::string& err);
+ /// `data` (a slot's bytes) back into the model: the state at the slot's position n_pos, as slot_save took it -
+ /// what a prompt that continues the slot's tokens reads on from (the caller checked the tokens are a prefix).
+ bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err);
+ /// What a saved conversation must match: this engine's version, this model's geometry and files and the drafter's
+ /// blocks - a slot naming anything else is not this model's state (a file's .meta says so; slot_load refuses).
+ std::string slot_identity() const;
/// The vision tower: the token every position of an image's span carries (-1: no tower).
int image_token() const;
diff --git a/src/dsv41/ds4_slotmeta.hpp b/src/dsv41/ds4_slotmeta.hpp
new file mode 100644
index 000000000..dab9fea3e
--- /dev/null
+++ b/src/dsv41/ds4_slotmeta.hpp
@@ -0,0 +1,98 @@
+// The kept-conversation (slot) bookkeeping that needs no GPU: a slot's .meta file (identity, size, tokens, checkpoint
+// positions) and the choice of which kept conversation a prompt continues, and from where. The engine's serve loop
+// (ds4_main.cpp) uses these; tests/dsv41/slot_test.cpp checks them alone.
+//
+// Slot .meta layout (little endian), "DS4META2":
+// u64 magic, u64 identity length, identity bytes, u64 .bin bytes, u64 token count, i32 tokens[],
+// u64 checkpoint count (0..4), i64 checkpoints[], u64 magic
+// The magic at both ends and the .bin size make a half-written or foreign file refused, never read.
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace strata::ds4::slotmeta {
+
+constexpr uint64_t kMagic = 0x324154454d345344ull; // "DS4META2"
+
+// Written under a temporary name and renamed over `path` (an interrupted write leaves no half file).
+inline bool write(const std::string& path, const std::string& identity, const std::vector<int32_t>& toks,
+ uint64_t bytes, const std::vector<int64_t>& cks) {
+ const std::string tmp = path + ".tmp";
+ {
+ std::ofstream f(tmp, std::ios::binary | std::ios::trunc);
+ const uint64_t il = identity.size(), nt = toks.size(), nc = cks.size();
+ f.write((const char*) &kMagic, 8);
+ f.write((const char*) &il, 8);
+ f.write(identity.data(), (std::streamsize) il);
+ f.write((const char*) &bytes, 8);
+ f.write((const char*) &nt, 8);
+ f.write((const char*) toks.data(), (std::streamsize) (nt * sizeof(int32_t)));
+ f.write((const char*) &nc, 8);
+ f.write((const char*) cks.data(), (std::streamsize) (nc * sizeof(int64_t)));
+ f.write((const char*) &kMagic, 8);
+ f.flush();
+ if (!f) return false;
+ }
+ std::error_code ec;
+ std::filesystem::rename(tmp, path, ec);
+ return !ec;
+}
+
+// false: foreign (another identity), damaged, cut short, or not plausible (no tokens, more than max_ctx, a checkpoint
+// outside the tokens).
+inline bool read(const std::string& path, const std::string& identity, int64_t max_ctx, std::vector<int32_t>& toks,
+ uint64_t& bytes, std::vector<int64_t>& cks) {
+ std::ifstream f(path, std::ios::binary);
+ uint64_t m0 = 0, il = 0, nt = 0, nc = 0, m1 = 0;
+ if (!f.read((char*) &m0, 8) || m0 != kMagic || !f.read((char*) &il, 8) || il != identity.size()) return false;
+ std::string id(il, '\0');
+ if (!f.read(id.data(), (std::streamsize) il) || id != identity) return false;
+ if (!f.read((char*) &bytes, 8) || !f.read((char*) &nt, 8) || nt == 0 || nt > (uint64_t) max_ctx) return false;
+ toks.resize(nt);
+ if (!f.read((char*) toks.data(), (std::streamsize) (nt * sizeof(int32_t))) || !f.read((char*) &nc, 8) || nc > 4)
+ return false;
+ cks.resize(nc);
+ if (!f.read((char*) cks.data(), (std::streamsize) (nc * sizeof(int64_t))) || !f.read((char*) &m1, 8) || m1 != kMagic)
+ return false;
+ for (int64_t c : cks)
+ if (c <= 0 || (uint64_t) c > nt) return false;
+ return true;
+}
+
+// The kept conversation `prompt` continues the furthest, and where: *len is the number of the slot's tokens that are
+// the prompt's (0: none). A conversation the prompt continues whole comes back whole (len == its size); one the prompt
+// leaves part way - a retried prompt, a history re-rendered without the earlier reasoning - comes back at its latest
+// checkpoint the prompt still agrees with. Only a match longer than `floor` (what the model itself can go back to)
+// counts. The last prompt token is never covered (a prompt always reads its last token). Ties keep the earlier slot.
+struct Cand {
+ const std::vector<int32_t>* tokens;
+ const std::vector<int64_t>* cks;
+};
+inline size_t pick(const std::vector<Cand>& slots, const std::vector<int>& prompt, size_t floor, size_t* len) {
+ *len = 0;
+ if (prompt.size() < 2) return SIZE_MAX;
+ size_t best = SIZE_MAX;
+ for (size_t i = 0; i < slots.size(); ++i) {
+ const std::vector<int32_t>& t = *slots[i].tokens;
+ const size_t lim = std::min(t.size(), prompt.size() - 1);
+ size_t c = 0;
+ while (c < lim && t[c] == prompt[c]) ++c;
+ size_t u = c == t.size() ? c : 0;
+ if (u == 0)
+ for (int64_t k : *slots[i].cks)
+ if ((size_t) k <= c && (size_t) k > u) u = (size_t) k;
+ if (u > *len && u > floor) {
+ best = i;
+ *len = u;
+ }
+ }
+ return best;
+}
+
+} // namespace strata::ds4::slotmeta
diff --git a/tests/dsv41/slot_test.cpp b/tests/dsv41/slot_test.cpp
new file mode 100644
index 000000000..0235b05c1
--- /dev/null
+++ b/tests/dsv41/slot_test.cpp
@@ -0,0 +1,153 @@
+// The kept conversations' bookkeeping without a GPU (src/dsv41/ds4_slotmeta.hpp): the .meta file's round trip and
+// every way it is refused, and which kept conversation a prompt continues, and from where.
+// ds4_slot_test exit 0: all passed
+#include "ds4_slotmeta.hpp"
+
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace sm = strata::ds4::slotmeta;
+namespace fs = std::filesystem;
+
+static int fails = 0;
+#define CHECK(c) \
+ do { \
+ if (!(c)) { \
+ std::printf("FAIL line %d: %s\n", __LINE__, #c); \
+ ++fails; \
+ } \
+ } while (0)
+
+static std::vector<int32_t> seq(int a, int n) {
+ std::vector<int32_t> v((size_t) n);
+ for (int i = 0; i < n; ++i) v[(size_t) i] = a + i;
+ return v;
+}
+static std::vector<int> as_int(const std::vector<int32_t>& v) { return std::vector<int>(v.begin(), v.end()); }
+
+static void patch(const std::string& p, size_t off, uint8_t b) {
+ std::FILE* f = std::fopen(p.c_str(), "r+b");
+ std::fseek(f, (long) off, SEEK_SET);
+ std::fwrite(&b, 1, 1, f);
+ std::fclose(f);
+}
+
+static void meta_tests(const fs::path& dir) {
+ const std::string id = "strata-ds4-session-2 engine=test", path = (dir / "slot-1.meta").string();
+ const std::vector<int32_t> toks = seq(100, 50);
+ const std::vector<int64_t> cks = {40, 33, 7};
+ CHECK(sm::write(path, id, toks, 123456789ull, cks));
+ CHECK(!fs::exists(path + ".tmp")); // (renamed over, no half file left)
+ std::vector<int32_t> t;
+ std::vector<int64_t> c;
+ uint64_t bytes = 0;
+ CHECK(sm::read(path, id, 1000, t, bytes, c));
+ CHECK(t == toks && c == cks && bytes == 123456789ull);
+ // no checkpoints is valid (a slot of a state that had none)
+ CHECK(sm::write(path, id, toks, 5, {}));
+ CHECK(sm::read(path, id, 1000, t, bytes, c) && c.empty() && bytes == 5);
+ // another model's / engine's identity: refused, never read
+ CHECK(!sm::read(path, id + "x", 1000, t, bytes, c));
+ CHECK(!sm::read(path, "other", 1000, t, bytes, c));
+ // more tokens than the context: refused
+ CHECK(!sm::read(path, id, 49, t, bytes, c));
+ CHECK(sm::read(path, id, 50, t, bytes, c));
+ // a missing file
+ CHECK(!sm::read((dir / "none.meta").string(), id, 1000, t, bytes, c));
+ // every cut length, a damaged magic at either end
+ CHECK(sm::write(path, id, toks, 9, cks));
+ const size_t full = (size_t) fs::file_size(path);
+ for (size_t keep = 0; keep < full; ++keep) {
+ fs::copy_file(path, dir / "cut.meta", fs::copy_options::overwrite_existing);
+ fs::resize_file(dir / "cut.meta", keep);
+ if (sm::read((dir / "cut.meta").string(), id, 1000, t, bytes, c)) {
+ std::printf("FAIL: a .meta cut to %zu of %zu bytes was read\n", keep, full);
+ ++fails;
+ break;
+ }
+ }
+ fs::copy_file(path, dir / "bad0.meta", fs::copy_options::overwrite_existing);
+ patch((dir / "bad0.meta").string(), 0, 0x55);
+ CHECK(!sm::read((dir / "bad0.meta").string(), id, 1000, t, bytes, c));
+ fs::copy_file(path, dir / "bad1.meta", fs::copy_options::overwrite_existing);
+ patch((dir / "bad1.meta").string(), full - 1, 0x55);
+ CHECK(!sm::read((dir / "bad1.meta").string(), id, 1000, t, bytes, c));
+ // a checkpoint outside the conversation, a zero one, more than four
+ CHECK(sm::write(path, id, toks, 9, {51}));
+ CHECK(!sm::read(path, id, 1000, t, bytes, c));
+ CHECK(sm::write(path, id, toks, 9, {0}));
+ CHECK(!sm::read(path, id, 1000, t, bytes, c));
+ CHECK(sm::write(path, id, toks, 9, {4, 3, 2, 1}));
+ CHECK(sm::read(path, id, 1000, t, bytes, c) && c.size() == 4);
+ CHECK(sm::write(path, id, toks, 9, {1, 2, 3, 4, 5}));
+ CHECK(!sm::read(path, id, 1000, t, bytes, c));
+ // no tokens at all
+ CHECK(sm::write(path, id, {}, 9, {}));
+ CHECK(!sm::read(path, id, 1000, t, bytes, c));
+ // an unwritable place: false, not a crash
+ CHECK(!sm::write((dir / "nodir" / "x.meta").string(), id, toks, 1, {}));
+}
+
+static size_t pick(const std::vector<std::vector<int32_t>>& toks, const std::vector<std::vector<int64_t>>& cks,
+ const std::vector<int>& prompt, size_t floor, size_t* len) {
+ std::vector<sm::Cand> c;
+ for (size_t i = 0; i < toks.size(); ++i) c.push_back({&toks[i], &cks[i]});
+ return sm::pick(c, prompt, floor, len);
+}
+
+static void pick_tests() {
+ size_t len = 99;
+ const std::vector<int32_t> a = seq(1000, 3000), b = seq(5000, 2000);
+ // the next turn of A: A whole, then new tokens
+ std::vector<int32_t> a_turn = a;
+ for (int32_t x : seq(9000, 40)) a_turn.push_back(x);
+ CHECK(pick({a, b}, {{}, {}}, as_int(a_turn), 0, &len) == 0 && len == a.size());
+ CHECK(pick({b, a}, {{}, {}}, as_int(a_turn), 0, &len) == 1 && len == a.size());
+ // nothing shared: none, len 0
+ CHECK(pick({a, b}, {{}, {}}, as_int(seq(7777, 100)), 0, &len) == SIZE_MAX && len == 0);
+ // a prompt equal to the slot's tokens: the last token is still read, so the slot is not taken whole
+ CHECK(pick({a}, {{}}, as_int(a), 0, &len) == SIZE_MAX && len == 0);
+ CHECK(pick({a}, {{2900}}, as_int(a), 0, &len) == 0 && len == 2900);
+ // a shorter prompt than the slot (a prefix of it): taken at a checkpoint within it, never past it
+ std::vector<int32_t> head(a.begin(), a.begin() + 2000);
+ CHECK(pick({a}, {{2900, 1500}}, as_int(head), 0, &len) == 0 && len == 1500);
+ // a retried prompt: A's prompt end (2950) is a checkpoint; the new prompt agrees up to 2960 then differs
+ std::vector<int32_t> retry(a.begin(), a.begin() + 2960);
+ for (int32_t x : seq(8000, 30)) retry.push_back(x);
+ CHECK(pick({a}, {{2950, 2000, 1000}}, as_int(retry), 0, &len) == 0 && len == 2950);
+ // diverging before the latest checkpoint: the latest one it still agrees with (the anchor)
+ std::vector<int32_t> early(a.begin(), a.begin() + 1200);
+ for (int32_t x : seq(8000, 30)) early.push_back(x);
+ CHECK(pick({a}, {{2950, 2000, 1000}}, as_int(early), 0, &len) == 0 && len == 1000);
+ // diverging before every checkpoint: none
+ std::vector<int32_t> first(a.begin(), a.begin() + 500);
+ for (int32_t x : seq(8000, 30)) first.push_back(x);
+ CHECK(pick({a}, {{2950, 2000, 1000}}, as_int(first), 0, &len) == SIZE_MAX && len == 0);
+ // the floor: what the model itself can go back to wins unless the slot goes further
+ CHECK(pick({a}, {{}}, as_int(a_turn), 3000, &len) == SIZE_MAX && len == 0);
+ CHECK(pick({a}, {{}}, as_int(a_turn), 2999, &len) == 0 && len == 3000);
+ // two slots, the longer match wins; a tie keeps the earlier one
+ std::vector<int32_t> a_short(a.begin(), a.begin() + 1000);
+ CHECK(pick({a_short, a}, {{}, {}}, as_int(a_turn), 0, &len) == 1 && len == 3000);
+ CHECK(pick({a, a}, {{}, {}}, as_int(a_turn), 0, &len) == 0 && len == 3000);
+ // too short a prompt
+ CHECK(pick({a}, {{}}, {1000}, 0, &len) == SIZE_MAX && len == 0);
+ CHECK(pick({a}, {{}}, {}, 0, &len) == SIZE_MAX && len == 0);
+ // an empty slot list
+ CHECK(pick({}, {}, as_int(a_turn), 0, &len) == SIZE_MAX && len == 0);
+}
+
+int main() {
+ const fs::path dir = fs::temp_directory_path() / "ds4-slot-test";
+ std::error_code ec;
+ fs::remove_all(dir, ec);
+ fs::create_directories(dir);
+ meta_tests(dir);
+ pick_tests();
+ fs::remove_all(dir, ec);
+ std::printf(fails ? "FAILED: %d\n" : "slot_test: all passed\n", fails);
+ return fails ? 1 : 0;
+}