Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-024-maya2 (6c6267f33) into f-024-rel: Windows batch submit, flow pump, Stop during a long prompt, lane-free, opt-in switches (all in src/dsv41)
Niko12216eaf32075cb9
diff --cc src/dsv41/ds4_main.cppindex a55b717e8,0bf6f2dbc..c253925e7--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@@ -11,16 -11,10 +11,17 @@@// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures-// (the file: see read_pics); the state is shared only where the same picture sits at the same place.+// (the file: see read_pics); the state is shared only where the same picture sits at the same place. With+// STRATA_DS4_SLOT_KEEP=1 the conversations are kept when another one takes the model (a slot each: the state, the+// compressed KV included), and across runs in STRATA_DS4_SLOT_DIR - so a request that does not continue the state but+// does continue a kept conversation reads only its new tokens.+// `pin=N` on a GEN marks the first N prompt tokens as a start other requests share (a system prompt and its tools): the+// read stops there for a checkpoint, so a request behind the same start - another agent - goes back to it instead of+// reading it again (also without slots). STRATA_DS4_REUSE=0 turns every reuse off (each request reads its whole prompt).#include "ds4_model.hpp"+ #include "ds4_os.hpp"#include "ds4_packinfo.hpp"+#include "ds4_slotmeta.hpp"#include <algorithm>#include <cmath>@@@ -193,9 -199,7 +210,10 @@@ struct Requnsigned long long seed = 0;int spec = -1; // spec=0|1: this request drafts or not (-1: as the engine was started)float conf = -1.0f; // dspark_conf=F: the drafts' confidence floor for this request (-1: the engine's)+ long long pin = 0; // pin=N: the first N prompt tokens are a start other requests share (a system prompt and its+ // tools): the state at N is kept as a checkpoint, so another agent behind the same start+ // goes back to it instead of reading it again+ float cum = -1.0f; // dspark_cum=F: the cut on the product of the confidences (-1: the engine's, dspark_cum_default)std::string file; // GENI: the picturesstd::vector<int> ids;};@@@ -291,7 -295,7 +309,8 @@@ bool parse_gen(const std::string& lineelse if (k == "seed") r.seed = std::strtoull(v.c_str(), nullptr, 10);else if (k == "spec") r.spec = std::atoi(v.c_str()) != 0;else if (k == "dspark_conf") r.conf = std::strtof(v.c_str(), nullptr);+ else if (k == "pin") r.pin = std::atoll(v.c_str());+ else if (k == "dspark_cum" || k == "spec_cum") r.cum = std::strtof(v.c_str(), nullptr);}return !r.ids.empty() && (!geni || !r.file.empty());}@@@ -753,12 -418,18 +772,22 @@@ int serve(strata::ds4::Model& m, const});// (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;+ // (pin=N: the read stops at N for a checkpoint of the shared start; only when this read crosses N, and N leaves+ // the prompt's last token to the usual end)+ const size_t pin_at = (r.pin >= 256 && (size_t) r.pin + 1 < total) ? (size_t) r.pin : 0;+ // a STOP while the prompt is read ends it there (the engine's run of chunks at a layer or chunk, windows between+ // feeds) and goes back to the state before this prompt - kept here first, in checkpoint 1, which the prompt's+ // own end overwrites as before - so the next request still shares the conversation up to it+ if (from > 0 && from < total) m.checkpoint(1);+ m.set_cancel([&in] { return in.stop.load(); });+ bool stopped = false;while (ok && done < total) {+ if (in.stop.load()) {+ stopped = true;+ break;+ }- const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);+ size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);+ if (pin_at > done && pin_at < done + n) n = pin_at - done;// (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)const bool fin = done + n == total;if (fin && last_window) {diff --cc src/dsv41/ds4_model.cuindex 610a0f94a,abbccf6c6..fa051a9f3--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@@ -7163,7 -6743,7 +7244,8 @@@ struct Model::ImplLayer& L0 = L[(size_t) il];const bool last_l = li + 1 == c.fl_layers.size();for (int k = 0; k < nk; ++k) {+ if (!run.runs(k, il)) continue; // (a skipped early chunk: nothing after the yoco layer)+ if (halt_now()) return halt_end(di, err); // (a Stop: one GPU, this one)const int64_t p0 = run.p0[(size_t) k];const int T = run.T[(size_t) k], slot = k % 2;const bool last_k = k + 1 == nk;@@@ -7335,7 -6910,8 +7418,8 @@@wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;bool started = false;std::vector<char> got(srcs.size(), 0);- for (int il = 0; il < g.nl; ++il) {+ for (int il = 0; il < g.nl && run.runs(k, il); ++il) {+ if (halt_now()) return halt_end(di, err); // (one GPU: after the layer it is on)Layer& L0 = L[(size_t) il];int mi = -1;for (int m = 0; m < L0.nm; ++m)