Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-024-maya2 (6c6267f33) into f-024-rel: Windows batch submit, flow pump, Stop during a long prompt, lane-free, opt-in switches (all in src/dsv41)

Niko12216eaf32075cb9
diff --cc src/dsv41/ds4_main.cpp
index a55b717e8,0bf6f2dbc..c253925e7
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@@ -11,16 -11,10 +11,17 @@@
// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that
// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to
// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures
-// (the file: see read_pics); the state is shared only where the same picture sits at the same place.
+// (the file: see read_pics); the state is shared only where the same picture sits at the same place. With
+// STRATA_DS4_SLOT_KEEP=1 the conversations are kept when another one takes the model (a slot each: the state, the
+// compressed KV included), and across runs in STRATA_DS4_SLOT_DIR - so a request that does not continue the state but
+// does continue a kept conversation reads only its new tokens.
+// `pin=N` on a GEN marks the first N prompt tokens as a start other requests share (a system prompt and its tools): the
+// read stops there for a checkpoint, so a request behind the same start - another agent - goes back to it instead of
+// reading it again (also without slots). STRATA_DS4_REUSE=0 turns every reuse off (each request reads its whole prompt).
#include "ds4_model.hpp"
+ #include "ds4_os.hpp"
#include "ds4_packinfo.hpp"
+#include "ds4_slotmeta.hpp"
#include <algorithm>
#include <cmath>
@@@ -193,9 -199,7 +210,10 @@@ struct Req
unsigned long long seed = 0;
int spec = -1; // spec=0|1: this request drafts or not (-1: as the engine was started)
float conf = -1.0f; // dspark_conf=F: the drafts' confidence floor for this request (-1: the engine's)
+ long long pin = 0; // pin=N: the first N prompt tokens are a start other requests share (a system prompt and its
+ // tools): the state at N is kept as a checkpoint, so another agent behind the same start
+ // goes back to it instead of reading it again
+ float cum = -1.0f; // dspark_cum=F: the cut on the product of the confidences (-1: the engine's, dspark_cum_default)
std::string file; // GENI: the pictures
std::vector<int> ids;
};
@@@ -291,7 -295,7 +309,8 @@@ bool parse_gen(const std::string& line
else if (k == "seed") r.seed = std::strtoull(v.c_str(), nullptr, 10);
else if (k == "spec") r.spec = std::atoi(v.c_str()) != 0;
else if (k == "dspark_conf") r.conf = std::strtof(v.c_str(), nullptr);
+ else if (k == "pin") r.pin = std::atoll(v.c_str());
+ else if (k == "dspark_cum" || k == "spec_cum") r.cum = std::strtof(v.c_str(), nullptr);
}
return !r.ids.empty() && (!geni || !r.file.empty());
}
@@@ -753,12 -418,18 +772,22 @@@ int serve(strata::ds4::Model& m, const
});
// (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)
static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;
+ // (pin=N: the read stops at N for a checkpoint of the shared start; only when this read crosses N, and N leaves
+ // the prompt's last token to the usual end)
+ const size_t pin_at = (r.pin >= 256 && (size_t) r.pin + 1 < total) ? (size_t) r.pin : 0;
+ // a STOP while the prompt is read ends it there (the engine's run of chunks at a layer or chunk, windows between
+ // feeds) and goes back to the state before this prompt - kept here first, in checkpoint 1, which the prompt's
+ // own end overwrites as before - so the next request still shares the conversation up to it
+ if (from > 0 && from < total) m.checkpoint(1);
+ m.set_cancel([&in] { return in.stop.load(); });
+ bool stopped = false;
while (ok && done < total) {
+ if (in.stop.load()) {
+ stopped = true;
+ break;
+ }
- const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);
+ size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);
+ if (pin_at > done && pin_at < done + n) n = pin_at - done;
// (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)
const bool fin = done + n == total;
if (fin && last_window) {
diff --cc src/dsv41/ds4_model.cu
index 610a0f94a,abbccf6c6..fa051a9f3
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@@ -7163,7 -6743,7 +7244,8 @@@ struct Model::Impl
Layer& L0 = L[(size_t) il];
const bool last_l = li + 1 == c.fl_layers.size();
for (int k = 0; k < nk; ++k) {
+ if (!run.runs(k, il)) continue; // (a skipped early chunk: nothing after the yoco layer)
+ if (halt_now()) return halt_end(di, err); // (a Stop: one GPU, this one)
const int64_t p0 = run.p0[(size_t) k];
const int T = run.T[(size_t) k], slot = k % 2;
const bool last_k = k + 1 == nk;
@@@ -7335,7 -6910,8 +7418,8 @@@
wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
bool started = false;
std::vector<char> got(srcs.size(), 0);
- for (int il = 0; il < g.nl; ++il) {
+ for (int il = 0; il < g.nl && run.runs(k, il); ++il) {
+ if (halt_now()) return halt_end(di, err); // (one GPU: after the layer it is on)
Layer& L0 = L[(size_t) il];
int mi = -1;
for (int m = 0; m < L0.nm; ++m)