Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 serve: Stop ends a long prompt's reading at a layer/chunk and keeps the conversation before it (set_cancel / cancelled)
Niko1221cffd24083445
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex ecd49de63..c3a140d94 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -418,7 +418,17 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c});// (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;+ // a STOP while the prompt is read ends it there (the engine's run of chunks at a layer or chunk, windows between+ // feeds) and goes back to the state before this prompt - kept here first, in checkpoint 1, which the prompt's+ // own end overwrites as before - so the next request still shares the conversation up to it+ if (from > 0 && from < total) m.checkpoint(1);+ m.set_cancel([&in] { return in.stop.load(); });+ bool stopped = false;while (ok && done < total) {+ if (in.stop.load()) {+ stopped = true;+ break;+ }const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);// (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)const bool fin = done + n == total;@@ -438,7 +448,20 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float cnext_pp = done + 64;}}+ m.set_cancel(nullptr);m.set_images({});+ if (stopped || (!ok && m.cancelled())) {+ const bool back = from > 0 && m.restore((int64_t) from) && (size_t) m.pos() == from;+ if (!back) {+ m.reset();+ held.clear();+ }+ std::fprintf(stderr, "ds4 serve: stopped while reading the prompt of %zu tokens, after %.0f ms%s\n",+ total, now_ms() - t0, back ? "; the conversation before it is kept" : "");+ std::printf("DONE 0 %zu %.1f 0.0 cancel 0 0 %zu 0 0\n", r.ids.size(), now_ms() - t0, back ? from : (size_t) 0);+ std::fflush(stdout);+ continue;+ }if (!ok) {std::fprintf(stderr, "ds4: %s\n", err.c_str());std::printf("ERR %s\n", err.c_str());diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 4cd00c699..b73c1ca4e 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -37,6 +37,7 @@#include <deque>#include <atomic>#include <chrono>+#include <climits>#include <cmath>#include <cstdio>#include <cstdlib>@@ -816,6 +817,36 @@ struct Model::Impl {bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)std::vector<float> chunk_logits;std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)+ // a Stop while a run of chunks goes through the layers (Model::set_cancel, asked while forward_chunks waits): one+ // GPU stops after the layer (or the layer's chunk) it is on (halt); several stop at a chunk every one of them+ // reaches - none starts chunk halt_k or later, halt_k one past the latest any started - so no device waits on a+ // hand-off that never comes (Project Maya v1.1.20)+ std::function<bool()> cancel;+ std::atomic<bool> halt{false};+ std::mutex halt_mu;+ int halt_k = INT_MAX, halt_seen = -1;+ bool halted = false; // the last run ended early on a Stop+ static constexpr const char* kHalted = "stopped";+ bool chunk_go(int k) { // chunk k may start on this device (several GPUs)+ std::lock_guard<std::mutex> lk(halt_mu);+ if (k >= halt_k) return false;+ halt_seen = std::max(halt_seen, k);+ return true;+ }+ bool halt_now() const { return devs.size() == 1 && halt.load(std::memory_order_acquire); }+ // a run's early end on device di: the expert reads still in flight land first (they count down items the next run+ // clears), then its streams are idle+ bool halt_end(int di, std::string& err) {+ Chunk& c = *chunks[(size_t) di];+ cudaSetDevice(devs[(size_t) di].id);+ if (c.flow)+ for (size_t i = c.base; i < c.n_read; ++i)+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);+ if (c.cs != nullptr) cudaStreamSynchronize(c.cs);+ cudaStreamSynchronize(devs[(size_t) di].st);+ err = kHalted;+ return false;+ }std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layerstd::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded// a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this@@ -6689,6 +6720,7 @@ struct Model::Impl {Layer& L0 = L[(size_t) il];const bool last_l = li + 1 == c.fl_layers.size();for (int k = 0; k < nk; ++k) {+ if (halt_now()) return halt_end(di, err); // (a Stop: one GPU, this one)const int64_t p0 = run.p0[(size_t) k];const int T = run.T[(size_t) k], slot = k % 2;const bool last_k = k + 1 == nk;@@ -6837,6 +6869,7 @@ struct Model::Impl {std::future<void> eg_next;if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);for (int k = 0; k < nk; ++k) {+ if (!chunk_go(k)) return halt_end(di, err); // (a Stop: the chunks every device reaches are done)const int64_t p0 = run.p0[(size_t) k];const int T = run.T[(size_t) k], slot = k % 2;const bool last_k = k + 1 == nk;@@ -6855,6 +6888,7 @@ struct Model::Impl {bool started = false;std::vector<char> got(srcs.size(), 0);for (int il = 0; il < g.nl; ++il) {+ if (halt_now()) return halt_end(di, err); // (one GPU: after the layer it is on)Layer& L0 = L[(size_t) il];int mi = -1;for (int m = 0; m < L0.nm; ++m)@@ -7355,6 +7389,14 @@ struct Model::Impl {}} lm_end_{this, lm};std::vector<std::string> errs(devs.size());+ halted = false;+ halt.store(false);+ {+ std::lock_guard<std::mutex> lk(halt_mu);+ halt_k = INT_MAX;+ halt_seen = -1;+ }+ bool halting = false;for (size_t di = 0; di < devs.size(); ++di)workers[di]->submit([this, di, &run, &errs] {std::string e;@@ -7364,6 +7406,12 @@ struct Model::Impl {for (;;) {bool all = true;for (auto& w : workers) all = all && w->idle();+ if (!all && !halting && cancel && cancel()) { // a Stop: the devices end the run early+ halting = true;+ std::lock_guard<std::mutex> lk(halt_mu);+ halt_k = halt_seen + 1;+ halt.store(true, std::memory_order_release);+ }if (progress && lm) { // (layer-major: the share of the layers every chunk went through)const int nd = lm_layers.load(std::memory_order_acquire);if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);@@ -7377,11 +7425,17 @@ struct Model::Impl {if (all) break;std::this_thread::sleep_for(std::chrono::milliseconds(50));}- for (const std::string& e : errs)- if (!e.empty()) {+ for (const std::string& e : errs) // (a device's failure first: a Stop's early end is not one)+ if (!e.empty() && e != kHalted) {err = e;return false;}+ for (const std::string& e : errs)+ if (e == kHalted) {+ halted = true;+ err = "stopped while reading the prompt";+ return false;+ }st.layer_ms += now_ms() - tl;eg_stats();Dev& d = devs[(size_t) head_dev];@@ -8376,6 +8430,7 @@ bool Model::step(int token, bool logits, std::string& err) { return p_->step(tokbool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {Impl& p = *p_;p.pm.last = -1;+ p.halted = false;if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path hereconst double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)@@ -8388,6 +8443,7 @@ bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {Impl& p = *p_;p.pm.last = -1;+ p.halted = false;const double t0 = now_ms();if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layersconst bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)@@ -8402,6 +8458,8 @@ bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::}int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }+void Model::set_cancel(std::function<bool()> f) { p_->cancel = std::move(f); }+bool Model::cancelled() const { return p_->halted; }int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {return p_->sample(t, tp, tk, mp, seed, counter);}diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex 929636d53..483f79382 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -134,6 +134,11 @@ public:int feed_unit() const;/// Called from feed() while a run of chunks goes through the layers, with the tokens done so far.void set_progress(std::function<void(int64_t)> f);+ /// Asked while a run of chunks goes through the layers: true (a Stop) ends it early - one GPU after the layer it is+ /// on, several at a chunk all of them reach - and feed() returns false with cancelled() true. The state past the+ /// prompt's start is then not usable: restore() a checkpoint kept before the feed, or reset().+ void set_cancel(std::function<bool()> f);+ bool cancelled() const; // the last feed() / feed_mark() ended early on a Stopint64_t pos() const;int64_t max_context() const;int vocab() const;