Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 serve: Stop ends a long prompt's reading at a layer/chunk and keeps the conversation before it (set_cancel / cancelled)

Niko1221cffd24083445
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index ecd49de63..c3a140d94 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -418,7 +418,17 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
});
// (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)
static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;
+ // a STOP while the prompt is read ends it there (the engine's run of chunks at a layer or chunk, windows between
+ // feeds) and goes back to the state before this prompt - kept here first, in checkpoint 1, which the prompt's
+ // own end overwrites as before - so the next request still shares the conversation up to it
+ if (from > 0 && from < total) m.checkpoint(1);
+ m.set_cancel([&in] { return in.stop.load(); });
+ bool stopped = false;
while (ok && done < total) {
+ if (in.stop.load()) {
+ stopped = true;
+ break;
+ }
const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);
// (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)
const bool fin = done + n == total;
@@ -438,7 +448,20 @@ int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float c
next_pp = done + 64;
}
}
+ m.set_cancel(nullptr);
m.set_images({});
+ if (stopped || (!ok && m.cancelled())) {
+ const bool back = from > 0 && m.restore((int64_t) from) && (size_t) m.pos() == from;
+ if (!back) {
+ m.reset();
+ held.clear();
+ }
+ std::fprintf(stderr, "ds4 serve: stopped while reading the prompt of %zu tokens, after %.0f ms%s\n",
+ total, now_ms() - t0, back ? "; the conversation before it is kept" : "");
+ std::printf("DONE 0 %zu %.1f 0.0 cancel 0 0 %zu 0 0\n", r.ids.size(), now_ms() - t0, back ? from : (size_t) 0);
+ std::fflush(stdout);
+ continue;
+ }
if (!ok) {
std::fprintf(stderr, "ds4: %s\n", err.c_str());
std::printf("ERR %s\n", err.c_str());
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 4cd00c699..b73c1ca4e 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -37,6 +37,7 @@
#include <deque>
#include <atomic>
#include <chrono>
+#include <climits>
#include <cmath>
#include <cstdio>
#include <cstdlib>
@@ -816,6 +817,36 @@ struct Model::Impl {
bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
std::vector<float> chunk_logits;
std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
+ // a Stop while a run of chunks goes through the layers (Model::set_cancel, asked while forward_chunks waits): one
+ // GPU stops after the layer (or the layer's chunk) it is on (halt); several stop at a chunk every one of them
+ // reaches - none starts chunk halt_k or later, halt_k one past the latest any started - so no device waits on a
+ // hand-off that never comes (Project Maya v1.1.20)
+ std::function<bool()> cancel;
+ std::atomic<bool> halt{false};
+ std::mutex halt_mu;
+ int halt_k = INT_MAX, halt_seen = -1;
+ bool halted = false; // the last run ended early on a Stop
+ static constexpr const char* kHalted = "stopped";
+ bool chunk_go(int k) { // chunk k may start on this device (several GPUs)
+ std::lock_guard<std::mutex> lk(halt_mu);
+ if (k >= halt_k) return false;
+ halt_seen = std::max(halt_seen, k);
+ return true;
+ }
+ bool halt_now() const { return devs.size() == 1 && halt.load(std::memory_order_acquire); }
+ // a run's early end on device di: the expert reads still in flight land first (they count down items the next run
+ // clears), then its streams are idle
+ bool halt_end(int di, std::string& err) {
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(devs[(size_t) di].id);
+ if (c.flow)
+ for (size_t i = c.base; i < c.n_read; ++i)
+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ if (c.cs != nullptr) cudaStreamSynchronize(c.cs);
+ cudaStreamSynchronize(devs[(size_t) di].st);
+ err = kHalted;
+ return false;
+ }
std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer
std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded
// a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this
@@ -6689,6 +6720,7 @@ struct Model::Impl {
Layer& L0 = L[(size_t) il];
const bool last_l = li + 1 == c.fl_layers.size();
for (int k = 0; k < nk; ++k) {
+ if (halt_now()) return halt_end(di, err); // (a Stop: one GPU, this one)
const int64_t p0 = run.p0[(size_t) k];
const int T = run.T[(size_t) k], slot = k % 2;
const bool last_k = k + 1 == nk;
@@ -6837,6 +6869,7 @@ struct Model::Impl {
std::future<void> eg_next;
if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);
for (int k = 0; k < nk; ++k) {
+ if (!chunk_go(k)) return halt_end(di, err); // (a Stop: the chunks every device reaches are done)
const int64_t p0 = run.p0[(size_t) k];
const int T = run.T[(size_t) k], slot = k % 2;
const bool last_k = k + 1 == nk;
@@ -6855,6 +6888,7 @@ struct Model::Impl {
bool started = false;
std::vector<char> got(srcs.size(), 0);
for (int il = 0; il < g.nl; ++il) {
+ if (halt_now()) return halt_end(di, err); // (one GPU: after the layer it is on)
Layer& L0 = L[(size_t) il];
int mi = -1;
for (int m = 0; m < L0.nm; ++m)
@@ -7355,6 +7389,14 @@ struct Model::Impl {
}
} lm_end_{this, lm};
std::vector<std::string> errs(devs.size());
+ halted = false;
+ halt.store(false);
+ {
+ std::lock_guard<std::mutex> lk(halt_mu);
+ halt_k = INT_MAX;
+ halt_seen = -1;
+ }
+ bool halting = false;
for (size_t di = 0; di < devs.size(); ++di)
workers[di]->submit([this, di, &run, &errs] {
std::string e;
@@ -7364,6 +7406,12 @@ struct Model::Impl {
for (;;) {
bool all = true;
for (auto& w : workers) all = all && w->idle();
+ if (!all && !halting && cancel && cancel()) { // a Stop: the devices end the run early
+ halting = true;
+ std::lock_guard<std::mutex> lk(halt_mu);
+ halt_k = halt_seen + 1;
+ halt.store(true, std::memory_order_release);
+ }
if (progress && lm) { // (layer-major: the share of the layers every chunk went through)
const int nd = lm_layers.load(std::memory_order_acquire);
if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);
@@ -7377,11 +7425,17 @@ struct Model::Impl {
if (all) break;
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
- for (const std::string& e : errs)
- if (!e.empty()) {
+ for (const std::string& e : errs) // (a device's failure first: a Stop's early end is not one)
+ if (!e.empty() && e != kHalted) {
err = e;
return false;
}
+ for (const std::string& e : errs)
+ if (e == kHalted) {
+ halted = true;
+ err = "stopped while reading the prompt";
+ return false;
+ }
st.layer_ms += now_ms() - tl;
eg_stats();
Dev& d = devs[(size_t) head_dev];
@@ -8376,6 +8430,7 @@ bool Model::step(int token, bool logits, std::string& err) { return p_->step(tok
bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
Impl& p = *p_;
p.pm.last = -1;
+ p.halted = false;
if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)
@@ -8388,6 +8443,7 @@ bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {
Impl& p = *p_;
p.pm.last = -1;
+ p.halted = false;
const double t0 = now_ms();
if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers
const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)
@@ -8402,6 +8458,8 @@ bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::
}
int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }
void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }
+void Model::set_cancel(std::function<bool()> f) { p_->cancel = std::move(f); }
+bool Model::cancelled() const { return p_->halted; }
int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {
return p_->sample(t, tp, tk, mp, seed, counter);
}
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index 929636d53..483f79382 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -134,6 +134,11 @@ public:
int feed_unit() const;
/// Called from feed() while a run of chunks goes through the layers, with the tokens done so far.
void set_progress(std::function<void(int64_t)> f);
+ /// Asked while a run of chunks goes through the layers: true (a Stop) ends it early - one GPU after the layer it is
+ /// on, several at a chunk all of them reach - and feed() returns false with cancelled() true. The state past the
+ /// prompt's start is then not usable: restore() a checkpoint kept before the feed, or reset().
+ void set_cancel(std::function<bool()> f);
+ bool cancelled() const; // the last feed() / feed_mark() ended early on a Stop
int64_t pos() const;
int64_t max_context() const;
int vocab() const;