Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: a checkpoint whose copy runs out of memory is skipped, it no longer ends the engine (#3350)
Niko1221f62589fdbe80
diff --git a/src/core/conversation_copy.hpp b/src/core/conversation_copy.hppindex 7e8341b3a..2a9c4c029 100644--- a/src/core/conversation_copy.hpp+++ b/src/core/conversation_copy.hpp@@ -6,6 +6,7 @@#include <cuda_runtime.h>#include <algorithm>+#include <atomic>#include <chrono>#include <cstddef>#include <cstdlib>@@ -35,9 +36,20 @@ inline cudaError_t private_stream(cudaStream_t& s) {return cudaSuccess;}+/// STRATA_TEST_COPY_FAIL_AT=N (tests only, #3350): the N-th snapshot / checkpoint copy of the process fails with+/// cudaErrorMemoryAllocation, as when a card has no memory left for the copy. Unset: nothing.+inline long test_copy_fail_at() {+ static const long v = [] { const char* e = std::getenv("STRATA_TEST_COPY_FAIL_AT"); return e ? std::atol(e) : 0L; }();+ return v;+}+/// `queued`: the copy is only queued on the private stream, and the caller synchronizes the device (or orders a/// later launch after it with order_after_queued) before anything reads the target or writes the source.inline cudaError_t copy_nonblocking(void* dst, const void* src, size_t bytes, bool queued = false) {+ if (const long at = test_copy_fail_at(); at > 0) {+ static std::atomic<long> copies{0};+ if (++copies == at) return cudaErrorMemoryAllocation;+ }cudaStream_t s = nullptr;cudaError_t e = private_stream(s);if (e != cudaSuccess) return e;diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 6452a35cc..75d404ac9 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -1745,6 +1745,15 @@ bool checkpoint_save(ConvCheckpoint& c, const strata::core::SessionState& ss, coreturn false;}+/// #3350: a conversation checkpoint is an optimization, like parking. After a failed save: true when the failure was+/// only the copy's memory (a host or driver allocation, which does not stick), so the caller goes on without that+/// checkpoint; false when the GPU itself is unwell (a sticky error comes back from the synchronize), which still ends+/// the request as before.+bool checkpoint_failure_is_survivable() {+ (void) cudaGetLastError(); // clears a failed allocation; a sticky error is not cleared and is returned again below+ return cudaDeviceSynchronize() == cudaSuccess;+}+/// Puts a checkpoint's running state back; the positional cells below it are the caller's to guarantee.bool checkpoint_restore(const ConvCheckpoint& c, strata::core::SessionState& ss, const strata::core::ModelGeometry& g) {std::string error;@@ -10060,7 +10069,15 @@ int main(int argc, char** argv) {} else {saved = checkpoint_at(done, nullptr, tail_checkpoint && !periodic_checkpoint);}- if (!saved) { e = "saving a conversation checkpoint failed" + ckpt_why; return false; }+ if (!saved) {+ // #3350: only a GPU that is unwell ends the request; a copy that ran out of memory skips this checkpoint+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) {+ e = "saving a conversation checkpoint failed" + ckpt_why;+ return false;+ }+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "+ "above); the request goes on without it\n", (long long) done);+ }if (periodic_checkpoint) pp_next_check = done + o.prompt_cache_every;if (tail_checkpoint) pp_tail_saved = true;}@@ -10077,7 +10094,16 @@ int main(int argc, char** argv) {part_next[i] = done + o.prompt_cache_every;ConvCheckpoint part; // this stage's state at `done` (its stream is synchronized)part.ids.assign(cur.begin(), cur.begin() + done);- if (!checkpoint_save(part, stage_ss, g)) { e = "saving a checkpoint part failed"; return false; }+ if (!checkpoint_save(part, stage_ss, g)) {+ // #3350: the copy ran out of memory (not the GPU): leave this part out. The set at `done` is then+ // incomplete and the checkpoint is skipped below, as for a part that was never made+ if (!checkpoint_failure_is_survivable()) { e = "saving a checkpoint part failed"; return false; }+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: stage %zu's state could not be copied "+ "out (see above); the request goes on without it\n", (long long) done, i);+ std::lock_guard<std::mutex> lk(part_mu);+ part_at[done].resize(stages.size() + 1); // (marks the set as seen; its empty parts make it incomplete)+ return true;+ }std::lock_guard<std::mutex> lk(part_mu);auto& v = part_at[done];v.resize(stages.size() + 1);@@ -14278,8 +14304,12 @@ int main(int argc, char** argv) {at = to;if ((to == turn_at || to == root_at || to == message_at || to == pin_at) &&!checkpoint_at(to, nullptr, to == message_at && to != turn_at && to != root_at && to != pin_at)) { // the tail kind leaves first- std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());- return 1;+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) { // #3350: only an unwell GPU ends the request+ std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());+ return 1;+ }+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "+ "above); the request goes on without it\n", (long long) to);}if (to == pin_at && !pin_checkpoint(pin_at))std::fprintf(stderr, "strata serve: pin=%lld: its checkpoint was not kept\n", (long long) pin_at);diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cppindex 442fd167c..a38529303 100644--- a/sycl/src/program/generate.cpp+++ b/sycl/src/program/generate.cpp@@ -1434,6 +1434,20 @@ bool checkpoint_save(ConvCheckpoint& c, const strata::core::SessionState& ss, coreturn false;}+/// #3350: a conversation checkpoint is an optimization, like parking. After a failed save: true when the failure was+/// only the copy's memory (a host or driver allocation, which does not stick), so the caller goes on without that+/// checkpoint; false when the GPU itself is unwell (a sticky error comes back from the synchronize), which still ends+/// the request as before.+bool checkpoint_failure_is_survivable() {+ // SYCL has no sticky error state to clear: the queues either drain or throw (a lost device)+ try {+ dpct::get_current_device().queues_wait_and_throw();+ return true;+ } catch (...) {+ return false;+ }+}+/// Puts a checkpoint's running state back; the positional cells below it are the caller's to guarantee.bool checkpoint_restore(const ConvCheckpoint& c, strata::core::SessionState& ss, const strata::core::ModelGeometry& g) {std::string error;@@ -7952,7 +7966,15 @@ int main(int argc, char **argv) try {} else {saved = checkpoint_at(done, nullptr, tail_checkpoint && !periodic_checkpoint);}- if (!saved) { e = "saving a conversation checkpoint failed" + ckpt_why; return false; }+ if (!saved) {+ // #3350: only a GPU that is unwell ends the request; a copy that ran out of memory skips this checkpoint+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) {+ e = "saving a conversation checkpoint failed" + ckpt_why;+ return false;+ }+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "+ "above); the request goes on without it\n", (long long) done);+ }if (periodic_checkpoint) pp_next_check = done + o.prompt_cache_every;if (tail_checkpoint) pp_tail_saved = true;}@@ -7969,7 +7991,16 @@ int main(int argc, char **argv) try {part_next[i] = done + o.prompt_cache_every;ConvCheckpoint part; // this stage's state at `done` (its stream is synchronized)part.ids.assign(cur.begin(), cur.begin() + done);- if (!checkpoint_save(part, stage_ss, g)) { e = "saving a checkpoint part failed"; return false; }+ if (!checkpoint_save(part, stage_ss, g)) {+ // #3350: the copy ran out of memory (not the GPU): leave this part out. The set at `done` is then+ // incomplete and the checkpoint is skipped below, as for a part that was never made+ if (!checkpoint_failure_is_survivable()) { e = "saving a checkpoint part failed"; return false; }+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: stage %zu's state could not be copied "+ "out (see above); the request goes on without it\n", (long long) done, i);+ std::lock_guard<std::mutex> lk(part_mu);+ part_at[done].resize(stages.size() + 1); // (marks the set as seen; its empty parts make it incomplete)+ return true;+ }std::lock_guard<std::mutex> lk(part_mu);auto& v = part_at[done];v.resize(stages.size() + 1);@@ -10680,8 +10711,12 @@ int main(int argc, char **argv) try {at = to;if ((to == turn_at || to == root_at || to == message_at) &&!checkpoint_at(to, nullptr, to == message_at && to != turn_at && to != root_at)) { // the tail kind leaves first- std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());- return 1;+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) { // #3350: only an unwell GPU ends the request+ std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());+ return 1;+ }+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "+ "above); the request goes on without it\n", (long long) to);}if (trace && to == message_at)std::fprintf(stderr, "strata serve: message boundary checkpoint: %lld tokens, %lld tail\n",