Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: a checkpoint whose copy runs out of memory is skipped, it no longer ends the engine (#3350)

Niko1221f62589fdbe80
diff --git a/src/core/conversation_copy.hpp b/src/core/conversation_copy.hpp
index 7e8341b3a..2a9c4c029 100644
--- a/src/core/conversation_copy.hpp
+++ b/src/core/conversation_copy.hpp
@@ -6,6 +6,7 @@
#include <cuda_runtime.h>
#include <algorithm>
+#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdlib>
@@ -35,9 +36,20 @@ inline cudaError_t private_stream(cudaStream_t& s) {
return cudaSuccess;
}
+/// STRATA_TEST_COPY_FAIL_AT=N (tests only, #3350): the N-th snapshot / checkpoint copy of the process fails with
+/// cudaErrorMemoryAllocation, as when a card has no memory left for the copy. Unset: nothing.
+inline long test_copy_fail_at() {
+ static const long v = [] { const char* e = std::getenv("STRATA_TEST_COPY_FAIL_AT"); return e ? std::atol(e) : 0L; }();
+ return v;
+}
+
/// `queued`: the copy is only queued on the private stream, and the caller synchronizes the device (or orders a
/// later launch after it with order_after_queued) before anything reads the target or writes the source.
inline cudaError_t copy_nonblocking(void* dst, const void* src, size_t bytes, bool queued = false) {
+ if (const long at = test_copy_fail_at(); at > 0) {
+ static std::atomic<long> copies{0};
+ if (++copies == at) return cudaErrorMemoryAllocation;
+ }
cudaStream_t s = nullptr;
cudaError_t e = private_stream(s);
if (e != cudaSuccess) return e;
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 6452a35cc..75d404ac9 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -1745,6 +1745,15 @@ bool checkpoint_save(ConvCheckpoint& c, const strata::core::SessionState& ss, co
return false;
}
+/// #3350: a conversation checkpoint is an optimization, like parking. After a failed save: true when the failure was
+/// only the copy's memory (a host or driver allocation, which does not stick), so the caller goes on without that
+/// checkpoint; false when the GPU itself is unwell (a sticky error comes back from the synchronize), which still ends
+/// the request as before.
+bool checkpoint_failure_is_survivable() {
+ (void) cudaGetLastError(); // clears a failed allocation; a sticky error is not cleared and is returned again below
+ return cudaDeviceSynchronize() == cudaSuccess;
+}
+
/// Puts a checkpoint's running state back; the positional cells below it are the caller's to guarantee.
bool checkpoint_restore(const ConvCheckpoint& c, strata::core::SessionState& ss, const strata::core::ModelGeometry& g) {
std::string error;
@@ -10060,7 +10069,15 @@ int main(int argc, char** argv) {
} else {
saved = checkpoint_at(done, nullptr, tail_checkpoint && !periodic_checkpoint);
}
- if (!saved) { e = "saving a conversation checkpoint failed" + ckpt_why; return false; }
+ if (!saved) {
+ // #3350: only a GPU that is unwell ends the request; a copy that ran out of memory skips this checkpoint
+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) {
+ e = "saving a conversation checkpoint failed" + ckpt_why;
+ return false;
+ }
+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "
+ "above); the request goes on without it\n", (long long) done);
+ }
if (periodic_checkpoint) pp_next_check = done + o.prompt_cache_every;
if (tail_checkpoint) pp_tail_saved = true;
}
@@ -10077,7 +10094,16 @@ int main(int argc, char** argv) {
part_next[i] = done + o.prompt_cache_every;
ConvCheckpoint part; // this stage's state at `done` (its stream is synchronized)
part.ids.assign(cur.begin(), cur.begin() + done);
- if (!checkpoint_save(part, stage_ss, g)) { e = "saving a checkpoint part failed"; return false; }
+ if (!checkpoint_save(part, stage_ss, g)) {
+ // #3350: the copy ran out of memory (not the GPU): leave this part out. The set at `done` is then
+ // incomplete and the checkpoint is skipped below, as for a part that was never made
+ if (!checkpoint_failure_is_survivable()) { e = "saving a checkpoint part failed"; return false; }
+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: stage %zu's state could not be copied "
+ "out (see above); the request goes on without it\n", (long long) done, i);
+ std::lock_guard<std::mutex> lk(part_mu);
+ part_at[done].resize(stages.size() + 1); // (marks the set as seen; its empty parts make it incomplete)
+ return true;
+ }
std::lock_guard<std::mutex> lk(part_mu);
auto& v = part_at[done];
v.resize(stages.size() + 1);
@@ -14278,8 +14304,12 @@ int main(int argc, char** argv) {
at = to;
if ((to == turn_at || to == root_at || to == message_at || to == pin_at) &&
!checkpoint_at(to, nullptr, to == message_at && to != turn_at && to != root_at && to != pin_at)) { // the tail kind leaves first
- std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());
- return 1;
+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) { // #3350: only an unwell GPU ends the request
+ std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());
+ return 1;
+ }
+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "
+ "above); the request goes on without it\n", (long long) to);
}
if (to == pin_at && !pin_checkpoint(pin_at))
std::fprintf(stderr, "strata serve: pin=%lld: its checkpoint was not kept\n", (long long) pin_at);
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cpp
index 442fd167c..a38529303 100644
--- a/sycl/src/program/generate.cpp
+++ b/sycl/src/program/generate.cpp
@@ -1434,6 +1434,20 @@ bool checkpoint_save(ConvCheckpoint& c, const strata::core::SessionState& ss, co
return false;
}
+/// #3350: a conversation checkpoint is an optimization, like parking. After a failed save: true when the failure was
+/// only the copy's memory (a host or driver allocation, which does not stick), so the caller goes on without that
+/// checkpoint; false when the GPU itself is unwell (a sticky error comes back from the synchronize), which still ends
+/// the request as before.
+bool checkpoint_failure_is_survivable() {
+ // SYCL has no sticky error state to clear: the queues either drain or throw (a lost device)
+ try {
+ dpct::get_current_device().queues_wait_and_throw();
+ return true;
+ } catch (...) {
+ return false;
+ }
+}
+
/// Puts a checkpoint's running state back; the positional cells below it are the caller's to guarantee.
bool checkpoint_restore(const ConvCheckpoint& c, strata::core::SessionState& ss, const strata::core::ModelGeometry& g) {
std::string error;
@@ -7952,7 +7966,15 @@ int main(int argc, char **argv) try {
} else {
saved = checkpoint_at(done, nullptr, tail_checkpoint && !periodic_checkpoint);
}
- if (!saved) { e = "saving a conversation checkpoint failed" + ckpt_why; return false; }
+ if (!saved) {
+ // #3350: only a GPU that is unwell ends the request; a copy that ran out of memory skips this checkpoint
+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) {
+ e = "saving a conversation checkpoint failed" + ckpt_why;
+ return false;
+ }
+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "
+ "above); the request goes on without it\n", (long long) done);
+ }
if (periodic_checkpoint) pp_next_check = done + o.prompt_cache_every;
if (tail_checkpoint) pp_tail_saved = true;
}
@@ -7969,7 +7991,16 @@ int main(int argc, char **argv) try {
part_next[i] = done + o.prompt_cache_every;
ConvCheckpoint part; // this stage's state at `done` (its stream is synchronized)
part.ids.assign(cur.begin(), cur.begin() + done);
- if (!checkpoint_save(part, stage_ss, g)) { e = "saving a checkpoint part failed"; return false; }
+ if (!checkpoint_save(part, stage_ss, g)) {
+ // #3350: the copy ran out of memory (not the GPU): leave this part out. The set at `done` is then
+ // incomplete and the checkpoint is skipped below, as for a part that was never made
+ if (!checkpoint_failure_is_survivable()) { e = "saving a checkpoint part failed"; return false; }
+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: stage %zu's state could not be copied "
+ "out (see above); the request goes on without it\n", (long long) done, i);
+ std::lock_guard<std::mutex> lk(part_mu);
+ part_at[done].resize(stages.size() + 1); // (marks the set as seen; its empty parts make it incomplete)
+ return true;
+ }
std::lock_guard<std::mutex> lk(part_mu);
auto& v = part_at[done];
v.resize(stages.size() + 1);
@@ -10680,8 +10711,12 @@ int main(int argc, char **argv) try {
at = to;
if ((to == turn_at || to == root_at || to == message_at) &&
!checkpoint_at(to, nullptr, to == message_at && to != turn_at && to != root_at)) { // the tail kind leaves first
- std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());
- return 1;
+ if (!ckpt_why.empty() || !checkpoint_failure_is_survivable()) { // #3350: only an unwell GPU ends the request
+ std::printf("ERR saving a conversation checkpoint failed%s\n", ckpt_why.c_str());
+ return 1;
+ }
+ std::fprintf(stderr, "strata serve: checkpoint at %lld skipped: its state could not be copied out (see "
+ "above); the request goes on without it\n", (long long) to);
}
if (trace && to == message_at)
std::fprintf(stderr, "strata serve: message boundary checkpoint: %lld tokens, %lld tail\n",