Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

conversation copies: the private copy streams and events are handed back to a pool when a thread ends (#3350, #3470)

Niko12213478382eeecd
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1cf372d97..6356e27b6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1486,6 +1486,13 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(pin_budget_test PRIVATE strata_core ${_strata_gpu_runtime_target})
add_test(NAME pin_budget_test COMMAND pin_budget_test)
+ # #3350 / #3470: the conversation copy streams and events are pooled across short-lived threads (needs a device, else skips)
+ add_executable(conversation_copy_streams_test src/core/conversation_copy_streams_test.cpp)
+ target_include_directories(conversation_copy_streams_test PRIVATE include)
+ target_link_libraries(conversation_copy_streams_test PRIVATE strata_core ${_strata_gpu_runtime_target})
+ add_test(NAME conversation_copy_streams_test COMMAND conversation_copy_streams_test)
+ set_tests_properties(conversation_copy_streams_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
+
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_executable(pinned_shared_test src/core/pinned_shared_test.cpp)
target_link_libraries(pinned_shared_test PRIVATE strata_core ${_strata_gpu_runtime_target})
diff --git a/src/core/conversation_copy.hpp b/src/core/conversation_copy.hpp
index 2a9c4c029..7ffb87f4a 100644
--- a/src/core/conversation_copy.hpp
+++ b/src/core/conversation_copy.hpp
@@ -10,27 +10,88 @@
#include <chrono>
#include <cstddef>
#include <cstdlib>
+#include <mutex>
#include <thread>
#include <vector>
namespace strata::core::conversation_detail {
+/// The private copy streams and ordering events (below) belong to a thread, but a thread can be short-lived: the layer
+/// split's later stages run each prompt chunk on a fresh std::async thread, and a checkpoint part is saved on it. The
+/// handles used to be created per thread and never destroyed, so every such thread left one stream and one event per
+/// device behind - a few MiB of VRAM per long cold prompt on the later cards, until a stream could not be created any
+/// more and the copy failed with "out of memory" (#3350, #3470). Now a thread hands its handles back to a process-wide
+/// pool when it ends and the next thread takes them: the number of handles is the number of threads copying at once.
+/// The pool is leaked on purpose (a thread may end after the statics are gone).
+struct CopyHandlePool {
+ std::mutex mu;
+ std::vector<std::vector<cudaStream_t>> streams; // free ones, by device
+ std::vector<std::vector<cudaEvent_t>> events;
+ long streams_created = 0, events_created = 0;
+};
+inline CopyHandlePool& copy_handle_pool() {
+ static CopyHandlePool* p = new CopyHandlePool;
+ return *p;
+}
+/// How many streams / events the process has created for these copies (tests: it stays at the peak number of threads
+/// copying at once, not at the number of threads that ever copied).
+inline long copy_streams_created() {
+ CopyHandlePool& p = copy_handle_pool();
+ std::lock_guard<std::mutex> lk(p.mu);
+ return p.streams_created;
+}
+inline long copy_events_created() {
+ CopyHandlePool& p = copy_handle_pool();
+ std::lock_guard<std::mutex> lk(p.mu);
+ return p.events_created;
+}
+struct ThreadCopyHandles {
+ std::vector<cudaStream_t> streams; // by device, nullptr = none yet
+ std::vector<cudaEvent_t> events;
+ ThreadCopyHandles() = default;
+ ThreadCopyHandles(const ThreadCopyHandles&) = delete;
+ ThreadCopyHandles& operator=(const ThreadCopyHandles&) = delete;
+ ~ThreadCopyHandles() {
+ CopyHandlePool& p = copy_handle_pool();
+ std::lock_guard<std::mutex> lk(p.mu);
+ if (p.streams.size() < streams.size()) p.streams.resize(streams.size());
+ if (p.events.size() < events.size()) p.events.resize(events.size());
+ for (size_t d = 0; d < streams.size(); ++d)
+ if (streams[d]) p.streams[d].push_back(streams[d]);
+ for (size_t d = 0; d < events.size(); ++d)
+ if (events[d]) p.events[d].push_back(events[d]);
+ }
+};
+inline ThreadCopyHandles& thread_copy_handles() {
+ thread_local ThreadCopyHandles t;
+ return t;
+}
+
/// Conversation snapshot / checkpoint copies go through a private non-blocking stream of the current device
/// instead of the legacy (null) stream. A plain cudaMemcpy is ordered against every blocking stream of the
/// device, and while another thread captures a graph on one (a drafter or a stage worker), HIP refuses it with
/// "operation would make the legacy stream depend on a capturing blocking stream". The caller's contract is
/// unchanged: the device was synchronized before the copy (the copy itself is complete on return).
-/// private_stream: this thread's stream on the current device, created on first use.
+/// private_stream: this thread's stream on the current device: one a finished thread handed back, else a new one on
+/// first use.
inline cudaError_t private_stream(cudaStream_t& s) {
int dev = 0;
cudaError_t e = cudaGetDevice(&dev);
if (e != cudaSuccess) return e;
- thread_local std::vector<cudaStream_t> streams;
+ std::vector<cudaStream_t>& streams = thread_copy_handles().streams;
if (dev < 0) dev = 0;
- if ((size_t) dev >= streams.size()) streams.resize((size_t) dev + 1, nullptr); // never destroyed: process lifetime
+ if ((size_t) dev >= streams.size()) streams.resize((size_t) dev + 1, nullptr);
if (!streams[(size_t) dev]) {
- e = cudaStreamCreateWithFlags(&streams[(size_t) dev], cudaStreamNonBlocking);
- if (e != cudaSuccess) { streams[(size_t) dev] = nullptr; return e; }
+ CopyHandlePool& p = copy_handle_pool();
+ std::lock_guard<std::mutex> lk(p.mu);
+ if ((size_t) dev < p.streams.size() && !p.streams[(size_t) dev].empty()) {
+ streams[(size_t) dev] = p.streams[(size_t) dev].back();
+ p.streams[(size_t) dev].pop_back();
+ } else {
+ e = cudaStreamCreateWithFlags(&streams[(size_t) dev], cudaStreamNonBlocking);
+ if (e != cudaSuccess) { streams[(size_t) dev] = nullptr; return e; }
+ ++p.streams_created;
+ }
}
s = streams[(size_t) dev];
return cudaSuccess;
@@ -118,12 +179,20 @@ inline cudaError_t order_after_queued() {
int dev = 0;
e = cudaGetDevice(&dev);
if (e != cudaSuccess) return e;
- thread_local std::vector<cudaEvent_t> events;
+ std::vector<cudaEvent_t>& events = thread_copy_handles().events;
if (dev < 0) dev = 0;
- if ((size_t) dev >= events.size()) events.resize((size_t) dev + 1, nullptr); // never destroyed: process lifetime
+ if ((size_t) dev >= events.size()) events.resize((size_t) dev + 1, nullptr);
if (!events[(size_t) dev]) {
- e = cudaEventCreateWithFlags(&events[(size_t) dev], cudaEventDisableTiming);
- if (e != cudaSuccess) { events[(size_t) dev] = nullptr; return e; }
+ CopyHandlePool& p = copy_handle_pool();
+ std::lock_guard<std::mutex> lk(p.mu);
+ if ((size_t) dev < p.events.size() && !p.events[(size_t) dev].empty()) {
+ events[(size_t) dev] = p.events[(size_t) dev].back();
+ p.events[(size_t) dev].pop_back();
+ } else {
+ e = cudaEventCreateWithFlags(&events[(size_t) dev], cudaEventDisableTiming);
+ if (e != cudaSuccess) { events[(size_t) dev] = nullptr; return e; }
+ ++p.events_created;
+ }
}
e = cudaEventRecord(events[(size_t) dev], s);
if (e != cudaSuccess) return e;
diff --git a/src/core/conversation_copy_streams_test.cpp b/src/core/conversation_copy_streams_test.cpp
new file mode 100644
index 000000000..6e709c84f
--- /dev/null
+++ b/src/core/conversation_copy_streams_test.cpp
@@ -0,0 +1,86 @@
+// The conversation copy streams and events are pooled across threads (#3350, #3470): short-lived threads (the layer
+// split's later stages run each prompt chunk on a fresh std::async thread) must not leave a stream and an event behind
+// each. Synthetic 4 KiB copies, no model; needs a CUDA device (skips, exit 77, without one).
+#include "conversation_copy.hpp"
+
+#include <atomic>
+#include <cstdio>
+#include <cstdlib>
+#include <thread>
+#include <vector>
+
+using namespace strata::core::conversation_detail;
+
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+ ++checks;
+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+
+// one thread's work on device `dev`: a queued copy, the ordering event, a synchronous copy
+bool one_thread(int dev, void* d, void* h, size_t bytes) {
+ if (cudaSetDevice(dev) != cudaSuccess) return false;
+ if (copy_nonblocking(d, h, bytes, true) != cudaSuccess) return false;
+ if (order_after_queued() != cudaSuccess) return false;
+ if (copy_nonblocking(h, d, bytes, false) != cudaSuccess) return false;
+ return cudaDeviceSynchronize() == cudaSuccess;
+}
+} // namespace
+
+int main() {
+ int n_dev = 0;
+ if (cudaGetDeviceCount(&n_dev) != cudaSuccess || n_dev < 1) {
+ (void) cudaGetLastError();
+ std::printf("conversation_copy_streams_test: no CUDA device, skipped\n");
+ return 77;
+ }
+ constexpr size_t bytes = 4096;
+ void* d = nullptr;
+ check(cudaMalloc(&d, bytes) == cudaSuccess, "device buffer");
+ std::vector<char> h(bytes, 7);
+
+ // 1. 200 threads, one after the other (a chunk per std::async thread): the handles are reused, not created per thread
+ const long s0 = copy_streams_created(), e0 = copy_events_created();
+ for (int i = 0; i < 200; ++i) {
+ bool ok = false;
+ std::thread t([&] { ok = one_thread(0, d, h.data(), bytes); });
+ t.join();
+ check(ok, "a short-lived thread's copies succeed");
+ }
+ check(copy_streams_created() - s0 <= 1, "200 sequential threads create at most one stream");
+ check(copy_events_created() - e0 <= 1, "200 sequential threads create at most one event");
+
+ // 2. threads at the same time each have their own (the number is bounded by the concurrency)
+ const long s1 = copy_streams_created();
+ std::atomic<int> fails{0};
+ for (int round = 0; round < 10; ++round) {
+ std::vector<std::thread> ts;
+ for (int k = 0; k < 6; ++k)
+ ts.emplace_back([&] { for (int j = 0; j < 5; ++j) if (!one_thread(0, d, h.data(), bytes)) ++fails; });
+ for (auto& t : ts) t.join();
+ }
+ check(fails.load() == 0, "concurrent threads' copies succeed");
+ check(copy_streams_created() - s1 <= 6, "ten rounds of 6 concurrent threads create at most 6 more streams");
+
+ // 3. a second device has its own
+ if (n_dev > 1) {
+ const long s2 = copy_streams_created();
+ for (int i = 0; i < 50; ++i) {
+ bool ok = false;
+ std::thread t([&] {
+ ok = cudaSetDevice(1) == cudaSuccess;
+ void* d1 = nullptr;
+ ok = ok && cudaMalloc(&d1, bytes) == cudaSuccess && one_thread(1, d1, h.data(), bytes);
+ if (d1) cudaFree(d1);
+ });
+ t.join();
+ check(ok, "device 1: a short-lived thread's copies succeed");
+ }
+ check(copy_streams_created() - s2 <= 1, "device 1: 50 sequential threads create at most one stream");
+ }
+ cudaFree(d);
+ std::printf("conversation_copy_streams_test: %d checks passed (%ld streams, %ld events created in all)\n", checks,
+ copy_streams_created(), copy_events_created());
+ return 0;
+}