Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
conversation copies: the private copy streams and events are handed back to a pool when a thread ends (#3350, #3470)
Niko12213478382eeecd
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 1cf372d97..6356e27b6 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1486,6 +1486,13 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(pin_budget_test PRIVATE strata_core ${_strata_gpu_runtime_target})add_test(NAME pin_budget_test COMMAND pin_budget_test)+ # #3350 / #3470: the conversation copy streams and events are pooled across short-lived threads (needs a device, else skips)+ add_executable(conversation_copy_streams_test src/core/conversation_copy_streams_test.cpp)+ target_include_directories(conversation_copy_streams_test PRIVATE include)+ target_link_libraries(conversation_copy_streams_test PRIVATE strata_core ${_strata_gpu_runtime_target})+ add_test(NAME conversation_copy_streams_test COMMAND conversation_copy_streams_test)+ set_tests_properties(conversation_copy_streams_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")add_executable(pinned_shared_test src/core/pinned_shared_test.cpp)target_link_libraries(pinned_shared_test PRIVATE strata_core ${_strata_gpu_runtime_target})diff --git a/src/core/conversation_copy.hpp b/src/core/conversation_copy.hppindex 2a9c4c029..7ffb87f4a 100644--- a/src/core/conversation_copy.hpp+++ b/src/core/conversation_copy.hpp@@ -10,27 +10,88 @@#include <chrono>#include <cstddef>#include <cstdlib>+#include <mutex>#include <thread>#include <vector>namespace strata::core::conversation_detail {+/// The private copy streams and ordering events (below) belong to a thread, but a thread can be short-lived: the layer+/// split's later stages run each prompt chunk on a fresh std::async thread, and a checkpoint part is saved on it. The+/// handles used to be created per thread and never destroyed, so every such thread left one stream and one event per+/// device behind - a few MiB of VRAM per long cold prompt on the later cards, until a stream could not be created any+/// more and the copy failed with "out of memory" (#3350, #3470). Now a thread hands its handles back to a process-wide+/// pool when it ends and the next thread takes them: the number of handles is the number of threads copying at once.+/// The pool is leaked on purpose (a thread may end after the statics are gone).+struct CopyHandlePool {+ std::mutex mu;+ std::vector<std::vector<cudaStream_t>> streams; // free ones, by device+ std::vector<std::vector<cudaEvent_t>> events;+ long streams_created = 0, events_created = 0;+};+inline CopyHandlePool& copy_handle_pool() {+ static CopyHandlePool* p = new CopyHandlePool;+ return *p;+}+/// How many streams / events the process has created for these copies (tests: it stays at the peak number of threads+/// copying at once, not at the number of threads that ever copied).+inline long copy_streams_created() {+ CopyHandlePool& p = copy_handle_pool();+ std::lock_guard<std::mutex> lk(p.mu);+ return p.streams_created;+}+inline long copy_events_created() {+ CopyHandlePool& p = copy_handle_pool();+ std::lock_guard<std::mutex> lk(p.mu);+ return p.events_created;+}+struct ThreadCopyHandles {+ std::vector<cudaStream_t> streams; // by device, nullptr = none yet+ std::vector<cudaEvent_t> events;+ ThreadCopyHandles() = default;+ ThreadCopyHandles(const ThreadCopyHandles&) = delete;+ ThreadCopyHandles& operator=(const ThreadCopyHandles&) = delete;+ ~ThreadCopyHandles() {+ CopyHandlePool& p = copy_handle_pool();+ std::lock_guard<std::mutex> lk(p.mu);+ if (p.streams.size() < streams.size()) p.streams.resize(streams.size());+ if (p.events.size() < events.size()) p.events.resize(events.size());+ for (size_t d = 0; d < streams.size(); ++d)+ if (streams[d]) p.streams[d].push_back(streams[d]);+ for (size_t d = 0; d < events.size(); ++d)+ if (events[d]) p.events[d].push_back(events[d]);+ }+};+inline ThreadCopyHandles& thread_copy_handles() {+ thread_local ThreadCopyHandles t;+ return t;+}+/// Conversation snapshot / checkpoint copies go through a private non-blocking stream of the current device/// instead of the legacy (null) stream. A plain cudaMemcpy is ordered against every blocking stream of the/// device, and while another thread captures a graph on one (a drafter or a stage worker), HIP refuses it with/// "operation would make the legacy stream depend on a capturing blocking stream". The caller's contract is/// unchanged: the device was synchronized before the copy (the copy itself is complete on return).-/// private_stream: this thread's stream on the current device, created on first use.+/// private_stream: this thread's stream on the current device: one a finished thread handed back, else a new one on+/// first use.inline cudaError_t private_stream(cudaStream_t& s) {int dev = 0;cudaError_t e = cudaGetDevice(&dev);if (e != cudaSuccess) return e;- thread_local std::vector<cudaStream_t> streams;+ std::vector<cudaStream_t>& streams = thread_copy_handles().streams;if (dev < 0) dev = 0;- if ((size_t) dev >= streams.size()) streams.resize((size_t) dev + 1, nullptr); // never destroyed: process lifetime+ if ((size_t) dev >= streams.size()) streams.resize((size_t) dev + 1, nullptr);if (!streams[(size_t) dev]) {- e = cudaStreamCreateWithFlags(&streams[(size_t) dev], cudaStreamNonBlocking);- if (e != cudaSuccess) { streams[(size_t) dev] = nullptr; return e; }+ CopyHandlePool& p = copy_handle_pool();+ std::lock_guard<std::mutex> lk(p.mu);+ if ((size_t) dev < p.streams.size() && !p.streams[(size_t) dev].empty()) {+ streams[(size_t) dev] = p.streams[(size_t) dev].back();+ p.streams[(size_t) dev].pop_back();+ } else {+ e = cudaStreamCreateWithFlags(&streams[(size_t) dev], cudaStreamNonBlocking);+ if (e != cudaSuccess) { streams[(size_t) dev] = nullptr; return e; }+ ++p.streams_created;+ }}s = streams[(size_t) dev];return cudaSuccess;@@ -118,12 +179,20 @@ inline cudaError_t order_after_queued() {int dev = 0;e = cudaGetDevice(&dev);if (e != cudaSuccess) return e;- thread_local std::vector<cudaEvent_t> events;+ std::vector<cudaEvent_t>& events = thread_copy_handles().events;if (dev < 0) dev = 0;- if ((size_t) dev >= events.size()) events.resize((size_t) dev + 1, nullptr); // never destroyed: process lifetime+ if ((size_t) dev >= events.size()) events.resize((size_t) dev + 1, nullptr);if (!events[(size_t) dev]) {- e = cudaEventCreateWithFlags(&events[(size_t) dev], cudaEventDisableTiming);- if (e != cudaSuccess) { events[(size_t) dev] = nullptr; return e; }+ CopyHandlePool& p = copy_handle_pool();+ std::lock_guard<std::mutex> lk(p.mu);+ if ((size_t) dev < p.events.size() && !p.events[(size_t) dev].empty()) {+ events[(size_t) dev] = p.events[(size_t) dev].back();+ p.events[(size_t) dev].pop_back();+ } else {+ e = cudaEventCreateWithFlags(&events[(size_t) dev], cudaEventDisableTiming);+ if (e != cudaSuccess) { events[(size_t) dev] = nullptr; return e; }+ ++p.events_created;+ }}e = cudaEventRecord(events[(size_t) dev], s);if (e != cudaSuccess) return e;diff --git a/src/core/conversation_copy_streams_test.cpp b/src/core/conversation_copy_streams_test.cppnew file mode 100644index 000000000..6e709c84f--- /dev/null+++ b/src/core/conversation_copy_streams_test.cpp@@ -0,0 +1,86 @@+// The conversation copy streams and events are pooled across threads (#3350, #3470): short-lived threads (the layer+// split's later stages run each prompt chunk on a fresh std::async thread) must not leave a stream and an event behind+// each. Synthetic 4 KiB copies, no model; needs a CUDA device (skips, exit 77, without one).+#include "conversation_copy.hpp"++#include <atomic>+#include <cstdio>+#include <cstdlib>+#include <thread>+#include <vector>++using namespace strata::core::conversation_detail;++namespace {+int checks = 0;+void check(bool ok, const char* label) {+ ++checks;+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }+}++// one thread's work on device `dev`: a queued copy, the ordering event, a synchronous copy+bool one_thread(int dev, void* d, void* h, size_t bytes) {+ if (cudaSetDevice(dev) != cudaSuccess) return false;+ if (copy_nonblocking(d, h, bytes, true) != cudaSuccess) return false;+ if (order_after_queued() != cudaSuccess) return false;+ if (copy_nonblocking(h, d, bytes, false) != cudaSuccess) return false;+ return cudaDeviceSynchronize() == cudaSuccess;+}+} // namespace++int main() {+ int n_dev = 0;+ if (cudaGetDeviceCount(&n_dev) != cudaSuccess || n_dev < 1) {+ (void) cudaGetLastError();+ std::printf("conversation_copy_streams_test: no CUDA device, skipped\n");+ return 77;+ }+ constexpr size_t bytes = 4096;+ void* d = nullptr;+ check(cudaMalloc(&d, bytes) == cudaSuccess, "device buffer");+ std::vector<char> h(bytes, 7);++ // 1. 200 threads, one after the other (a chunk per std::async thread): the handles are reused, not created per thread+ const long s0 = copy_streams_created(), e0 = copy_events_created();+ for (int i = 0; i < 200; ++i) {+ bool ok = false;+ std::thread t([&] { ok = one_thread(0, d, h.data(), bytes); });+ t.join();+ check(ok, "a short-lived thread's copies succeed");+ }+ check(copy_streams_created() - s0 <= 1, "200 sequential threads create at most one stream");+ check(copy_events_created() - e0 <= 1, "200 sequential threads create at most one event");++ // 2. threads at the same time each have their own (the number is bounded by the concurrency)+ const long s1 = copy_streams_created();+ std::atomic<int> fails{0};+ for (int round = 0; round < 10; ++round) {+ std::vector<std::thread> ts;+ for (int k = 0; k < 6; ++k)+ ts.emplace_back([&] { for (int j = 0; j < 5; ++j) if (!one_thread(0, d, h.data(), bytes)) ++fails; });+ for (auto& t : ts) t.join();+ }+ check(fails.load() == 0, "concurrent threads' copies succeed");+ check(copy_streams_created() - s1 <= 6, "ten rounds of 6 concurrent threads create at most 6 more streams");++ // 3. a second device has its own+ if (n_dev > 1) {+ const long s2 = copy_streams_created();+ for (int i = 0; i < 50; ++i) {+ bool ok = false;+ std::thread t([&] {+ ok = cudaSetDevice(1) == cudaSuccess;+ void* d1 = nullptr;+ ok = ok && cudaMalloc(&d1, bytes) == cudaSuccess && one_thread(1, d1, h.data(), bytes);+ if (d1) cudaFree(d1);+ });+ t.join();+ check(ok, "device 1: a short-lived thread's copies succeed");+ }+ check(copy_streams_created() - s2 <= 1, "device 1: 50 sequential threads create at most one stream");+ }+ cudaFree(d);+ std::printf("conversation_copy_streams_test: %d checks passed (%ld streams, %ld events created in all)\n", checks,+ copy_streams_created(), copy_events_created());+ return 0;+}