Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 Windows: queued GPU work submitted at once on NVIDIA too (win_submit), after every flag hand-off; DS4_WIN_SUBMIT=0 turns it off

Niko12215ec845f6bcb4
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index b04e9eec9..58df7c19b 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -2123,6 +2123,23 @@ void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaSt
for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;
const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));
xfer_put_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq, cnt);
+ // (another GPU, or the host, waits for this flag: on Windows the put must not sit in the queue - #303: two RTX 3080s
+ // hung at the first hand-off, the receiver spinning on a flag whose writer had never been submitted)
+ win_submit(s);
+}
+void win_submit(cudaStream_t s) {
+#if defined(_WIN32)
+ static const bool on = [] {
+ const char* v = std::getenv("DS4_WIN_SUBMIT");
+ return v == nullptr || std::atoi(v) != 0;
+ }();
+ if (on) {
+ (void) cudaStreamQuery(s);
+ (void) cudaGetLastError();
+ }
+#else
+ (void) s;
+#endif
}
void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s) {
size_t total = 0;
@@ -2133,7 +2150,10 @@ void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s)
void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s) {
if (n > 0) wait_flags_kernel<<<1, 32, 0, s>>>(flags, n, stride, seq);
}
-void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { set_flag_kernel<<<1, 1, 0, s>>>(flag, v); }
+void set_flag(unsigned* flag, unsigned v, cudaStream_t s) {
+ set_flag_kernel<<<1, 1, 0, s>>>(flag, v);
+ win_submit(s); // (the writer on another GPU waits for this acknowledgement before its next put - #384)
+}
void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s) {
const int slot = (int) (seq % (unsigned) a.slots);
if (a.nm > 2 && a.sums != nullptr && a.bflags != nullptr) { // (the reduce-scatter: ar_rs_kernel)
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
index 72488e9b3..9bc16233d 100644
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@ -279,6 +279,9 @@ struct Xfer {
int n = 0;
};
void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s);
+// Windows (WDDM, CUDA and HIP alike) holds launches back until the host waits or queries the stream: the work queued on
+// `s` submitted now (cudaStreamQuery, which does not wait). Elsewhere nothing. DS4_WIN_SUBMIT=0: off (A/B tests)
+void win_submit(cudaStream_t s);
void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s);
/// Waits (one thread) until flags[i * stride] >= seq for i < n (a pipelined chunk's readers have taken a buffer);
/// set_flag: *flag = v after a system fence (a reader's acknowledgement).
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index a9e370441..1eaeccebc 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -3047,16 +3047,10 @@ struct Model::Impl {
}
// ---- the flow (see Chunk)
- // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute
- // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing
- static void kick(cudaStream_t s) {
-#if defined(_WIN32) && defined(STRATA_USE_HIP)
- (void) cudaStreamQuery(s);
- (void) cudaGetLastError();
-#else
- (void) s;
-#endif
- }
+ // Windows (WDDM: HIP's runtime and CUDA's driver alike) submits queued work only when the host waits on the GPU:
+ // the flow's copies and the compute it queued between its waits are submitted at once (win_submit); elsewhere
+ // nothing. (CUDA too since #303: two RTX 3080s hung before the first layer, one GPU spinning on the other's flow)
+ static void kick(cudaStream_t s) { win_submit(s); }
size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)
return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;
}