Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: slot_load() writes each tensor-parallel member's window ring and compressor group under its own device (#4350)

Niko122135c0c9a8ae19
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 57dfd860b..d1364af8e 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -5792,6 +5792,7 @@ struct Model::Impl {
wrote = true;
for (int m = 0; m < l.nm; ++m) {
Layer& lm = member(l, m);
+ cudaSetDevice(devs[(size_t) lm.dev].id); // (a tensor-parallel group's members sit on different GPUs: #4350)
if (cudaMemcpy(lm.win, data + off, (size_t) wb, cudaMemcpyHostToDevice) != cudaSuccess) {
err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());
return false;
@@ -5812,6 +5813,7 @@ struct Model::Impl {
}
for (int m = 0; m < l.nm; ++m) {
Layer& lm = member(l, m);
+ cudaSetDevice(devs[(size_t) lm.dev].id);
if (cudaMemcpy(lm.c_kv, data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess ||
cudaMemcpy(lm.c_score, data + off + cb, cb, cudaMemcpyHostToDevice) != cudaSuccess) {
err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());