Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: slot_load() writes each tensor-parallel member's window ring and compressor group under its own device (#4350)
Niko122135c0c9a8ae19
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 57dfd860b..d1364af8e 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -5792,6 +5792,7 @@ struct Model::Impl {wrote = true;for (int m = 0; m < l.nm; ++m) {Layer& lm = member(l, m);+ cudaSetDevice(devs[(size_t) lm.dev].id); // (a tensor-parallel group's members sit on different GPUs: #4350)if (cudaMemcpy(lm.win, data + off, (size_t) wb, cudaMemcpyHostToDevice) != cudaSuccess) {err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());return false;@@ -5812,6 +5813,7 @@ struct Model::Impl {}for (int m = 0; m < l.nm; ++m) {Layer& lm = member(l, m);+ cudaSetDevice(devs[(size_t) lm.dev].id);if (cudaMemcpy(lm.c_kv, data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess ||cudaMemcpy(lm.c_score, data + off + cb, cb, cudaMemcpyHostToDevice) != cudaSuccess) {err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());