Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: DS4_PROF events per device (invalid resource handle on several GPUs), exl3_legacy.cu in CUDA builds (reported by the CUDA lab)

Niko12212189884a59b3
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 786cf9d51..5ea05b40b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -770,6 +770,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
if(STRATA_BUILD_DSV41 AND NOT STRATA_HIP_GFX906)
add_library(strata_dsv41_kernels STATIC src/dsv41/vendor/exl3.cu src/dsv41/vendor/sampler.cu src/dsv41/vendor/moe_win.cu
src/dsv41/vendor/exl3_cpu.cpp)
+ if(NOT STRATA_ENABLE_HIP) # (the pre-sm_80 NVIDIA decode kernels exl3.cu dispatches to)
+ target_sources(strata_dsv41_kernels PRIVATE src/dsv41/vendor/exl3_legacy.cu)
+ endif()
target_include_directories(strata_dsv41_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})
add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 4ed6ef3aa..369b51387 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1078,7 +1078,7 @@ struct Model::Impl {
};
std::mutex prof_mu;
std::vector<ProfMark> prof_marks;
- std::vector<cudaEvent_t> prof_pool;
+ std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on
bool prof_window = false; // (decode windows only: T == 1)
// ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared
// expert, after its experts (the streamed ones' waits for their reads in it); printed after the run
@@ -1087,13 +1087,18 @@ struct Model::Impl {
void prof_mark(const Dev& d, int stage) {
if (!prof_window && !prof_chunk) return;
std::lock_guard<std::mutex> lk(prof_mu);
+ int cur = 0;
+ cudaGetDevice(&cur);
+ cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")
cudaEvent_t e = nullptr;
- if (prof_pool.empty()) cudaEventCreate(&e);
+ std::vector<cudaEvent_t>& pool = prof_pool[d.id];
+ if (pool.empty()) cudaEventCreate(&e);
else {
- e = prof_pool.back();
- prof_pool.pop_back();
+ e = pool.back();
+ pool.pop_back();
}
cudaEventRecord(e, d.st);
+ cudaSetDevice(cur);
prof_marks.push_back({&d, e, stage});
}
void prof_collect() {
@@ -1104,6 +1109,7 @@ struct Model::Impl {
auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });
if (it != last.end()) {
float ms = 0.0f;
+ cudaSetDevice(m.d->id);
if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)
(prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;
it->second = m.ev;
@@ -1111,7 +1117,7 @@ struct Model::Impl {
last.push_back({m.d, m.ev});
}
}
- for (const ProfMark& m : prof_marks) prof_pool.push_back(m.ev);
+ for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);
prof_marks.clear();
if (!prof_chunk) ++st.prof_tokens;
}
@@ -1730,7 +1736,7 @@ struct Model::Impl {
const int ty = gg_type_of(kv.second.dtype);
if (ty >= 0) blk = std::max(blk, gg::type_block(ty));
}
- while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % 128 != 0 || g.heads % tp != 0 ||
+ while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||
g.o_groups % tp != 0))
--tp;
if (tp != o.tp)