Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
HIP prefill: no start-up hipBLAS warm-up when the cards are of more than one architecture (#3204)
Niko12214e76d6c9a6b7
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 7ba7ee634..9567822da 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -2095,6 +2095,10 @@ if(STRATA_BUILD_TESTS)add_executable(pool_plan_test src/kernels/cpu/pool_plan_test.cpp)target_include_directories(pool_plan_test PRIVATE include)add_test(NAME pool_plan_test COMMAND pool_plan_test)+ # #3204: when the HIP start-up warm-up of hipBLAS may run (pure policy, no GPU)+ add_executable(warm_policy_test src/prefill/warm_policy_test.cpp)+ target_include_directories(warm_policy_test PRIVATE include)+ add_test(NAME warm_policy_test COMMAND warm_policy_test)# issue #29: tiny batches with sleeping workers, as on a big-VRAM card; a hang fails itadd_executable(pool_stress src/kernels/cpu/pool_stress.cpp)target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)diff --git a/include/strata/prefill/warm_policy.hpp b/include/strata/prefill/warm_policy.hppnew file mode 100644index 000000000..258c65190--- /dev/null+++ b/include/strata/prefill/warm_policy.hpp@@ -0,0 +1,30 @@+// include/strata/prefill/warm_policy.hpp - whether the HIP start-up warm-up of hipBLAS (prefill.cpp warm_gemm_libraries)+// may run. Pure, so it is unit tested without a GPU (warm_policy_test).+//+// #3204: on a layer split across a gfx1151 and a gfx1201 card, prompts read ~25% slower with the warm-up (1,250 against+// 1,710 tok/s at 64K; STRATA_PREFILL_WARM=0 restores it; hc read of the gfx1151 stage 2.7 -> 9.0 s a 32K block). The+// same warm-up on two cards of ONE architecture (2x R9700) costs nothing (021-iD, interleaved pairs). Cards of different+// architectures share one process-wide hipBLAS / hipBLASLt library state, and the warm-up of the second architecture runs+// before the first card's own tuned products resolve their solutions. So: no warm-up when the process sees more than one+// architecture (the first prompt pays the one-time 1-2.5 s load, as before 0.2.0), unless STRATA_PREFILL_WARM=1 asks.+#pragma once++#include <string>+#include <vector>++namespace strata::prefill {++/// `gcn_archs`: gcnArchName of every visible HIP device (a ":sramecc+:xnack-" feature suffix is ignored);+/// `env`: the value of STRATA_PREFILL_WARM or nullptr. "0" turns the warm-up off, "1" on whatever the cards are.+inline bool warm_gemm_wanted(const std::vector<std::string>& gcn_archs, const char* env) {+ if (env != nullptr && env[0] == '0') return false;+ if (env != nullptr && env[0] == '1') return true;+ if (gcn_archs.empty()) return true;+ const auto base = [](const std::string& a) { return a.substr(0, a.find(':')); };+ const std::string first = base(gcn_archs.front());+ for (const std::string& a : gcn_archs)+ if (base(a) != first) return false;+ return true;+}++} // namespace strata::prefilldiff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex f00f1cc13..14b23bccb 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1,6 +1,7 @@// src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.#include "strata/kernels/fused_gr.hpp"#include "strata/prefill/prefill.hpp"+#include "strata/prefill/warm_policy.hpp"#include <atomic>#if defined(__cpp_lib_atomic_wait)#define STRATA_ATOMIC_WAIT 1 // #1488: std::atomic::wait / notify_all are a GCC 11 library feature; GCC 10 spins instead@@ -1286,6 +1287,24 @@ void warm_gemm_libraries(int device) {static const bool off = [] { const char* e = std::getenv("STRATA_PREFILL_WARM"); return e != nullptr && e[0] == '0'; }();if (off) return;#if defined(STRATA_USE_HIP)+ // #3204: not when the process sees cards of more than one architecture (gfx1151 + gfx1201: prompts -25%); see+ // warm_policy.hpp. STRATA_PREFILL_WARM=1 asks for it anyway.+ static const bool arch_ok = [] {+ std::vector<std::string> archs;+ int n = 0;+ if (cudaGetDeviceCount(&n) != cudaSuccess) { (void) cudaGetLastError(); return true; }+ for (int i = 0; i < n && i < 32; ++i) {+ cudaDeviceProp p{};+ if (cudaGetDeviceProperties(&p, i) == cudaSuccess) archs.emplace_back(p.gcnArchName);+ else (void) cudaGetLastError();+ }+ const bool wanted = warm_gemm_wanted(archs, std::getenv("STRATA_PREFILL_WARM"));+ if (!wanted)+ std::fprintf(stderr, "strata prefill: no hipBLAS warm-up: the cards are of more than one architecture "+ "(it slows prompts there, #3204); the first prompt loads the libraries (STRATA_PREFILL_WARM=1 asks for it)\n");+ return wanted;+ }();+ if (!arch_ok) return;static std::mutex mu;static unsigned done_mask = 0; // per device: each card loads its own code objectsif (device < 0 || device >= 32) return;diff --git a/src/prefill/warm_policy_test.cpp b/src/prefill/warm_policy_test.cppnew file mode 100644index 000000000..5ddfc869c--- /dev/null+++ b/src/prefill/warm_policy_test.cpp@@ -0,0 +1,31 @@+// warm_policy_test: when the HIP start-up warm-up of hipBLAS may run (include/strata/prefill/warm_policy.hpp). Pure.+#include "strata/prefill/warm_policy.hpp"++#include <cstdio>++using strata::prefill::warm_gemm_wanted;++static int g_fail = 0;+static void expect(bool ok, const char* what) {+ if (!ok) {+ ++g_fail;+ std::fprintf(stderr, "FAIL: %s\n", what);+ }+}++int main() {+ expect(warm_gemm_wanted({}, nullptr), "no device list: the warm-up runs");+ expect(warm_gemm_wanted({"gfx1201"}, nullptr), "one R9700: on");+ expect(warm_gemm_wanted({"gfx1201", "gfx1201"}, nullptr), "2x R9700 (same architecture): on (021-iD: no loss)");+ expect(warm_gemm_wanted({"gfx1201:sramecc+:xnack-", "gfx1201"}, nullptr), "feature suffixes do not make a mixed pair");+ expect(!warm_gemm_wanted({"gfx1151", "gfx1201"}, nullptr), "8060S + R9700 (#3204): off");+ expect(!warm_gemm_wanted({"gfx1201", "gfx1151"}, nullptr), "the order does not matter");+ expect(!warm_gemm_wanted({"gfx1100", "gfx1100", "gfx1030"}, nullptr), "any second architecture turns it off");+ expect(!warm_gemm_wanted({"gfx1201"}, "0"), "STRATA_PREFILL_WARM=0: off");+ expect(!warm_gemm_wanted({"gfx1201", "gfx1201"}, "0"), "STRATA_PREFILL_WARM=0 on two cards: off");+ expect(warm_gemm_wanted({"gfx1151", "gfx1201"}, "1"), "STRATA_PREFILL_WARM=1 forces it on a mixed pair");+ expect(!warm_gemm_wanted({"gfx1151", "gfx1201"}, ""), "an empty value is unset: a mixed pair stays off");+ if (g_fail) { std::fprintf(stderr, "warm_policy_test: %d failure(s)\n", g_fail); return 1; }+ std::printf("warm_policy_test: OK\n");+ return 0;+}