Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

HIP prefill: no start-up hipBLAS warm-up when the cards are of more than one architecture (#3204)

Niko12214e76d6c9a6b7
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7ba7ee634..9567822da 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2095,6 +2095,10 @@ if(STRATA_BUILD_TESTS)
add_executable(pool_plan_test src/kernels/cpu/pool_plan_test.cpp)
target_include_directories(pool_plan_test PRIVATE include)
add_test(NAME pool_plan_test COMMAND pool_plan_test)
+ # #3204: when the HIP start-up warm-up of hipBLAS may run (pure policy, no GPU)
+ add_executable(warm_policy_test src/prefill/warm_policy_test.cpp)
+ target_include_directories(warm_policy_test PRIVATE include)
+ add_test(NAME warm_policy_test COMMAND warm_policy_test)
# issue #29: tiny batches with sleeping workers, as on a big-VRAM card; a hang fails it
add_executable(pool_stress src/kernels/cpu/pool_stress.cpp)
target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)
diff --git a/include/strata/prefill/warm_policy.hpp b/include/strata/prefill/warm_policy.hpp
new file mode 100644
index 000000000..258c65190
--- /dev/null
+++ b/include/strata/prefill/warm_policy.hpp
@@ -0,0 +1,30 @@
+// include/strata/prefill/warm_policy.hpp - whether the HIP start-up warm-up of hipBLAS (prefill.cpp warm_gemm_libraries)
+// may run. Pure, so it is unit tested without a GPU (warm_policy_test).
+//
+// #3204: on a layer split across a gfx1151 and a gfx1201 card, prompts read ~25% slower with the warm-up (1,250 against
+// 1,710 tok/s at 64K; STRATA_PREFILL_WARM=0 restores it; hc read of the gfx1151 stage 2.7 -> 9.0 s a 32K block). The
+// same warm-up on two cards of ONE architecture (2x R9700) costs nothing (021-iD, interleaved pairs). Cards of different
+// architectures share one process-wide hipBLAS / hipBLASLt library state, and the warm-up of the second architecture runs
+// before the first card's own tuned products resolve their solutions. So: no warm-up when the process sees more than one
+// architecture (the first prompt pays the one-time 1-2.5 s load, as before 0.2.0), unless STRATA_PREFILL_WARM=1 asks.
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace strata::prefill {
+
+/// `gcn_archs`: gcnArchName of every visible HIP device (a ":sramecc+:xnack-" feature suffix is ignored);
+/// `env`: the value of STRATA_PREFILL_WARM or nullptr. "0" turns the warm-up off, "1" on whatever the cards are.
+inline bool warm_gemm_wanted(const std::vector<std::string>& gcn_archs, const char* env) {
+ if (env != nullptr && env[0] == '0') return false;
+ if (env != nullptr && env[0] == '1') return true;
+ if (gcn_archs.empty()) return true;
+ const auto base = [](const std::string& a) { return a.substr(0, a.find(':')); };
+ const std::string first = base(gcn_archs.front());
+ for (const std::string& a : gcn_archs)
+ if (base(a) != first) return false;
+ return true;
+}
+
+} // namespace strata::prefill
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index f00f1cc13..14b23bccb 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -1,6 +1,7 @@
// src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.
#include "strata/kernels/fused_gr.hpp"
#include "strata/prefill/prefill.hpp"
+#include "strata/prefill/warm_policy.hpp"
#include <atomic>
#if defined(__cpp_lib_atomic_wait)
#define STRATA_ATOMIC_WAIT 1 // #1488: std::atomic::wait / notify_all are a GCC 11 library feature; GCC 10 spins instead
@@ -1286,6 +1287,24 @@ void warm_gemm_libraries(int device) {
static const bool off = [] { const char* e = std::getenv("STRATA_PREFILL_WARM"); return e != nullptr && e[0] == '0'; }();
if (off) return;
#if defined(STRATA_USE_HIP)
+ // #3204: not when the process sees cards of more than one architecture (gfx1151 + gfx1201: prompts -25%); see
+ // warm_policy.hpp. STRATA_PREFILL_WARM=1 asks for it anyway.
+ static const bool arch_ok = [] {
+ std::vector<std::string> archs;
+ int n = 0;
+ if (cudaGetDeviceCount(&n) != cudaSuccess) { (void) cudaGetLastError(); return true; }
+ for (int i = 0; i < n && i < 32; ++i) {
+ cudaDeviceProp p{};
+ if (cudaGetDeviceProperties(&p, i) == cudaSuccess) archs.emplace_back(p.gcnArchName);
+ else (void) cudaGetLastError();
+ }
+ const bool wanted = warm_gemm_wanted(archs, std::getenv("STRATA_PREFILL_WARM"));
+ if (!wanted)
+ std::fprintf(stderr, "strata prefill: no hipBLAS warm-up: the cards are of more than one architecture "
+ "(it slows prompts there, #3204); the first prompt loads the libraries (STRATA_PREFILL_WARM=1 asks for it)\n");
+ return wanted;
+ }();
+ if (!arch_ok) return;
static std::mutex mu;
static unsigned done_mask = 0; // per device: each card loads its own code objects
if (device < 0 || device >= 32) return;
diff --git a/src/prefill/warm_policy_test.cpp b/src/prefill/warm_policy_test.cpp
new file mode 100644
index 000000000..5ddfc869c
--- /dev/null
+++ b/src/prefill/warm_policy_test.cpp
@@ -0,0 +1,31 @@
+// warm_policy_test: when the HIP start-up warm-up of hipBLAS may run (include/strata/prefill/warm_policy.hpp). Pure.
+#include "strata/prefill/warm_policy.hpp"
+
+#include <cstdio>
+
+using strata::prefill::warm_gemm_wanted;
+
+static int g_fail = 0;
+static void expect(bool ok, const char* what) {
+ if (!ok) {
+ ++g_fail;
+ std::fprintf(stderr, "FAIL: %s\n", what);
+ }
+}
+
+int main() {
+ expect(warm_gemm_wanted({}, nullptr), "no device list: the warm-up runs");
+ expect(warm_gemm_wanted({"gfx1201"}, nullptr), "one R9700: on");
+ expect(warm_gemm_wanted({"gfx1201", "gfx1201"}, nullptr), "2x R9700 (same architecture): on (021-iD: no loss)");
+ expect(warm_gemm_wanted({"gfx1201:sramecc+:xnack-", "gfx1201"}, nullptr), "feature suffixes do not make a mixed pair");
+ expect(!warm_gemm_wanted({"gfx1151", "gfx1201"}, nullptr), "8060S + R9700 (#3204): off");
+ expect(!warm_gemm_wanted({"gfx1201", "gfx1151"}, nullptr), "the order does not matter");
+ expect(!warm_gemm_wanted({"gfx1100", "gfx1100", "gfx1030"}, nullptr), "any second architecture turns it off");
+ expect(!warm_gemm_wanted({"gfx1201"}, "0"), "STRATA_PREFILL_WARM=0: off");
+ expect(!warm_gemm_wanted({"gfx1201", "gfx1201"}, "0"), "STRATA_PREFILL_WARM=0 on two cards: off");
+ expect(warm_gemm_wanted({"gfx1151", "gfx1201"}, "1"), "STRATA_PREFILL_WARM=1 forces it on a mixed pair");
+ expect(!warm_gemm_wanted({"gfx1151", "gfx1201"}, ""), "an empty value is unset: a mixed pair stays off");
+ if (g_fail) { std::fprintf(stderr, "warm_policy_test: %d failure(s)\n", g_fail); return 1; }
+ std::printf("warm_policy_test: OK\n");
+ return 0;
+}