Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: warm the first BF16 and FP16 hipBLAS products on a thread at start (STRATA_PREFILL_WARM=0 skips), STRATA_COLD_TRACE diagnostic
Niko1221f3be974359fa
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hppindex 6c4260824..931e2479c 100644--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@ -45,6 +45,12 @@ struct PrefillStats {namespace strata::core { class MtpDrafter; }namespace strata::prefill {+/// HIP: one tiny BF16 and one tiny FP16 product on a private handle, so the first prompt does not pay hipBLAS's one-time+/// library loads (about 2.5 s on an R9700); once per device, later calls wait for the first and return. Safe to call from a+/// thread before the expert arena loads. STRATA_PREFILL_WARM=0 makes it a no-op. Nothing on CUDA / SYCL.+void warm_gemm_libraries(int device);++class Prefill {public:/// E-9: the draft layer's K/V for prompt cells [cell0, cell0 + n) from their final residual rows `R_rows`diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 7138b85b3..5e62c37f3 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1279,6 +1279,58 @@ uint64_t moe_set_bytes(size_t T, int64_t n_expert, bool fused) {}}+// The first BF16 and the first FP16 GEMM of a process each pay a one-time library / code-object load in hipBLAS (1.2 s and+// 1.0 s on an R9700, hipBLASLt 1.2.0: STRATA_COLD_TRACE), which the first prompt then carries (a 700-token prompt: 5.9 s,+// 3.5 s of it these). One tiny product of each type, on a handle and stream of its own, moves that cost out of the first+// request; generate.cpp starts it on a thread before the expert arena loads, so it costs no start-up time, and+// Prefill::init waits for it. The outputs go to scratch buffers of their own, so no later result changes.+// STRATA_PREFILL_WARM=0 turns it off.+void warm_gemm_libraries(int device) {+ static const bool off = [] { const char* e = std::getenv("STRATA_PREFILL_WARM"); return e != nullptr && e[0] == '0'; }();+ if (off) return;+#if defined(STRATA_USE_HIP)+ static std::mutex mu;+ static unsigned done_mask = 0; // per device: each card loads its own code objects+ if (device < 0 || device >= 32) return;+ std::lock_guard<std::mutex> lk(mu);+ if (done_mask & (1u << device)) return;+ done_mask |= 1u << device;+ int prev = 0;+ if (cudaGetDevice(&prev) != cudaSuccess) { (void) cudaGetLastError(); return; }+ if (prev != device && cudaSetDevice(device) != cudaSuccess) { (void) cudaGetLastError(); return; }+ const auto t0 = std::chrono::steady_clock::now();+ constexpr int64_t T = 64, Nn = 64, K = 320;+ uint16_t *x = nullptr, *w = nullptr;+ float* y = nullptr;+ cudaStream_t st = nullptr;+ bool ok = cudaStreamCreate(&st) == cudaSuccess && cudaMalloc((void**) &x, (size_t) T * K * 2) == cudaSuccess &&+ cudaMalloc((void**) &w, (size_t) Nn * K * 2) == cudaSuccess && cudaMalloc((void**) &y, (size_t) T * Nn * 4) == cudaSuccess;+ if (ok) {+ Gemm gm;+ std::string err;+ ok = gm.init((void*) st, T * K, err);+ if (ok) {+ cudaMemsetAsync(x, 0, (size_t) T * K * 2, st);+ cudaMemsetAsync(w, 0, (size_t) Nn * K * 2, st);+ gm.bf16(x, w, y, T, Nn, K);+ gm.f16(x, w, y, T, Nn, K);+ cudaStreamSynchronize(st);+ }+ }+ (void) cudaGetLastError();+ if (x) cudaFree(x);+ if (w) cudaFree(w);+ if (y) cudaFree(y);+ if (st) cudaStreamDestroy(st);+ if (prev != device) cudaSetDevice(prev);+ if (ok)+ std::fprintf(stderr, "strata prefill: warmed the BF16 and FP16 GEMM libraries on GPU %d in %.0f ms (STRATA_PREFILL_WARM=0 skips)\n",+ device, std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count());+#else+ (void) device;+#endif+}+bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, core::SessionState& ss,core::ExpertSource* src, const core::ExpertCache* cache, const int32_t* host_res, int64_t chunk,void* stream, std::string& err, void* borrow, uint64_t borrow_bytes, bool borrow_spill) {@@ -1470,6 +1522,7 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, costd::to_string(o.failed_bytes >> 20) + " MiB)";return false;}+ { int wd = 0; if (cudaGetDevice(&wd) == cudaSuccess) warm_gemm_libraries(wd); else (void) cudaGetLastError(); }return true;}@@ -2345,10 +2398,32 @@ const core::WeightRef* need(const core::LayerView& v, const char* suffix, std::sif (!r) err = v.name(suffix) + " is missing";return r;}+// STRATA_COLD_TRACE=1 (diagnostic): the wall time of the first 80 dense projections of the process, each synchronized, to show+// which first calls pay a one-time cost (library / code object loads).+struct ColdTrace {+ bool on; int n = 0; std::chrono::steady_clock::time_point t0;+ ColdTrace() { const char* e = std::getenv("STRATA_COLD_TRACE"); on = e != nullptr && e[0] == '1'; }+ bool begin() {+ if (!on || n >= 80) return false;+ cudaDeviceSynchronize();+ t0 = std::chrono::steady_clock::now();+ return true;+ }+ void end(const char* what, const std::string& name, int64_t T, int64_t N, int64_t K) {+ cudaDeviceSynchronize();+ const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();+ ++n;+ std::fprintf(stderr, "strata cold-trace: #%d %s %s T=%lld N=%lld K=%lld %.2f ms\n", n, what, name.c_str(), (long long) T,+ (long long) N, (long long) K, ms);+ }+};+ColdTrace g_cold;bool native_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y, int64_t T, const std::string& name,std::string& err, int64_t ldy = 0, int64_t ldx = 0) {if (!w->native_data) { err = "prefill: " + name + " has no native GGUF blocks (run with --native)"; return false; }+ const bool ct = g_cold.begin();gm.native(X, w->native_type, w->native_data, Y, T, w->ne1, w->ne0, ldy, 0.0f, ldx);+ if (ct) g_cold.end("native", name, T, w->ne1, w->ne0);return true;}bool bf16_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y, int64_t T, const std::string& name,@@ -2370,7 +2445,9 @@ bool bf16_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y,strata::kernels::hc_q8_to_bf16(w->data, w->q8_scale, scratch, (long long) w->elements, stream);wb = scratch;}+ const bool ct = g_cold.begin();gm.bf16(X, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx, x_f16);+ if (ct) g_cold.end("bf16", name, T, w->ne1, w->ne0);if (X_lo) gm.bf16(X_lo, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 1.0f);return true;}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 3035bf166..531b85725 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -2115,6 +2115,10 @@ strata::kernels::cpu::PoolAffinity measure_pool_affinity(strata::core::ExpertSoureturn take_p ? cpu::PoolAffinity::PCores : cpu::PoolAffinity::All;}+// HIP: the thread that warms hipBLAS's libraries while the expert arena loads (see warm_gemm_libraries); joined at exit.+std::thread g_warm_thr;+struct WarmJoin { ~WarmJoin() { if (g_warm_thr.joinable()) g_warm_thr.join(); } } g_warm_join;+// 0.1.42, CUDA single-GPU serve, no --pcie-frac: the share refined from what the CPU pool measures in the first decode// windows. The balance share of the missed experts is link / (link + CPU rate). The link is the start-up probe; the CPU// rate is the expert bytes over the pool's wall time per missed expert (ExpertDispatch::ms_run / multi_misses), which the@@ -5354,6 +5358,8 @@ int main(int argc, char** argv) {const char* name = named && p.name[0] ? p.name : "(an unnamed GPU)";#if defined(STRATA_USE_HIP)std::fprintf(stderr, "strata generate: GPU %d: %s (%s)\n", dev, name, named ? p.gcnArchName : "?");+ // the first prompt's one-time hipBLAS library loads (about 2.5 s), done on a thread while the expert arena loads+ if (named) g_warm_thr = std::thread([dev] { strata::prefill::warm_gemm_libraries(dev); });// A hint only, never an action: on RDNA4 with an int8 KV the opt-in matrix-core prompt attention reads prompts// about 30% faster (R9700, PR #1318), but its output differs from the default kernel's in the last bits, so it// stays the user's choice. Silent when the variable is set to anything (STRATA_HIP_WMMA=0 included).