Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine tests: the gfx11 / gfx12 matrix-core prompt tiles are forced on and off in ds4_iq_test (launch counted, both against the CPU reference and each other); umaprobe m1/ver probes; test knobs DS4_ZC_COARSE and DS4_STORE_CHUNK_SLOTS for the zero-copy registration
Niko12214d8ff9646f82
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex b116133d8..e5f668876 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -1425,6 +1425,23 @@ bool light_down(const SetArgs& a, cudaStream_t s) {return light_set(a, false, s);}+bool hipk_force(int mode) {+#if defined(STRATA_USE_HIP)+ hipk::force_mode().store(mode, std::memory_order_relaxed);+ return mode < 0 || hipk::ok_device() || mode == 0;+#else+ (void) mode;+ return false;+#endif+}+long long hipk_launches() {+#if defined(STRATA_USE_HIP)+ return hipk::launches().load(std::memory_order_relaxed);+#else+ return -1;+#endif+}+bool set_gate_up(const SetArgs& a, cudaStream_t s) {if (is_tq(a.tg)) return tq::set_gate_up(a, s);if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 5fc415499..c2b74ff11 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -95,6 +95,10 @@ struct SetArgs {};bool set_gate_up(const SetArgs& a, cudaStream_t s);bool set_down(const SetArgs& a, cudaStream_t s);+/// Test hooks for the matrix-core (HIP gfx11 / gfx12) prompt tiles: hipk_force(-1 default rules, 0 scalar tiles, 1 matrix-core tiles) and the+/// count of matrix-core launches so far (both do nothing / return -1 where there are no such tiles; hipk_force returns whether this device has them)+bool hipk_force(int mode);+long long hipk_launches();/// The light form alone (the set's experts of at most 16 rows; the others untouched), for any of these types and MXFP4/// (tg / td 39: ds4_gguf.cu's experts): false (nothing launched) when it is switched off - the caller's tiles take every/// expert then; true: they take those of more rows (skip_le 16).diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.incindex 4fd9c3f40..4e62dffd8 100644--- a/src/dsv41/ds4_iq_hip.inc+++ b/src/dsv41/ds4_iq_hip.inc@@ -10,6 +10,9 @@#if defined(STRATA_USE_HIP)namespace hipk {+// test hooks: force_mode -1 = the default rules, 0 = never, 1 = on wherever the device has the WMMA tiles (gfx11 / gfx12); launches counts the tiles taken+inline std::atomic<int>& force_mode() { static std::atomic<int> f{-1}; return f; }+inline std::atomic<long long>& launches() { static std::atomic<long long> n{0}; return n; }constexpr int kStH = 72; // halves a shared row: 64 + 8 (the pad spreads the banks; rows stay 16-byte aligned)using hv2 = _Float16 __attribute__((ext_vector_type(2)));using hv8 = _Float16 __attribute__((ext_vector_type(8)));@@ -233,9 +236,21 @@ __global__ void __launch_bounds__(256) set_kernel(const SetArgs a, int dbg) {inline bool ok_device() { // gfx12 unless STRATA_DS4_HIPK=0static std::atomic<int> state[16];+ static std::atomic<int> capable[16]; // 1: gfx11 / gfx12int dev = 0;cudaGetDevice(&dev);if (dev < 0 || dev >= 16) return false;+ if (const int f = force_mode().load(std::memory_order_relaxed); f >= 0) {+ if (f == 0) return false;+ int c = capable[dev].load(std::memory_order_relaxed);+ if (c == 0) {+ cudaDeviceProp pr{};+ const bool g = cudaGetDeviceProperties(&pr, dev) == cudaSuccess && (std::strncmp(pr.gcnArchName, "gfx12", 5) == 0 || std::strncmp(pr.gcnArchName, "gfx11", 5) == 0);+ c = g ? 1 : 2;+ capable[dev].store(c, std::memory_order_relaxed);+ }+ return c == 1;+ }int v = state[dev].load(std::memory_order_relaxed);if (v == 0) {cudaDeviceProp prop{};@@ -274,6 +289,7 @@ inline bool run(const SetArgs& a, cudaStream_t s) {const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;set_kernel<T, GU, BM, BN, WM, WN><<<grid, 256, 0, s>>>(a, dbg);+ launches().fetch_add(1, std::memory_order_relaxed);return cudaPeekAtLastError() == cudaSuccess;}// false: not taken (the caller's tiles do it)diff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.incindex 38747c0f5..0844177be 100644--- a/src/dsv41/ds4_iq_hip_r.inc+++ b/src/dsv41/ds4_iq_hip_r.inc@@ -205,5 +205,6 @@ inline bool runr(const SetArgs& a, cudaStream_t s) {const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;set_kernel_r<T, GU, BM, NW, NTW><<<grid, NW * 32, 0, s>>>(a, dbg);+ launches().fetch_add(1, std::memory_order_relaxed);return cudaPeekAtLastError() == cudaSuccess;}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 3f2e9c367..1fd077235 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4531,7 +4531,12 @@ struct Model::Impl {const int dev0 = devs[0].id;cfg.map_chunk = [dev0](void* h, size_t n) -> void* {cudaSetDevice(dev0);- if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {+ unsigned reg_flags = cudaHostRegisterMapped | cudaHostRegisterPortable;+#if defined(STRATA_USE_HIP)+ // (test arm: DS4_ZC_COARSE=1, hipExtHostRegisterCoarseGrained, which ROCm Core SDK 10.1 honors on Windows; the store is read-only to the GPU)+ if (std::getenv("DS4_ZC_COARSE") && std::atoi(std::getenv("DS4_ZC_COARSE")) != 0) reg_flags |= 0x8u;+#endif+ if (cudaHostRegister(h, n, reg_flags) != cudaSuccess) {(void) cudaGetLastError();return nullptr;}diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppindex 0fb77af7c..45b3fea8e 100644--- a/src/dsv41/ds4_store.hpp+++ b/src/dsv41/ds4_store.hpp@@ -375,7 +375,13 @@ public:private:static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;- static constexpr size_t kChunk = 64; // slots a chunk of the arena+ // slots a chunk of the arena (the unit the GPU mapping registers; DS4_STORE_CHUNK_SLOTS: a test knob for the registration segment size)+ static size_t chunk_slots() {+ const char* e = std::getenv("DS4_STORE_CHUNK_SLOTS");+ const long v = e ? std::atol(e) : 0;+ return v >= 8 && v <= 4096 ? (size_t) v : 64;+ }+ const size_t kChunk = chunk_slots();struct Slot {uint8_t* base = nullptr; // the blob's first byteint layer = -1, expert = -1, pin = 0;diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppindex 0d9698a22..079b8a3ff 100644--- a/tests/dsv41/iq_expert_test.cpp+++ b/tests/dsv41/iq_expert_test.cpp@@ -513,6 +513,57 @@ int main(int argc, char** argv) {}// (the prompt sets' tiles take F16 operands for the i-quants: the tile's own rounding, not the token path's)report("set down", type, w2, 3e-3);++ // ---- the matrix-core tiles (HIP gfx11 / gfx12) forced on, against the scalar tiles forced on the same inputs: each within the+ // tolerances above of the CPU reference, the two within the same tolerance of each other, and the forced-on run must really+ // have launched them (STRATA_DS4_HIPK is not needed)+ if (iq::hipk_force(1)) {+ const long long n0 = iq::hipk_launches();+ iq::SetArgs qm = q;+ uint16_t* mh; CK(hipMalloc(&mh, (size_t) NR * FF * 2)); float* my; CK(hipMalloc(&my, (size_t) NR * E * 4));+ qm.h = mh; qm.ldx = E; qm.ldo = FF; qm.xh = nullptr; qm.y = nullptr;+ if (!iq::set_gate_up(qm, 0)) { std::printf("matrix-core set_gate_up refused\n"); return 1; }+ qm.xh = mh; qm.ldx = FF; qm.y = my; qm.ldo = E;+ if (!iq::set_down(qm, 0)) { std::printf("matrix-core set_down refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const long long taken = iq::hipk_launches() - n0;+ iq::hipk_force(0);+ const long long n1 = iq::hipk_launches();+ uint16_t* sh; CK(hipMalloc(&sh, (size_t) NR * FF * 2)); float* sy; CK(hipMalloc(&sy, (size_t) NR * E * 4));+ iq::SetArgs qs = q;+ qs.h = sh; qs.ldx = E; qs.ldo = FF; qs.xh = nullptr; qs.y = nullptr;+ if (!iq::set_gate_up(qs, 0)) { std::printf("scalar set_gate_up refused\n"); return 1; }+ qs.xh = sh; qs.ldx = FF; qs.y = sy; qs.ldo = E;+ if (!iq::set_down(qs, 0)) { std::printf("scalar set_down refused\n"); return 1; }+ CK(hipDeviceSynchronize());+ const long long scal = iq::hipk_launches() - n1;+ iq::hipk_force(-1);+ const auto hm = down_(mh, (size_t) NR * FF), hs = down_(sh, (size_t) NR * FF);+ const auto ym = down_(my, (size_t) NR * E), ys = down_(sy, (size_t) NR * E);+ double wg = 0, wd = 0, wx = 0, wy = 0;+ for (int e = 0; e < 3; ++e)+ for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {+ std::vector<float> hf(FF);+ for (int c = 0; c < FF; ++c) hf[c] = h2f(hm[(size_t) lr * FF + c]);+ for (int r = 0; r < FF; ++r) {+ double sc;+ const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);+ wg = std::max(wg, std::fabs((double) h2f(hm[(size_t) lr * FF + r]) - ref) / sc);+ wx = std::max(wx, std::fabs((double) h2f(hm[(size_t) lr * FF + r]) - (double) h2f(hs[(size_t) lr * FF + r])) / sc);+ }+ for (int r = 0; r < E; ++r) {+ double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);+ wd = std::max(wd, std::fabs(ym[(size_t) lr * E + r] - ref) / (sa + 1e-30));+ wy = std::max(wy, std::fabs((double) ym[(size_t) lr * E + r] - (double) ys[(size_t) lr * E + r]) / (sa + 1e-30));+ }+ }+ std::printf("%-7s matrix-core tiles taken %lld (scalar run: %lld)\n", iq::name(type), taken, scal);+ if (taken < 2 || scal != 0) { std::printf("%-7s FAIL: the matrix-core tiles did not run as forced\n", iq::name(type)); ++fails; }+ report("wmma gate_up vs CPU", type, wg, type == 19 ? 1.5e-2 : 1e-2);+ report("wmma down vs CPU", type, wd, 3e-3);+ report("wmma vs scalar h", type, wx, type == 19 ? 1.5e-2 : 1e-2);+ report("wmma vs scalar y", type, wy, 3e-3);+ }}// ---- the CPU lane on the same experts and inputs (ggml-cpu's rows, or Strata's AVX2 ones with DS4_GG_IQ256=1)diff --git a/tools/dsv41/win/build_umaprobe.bat b/tools/dsv41/win/build_umaprobe.batindex 5db3323fd..5214219cd 100644--- a/tools/dsv41/win/build_umaprobe.bat+++ b/tools/dsv41/win/build_umaprobe.bat@@ -7,6 +7,6 @@ for /f "delims=" %%R in ('"%ROCM_VENV%\Scripts\rocm-sdk.exe" path --root') do sefor /f "delims=" %%V in ('"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath') do set "VS=%%V"call "%VS%\VC\Auxiliary\Build\vcvars64.bat" >nul || exit /b 1set "PATH=%ROCM%\bin;%ROCM%\lib\llvm\bin;%PATH%"-"%ROCM%\lib\llvm\bin\clang++.exe" -x hip -O2 --offload-arch=gfx1151 --rocm-path="%ROCM%" --rocm-device-lib-path="%ROCM%\lib\llvm\amdgcn\bitcode" -I"%ROCM%\include" "%~dp0umaprobe.cpp" -o "%OUT%" -L"%ROCM%\lib" -lamdhip64 -ldxgi -Wno-unused-command-line-argument+"%ROCM%\lib\llvm\bin\clang++.exe" -x hip -O2 --offload-arch=gfx1151 --rocm-path="%ROCM%" --rocm-device-lib-path="%ROCM%\lib\llvm\amdgcn\bitcode" -I"%ROCM%\include" "%~dp0umaprobe.cpp" -o "%OUT%" -L"%ROCM%\lib" -lamdhip64 -ldxgi -lversion -lpsapi -Wno-unused-command-line-argumentecho BUILD-EXIT %errorlevel%endlocaldiff --git a/tools/dsv41/win/umaprobe.cpp b/tools/dsv41/win/umaprobe.cppindex cb84d6c52..48b703dff 100644--- a/tools/dsv41/win/umaprobe.cpp+++ b/tools/dsv41/win/umaprobe.cpp@@ -8,6 +8,7 @@// Build: tools\dsv41\win\build_umaprobe.bat (ROCm clang, gfx1151). Internal tool, not shipped.#include <hip/hip_runtime.h>#include <windows.h>+#include <psapi.h>#include <dxgi1_4.h>#include <chrono>#include <cstdint>@@ -171,6 +172,32 @@ static void regchunk(int gib, int chunk) {for (void* p : v) { hipHostUnregister(p); VirtualFree(p, 0, MEM_RELEASE); }}++// regsub: the engine's store pattern: one big VirtualAlloc(MEM_RESERVE), committed in chunks of N MiB (untouched or touched), each registered mapped+static void regsub(int chunk_mib, int nchunks, int touch) {+ info();+ const size_t C = (size_t) chunk_mib << 20;+ uint8_t* base = (uint8_t*) VirtualAlloc(nullptr, C * (size_t) nchunks, MEM_RESERVE, PAGE_READWRITE);+ if (!base) { std::printf("reserve failed\n"); return; }+ std::vector<void*> dev;+ for (int i = 0; i < nchunks; ++i) {+ uint8_t* p = base + C * (size_t) i;+ if (!VirtualAlloc(p, C, MEM_COMMIT, PAGE_READWRITE)) { std::printf("commit failed at %d\n", i); break; }+ if (touch) std::memset(p, 7, C);+ hipError_t e = hipHostRegister(p, C, hipHostRegisterPortable | hipHostRegisterMapped);+ void* dp = nullptr;+ if (e == hipSuccess) e = hipHostGetDevicePointer(&dp, p, 0);+ std::printf("chunk %d: register+devptr %s, dev %p host %p\n", i, hipGetErrorString(e), dp, (void*) p);+ if (e != hipSuccess) { (void) hipGetLastError(); break; }+ dev.push_back(dp);+ if (!touch) std::memset(p, 7, C); // (the store's reads fill it after the registration)+ }+ for (size_t i = 0; i < dev.size(); ++i) {+ std::printf("GPU read chunk %zu: %.1f GB/s\n", i, gpu_read_gbps(dev[i], C));+ std::fflush(stdout);+ }+}+static void hostalloc(int gib) {info();void* p = nullptr;@@ -203,12 +230,94 @@ static void copy() {}}+// ver: which amdhip64 this process really loaded (path, file version), the HIP runtime / driver versions+static void ver() {+ int rv = 0, dv = 0; hipRuntimeGetVersion(&rv); hipDriverGetVersion(&dv);+ std::printf("hipRuntimeGetVersion %d, hipDriverGetVersion %d\n", rv, dv);+ HMODULE h = GetModuleHandleA("amdhip64_7.dll");+ if (!h) h = GetModuleHandleA("amdhip64.dll");+ char path[MAX_PATH] = {0};+ if (h) GetModuleFileNameA(h, path, MAX_PATH);+ std::printf("loaded amdhip64: %s\n", path);+ DWORD hd = 0, sz = path[0] ? GetFileVersionInfoSizeA(path, &hd) : 0;+ if (sz) {+ std::vector<char> b(sz);+ VS_FIXEDFILEINFO* fi = nullptr; UINT n = 0;+ if (GetFileVersionInfoA(path, 0, sz, b.data()) && VerQueryValueA(b.data(), "\\", (void**) &fi, &n) && fi)+ std::printf("file version %u.%u.%u.%u\n", HIWORD(fi->dwFileVersionMS), LOWORD(fi->dwFileVersionMS), HIWORD(fi->dwFileVersionLS), LOWORD(fi->dwFileVersionLS));+ }+ hipDeviceProp_t p{}; hipGetDeviceProperties(&p, 0);+ std::printf("device %s %s\n", p.name, p.gcnArchName);+ info();+}++__global__ void sum_kernel(const unsigned long long* __restrict__ p, size_t n, unsigned long long* out) {+ unsigned long long acc = 0;+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (size_t) gridDim.x * blockDim.x) acc += p[i];+ atomicAdd(out, acc);+}+static void m1_line(const char* what, int seg, double secs, double avail0, size_t reg_total) {+ PROCESS_MEMORY_COUNTERS_EX pm{}; pm.cb = sizeof pm; GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*) &pm, sizeof pm);+ const double avail = ram_avail_gib();+ std::printf("M1 %s seg %d: %.2fs | registered %.1f GiB | avail %.1f GiB (fell %.1f = %.2fx the registered) | private %.1f GiB ws %.1f GiB faults %lu\n", what, seg, secs,+ reg_total / 1073741824.0, avail, avail0 - avail, reg_total ? (avail0 - avail) / (reg_total / 1073741824.0) : 0.0, pm.PrivateUsage / 1073741824.0,+ pm.WorkingSetSize / 1073741824.0, pm.PageFaultCount);+ dxgi_line(" ");+ std::fflush(stdout);+}+// m1 <segment MiB> <total GiB> <mapped|coarse|plain>: reserve total, then per segment commit + touch (a known pattern), register (flags per mode),+// device pointer, a checksum kernel that reads every page and compares with the pattern's sum; every step logs DXGI, RAM, commit, faults.+// 'coarse' = hipHostRegisterMapped | hipExtHostRegisterCoarseGrained; 'plain' = portable only (no map). A fresh process per arm, a watchdog outside.+static void m1(int seg_mib, int total_gib, const std::string& mode) {+ ver();+ const size_t S = (size_t) seg_mib << 20, T = (size_t) total_gib << 30;+ const int nseg = (int) (T / S);+ uint8_t* base = (uint8_t*) VirtualAlloc(nullptr, T, MEM_RESERVE, PAGE_READWRITE);+ if (!base) { std::printf("reserve failed\n"); return; }+ unsigned flags = hipHostRegisterPortable;+ if (mode == "mapped") flags |= hipHostRegisterMapped;+ if (mode == "coarse") flags |= hipHostRegisterMapped | hipExtHostRegisterCoarseGrained;+ unsigned long long* dsum = nullptr; CK(hipMalloc(&dsum, 8));+ const double avail0 = ram_avail_gib();+ m1_line("start", 0, 0, avail0, 0);+ size_t reg = 0;+ for (int i = 0; i < nseg; ++i) {+ uint8_t* p = base + S * (size_t) i;+ double t0 = now();+ if (!VirtualAlloc(p, S, MEM_COMMIT, PAGE_READWRITE)) { std::printf("commit failed at segment %d\n", i); break; }+ unsigned long long* w = (unsigned long long*) p; const size_t nw = S / 8;+ unsigned long long expect = 0;+ for (size_t k = 0; k < nw; ++k) { w[k] = (unsigned long long) i * 1000003ull + k; expect += w[k]; }+ const double tt = now() - t0;+ t0 = now();+ hipError_t e = hipHostRegister(p, S, flags);+ void* dp = p;+ if (e == hipSuccess && (flags & hipHostRegisterMapped)) { e = hipHostGetDevicePointer(&dp, p, 0); }+ const double tr = now() - t0;+ if (e != hipSuccess) { std::printf("M1 register segment %d FAILED: %s (after %.2fs)\n", i, hipGetErrorString(e), tr); (void) hipGetLastError(); m1_line("failed", i, tr, avail0, reg); break; }+ reg += S;+ unsigned long long zero = 0; CK(hipMemcpy(dsum, &zero, 8, hipMemcpyHostToDevice));+ t0 = now();+ sum_kernel<<<2048, 256>>>((const unsigned long long*) dp, nw, dsum);+ hipError_t ek = hipDeviceSynchronize();+ const double tk = now() - t0;+ unsigned long long got = 0; hipMemcpy(&got, dsum, 8, hipMemcpyDeviceToHost);+ std::printf("M1 segment %d: touch %.2fs register %.2fs read %.3fs (%.0f GB/s) checksum %s\n", i, tt, tr, tk, S / 1e9 / (std::max)(tk, 1e-9), ek != hipSuccess ? hipGetErrorString(ek) : (got == expect ? "OK" : "MISMATCH"));+ if (ek != hipSuccess) { m1_line("kernel fault", i, tk, avail0, reg); break; }+ if (i % (std::max)(1, nseg / 8) == 0 || i == nseg - 1) m1_line("step", i, tr, avail0, reg);+ }+ m1_line("end", nseg, 0, avail0, reg);+}+int main(int argc, char** argv) {const std::string m = argc > 1 ? argv[1] : "";+ if (m == "ver") { ver(); return 0; }+ if (m == "m1") { m1(argc > 2 ? std::atoi(argv[2]) : 1024, argc > 3 ? std::atoi(argv[3]) : 8, argc > 4 ? argv[4] : "mapped"); return 0; }if (m == "info") info();else if (m == "alloc") alloc(argc > 2 ? std::atoi(argv[2]) : 80);else if (m == "reg") reg(argc > 2 ? std::atoi(argv[2]) : 8, argc > 3 ? std::atoi(argv[3]) : 0);else if (m == "regchunk") regchunk(argc > 2 ? std::atoi(argv[2]) : 32, argc > 3 ? std::atoi(argv[3]) : 4);+ else if (m == "regsub") regsub(argc > 2 ? std::atoi(argv[2]) : 660, argc > 3 ? std::atoi(argv[3]) : 8, argc > 4 ? std::atoi(argv[4]) : 0);else if (m == "hostalloc") hostalloc(argc > 2 ? std::atoi(argv[2]) : 8);else if (m == "copy") copy();else std::printf("usage: umaprobe info|alloc N|reg GiB|hostalloc GiB|copy\n");