Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 iq (CUDA): per-device light-set scratch and tensor-core check (stream 0 is one handle on every device: the scratch of the first GPU was reused on the second), block/sub-block loop form in the token kernels; test: --devices (in-process multi-GPU, bit-compared)
Niko12211cace5c864b2
diff --git a/include/strata/dsv41/hip_or_cuda.h b/include/strata/dsv41/hip_or_cuda.hindex b0b835f95..fb5630df0 100644--- a/include/strata/dsv41/hip_or_cuda.h+++ b/include/strata/dsv41/hip_or_cuda.h@@ -36,6 +36,7 @@ inline hipError_t hipMemcpy(T* dst, const T* src, size_t n, int kind) {}inline hipError_t hipMemset(void* p, int v, size_t n) { return cudaMemset(p, v, n); }inline hipError_t hipDeviceSynchronize() { return cudaDeviceSynchronize(); }+inline hipError_t hipSetDevice(int d) { return cudaSetDevice(d); }inline const char* hipGetErrorString(hipError_t e) { return cudaGetErrorString(e); }inline hipError_t hipEventCreate(hipEvent_t* e) { return cudaEventCreate(e); }inline hipError_t hipEventRecord(hipEvent_t e, hipStream_t s = nullptr) { return cudaEventRecord(e, s); }diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex d85f13868..cc486f3c6 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -18,6 +18,7 @@#include <atomic>#include <cstdio>#include <cstdlib>+#include <map>#include <mutex>#include <unordered_map>@@ -144,10 +145,9 @@ struct Sub {float d, m0, m1;int s0, s1;};+// (b: the block's address, ib: the sub-block in it (0 for the 32-value types); the token kernels keep ib fixed a lane)template <int T>-__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {- const int ib = kb & 7;- const uint8_t* b = row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B;+__device__ __forceinline__ void decode_b(const uint8_t* __restrict__ b, int ib, const uint32_t* tab, Sub& s) {if constexpr (T == kMXFP4) { // codes: value j in byte j's low nibble, j + 16 in its high one (any alignment)const uint32_t t[4] = {0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu}; // ggml's kvalues_mxfp4#pragma unroll@@ -341,6 +341,10 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,s.d = h2f(b + 108);}}+template <int T>+__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {+ decode_b<T>(row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B, kb & 7, tab, s);+}// the sub-block against 32 int8 activations (xa: values 0..15, xb: 16..31), before their scale: an exact integer dot,// its scales in F32 (explicit roundings: every instantiation the same bits)template <int T>@@ -517,12 +521,10 @@ __device__ __forceinline__ void load_tab_fast(uint32_t* tab) {}}template <int T>-__device__ __forceinline__ void decode_fast(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {+__device__ __forceinline__ void decode_fast_b(const uint8_t* __restrict__ b, int ib, const uint32_t* tab, Sub& s) {if constexpr (Fast<T>::WORDS == 0) {- decode<T>(row, kb, tab, s);+ decode_b<T>(b, ib, tab, s);} else if constexpr (T == kIQ1_M) { // (56-byte blocks in 1120-byte rows at 256-aligned offsets: 8-byte aligned)- const int ib = kb & 7;- const uint8_t* b = row + (size_t) (kb >> 3) * Ty<T>::B;const uint32_t qsw = *(const uint32_t*) (b + 4 * ib);const uint32_t qhw = *(const uint16_t*) (b + 32 + 2 * ib);const uint2 sw = *(const uint2*) (b + 48);@@ -586,6 +588,8 @@ constexpr int kGuRW = 1, kDnRW = 2; // rows a warp: gate / up (n_embd 5120: 10template <int T, int NE, int RW = kGuRW>__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {constexpr int LPR = 32 / RW;+ constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = LPR / kBPS; // sub-blocks a block; blocks a lane steps+ static_assert(LPR % kBPS == 0, "a lane keeps one sub-block position");__shared__ __align__(16) uint32_t tab[kTabWords<T>];__shared__ int es[NE], xr[NE];const uint8_t* blob;@@ -594,7 +598,7 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {load_tab_fast<T>(tab);const int lane32 = threadIdx.x & 31;const size_t rb = row_bytes_t<T>(a.n_embd);- const int nsb = a.n_embd / 32;+ const int nsb = a.n_embd / 32, nblk = nsb / kBPS;const int ntile = a.n_ff / (kWarps * RW);for (int mm = m0; mm < m1; mm += NE) {const int ne = min(NE, m1 - mm);@@ -608,15 +612,18 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {// (a persistent grid: blockIdx.x strides over the row tiles, the codebook is loaded once a block)for (int tile = blockIdx.x; tile < ntile; tile += gridDim.x) {const int row = (tile * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;+ const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;float ag[NE], au[NE];#pragma unrollfor (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;- for (int kb = lane; kb < nsb; kb += LPR) {+ // a lane's sub-blocks: kb = lane + LPR j = 8 blk + ib with ib fixed (the 32-value types: blk = kb)+ for (int blk = blk0; blk < nblk; blk += kStep) {+ const int kb = blk * kBPS + ib0;Sub sg, su;- decode_fast<T>(g, kb, tab, sg);- decode_fast<T>(u, kb, tab, su);+ decode_fast_b<T>(g + (size_t) blk * Ty<T>::B, ib0, tab, sg);+ decode_fast_b<T>(u + (size_t) blk * Ty<T>::B, ib0, tab, su);#pragma unrollfor (int i = 0; i < NE; ++i)if (i < ne) {@@ -641,6 +648,8 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {template <int T, int NE, int RW = kDnRW>__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {constexpr int LPR = 32 / RW;+ constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = LPR / kBPS; // sub-blocks a block; blocks a lane steps+ static_assert(LPR % kBPS == 0, "a lane keeps one sub-block position");__shared__ __align__(16) uint32_t tab[kTabWords<T>];__shared__ int es[NE];const uint8_t* blob;@@ -648,7 +657,7 @@ __global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {if (!group(a, blockIdx.y, blob, m0, m1)) return;load_tab_fast<T>(tab);const int lane32 = threadIdx.x & 31;- const int nsb = a.n_ff / 32;+ const int nsb = a.n_ff / 32, nblk = nsb / kBPS;const int ntile = a.n_embd / (kWarps * RW);for (int mm = m0; mm < m1; mm += NE) {const int ne = min(NE, m1 - mm);@@ -657,13 +666,15 @@ __global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {__syncthreads();for (int tile = blockIdx.x; tile < ntile; tile += gridDim.x) {const int row = (tile * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;+ const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;const uint8_t* d = blob + a.tr[2] + (size_t) row * row_bytes_t<T>(a.n_ff);float ad[NE];#pragma unrollfor (int i = 0; i < NE; ++i) ad[i] = 0.0f;- for (int kb = lane; kb < nsb; kb += LPR) {+ for (int blk = blk0; blk < nblk; blk += kStep) {+ const int kb = blk * kBPS + ib0;Sub sd;- decode_fast<T>(d, kb, tab, sd);+ decode_fast_b<T>(d + (size_t) blk * Ty<T>::B, ib0, tab, sd);#pragma unrollfor (int i = 0; i < NE; ++i)if (i < ne) {@@ -980,14 +991,20 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps#if defined(STRATA_USE_HIP)return false;#else- static const bool ok = [] {- int dev = 0, major = 0;- cudaGetDevice(&dev);+ // (per device: the cards of a multi-GPU run need not be the same generation)+ static std::atomic<int> state[16]; // 0: unknown, 1: tensor cores, 2: the F32 kernel+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 16) return false;+ int v = state[dev].load(std::memory_order_relaxed);+ if (v == 0) {+ int major = 0;cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);- const char* v = std::getenv("STRATA_DS4_NO_TC");- return major >= 7 && !(v != nullptr && v[0] == '1');- }();- return ok;+ const char* e = std::getenv("STRATA_DS4_NO_TC");+ v = major >= 7 && !(e != nullptr && e[0] == '1') ? 1 : 2;+ state[dev].store(v, std::memory_order_relaxed);+ }+ return v == 1;#endif}@@ -1080,10 +1097,12 @@ struct LightBuf {size_t cap = 0;};std::mutex g_lmu;-std::unordered_map<cudaStream_t, LightBuf> g_light;+std::map<std::pair<int, cudaStream_t>, LightBuf> g_light; // (device, stream): the default stream 0 is one handle on every deviceLightBuf* light_buf(cudaStream_t s, int k) {std::lock_guard<std::mutex> lk(g_lmu);- LightBuf& b = g_light[s];+ int dev = 0;+ cudaGetDevice(&dev);+ LightBuf& b = g_light[{dev, s}];const size_t need = (size_t) kLightMaxExp * kLightRows * (size_t) std::max(k, 8192);if (b.cap < need) {if (b.q != nullptr) cudaFree(b.q);diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppindex 4d0bcfab3..17faa7daf 100644--- a/tests/dsv41/iq_expert_test.cpp+++ b/tests/dsv41/iq_expert_test.cpp@@ -251,6 +251,7 @@ static void bench_big(int type, int mib) {int main(int argc, char** argv) {std::vector<int> types = {16, 17, 22, 18, 21, 23, 29};+ std::vector<int> devs;for (int i = 1; i < argc; ++i) {if (std::string(argv[i]) == "--bench-big") { // [types] [MiB]std::vector<int> bt; int mib = 640;@@ -265,6 +266,8 @@ int main(int argc, char** argv) {else bt = types;for (int t : bt) bench(t);return 0;+ } else if (std::string(argv[i]) == "--devices" && i + 1 < argc) { // every check on each device in turn, in this process;+ devs.clear(); for (const char* p = argv[++i]; *p;) { devs.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } // the later ones bit-compared to the first} else if (std::string(argv[i]) == "--dump" && i + 1 < argc) {g_dump = std::fopen(argv[++i], "wb"); if (!g_dump) { std::printf("cannot write %s\n", argv[i]); return 2; }} else if (std::string(argv[i]) == "--cmp" && i + 1 < argc) {@@ -275,9 +278,21 @@ int main(int argc, char** argv) {gg::gg_cpu_set_iq256(std::atoi(argv[++i]));}}+ int fails = 0;+ if (devs.size() > 1) { // the first device writes its outputs, the others compare to them+ if (g_dump || g_cmp) { std::printf("--devices takes neither --dump nor --cmp\n"); return 2; }+ }+ for (size_t di = 0; di < std::max<size_t>(1, devs.size()); ++di) {+ if (!devs.empty()) {+ CK(hipSetDevice(devs[di]));+ if (devs.size() > 1) {+ if (g_dump) { std::fclose(g_dump); g_dump = nullptr; }+ if (di == 0) g_dump = std::fopen("iq_devices.bin", "wb"); else { g_cmp = std::fopen("iq_devices.bin", "rb"); g_cmp_n = g_cmp_bad = 0; }+ }+ std::printf("---- device %d\n", devs[di]);+ }std::mt19937_64 rng(11);std::normal_distribution<float> nd(0.f, 1.f);- int fails = 0;auto report = [&](const char* what, int type, double worst, double tol) {const bool ok = worst <= tol;std::printf("%-7s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");@@ -437,6 +452,8 @@ int main(int argc, char** argv) {}CK(hipFree(dblob));}+ if (g_cmp && devs.size() > 1) { std::fclose(g_cmp); g_cmp = nullptr; std::printf("device %d vs device %d: %ld buffers, %ld differ\n", devs[di], devs[0], g_cmp_n, g_cmp_bad); fails += g_cmp_bad != 0; }+ }if (g_cmp) { std::printf("bit-compare vs golden: %ld buffers, %ld differ -> %s\n", g_cmp_n, g_cmp_bad, g_cmp_bad ? "DIFFERENT" : "IDENTICAL"); fails += g_cmp_bad != 0; }if (g_dump) std::fclose(g_dump);std::printf("%s\n", fails ? "FAILED" : "ALL OK");