Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 iq (CUDA): per-device light-set scratch and tensor-core check (stream 0 is one handle on every device: the scratch of the first GPU was reused on the second), block/sub-block loop form in the token kernels; test: --devices (in-process multi-GPU, bit-compared)

Niko12211cace5c864b2
diff --git a/include/strata/dsv41/hip_or_cuda.h b/include/strata/dsv41/hip_or_cuda.h
index b0b835f95..fb5630df0 100644
--- a/include/strata/dsv41/hip_or_cuda.h
+++ b/include/strata/dsv41/hip_or_cuda.h
@@ -36,6 +36,7 @@ inline hipError_t hipMemcpy(T* dst, const T* src, size_t n, int kind) {
}
inline hipError_t hipMemset(void* p, int v, size_t n) { return cudaMemset(p, v, n); }
inline hipError_t hipDeviceSynchronize() { return cudaDeviceSynchronize(); }
+inline hipError_t hipSetDevice(int d) { return cudaSetDevice(d); }
inline const char* hipGetErrorString(hipError_t e) { return cudaGetErrorString(e); }
inline hipError_t hipEventCreate(hipEvent_t* e) { return cudaEventCreate(e); }
inline hipError_t hipEventRecord(hipEvent_t e, hipStream_t s = nullptr) { return cudaEventRecord(e, s); }
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index d85f13868..cc486f3c6 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -18,6 +18,7 @@
#include <atomic>
#include <cstdio>
#include <cstdlib>
+#include <map>
#include <mutex>
#include <unordered_map>
@@ -144,10 +145,9 @@ struct Sub {
float d, m0, m1;
int s0, s1;
};
+// (b: the block's address, ib: the sub-block in it (0 for the 32-value types); the token kernels keep ib fixed a lane)
template <int T>
-__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {
- const int ib = kb & 7;
- const uint8_t* b = row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B;
+__device__ __forceinline__ void decode_b(const uint8_t* __restrict__ b, int ib, const uint32_t* tab, Sub& s) {
if constexpr (T == kMXFP4) { // codes: value j in byte j's low nibble, j + 16 in its high one (any alignment)
const uint32_t t[4] = {0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu}; // ggml's kvalues_mxfp4
#pragma unroll
@@ -341,6 +341,10 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,
s.d = h2f(b + 108);
}
}
+template <int T>
+__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {
+ decode_b<T>(row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B, kb & 7, tab, s);
+}
// the sub-block against 32 int8 activations (xa: values 0..15, xb: 16..31), before their scale: an exact integer dot,
// its scales in F32 (explicit roundings: every instantiation the same bits)
template <int T>
@@ -517,12 +521,10 @@ __device__ __forceinline__ void load_tab_fast(uint32_t* tab) {
}
}
template <int T>
-__device__ __forceinline__ void decode_fast(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {
+__device__ __forceinline__ void decode_fast_b(const uint8_t* __restrict__ b, int ib, const uint32_t* tab, Sub& s) {
if constexpr (Fast<T>::WORDS == 0) {
- decode<T>(row, kb, tab, s);
+ decode_b<T>(b, ib, tab, s);
} else if constexpr (T == kIQ1_M) { // (56-byte blocks in 1120-byte rows at 256-aligned offsets: 8-byte aligned)
- const int ib = kb & 7;
- const uint8_t* b = row + (size_t) (kb >> 3) * Ty<T>::B;
const uint32_t qsw = *(const uint32_t*) (b + 4 * ib);
const uint32_t qhw = *(const uint16_t*) (b + 32 + 2 * ib);
const uint2 sw = *(const uint2*) (b + 48);
@@ -586,6 +588,8 @@ constexpr int kGuRW = 1, kDnRW = 2; // rows a warp: gate / up (n_embd 5120: 10
template <int T, int NE, int RW = kGuRW>
__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
constexpr int LPR = 32 / RW;
+ constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = LPR / kBPS; // sub-blocks a block; blocks a lane steps
+ static_assert(LPR % kBPS == 0, "a lane keeps one sub-block position");
__shared__ __align__(16) uint32_t tab[kTabWords<T>];
__shared__ int es[NE], xr[NE];
const uint8_t* blob;
@@ -594,7 +598,7 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
load_tab_fast<T>(tab);
const int lane32 = threadIdx.x & 31;
const size_t rb = row_bytes_t<T>(a.n_embd);
- const int nsb = a.n_embd / 32;
+ const int nsb = a.n_embd / 32, nblk = nsb / kBPS;
const int ntile = a.n_ff / (kWarps * RW);
for (int mm = m0; mm < m1; mm += NE) {
const int ne = min(NE, m1 - mm);
@@ -608,15 +612,18 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
// (a persistent grid: blockIdx.x strides over the row tiles, the codebook is loaded once a block)
for (int tile = blockIdx.x; tile < ntile; tile += gridDim.x) {
const int row = (tile * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;
+ const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;
const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
float ag[NE], au[NE];
#pragma unroll
for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;
- for (int kb = lane; kb < nsb; kb += LPR) {
+ // a lane's sub-blocks: kb = lane + LPR j = 8 blk + ib with ib fixed (the 32-value types: blk = kb)
+ for (int blk = blk0; blk < nblk; blk += kStep) {
+ const int kb = blk * kBPS + ib0;
Sub sg, su;
- decode_fast<T>(g, kb, tab, sg);
- decode_fast<T>(u, kb, tab, su);
+ decode_fast_b<T>(g + (size_t) blk * Ty<T>::B, ib0, tab, sg);
+ decode_fast_b<T>(u + (size_t) blk * Ty<T>::B, ib0, tab, su);
#pragma unroll
for (int i = 0; i < NE; ++i)
if (i < ne) {
@@ -641,6 +648,8 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
template <int T, int NE, int RW = kDnRW>
__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
constexpr int LPR = 32 / RW;
+ constexpr int kBPS = kQK<T> == 32 ? 1 : 8, kStep = LPR / kBPS; // sub-blocks a block; blocks a lane steps
+ static_assert(LPR % kBPS == 0, "a lane keeps one sub-block position");
__shared__ __align__(16) uint32_t tab[kTabWords<T>];
__shared__ int es[NE];
const uint8_t* blob;
@@ -648,7 +657,7 @@ __global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
if (!group(a, blockIdx.y, blob, m0, m1)) return;
load_tab_fast<T>(tab);
const int lane32 = threadIdx.x & 31;
- const int nsb = a.n_ff / 32;
+ const int nsb = a.n_ff / 32, nblk = nsb / kBPS;
const int ntile = a.n_embd / (kWarps * RW);
for (int mm = m0; mm < m1; mm += NE) {
const int ne = min(NE, m1 - mm);
@@ -657,13 +666,15 @@ __global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
__syncthreads();
for (int tile = blockIdx.x; tile < ntile; tile += gridDim.x) {
const int row = (tile * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;
+ const int ib0 = kBPS == 8 ? lane & 7 : 0, blk0 = kBPS == 8 ? lane >> 3 : lane;
const uint8_t* d = blob + a.tr[2] + (size_t) row * row_bytes_t<T>(a.n_ff);
float ad[NE];
#pragma unroll
for (int i = 0; i < NE; ++i) ad[i] = 0.0f;
- for (int kb = lane; kb < nsb; kb += LPR) {
+ for (int blk = blk0; blk < nblk; blk += kStep) {
+ const int kb = blk * kBPS + ib0;
Sub sd;
- decode_fast<T>(d, kb, tab, sd);
+ decode_fast_b<T>(d + (size_t) blk * Ty<T>::B, ib0, tab, sd);
#pragma unroll
for (int i = 0; i < NE; ++i)
if (i < ne) {
@@ -980,14 +991,20 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps
#if defined(STRATA_USE_HIP)
return false;
#else
- static const bool ok = [] {
- int dev = 0, major = 0;
- cudaGetDevice(&dev);
+ // (per device: the cards of a multi-GPU run need not be the same generation)
+ static std::atomic<int> state[16]; // 0: unknown, 1: tensor cores, 2: the F32 kernel
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 16) return false;
+ int v = state[dev].load(std::memory_order_relaxed);
+ if (v == 0) {
+ int major = 0;
cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
- const char* v = std::getenv("STRATA_DS4_NO_TC");
- return major >= 7 && !(v != nullptr && v[0] == '1');
- }();
- return ok;
+ const char* e = std::getenv("STRATA_DS4_NO_TC");
+ v = major >= 7 && !(e != nullptr && e[0] == '1') ? 1 : 2;
+ state[dev].store(v, std::memory_order_relaxed);
+ }
+ return v == 1;
#endif
}
@@ -1080,10 +1097,12 @@ struct LightBuf {
size_t cap = 0;
};
std::mutex g_lmu;
-std::unordered_map<cudaStream_t, LightBuf> g_light;
+std::map<std::pair<int, cudaStream_t>, LightBuf> g_light; // (device, stream): the default stream 0 is one handle on every device
LightBuf* light_buf(cudaStream_t s, int k) {
std::lock_guard<std::mutex> lk(g_lmu);
- LightBuf& b = g_light[s];
+ int dev = 0;
+ cudaGetDevice(&dev);
+ LightBuf& b = g_light[{dev, s}];
const size_t need = (size_t) kLightMaxExp * kLightRows * (size_t) std::max(k, 8192);
if (b.cap < need) {
if (b.q != nullptr) cudaFree(b.q);
diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cpp
index 4d0bcfab3..17faa7daf 100644
--- a/tests/dsv41/iq_expert_test.cpp
+++ b/tests/dsv41/iq_expert_test.cpp
@@ -251,6 +251,7 @@ static void bench_big(int type, int mib) {
int main(int argc, char** argv) {
std::vector<int> types = {16, 17, 22, 18, 21, 23, 29};
+ std::vector<int> devs;
for (int i = 1; i < argc; ++i) {
if (std::string(argv[i]) == "--bench-big") { // [types] [MiB]
std::vector<int> bt; int mib = 640;
@@ -265,6 +266,8 @@ int main(int argc, char** argv) {
else bt = types;
for (int t : bt) bench(t);
return 0;
+ } else if (std::string(argv[i]) == "--devices" && i + 1 < argc) { // every check on each device in turn, in this process;
+ devs.clear(); for (const char* p = argv[++i]; *p;) { devs.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } // the later ones bit-compared to the first
} else if (std::string(argv[i]) == "--dump" && i + 1 < argc) {
g_dump = std::fopen(argv[++i], "wb"); if (!g_dump) { std::printf("cannot write %s\n", argv[i]); return 2; }
} else if (std::string(argv[i]) == "--cmp" && i + 1 < argc) {
@@ -275,9 +278,21 @@ int main(int argc, char** argv) {
gg::gg_cpu_set_iq256(std::atoi(argv[++i]));
}
}
+ int fails = 0;
+ if (devs.size() > 1) { // the first device writes its outputs, the others compare to them
+ if (g_dump || g_cmp) { std::printf("--devices takes neither --dump nor --cmp\n"); return 2; }
+ }
+ for (size_t di = 0; di < std::max<size_t>(1, devs.size()); ++di) {
+ if (!devs.empty()) {
+ CK(hipSetDevice(devs[di]));
+ if (devs.size() > 1) {
+ if (g_dump) { std::fclose(g_dump); g_dump = nullptr; }
+ if (di == 0) g_dump = std::fopen("iq_devices.bin", "wb"); else { g_cmp = std::fopen("iq_devices.bin", "rb"); g_cmp_n = g_cmp_bad = 0; }
+ }
+ std::printf("---- device %d\n", devs[di]);
+ }
std::mt19937_64 rng(11);
std::normal_distribution<float> nd(0.f, 1.f);
- int fails = 0;
auto report = [&](const char* what, int type, double worst, double tol) {
const bool ok = worst <= tol;
std::printf("%-7s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");
@@ -437,6 +452,8 @@ int main(int argc, char** argv) {
}
CK(hipFree(dblob));
}
+ if (g_cmp && devs.size() > 1) { std::fclose(g_cmp); g_cmp = nullptr; std::printf("device %d vs device %d: %ld buffers, %ld differ\n", devs[di], devs[0], g_cmp_n, g_cmp_bad); fails += g_cmp_bad != 0; }
+ }
if (g_cmp) { std::printf("bit-compare vs golden: %ld buffers, %ld differ -> %s\n", g_cmp_n, g_cmp_bad, g_cmp_bad ? "DIFFERENT" : "IDENTICAL"); fails += g_cmp_bad != 0; }
if (g_dump) std::fclose(g_dump);
std::printf("%s\n", fails ? "FAILED" : "ALL OK");