Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cudalab: tensor-core trellis GEMV (fp16 point table, TQL coalesced layout, prepared B fragments), K4-K7, widths 1-5

Niko1221770292e41cbe
diff --git a/tools/dsv41_lab/tqk_gemv.cuh b/tools/dsv41_lab/tqk_gemv.cuh
index 4ffab23b7..8b96aaee5 100644
--- a/tools/dsv41_lab/tqk_gemv.cuh
+++ b/tools/dsv41_lab/tqk_gemv.cuh
@@ -27,6 +27,8 @@ struct Job {
float* y; // [NB][ldy]
int rows;
int ldx, ldy;
+ const uint64_t* sc; // v3 (TQL layout) only: the scales, see tqk_repack.hpp
+ int in = 0; // v3: weights per row (x is then the prepared B-fragment array, see prep_b)
};
// the four unscaled weights of the step with state s from the float2 point table
@@ -138,4 +140,97 @@ __global__ void dequant_f32(const uint8_t* __restrict__ w, const float2* __restr
}
}
+
+// ------------------------------------------------------------------------------------------------------------
+// v2: tensor cores. The point table is exactly representable in fp16 (int16 table / 8192, |n| < 1024 or <= 11
+// significant bits), so a trellis step's four weights are two half2 words straight out of an 8 KB u32 table: no
+// conversion, and they are the A fragment of mma.m16n8k16 as they are. Thread (g = lane/4, q = lane%4) of a 16-row
+// tile holds rows g and g+8; mma m of a 128-weight block takes step t = 8q + m of both rows (k slots 2q, 2q+1, 2q+8,
+// 2q+9 = weights 4t..4t+3) against the activations x[col g][4t..4t+3] (B fragment, columns 0..NB-1 of the 8). The k
+// order inside an mma is free as long as A and B agree. The activations are split x = hi + lo (two fp16) so the
+// product is accurate to ~2^-22 relative (EXACT = 1; EXACT = 0 uses hi only, fp16-rounded activations). The block
+// scale d is applied per row after the block's 8 (16) mma. D lands in rows g / g+8, columns 2q, 2q+1: stored directly.
+__device__ __forceinline__ void mma16816(float* c, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) {
+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n"
+ : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+ : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));
+}
+__device__ __forceinline__ uint32_t h2bits(__half2 h) { return *(uint32_t*) &h; }
+
+// the 4 states of steps 4l..4l+3 -> the 8 table entries needed: es[i][0..1]
+template <int K, int NB, int T, int EXACT>
+__global__ void __launch_bounds__(128) gemv_v2(const Job* __restrict__ jobs, const uint32_t* __restrict__ lut_g, int in) {
+ __shared__ uint32_t lut[2048];
+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];
+ __syncthreads();
+ const Job job = jobs[blockIdx.y];
+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, q = lane & 3;
+ const int row0 = (blockIdx.x * (blockDim.x >> 5) + warp) * 16 * T;
+ if (row0 >= job.rows) return;
+ constexpr int bsz = Fmt<K>::kBlockBytes;
+ const int nblk = in / 128;
+ const size_t row_bytes = (size_t) nblk * bsz;
+ float tot[T][4];
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 4; ++i) tot[t][i] = 0.f;
+ for (int b = 0; b < nblk; ++b) {
+ uint32_t bh[8][2], bl[8][2];
+ if (g < NB) {
+ const float4* xp = (const float4*) (job.x + (size_t) g * job.ldx + b * 128 + 32 * q);
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ const float4 v = __ldg(xp + m);
+ const __half2 h0 = __floats2half2_rn(v.x, v.y), h1 = __floats2half2_rn(v.z, v.w);
+ bh[m][0] = h2bits(h0); bh[m][1] = h2bits(h1);
+ if (EXACT) {
+ bl[m][0] = h2bits(__floats2half2_rn(v.x - __low2float(h0), v.y - __high2float(h0)));
+ bl[m][1] = h2bits(__floats2half2_rn(v.z - __low2float(h1), v.w - __high2float(h1)));
+ }
+ }
+ } else {
+#pragma unroll
+ for (int m = 0; m < 8; ++m) { bh[m][0] = bh[m][1] = bl[m][0] = bl[m][1] = 0u; }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t) {
+ uint32_t ea[2][8], eb[2][8]; // [row half][m]
+ float dd[2];
+#pragma unroll
+ for (int h = 0; h < 2; ++h) {
+ const int row = row0 + t * 16 + g + 8 * h;
+ const uint8_t* blk = job.w + (size_t) row * row_bytes + (size_t) b * bsz;
+ dd[h] = __half2float(__ushort_as_half(*(const uint16_t*) blk));
+#pragma unroll
+ for (int u = 0; u < 2; ++u) {
+ uint32_t s[4];
+ states4<K>((const uint16_t*) (blk + 2), 2 * q + u, s);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const uint32_t x = s[i] * kHash;
+ ea[h][4 * u + i] = lut[x >> 21];
+ eb[h][4 * u + i] = lut[(x >> 10) & 2047u];
+ }
+ }
+ }
+ float acc[4] = {0.f, 0.f, 0.f, 0.f};
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ mma16816(acc, ea[0][m], ea[1][m], eb[0][m], eb[1][m], bh[m][0], bh[m][1]);
+ if (EXACT) mma16816(acc, ea[0][m], ea[1][m], eb[0][m], eb[1][m], bl[m][0], bl[m][1]);
+ }
+ tot[t][0] += dd[0] * acc[0]; tot[t][1] += dd[0] * acc[1];
+ tot[t][2] += dd[1] * acc[2]; tot[t][3] += dd[1] * acc[3];
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int col = 2 * q + (i & 1), row = row0 + t * 16 + g + 8 * (i >> 1);
+ if (col < NB) job.y[(size_t) col * job.ldy + row] = tot[t][i];
+ }
+}
+
} // namespace tqlab
diff --git a/tools/dsv41_lab/tqk_gemv3.cuh b/tools/dsv41_lab/tqk_gemv3.cuh
new file mode 100644
index 000000000..cc2ff36c6
--- /dev/null
+++ b/tools/dsv41_lab/tqk_gemv3.cuh
@@ -0,0 +1,118 @@
+// tools/dsv41_lab/tqk_gemv3.cuh - v3: tensor-core GEMV over the TQL layout (tqk_repack.hpp); include after tqk_gemv.cuh.
+// Every lane's stream bits and the scales arrive in K + 1 fully coalesced loads per (16-row tile, block pair) - K words and one
+// u64 shared by the quad - instead of ~18 u16 loads over 8 cache lines. The 15-bit states of a lane's 8 steps need 15 - K
+// bits from lane q-1's chunk (circular): one shuffle per chunk.
+#pragma once
+#include "tqk_gemv.cuh"
+
+namespace tqlab {
+
+template <int K, int NB, int T, int EXACT>
+__global__ void __launch_bounds__(128) gemv_v3(const Job* __restrict__ jobs, const uint32_t* __restrict__ lut_g, int in) {
+ __shared__ uint32_t lut[2048];
+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];
+ __syncthreads();
+ const Job job = jobs[blockIdx.y];
+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, q = lane & 3;
+ const int tile0 = (blockIdx.x * (blockDim.x >> 5) + warp) * T;
+ if (tile0 * 16 >= job.rows) return;
+ const int nblk = job.in / 128, NP = nblk / 2;
+ float tot[T][4];
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 4; ++i) tot[t][i] = 0.f;
+ const int src_lane = 4 * g + ((q + 3) & 3);
+ for (int P = 0; P < NP; ++P) {
+ uint32_t bh[2][8][2], bl[2][8][2];
+#pragma unroll
+ for (int bk = 0; bk < 2; ++bk) {
+ if (g < NB) {
+ const uint4* xp = ((const uint4*) job.x) + (((size_t) (2 * P + bk) * 8) * NB + g) * 4 + q;
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ const uint4 v = __ldg(xp + (size_t) m * NB * 4);
+ bh[bk][m][0] = v.x;
+ bh[bk][m][1] = v.y;
+ if (EXACT) { bl[bk][m][0] = v.z; bl[bk][m][1] = v.w; }
+ }
+ } else {
+#pragma unroll
+ for (int m = 0; m < 8; ++m) { bh[bk][m][0] = bh[bk][m][1] = bl[bk][m][0] = bl[bk][m][1] = 0u; }
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t) {
+ const size_t base = (size_t) (tile0 + t) * NP + P;
+ uint32_t r[K];
+#pragma unroll
+ for (int j = 0; j < K; ++j) r[j] = __ldcs(((const uint32_t*) job.w) + (base * K + j) * 32 + lane);
+ const uint64_t dq = __ldcs(job.sc + base * 8 + g);
+ // windows of the 4 chunks: own 8K bits + the low 15-K bits of lane q-1's chunk
+ uint64_t win[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int a = (8 * K * i) >> 5, sh = (8 * K * i) & 31;
+ const uint32_t w0 = r[a];
+ const uint32_t w1 = a + 1 < K ? r[a + 1 < K ? a + 1 : 0] : 0u;
+ const uint32_t w2 = a + 2 < K ? r[a + 2 < K ? a + 2 : 0] : 0u;
+ const uint32_t lo = __funnelshift_r(w0, w1, sh), hi = __funnelshift_r(w1, w2, sh);
+ const uint64_t ch = (((uint64_t) hi << 32) | lo) & ((1ull << (8 * K)) - 1ull);
+ const uint32_t fw = __shfl_sync(0xffffffffu, lo, src_lane);
+ win[i] = ch | ((uint64_t) (fw & ((1u << (15 - K)) - 1u)) << (8 * K));
+ }
+ uint32_t ea[2][2][8], eb[2][2][8]; // [bk][h][m]
+#pragma unroll
+ for (int i = 0; i < 4; ++i)
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ const uint32_t s = (uint32_t) (win[i] >> ((7 - m) * K)) & kMask;
+ const uint32_t x = s * kHash;
+ ea[i >> 1][i & 1][m] = lut[x >> 21];
+ eb[i >> 1][i & 1][m] = lut[(x >> 10) & 2047u];
+ }
+#pragma unroll
+ for (int bk = 0; bk < 2; ++bk) {
+ float acc[4] = {0.f, 0.f, 0.f, 0.f};
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ mma16816(acc, ea[bk][0][m], ea[bk][1][m], eb[bk][0][m], eb[bk][1][m], bh[bk][m][0], bh[bk][m][1]);
+ if (EXACT) mma16816(acc, ea[bk][0][m], ea[bk][1][m], eb[bk][0][m], eb[bk][1][m], bl[bk][m][0], bl[bk][m][1]);
+ }
+ const float d0 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 0)))));
+ const float d1 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 1)))));
+ tot[t][0] += d0 * acc[0];
+ tot[t][1] += d0 * acc[1];
+ tot[t][2] += d1 * acc[2];
+ tot[t][3] += d1 * acc[3];
+ }
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int col = 2 * q + (i & 1), row = (tile0 + t) * 16 + g + 8 * (i >> 1);
+ if (col < NB) job.y[(size_t) col * job.ldy + row] = tot[t][i];
+ }
+}
+
+
+} // namespace tqlab
+
+namespace tqlab {
+// activations -> B fragments, once per call: out[(((blk * 8 + m) * nb + c) * 4 + q)] = {hi(x0,x1), hi(x2,x3), lo(x0,x1), lo(x2,x3)}
+// for x[c][blk * 128 + 32 q + 4 m .. +3] (hi = fp16(x), lo = fp16(x - hi): hi + lo is x to ~2^-22).
+__global__ void prep_b(const float* __restrict__ x, int ldx, int nb, int in, uint4* __restrict__ out) {
+ const int nblk = in / 128;
+ const size_t gid = (size_t) blockIdx.x * blockDim.x + threadIdx.x, n = (size_t) nb * nblk * 32;
+ if (gid >= n) return;
+ const int m = gid & 7, q = (gid >> 3) & 3;
+ const size_t cb = gid >> 5; // c * nblk + blk
+ const int c = (int) (cb / nblk), blk = (int) (cb % nblk);
+ const float4 v = *(const float4*) (x + (size_t) c * ldx + blk * 128 + 32 * q + 4 * m);
+ const __half2 h0 = __floats2half2_rn(v.x, v.y), h1 = __floats2half2_rn(v.z, v.w);
+ out[(((size_t) blk * 8 + m) * nb + c) * 4 + q] = make_uint4(h2bits(h0), h2bits(h1), h2bits(__floats2half2_rn(v.x - __low2float(h0), v.y - __high2float(h0))),
+ h2bits(__floats2half2_rn(v.z - __low2float(h1), v.w - __high2float(h1))));
+}
+} // namespace tqlab
diff --git a/tools/dsv41_lab/tqk_lab.cu b/tools/dsv41_lab/tqk_lab.cu
index f227443a9..d0a4fe2b0 100644
--- a/tools/dsv41_lab/tqk_lab.cu
+++ b/tools/dsv41_lab/tqk_lab.cu
@@ -3,6 +3,8 @@
// Parity: dequant vs tqenc::decode_blocks (bit exact), GEMV vs a double reference on the host-decoded weights.
// Bench: one decode token of a routed layer = top-6 experts x (gate, up: 2304 x 5120; down: 5120 x 2304).
#include "tqk_gemv.cuh"
+#include "tqk_gemv3.cuh"
+#include "tqk_repack.hpp"
#include "../trellis/tqenc.hpp"
#include <algorithm>
@@ -39,15 +41,23 @@ __global__ void rd_kernel(const float4* p, size_t n4, float* o) {
for (size_t i = blockIdx.x * (size_t) blockDim.x + threadIdx.x; i < n4; i += (size_t) gridDim.x * blockDim.x) { float4 v = p[i]; s += v.x + v.y + v.z + v.w; }
if (s == 1234.5f) *o = s;
}
-typedef void (*gemv_fn)(const Job*, const float2*, int);
-template <int K, int NB, int RPW> static gemv_fn pick_rpw(int rpw) { (void) rpw; return gemv_v1<K, NB, RPW>; }
-template <int K, int NB> static gemv_fn pick(int variant) {
+struct KFn { const void* f; int threads; int rows_per_cta; int lutkind; int tql = 0; }; // lutkind 0: float2 table, 1: u32 half2 table
+template <int K, int NB> static KFn pick(int variant) {
switch (variant) {
- case 1: return gemv_v1<K, NB, 1>;
- default: return gemv_v1<K, NB, 2>;
+ case 1: return {(const void*) gemv_v1<K, NB, 1>, 256, 8, 0};
+ case 2: return {(const void*) gemv_v1<K, NB, 2>, 256, 16, 0};
+ case 3: return {(const void*) gemv_v2<K, NB, 1, 1>, 128, 64, 1};
+ case 4: return {(const void*) gemv_v2<K, NB, 2, 1>, 128, 128, 1};
+ case 5: return {(const void*) gemv_v2<K, NB, 4, 1>, 128, 256, 1};
+ case 6: return {(const void*) gemv_v2<K, NB, 2, 0>, 128, 128, 1};
+ case 7: return {(const void*) gemv_v2<K, NB, 4, 0>, 128, 256, 1};
+ case 8: return {(const void*) gemv_v3<K, NB, 1, 1>, 128, 64, 1, 1};
+ case 9: return {(const void*) gemv_v3<K, NB, 2, 1>, 128, 128, 1, 1};
+ case 10: return {(const void*) gemv_v3<K, NB, 4, 1>, 128, 256, 1, 1};
+ default: return {(const void*) gemv_v3<K, NB, 2, 0>, 128, 128, 1, 1};
}
}
-template <int K> static gemv_fn pick_nb(int nb, int variant) {
+template <int K> static KFn pick_nb(int nb, int variant) {
switch (nb) {
case 1: return pick<K, 1>(variant);
case 2: return pick<K, 2>(variant);
@@ -56,7 +66,7 @@ template <int K> static gemv_fn pick_nb(int nb, int variant) {
default: return pick<K, 5>(variant);
}
}
-static gemv_fn pick_k(int K, int nb, int variant) {
+static KFn pick_k(int K, int nb, int variant) {
switch (K) {
case 4: return pick_nb<4>(nb, variant);
case 5: return pick_nb<5>(nb, variant);
@@ -64,12 +74,12 @@ static gemv_fn pick_k(int K, int nb, int variant) {
default: return pick_nb<7>(nb, variant);
}
}
-static int variant_rpw(int variant) { return variant == 1 ? 1 : 2; }
-
-static void launch(gemv_fn f, int variant, const Job* jd, int njobs, int rows, int in, const float2* lut, cudaStream_t s = 0) {
- const int rpw = variant_rpw(variant), per_cta = 8 * rpw;
- dim3 grid((rows + per_cta - 1) / per_cta, njobs);
- f<<<grid, 256, 0, s>>>(jd, lut, in);
+static const float2* g_lut_f2; static const uint32_t* g_lut_u32;
+static void launch(const KFn& k, const Job* jd, int njobs, int rows, int in, cudaStream_t s = 0) {
+ dim3 grid((rows + k.rows_per_cta - 1) / k.rows_per_cta, njobs);
+ const void* lut = k.lutkind ? (const void*) g_lut_u32 : (const void*) g_lut_f2;
+ void* args[] = {(void*) &jd, (void*) &lut, (void*) &in};
+ cudaLaunchKernel(k.f, grid, dim3(k.threads), args, 0, s);
}
static float* dev_f32(const std::vector<float>& h) {
@@ -98,6 +108,17 @@ int main(int argc, char** argv) {
// point table (x, y) as the CPU decoder sees it
const float* pt = tqenc::point_table();
float2* lut; CK(cudaMalloc(&lut, 2048 * sizeof(float2))); CK(cudaMemcpy(lut, pt, 2048 * sizeof(float2), cudaMemcpyHostToDevice));
+ g_lut_f2 = lut;
+ { // half2 table: must be exact (the claim the tensor-core kernels rest on)
+ std::vector<uint32_t> hl(2048); int inexact = 0;
+ for (int i = 0; i < 2048; ++i) {
+ const uint16_t a = tqenc::float_to_half(pt[2 * i]), b = tqenc::float_to_half(pt[2 * i + 1]);
+ inexact += tqenc::half_to_float(a) != pt[2 * i] || tqenc::half_to_float(b) != pt[2 * i + 1];
+ hl[i] = a | ((uint32_t) b << 16);
+ }
+ std::printf("fp16 point table inexact entries: %d\n", inexact);
+ uint32_t* d; CK(cudaMalloc(&d, 8192)); CK(cudaMemcpy(d, hl.data(), 8192, cudaMemcpyHostToDevice)); g_lut_u32 = d;
+ }
// streaming bandwidth baseline: read-only sum over 512 MB
{
@@ -144,7 +165,7 @@ int main(int argc, char** argv) {
fails += bad != 0;
// ---- parity 2: GEMV vs double reference on the decoded weights, rows x in small
for (int nb : NBs) {
- const int rows = 96, in = 640, nblk = in / 128;
+ const int rows = 256, in = 768, nblk = in / 128;
std::vector<uint8_t> mw; fill_random(mw, K, (int64_t) rows * nblk);
std::vector<float> dec((size_t) rows * in); tqenc::decode_blocks(type, mw.data(), (int64_t) rows * nblk, dec.data());
std::vector<float> x((size_t) nb * in); for (auto& v : x) v = frnd();
@@ -152,8 +173,13 @@ int main(int argc, char** argv) {
for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); } ref[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }
uint8_t* mwd; CK(cudaMalloc(&mwd, mw.size())); CK(cudaMemcpy(mwd, mw.data(), mw.size(), cudaMemcpyHostToDevice));
float* xd = dev_f32(x); float* yd; CK(cudaMalloc(&yd, (size_t) nb * rows * 4));
- Job j{mwd, xd, yd, rows, in, rows}; Job* jd; CK(cudaMalloc(&jd, sizeof(Job))); CK(cudaMemcpy(jd, &j, sizeof(Job), cudaMemcpyHostToDevice));
- launch(pick_k(K, nb, variant), variant, jd, 1, rows, in, lut); CK(cudaGetLastError());
+ const KFn kf = pick_k(K, nb, variant); const uint64_t* scd = nullptr; uint8_t* wd = mwd;
+ if (kf.tql) { std::vector<uint32_t> words; std::vector<uint64_t> sc; repack_tql(K, mw.data(), rows, nblk, words, sc);
+ CK(cudaMalloc(&wd, words.size() * 4)); CK(cudaMemcpy(wd, words.data(), words.size() * 4, cudaMemcpyHostToDevice));
+ uint64_t* t; CK(cudaMalloc(&t, sc.size() * 8)); CK(cudaMemcpy(t, sc.data(), sc.size() * 8, cudaMemcpyHostToDevice)); scd = t; }
+ if (kf.tql) { const size_t n = (size_t) nb * (in / 128) * 32; uint4* xb; CK(cudaMalloc(&xb, n * 16)); prep_b<<<(int) ((n + 255) / 256), 256>>>(xd, in, nb, in, xb); CK(cudaDeviceSynchronize()); xd = (float*) xb; }
+ Job j{wd, xd, yd, rows, in, rows, scd, in}; Job* jd; CK(cudaMalloc(&jd, sizeof(Job))); CK(cudaMemcpy(jd, &j, sizeof(Job), cudaMemcpyHostToDevice));
+ launch(kf, jd, 1, rows, in); CK(cudaGetLastError());
std::vector<float> y((size_t) nb * rows); CK(cudaMemcpy(y.data(), yd, y.size() * 4, cudaMemcpyDeviceToHost));
double worst = 0; for (size_t i = 0; i < y.size(); ++i) worst = std::max(worst, std::fabs(y[i] - ref[i]) / (refabs[i] + 1e-30));
const bool ok = worst < 2e-6;
@@ -183,28 +209,41 @@ int main(int argc, char** argv) {
for (int m = 0; m < 2; ++m) {
uint8_t* p = big + (size_t) (e * 2 + m) * mat_gu;
CK(cudaMemcpy(p, hw.data(), mat_gu, cudaMemcpyHostToDevice));
- hj_gu.push_back(Job{p, xg_d, yg_d + (size_t) (e * 2 + m) * maxnb * rows_gu, rows_gu, in_gu, rows_gu});
+ hj_gu.push_back(Job{p, xg_d, yg_d + (size_t) (e * 2 + m) * maxnb * rows_gu, rows_gu, in_gu, rows_gu, (const uint64_t*) (p + (size_t) rows_gu * (in_gu / 128) * 4 * K)});
}
uint8_t* p = big + 2 * E * mat_gu + (size_t) e * mat_dn;
std::vector<uint8_t> hd; fill_random(hd, K, (int64_t) rows_dn * (in_dn / 128));
CK(cudaMemcpy(p, hd.data(), mat_dn, cudaMemcpyHostToDevice));
- hj_dn.push_back(Job{p, xdn_d + (size_t) e * maxnb * in_dn, yd_d + (size_t) e * maxnb * rows_dn, rows_dn, in_dn, rows_dn});
+ hj_dn.push_back(Job{p, xdn_d + (size_t) e * maxnb * in_dn, yd_d + (size_t) e * maxnb * rows_dn, rows_dn, in_dn, rows_dn, (const uint64_t*) (p + (size_t) rows_dn * (in_dn / 128) * 4 * K)});
}
Job *jgu, *jdn; CK(cudaMalloc(&jgu, sizeof(Job) * hj_gu.size())); CK(cudaMalloc(&jdn, sizeof(Job) * hj_dn.size()));
CK(cudaMemcpy(jgu, hj_gu.data(), sizeof(Job) * hj_gu.size(), cudaMemcpyHostToDevice)); CK(cudaMemcpy(jdn, hj_dn.data(), sizeof(Job) * hj_dn.size(), cudaMemcpyHostToDevice));
const double bytes_layer = (double) (mat_gu * 2 + mat_dn) * E, weights_layer = (double) (rows_gu * (double) in_gu * 2 + rows_dn * (double) in_dn) * E;
// flush L2 between reps would only matter for tiny sets: 6 x 3 matrices exceed the 4 MB L2 many times over.
for (int nb : NBs) {
- gemv_fn f = pick_k(K, nb, variant);
+ KFn f = pick_k(K, nb, variant);
cudaEvent_t e0, e1; cudaEventCreate(&e0); cudaEventCreate(&e1);
- for (int w = 0; w < 5; ++w) { launch(f, variant, jgu, 2 * E, rows_gu, in_gu, lut); launch(f, variant, jdn, E, rows_dn, in_dn, lut); }
+ Job* jall = nullptr; uint4* xb_gu = nullptr; uint4* xb_dn = nullptr;
+ if (f.tql) { // one launch for all 18 matrices; activations prepared once (prep_b)
+ const size_t gu_n = (size_t) nb * (in_gu / 128) * 32, dn_n = (size_t) nb * (in_dn / 128) * 32;
+ CK(cudaMalloc(&xb_gu, gu_n * 16)); CK(cudaMalloc(&xb_dn, dn_n * 16 * E));
+ prep_b<<<(int) ((gu_n + 255) / 256), 256>>>(xg_d, in_gu, nb, in_gu, xb_gu);
+ for (int e = 0; e < E; ++e) prep_b<<<(int) ((dn_n + 255) / 256), 256>>>(xdn_d + (size_t) e * maxnb * in_dn, in_dn, nb, in_dn, xb_dn + e * dn_n);
+ std::vector<Job> all = hj_gu; all.insert(all.end(), hj_dn.begin(), hj_dn.end());
+ for (size_t i = 0; i < all.size(); ++i) { const bool gu = i < hj_gu.size(); all[i].in = gu ? in_gu : in_dn; all[i].x = (const float*) (gu ? xb_gu : xb_dn + (i - hj_gu.size()) * dn_n); }
+ CK(cudaMalloc(&jall, sizeof(Job) * all.size())); CK(cudaMemcpy(jall, all.data(), sizeof(Job) * all.size(), cudaMemcpyHostToDevice));
+ CK(cudaDeviceSynchronize());
+ }
+ auto one = [&]() { if (f.tql) launch(f, jall, 3 * E, rows_dn, 0); else { launch(f, jgu, 2 * E, rows_gu, in_gu); launch(f, jdn, E, rows_dn, in_dn); } };
+ for (int w = 0; w < 5; ++w) one();
CK(cudaDeviceSynchronize()); cudaEventRecord(e0);
- for (int r = 0; r < reps; ++r) { launch(f, variant, jgu, 2 * E, rows_gu, in_gu, lut); launch(f, variant, jdn, E, rows_dn, in_dn, lut); }
+ for (int r = 0; r < reps; ++r) one();
cudaEventRecord(e1); CK(cudaEventSynchronize(e1)); CK(cudaGetLastError());
float ms; cudaEventElapsedTime(&ms, e0, e1); const double us = ms * 1e3 / reps;
const double gbs = bytes_layer / (us * 1e-6) / 1e9, roof_us = bytes_layer / 448e9 * 1e6;
std::printf("bench nb=%d: %8.1f us / layer (top-6 gate+up+down) %6.1f GB/s %6.2f Gweights/s roof(448 GB/s) %7.1f us = %4.1f%% of roof | 43 layers: %.2f ms\n",
nb, us, gbs, weights_layer / (us * 1e-6) / 1e9, roof_us, 100 * roof_us / us, us * 43 / 1e3);
+ if (jall) { cudaFree(jall); cudaFree(xb_gu); cudaFree(xb_dn); }
}
cudaFree(big); cudaFree(jgu); cudaFree(jdn); cudaFree(xg_d); cudaFree(xdn_d); cudaFree(yg_d); cudaFree(yd_d);
}
diff --git a/tools/dsv41_lab/tqk_repack.hpp b/tools/dsv41_lab/tqk_repack.hpp
new file mode 100644
index 000000000..405375d9d
--- /dev/null
+++ b/tools/dsv41_lab/tqk_repack.hpp
@@ -0,0 +1,50 @@
+// tools/dsv41_lab/tqk_repack.hpp - host reference of the "TQL" kernel layout of a trellis matrix (lossless bit permutation of
+// the file layout; rows % 16 == 0, blocks per row even). See tqk_gemv3.cuh for why.
+//
+// A tile = 16 rows, a pair P = blocks 2P, 2P+1. Lane (g = lane/4, q = lane%4) of the mma tile owns, of each of the 4
+// (block bk, row g + 8 h) stream chunks, the 8K bits at stream bit (24 - 8q) K (steps 8q .. 8q+7 of the block), chunk
+// i = 2 bk + h. Its 4 chunks are 32 K bits = K words: words[((tile * NP + P) * K + j) * 32 + lane]. The scales:
+// sc[(tile * NP + P) * 8 + g] = 4 x u16, slot 2 bk + h = fp16 d of (row g + 8 h, block 2P + bk).
+#pragma once
+#include <cstdint>
+#include <cstring>
+#include <vector>
+
+namespace tqlab {
+
+// file layout -> TQL: words (K * 32 per tile-pair) and scales (8 u64 per tile-pair)
+inline void repack_tql(int K, const uint8_t* w, int rows, int nblk, std::vector<uint32_t>& words, std::vector<uint64_t>& sc) {
+ const int bsz = 2 + 4 * K, NP = nblk / 2, NT = rows / 16;
+ words.assign((size_t) NT * NP * K * 32, 0u);
+ sc.assign((size_t) NT * NP * 8, 0ull);
+ for (int t = 0; t < NT; ++t)
+ for (int P = 0; P < NP; ++P) {
+ const size_t base = (size_t) t * NP + P;
+ for (int g = 0; g < 8; ++g) {
+ uint64_t dq = 0;
+ for (int c = 0; c < 4; ++c) {
+ const int bk = c >> 1, h = c & 1;
+ const uint8_t* blk = w + ((size_t) (t * 16 + g + 8 * h) * nblk + 2 * P + bk) * bsz;
+ uint16_t d;
+ std::memcpy(&d, blk, 2);
+ dq |= (uint64_t) d << (16 * c);
+ }
+ sc[base * 8 + g] = dq;
+ for (int q = 0; q < 4; ++q) {
+ const int lane = 4 * g + q;
+ for (int i = 0; i < 4; ++i) {
+ const int bk = i >> 1, h = i & 1;
+ const uint8_t* blk = w + ((size_t) (t * 16 + g + 8 * h) * nblk + 2 * P + bk) * bsz;
+ for (int b = 0; b < 8 * K; ++b) { // chunk bit b -> record bit 8 K i + b
+ const int src = (24 - 8 * q) * K + b;
+ const uint32_t bit = (blk[2 + (src >> 3)] >> (src & 7)) & 1u;
+ const int dst = 8 * K * i + b;
+ words[(base * K + (dst >> 5)) * 32 + lane] |= bit << (dst & 31);
+ }
+ }
+ }
+ }
+ }
+}
+
+} // namespace tqlab