Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 cudalab: tensor-core trellis GEMV (fp16 point table, TQL coalesced layout, prepared B fragments), K4-K7, widths 1-5
Niko1221770292e41cbe
diff --git a/tools/dsv41_lab/tqk_gemv.cuh b/tools/dsv41_lab/tqk_gemv.cuhindex 4ffab23b7..8b96aaee5 100644--- a/tools/dsv41_lab/tqk_gemv.cuh+++ b/tools/dsv41_lab/tqk_gemv.cuh@@ -27,6 +27,8 @@ struct Job {float* y; // [NB][ldy]int rows;int ldx, ldy;+ const uint64_t* sc; // v3 (TQL layout) only: the scales, see tqk_repack.hpp+ int in = 0; // v3: weights per row (x is then the prepared B-fragment array, see prep_b)};// the four unscaled weights of the step with state s from the float2 point table@@ -138,4 +140,97 @@ __global__ void dequant_f32(const uint8_t* __restrict__ w, const float2* __restr}}++// ------------------------------------------------------------------------------------------------------------+// v2: tensor cores. The point table is exactly representable in fp16 (int16 table / 8192, |n| < 1024 or <= 11+// significant bits), so a trellis step's four weights are two half2 words straight out of an 8 KB u32 table: no+// conversion, and they are the A fragment of mma.m16n8k16 as they are. Thread (g = lane/4, q = lane%4) of a 16-row+// tile holds rows g and g+8; mma m of a 128-weight block takes step t = 8q + m of both rows (k slots 2q, 2q+1, 2q+8,+// 2q+9 = weights 4t..4t+3) against the activations x[col g][4t..4t+3] (B fragment, columns 0..NB-1 of the 8). The k+// order inside an mma is free as long as A and B agree. The activations are split x = hi + lo (two fp16) so the+// product is accurate to ~2^-22 relative (EXACT = 1; EXACT = 0 uses hi only, fp16-rounded activations). The block+// scale d is applied per row after the block's 8 (16) mma. D lands in rows g / g+8, columns 2q, 2q+1: stored directly.+__device__ __forceinline__ void mma16816(float* c, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) {+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n"+ : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])+ : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));+}+__device__ __forceinline__ uint32_t h2bits(__half2 h) { return *(uint32_t*) &h; }++// the 4 states of steps 4l..4l+3 -> the 8 table entries needed: es[i][0..1]+template <int K, int NB, int T, int EXACT>+__global__ void __launch_bounds__(128) gemv_v2(const Job* __restrict__ jobs, const uint32_t* __restrict__ lut_g, int in) {+ __shared__ uint32_t lut[2048];+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];+ __syncthreads();+ const Job job = jobs[blockIdx.y];+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, q = lane & 3;+ const int row0 = (blockIdx.x * (blockDim.x >> 5) + warp) * 16 * T;+ if (row0 >= job.rows) return;+ constexpr int bsz = Fmt<K>::kBlockBytes;+ const int nblk = in / 128;+ const size_t row_bytes = (size_t) nblk * bsz;+ float tot[T][4];+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 4; ++i) tot[t][i] = 0.f;+ for (int b = 0; b < nblk; ++b) {+ uint32_t bh[8][2], bl[8][2];+ if (g < NB) {+ const float4* xp = (const float4*) (job.x + (size_t) g * job.ldx + b * 128 + 32 * q);+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ const float4 v = __ldg(xp + m);+ const __half2 h0 = __floats2half2_rn(v.x, v.y), h1 = __floats2half2_rn(v.z, v.w);+ bh[m][0] = h2bits(h0); bh[m][1] = h2bits(h1);+ if (EXACT) {+ bl[m][0] = h2bits(__floats2half2_rn(v.x - __low2float(h0), v.y - __high2float(h0)));+ bl[m][1] = h2bits(__floats2half2_rn(v.z - __low2float(h1), v.w - __high2float(h1)));+ }+ }+ } else {+#pragma unroll+ for (int m = 0; m < 8; ++m) { bh[m][0] = bh[m][1] = bl[m][0] = bl[m][1] = 0u; }+ }+#pragma unroll+ for (int t = 0; t < T; ++t) {+ uint32_t ea[2][8], eb[2][8]; // [row half][m]+ float dd[2];+#pragma unroll+ for (int h = 0; h < 2; ++h) {+ const int row = row0 + t * 16 + g + 8 * h;+ const uint8_t* blk = job.w + (size_t) row * row_bytes + (size_t) b * bsz;+ dd[h] = __half2float(__ushort_as_half(*(const uint16_t*) blk));+#pragma unroll+ for (int u = 0; u < 2; ++u) {+ uint32_t s[4];+ states4<K>((const uint16_t*) (blk + 2), 2 * q + u, s);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const uint32_t x = s[i] * kHash;+ ea[h][4 * u + i] = lut[x >> 21];+ eb[h][4 * u + i] = lut[(x >> 10) & 2047u];+ }+ }+ }+ float acc[4] = {0.f, 0.f, 0.f, 0.f};+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ mma16816(acc, ea[0][m], ea[1][m], eb[0][m], eb[1][m], bh[m][0], bh[m][1]);+ if (EXACT) mma16816(acc, ea[0][m], ea[1][m], eb[0][m], eb[1][m], bl[m][0], bl[m][1]);+ }+ tot[t][0] += dd[0] * acc[0]; tot[t][1] += dd[0] * acc[1];+ tot[t][2] += dd[1] * acc[2]; tot[t][3] += dd[1] * acc[3];+ }+ }+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int col = 2 * q + (i & 1), row = row0 + t * 16 + g + 8 * (i >> 1);+ if (col < NB) job.y[(size_t) col * job.ldy + row] = tot[t][i];+ }+}+} // namespace tqlabdiff --git a/tools/dsv41_lab/tqk_gemv3.cuh b/tools/dsv41_lab/tqk_gemv3.cuhnew file mode 100644index 000000000..cc2ff36c6--- /dev/null+++ b/tools/dsv41_lab/tqk_gemv3.cuh@@ -0,0 +1,118 @@+// tools/dsv41_lab/tqk_gemv3.cuh - v3: tensor-core GEMV over the TQL layout (tqk_repack.hpp); include after tqk_gemv.cuh.+// Every lane's stream bits and the scales arrive in K + 1 fully coalesced loads per (16-row tile, block pair) - K words and one+// u64 shared by the quad - instead of ~18 u16 loads over 8 cache lines. The 15-bit states of a lane's 8 steps need 15 - K+// bits from lane q-1's chunk (circular): one shuffle per chunk.+#pragma once+#include "tqk_gemv.cuh"++namespace tqlab {++template <int K, int NB, int T, int EXACT>+__global__ void __launch_bounds__(128) gemv_v3(const Job* __restrict__ jobs, const uint32_t* __restrict__ lut_g, int in) {+ __shared__ uint32_t lut[2048];+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];+ __syncthreads();+ const Job job = jobs[blockIdx.y];+ const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, q = lane & 3;+ const int tile0 = (blockIdx.x * (blockDim.x >> 5) + warp) * T;+ if (tile0 * 16 >= job.rows) return;+ const int nblk = job.in / 128, NP = nblk / 2;+ float tot[T][4];+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 4; ++i) tot[t][i] = 0.f;+ const int src_lane = 4 * g + ((q + 3) & 3);+ for (int P = 0; P < NP; ++P) {+ uint32_t bh[2][8][2], bl[2][8][2];+#pragma unroll+ for (int bk = 0; bk < 2; ++bk) {+ if (g < NB) {+ const uint4* xp = ((const uint4*) job.x) + (((size_t) (2 * P + bk) * 8) * NB + g) * 4 + q;+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ const uint4 v = __ldg(xp + (size_t) m * NB * 4);+ bh[bk][m][0] = v.x;+ bh[bk][m][1] = v.y;+ if (EXACT) { bl[bk][m][0] = v.z; bl[bk][m][1] = v.w; }+ }+ } else {+#pragma unroll+ for (int m = 0; m < 8; ++m) { bh[bk][m][0] = bh[bk][m][1] = bl[bk][m][0] = bl[bk][m][1] = 0u; }+ }+ }+#pragma unroll+ for (int t = 0; t < T; ++t) {+ const size_t base = (size_t) (tile0 + t) * NP + P;+ uint32_t r[K];+#pragma unroll+ for (int j = 0; j < K; ++j) r[j] = __ldcs(((const uint32_t*) job.w) + (base * K + j) * 32 + lane);+ const uint64_t dq = __ldcs(job.sc + base * 8 + g);+ // windows of the 4 chunks: own 8K bits + the low 15-K bits of lane q-1's chunk+ uint64_t win[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int a = (8 * K * i) >> 5, sh = (8 * K * i) & 31;+ const uint32_t w0 = r[a];+ const uint32_t w1 = a + 1 < K ? r[a + 1 < K ? a + 1 : 0] : 0u;+ const uint32_t w2 = a + 2 < K ? r[a + 2 < K ? a + 2 : 0] : 0u;+ const uint32_t lo = __funnelshift_r(w0, w1, sh), hi = __funnelshift_r(w1, w2, sh);+ const uint64_t ch = (((uint64_t) hi << 32) | lo) & ((1ull << (8 * K)) - 1ull);+ const uint32_t fw = __shfl_sync(0xffffffffu, lo, src_lane);+ win[i] = ch | ((uint64_t) (fw & ((1u << (15 - K)) - 1u)) << (8 * K));+ }+ uint32_t ea[2][2][8], eb[2][2][8]; // [bk][h][m]+#pragma unroll+ for (int i = 0; i < 4; ++i)+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ const uint32_t s = (uint32_t) (win[i] >> ((7 - m) * K)) & kMask;+ const uint32_t x = s * kHash;+ ea[i >> 1][i & 1][m] = lut[x >> 21];+ eb[i >> 1][i & 1][m] = lut[(x >> 10) & 2047u];+ }+#pragma unroll+ for (int bk = 0; bk < 2; ++bk) {+ float acc[4] = {0.f, 0.f, 0.f, 0.f};+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ mma16816(acc, ea[bk][0][m], ea[bk][1][m], eb[bk][0][m], eb[bk][1][m], bh[bk][m][0], bh[bk][m][1]);+ if (EXACT) mma16816(acc, ea[bk][0][m], ea[bk][1][m], eb[bk][0][m], eb[bk][1][m], bl[bk][m][0], bl[bk][m][1]);+ }+ const float d0 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 0)))));+ const float d1 = __half2float(__ushort_as_half((unsigned short) (dq >> (16 * (2 * bk + 1)))));+ tot[t][0] += d0 * acc[0];+ tot[t][1] += d0 * acc[1];+ tot[t][2] += d1 * acc[2];+ tot[t][3] += d1 * acc[3];+ }+ }+ }+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int col = 2 * q + (i & 1), row = (tile0 + t) * 16 + g + 8 * (i >> 1);+ if (col < NB) job.y[(size_t) col * job.ldy + row] = tot[t][i];+ }+}+++} // namespace tqlab++namespace tqlab {+// activations -> B fragments, once per call: out[(((blk * 8 + m) * nb + c) * 4 + q)] = {hi(x0,x1), hi(x2,x3), lo(x0,x1), lo(x2,x3)}+// for x[c][blk * 128 + 32 q + 4 m .. +3] (hi = fp16(x), lo = fp16(x - hi): hi + lo is x to ~2^-22).+__global__ void prep_b(const float* __restrict__ x, int ldx, int nb, int in, uint4* __restrict__ out) {+ const int nblk = in / 128;+ const size_t gid = (size_t) blockIdx.x * blockDim.x + threadIdx.x, n = (size_t) nb * nblk * 32;+ if (gid >= n) return;+ const int m = gid & 7, q = (gid >> 3) & 3;+ const size_t cb = gid >> 5; // c * nblk + blk+ const int c = (int) (cb / nblk), blk = (int) (cb % nblk);+ const float4 v = *(const float4*) (x + (size_t) c * ldx + blk * 128 + 32 * q + 4 * m);+ const __half2 h0 = __floats2half2_rn(v.x, v.y), h1 = __floats2half2_rn(v.z, v.w);+ out[(((size_t) blk * 8 + m) * nb + c) * 4 + q] = make_uint4(h2bits(h0), h2bits(h1), h2bits(__floats2half2_rn(v.x - __low2float(h0), v.y - __high2float(h0))),+ h2bits(__floats2half2_rn(v.z - __low2float(h1), v.w - __high2float(h1))));+}+} // namespace tqlabdiff --git a/tools/dsv41_lab/tqk_lab.cu b/tools/dsv41_lab/tqk_lab.cuindex f227443a9..d0a4fe2b0 100644--- a/tools/dsv41_lab/tqk_lab.cu+++ b/tools/dsv41_lab/tqk_lab.cu@@ -3,6 +3,8 @@// Parity: dequant vs tqenc::decode_blocks (bit exact), GEMV vs a double reference on the host-decoded weights.// Bench: one decode token of a routed layer = top-6 experts x (gate, up: 2304 x 5120; down: 5120 x 2304).#include "tqk_gemv.cuh"+#include "tqk_gemv3.cuh"+#include "tqk_repack.hpp"#include "../trellis/tqenc.hpp"#include <algorithm>@@ -39,15 +41,23 @@ __global__ void rd_kernel(const float4* p, size_t n4, float* o) {for (size_t i = blockIdx.x * (size_t) blockDim.x + threadIdx.x; i < n4; i += (size_t) gridDim.x * blockDim.x) { float4 v = p[i]; s += v.x + v.y + v.z + v.w; }if (s == 1234.5f) *o = s;}-typedef void (*gemv_fn)(const Job*, const float2*, int);-template <int K, int NB, int RPW> static gemv_fn pick_rpw(int rpw) { (void) rpw; return gemv_v1<K, NB, RPW>; }-template <int K, int NB> static gemv_fn pick(int variant) {+struct KFn { const void* f; int threads; int rows_per_cta; int lutkind; int tql = 0; }; // lutkind 0: float2 table, 1: u32 half2 table+template <int K, int NB> static KFn pick(int variant) {switch (variant) {- case 1: return gemv_v1<K, NB, 1>;- default: return gemv_v1<K, NB, 2>;+ case 1: return {(const void*) gemv_v1<K, NB, 1>, 256, 8, 0};+ case 2: return {(const void*) gemv_v1<K, NB, 2>, 256, 16, 0};+ case 3: return {(const void*) gemv_v2<K, NB, 1, 1>, 128, 64, 1};+ case 4: return {(const void*) gemv_v2<K, NB, 2, 1>, 128, 128, 1};+ case 5: return {(const void*) gemv_v2<K, NB, 4, 1>, 128, 256, 1};+ case 6: return {(const void*) gemv_v2<K, NB, 2, 0>, 128, 128, 1};+ case 7: return {(const void*) gemv_v2<K, NB, 4, 0>, 128, 256, 1};+ case 8: return {(const void*) gemv_v3<K, NB, 1, 1>, 128, 64, 1, 1};+ case 9: return {(const void*) gemv_v3<K, NB, 2, 1>, 128, 128, 1, 1};+ case 10: return {(const void*) gemv_v3<K, NB, 4, 1>, 128, 256, 1, 1};+ default: return {(const void*) gemv_v3<K, NB, 2, 0>, 128, 128, 1, 1};}}-template <int K> static gemv_fn pick_nb(int nb, int variant) {+template <int K> static KFn pick_nb(int nb, int variant) {switch (nb) {case 1: return pick<K, 1>(variant);case 2: return pick<K, 2>(variant);@@ -56,7 +66,7 @@ template <int K> static gemv_fn pick_nb(int nb, int variant) {default: return pick<K, 5>(variant);}}-static gemv_fn pick_k(int K, int nb, int variant) {+static KFn pick_k(int K, int nb, int variant) {switch (K) {case 4: return pick_nb<4>(nb, variant);case 5: return pick_nb<5>(nb, variant);@@ -64,12 +74,12 @@ static gemv_fn pick_k(int K, int nb, int variant) {default: return pick_nb<7>(nb, variant);}}-static int variant_rpw(int variant) { return variant == 1 ? 1 : 2; }--static void launch(gemv_fn f, int variant, const Job* jd, int njobs, int rows, int in, const float2* lut, cudaStream_t s = 0) {- const int rpw = variant_rpw(variant), per_cta = 8 * rpw;- dim3 grid((rows + per_cta - 1) / per_cta, njobs);- f<<<grid, 256, 0, s>>>(jd, lut, in);+static const float2* g_lut_f2; static const uint32_t* g_lut_u32;+static void launch(const KFn& k, const Job* jd, int njobs, int rows, int in, cudaStream_t s = 0) {+ dim3 grid((rows + k.rows_per_cta - 1) / k.rows_per_cta, njobs);+ const void* lut = k.lutkind ? (const void*) g_lut_u32 : (const void*) g_lut_f2;+ void* args[] = {(void*) &jd, (void*) &lut, (void*) &in};+ cudaLaunchKernel(k.f, grid, dim3(k.threads), args, 0, s);}static float* dev_f32(const std::vector<float>& h) {@@ -98,6 +108,17 @@ int main(int argc, char** argv) {// point table (x, y) as the CPU decoder sees itconst float* pt = tqenc::point_table();float2* lut; CK(cudaMalloc(&lut, 2048 * sizeof(float2))); CK(cudaMemcpy(lut, pt, 2048 * sizeof(float2), cudaMemcpyHostToDevice));+ g_lut_f2 = lut;+ { // half2 table: must be exact (the claim the tensor-core kernels rest on)+ std::vector<uint32_t> hl(2048); int inexact = 0;+ for (int i = 0; i < 2048; ++i) {+ const uint16_t a = tqenc::float_to_half(pt[2 * i]), b = tqenc::float_to_half(pt[2 * i + 1]);+ inexact += tqenc::half_to_float(a) != pt[2 * i] || tqenc::half_to_float(b) != pt[2 * i + 1];+ hl[i] = a | ((uint32_t) b << 16);+ }+ std::printf("fp16 point table inexact entries: %d\n", inexact);+ uint32_t* d; CK(cudaMalloc(&d, 8192)); CK(cudaMemcpy(d, hl.data(), 8192, cudaMemcpyHostToDevice)); g_lut_u32 = d;+ }// streaming bandwidth baseline: read-only sum over 512 MB{@@ -144,7 +165,7 @@ int main(int argc, char** argv) {fails += bad != 0;// ---- parity 2: GEMV vs double reference on the decoded weights, rows x in smallfor (int nb : NBs) {- const int rows = 96, in = 640, nblk = in / 128;+ const int rows = 256, in = 768, nblk = in / 128;std::vector<uint8_t> mw; fill_random(mw, K, (int64_t) rows * nblk);std::vector<float> dec((size_t) rows * in); tqenc::decode_blocks(type, mw.data(), (int64_t) rows * nblk, dec.data());std::vector<float> x((size_t) nb * in); for (auto& v : x) v = frnd();@@ -152,8 +173,13 @@ int main(int argc, char** argv) {for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); } ref[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }uint8_t* mwd; CK(cudaMalloc(&mwd, mw.size())); CK(cudaMemcpy(mwd, mw.data(), mw.size(), cudaMemcpyHostToDevice));float* xd = dev_f32(x); float* yd; CK(cudaMalloc(&yd, (size_t) nb * rows * 4));- Job j{mwd, xd, yd, rows, in, rows}; Job* jd; CK(cudaMalloc(&jd, sizeof(Job))); CK(cudaMemcpy(jd, &j, sizeof(Job), cudaMemcpyHostToDevice));- launch(pick_k(K, nb, variant), variant, jd, 1, rows, in, lut); CK(cudaGetLastError());+ const KFn kf = pick_k(K, nb, variant); const uint64_t* scd = nullptr; uint8_t* wd = mwd;+ if (kf.tql) { std::vector<uint32_t> words; std::vector<uint64_t> sc; repack_tql(K, mw.data(), rows, nblk, words, sc);+ CK(cudaMalloc(&wd, words.size() * 4)); CK(cudaMemcpy(wd, words.data(), words.size() * 4, cudaMemcpyHostToDevice));+ uint64_t* t; CK(cudaMalloc(&t, sc.size() * 8)); CK(cudaMemcpy(t, sc.data(), sc.size() * 8, cudaMemcpyHostToDevice)); scd = t; }+ if (kf.tql) { const size_t n = (size_t) nb * (in / 128) * 32; uint4* xb; CK(cudaMalloc(&xb, n * 16)); prep_b<<<(int) ((n + 255) / 256), 256>>>(xd, in, nb, in, xb); CK(cudaDeviceSynchronize()); xd = (float*) xb; }+ Job j{wd, xd, yd, rows, in, rows, scd, in}; Job* jd; CK(cudaMalloc(&jd, sizeof(Job))); CK(cudaMemcpy(jd, &j, sizeof(Job), cudaMemcpyHostToDevice));+ launch(kf, jd, 1, rows, in); CK(cudaGetLastError());std::vector<float> y((size_t) nb * rows); CK(cudaMemcpy(y.data(), yd, y.size() * 4, cudaMemcpyDeviceToHost));double worst = 0; for (size_t i = 0; i < y.size(); ++i) worst = std::max(worst, std::fabs(y[i] - ref[i]) / (refabs[i] + 1e-30));const bool ok = worst < 2e-6;@@ -183,28 +209,41 @@ int main(int argc, char** argv) {for (int m = 0; m < 2; ++m) {uint8_t* p = big + (size_t) (e * 2 + m) * mat_gu;CK(cudaMemcpy(p, hw.data(), mat_gu, cudaMemcpyHostToDevice));- hj_gu.push_back(Job{p, xg_d, yg_d + (size_t) (e * 2 + m) * maxnb * rows_gu, rows_gu, in_gu, rows_gu});+ hj_gu.push_back(Job{p, xg_d, yg_d + (size_t) (e * 2 + m) * maxnb * rows_gu, rows_gu, in_gu, rows_gu, (const uint64_t*) (p + (size_t) rows_gu * (in_gu / 128) * 4 * K)});}uint8_t* p = big + 2 * E * mat_gu + (size_t) e * mat_dn;std::vector<uint8_t> hd; fill_random(hd, K, (int64_t) rows_dn * (in_dn / 128));CK(cudaMemcpy(p, hd.data(), mat_dn, cudaMemcpyHostToDevice));- hj_dn.push_back(Job{p, xdn_d + (size_t) e * maxnb * in_dn, yd_d + (size_t) e * maxnb * rows_dn, rows_dn, in_dn, rows_dn});+ hj_dn.push_back(Job{p, xdn_d + (size_t) e * maxnb * in_dn, yd_d + (size_t) e * maxnb * rows_dn, rows_dn, in_dn, rows_dn, (const uint64_t*) (p + (size_t) rows_dn * (in_dn / 128) * 4 * K)});}Job *jgu, *jdn; CK(cudaMalloc(&jgu, sizeof(Job) * hj_gu.size())); CK(cudaMalloc(&jdn, sizeof(Job) * hj_dn.size()));CK(cudaMemcpy(jgu, hj_gu.data(), sizeof(Job) * hj_gu.size(), cudaMemcpyHostToDevice)); CK(cudaMemcpy(jdn, hj_dn.data(), sizeof(Job) * hj_dn.size(), cudaMemcpyHostToDevice));const double bytes_layer = (double) (mat_gu * 2 + mat_dn) * E, weights_layer = (double) (rows_gu * (double) in_gu * 2 + rows_dn * (double) in_dn) * E;// flush L2 between reps would only matter for tiny sets: 6 x 3 matrices exceed the 4 MB L2 many times over.for (int nb : NBs) {- gemv_fn f = pick_k(K, nb, variant);+ KFn f = pick_k(K, nb, variant);cudaEvent_t e0, e1; cudaEventCreate(&e0); cudaEventCreate(&e1);- for (int w = 0; w < 5; ++w) { launch(f, variant, jgu, 2 * E, rows_gu, in_gu, lut); launch(f, variant, jdn, E, rows_dn, in_dn, lut); }+ Job* jall = nullptr; uint4* xb_gu = nullptr; uint4* xb_dn = nullptr;+ if (f.tql) { // one launch for all 18 matrices; activations prepared once (prep_b)+ const size_t gu_n = (size_t) nb * (in_gu / 128) * 32, dn_n = (size_t) nb * (in_dn / 128) * 32;+ CK(cudaMalloc(&xb_gu, gu_n * 16)); CK(cudaMalloc(&xb_dn, dn_n * 16 * E));+ prep_b<<<(int) ((gu_n + 255) / 256), 256>>>(xg_d, in_gu, nb, in_gu, xb_gu);+ for (int e = 0; e < E; ++e) prep_b<<<(int) ((dn_n + 255) / 256), 256>>>(xdn_d + (size_t) e * maxnb * in_dn, in_dn, nb, in_dn, xb_dn + e * dn_n);+ std::vector<Job> all = hj_gu; all.insert(all.end(), hj_dn.begin(), hj_dn.end());+ for (size_t i = 0; i < all.size(); ++i) { const bool gu = i < hj_gu.size(); all[i].in = gu ? in_gu : in_dn; all[i].x = (const float*) (gu ? xb_gu : xb_dn + (i - hj_gu.size()) * dn_n); }+ CK(cudaMalloc(&jall, sizeof(Job) * all.size())); CK(cudaMemcpy(jall, all.data(), sizeof(Job) * all.size(), cudaMemcpyHostToDevice));+ CK(cudaDeviceSynchronize());+ }+ auto one = [&]() { if (f.tql) launch(f, jall, 3 * E, rows_dn, 0); else { launch(f, jgu, 2 * E, rows_gu, in_gu); launch(f, jdn, E, rows_dn, in_dn); } };+ for (int w = 0; w < 5; ++w) one();CK(cudaDeviceSynchronize()); cudaEventRecord(e0);- for (int r = 0; r < reps; ++r) { launch(f, variant, jgu, 2 * E, rows_gu, in_gu, lut); launch(f, variant, jdn, E, rows_dn, in_dn, lut); }+ for (int r = 0; r < reps; ++r) one();cudaEventRecord(e1); CK(cudaEventSynchronize(e1)); CK(cudaGetLastError());float ms; cudaEventElapsedTime(&ms, e0, e1); const double us = ms * 1e3 / reps;const double gbs = bytes_layer / (us * 1e-6) / 1e9, roof_us = bytes_layer / 448e9 * 1e6;std::printf("bench nb=%d: %8.1f us / layer (top-6 gate+up+down) %6.1f GB/s %6.2f Gweights/s roof(448 GB/s) %7.1f us = %4.1f%% of roof | 43 layers: %.2f ms\n",nb, us, gbs, weights_layer / (us * 1e-6) / 1e9, roof_us, 100 * roof_us / us, us * 43 / 1e3);+ if (jall) { cudaFree(jall); cudaFree(xb_gu); cudaFree(xb_dn); }}cudaFree(big); cudaFree(jgu); cudaFree(jdn); cudaFree(xg_d); cudaFree(xdn_d); cudaFree(yg_d); cudaFree(yd_d);}diff --git a/tools/dsv41_lab/tqk_repack.hpp b/tools/dsv41_lab/tqk_repack.hppnew file mode 100644index 000000000..405375d9d--- /dev/null+++ b/tools/dsv41_lab/tqk_repack.hpp@@ -0,0 +1,50 @@+// tools/dsv41_lab/tqk_repack.hpp - host reference of the "TQL" kernel layout of a trellis matrix (lossless bit permutation of+// the file layout; rows % 16 == 0, blocks per row even). See tqk_gemv3.cuh for why.+//+// A tile = 16 rows, a pair P = blocks 2P, 2P+1. Lane (g = lane/4, q = lane%4) of the mma tile owns, of each of the 4+// (block bk, row g + 8 h) stream chunks, the 8K bits at stream bit (24 - 8q) K (steps 8q .. 8q+7 of the block), chunk+// i = 2 bk + h. Its 4 chunks are 32 K bits = K words: words[((tile * NP + P) * K + j) * 32 + lane]. The scales:+// sc[(tile * NP + P) * 8 + g] = 4 x u16, slot 2 bk + h = fp16 d of (row g + 8 h, block 2P + bk).+#pragma once+#include <cstdint>+#include <cstring>+#include <vector>++namespace tqlab {++// file layout -> TQL: words (K * 32 per tile-pair) and scales (8 u64 per tile-pair)+inline void repack_tql(int K, const uint8_t* w, int rows, int nblk, std::vector<uint32_t>& words, std::vector<uint64_t>& sc) {+ const int bsz = 2 + 4 * K, NP = nblk / 2, NT = rows / 16;+ words.assign((size_t) NT * NP * K * 32, 0u);+ sc.assign((size_t) NT * NP * 8, 0ull);+ for (int t = 0; t < NT; ++t)+ for (int P = 0; P < NP; ++P) {+ const size_t base = (size_t) t * NP + P;+ for (int g = 0; g < 8; ++g) {+ uint64_t dq = 0;+ for (int c = 0; c < 4; ++c) {+ const int bk = c >> 1, h = c & 1;+ const uint8_t* blk = w + ((size_t) (t * 16 + g + 8 * h) * nblk + 2 * P + bk) * bsz;+ uint16_t d;+ std::memcpy(&d, blk, 2);+ dq |= (uint64_t) d << (16 * c);+ }+ sc[base * 8 + g] = dq;+ for (int q = 0; q < 4; ++q) {+ const int lane = 4 * g + q;+ for (int i = 0; i < 4; ++i) {+ const int bk = i >> 1, h = i & 1;+ const uint8_t* blk = w + ((size_t) (t * 16 + g + 8 * h) * nblk + 2 * P + bk) * bsz;+ for (int b = 0; b < 8 * K; ++b) { // chunk bit b -> record bit 8 K i + b+ const int src = (24 - 8 * q) * K + b;+ const uint32_t bit = (blk[2 + (src >> 3)] >> (src & 7)) & 1u;+ const int dst = 8 * K * i + b;+ words[(base * K + (dst >> 5)) * 32 + lane] |= bit << (dst & 31);+ }+ }+ }+ }+ }+}++} // namespace tqlab