LLM inference in C/C++
C++ ggml
vulkan: handle larger batch sizes (>4) efficiently for IQ3_S mat-vec (#27449)
Laurent Zuijdwijkba8818cbf3ad
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.compindex 5cdf2a89d0..42f52b4a12 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp@@ -7,7 +7,14 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];-void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {+// invocations per superblock. with many columns, 8 invocations need too many+// registers and spill, so use 16 to halve the per-invocation B working set+const uint TPB = NUM_COLS <= 4 ? 8 : 16;+const uint NL = 32 / TPB; // l steps per invocation++void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {+ const uint ib32 = itid / (TPB / 8);+ const uint l0 = (itid % (TPB / 8)) * NL;const uint y_idx = i * QUANT_K + 32 * ib32;uint ibi = a_offset + first_row * num_blocks_per_row + i;@@ -16,11 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,const uint scale = (data_a[ibi].scales[ib32/2] >> (4 * (ib32 & 1))) & 0xF;const float dscale = d * (1 + 2 * scale);const uint qh = data_a[ibi].qh[ib32];- FLOAT_TYPE sum[NUM_COLS];- [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {- sum[j] = 0.0;- }- [[unroll]] for (uint l = 0; l < 4; ++l) {+ [[unroll]] for (uint ll = 0; ll < NL; ++ll) {+ const uint l = l0 + ll;const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147const uint sign = data_a[ibi].signs[4 * ib32 + l];const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));@@ -30,7 +34,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);- sum[j] =+ const FLOAT_TYPE sum =fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),@@ -39,12 +43,11 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),- sum[j]))))))));+ FLOAT_TYPE(0.0)))))))));++ temp[j][n] = fma(dscale, sum, temp[j][n]);}}- [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {- temp[j][n] = fma(dscale, sum[j], temp[j][n]);- }ibi += num_blocks_per_row;}}@@ -55,11 +58,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {const uint num_blocks_per_row = p.ncols / QUANT_K;- // 8 threads are used to process each block- const uint blocks_per_wg = gl_WorkGroupSize.x/8;+ // TPB invocations are used to process each block+ const uint blocks_per_wg = gl_WorkGroupSize.x/TPB;const uint tid = gl_LocalInvocationID.x;- const uint itid = tid % 8; // 0...7- const uint ix = tid / 8;+ const uint itid = tid % TPB;+ const uint ix = tid / TPB;[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex dc28aba2d8..4d9808562d 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9395,6 +9395,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {//test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 18, i, 32*256, { 1, 1}, {8, 1}));//test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 19, i, 33*256, { 1, 1}, {1, 1}));}+ // mat-vec shaders split k across lanes and loop over the blocks in strides. k must be+ // long enough that the loop wraps, else the stride is never exercised+ test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 1, 16*256, { 1, 1}, {1, 1}));+ test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1}));}test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));