LLM inference in C/C++
C++ ggml
vulkan: add SHMEM_STRIDE_PAD/APPLY_SLM_A_RESHAPE for coopmat1 on Intel Xe (#25380)
Jiang, Fish9b0a2ce859d3
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 7d72ae34fb..585e10d45a 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -3962,7 +3962,10 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec}// Needs to be kept up to date on shader changes- const uint32_t bank_conflict_offset = device->coopmat_support ? 8 : 1;+ // Needs to stay aligned with ggml_vk_mul_mm_spec.+ const bool intel_shmem_stride_pad_zero = device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support &&+ device->driver_id == vk::DriverId::eIntelProprietaryWindows;+ const uint32_t bank_conflict_offset = intel_shmem_stride_pad_zero ? 0 : (device->coopmat_support ? 8 : 1);const uint32_t type_size = device->fp16 ? sizeof(ggml_fp16_t) : sizeof(float);const uint32_t warps = warptile[0] / warptile[10];@@ -4579,8 +4582,13 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {}#endif- auto const &ggml_vk_mul_mm_spec = [](std::vector<uint32_t> spec, bool aligned) {- spec.push_back(aligned ? 1u : 0u);+ auto const &ggml_vk_mul_mm_spec = [&device](std::vector<uint32_t> spec, bool aligned) {+ spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED+ if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support &&+ device->driver_id == vk::DriverId::eIntelProprietaryWindows) {+ spec.push_back(0u); // constantID=12: SHMEM_STRIDE_PAD = 0+ spec.push_back(1u); // constantID=13: APPLY_SLM_A_RESHAPE = true+ }return spec;};diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.compindex 57c0410e45..3df88044a5 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp@@ -119,10 +119,13 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit#endif#ifdef COOPMAT-#define SHMEM_STRIDE (BK / 2 + 4)+layout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4;+layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false;#else-#define SHMEM_STRIDE (BK / 2 + 1)+const uint SHMEM_STRIDE_PAD = 1;+const bool APPLY_SLM_A_RESHAPE = false;#endif+#define SHMEM_STRIDE (BK / 2 + SHMEM_STRIDE_PAD)shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE];@@ -302,7 +305,7 @@ void main() {[[unroll]] for (uint i = 0; i < BK; i += TK) {[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {// Load from shared into cache- coopMatLoad(cache_a, buf_a, (warp_r * WM + cm_row * TM) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutRowMajor);+ coopMatLoad(cache_a, buf_a, a_shmem_index(warp_r * WM + cm_row * TM, i / 2), a_shmem_stride(), gl_CooperativeMatrixLayoutRowMajor);[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {coopMatLoad(cache_b, buf_b, (warp_c * WN + cm_col * TN) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor);diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glslindex 63af2ce685..7d852dced8 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl@@ -1,60 +1,76 @@+// k_pair is the K coordinate measured in FLOAT_TYPEV2 elements.+uint a_shmem_index(uint m, uint k_pair) {+ if (APPLY_SLM_A_RESHAPE) {+ const uint tile_width = TK / 2;+ return (k_pair / tile_width) * BM * tile_width+ + m * tile_width+ + k_pair % tile_width;+ }+ return m * SHMEM_STRIDE + k_pair;+}++uint a_shmem_stride() {+ return APPLY_SLM_A_RESHAPE ? TK / 2 : SHMEM_STRIDE;+}++void store_a(uint m, uint k_pair, FLOAT_TYPEV2 value) {+ buf_a[a_shmem_index(m, k_pair)] = value;+}+void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) {#if defined(DATA_A_F32) || defined(DATA_A_F16)#if LOAD_VEC_A == 8if (ALIGNED != 0) {const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;+ const uint k_pair = row * LOAD_VEC_A / 2;FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]);- buf_a[buf_idx ] = aa[0].xy;- buf_a[buf_idx + 1] = aa[0].zw;- buf_a[buf_idx + 2] = aa[1].xy;- buf_a[buf_idx + 3] = aa[1].zw;+ store_a(col, k_pair, aa[0].xy);+ store_a(col, k_pair + 1, aa[0].zw);+ store_a(col, k_pair + 2, aa[1].xy);+ store_a(col, k_pair + 3, aa[1].zw);return;}#elif LOAD_VEC_A == 4if (ALIGNED != 0) {const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;+ const uint k_pair = row * LOAD_VEC_A / 2;FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]);- buf_a[buf_idx ] = aa.xy;- buf_a[buf_idx + 1] = aa.zw;+ store_a(col, k_pair, aa.xy);+ store_a(col, k_pair + 1, aa.zw);return;}#endifconst uint idx = pos_a + col * p.stride_a + row * 2;- const uint buf_idx = col * SHMEM_STRIDE + row;if (idx_m < p.M && block + row * 2 + 1 < end_k) {- buf_a[buf_idx] = FLOAT_TYPEV2(data_a_scalar[idx],- data_a_scalar[idx + 1]);+ store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx],+ data_a_scalar[idx + 1]));} else if (idx_m < p.M && block + row * 2 < end_k) {- buf_a[buf_idx] = FLOAT_TYPEV2(data_a_scalar[idx], 0.0f);+ store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f));} else {- buf_a[buf_idx] = FLOAT_TYPEV2(0.0f);+ store_a(col, row, FLOAT_TYPEV2(0.0f));}#elif defined(DATA_A_BF16)#if LOAD_VEC_A == 4if (ALIGNED != 0) {const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;+ const uint k_pair = row * LOAD_VEC_A / 2;FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx]));- buf_a[buf_idx ] = aa.xy;- buf_a[buf_idx + 1] = aa.zw;+ store_a(col, k_pair, aa.xy);+ store_a(col, k_pair + 1, aa.zw);return;}#endifconst uint idx = pos_a + col * p.stride_a + row * 2;- const uint buf_idx = col * SHMEM_STRIDE + row;if (idx_m < p.M && block + row * 2 + 1 < end_k) {- buf_a[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]),- TO_FLOAT_TYPE(data_a_scalar[idx + 1]));+ store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]),+ TO_FLOAT_TYPE(data_a_scalar[idx + 1])));} else if (idx_m < p.M && block + row * 2 < end_k) {- buf_a[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f);+ store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f));} else {- buf_a[buf_idx] = FLOAT_TYPEV2(0.0f);+ store_a(col, row, FLOAT_TYPEV2(0.0f));}#elif defined(DATA_A_Q4_0)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;const uint ib = idx / 4;const uint iqs = idx & 0x03;@@ -64,13 +80,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d;const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d;- buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xy);- buf_a[buf_idx + 1] = FLOAT_TYPEV2(v0.zw);- buf_a[buf_idx + 8] = FLOAT_TYPEV2(v1.xy);- buf_a[buf_idx + 9] = FLOAT_TYPEV2(v1.zw);+ const uint k_pair = row * LOAD_VEC_A / 4;+ store_a(col, k_pair, FLOAT_TYPEV2(v0.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy));+ store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw));#elif defined(DATA_A_Q4_1)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;const uint ib = idx / 4;const uint iqs = idx & 0x03;@@ -80,13 +96,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y;const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y;- buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xy);- buf_a[buf_idx + 1 ] = FLOAT_TYPEV2(v0.zw);- buf_a[buf_idx + 8 ] = FLOAT_TYPEV2(v1.xy);- buf_a[buf_idx + 9 ] = FLOAT_TYPEV2(v1.zw);+ const uint k_pair = row * LOAD_VEC_A / 4;+ store_a(col, k_pair, FLOAT_TYPEV2(v0.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy));+ store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw));#elif defined(DATA_A_Q5_0)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;const uint ib = idx / 8;const uint iqs = idx & 0x07;@@ -98,12 +114,10 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst uint vui = uint(data_a_packed16[ib].qs[iqs]);const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d;-- buf_a[buf_idx ] = FLOAT_TYPEV2(v.xz);- buf_a[buf_idx + 8] = FLOAT_TYPEV2(v.yw);+ store_a(col, row, FLOAT_TYPEV2(v.xz));+ store_a(col, row + 8, FLOAT_TYPEV2(v.yw));#elif defined(DATA_A_Q5_1)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;const uint ib = idx / 4;const uint iqs = idx & 0x03;@@ -119,13 +133,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y;const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y;- buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xz);- buf_a[buf_idx + 1] = FLOAT_TYPEV2(v1.xz);- buf_a[buf_idx + 8] = FLOAT_TYPEV2(v0.yw);- buf_a[buf_idx + 9] = FLOAT_TYPEV2(v1.yw);+ const uint k_pair = row * LOAD_VEC_A / 4;+ store_a(col, k_pair, FLOAT_TYPEV2(v0.xz));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw));+ store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw));#elif defined(DATA_A_Q8_0)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 8;const uint iqs = idx & 0x07;@@ -135,11 +149,11 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy;const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;- buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);- buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));#elif defined(DATA_A_Q1_0)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 16;const uint iqs = idx & 0xfu;@@ -147,13 +161,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst float d = float(data_a[ib].d);const uint bits = uint(data_a[ib].qs[iqs]);- buf_a[buf_idx ] = FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d);- buf_a[buf_idx + 1] = FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d);- buf_a[buf_idx + 2] = FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d);- buf_a[buf_idx + 3] = FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d));+ store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d));+ store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d));+ store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d));#elif defined(DATA_A_Q2_0)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 16;const uint iqs = idx & 0xfu;@@ -161,11 +175,11 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst FLOAT_TYPE d = FLOAT_TYPE(data_a[ib].d);const uint bits = uint(data_a[ib].qs[iqs]);- buf_a[buf_idx ] = d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f));- buf_a[buf_idx + 1] = d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f));+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f)));+ store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f)));#elif defined(DATA_A_Q2_K)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 64; // 4 values per idxconst uint iqs = (idx % 64) * 2; // 0,2,4..126@@ -180,11 +194,11 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4);- buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);- buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));#elif defined(DATA_A_TQ2_0)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 128; // 2 values per idxconst uint iqs = (idx % 128) * 2; // elem 0,2,4..254@@ -197,10 +211,10 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec2 v = d * (vec2((qs >> shift) & 3) - 1.0);- buf_a[buf_idx] = FLOAT_TYPEV2(v.xy);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));#elif defined(DATA_A_Q3_K)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 128; // 2 values per idxconst uint iqs = idx % 128; // 0..127@@ -220,11 +234,10 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy);const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy);- buf_a[buf_idx] = FLOAT_TYPEV2(dl * (qs.x - hm.x),- dl * (qs.y - hm.y));+ store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(dl * (qs.x - hm.x),+ dl * (qs.y - hm.y)));#elif defined(DATA_A_Q4_K)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 64; // 4 values per idxconst uint iqs = (idx % 64) * 2; // 0,2,4..126@@ -256,11 +269,11 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F));- buf_a[buf_idx ] = FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m));- buf_a[buf_idx + 1] = FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m));+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m)));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m)));#elif defined(DATA_A_Q5_K)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 64; // 4 values per idxconst uint iqs = (idx % 64) * 2; // 0,2,4..126@@ -295,11 +308,11 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4;const vec4 q = vec4(unpack8(qs | qh));- buf_a[buf_idx ] = FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m));- buf_a[buf_idx + 1] = FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m));+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m)));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m)));#elif defined(DATA_A_Q6_K)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 128; // 2 values per idxconst uint iqs = idx % 128; // 0..127@@ -318,10 +331,9 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303;const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale;- buf_a[buf_idx] = FLOAT_TYPEV2(q.x, q.y);+ store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(q.x, q.y));#elif defined(DATA_A_IQ1_S)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 32; // 8 values per idxconst uint ib32 = (idx % 32) / 4; // 0..7@@ -334,13 +346,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]);+ const uint k_pair = row * LOAD_VEC_A / 2;[[unroll]] for (int k = 0; k < 4; ++k) {- buf_a[buf_idx + k] = FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),- dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta));+ store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),+ dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta)));}#elif defined(DATA_A_IQ1_M)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 32; // 8 values per idxconst uint ib8 = idx % 32;@@ -356,13 +368,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]);+ const uint k_pair = row * LOAD_VEC_A / 2;[[unroll]] for (int k = 0; k < 4; ++k) {- buf_a[buf_idx + k] = FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),- dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta));+ store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),+ dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta)));}#elif defined(DATA_A_IQ2_XXS)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 32; // 8 values per idxconst uint ib32 = (idx % 32) / 4; // 0..7@@ -383,17 +395,17 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 grid0 = vec4(unpack8(grid.x));const vec4 grid1 = vec4(unpack8(grid.y));- buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,- (sign & 2) != 0 ? -grid0.y : grid0.y);- buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,- (sign & 8) != 0 ? -grid0.w : grid0.w);- buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,- (sign & 32) != 0 ? -grid1.y : grid1.y);- buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,- (sign & 128) != 0 ? -grid1.w : grid1.w);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,+ (sign & 2) != 0 ? -grid0.y : grid0.y));+ store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,+ (sign & 8) != 0 ? -grid0.w : grid0.w));+ store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,+ (sign & 32) != 0 ? -grid1.y : grid1.y));+ store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,+ (sign & 128) != 0 ? -grid1.w : grid1.w));#elif defined(DATA_A_IQ2_XS)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 32; // 8 values per idxconst uint ib32 = (idx % 32) / 4; // 0..7@@ -409,17 +421,17 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 grid0 = vec4(unpack8(grid.x));const vec4 grid1 = vec4(unpack8(grid.y));- buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,- (sign & 2) != 0 ? -grid0.y : grid0.y);- buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,- (sign & 8) != 0 ? -grid0.w : grid0.w);- buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,- (sign & 32) != 0 ? -grid1.y : grid1.y);- buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,- (sign & 128) != 0 ? -grid1.w : grid1.w);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,+ (sign & 2) != 0 ? -grid0.y : grid0.y));+ store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,+ (sign & 8) != 0 ? -grid0.w : grid0.w));+ store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,+ (sign & 32) != 0 ? -grid1.y : grid1.y));+ store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,+ (sign & 128) != 0 ? -grid1.w : grid1.w));#elif defined(DATA_A_IQ2_S)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 32; // 8 values per idxconst uint ib8 = idx % 32; // 0..31@@ -437,17 +449,17 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst vec4 grid0 = vec4(unpack8(grid.x));const vec4 grid1 = vec4(unpack8(grid.y));- buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,- (sign & 2) != 0 ? -grid0.y : grid0.y);- buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,- (sign & 8) != 0 ? -grid0.w : grid0.w);- buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,- (sign & 32) != 0 ? -grid1.y : grid1.y);- buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,- (sign & 128) != 0 ? -grid1.w : grid1.w);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,+ (sign & 2) != 0 ? -grid0.y : grid0.y));+ store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,+ (sign & 8) != 0 ? -grid0.w : grid0.w));+ store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,+ (sign & 32) != 0 ? -grid1.y : grid1.y));+ store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,+ (sign & 128) != 0 ? -grid1.w : grid1.w));#elif defined(DATA_A_IQ3_XXS)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 64; // 4 values per idxconst uint iqs = idx % 64; // 0..63@@ -465,13 +477,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst uint grid = iq3xxs_grid[qs];const vec4 v = db * vec4(unpack8(grid));- buf_a[buf_idx ] = FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,- (sign & 2) != 0 ? -v.y : v.y);- buf_a[buf_idx + 1] = FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,- (sign & 8) != 0 ? -v.w : v.w);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,+ (sign & 2) != 0 ? -v.y : v.y));+ store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,+ (sign & 8) != 0 ? -v.w : v.w));#elif defined(DATA_A_IQ3_S)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 64; // 4 values per idxconst uint iqs = idx % 64; // 0..63@@ -487,13 +499,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];const vec4 v = db * vec4(unpack8(grid));- buf_a[buf_idx ] = FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,- (sign & 2) != 0 ? -v.y : v.y);- buf_a[buf_idx + 1] = FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,- (sign & 8) != 0 ? -v.w : v.w);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,+ (sign & 2) != 0 ? -v.y : v.y));+ store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,+ (sign & 8) != 0 ? -v.w : v.w));#elif defined(DATA_A_IQ4_XS)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;const uint ib = idx / 64; // 4 values per idxconst uint ib32 = (idx % 64) / 8; // 0..7@@ -507,11 +519,11 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst float d = float(data_a[ib].d);const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]);- buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);- buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);+ const uint k_pair = row * LOAD_VEC_A / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));#elif defined(DATA_A_IQ4_NL)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;const uint ib = idx / 8;const uint iqs = idx & 0x07;@@ -519,13 +531,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uinconst FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d);const uint vui = uint(data_a_packed16[ib].qs[iqs]);- buf_a[buf_idx ] = d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],- kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]);- buf_a[buf_idx + 8] = d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)],- kvalues_iq4nl[vui >> 12]);+ const uint k_pair = row * LOAD_VEC_A / 4;+ store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],+ kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]));+ store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)],+ kvalues_iq4nl[vui >> 12]));#elif defined(DATA_A_MXFP4)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;const uint ib = idx / 8;const uint iqs = (idx & 0x07) * 2;@@ -536,38 +548,37 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin#ifdef USE_OCP_FP4const float d = e8m0_to_fp32(data_a[ib].e);const u8vec2 packed = u8vec2(vui, vui2);- buf_a[buf_idx ] = FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d);- buf_a[buf_idx + 8] = FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d);+ store_a(col, row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d));+ store_a(col, row + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d));#elseconst float d = e8m0_to_fp32(data_a[ib].e) * 0.5;- buf_a[buf_idx ] = FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,- kvalues_mxfp4[vui2 & 0xF] * d);- buf_a[buf_idx + 8] = FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,- kvalues_mxfp4[vui2 >> 4] * d);+ store_a(col, row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,+ kvalues_mxfp4[vui2 & 0xF] * d));+ store_a(col, row + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,+ kvalues_mxfp4[vui2 >> 4] * d));#endif#elif defined(DATA_A_NVFP4)const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- // lo and hi nibbles are 8 elements apart, which doesn't quite line up with- // how the thread mapping and buf_idx calculation works for other types.- const uint buf_idx = col * SHMEM_STRIDE + (row & 3) + (row & ~3) * 2;-const uint ib = idx / 16u;const uint sub = (idx & 0xC) >> 2;const uint iqs = (idx & 0xF) * 2;const uint vui = uint(data_a[ib].qs[iqs]);const uint vui2 = uint(data_a[ib].qs[iqs+1]);+ // lo and hi nibbles are 8 elements apart, which doesn't quite line up with+ // how the thread mapping and buf_idx calculation works for other types.+ const uint eff_row = (row & 3) + (row & ~3) * 2;#ifdef USE_OCP_FP4const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub]));const u8vec2 packed = u8vec2(vui, vui2);- buf_a[buf_idx ] = FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d;- buf_a[buf_idx + 4] = FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d;+ store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d);+ store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d);#elseconst float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5;- buf_a[buf_idx ] = FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,- kvalues_mxfp4[vui2 & 0xF] * d);- buf_a[buf_idx + 4] = FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,- kvalues_mxfp4[vui2 >> 4] * d);+ store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,+ kvalues_mxfp4[vui2 & 0xF] * d));+ store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,+ kvalues_mxfp4[vui2 >> 4] * d));#endif#endif}