LLM inference in C/C++
C++ ggml
vulkan: use spec constant for matrix matrix multiplication A-type (#25773)
Ruben Ortlam91f6a6cf3613
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 9eae8dab9c..235b35c6ce 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -258,27 +258,34 @@ typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref;static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);-struct vk_matmul_pipeline_struct {- vk_pipeline l, m, s;- vk_pipeline a_l, a_m, a_s;- // Returns true when all unaligned pipelines are null.- // We only check for unaligned variants since one of the unaligned pipelines must exist- // while aligned pipelines are optional- bool is_empty() const {- return l == nullptr && m == nullptr && s == nullptr;+struct vk_matmul_pipeline_key {+ ggml_type type_a;+ ggml_type type_b;+ bool mul_mat_id;+ bool f16acc;++ bool operator<(const vk_matmul_pipeline_key & o) const {+ return std::tie(type_a, type_b, mul_mat_id, f16acc)+ < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);}};-typedef std::shared_ptr<vk_matmul_pipeline_struct> vk_matmul_pipeline;-struct vk_matmul_pipeline2 {- vk_matmul_pipeline2() {- f16acc = std::make_shared<vk_matmul_pipeline_struct>();- f32acc = std::make_shared<vk_matmul_pipeline_struct>();- }- vk_matmul_pipeline f32acc;- vk_matmul_pipeline f16acc;+struct vk_matmul_pipeline_pair {+ vk_pipeline unaligned;+ vk_pipeline aligned;+ uint32_t align;+};++struct vk_tile_config {+ std::vector<uint32_t> warptile;+ std::array<uint32_t, 3> wg_denoms;+ uint32_t align;};+using matmul_tile_selector_t = std::function<uint32_t(+ uint32_t m, uint32_t n, uint32_t k, uint32_t shader_core_count,+ const std::vector<vk_matmul_pipeline_pair>& configs)>;+struct vk_device_struct;typedef std::shared_ptr<vk_device_struct> vk_device;typedef std::weak_ptr<vk_device_struct> vk_device_ref;@@ -949,24 +956,9 @@ struct vk_device_struct {vk::DescriptorSetLayout dsl;- vk_matmul_pipeline pipeline_matmul_f32 {};- vk_matmul_pipeline pipeline_matmul_f32_f16 {};- vk_matmul_pipeline pipeline_matmul_bf16 {};- vk_matmul_pipeline2 pipeline_matmul_f16;- vk_matmul_pipeline2 pipeline_matmul_f16_f32;-- vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat[GGML_TYPE_COUNT];- vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_COUNT];- vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_COUNT];-- vk_matmul_pipeline pipeline_matmul_id_f32 {};- vk_matmul_pipeline pipeline_matmul_id_bf16 {};- vk_matmul_pipeline2 pipeline_matmul_id_f16;- vk_matmul_pipeline2 pipeline_matmul_id_f16_f32;-- vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id[GGML_TYPE_COUNT];- vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_COUNT]; // f16 B-type variant (coopmat1 only)- vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_COUNT];+ std::map<vk_matmul_pipeline_key, std::vector<vk_matmul_pipeline_pair>> pipeline_matmul;+ matmul_tile_selector_t matmul_tile_selector;+ matmul_tile_selector_t matmul_id_tile_selector;vk_pipeline pipeline_matmul_split_k_reduce;vk_pipeline pipeline_quantize_q8_1_x4;@@ -4411,8 +4403,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {// Warptile layout (indices match mul_mm.comp constantIDs):// [0..9] : BLOCK_SIZE, BM, BN, BK, WM, WN, WMITER, TM, TN, TK// [10] : WARP / required_subgroup_size (read via WARP_SIZE_IDX)- // [11] : ALIGNED (appended by ggml_vk_mul_mm_spec)- // [12,13] : SHMEM_STRIDE_PAD, APPLY_SLM_A_RESHAPEstatic constexpr size_t WARP_SIZE_IDX = 10;std::vector<uint32_t> l_warptile, m_warptile, s_warptile,l_warptile_id, m_warptile_id, s_warptile_id,@@ -4465,9 +4455,9 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {// spec constants and tile sizes for quant matmul_idconst uint32_t mmqid_bk = device->coopmat2_decode_vector ? 64u : 32u;- l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1, device->subgroup_size };- m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size };- s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0, device->subgroup_size };+ l_warptile_mmqid = { 256, 128, 128, mmqid_bk, 1 };+ m_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 };+ s_warptile_mmqid = { 256, 128, 64, mmqid_bk, 0 };l_mmqid_wg_denoms = { 128, 128, 1 };m_mmqid_wg_denoms = { 128, 64, 1 };s_mmqid_wg_denoms = { 128, 64, 1 };@@ -4618,22 +4608,6 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {}}- if (!device->pipeline_matmul_f32) {- device->pipeline_matmul_f32 = std::make_shared<vk_matmul_pipeline_struct>();- }- if (!device->pipeline_matmul_f32_f16) {- device->pipeline_matmul_f32_f16 = std::make_shared<vk_matmul_pipeline_struct>();- }- if (!device->pipeline_matmul_id_f32) {- device->pipeline_matmul_id_f32 = std::make_shared<vk_matmul_pipeline_struct>();- }- if (!device->pipeline_matmul_bf16) {- device->pipeline_matmul_bf16 = std::make_shared<vk_matmul_pipeline_struct>();- }- if (!device->pipeline_matmul_id_bf16) {- device->pipeline_matmul_id_bf16 = std::make_shared<vk_matmul_pipeline_struct>();- }-auto const &ggml_vk_create_pipeline = [&](vk_device& device, vk_pipeline& base_pipeline, const char *name, size_t spv_size, const void* spv_data, const char *entrypoint,uint32_t parameter_count, uint32_t push_constant_size, std::array<uint32_t, 3> wg_denoms, const std::vector<uint32_t>& specialization_constants,uint32_t align, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0) {@@ -4842,685 +4816,639 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {return spec;};+ auto const &ggml_vk_mul_mm_spec_quant = [&device](std::vector<uint32_t> spec, bool aligned, uint32_t type) {+ spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED+ spec.push_back(type); // constantID=12: MmTypeA+ if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support &&+ device->driver_id == vk::DriverId::eIntelProprietaryWindows) {+ spec.push_back(0u); // constantID=13: SHMEM_STRIDE_PAD = 0+ spec.push_back(1u); // constantID=14: APPLY_SLM_A_RESHAPE = true+ }+ return spec;+ };++ static const ggml_type non_lut_quant_types[] = {+ GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0,+ GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0,+ };++#define FOR_EACH_LUT_TYPE_NONFP4(X) \+ X(GGML_TYPE_IQ1_S, iq1_s) \+ X(GGML_TYPE_IQ1_M, iq1_m) \+ X(GGML_TYPE_IQ2_XXS, iq2_xxs) \+ X(GGML_TYPE_IQ2_XS, iq2_xs) \+ X(GGML_TYPE_IQ2_S, iq2_s) \+ X(GGML_TYPE_IQ3_XXS, iq3_xxs) \+ X(GGML_TYPE_IQ3_S, iq3_s) \+ X(GGML_TYPE_IQ4_XS, iq4_xs) \+ X(GGML_TYPE_IQ4_NL, iq4_nl)+#define FOR_EACH_LUT_FP4_TYPE(X) \+ X(GGML_TYPE_MXFP4, mxfp4) \+ X(GGML_TYPE_NVFP4, nvfp4)+#define FOR_EACH_LUT_TYPE(X) \+ FOR_EACH_LUT_TYPE_NONFP4(X) \+ FOR_EACH_LUT_FP4_TYPE(X)+const int mul_mat_id_param_count = 5;+ using spec_fn_t = std::function<std::vector<uint32_t>(const std::vector<uint32_t>&, bool)>;+ auto const &create_mm_pipelines = [&](+ const vk_matmul_pipeline_key& key,+ const std::vector<vk_tile_config>& tile_configs,+ const std::string& shader_name, size_t spv_len, const void* spv_data,+ uint32_t push_constant_size, uint32_t param_count,+ const spec_fn_t& spec_fn,+ bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0,+ bool create_aligned = true, bool pin_subgroup_to_warp = false+ ) {+ auto& vec = device->pipeline_matmul[key];+ const bool first_call = vec.empty();+ for (size_t i = 0; i < tile_configs.size(); i++) {+ const auto& tc = tile_configs[i];++ // Intel coopmat1 pins the required subgroup size to each warptile's WARP element.+ const uint32_t rsgs = pin_subgroup_to_warp ? tc.warptile[WARP_SIZE_IDX] : required_subgroup_size;+ const bool rfs = require_full_subgroups || pin_subgroup_to_warp;++ if (first_call) {+ vk_matmul_pipeline_pair pair{};+ pair.align = tc.align;+ std::string suffix = "_" + std::to_string(i);+ pair.unaligned = std::make_shared<vk_pipeline_struct>();+ if (create_aligned) {+ pair.aligned = std::make_shared<vk_pipeline_struct>();+ }+ vec.push_back(pair);+ }++ ggml_vk_create_pipeline(device, vec[i].unaligned,+ vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),+ spv_len, spv_data, "main", param_count, push_constant_size,+ tc.wg_denoms, spec_fn(tc.warptile, false), 1,+ disable_robustness, rfs, rsgs);++ if (vec[i].aligned) {+ ggml_vk_create_pipeline(device, vec[i].aligned,+ vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(),+ spv_len, spv_data, "main", param_count, push_constant_size,+ tc.wg_denoms, spec_fn(tc.warptile, true), tc.align,+ disable_robustness, rfs, rsgs);+ }+ }+ };++ auto filter_tc = [&](const std::vector<vk_tile_config>& configs, ggml_type type, bool is_id, bool is_int = false) -> std::vector<vk_tile_config> {+ std::vector<vk_tile_config> result;+ bool enabled[3];+ if (is_int) {+ enabled[0] = is_id ? device->mul_mat_id_s_int[type] : device->mul_mat_s_int[type];+ enabled[1] = is_id ? device->mul_mat_id_m_int[type] : device->mul_mat_m_int[type];+ enabled[2] = is_id ? device->mul_mat_id_l_int[type] : device->mul_mat_l_int[type];+ } else {+ enabled[0] = is_id ? device->mul_mat_id_s[type] : device->mul_mat_s[type];+ enabled[1] = is_id ? device->mul_mat_id_m[type] : device->mul_mat_m[type];+ enabled[2] = is_id ? device->mul_mat_id_l[type] : device->mul_mat_l[type];+ }+ for (size_t i = 0; i < configs.size() && i < 3; i++) {+ if (enabled[i]) result.push_back(configs[i]);+ }+ return result;+ };++ std::vector<vk_tile_config> tc_mm = {{s_warptile, s_wg_denoms, s_align}, {m_warptile, m_wg_denoms, m_align}, {l_warptile, l_wg_denoms, l_align}};+ std::vector<vk_tile_config> tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, m_mmq_wg_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}};+#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)if (device->coopmat2) {- auto const &ggml_vk_mul_mm_cm2_spec = [](std::vector<uint32_t> spec, bool aligned, bool mul_mat_id) {- if (mul_mat_id && spec.size() > 5) {- spec.insert(spec.begin() + 5, aligned ? 1u : 0u);- } else {- spec.push_back(aligned ? 1u : 0u);- }- if (mul_mat_id && spec.size() == 6) {- spec.push_back(32);+ auto const &ggml_vk_mul_mm_cm2_spec = [&](std::vector<uint32_t> spec, bool aligned, uint32_t type = UINT32_MAX) {+ spec.push_back(aligned ? 1u : 0u); // ALIGNED+ spec.push_back(device->subgroup_size); // subgroup_size+ if (type != UINT32_MAX) {+ spec.push_back(type); // MmTypeA+ spec.push_back((uint32_t)ggml_type_size((ggml_type)type)); // MmABlockBytes}return spec;};- // Create 6 variants, {s,m,l}x{unaligned,aligned}-#define CREATE_MM(PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, false, PARAMCOUNT == mul_mat_id_param_count), 1, true); \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(l_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), l_align, true); \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(m_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), m_align, true); \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm2_len, NAMELC ## F16ACC ## _cm2_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_cm2_spec(s_ ## WARPTILE, true, PARAMCOUNT == mul_mat_id_param_count), s_align, true); \-- // Create 2 variants, {f16,f32} accumulator-#define CREATE_MM2(PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \- CREATE_MM(PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \- CREATE_MM(PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT) \-- CREATE_MM2(pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3)+ std::vector<vk_tile_config> tc_mmq_k = {{s_warptile_mmq_k, s_mmq_wg_denoms_k, s_align}, {m_warptile_mmq_k, m_mmq_wg_denoms_k, m_align}, {l_warptile_mmq_k, l_mmq_wg_denoms_k, l_align}};+ std::vector<vk_tile_config> tc_mmqid = {{s_warptile_mmqid, s_mmqid_wg_denoms, s_align}, {m_warptile_mmqid, m_mmqid_wg_denoms, m_align}, {l_warptile_mmqid, l_mmqid_wg_denoms, l_align}};++ spec_fn_t cm2_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); };++ // F16 x F16+ create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)if (device->coopmat_bf16_support) {- CREATE_MM(pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3)+ create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);}#endif- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q1_0], matmul_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_0], matmul_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_0], matmul_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_1], matmul_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_0], matmul_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_1], matmul_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q8_0], matmul_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_K], matmul_q2_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ2_0], matmul_tq2_0_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ1_0], matmul_tq1_0_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q3_K], matmul_q3_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_K], matmul_q4_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_K], matmul_q5_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q6_K], matmul_q6_k_f16, mmq_wg_denoms_k, warptile_mmq_k, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_S], matmul_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_M], matmul_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_S], matmul_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_S], matmul_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)+ for (const auto type : non_lut_quant_types) {+ // regression in unified shader on Ampere+ if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) {+ continue;+ }+ auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq;+ spec_fn_t qs = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); };+ create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);+ create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);+ }+ create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);+#define X_CM2(TYPE, tstr) \+ { auto tc = filter_tc(tc_mmq, TYPE, false); \+ if (!tc.empty()) { \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \+ } }+ FOR_EACH_LUT_TYPE_NONFP4(X_CM2)#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)if (device->ocp_fp4) {- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)+#define X_CM2_OCP(TYPE, tstr) \+ { auto tc = filter_tc(tc_mmq, TYPE, false); \+ if (!tc.empty()) { \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \+ } }+ FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP)+#undef X_CM2_OCP} else#endif{- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)- CREATE_MM2(pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3)+ FOR_EACH_LUT_FP4_TYPE(X_CM2)}+#undef X_CM2GGML_ASSERT(device->subgroup_ballot);- CREATE_MM2(pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, 5)+ create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm2_len, matmul_id_subgroup_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm2_len, matmul_id_subgroup_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)if (device->coopmat_bf16_support) {- CREATE_MM(pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, 5)+ create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm2_len, matmul_id_subgroup_bf16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);}#endif- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)+ for (const auto type : non_lut_quant_types) {+ if (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) {+ continue;+ }+ spec_fn_t qs_id = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); };+ create_mm_pipelines({type, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm2_len, matmul_id_subgroup_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true);+ create_mm_pipelines({type, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm2_len, matmul_id_subgroup_quant_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, qs_id, true);+ }+ create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q4_k_f16_f16acc", matmul_id_subgroup_q4_k_f16_f16acc_cm2_len, matmul_id_subgroup_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q4_k_f16", matmul_id_subgroup_q4_k_f16_cm2_len, matmul_id_subgroup_q4_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, true}, tc_mmqid, "matmul_id_subgroup_q5_k_f16_f16acc", matmul_id_subgroup_q5_k_f16_f16acc_cm2_len, matmul_id_subgroup_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);+ create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, true, false}, tc_mmqid, "matmul_id_subgroup_q5_k_f16", matmul_id_subgroup_q5_k_f16_cm2_len, matmul_id_subgroup_q5_k_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true);+#define X_CM2_ID(TYPE, tstr) \+ { auto tc = filter_tc(tc_mmqid, TYPE, true); \+ if (!tc.empty()) { \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm2_len, matmul_id_subgroup_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \+ } }+ FOR_EACH_LUT_TYPE_NONFP4(X_CM2_ID)#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)if (device->ocp_fp4) {- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16_ocp, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)+#define X_CM2_ID_OCP(TYPE, tstr) \+ { auto tc = filter_tc(tc_mmqid, TYPE, true); \+ if (!tc.empty()) { \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, true, true}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \+ create_mm_pipelines({TYPE, GGML_TYPE_F16, true, false}, tc, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm2_len, matmul_id_subgroup_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_id_push_constants), 5, cm2_spec, true); \+ } }+ FOR_EACH_LUT_FP4_TYPE(X_CM2_ID_OCP)+#undef X_CM2_ID_OCP} else#endif{- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)- CREATE_MM2(pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16, mmqid_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, 5)+ FOR_EACH_LUT_FP4_TYPE(X_CM2_ID)}-#undef CREATE_MM-#undef CREATE_MM2+#undef X_CM2_ID} else#endif // defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)if (device->coopmat_support) {- // Create 6 variants, {s,m,l}x{unaligned,aligned}- // Only Intel needs required_subgroup_size pinned to the warptile's WARP element.-#define REQUIRED_SUBGROUP_SIZE(WARPTILE) (device->vendor_id == VK_VENDOR_ID_INTEL ? (WARPTILE)[WARP_SIZE_IDX] : 0)-#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, true, REQUIRED_SUBGROUP_SIZE(l_ ## WARPTILE)); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, true, REQUIRED_SUBGROUP_SIZE(m_ ## WARPTILE)); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, true, REQUIRED_SUBGROUP_SIZE(s_ ## WARPTILE)); \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, true, REQUIRED_SUBGROUP_SIZE(l_ ## WARPTILE)); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, true, REQUIRED_SUBGROUP_SIZE(m_ ## WARPTILE)); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _cm1_len, NAMELC ## F16ACC ## _cm1_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, true, REQUIRED_SUBGROUP_SIZE(s_ ## WARPTILE)); \-- // Create 2 variants, {f16,f32} accumulator-#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \- if (device->coopmat_acc_f16_support) { \- CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \- } \- if (device->coopmat_acc_f32_support) { \- CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \- } \+ spec_fn_t cm1_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); };- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, );- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, );-#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)- if (device->coopmat_bf16_support) {- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, )+ // Intel coopmat1 pins each pipeline's required subgroup size to its warptile WARP element.+ const bool cm1_pin = device->vendor_id == VK_VENDOR_ID_INTEL;++ // Intel coopmat1 uses a dedicated large-tile config for quant matmul_id.+ std::vector<vk_tile_config> tc_mmq_id = tc_mmq;+ if (cm1_pin) {+ tc_mmq_id[2] = { { 512, 128, 128, 32, 32, 32, 2, device->coopmat_m, device->coopmat_n, device->coopmat_k, 32 }, { 128, 128, 1 }, 32 };}-#endif- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );-- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ1_0], matmul_tq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );+ auto cm1_create = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base,+ const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) {+ auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id);+ if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, cm1_spec, false, true, 0, true, cm1_pin);+ };+ auto cm1_create_quant = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base,+ const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) {+ spec_fn_t qs = [&, type_a=key.type_a](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); };+ auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id);+ if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, true, 0, true, cm1_pin);+ };-#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)- if (device->ocp_fp4) {- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- } else+ cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_cm1_len, matmul_f32_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ cm1_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_cm1_len, matmul_f32_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ if (device->coopmat_acc_f16_support) {+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm1_len, matmul_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", matmul_f16_f32_f16acc_cm1_len, matmul_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ }+ if (device->coopmat_acc_f32_support) {+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm1_len, matmul_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_cm1_len, matmul_f16_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ }+#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)+ if (device->coopmat_bf16_support) {+ cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm1_len, matmul_bf16_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ }#endif- {- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- }-- // f16 B-type dense GEMM pipelines for coopmat1 (used when y_non_contig auto-converts f32->f16)- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q1_0], matmul_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_0], matmul_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ1_0], matmul_tq1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_TQ2_0], matmul_tq2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_0], matmul_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_1], matmul_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_0], matmul_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_1], matmul_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q8_0], matmul_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q2_K], matmul_q2_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q3_K], matmul_q3_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q4_K], matmul_q4_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q5_K], matmul_q5_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_Q6_K], matmul_q6_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_S], matmul_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ1_M], matmul_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ2_S], matmul_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ3_S], matmul_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );+ for (const auto type : non_lut_quant_types) {+ if (device->coopmat_acc_f16_support) {+ cm1_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", matmul_quant_f32_f16acc_cm1_len, matmul_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ cm1_create_quant({type, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm1_len, matmul_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ }+ if (device->coopmat_acc_f32_support) {+ cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ cm1_create_quant({type, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_quant_f16", matmul_quant_f16_cm1_len, matmul_quant_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3);+ }+ }+ // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16.+#define X_CM1(TYPE, tstr) \+ if (device->coopmat_acc_f16_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", matmul_##tstr##_f32_f16acc_cm1_len, matmul_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm1_len, matmul_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ } \+ if (device->coopmat_acc_f32_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_cm1_len, matmul_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm1_len, matmul_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ }+ FOR_EACH_LUT_TYPE_NONFP4(X_CM1)#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)if (device->ocp_fp4) {- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );+#define X_CM1_OCP(TYPE, tstr) \+ if (device->coopmat_acc_f16_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_ocp_f16acc", matmul_##tstr##_f32_ocp_f16acc_cm1_len, matmul_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm1_len, matmul_##tstr##_f16_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ } \+ if (device->coopmat_acc_f32_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32_ocp", matmul_##tstr##_f32_ocp_cm1_len, matmul_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm1_len, matmul_##tstr##_f16_ocp_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \+ }+ FOR_EACH_LUT_FP4_TYPE(X_CM1_OCP)+#undef X_CM1_OCP} else#endif{- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_MXFP4], matmul_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_NVFP4], matmul_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, );+ FOR_EACH_LUT_FP4_TYPE(X_CM1)}-- // Intel matmul_id warptile tuning- if (device->vendor_id == VK_VENDOR_ID_INTEL) {- l_warptile_mmq = { 512, 128, 128, 32, 32, 32, 2, device->coopmat_m, device->coopmat_n, device->coopmat_k, 32 };- l_mmq_wg_denoms = { 128, 128, 1 };- l_align = 32; //set as BK- }-+#undef X_CM1GGML_ASSERT(device->subgroup_ballot);- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);+ cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_cm1_len, matmul_id_subgroup_f32_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ if (device->coopmat_acc_f16_support) {+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm1_len, matmul_id_subgroup_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_subgroup_f16_f32_f16acc", matmul_id_subgroup_f16_f32_f16acc_cm1_len, matmul_id_subgroup_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }+ if (device->coopmat_acc_f32_support) {+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm1_len, matmul_id_subgroup_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_cm1_len, matmul_id_subgroup_f16_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)if (device->coopmat_bf16_support) {- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);+ cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);}#endif-- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);-#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)- if (device->ocp_fp4) {- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- } else-#endif- {- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- }-- // f16 B-type MoE GEMM pipelines for coopmat1 (used when y_non_contig auto-converts f32->f16)- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);+ for (const auto type : non_lut_quant_types) {+ if (device->coopmat_acc_f16_support) {+ cm1_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f32_f16acc", matmul_id_subgroup_quant_f32_f16acc_cm1_len, matmul_id_subgroup_quant_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ cm1_create_quant({type, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm1_len, matmul_id_subgroup_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }+ if (device->coopmat_acc_f32_support) {+ cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ cm1_create_quant({type, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm1_len, matmul_id_subgroup_quant_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }+ }+ // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16.+#define X_CM1_ID(TYPE, tstr) \+ if (device->coopmat_acc_f16_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_f16acc", matmul_id_subgroup_##tstr##_f32_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ } \+ if (device->coopmat_acc_f32_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_cm1_len, matmul_id_subgroup_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm1_len, matmul_id_subgroup_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ }+ FOR_EACH_LUT_TYPE_NONFP4(X_CM1_ID)#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)if (device->ocp_fp4) {- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16_ocp, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);+#define X_CM1_ID_OCP(TYPE, tstr) \+ if (device->coopmat_acc_f16_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_ocp_f16acc", matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_ocp_f16acc", matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_ocp_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ } \+ if (device->coopmat_acc_f32_support) { \+ cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_ocp", matmul_id_subgroup_##tstr##_f32_ocp_cm1_len, matmul_id_subgroup_##tstr##_f32_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_ocp", matmul_id_subgroup_##tstr##_f16_ocp_cm1_len, matmul_id_subgroup_##tstr##_f16_ocp_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ }+ FOR_EACH_LUT_FP4_TYPE(X_CM1_ID_OCP)+#undef X_CM1_ID_OCP} else#endif{- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id_f16b[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f16, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id);+ FOR_EACH_LUT_FP4_TYPE(X_CM1_ID)}-#undef CREATE_MM2-#undef CREATE_MM-#undef REQUIRED_SUBGROUP_SIZE+#undef X_CM1_ID} else#endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)- if (device->fp16) {- // Create 6 variants, {s,m,l}x{unaligned,aligned}- // Selects dot2 SPIR-V variant at runtime when device->dot2_f16 is true-#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _len : NAMELC ## F16ACC ## _len), (device->dot2_f16 ? NAMELC ## _dot2 ## F16ACC ## _data : NAMELC ## F16ACC ## _data), "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \-- // bf16 scalar path promotes to f32, no dot2 variant-#define CREATE_MM_NODOT2(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _len, NAMELC ## F16ACC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \--#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \- if (device->mul_mat ## ID ## _l_int[TYPE]) { \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->l, #NAMELC "_l", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- } \- if (device->mul_mat ## ID ## _m_int[TYPE]) { \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->m, #NAMELC "_m", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- } \- if (device->mul_mat ## ID ## _s_int[TYPE]) { \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME .f32acc->s, #NAMELC "_s", NAMELC ## _len, NAMELC ## _data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- } \-- // Create 2 variants, {f16,f32} accumulator-#define CREATE_MM2(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \- CREATE_MM(TYPE, PIPELINE_NAME . f16acc, NAMELC, _f16acc, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \- CREATE_MM(TYPE, PIPELINE_NAME . f32acc, NAMELC, , WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \-- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16, matmul_f16, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_f16_f32, matmul_f16_f32, wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0], matmul_q5_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1], matmul_q5_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0], matmul_q8_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K], matmul_q2_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0], matmul_tq2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ1_0], matmul_tq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K], matmul_q3_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K], matmul_q4_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K], matmul_q5_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K], matmul_q6_k_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S], matmul_iq1_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M], matmul_iq1_m_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS], matmul_iq2_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS], matmul_iq2_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S], matmul_iq2_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS], matmul_iq3_xxs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S], matmul_iq3_s_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS], matmul_iq4_xs_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL], matmul_iq4_nl_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4], matmul_mxfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4], matmul_nvfp4_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);-+ {+ // Helper for subgroup path with dot2 selection and filtering+ auto sg_create = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base,+ const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc,+ uint32_t rsgs = 0) {+ auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id);+ if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc,+ [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); },+ false, rsgs > 0, rsgs);+ };+ auto sg_create_quant = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base,+ const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc,+ uint32_t rsgs = 0) {+ auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id);+ if (!tc.empty()) {+ spec_fn_t qs = [&, type_a=key.type_a](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); };+ create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, rsgs > 0, rsgs);+ }+ };#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)- if (device->integer_dot_product) {- CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0], matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0], matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1], matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0], matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1], matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0], matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_MXFP4], matmul_mxfp4_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K], matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K], matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K], matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K], matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0);- CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K], matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, , 0);- }+ auto sg_create_mmq = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc_base,+ const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc,+ uint32_t rsgs = 0) {+ auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true);+ if (!tc.empty()) {+ spec_fn_t identity = [](const std::vector<uint32_t>& wt, bool) { return wt; };+ create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false);+ }+ };#endif- if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) {- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_subgroup_f16, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_subgroup_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_subgroup_tq1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_subgroup_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_subgroup_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_subgroup_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_subgroup_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_subgroup_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_subgroup_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_subgroup_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_subgroup_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_subgroup_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_subgroup_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);+ std::vector<vk_tile_config> tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}};+ std::vector<vk_tile_config> tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}};++ if (device->fp16) {+ // FP16 subgroup path - with dot2 runtime selection+ #define SPV_DOT2(NAME) (device->dot2_f16 ? NAME ## _dot2_len : NAME ## _len), (device->dot2_f16 ? NAME ## _dot2_data : NAME ## _data)+ #define SPV_DOT2_F16ACC(NAME) (device->dot2_f16 ? NAME ## _dot2_f16acc_len : NAME ## _f16acc_len), (device->dot2_f16 ? NAME ## _dot2_f16acc_data : NAME ## _f16acc_data)++ sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", SPV_DOT2(matmul_f32_f32), sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", SPV_DOT2(matmul_f32_f16), sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", SPV_DOT2_F16ACC(matmul_f16), sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", SPV_DOT2(matmul_f16), sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", SPV_DOT2(matmul_f16_f32), sizeof(vk_mat_mat_push_constants), 3);+ // BF16 - no dot2+ sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_len, matmul_bf16_data, sizeof(vk_mat_mat_push_constants), 3);++ for (const auto type : non_lut_quant_types) {+ sg_create_quant({type, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3);+ sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", SPV_DOT2(matmul_quant_f32), sizeof(vk_mat_mat_push_constants), 3);+ }+ #define X_SG(TYPE, tstr) \+ sg_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); \+ sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", SPV_DOT2(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3);+ FOR_EACH_LUT_TYPE(X_SG)+#undef X_SG#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)if (device->integer_dot_product) {- CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_subgroup_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_subgroup_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_subgroup_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);-- CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_subgroup_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);-- CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_subgroup_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_subgroup_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_subgroup_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_subgroup_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_subgroup_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);+ std::vector<vk_tile_config> tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}};+ std::vector<vk_tile_config> tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}};+ sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_len, matmul_q2_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_len, matmul_q4_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_len, matmul_q4_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_len, matmul_q5_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_len, matmul_q5_1_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_len, matmul_q8_0_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_mxfp4_q8_1", matmul_mxfp4_q8_1_len, matmul_mxfp4_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_len, matmul_q2_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_len, matmul_q3_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_len, matmul_q4_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_len, matmul_q5_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_len, matmul_q6_k_q8_1_data, sizeof(vk_mat_mat_push_constants), 3);}#endif++ if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) {+ sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", SPV_DOT2(matmul_id_subgroup_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_id, "matmul_id_subgroup_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", SPV_DOT2(matmul_id_subgroup_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_id, "matmul_id_subgroup_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", SPV_DOT2(matmul_id_subgroup_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ // BF16 id - no dot2+ sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_len, matmul_id_subgroup_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ for (const auto type : non_lut_quant_types) {+ sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", SPV_DOT2(matmul_id_subgroup_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ }+ #define X_SG_ID_SUB(TYPE, tstr) \+ sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); \+ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", SPV_DOT2(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ FOR_EACH_LUT_TYPE(X_SG_ID_SUB)+#undef X_SG_ID_SUB+#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)+ if (device->integer_dot_product) {+ std::vector<vk_tile_config> tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}};+ std::vector<vk_tile_config> tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}};+ sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q2_0_q8_1", matmul_id_subgroup_q2_0_q8_1_len, matmul_id_subgroup_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_0_q8_1", matmul_id_subgroup_q4_0_q8_1_len, matmul_id_subgroup_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q4_1_q8_1", matmul_id_subgroup_q4_1_q8_1_len, matmul_id_subgroup_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_0_q8_1", matmul_id_subgroup_q5_0_q8_1_len, matmul_id_subgroup_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q5_1_q8_1", matmul_id_subgroup_q5_1_q8_1_len, matmul_id_subgroup_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_q8_0_q8_1", matmul_id_subgroup_q8_0_q8_1_len, matmul_id_subgroup_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_subgroup_mxfp4_q8_1", matmul_id_subgroup_mxfp4_q8_1_len, matmul_id_subgroup_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q2_k_q8_1", matmul_id_subgroup_q2_k_q8_1_len, matmul_id_subgroup_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q3_k_q8_1", matmul_id_subgroup_q3_k_q8_1_len, matmul_id_subgroup_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q4_k_q8_1", matmul_id_subgroup_q4_k_q8_1_len, matmul_id_subgroup_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q5_k_q8_1", matmul_id_subgroup_q5_k_q8_1_len, matmul_id_subgroup_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_subgroup_q6_k_q8_1", matmul_id_subgroup_q6_k_q8_1_len, matmul_id_subgroup_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ }+#endif+ } else {+ sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", SPV_DOT2(matmul_id_f32_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_f16_f16acc", SPV_DOT2_F16ACC(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", SPV_DOT2(matmul_id_f16), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_f16_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", SPV_DOT2(matmul_id_f16_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ // BF16 id - no dot2+ sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_len, matmul_id_bf16_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ for (const auto type : non_lut_quant_types) {+ sg_create_quant({type, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_quant_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", SPV_DOT2(matmul_id_quant_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }+ #define X_SG_ID(TYPE, tstr) \+ sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \+ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", SPV_DOT2(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ FOR_EACH_LUT_TYPE(X_SG_ID)+#undef X_SG_ID+#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)+ if (device->integer_dot_product) {+ std::vector<vk_tile_config> tc_mmqid_int = {{s_warptile_mmqid_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int, l_mmq_wg_denoms, l_align}};+ std::vector<vk_tile_config> tc_mmqid_int_k = {{s_warptile_mmqid_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid_int_k, l_mmq_wg_denoms, l_align}};+ sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q2_0_q8_1", matmul_id_q2_0_q8_1_len, matmul_id_q2_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_0_q8_1", matmul_id_q4_0_q8_1_len, matmul_id_q4_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q4_1_q8_1", matmul_id_q4_1_q8_1_len, matmul_id_q4_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_0_q8_1", matmul_id_q5_0_q8_1_len, matmul_id_q5_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q5_1_q8_1", matmul_id_q5_1_q8_1_len, matmul_id_q5_1_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_q8_0_q8_1", matmul_id_q8_0_q8_1_len, matmul_id_q8_0_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_MXFP4, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int, "matmul_id_mxfp4_q8_1", matmul_id_mxfp4_q8_1_len, matmul_id_mxfp4_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q2_k_q8_1", matmul_id_q2_k_q8_1_len, matmul_id_q2_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q3_k_q8_1", matmul_id_q3_k_q8_1_len, matmul_id_q3_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q4_k_q8_1", matmul_id_q4_k_q8_1_len, matmul_id_q4_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q5_k_q8_1", matmul_id_q5_k_q8_1_len, matmul_id_q5_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmqid_int_k, "matmul_id_q6_k_q8_1", matmul_id_q6_k_q8_1_len, matmul_id_q6_k_q8_1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }+#endif+ }+ #undef SPV_DOT2+ #undef SPV_DOT2_F16ACC} else {- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16, matmul_id_f16, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0], matmul_id_q5_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1], matmul_id_q5_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0], matmul_id_q8_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K], matmul_id_q2_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0], matmul_id_tq2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0], matmul_id_tq1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K], matmul_id_q3_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K], matmul_id_q4_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K], matmul_id_q5_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K], matmul_id_q6_k_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S], matmul_id_iq1_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M], matmul_id_iq1_m_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS], matmul_id_iq2_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS], matmul_id_iq2_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S], matmul_id_iq2_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS], matmul_id_iq3_xxs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S], matmul_id_iq3_s_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS], matmul_id_iq4_xs_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL], matmul_id_iq4_nl_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4], matmul_id_mxfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM2(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4], matmul_id_nvfp4_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);+ // FP32-only fallback path+ sg_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_fp32_len, matmul_f32_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_fp32_len, matmul_f16_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_fp32_len, matmul_f16_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3);++ for (const auto type : non_lut_quant_types) {+ sg_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_fp32_len, matmul_quant_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ }+ #define X_SG_FP32(TYPE, tstr) \+ sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_fp32_len, matmul_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ FOR_EACH_LUT_TYPE(X_SG_FP32)+#undef X_SG_FP32#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)if (device->integer_dot_product) {- CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_0], matmul_id_q2_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_0], matmul_id_q4_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_1], matmul_id_q4_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_0], matmul_id_q5_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_1], matmul_id_q5_1_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q8_0], matmul_id_q8_0_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);-- CREATE_MMQ(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_MXFP4], matmul_id_mxfp4_q8_1, mmq_wg_denoms, warptile_mmqid_int, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);-- CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q2_K], matmul_id_q2_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q3_K], matmul_id_q3_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q4_K], matmul_id_q4_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q5_K], matmul_id_q5_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id_q8_1[GGML_TYPE_Q6_K], matmul_id_q6_k_q8_1, mmq_wg_denoms, warptile_mmqid_int_k, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);+ std::vector<vk_tile_config> tc_mmq_int = {{s_warptile_mmq_int, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int, l_mmq_wg_denoms, l_align}};+ std::vector<vk_tile_config> tc_mmq_int_k = {{s_warptile_mmq_int_k, s_mmq_wg_denoms, s_align}, {m_warptile_mmq_int_k, m_mmq_wg_denoms, m_align}, {l_warptile_mmq_int_k, l_mmq_wg_denoms, l_align}};+ sg_create_mmq({GGML_TYPE_Q2_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q2_0_q8_1", matmul_q2_0_q8_1_fp32_len, matmul_q2_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q4_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_0_q8_1", matmul_q4_0_q8_1_fp32_len, matmul_q4_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q4_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q4_1_q8_1", matmul_q4_1_q8_1_fp32_len, matmul_q4_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q5_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_0_q8_1", matmul_q5_0_q8_1_fp32_len, matmul_q5_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q5_1, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q5_1_q8_1", matmul_q5_1_q8_1_fp32_len, matmul_q5_1_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, false, false}, tc_mmq_int, "matmul_q8_0_q8_1", matmul_q8_0_q8_1_fp32_len, matmul_q8_0_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q2_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q2_k_q8_1", matmul_q2_k_q8_1_fp32_len, matmul_q2_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q3_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q3_k_q8_1", matmul_q3_k_q8_1_fp32_len, matmul_q3_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q4_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q4_k_q8_1", matmul_q4_k_q8_1_fp32_len, matmul_q4_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q5_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q5_k_q8_1", matmul_q5_k_q8_1_fp32_len, matmul_q5_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);+ sg_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, false, false}, tc_mmq_int_k, "matmul_q6_k_q8_1", matmul_q6_k_q8_1_fp32_len, matmul_q6_k_q8_1_fp32_data, sizeof(vk_mat_mat_push_constants), 3);}#endif- }-#undef CREATE_MM2-#undef CREATE_MMQ-#undef CREATE_MM-#undef CREATE_MM_NODOT2- } else {- // Create 6 variants, {s,m,l}x{unaligned,aligned}-#define CREATE_MM(TYPE, PIPELINE_NAME, NAMELC, F16ACC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID, REQSUBGROUPSIZE) \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC #F16ACC "_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC #F16ACC "_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC #F16ACC "_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, false), 1, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _l[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_l, #NAMELC #F16ACC "_aligned_l", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, ggml_vk_mul_mm_spec(l_ ## WARPTILE, true), l_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _m[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_m, #NAMELC #F16ACC "_aligned_m", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, ggml_vk_mul_mm_spec(m_ ## WARPTILE, true), m_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \- if (device->mul_mat ## ID ## _s[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->a_s, #NAMELC #F16ACC "_aligned_s", NAMELC ## F16ACC ## _fp32_len, NAMELC ## F16ACC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, ggml_vk_mul_mm_spec(s_ ## WARPTILE, true), s_align, false, REQSUBGROUPSIZE > 0, REQSUBGROUPSIZE); \--#define CREATE_MMQ(TYPE, PIPELINE_NAME, NAMELC, WG_DENOMS, WARPTILE, PUSHCONST, PARAMCOUNT, ID) \- if (device->mul_mat ## ID ## _l_int[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->l, #NAMELC "_l", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), l_ ## WG_DENOMS, l_ ## WARPTILE, 1); \- if (device->mul_mat ## ID ## _m_int[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->m, #NAMELC "_m", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), m_ ## WG_DENOMS, m_ ## WARPTILE, 1); \- if (device->mul_mat ## ID ## _s_int[TYPE]) \- ggml_vk_create_pipeline(device, device-> PIPELINE_NAME ->s, #NAMELC "_s", NAMELC ## _fp32_len, NAMELC ## _fp32_data, "main", PARAMCOUNT, sizeof(PUSHCONST), s_ ## WG_DENOMS, s_ ## WARPTILE, 1); \-- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32, matmul_f32_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_f32_f16, matmul_f32_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16.f32acc, matmul_f16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_F16, pipeline_matmul_f16_f32.f32acc, matmul_f16_f32, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0].f32acc, matmul_q1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);-- CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ2_0].f32acc, matmul_tq2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_TQ1_0].f32acc, matmul_tq1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_S].f32acc, matmul_iq1_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ1_M].f32acc, matmul_iq1_m_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XXS].f32acc, matmul_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_XS].f32acc, matmul_iq2_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ2_S].f32acc, matmul_iq2_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_XXS].f32acc, matmul_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ3_S].f32acc, matmul_iq3_s_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_XS].f32acc, matmul_iq4_xs_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat[GGML_TYPE_IQ4_NL].f32acc, matmul_iq4_nl_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_MXFP4].f32acc, matmul_mxfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat[GGML_TYPE_NVFP4].f32acc, matmul_nvfp4_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0);-#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)- if (device->integer_dot_product) {- CREATE_MMQ(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_0].f32acc, matmul_q5_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_1].f32acc, matmul_q5_1_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q8_0].f32acc, matmul_q8_0_q8_1, mmq_wg_denoms, warptile_mmq_int, vk_mat_mat_push_constants, 3, );-- CREATE_MMQ(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q2_K].f32acc, matmul_q2_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q3_K].f32acc, matmul_q3_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q4_K].f32acc, matmul_q4_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q5_K].f32acc, matmul_q5_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, );- CREATE_MMQ(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_q8_1[GGML_TYPE_Q6_K].f32acc, matmul_q6_k_q8_1, mmq_wg_denoms, warptile_mmq_int_k, vk_mat_mat_push_constants, 3, );+ if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) {+ sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_fp32_len, matmul_id_subgroup_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_fp32_len, matmul_id_subgroup_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_fp32_len, matmul_id_subgroup_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_fp32_len, matmul_id_subgroup_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size_16);+ for (const auto type : non_lut_quant_types) {+ sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_fp32_len, matmul_id_subgroup_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ }+ #define X_SG_ID_SUB_FP32(TYPE, tstr) \+ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_fp32_len, matmul_id_subgroup_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size);+ FOR_EACH_LUT_TYPE(X_SG_ID_SUB_FP32)+#undef X_SG_ID_SUB_FP32+ } else {+ sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", matmul_id_f32_f32_fp32_len, matmul_id_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_f16", matmul_id_f16_fp32_len, matmul_id_f16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f16_f32", matmul_id_f16_f32_fp32_len, matmul_id_f16_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ sg_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ for (const auto type : non_lut_quant_types) {+ sg_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_quant_f32", matmul_id_quant_f32_fp32_len, matmul_id_quant_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ }+ #define X_SG_ID_FP32(TYPE, tstr) \+ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", matmul_id_##tstr##_f32_fp32_len, matmul_id_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count);+ FOR_EACH_LUT_TYPE(X_SG_ID_FP32)+#undef X_SG_ID_FP32+ }}-#endif-- if (device->subgroup_ballot && device->subgroup_require_full_support && subgroup_min_size_16) {- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_subgroup_f32_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_subgroup_f16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_subgroup_f16_f32, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16);-- CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_subgroup_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_subgroup_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_subgroup_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_subgroup_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_subgroup_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_subgroup_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0].f32acc, matmul_id_subgroup_tq1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_subgroup_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_subgroup_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_subgroup_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_subgroup_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_subgroup_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_subgroup_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_subgroup_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_subgroup_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_subgroup_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_subgroup_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_subgroup_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_subgroup_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_subgroup_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_subgroup_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_subgroup_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size);- } else {- CREATE_MM(GGML_TYPE_F32, pipeline_matmul_id_f32, matmul_id_f32_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16.f32acc, matmul_id_f16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_F16, pipeline_matmul_id_f16_f32.f32acc, matmul_id_f16_f32, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);-- CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q5_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_0].f32acc, matmul_id_q5_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q5_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_1].f32acc, matmul_id_q5_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q8_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q8_0].f32acc, matmul_id_q8_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q2_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_K].f32acc, matmul_id_q2_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_TQ2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ2_0].f32acc, matmul_id_tq2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_TQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_TQ1_0].f32acc, matmul_id_tq1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q3_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q3_K].f32acc, matmul_id_q3_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q4_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_K].f32acc, matmul_id_q4_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q5_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q5_K].f32acc, matmul_id_q5_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_Q6_K, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q6_K].f32acc, matmul_id_q6_k_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ1_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_S].f32acc, matmul_id_iq1_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ1_M, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ1_M].f32acc, matmul_id_iq1_m_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ2_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XXS].f32acc, matmul_id_iq2_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ2_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_XS].f32acc, matmul_id_iq2_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ2_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ2_S].f32acc, matmul_id_iq2_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ3_XXS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_XXS].f32acc, matmul_id_iq3_xxs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ3_S, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ3_S].f32acc, matmul_id_iq3_s_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ4_XS, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_XS].f32acc, matmul_id_iq4_xs_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_IQ4_NL, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_IQ4_NL].f32acc, matmul_id_iq4_nl_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_MXFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_MXFP4].f32acc, matmul_id_mxfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- CREATE_MM(GGML_TYPE_NVFP4, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_NVFP4].f32acc, matmul_id_nvfp4_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);- }- }- // reusing CREATE_MM from the fp32 path+ }+#undef FOR_EACH_LUT_TYPE+#undef FOR_EACH_LUT_TYPE_NONFP4+#undef FOR_EACH_LUT_FP4_TYPE+ // BF16 fallback for coopmat devices without bf16 coopmat supportif ((device->coopmat2 || device->coopmat_support)#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)&& !device->coopmat_bf16_support#endif) {- const uint32_t s_warptile_wm = device->subgroup_size == 8 ? 8 : 32;-- // use scalar tile sizes- l_warptile = { 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 };- m_warptile = { 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 };- s_warptile = { subgroup_size_32, 32, 32, 16, s_warptile_wm, 32, 2, 2, 2, 1, subgroup_size_8 };-- l_wg_denoms = {128, 128, 1 };- m_wg_denoms = { 64, 64, 1 };- s_wg_denoms = { 32, 32, 1 };+ const uint32_t s_warptile_wm_bf16 = device->subgroup_size == 8 ? 8 : 32;+ std::vector<vk_tile_config> tc_bf16_fb = {+ {{ subgroup_size_32, 32, 32, 16, s_warptile_wm_bf16, 32, 2, 2, 2, 1, subgroup_size_8 }, {32, 32, 1}, s_align},+ {{ 128, 64, 64, 16, mm_warp_8, 32, 2, 4, 2, 1, mm_warp_8 }, {64, 64, 1}, m_align},+ {{ 128, 128, 128, 16, mm_warp_8 * 2, 64, 2, 4, 4, 1, mm_warp_8 }, {128, 128, 1}, l_align},+ };+ auto tc_bf16_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, false);+ auto tc_bf16_id_filtered = filter_tc(tc_bf16_fb, GGML_TYPE_BF16, true);+ spec_fn_t bf16_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_spec(wt, a); };+ if (!tc_bf16_filtered.empty()) {+ create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_bf16_filtered, "matmul_bf16", matmul_bf16_fp32_len, matmul_bf16_fp32_data, sizeof(vk_mat_mat_push_constants), 3, bf16_spec);+ }+ if (!tc_bf16_id_filtered.empty()) {+ create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_bf16_id_filtered, "matmul_id_bf16", matmul_id_bf16_fp32_len, matmul_id_bf16_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, bf16_spec);+ }+ }- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0);- CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0);+ // Set up tile selector functions+ if (device->coopmat2) {+ device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t shader_core_count,+ const std::vector<vk_matmul_pipeline_pair>& configs) -> uint32_t {+ if (configs.size() <= 1) return 0;+ uint32_t last = (uint32_t)configs.size() - 1;+ if (configs.size() == 2) {+ uint32_t crossover = configs[0].unaligned->wg_denoms[1];+ return (n > crossover) ? 1 : 0;+ }+ // 3+ configs: s=0, m=1, l=2+ const uint32_t tiles_l = CEIL_DIV(m, configs[last].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[last].unaligned->wg_denoms[1]);+ const uint32_t tiles_m = CEIL_DIV(m, configs[1].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[1].unaligned->wg_denoms[1]);+ uint32_t crossover_large = configs[1].unaligned->wg_denoms[1];+ bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count ||+ (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2);+ if (n > crossover_large && prefer_large) return last;+ uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1];+ if (n > crossover_medium) return 1;+ return 0;+ };+ device->matmul_id_tile_selector = [](uint32_t /*m*/, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/,+ const std::vector<vk_matmul_pipeline_pair>& configs) -> uint32_t {+ if (configs.size() <= 1) return 0;+ uint32_t last = (uint32_t)configs.size() - 1;+ if (configs.size() == 2) {+ uint32_t crossover = configs[0].unaligned->wg_denoms[1];+ return (n > crossover) ? 1 : 0;+ }+ uint32_t crossover_large = configs[1].unaligned->wg_denoms[1];+ if (n > crossover_large) return last;+ uint32_t crossover_medium = configs[0].unaligned->wg_denoms[1];+ if (n > crossover_medium) return 1;+ return 0;+ };+ } else {+ device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/,+ const std::vector<vk_matmul_pipeline_pair>& configs) -> uint32_t {+ if (configs.size() <= 1) return 0;+ if (m <= 32 || n <= 32) return 0;+ if (configs.size() == 2) return 1;+ if (m <= 64 || n <= 64) return 1;+ return (uint32_t)configs.size() - 1;+ };+ device->matmul_id_tile_selector = device->matmul_tile_selector;}-#undef CREATE_MM// mul mat vec@@ -5591,12 +5519,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);- ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);- ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);+ ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);+ ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);@@ -5620,12 +5548,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);- ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);- ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);+ ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);+ ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16);@@ -5676,12 +5604,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq}, 1, true, use_subgroups, force_subgroup_size);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);- ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);- ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);+ ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);+ ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16);ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16);@@ -5743,12 +5671,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q8_0], "dequant_q8_0", dequant_q8_0_len, dequant_q8_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant_transpose[GGML_TYPE_Q8_0], "dequant_q8_0_transpose", dequant_q8_0_transpose_len, dequant_q8_0_transpose_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_K], "dequant_q2_k", dequant_q2_k_len, dequant_q2_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ1_0], "dequant_tq1_0", dequant_tq1_0_len, dequant_tq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q3_K], "dequant_q3_k", dequant_q3_k_len, dequant_q3_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_K], "dequant_q4_k", dequant_q4_k_len, dequant_q4_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q5_K], "dequant_q5_k", dequant_q5_k_len, dequant_q5_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q6_K], "dequant_q6_k", dequant_q6_k_len, dequant_q6_k_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ1_0], "dequant_tq1_0", dequant_tq1_0_len, dequant_tq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ2_0], "dequant_tq2_0", dequant_tq2_0_len, dequant_tq2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 64, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_S], "dequant_iq1_s", dequant_iq1_s_len, dequant_iq1_s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ1_M], "dequant_iq1_m", dequant_iq1_m_len, dequant_iq1_m_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_IQ2_XXS], "dequant_iq2_xxs", dequant_iq2_xxs_len, dequant_iq2_xxs_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1);@@ -5773,12 +5701,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_1], "get_rows_q5_1", get_rows_q5_1_len, get_rows_q5_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q8_0], "get_rows_q8_0", get_rows_q8_0_len, get_rows_q8_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_K], "get_rows_q2_k", get_rows_q2_k_len, get_rows_q2_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ2_0], "get_rows_tq2_0", get_rows_tq2_0_len, get_rows_tq2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ1_0], "get_rows_tq1_0", get_rows_tq1_0_len, get_rows_tq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q3_K], "get_rows_q3_k", get_rows_q3_k_len, get_rows_q3_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_K], "get_rows_q4_k", get_rows_q4_k_len, get_rows_q4_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q5_K], "get_rows_q5_k", get_rows_q5_k_len, get_rows_q5_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q6_K], "get_rows_q6_k", get_rows_q6_k_len, get_rows_q6_k_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ1_0], "get_rows_tq1_0", get_rows_tq1_0_len, get_rows_tq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TQ2_0], "get_rows_tq2_0", get_rows_tq2_0_len, get_rows_tq2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_S], "get_rows_iq1_s", get_rows_iq1_s_len, get_rows_iq1_s_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ1_M], "get_rows_iq1_m", get_rows_iq1_m_len, get_rows_iq1_m_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs", get_rows_iq2_xxs_len, get_rows_iq2_xxs_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);@@ -5803,12 +5731,12 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_1], "get_rows_q5_1_f32", get_rows_q5_1_f32_len, get_rows_q5_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q8_0], "get_rows_q8_0_f32", get_rows_q8_0_f32_len, get_rows_q8_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_K], "get_rows_q2_k_f32", get_rows_q2_k_f32_len, get_rows_q2_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ2_0], "get_rows_tq2_0_f32", get_rows_tq2_0_f32_len, get_rows_tq2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);- ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ1_0], "get_rows_tq1_0_f32", get_rows_tq1_0_f32_len, get_rows_tq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q3_K], "get_rows_q3_k_f32", get_rows_q3_k_f32_len, get_rows_q3_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_K], "get_rows_q4_k_f32", get_rows_q4_k_f32_len, get_rows_q4_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q5_K], "get_rows_q5_k_f32", get_rows_q5_k_f32_len, get_rows_q5_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q6_K], "get_rows_q6_k_f32", get_rows_q6_k_f32_len, get_rows_q6_k_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ1_0], "get_rows_tq1_0_f32", get_rows_tq1_0_f32_len, get_rows_tq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TQ2_0], "get_rows_tq2_0_f32", get_rows_tq2_0_f32_len, get_rows_tq2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_S], "get_rows_iq1_s_f32", get_rows_iq1_s_f32_len, get_rows_iq1_s_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ1_M], "get_rows_iq1_m_f32", get_rows_iq1_m_f32_len, get_rows_iq1_m_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ2_XXS], "get_rows_iq2_xxs_f32", get_rows_iq2_xxs_f32_len, get_rows_iq2_xxs_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1);@@ -8079,115 +8007,14 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_typecase GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:case GGML_TYPE_NVFP4:- case GGML_TYPE_TQ2_0:case GGML_TYPE_TQ1_0:- break;- default:- return nullptr;- }-- return ctx->device->pipeline_dequant[type];-}--static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) {- VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_pipeline(" << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ", " << prec << ")");- if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {- return ctx->device->pipeline_matmul_f32;- }- if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) {- return ctx->device->pipeline_matmul_f32_f16;- }- if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) {- return ctx->device->pipeline_matmul_bf16;- }- if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) {- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {- return ctx->device->pipeline_matmul_f16_f32.f16acc;- }- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {- return ctx->device->pipeline_matmul_f16.f16acc;- }- } else {- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {- return ctx->device->pipeline_matmul_f16_f32.f32acc;- }- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {- return ctx->device->pipeline_matmul_f16.f32acc;- }- }-- // MMQ- if (src1_type == GGML_TYPE_Q8_1) {- vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1[src0_type].f32acc;-- if (pipelines->is_empty()) {- return nullptr;- }-- return pipelines;- }-- // f16 B on coopmat1- if (src1_type == GGML_TYPE_F16 && ctx->device->coopmat_support && !ctx->device->coopmat2) {- vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type];- bool prefer_fp16acc = ctx->device->fp16 && prec == GGML_PREC_DEFAULT;- bool support_fp16acc = !mmp.f16acc->is_empty();- bool support_fp32acc = !mmp.f32acc->is_empty();-- if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) {- return mmp.f16acc;- } else if (support_fp32acc) {- return mmp.f32acc;- }- return nullptr;- }-- if (src1_type != GGML_TYPE_F32 &&- !(src1_type == GGML_TYPE_F16 && ctx->device->coopmat_support) &&- !ctx->device->coopmat2) {- return nullptr;- }-- switch (src0_type) {- case GGML_TYPE_Q1_0:- case GGML_TYPE_Q2_0:- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- case GGML_TYPE_IQ1_S:- case GGML_TYPE_IQ1_M:- case GGML_TYPE_IQ2_XXS:- case GGML_TYPE_IQ2_XS:- case GGML_TYPE_IQ2_S:- case GGML_TYPE_IQ3_XXS:- case GGML_TYPE_IQ3_S:- case GGML_TYPE_IQ4_XS:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_MXFP4:- case GGML_TYPE_NVFP4:case GGML_TYPE_TQ2_0:- case GGML_TYPE_TQ1_0:break;default:return nullptr;}- if (ctx->device->coopmat2) {- assert(src1_type == GGML_TYPE_F16);- return prec == GGML_PREC_DEFAULT ? ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat_f16[src0_type].f32acc;- }-- if (ctx->device->coopmat_support) {- return (ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc;- }- return (ctx->device->fp16 && prec == GGML_PREC_DEFAULT) ? ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f16acc : ctx->device->pipeline_dequant_mul_mat_mat[src0_type].f32acc;+ return ctx->device->pipeline_dequant[type];}static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t num_cols, uint32_t m, uint32_t k) {@@ -8244,8 +8071,8 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context *case GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:case GGML_TYPE_NVFP4:- case GGML_TYPE_TQ2_0:case GGML_TYPE_TQ1_0:+ case GGML_TYPE_TQ2_0:break;default:return nullptr;@@ -8278,103 +8105,6 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context *return b_type == GGML_TYPE_F32 ? ctx->device->pipeline_dequant_mul_mat_vec_f32_f32[dmmv_wg][a_type][num_cols-1] : ctx->device->pipeline_dequant_mul_mat_vec_f16_f32[dmmv_wg][a_type][num_cols-1];}-static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_id_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) {- VK_LOG_DEBUG("ggml_vk_get_mul_mat_mat_id_pipeline()");- if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {- return ctx->device->pipeline_matmul_id_f32;- }- if (src0_type == GGML_TYPE_BF16 && src1_type == GGML_TYPE_BF16) {- return ctx->device->pipeline_matmul_id_bf16;- }- if (prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support)) {- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {- return ctx->device->pipeline_matmul_id_f16_f32.f16acc;- }- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {- return ctx->device->pipeline_matmul_id_f16.f16acc;- }- } else {- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {- return ctx->device->pipeline_matmul_id_f16_f32.f32acc;- }- if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {- return ctx->device->pipeline_matmul_id_f16.f32acc;- }- }-- // MMQ- if (src1_type == GGML_TYPE_Q8_1) {- vk_matmul_pipeline pipelines = ctx->device->pipeline_dequant_mul_mat_mat_id_q8_1[src0_type].f32acc;-- if (pipelines->is_empty()) {- return nullptr;- }-- return pipelines;- }-- // f16 B on coopmat1- if (src1_type == GGML_TYPE_F16 && ctx->device->coopmat_support && !ctx->device->coopmat2) {- vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_id_f16b[src0_type];- bool prefer_fp16acc = ctx->device->fp16;- bool support_fp16acc = !mmp.f16acc->is_empty();- bool support_fp32acc = !mmp.f32acc->is_empty();-- if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) {- return mmp.f16acc;- } else if (support_fp32acc) {- return mmp.f32acc;- }- return nullptr;- }-- GGML_ASSERT(src1_type == GGML_TYPE_F32 || (ctx->device->coopmat2 && src1_type == GGML_TYPE_F16));-- switch (src0_type) {- case GGML_TYPE_Q1_0:- case GGML_TYPE_Q2_0:- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- case GGML_TYPE_IQ1_S:- case GGML_TYPE_IQ1_M:- case GGML_TYPE_IQ2_XXS:- case GGML_TYPE_IQ2_XS:- case GGML_TYPE_IQ2_S:- case GGML_TYPE_IQ3_XXS:- case GGML_TYPE_IQ3_S:- case GGML_TYPE_IQ4_XS:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_MXFP4:- case GGML_TYPE_NVFP4:- case GGML_TYPE_TQ2_0:- case GGML_TYPE_TQ1_0:- break;- default:- return nullptr;- }-- vk_matmul_pipeline2& mmp = ctx->device->pipeline_dequant_mul_mat_mat_id[src0_type];- // XXX TODO 'prec' is not actually allowed in mul_mat_id.- bool prefer_fp16acc = ctx->device->fp16 /*&& prec == GGML_PREC_DEFAULT*/;- bool support_fp16acc = !mmp.f16acc->is_empty();- bool support_fp32acc = !mmp.f32acc->is_empty();-- if (support_fp16acc && (prefer_fp16acc || !support_fp32acc)) {- return mmp.f16acc;- } else {- GGML_ASSERT(support_fp32acc);- return mmp.f32acc;- }-}-static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t m, uint32_t k) {VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec_id()");GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_Q8_1);@@ -8428,8 +8158,8 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_contextcase GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:case GGML_TYPE_NVFP4:- case GGML_TYPE_TQ2_0:case GGML_TYPE_TQ1_0:+ case GGML_TYPE_TQ2_0:break;default:return nullptr;@@ -9211,58 +8941,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,return split_k;}-static vk_pipeline ggml_vk_guess_matmul_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) {- VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")");-- // The q8_1 (integer dot) mmq path uses a different shader with its own- // shared-memory layout, so use the int-specific availability flags.- const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1);- const bool mm_l = is_q8_1 ? ctx->device->mul_mat_l_int[src0_type] : ctx->device->mul_mat_l[src0_type];- const bool mm_m = is_q8_1 ? ctx->device->mul_mat_m_int[src0_type] : ctx->device->mul_mat_m[src0_type];- const bool mm_s = is_q8_1 ? ctx->device->mul_mat_s_int[src0_type] : ctx->device->mul_mat_s[src0_type];- if (ctx->device->coopmat2) {- const uint32_t shader_core_count = ctx->device->shader_core_count;- const uint32_t tiles_l = CEIL_DIV(m, mmp->a_l->wg_denoms[0]) * CEIL_DIV(n, mmp->a_l->wg_denoms[1]);- const uint32_t tiles_m = CEIL_DIV(m, mmp->a_m->wg_denoms[0]) * CEIL_DIV(n, mmp->a_m->wg_denoms[1]);-- // Use large shader when the N dimension is greater than the medium shader's tile size- uint32_t crossover_large = mmp->m->wg_denoms[1];-- // Prefer large over medium if either:- // - medium or large tiles would overfill the GPU- // - large tiles with a split_k==3 fits in the GPU and medium tiles with split_k==2 does not- // (medium with split_k==2 is probably better if it fits - more workgroups running and less split_k overhead)- bool prefer_large = tiles_m > shader_core_count || tiles_l > shader_core_count ||- // split_k==3 with large tiles likely better than medium tiles with no split_k.- (tiles_l <= shader_core_count / 3 && tiles_m > shader_core_count / 2);-- if ((mm_l && (n > crossover_large && prefer_large)) || (!mm_m && !mm_s)) {- return aligned ? mmp->a_l : mmp->l;- }- // Use medium shader when the N dimension is greater than the small shader's tile size- uint32_t crossover_medium = mmp->s->wg_denoms[1];- if ((mm_m && (n > crossover_medium)) || !mm_s) {- return aligned ? mmp->a_m : mmp->m;- }- return aligned ? mmp->a_s : mmp->s;- }-- if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) {- return aligned ? mmp->a_s : mmp->s;- }- if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) {- return aligned ? mmp->a_m : mmp->m;- }- return aligned ? mmp->a_l : mmp->l;-}--static uint32_t ggml_vk_guess_matmul_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) {- VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")");- vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, m, n, true, src0_type, src1_type);- GGML_ASSERT(pipeline != nullptr && "missing matmul pipeline - check pipeline registration in ggml_vk_load_shaders for this type combo");- return pipeline->align;-}static void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,@@ -9313,42 +8992,56 @@ static void ggml_vk_matmul(ctx->prealloc_split_k_need_sync = true;}-static vk_pipeline ggml_vk_guess_matmul_id_pipeline(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, uint32_t m, uint32_t n, bool aligned, ggml_type src0_type, ggml_type src1_type) {- VK_LOG_DEBUG("ggml_vk_guess_matmul_id_pipeline(" << m << ", " << n << ", " << aligned << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")");-- // The q8_1 (integer dot) mmq path uses a different shader with its own- // shared-memory layout, so use the int-specific availability flags.- const bool is_q8_1 = (src1_type == GGML_TYPE_Q8_1);- const bool mm_l = is_q8_1 ? ctx->device->mul_mat_id_l_int[src0_type] : ctx->device->mul_mat_id_l[src0_type];- const bool mm_m = is_q8_1 ? ctx->device->mul_mat_id_m_int[src0_type] : ctx->device->mul_mat_id_m[src0_type];- const bool mm_s = is_q8_1 ? ctx->device->mul_mat_id_s_int[src0_type] : ctx->device->mul_mat_id_s[src0_type];+static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) {+ if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false;+ if (src1_type == GGML_TYPE_Q8_1) return false;+ if (src0_type == GGML_TYPE_F16) {+ return prec == GGML_PREC_DEFAULT && ctx->device->fp16 && !(ctx->device->coopmat_support && !ctx->device->coopmat_acc_f16_support);+ }+ // quant typesif (ctx->device->coopmat2) {- // Use large shader when the N dimension is greater than the medium shader's tile size- uint32_t crossover_large = mmp->m->wg_denoms[1];- if ((mm_l && (n > crossover_large)) || (!mm_m && !mm_s)) {- return aligned ? mmp->a_l : mmp->l;- }- // Use medium shader when the N dimension is greater than the small shader's tile size- uint32_t crossover_medium = mmp->s->wg_denoms[1];- if ((mm_m && (n > crossover_medium)) || !mm_s) {- return aligned ? mmp->a_m : mmp->m;- }- return aligned ? mmp->a_s : mmp->s;+ return prec == GGML_PREC_DEFAULT;}-- if ((mm_s && (m <= 32 || n <= 32)) || (!mm_m && !mm_l)) {- return aligned ? mmp->a_s : mmp->s;+ if (ctx->device->coopmat_support) {+ return ctx->device->fp16 && ctx->device->coopmat_acc_f16_support && prec == GGML_PREC_DEFAULT;}- if ((mm_m && (m <= 64 || n <= 64)) || !mm_l) {- return aligned ? mmp->a_m : mmp->m;+ return ctx->device->fp16 && prec == GGML_PREC_DEFAULT;+}++static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map(+ ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) {+ bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec);+ vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc};+ auto it = ctx->device->pipeline_matmul.find(key);+ if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) {+ // Try without f16acc+ if (f16acc) {+ key.f16acc = false;+ it = ctx->device->pipeline_matmul.find(key);+ if (it != ctx->device->pipeline_matmul.end() && !it->second.empty()) return &it->second;+ }+ return nullptr;}- return aligned ? mmp->a_l : mmp->l;+ return &it->second;}-static uint32_t ggml_vk_guess_matmul_id_pipeline_align(ggml_backend_vk_context * ctx, vk_matmul_pipeline& mmp, int m, int n, ggml_type src0_type, ggml_type src1_type) {- VK_LOG_DEBUG("ggml_vk_guess_matmul_pipeline_align(" << m << ", " << n << ", " << ggml_type_name(src0_type) << ", " << ggml_type_name(src1_type) << ")");- return ggml_vk_guess_matmul_id_pipeline(ctx, mmp, m, n, true, src0_type, src1_type)->align;+static vk_pipeline ggml_vk_guess_matmul_pipeline_map(ggml_backend_vk_context * ctx,+ const std::vector<vk_matmul_pipeline_pair>& configs,+ uint32_t m, uint32_t n, bool aligned, bool mul_mat_id) {+ auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector;+ uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs);+ if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1;+ return (aligned && configs[idx].aligned) ? configs[idx].aligned : configs[idx].unaligned;+}++static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_context * ctx,+ const std::vector<vk_matmul_pipeline_pair>& configs,+ uint32_t m, uint32_t n, bool mul_mat_id) {+ auto& selector = mul_mat_id ? ctx->device->matmul_id_tile_selector : ctx->device->matmul_tile_selector;+ uint32_t idx = selector(m, n, 0, ctx->device->shader_core_count, configs);+ if (idx >= configs.size()) idx = (uint32_t)configs.size() - 1;+ return configs[idx].align;}static void ggml_vk_matmul_id(@@ -9657,10 +9350,12 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subsrc1_uma = d_Qy != nullptr;}+ // TODO: Clean up this logic to pick src1 type by capability// Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perfconst bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) ||!ggml_vk_dim01_contiguous(src0);const bool y_non_contig = (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) ||+ // coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used.(ctx->device->coopmat_support && !ctx->device->coopmat2 &&ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) ||(src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) ||@@ -9674,31 +9369,31 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subbool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0;// Check for mmq first- vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr;+ const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr;- if (mmp == nullptr) {+ if (mmp_map == nullptr) {// Fall back to f16 dequant mul mat- mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]);+ mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]);quantize_y = false;}- const bool qx_needs_dequant = mmp == nullptr || x_non_contig;+ const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig);if (qx_needs_dequant) {// Fall back to dequant + f16 mulmat- mmp = ggml_vk_get_mul_mat_mat_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]);+ mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]);}// Not implementedGGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT- const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type);+ GGML_ASSERT(mmp_map != nullptr);- const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align(ctx, mmp, ne01, ne11, qx_needs_dequant ? f16_type : src0->type, effective_src1_type));+ const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false));const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8;- vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline(ctx, mmp, ne01, ne11, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type);+ vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false);if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);@@ -10698,7 +10393,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&#endifconst bool y_non_contig = y_decode_vector_staging ||(ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) ||- // Intel coopmat1: force f32->f16 conversion so the f16-B-type pipeline is used.+ // Intel coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used.(ctx->device->coopmat_support && !ctx->device->coopmat2 &&ctx->device->vendor_id == VK_VENDOR_ID_INTEL &&ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) ||@@ -10710,25 +10405,22 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0;// Check for mmq first- vk_matmul_pipeline mmp = quantize_y ? ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr;+ const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0], true) : nullptr;- if (mmp == nullptr) {+ if (mmp_map == nullptr) {// Fall back to f16 dequant mul mat- mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0]);+ mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0], true);quantize_y = false;}- const bool qx_needs_dequant = mmp == nullptr || x_non_contig;+ const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig);if (qx_needs_dequant) {// Fall back to dequant + f16 mulmat- mmp = ggml_vk_get_mul_mat_mat_id_pipeline(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0]);+ mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0], true);}- const ggml_type effective_src1_type = quantize_y ? GGML_TYPE_Q8_1 : (y_f32_kernel ? GGML_TYPE_F32 : src1->type);-- const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_id_pipeline_align(ctx, mmp, ne01, nei1, qx_needs_dequant ? f16_type : src0->type, effective_src1_type));// Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64.const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10;const bool y_needs_k_padding = ne10 != y_staged_row_stride;@@ -10738,9 +10430,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&// Not implementedGGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT+ GGML_ASSERT(mmp_map != nullptr);++ const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, nei1, true));const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && nei1 > 8;- vk_pipeline pipeline = ggml_vk_guess_matmul_id_pipeline(ctx, mmp, ne01, nei1, aligned, qx_needs_dequant ? f16_type : src0->type, effective_src1_type);+ vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true);if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);@@ -15330,106 +15025,22 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_tconst size_t y_ne = k * n * batch;const size_t d_ne = m * n * batch;- vk_pipeline p;- std::string shname;- if (shader_size == 0) {- if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32->a_s;- shname = "F32_ALIGNED_S";- } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32_f16->a_s;- shname = "F32_F16_ALIGNED_S";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_s;- shname = "F16_F32_ALIGNED_S";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16.f32acc->a_s;- shname = "F16_ALIGNED_S";- } else {- GGML_ABORT("fatal error");- }- } else if (shader_size == 1) {- if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32->a_m;- shname = "F32_ALIGNED_M";- } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32_f16->a_m;- shname = "F32_F16_ALIGNED_M";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_m;- shname = "F16_F32_ALIGNED_M";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16.f32acc->a_m;- shname = "F16_ALIGNED_M";- } else {- GGML_ABORT("fatal error");- }- } else if (shader_size == 2) {- if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32->a_l;- shname = "F32_ALIGNED_L";- } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32_f16->a_l;- shname = "F32_F16_ALIGNED_L";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16_f32.f32acc->a_l;- shname = "F16_F32_ALIGNED_L";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16.f32acc->a_l;- shname = "F16_ALIGNED_L";- } else {- GGML_ABORT("fatal error");- }- } else {- GGML_ASSERT(0);- }+ ggml_type x_type = std::is_same<float, X_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;+ ggml_type y_type = std::is_same<float, Y_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;+ vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false};+ auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key);+ GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty());+ auto& mm_test_configs = mm_test_it->second;+ GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size());++ std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size);+ vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned;- const size_t kpad = ggml_vk_align_size(k, p->align);+ const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align);if (k != kpad) {- if (shader_size == 0) {- if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32->s;- shname = "F32_S";- } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32_f16->s;- shname = "F32_F16_S";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16_f32.f32acc->s;- shname = "F16_F32_S";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16.f32acc->s;- shname = "F16_S";- }- } else if (shader_size == 1) {- if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32->m;- shname = "F32_M";- } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32_f16->m;- shname = "F32_F16_M";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16_f32.f32acc->m;- shname = "F16_F32_M";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16.f32acc->m;- shname = "F16_M";- }- } else if (shader_size == 2) {- if (std::is_same<float, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32->l;- shname = "F32_L";- } else if (std::is_same<float, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f32_f16->l;- shname = "F32_F16_L";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<float, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16_f32.f32acc->l;- shname = "F16_F32_L";- } else if (std::is_same<ggml_fp16_t, X_TYPE>() && std::is_same<ggml_fp16_t, Y_TYPE>()) {- p = ctx->device->pipeline_matmul_f16.f32acc->l;- shname = "F16_L";- }- }+ p = mm_test_configs[shader_size].unaligned;+ shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size);}if (split_k > 1) {@@ -15864,46 +15475,34 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m,const size_t y_ne = k * n * batch;const size_t d_ne = m * n * batch;- vk_matmul_pipeline2 * pipelines;-- if (mmq) {- pipelines = ctx->device->pipeline_dequant_mul_mat_mat_q8_1;- } else {- pipelines = ctx->device->pipeline_dequant_mul_mat_mat;- }-- const bool fp16acc = ctx->device->fp16;-- vk_pipeline p;- std::string shname;- if (shader_size == 0) {- p = fp16acc ? pipelines[quant].f16acc->a_s : pipelines[quant].f32acc->a_s;- shname = std::string(ggml_type_name(quant)) + "_ALIGNED_S";- } else if (shader_size == 1) {- p = fp16acc ? pipelines[quant].f16acc->a_m : pipelines[quant].f32acc->a_m;- shname = std::string(ggml_type_name(quant)) + "_ALIGNED_M";- } else if (shader_size == 2) {- p = fp16acc ? pipelines[quant].f16acc->a_l : pipelines[quant].f32acc->a_l;- shname = std::string(ggml_type_name(quant)) + "_ALIGNED_L";- } else {- GGML_ASSERT(0);+ ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32;+ bool f16acc = ctx->device->fp16 && !mmq;+ vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc};+ auto dq_it = ctx->device->pipeline_matmul.find(dq_key);+ if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {+ if (f16acc) {+ dq_key.f16acc = false;+ dq_it = ctx->device->pipeline_matmul.find(dq_key);+ }+ }+ if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {+ std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;+ return;+ }+ auto& dq_configs = dq_it->second;+ if (shader_size >= (int)dq_configs.size()) {+ std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl;+ return;}- const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, p->align);+ std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size);+ vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned;++ const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align);if (mmq || k != kpad) {- if (shader_size == 0) {- p = fp16acc ? pipelines[quant].f16acc->s : pipelines[quant].f32acc->s;- shname = std::string(ggml_type_name(quant)) + "_S";- } else if (shader_size == 1) {- p = fp16acc ? pipelines[quant].f16acc->m : pipelines[quant].f32acc->m;- shname = std::string(ggml_type_name(quant)) + "_M";- } else if (shader_size == 2) {- p = fp16acc ? pipelines[quant].f16acc->l : pipelines[quant].f32acc->l;- shname = std::string(ggml_type_name(quant)) + "_L";- } else {- GGML_ASSERT(0);- }+ p = dq_configs[shader_size].unaligned;+ shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size);}if (p == nullptr) {@@ -19464,8 +19063,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:case GGML_TYPE_NVFP4:- case GGML_TYPE_TQ2_0:case GGML_TYPE_TQ1_0:+ case GGML_TYPE_TQ2_0:break;default:return false;@@ -19571,8 +19170,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_TYPE_IQ4_NL:case GGML_TYPE_MXFP4:case GGML_TYPE_NVFP4:- case GGML_TYPE_TQ2_0:case GGML_TYPE_TQ1_0:+ case GGML_TYPE_TQ2_0:case GGML_TYPE_I32:return true;default:diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glslindex ef53264a77..cc6e242a90 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl@@ -260,6 +260,20 @@ float16_t dequantFuncTQ1_0(const in decodeBufTQ1_0 bl, const in uint blockCoordsreturn bl.block.d * (float16_t(int(xi)) - float16_t(1.0));}+f16vec4 dequantFuncTQ1_0_v(const in decodeBufTQ1_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])+{+ const uint e = coordInBlock[1];+ f16vec4 v;+ [[unroll]] for (uint k = 0u; k < 4u; ++k) {+ const uint ee = e + k;+ const uint bidx = tq1_0_byte_of(ee);+ const uint qbyte = uint(bidx < 48u ? bl.block.qs[bidx] : bl.block.qh[bidx - 48u]);+ const uint xi = tq1_0_trit(qbyte, tq1_0_digit_of(ee));+ v[k] = bl.block.d * (float16_t(int(xi)) - float16_t(1.0));+ }+ return v;+}+layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0 {block_tq2_0 block;};@@ -1054,7 +1068,7 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoordsconst uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf;const uint qs = bl.block.qs[ib8];const uint qh = bl.block.qh[ib32];- const uint sign = bl.block.qs[QUANT_K / 8 + ib8] >> (idx & 0x6);+ const uint sign = bl.block.qs[QUANT_K_IQ2_S / 8 + ib8] >> (idx & 0x6);const float d = float(bl.block.d);const float db = d * 0.25 * (0.5 + scale);@@ -1076,7 +1090,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoordsconst uint scale = (bl.block.scales[ib32] >> ((idx & 0x10) >> 2)) & 0xf;const uint qs = bl.block.qs[ib8];const uint qh = bl.block.qh[ib32];- const uint sb = uint(bl.block.qs[QUANT_K / 8 + ib8]) >> (idx & 0x6u);+ const uint sb = uint(bl.block.qs[QUANT_K_IQ2_S / 8 + ib8]) >> (idx & 0x6u);const float d = float(bl.block.d);const float db = d * 0.25 * (0.5 + scale);@@ -1107,7 +1121,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCouint idx = coordInBlock[1];const uint iqs = (idx & 0xFC) >> 2; // 0..63- const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3);// 8 values+ const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3);// 8 valuesconst float d = float(bl.block.d);const uint qs = bl.block.qs[iqs];@@ -1130,7 +1144,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCoconst uint idx = coordInBlock[1];const uint iqs = idx >> 2;- const uint is = QUANT_K / 4 + ((idx & 0xE0) >> 3);+ const uint is = QUANT_K_IQ3_XXS / 4 + ((idx & 0xE0) >> 3);const float d = float(bl.block.d);const uint qs = bl.block.qs[iqs];diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glslindex 6f414ded12..1e732a9a29 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl@@ -1,32 +1,22 @@#if !defined(GGML_FA_TYPES_COMP)#define GGML_FA_TYPES_COMP-// FaTypeK / FaTypeV spec constant values. These mirror enum ggml_type so the-// host can pass the type directly. Keep in sync with ggml.h.-#define FA_TYPE_F32 0u-#define FA_TYPE_F16 1u-#define FA_TYPE_Q4_0 2u-#define FA_TYPE_Q4_1 3u-#define FA_TYPE_Q5_0 6u-#define FA_TYPE_Q5_1 7u-#define FA_TYPE_Q8_0 8u-#define FA_TYPE_IQ4_NL 20u-#define FA_TYPE_BF16 30u+#include "ggml_type_ids.glsl"// Number of matrix elements per buffer block, derived from the K/V type spec// constant. F32 is treated as a vec4 "block" of 4 floats. F16 uses block size 1// and bypasses the dequant path entirely. Quants follow their ggml block sizes.uint fa_block_elems(uint ty) {switch (ty) {- case FA_TYPE_F32: return 4u;- case FA_TYPE_F16: return 1u;- case FA_TYPE_Q4_0: return uint(QUANT_K_Q4_0);- case FA_TYPE_Q4_1: return uint(QUANT_K_Q4_1);- case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0);- case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1);- case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0);- case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL);- case FA_TYPE_BF16: return 1u;+ case GGML_TYPE_F32: return 4u;+ case GGML_TYPE_F16: return 1u;+ case GGML_TYPE_Q4_0: return uint(QUANT_K_Q4_0);+ case GGML_TYPE_Q4_1: return uint(QUANT_K_Q4_1);+ case GGML_TYPE_Q5_0: return uint(QUANT_K_Q5_0);+ case GGML_TYPE_Q5_1: return uint(QUANT_K_Q5_1);+ case GGML_TYPE_Q8_0: return uint(QUANT_K_Q8_0);+ case GGML_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL);+ case GGML_TYPE_BF16: return 1u;default: return 1u;}}@@ -36,18 +26,18 @@ uint fa_block_elems(uint ty) {// of int32s per 32-element block on the MMQ K path: ints_per_block == 8 / R.uint fa_quant_r_mmq(uint ty) {switch (ty) {- case FA_TYPE_Q4_0: return uint(QUANT_R_Q4_0);- case FA_TYPE_Q4_1: return uint(QUANT_R_Q4_1);- case FA_TYPE_Q5_0: return uint(QUANT_R_Q5_0);- case FA_TYPE_Q5_1: return uint(QUANT_R_Q5_1);- case FA_TYPE_Q8_0: return uint(QUANT_R_Q8_0);+ case GGML_TYPE_Q4_0: return uint(QUANT_R_Q4_0);+ case GGML_TYPE_Q4_1: return uint(QUANT_R_Q4_1);+ case GGML_TYPE_Q5_0: return uint(QUANT_R_Q5_0);+ case GGML_TYPE_Q5_1: return uint(QUANT_R_Q5_1);+ case GGML_TYPE_Q8_0: return uint(QUANT_R_Q8_0);default: return 1u;}}bool fa_type_needs_shmem(uint ty) {switch (ty) {- case FA_TYPE_IQ4_NL: return true;+ case GGML_TYPE_IQ4_NL: return true;default: return false;}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.compindex 0c1b6d0673..9a12cdfb88 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp@@ -134,7 +134,7 @@ void main() {// Q8_0 K only needs (qd, _); the asymmetric Q4_*/Q5_* family also stores// the row-sum scaled by qd, used in k_dot_correction.- if (FaTypeK == FA_TYPE_Q8_0) {+ if (FaTypeK == GGML_TYPE_Q8_0) {if (buf_iqs == 0) {Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f);}@@ -367,7 +367,7 @@ void main() {// Q4_*/Q5_* take the block-8 fast path when one step covers a full// block; Q8_0 always goes through the per-int get_k_qs* helpers// (its qs is byte-packed, not nibble-packed).- const bool block8_fast = (d_per_step == 8) && (FaTypeK != FA_TYPE_Q8_0);+ const bool block8_fast = (d_per_step == 8) && (FaTypeK != GGML_TYPE_Q8_0);if (SHMEM_STAGING != 0) {const uint k_block_idx = (d_tid * (HSK_per_thread / 4) + d_block) / 8;@@ -375,7 +375,7 @@ void main() {k_dm = ACC_TYPEV2(kblocksh[buf_ib].dm);if (block8_fast) {- const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1);+ const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1);[[unroll]] for (uint32_t d = 0; d < 4; d++) {uint vui = kblocksh[buf_ib].qs[d];k_quants[d ] = int32_t( vui & 0x0F0F0F0F);diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glslindex 0ce4503a88..a4be1ebf98 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl@@ -105,8 +105,8 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];};#define BLOCK_SIZE_V fa_block_elems(FaTypeV)// F16 reads f16 elements directly from the binding; everything else routes// through dequantize4 / the MMQ helpers to unpack from the packed block layout.-#define USE_DECODE_K (FaTypeK != FA_TYPE_F16)-#define USE_DECODE_V (FaTypeV != FA_TYPE_F16)+#define USE_DECODE_K (FaTypeK != GGML_TYPE_F16)+#define USE_DECODE_V (FaTypeV != GGML_TYPE_F16)#define CEIL_DIV(a, b) (((a) + (b) - 1) / (b))diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.compindex 3174111530..5a9abe2265 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp@@ -40,26 +40,28 @@ layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufFA_#if !defined(BFLOAT16)float16_t faDecodeK(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {switch (FaTypeK) {- case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);default: return float16_t(0);}}float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {switch (FaTypeV) {- case FA_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_F32: return dequantFuncF32 (decodeBufF32 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_0: return dequantFuncQ4_0(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_1: return dequantFuncQ4_1(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);default: return float16_t(0);}}@@ -67,26 +69,26 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const// V=4 vector decode for K/V; dispatches to per-format _v decoders.f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {switch (FaTypeK) {- case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block);- case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block);+ case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);default: return f16vec4(0);}}f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {switch (FaTypeV) {- case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block);- case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block);+ case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);default: return f16vec4(0);}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glslindex 8ba4725f33..4fcf7c1f4f 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl@@ -121,25 +121,25 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1_P32 { block_q5_1_packed32 datFLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) {if (binding_idx == BINDING_IDX_K) {switch (FaTypeK) {- case FA_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32)- case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0)- case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1)- case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0)- case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1)- case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0)- case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl)- case FA_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16)+ case GGML_TYPE_F32: FA_DEQUANT4_F32 (k_packed_f32)+ case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(k_packed_q4_0)+ case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(k_packed_q4_1)+ case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0)+ case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1)+ case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0)+ case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl)+ case GGML_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16)}} else {switch (FaTypeV) {- case FA_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32)- case FA_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0)- case FA_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1)- case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0)- case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1)- case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0)- case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl)- case FA_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16)+ case GGML_TYPE_F32: FA_DEQUANT4_F32 (v_packed_f32)+ case GGML_TYPE_Q4_0: FA_DEQUANT4_Q4_0(v_packed_q4_0)+ case GGML_TYPE_Q4_1: FA_DEQUANT4_Q4_1(v_packed_q4_1)+ case GGML_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0)+ case GGML_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1)+ case GGML_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0)+ case GGML_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl)+ case GGML_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16)}}return FLOAT_TYPEV4(0);diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glslindex 6bf10a7cff..49900aa5ae 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl@@ -4,20 +4,20 @@int32_t get_k_qs(uint ib, uint iqs, uint a_offset) {switch (FaTypeK) {- case FA_TYPE_Q4_0: {+ case GGML_TYPE_Q4_0: {uint vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0],k_packed_q4_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1]));uint shift = (iqs & 0x10) >> 2;vui >>= shift;return int32_t(vui & 0x0F0F0F0F);}- case FA_TYPE_Q4_1: { // uses packed32 alias+ case GGML_TYPE_Q4_1: { // uses packed32 aliasuint vui = k_packed_q4_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4];uint shift = (iqs & 0x10) >> 2;vui >>= shift;return int32_t(vui & 0x0F0F0F0F);}- case FA_TYPE_Q5_0: {+ case GGML_TYPE_Q5_0: {uint vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 0],k_packed_q5_0.data[a_offset + ib].qs[(iqs & 0xF) / 2 + 1]));uint qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0],@@ -27,7 +27,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) {uint qh_bits = (qh >> iqs) & 0xF;return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u);}- case FA_TYPE_Q5_1: { // qs via packed32, qh via packed16+ case GGML_TYPE_Q5_1: { // qs via packed32, qh via packed16uint vui = k_packed_q5_1_p32.data[a_offset + ib].qs[(iqs & 0xF) / 4];uint qh = k_packed_q5_1.data[a_offset + ib].qh;uint shift = (iqs & 0x10) >> 2;@@ -35,7 +35,7 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) {uint qh_bits = (qh >> iqs) & 0xF;return int32_t(vui & 0x0F0F0F0F) | int32_t((qh_bits * 0x02040810u) & 0x10101010u);}- case FA_TYPE_Q8_0: {+ case GGML_TYPE_Q8_0: {return pack32(i16vec2(k_packed_q8_0.data[a_offset + ib].qs[iqs / 2],k_packed_q8_0.data[a_offset + ib].qs[iqs / 2 + 1]));}@@ -47,11 +47,11 @@ int32_t get_k_qs(uint ib, uint iqs, uint a_offset) {// return (d, 0) so call sites always see the same shape.FLOAT_TYPEV2 get_k_scale(uint ib, uint a_offset) {switch (FaTypeK) {- case FA_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0);- case FA_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm);- case FA_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0);- case FA_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm);- case FA_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0);+ case GGML_TYPE_Q4_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + ib].d), 0.0);+ case GGML_TYPE_Q4_1: return FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + ib].dm);+ case GGML_TYPE_Q5_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + ib].d), 0.0);+ case GGML_TYPE_Q5_1: return FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + ib].dm);+ case GGML_TYPE_Q8_0: return FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + ib].d), 0.0);default: return FLOAT_TYPEV2(0);}}@@ -61,16 +61,16 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c// explicit casts. The bit pattern is what we care about here -- the actual// signed/unsigned interpretation happens downstream in the dot product.switch (FaTypeK) {- case FA_TYPE_Q4_0: {+ case GGML_TYPE_Q4_0: {kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2],k_packed_q4_0.data[a_offset + global_ib].qs[iqs * 2 + 1])));break;}- case FA_TYPE_Q4_1: {+ case GGML_TYPE_Q4_1: {kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q4_1_p32.data[a_offset + global_ib].qs[iqs]);break;}- case FA_TYPE_Q5_0: {+ case GGML_TYPE_Q5_0: {kblocksh[buf_ib].qs[iqs] = int32_t(pack32(u16vec2(k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2],k_packed_q5_0.data[a_offset + global_ib].qs[iqs * 2 + 1])));if (iqs == 0) {@@ -79,14 +79,14 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, c}break;}- case FA_TYPE_Q5_1: {+ case GGML_TYPE_Q5_1: {kblocksh[buf_ib].qs[iqs] = int32_t(k_packed_q5_1_p32.data[a_offset + global_ib].qs[iqs]);if (iqs == 0) {kblocksh[buf_ib].qh = k_packed_q5_1.data[a_offset + global_ib].qh;}break;}- case FA_TYPE_Q8_0: {+ case GGML_TYPE_Q8_0: {kblocksh[buf_ib].qs[iqs] = pack32(i16vec2(k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2],k_packed_q8_0.data[a_offset + global_ib].qs[iqs * 2 + 1]));break;@@ -96,11 +96,11 @@ void k_block_to_shmem(const uint buf_ib, const uint global_ib, const uint iqs, cif (iqs == 0) {// Q4_0/Q5_0/Q8_0 store dm.x = d; Q4_1/Q5_1 store dm = (d, m) pair.switch (FaTypeK) {- case FA_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break;- case FA_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break;- case FA_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break;- case FA_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break;- case FA_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break;+ case GGML_TYPE_Q4_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q4_0.data[a_offset + global_ib].d), 0.0); break;+ case GGML_TYPE_Q4_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q4_1_p32.data[a_offset + global_ib].dm); break;+ case GGML_TYPE_Q5_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q5_0.data[a_offset + global_ib].d), 0.0); break;+ case GGML_TYPE_Q5_1: kblocksh[buf_ib].dm = FLOAT_TYPEV2(k_packed_q5_1_p32.data[a_offset + global_ib].dm); break;+ case GGML_TYPE_Q8_0: kblocksh[buf_ib].dm = FLOAT_TYPEV2(FLOAT_TYPE(k_packed_q8_0.data[a_offset + global_ib].d), 0.0); break;}}}@@ -121,31 +121,31 @@ struct fa_k_qs_block8 {fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) {fa_k_qs_block8 r;uint qh = 0;- if (FaTypeK == FA_TYPE_Q5_0) {+ if (FaTypeK == GGML_TYPE_Q5_0) {qh = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qh[0],k_packed_q5_0.data[a_offset + ib].qh[1]));- } else if (FaTypeK == FA_TYPE_Q5_1) {+ } else if (FaTypeK == GGML_TYPE_Q5_1) {qh = k_packed_q5_1.data[a_offset + ib].qh;}- const bool has_qh = (FaTypeK == FA_TYPE_Q5_0) || (FaTypeK == FA_TYPE_Q5_1);+ const bool has_qh = (FaTypeK == GGML_TYPE_Q5_0) || (FaTypeK == GGML_TYPE_Q5_1);[[unroll]] for (uint32_t d = 0; d < 4; d++) {uint vui = 0;switch (FaTypeK) {- case FA_TYPE_Q4_0: { // packed16+ case GGML_TYPE_Q4_0: { // packed16vui = pack32(u16vec2(k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 0],k_packed_q4_0.data[a_offset + ib].qs[d * 2 + 1]));break;}- case FA_TYPE_Q4_1: { // packed32 alias+ case GGML_TYPE_Q4_1: { // packed32 aliasvui = k_packed_q4_1_p32.data[a_offset + ib].qs[d];break;}- case FA_TYPE_Q5_0: { // packed16+ case GGML_TYPE_Q5_0: { // packed16vui = pack32(u16vec2(k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 0],k_packed_q5_0.data[a_offset + ib].qs[d * 2 + 1]));break;}- case FA_TYPE_Q5_1: { // packed32 alias+ case GGML_TYPE_Q5_1: { // packed32 aliasvui = k_packed_q5_1_p32.data[a_offset + ib].qs[d];break;}@@ -164,21 +164,21 @@ fa_k_qs_block8 get_k_qs_block8(uint ib, uint a_offset) {int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) {switch (FaTypeK) {- case FA_TYPE_Q4_0:- case FA_TYPE_Q4_1: {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1: {uint sub = pos % 4;uint shift = ((pos % 8) >= 4) ? 4u : 0u;return int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu);}- case FA_TYPE_Q5_0:- case FA_TYPE_Q5_1: {+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1: {uint sub = pos % 4;uint shift = ((pos % 8) >= 4) ? 4u : 0u;int32_t result = int32_t((uint(kblocksh[buf_ib].qs[sub]) >> shift) & 0x0F0F0F0Fu);uint qh_bits = (kblocksh[buf_ib].qh >> (pos * 4u)) & 0xFu;return result | int32_t((qh_bits * 0x02040810u) & 0x10101010u);}- case FA_TYPE_Q8_0: {+ case GGML_TYPE_Q8_0: {return kblocksh[buf_ib].qs[pos];}default: return 0;@@ -187,10 +187,10 @@ int32_t get_k_qs_shmem(const uint buf_ib, const uint pos) {ACC_TYPE k_dot_correction(const uint qib, const ACC_TYPEV2 k_dm) {switch (FaTypeK) {- case FA_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x;- case FA_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x;- case FA_TYPE_Q4_1:- case FA_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y;+ case GGML_TYPE_Q4_0: return -ACC_TYPE(8.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x;+ case GGML_TYPE_Q5_0: return -ACC_TYPE(16.0) * ACC_TYPE(Qf[qib].ds.y) * k_dm.x;+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_1: return ACC_TYPE(Qf[qib].ds.y) * k_dm.y;default: return ACC_TYPE(0.0);}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glslnew file mode 100644index 0000000000..0f10c733dd--- /dev/null+++ b/ggml/src/ggml-vulkan/vulkan-shaders/ggml_type_ids.glsl@@ -0,0 +1,34 @@+#if !defined(GGML_TYPE_IDS_COMP)+#define GGML_TYPE_IDS_COMP++// ggml_type enum values — must match ggml.h+#define GGML_TYPE_F32 0u+#define GGML_TYPE_F16 1u+#define GGML_TYPE_Q4_0 2u+#define GGML_TYPE_Q4_1 3u+#define GGML_TYPE_Q5_0 6u+#define GGML_TYPE_Q5_1 7u+#define GGML_TYPE_Q8_0 8u+#define GGML_TYPE_Q2_K 10u+#define GGML_TYPE_Q3_K 11u+#define GGML_TYPE_Q4_K 12u+#define GGML_TYPE_Q5_K 13u+#define GGML_TYPE_Q6_K 14u+#define GGML_TYPE_IQ2_XXS 16u+#define GGML_TYPE_IQ2_XS 17u+#define GGML_TYPE_IQ3_XXS 18u+#define GGML_TYPE_IQ1_S 19u+#define GGML_TYPE_IQ4_NL 20u+#define GGML_TYPE_IQ3_S 21u+#define GGML_TYPE_IQ2_S 22u+#define GGML_TYPE_IQ4_XS 23u+#define GGML_TYPE_IQ1_M 29u+#define GGML_TYPE_BF16 30u+#define GGML_TYPE_TQ1_0 34u+#define GGML_TYPE_TQ2_0 35u+#define GGML_TYPE_MXFP4 39u+#define GGML_TYPE_NVFP4 40u+#define GGML_TYPE_Q1_0 41u+#define GGML_TYPE_Q2_0 42u++#endif // !defined(GGML_TYPE_IDS_COMP)diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glslnew file mode 100644index 0000000000..12e50ee9eb--- /dev/null+++ b/ggml/src/ggml-vulkan/vulkan-shaders/iq_shmem_init.glsl@@ -0,0 +1,2 @@+void init_iq_shmem(uvec3 wgsize) {+}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.compindex ba76ec72ca..9b34d83662 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp@@ -12,9 +12,9 @@#include "types.glsl"#include "fa_types.glsl"-#define FaTypeV FA_TYPE_F32+#define FaTypeV GGML_TYPE_F32-layout(constant_id = 0) const uint FaTypeK = FA_TYPE_F32;+layout(constant_id = 0) const uint FaTypeK = GGML_TYPE_F32;layout(constant_id = 1) const uint FaBlockBytesK = 4;layout(constant_id = 2) const uint SUBGROUP_SIZE = 32;@@ -84,11 +84,11 @@ void main() {const uint k_block_elems = fa_block_elems(FaTypeK);const uint k_elem_bytes = FaBlockBytesK / k_block_elems;- if (FaTypeK == FA_TYPE_F16) {+ if (FaTypeK == GGML_TYPE_F16) {k_row[tid] = float(k_f16[k_offset / k_elem_bytes + tid]);- } else if (FaTypeK == FA_TYPE_F32) {+ } else if (FaTypeK == GGML_TYPE_F32) {k_row[tid] = k_f32[k_offset / k_elem_bytes + tid];- } else if (FaTypeK == FA_TYPE_BF16) {+ } else if (FaTypeK == GGML_TYPE_BF16) {k_row[tid] = bf16_to_fp32(uint(k_bf16[k_offset / k_elem_bytes + tid]));} else if (4 * tid < HEAD_SIZE) {const uint coord = 4 * tid;diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.compindex 63c4aaebcb..90e4e11cde 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp@@ -9,6 +9,9 @@#if defined(DATA_A_IQ1_M)#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require#endif+#if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16)+#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require+#endif#if defined(DATA_A_BF16) && defined(COOPMAT)#extension GL_EXT_bfloat16 : enable@@ -28,24 +31,54 @@#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require#endif+#ifdef MULMAT_QUANT+#include "ggml_type_ids.glsl"+layout (constant_id = 12) const uint MmTypeA = 0;+#endif+#include "types.glsl"#include "dot_product_funcs.glsl"+#ifndef MULMAT_QUANT#ifndef LOAD_VEC_A#define LOAD_VEC_A 1#endif+#endif#ifndef LOAD_VEC_B#define LOAD_VEC_B 1#endiflayout (constant_id = 11) const uint ALIGNED = 0;+#ifdef MULMAT_QUANT++uint mm_load_vec_a() {+ switch (MmTypeA) {+ case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_1:+ return 8u;+ case GGML_TYPE_Q2_0:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ return 4u;+ default:+ return 2u;+ }+}+#endif+#if !defined(TO_FLOAT_TYPE)#define TO_FLOAT_TYPE FLOAT_TYPE#endiflayout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;+#ifndef MULMAT_QUANTlayout (binding = 0) readonly buffer A {A_TYPE data_a[];};#if defined(DATA_A_F32)layout (binding = 0) readonly buffer A_SCALAR {float data_a_scalar[];};@@ -60,6 +93,30 @@ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16#if defined(A_TYPE_PACKED32)layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};#endif+#else+// Unpacked struct aliases+layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0;+layout (binding = 0) readonly buffer BUF_Q2_0 { block_q2_0 data[]; } a_q2_0;+layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k;+layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k;+layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k;+layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k;+layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k;+layout (binding = 0) readonly buffer BUF_TQ1_0 { block_tq1_0 data[]; } a_tq1_0;+layout (binding = 0) readonly buffer BUF_TQ2_0 { block_tq2_0 data[]; } a_tq2_0;+// Packed16 aliases+layout (binding = 0) readonly buffer BUF_Q4_0_P16 { block_q4_0_packed16 data[]; } a_q4_0_p16;+layout (binding = 0) readonly buffer BUF_Q5_0_P16 { block_q5_0_packed16 data[]; } a_q5_0_p16;+layout (binding = 0) readonly buffer BUF_Q8_0_P16 { block_q8_0_packed16 data[]; } a_q8_0_p16;+layout (binding = 0) readonly buffer BUF_Q3_K_P16 { block_q3_K_packed16 data[]; } a_q3_k_p16;+layout (binding = 0) readonly buffer BUF_Q6_K_P16 { block_q6_K_packed16 data[]; } a_q6_k_p16;+// Packed32 aliases+layout (binding = 0) readonly buffer BUF_Q4_1_P32 { block_q4_1_packed32 data[]; } a_q4_1_p32;+layout (binding = 0) readonly buffer BUF_Q5_1_P32 { block_q5_1_packed32 data[]; } a_q5_1_p32;+layout (binding = 0) readonly buffer BUF_Q2_K_P32 { block_q2_K_packed32 data[]; } a_q2_k_p32;+layout (binding = 0) readonly buffer BUF_Q4_K_P32 { block_q4_K_packed32 data[]; } a_q4_k_p32;+layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]; } a_q5_k_p32;+#endiflayout (binding = 1) readonly buffer B {B_TYPE data_b[];};layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];};@@ -121,8 +178,13 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit#endif#ifdef COOPMAT+#ifdef MULMAT_QUANT+layout(constant_id = 13) const uint SHMEM_STRIDE_PAD = 4;+layout(constant_id = 14) const bool APPLY_SLM_A_RESHAPE = false;+#elselayout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4;layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false;+#endif#elseconst uint SHMEM_STRIDE_PAD = 1;const bool APPLY_SLM_A_RESHAPE = false;@@ -141,6 +203,10 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS];#include "mul_mm_id_funcs.glsl"#include "mul_mm_funcs.glsl"+#ifdef MULMAT_QUANT+#include "iq_shmem_init.glsl"+#endif+void main() {const uint ic = gl_WorkGroupID.y;@@ -150,7 +216,7 @@ void main() {return;}#endif-#ifdef NEEDS_INIT_IQ_SHMEM+#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)init_iq_shmem(gl_WorkGroupSize);#endif@@ -200,9 +266,12 @@ void main() {#if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)const uint LOAD_VEC_A_EFF = (ALIGNED != 0) ? LOAD_VEC_A : 1;const uint LOAD_VEC_BATCH_A = (ALIGNED != 0) ? 1 : 2;-#else+#elif !defined(MULMAT_QUANT)const uint LOAD_VEC_A_EFF = LOAD_VEC_A;const uint LOAD_VEC_BATCH_A = 1;+#else+ const uint LOAD_VEC_A_EFF = mm_load_vec_a();+ const uint LOAD_VEC_BATCH_A = 1;#endifconst uint LOAD_VEC_B_EFF = (ALIGNED != 0) ? LOAD_VEC_B : 1;const uint LOAD_VEC_BATCH_B = (ALIGNED != 0) ? 1 : 2;diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.compindex 27f3178e7f..189788a862 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp@@ -21,6 +21,13 @@#extension GL_EXT_bfloat16 : enable#endif+#include "ggml_type_ids.glsl"++#ifdef MULMAT_QUANT+layout (constant_id = 7) const uint MmTypeA = 0;+layout (constant_id = 8) const uint MmABlockBytes = 2;+#endif+#include "types.glsl"#include "utils.glsl"@@ -37,6 +44,24 @@ layout (constant_id = 4) const bool enable_smaller_matrices = false;const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN;const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN;layout (constant_id = 5) const uint ALIGNED = 0;+layout (constant_id = 6) const uint subgroup_size = 32;++#ifdef MULMAT_QUANT++uint mm_quant_k() {+ switch (MmTypeA) {+ case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return 32u;+ case GGML_TYPE_Q1_0:+ return 128u;+ case GGML_TYPE_Q2_0:+ return 64u;+ default:+ return 256u;+ }+}+#endiflayout (push_constant) uniform parameter{@@ -72,21 +97,73 @@ layout (push_constant) uniform parameter} p;+#ifndef MULMAT_QUANTlayout (binding = 0) readonly buffer A {A_TYPE data_a[];};+#else+layout (binding = 0) readonly buffer A {uint8_t data_a[];};+#endiflayout (binding = 1) readonly buffer B {B_TYPE data_b[];};layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};#if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];};#endif-#if QUANT_K > 1+#if defined(MULMAT_QUANT) || QUANT_K > 1#include "dequant_funcs_cm2.glsl"+#ifndef MULMAT_QUANT+// Per-type path: use the alias set by dequant_funcs_cm2.glsl#if defined(dequantFuncA_v) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)#define DECODEFUNCA , dequantFuncA, dequantFuncA_v#else#define DECODEFUNCA , dequantFuncA#endif#else+layout(buffer_reference, std430, buffer_reference_align = 1) buffer decodeBufA {+ uint8_t raw[MmABlockBytes];+};++float16_t mmDecodeA(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) {+ switch (MmTypeA) {+ case GGML_TYPE_Q1_0: return dequantFuncQ1_0 (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q2_0: return dequantFuncQ2_0 (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_0: return dequantFuncQ4_0 (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_1: return dequantFuncQ4_1 (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_0: return dequantFuncQ5_0 (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_1: return dequantFuncQ5_1 (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q8_0: return dequantFuncQ8_0 (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q2_K: return dequantFuncQ2_K (decodeBufQ2_K (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q3_K: return dequantFuncQ3_K (decodeBufQ3_K (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q6_K: return dequantFuncQ6_K (decodeBufQ6_K (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_TQ1_0: return dequantFuncTQ1_0(decodeBufTQ1_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock);+ default: return float16_t(0);+ }+}++#ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR+f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], const in uint coordInBlock[2]) {+ switch (MmTypeA) {+ case GGML_TYPE_Q1_0: return dequantFuncQ1_0_v (decodeBufQ1_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q2_0: return dequantFuncQ2_0_v (decodeBufQ2_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_0: return dequantFuncQ4_0_v (decodeBufQ4_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q4_1: return dequantFuncQ4_1_v (decodeBufQ4_1 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_0: return dequantFuncQ5_0_v (decodeBufQ5_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q5_1: return dequantFuncQ5_1_v (decodeBufQ5_1 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q8_0: return dequantFuncQ8_0_v (decodeBufQ8_0 (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q2_K: return dequantFuncQ2_K_v (decodeBufQ2_K (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q3_K: return dequantFuncQ3_K_v (decodeBufQ3_K (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_Q6_K: return dequantFuncQ6_K_v (decodeBufQ6_K (bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_TQ1_0: return dequantFuncTQ1_0_v(decodeBufTQ1_0(bl_in), blockCoords, coordInBlock);+ case GGML_TYPE_TQ2_0: return dequantFuncTQ2_0_v(decodeBufTQ2_0(bl_in), blockCoords, coordInBlock);+ default: return f16vec4(0);+ }+}+#define DECODEFUNCA , mmDecodeA, mmDecodeA_v+#else+#define DECODEFUNCA , mmDecodeA+#endif+#endif+#else#define DECODEFUNCA#endif@@ -114,7 +191,6 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB {};uint _ne1;-layout (constant_id = 6) const uint subgroup_size = 32;shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size];B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2])@@ -246,6 +322,10 @@ void load_row_ids_hoisted(uint expert_idx, uint ic) {}#endif+#ifdef MULMAT_QUANT+#include "iq_shmem_init.glsl"+#endif+void main() {const uint tid = gl_LocalInvocationIndex;const uint ic = gl_WorkGroupID.y;@@ -264,7 +344,7 @@ void main() {#endif#endif-#ifdef NEEDS_INIT_IQ_SHMEM+#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)init_iq_shmem(gl_WorkGroupSize);#endif@@ -305,22 +385,33 @@ void main() {const uint end_k = min(p.K, (ik + 1) * p.k_split);#endif+#ifdef MULMAT_QUANT+ const uint qk = mm_quant_k();+#else+ const uint qk = QUANT_K;+#endif+#ifdef MUL_MAT_ID- uint pos_a = expert_idx * (p.batch_stride_a / QUANT_K);+ uint pos_a = expert_idx * (p.batch_stride_a / qk);uint pos_b = 0;#else- uint pos_a = batch_idx_a * (p.batch_stride_a / QUANT_K);+ uint pos_a = batch_idx_a * (p.batch_stride_a / qk);uint pos_b = batch_idx * p.batch_stride_b;uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;#endif- uint stride_a = p.stride_a / QUANT_K;+#ifdef MULMAT_QUANT+ // pos_a is a byte offset into the raw buffer; strides stay in block units+ pos_a *= MmABlockBytes;+#endif++ uint stride_a = p.stride_a / qk;uint stride_b = p.stride_b;// Hint to the compiler that values are aligned (want 16B alignment).// Quants are always block-aligned, no alignment needed.if (ALIGNED != 0) {-#if QUANT_K == 1+#if !defined(MULMAT_QUANT) && QUANT_K == 1stride_a &= ~7;#endifstride_b &= ~7;@@ -335,10 +426,8 @@ void main() {#endiftensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);-#if QUANT_K > 1- tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K);- tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K);-#endif+ tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, qk);+ tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, qk);#if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK);#endif@@ -368,19 +457,19 @@ void main() {const uint START_ALIGN_K = 256;// For Qi_K (block size 256), unroll whole 256 element tiles.// For legacy quants (block size 32), unroll 8x.- const uint UNROLL_K = (QUANT_K == 256) ? 256 : (BK * 8);+ const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8);const uint unroll_count = UNROLL_K / BK;// Detect a fast path where all loads are entirely in bounds and no clamping is requiredif ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&-#if QUANT_K == 1+#if !defined(MULMAT_QUANT) && QUANT_K == 1(stride_a % 8) == 0 &&#endif(stride_b % 8) == 0) {// Hint to the compiler that values are aligned (want 16B alignment)start_k &= ~(START_ALIGN_K-1);stride_b &= ~7;-#if QUANT_K == 1+#if !defined(MULMAT_QUANT) && QUANT_K == 1stride_a &= ~7;#endif@@ -551,10 +640,10 @@ void main() {[[dont_unroll]]for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {- if ((block_k % QUANT_K) == 0) {+ if ((block_k % qk) == 0) {store_scales(tid);}- if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {+ if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) {fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);}@@ -595,10 +684,10 @@ void main() {[[dont_unroll]]for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {- if ((block_k % QUANT_K) == 0) {+ if ((block_k % qk) == 0) {store_scales(tid);}- if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {+ if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) {fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);}@@ -639,10 +728,10 @@ void main() {[[dont_unroll]]for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {- if ((block_k % QUANT_K) == 0) {+ if ((block_k % qk) == 0) {store_scales(tid);}- if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {+ if (block_k + BK < end_k && ((block_k + BK) % qk) == 0) {fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glslindex bdc70af140..dd05fb1bde 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl@@ -20,669 +20,698 @@ void store_a(uint m, uint k_pair, FLOAT_TYPEV2 value) {void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) {#if defined(DATA_A_F32) || defined(DATA_A_F16)#if LOAD_VEC_A == 8- if (ALIGNED != 0) {- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint k_pair = row * LOAD_VEC_A / 2;- FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]);- store_a(col, k_pair, aa[0].xy);- store_a(col, k_pair + 1, aa[0].zw);- store_a(col, k_pair + 2, aa[1].xy);- store_a(col, k_pair + 3, aa[1].zw);- return;- }+ if (ALIGNED != 0) {+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;+ FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]);+ store_a(col, k_pair, aa[0].xy);+ store_a(col, k_pair + 1, aa[0].zw);+ store_a(col, k_pair + 2, aa[1].xy);+ store_a(col, k_pair + 3, aa[1].zw);+ return;+ }#elif LOAD_VEC_A == 4- if (ALIGNED != 0) {- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint k_pair = row * LOAD_VEC_A / 2;- FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]);- store_a(col, k_pair, aa.xy);- store_a(col, k_pair + 1, aa.zw);- return;- }+ if (ALIGNED != 0) {+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;+ FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]);+ store_a(col, k_pair, aa.xy);+ store_a(col, k_pair + 1, aa.zw);+ return;+ }#endif- const uint idx = pos_a + col * p.stride_a + row * 2;- if (idx_m < p.M && block + row * 2 + 1 < end_k) {- store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx],- data_a_scalar[idx + 1]));- } else if (idx_m < p.M && block + row * 2 < end_k) {- store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f));- } else {- store_a(col, row, FLOAT_TYPEV2(0.0f));- }+ const uint idx = pos_a + col * p.stride_a + row * 2;+ if (idx_m < p.M && block + row * 2 + 1 < end_k) {+ store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx],+ data_a_scalar[idx + 1]));+ } else if (idx_m < p.M && block + row * 2 < end_k) {+ store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f));+ } else {+ store_a(col, row, FLOAT_TYPEV2(0.0f));+ }#elif defined(DATA_A_BF16)#if LOAD_VEC_A == 4- if (ALIGNED != 0) {- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint k_pair = row * LOAD_VEC_A / 2;- FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx]));- store_a(col, k_pair, aa.xy);- store_a(col, k_pair + 1, aa.zw);- return;- }+ if (ALIGNED != 0) {+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;+ FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx]));+ store_a(col, k_pair, aa.xy);+ store_a(col, k_pair + 1, aa.zw);+ return;+ }#endif- const uint idx = pos_a + col * p.stride_a + row * 2;- if (idx_m < p.M && block + row * 2 + 1 < end_k) {- store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]),- TO_FLOAT_TYPE(data_a_scalar[idx + 1])));- } else if (idx_m < p.M && block + row * 2 < end_k) {- store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f));- } else {- store_a(col, row, FLOAT_TYPEV2(0.0f));- }-#elif defined(DATA_A_Q4_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 4;- const uint iqs = idx & 0x03;-- const float d = float(data_a_packed16[ib].d);- const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16);- const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d;- const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d;-- const uint k_pair = row * LOAD_VEC_A / 4;- store_a(col, k_pair, FLOAT_TYPEV2(v0.xy));- store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw));- store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy));- store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw));-#elif defined(DATA_A_Q4_1)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 4;- const uint iqs = idx & 0x03;-- const vec2 dm = vec2(data_a_packed32[ib].dm);- const uint vui = data_a_packed32[ib].qs[iqs];- const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y;- const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y;-- const uint k_pair = row * LOAD_VEC_A / 4;- store_a(col, k_pair, FLOAT_TYPEV2(v0.xy));- store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw));- store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy));- store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw));-#elif defined(DATA_A_Q5_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 8;- const uint iqs = idx & 0x07;-- const float d = float(data_a_packed16[ib].d);- const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]);- const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10);- const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10);-- const uint vui = uint(data_a_packed16[ib].qs[iqs]);- const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d;- store_a(col, row, FLOAT_TYPEV2(v.xz));- store_a(col, row + 8, FLOAT_TYPEV2(v.yw));-#elif defined(DATA_A_Q5_1)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 4;- const uint iqs = idx & 0x03;-- const vec2 dm = vec2(data_a_packed32[ib].dm);- const uint uint_qh = data_a_packed32[ib].qh;- const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10);- const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10);- const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10);- const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10);-- const uint vui = data_a_packed32[ib].qs[iqs];- const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y;- const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y;-- const uint k_pair = row * LOAD_VEC_A / 4;- store_a(col, k_pair, FLOAT_TYPEV2(v0.xz));- store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz));- store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw));- store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw));-#elif defined(DATA_A_Q8_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 8;- const uint iqs = idx & 0x07;-- const float d = float(data_a_packed16[ib].d);- const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147- const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy;- const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(v.xy));- store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));-#elif defined(DATA_A_Q1_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 16;- const uint iqs = idx & 0xfu;-- const float d = float(data_a[ib].d);- const uint bits = uint(data_a[ib].qs[iqs]);-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d));- store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d));- store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d));- store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d));-#elif defined(DATA_A_Q2_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 16;- const uint iqs = idx & 0xfu;-- const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib].d);- const uint bits = uint(data_a[ib].qs[iqs]);-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f)));- store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f)));-#elif defined(DATA_A_Q2_K)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 64; // 4 values per idx- const uint iqs = (idx % 64) * 2; // 0,2,4..126-- const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15- const uint scalesi = iqs / 8; // 0..15- const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6-- const vec4 qs = vec4(unpack8((data_a_packed32[ib].qs[qsi / 2] >> qsshift) & 0x03030303));- const uint scales = data_a[ib].scales[scalesi];- const vec2 dm = vec2(data_a[ib].dm);-- const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4);-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(v.xy));- store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));-#elif defined(DATA_A_TQ1_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 128; // 2 values per idx- const uint iqs = (idx % 128) * 2; // element 0,2,4..254-- const float d = float(data_a[ib].d);- vec2 v;- for (uint kk = 0u; kk < 2u; ++kk) {- const uint e = iqs + kk;- const uint bidx = tq1_0_byte_of(e);- const uint qbyte = uint(bidx < 48u ? data_a[ib].qs[bidx]- : data_a[ib].qh[bidx - 48u]);- v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0);- }-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(v.xy));-#elif defined(DATA_A_TQ2_0)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 128; // 2 values per idx- const uint iqs = (idx % 128) * 2; // elem 0,2,4..254-- const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start- const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6-- const uvec2 qs = uvec2(data_a[ib].qs[qsi], data_a[ib].qs[qsi + 1]);- const float d = float(data_a[ib].d);-- const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0);-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(v.xy));-#elif defined(DATA_A_Q3_K)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 128; // 2 values per idx- const uint iqs = idx % 128; // 0..127-- const uint n = iqs / 64; // 0,1- const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62- const uint hmi = (iqs % 16) * 2; // 0,2,4..30- const uint j = (iqs % 64) / 4; // 0..3- const uint is = iqs / 8; // 0..15- const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3- const uint qsshift = halfsplit * 2; // 0,2,4,6-- const int8_t us = int8_t(((data_a[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF)- | (((data_a[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4));- const float dl = float(data_a[ib].d) * float(us - 32);-- const vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy);- const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy);-- store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(dl * (qs.x - hm.x),- dl * (qs.y - hm.y)));-#elif defined(DATA_A_Q4_K)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 64; // 4 values per idx- const uint iqs = (idx % 64) * 2; // 0,2,4..126-- const uint n = iqs / 32; // 0,1,2,3- const uint b = (iqs % 32) / 16; // 0,1- const uint is = 2 * n + b; // 0..7- const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126-- const vec2 loadd = vec2(data_a[ib].dm);-- const uvec3 scales = uvec3(data_a_packed32[ib].scales[0],- data_a_packed32[ib].scales[1],- data_a_packed32[ib].scales[2]);- const uint scalesoffs = (is & 3) * 8;-- const uint scidx0 = (is < 4) ? 0 : 2;- const uint scidxshift0 = scalesoffs;- const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;- const uint mbidx0 = (is < 4) ? 1 : 2;- const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;- const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;-- const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));- const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));-- const float d = loadd.x * sc;- const float m = -loadd.y * mbyte;-- const vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m)));- store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m)));-#elif defined(DATA_A_Q5_K)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 64; // 4 values per idx- const uint iqs = (idx % 64) * 2; // 0,2,4..126-- const uint n = iqs / 32; // 0,1,2,3- const uint b = (iqs % 32) / 16; // 0,1- const uint is = 2 * n + b; // 0..7- const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126- const uint qhi = (iqs % 16) * 2; // 0,2,4..30-- const vec2 loadd = vec2(data_a[ib].dm);-- const uvec3 scales = uvec3(data_a_packed32[ib].scales[0],- data_a_packed32[ib].scales[1],- data_a_packed32[ib].scales[2]);- const uint scalesoffs = (is & 3) * 8;-- const uint scidx0 = (is < 4) ? 0 : 2;- const uint scidxshift0 = scalesoffs;- const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;- const uint mbidx0 = (is < 4) ? 1 : 2;- const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;- const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;+ const uint idx = pos_a + col * p.stride_a + row * 2;+ if (idx_m < p.M && block + row * 2 + 1 < end_k) {+ store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]),+ TO_FLOAT_TYPE(data_a_scalar[idx + 1])));+ } else if (idx_m < p.M && block + row * 2 < end_k) {+ store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f));+ } else {+ store_a(col, row, FLOAT_TYPEV2(0.0f));+ }+#elif defined(DATA_A_IQ1_S)+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;- const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));- const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));-- const float d = loadd.x * sc;- const float m = -loadd.y * mbyte;-- const uint qs = (data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F;- const uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4;- const vec4 q = vec4(unpack8(qs | qh));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m)));- store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m)));-#elif defined(DATA_A_Q6_K)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 128; // 2 values per idx- const uint iqs = idx % 128; // 0..127-- const uint n = iqs / 64; // 0,1- const uint b = ((iqs % 64) / 32) * 4; // 0,4- const uint is_b = (iqs % 16) / 8; // 0,1- const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6- const uint is = 8 * n + qhshift + is_b; // 0..15- const uint qsi = n * 32 + (iqs % 32); // 0..63- const uint qhi = n * 16 + (iqs % 16); // 0..31-- const float dscale = float(data_a[ib].d) * float(data_a[ib].scales[is]);+ const uint ib = idx / 32;+ const uint ib32 = (idx % 32) / 4;+ const uint ib8 = idx % 32;- const uint ql = (uint(data_a_packed16[ib].ql[qsi]) >> b) & 0x0F0F;- const uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303;- const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale;+ const float d = float(data_a[ib].d);+ const uint qh = data_a[ib].qh[ib32];+ const uint qs = data_a[ib].qs[ib8];+ const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1);+ const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;+ const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]);- store_a(col, row * LOAD_VEC_A / 2, FLOAT_TYPEV2(q.x, q.y));-#elif defined(DATA_A_IQ1_S)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 32; // 8 values per idx- const uint ib32 = (idx % 32) / 4; // 0..7- const uint ib8 = idx % 32;-- const float d = float(data_a[ib].d);- const uint qh = data_a[ib].qh[ib32];- const uint qs = data_a[ib].qs[ib8];- const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1);- const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;- const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]);-- const uint k_pair = row * LOAD_VEC_A / 2;- [[unroll]] for (int k = 0; k < 4; ++k) {- store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),- dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta)));- }+ [[unroll]] for (int k = 0; k < 4; ++k) {+ store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),+ dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta)));++ }#elif defined(DATA_A_IQ1_M)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 32; // 8 values per idx- const uint ib8 = idx % 32;- const uint ib16 = ib8 / 2;-- const uint16_t[4] scales = data_a[ib].scales;- const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;- const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);- const uint sc = scales[ib8 / 8];- const uint qs = data_a[ib].qs[ib8];- const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1));- const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1);- const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;- const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]);-- const uint k_pair = row * LOAD_VEC_A / 2;- [[unroll]] for (int k = 0; k < 4; ++k) {- store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),- dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta)));- }+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;++ const uint ib = idx / 32;+ const uint ib8 = idx % 32;+ const uint ib16 = ib8 / 2;++ const uint16_t[4] scales = data_a[ib].scales;+ const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;+ const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);+ const uint sc = scales[ib8 / 8];+ const uint qs = data_a[ib].qs[ib8];+ const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1));+ const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1);+ const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;+ const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]);++ [[unroll]] for (int k = 0; k < 4; ++k) {+ store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),+ dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta)));++ }#elif defined(DATA_A_IQ2_XXS)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 32; // 8 values per idx- const uint ib32 = (idx % 32) / 4; // 0..7- const uint ib8 = idx % 4;-- const float d = float(data_a[ib].d);- const uint qs = data_a[ib].qs[8 * ib32 + ib8];- const uint signs = pack32(u8vec4(- data_a[ib].qs[8*ib32 + 4],- data_a[ib].qs[8*ib32 + 5],- data_a[ib].qs[8*ib32 + 6],- data_a[ib].qs[8*ib32 + 7]- ));- const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28)));- const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7);- const uint sign = sign7 | (bitCount(sign7) << 7);- const uvec2 grid = iq2xxs_grid[qs];- const vec4 grid0 = vec4(unpack8(grid.x));- const vec4 grid1 = vec4(unpack8(grid.y));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,- (sign & 2) != 0 ? -grid0.y : grid0.y));- store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,- (sign & 8) != 0 ? -grid0.w : grid0.w));- store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,- (sign & 32) != 0 ? -grid1.y : grid1.y));- store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,- (sign & 128) != 0 ? -grid1.w : grid1.w));+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;++ const uint ib = idx / 32;+ const uint ib32 = (idx % 32) / 4;+ const uint ib8 = idx % 4;++ const float d = float(data_a[ib].d);+ const uint qs = data_a[ib].qs[8 * ib32 + ib8];+ const uint signs = pack32(u8vec4(+ data_a[ib].qs[8*ib32 + 4],+ data_a[ib].qs[8*ib32 + 5],+ data_a[ib].qs[8*ib32 + 6],+ data_a[ib].qs[8*ib32 + 7]+ ));+ const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28)));+ const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7);+ const uint sign = sign7 | (bitCount(sign7) << 7);+ const uvec2 grid = iq2xxs_grid[qs];+ const vec4 grid0 = vec4(unpack8(grid.x));+ const vec4 grid1 = vec4(unpack8(grid.y));++ store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,+ (sign & 2) != 0 ? -grid0.y : grid0.y));++ store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,+ (sign & 8) != 0 ? -grid0.w : grid0.w));++ store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,+ (sign & 32) != 0 ? -grid1.y : grid1.y));++ store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,+ (sign & 128) != 0 ? -grid1.w : grid1.w));+#elif defined(DATA_A_IQ2_XS)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 32; // 8 values per idx- const uint ib32 = (idx % 32) / 4; // 0..7- const uint ib8 = idx % 4; // 0..3-- const float d = float(data_a[ib].d);- const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;- const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));- const uint qs = data_a[ib].qs[4 * ib32 + ib8];- const uint sign7 = qs >> 9;- const uint sign = sign7 | (bitCount(sign7) << 7);- const uvec2 grid = iq2xs_grid[qs & 511];- const vec4 grid0 = vec4(unpack8(grid.x));- const vec4 grid1 = vec4(unpack8(grid.y));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,- (sign & 2) != 0 ? -grid0.y : grid0.y));- store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,- (sign & 8) != 0 ? -grid0.w : grid0.w));- store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,- (sign & 32) != 0 ? -grid1.y : grid1.y));- store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,- (sign & 128) != 0 ? -grid1.w : grid1.w));+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;++ const uint ib = idx / 32;+ const uint ib32 = (idx % 32) / 4;+ const uint ib8 = idx % 4;++ const float d = float(data_a[ib].d);+ const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;+ const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));+ const uint qs = data_a[ib].qs[4 * ib32 + ib8];+ const uint sign7 = qs >> 9;+ const uint sign = sign7 | (bitCount(sign7) << 7);+ const uvec2 grid = iq2xs_grid[qs & 511];+ const vec4 grid0 = vec4(unpack8(grid.x));+ const vec4 grid1 = vec4(unpack8(grid.y));++ store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,+ (sign & 2) != 0 ? -grid0.y : grid0.y));++ store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,+ (sign & 8) != 0 ? -grid0.w : grid0.w));++ store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,+ (sign & 32) != 0 ? -grid1.y : grid1.y));++ store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,+ (sign & 128) != 0 ? -grid1.w : grid1.w));+#elif defined(DATA_A_IQ2_S)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 32; // 8 values per idx- const uint ib8 = idx % 32; // 0..31- const uint ib32 = ib8 / 4; // 0..7-- const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;- const uint qs = data_a[ib].qs[ib8];- const uint qh = data_a[ib].qh[ib32];- const uint qhshift = 2 * (ib8 % 4);- const uint sign = data_a[ib].qs[QUANT_K / 8 + ib8];-- const float d = float(data_a[ib].d);- const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));- const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)];- const vec4 grid0 = vec4(unpack8(grid.x));- const vec4 grid1 = vec4(unpack8(grid.y));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,- (sign & 2) != 0 ? -grid0.y : grid0.y));- store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,- (sign & 8) != 0 ? -grid0.w : grid0.w));- store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,- (sign & 32) != 0 ? -grid1.y : grid1.y));- store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,- (sign & 128) != 0 ? -grid1.w : grid1.w));+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;++ const uint ib = idx / 32;+ const uint ib8 = idx % 32;+ const uint ib32 = ib8 / 4;++ const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;+ const uint qs = data_a[ib].qs[ib8];+ const uint qh = data_a[ib].qh[ib32];+ const uint qhshift = 2 * (ib8 % 4);+ const uint sign = data_a[ib].qs[QUANT_K_IQ2_S / 8 + ib8];++ const float d = float(data_a[ib].d);+ const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));+ const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)];+ const vec4 grid0 = vec4(unpack8(grid.x));+ const vec4 grid1 = vec4(unpack8(grid.y));++ store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,+ (sign & 2) != 0 ? -grid0.y : grid0.y));++ store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,+ (sign & 8) != 0 ? -grid0.w : grid0.w));++ store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,+ (sign & 32) != 0 ? -grid1.y : grid1.y));++ store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,+ (sign & 128) != 0 ? -grid1.w : grid1.w));+#elif defined(DATA_A_IQ3_XXS)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 64; // 4 values per idx- const uint iqs = idx % 64; // 0..63- const uint is = QUANT_K / 4 + 4 * (iqs / 8); // 8 values-- const float d = float(data_a[ib].d);- const uint qs = data_a[ib].qs[iqs];- const uint signs = pack32(u16vec2(- data_a_packed16[ib].qs[is/2],- data_a_packed16[ib].qs[is/2+1]- ));- const float db = d * 0.5 * (0.5 + (signs >> 28));- const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);- const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2));- const uint grid = iq3xxs_grid[qs];- const vec4 v = db * vec4(unpack8(grid));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,- (sign & 2) != 0 ? -v.y : v.y));- store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,- (sign & 8) != 0 ? -v.w : v.w));+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;++ const uint ib = idx / 64;+ const uint iqs = idx % 64;+ const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8);++ const float d = float(data_a[ib].d);+ const uint qs = data_a[ib].qs[iqs];+ const uint signs = pack32(u16vec2(+ data_a_packed16[ib].qs[is/2],+ data_a_packed16[ib].qs[is/2+1]+ ));+ const float db = d * 0.5 * (0.5 + (signs >> 28));+ const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);+ const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2));+ const uint grid = iq3xxs_grid[qs];+ const vec4 v = db * vec4(unpack8(grid));++ store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,+ (sign & 2) != 0 ? -v.y : v.y));++ store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,+ (sign & 8) != 0 ? -v.w : v.w));+#elif defined(DATA_A_IQ3_S)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;-- const uint ib = idx / 64; // 4 values per idx- const uint iqs = idx % 64; // 0..63- const uint iqh = iqs / 8;-- const float d = float(data_a[ib].d);- const uint qs = data_a[ib].qs[iqs];- const uint qh = data_a[ib].qh[iqh];- const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2)));- const uint scale = data_a[ib].scales[iqs / 16];- const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign)));- const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));- const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];- const vec4 v = db * vec4(unpack8(grid));-- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,- (sign & 2) != 0 ? -v.y : v.y));- store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,- (sign & 8) != 0 ? -v.w : v.w));+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;++ const uint ib = idx / 64;+ const uint iqs = idx % 64;+ const uint iqh = iqs / 8;++ const float d = float(data_a[ib].d);+ const uint qs = data_a[ib].qs[iqs];+ const uint qh = data_a[ib].qh[iqh];+ const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2)));+ const uint scale = data_a[ib].scales[iqs / 16];+ const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign)));+ const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));+ const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];+ const vec4 v = db * vec4(unpack8(grid));++ store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,+ (sign & 2) != 0 ? -v.y : v.y));++ store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,+ (sign & 8) != 0 ? -v.w : v.w));+#elif defined(DATA_A_IQ4_XS)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 2;- const uint ib = idx / 64; // 4 values per idx- const uint ib32 = (idx % 64) / 8; // 0..7- const uint iq = 4 * ib32 + (idx % 4);+ const uint ib = idx / 64;+ const uint ib32 = (idx % 64) / 8;+ const uint iq = 4 * ib32 + (idx % 4);- const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF;- const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3;- const uint qshift = idx & 4;- u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F);+ const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF;+ const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3;+ const uint qshift = idx & 4;+ u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F);- const float d = float(data_a[ib].d);- const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]);+ const float d = float(data_a[ib].d);+ const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]);- const uint k_pair = row * LOAD_VEC_A / 2;- store_a(col, k_pair, FLOAT_TYPEV2(v.xy));- store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));#elif defined(DATA_A_IQ4_NL)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 4;++ const uint ib = idx / 8;+ const uint iqs = idx & 0x07;- const uint ib = idx / 8;- const uint iqs = idx & 0x07;+ const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d);+ const uint vui = uint(data_a_packed16[ib].qs[iqs]);- const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d);- const uint vui = uint(data_a_packed16[ib].qs[iqs]);+ store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],+ kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]));++ store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)],+ kvalues_iq4nl[vui >> 12]));- const uint k_pair = row * LOAD_VEC_A / 4;- store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],- kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]));- store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)],- kvalues_iq4nl[vui >> 12]));#elif defined(DATA_A_MXFP4)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint k_pair = row * LOAD_VEC_A / 4;- const uint ib = idx / 8;- const uint iqs = (idx & 0x07) * 2;+ const uint ib = idx / 8;+ const uint iqs = (idx & 0x07) * 2;- const uint vui = uint(data_a[ib].qs[iqs]);- const uint vui2 = uint(data_a[ib].qs[iqs+1]);+ const uint vui = uint(data_a[ib].qs[iqs]);+ const uint vui2 = uint(data_a[ib].qs[iqs+1]);#ifdef USE_OCP_FP4- const float d = e8m0_to_fp32(data_a[ib].e);- const u8vec2 packed = u8vec2(vui, vui2);- store_a(col, row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d));- store_a(col, row + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d));+ const float d = e8m0_to_fp32(data_a[ib].e);+ const u8vec2 packed = u8vec2(vui, vui2);+ store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d));#else- const float d = e8m0_to_fp32(data_a[ib].e) * 0.5;- store_a(col, row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,- kvalues_mxfp4[vui2 & 0xF] * d));- store_a(col, row + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,- kvalues_mxfp4[vui2 >> 4] * d));+ const float d = e8m0_to_fp32(data_a[ib].e) * 0.5;+ store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,+ kvalues_mxfp4[vui2 & 0xF] * d));++ store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,+ kvalues_mxfp4[vui2 >> 4] * d));+#endif#elif defined(DATA_A_NVFP4)- const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;- const uint ib = idx / 16u;- const uint sub = (idx & 0xC) >> 2;- const uint iqs = (idx & 0xF) * 2;- const uint vui = uint(data_a[ib].qs[iqs]);- const uint vui2 = uint(data_a[ib].qs[iqs+1]);-- // lo and hi nibbles are 8 elements apart, which doesn't quite line up with- // how the thread mapping and buf_idx calculation works for other types.- const uint eff_row = (row & 3) + (row & ~3) * 2;+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;+ const uint eff_row = (row & 3) + (row & ~3) * 2;++ const uint ib = idx / 16u;+ const uint sub = (idx & 0xC) >> 2;+ const uint iqs = (idx & 0xF) * 2;+ const uint vui = uint(data_a[ib].qs[iqs]);+ const uint vui2 = uint(data_a[ib].qs[iqs+1]);+#ifdef USE_OCP_FP4- const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub]));- const u8vec2 packed = u8vec2(vui, vui2);- store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d);- store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d);+ const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub]));+ const u8vec2 packed = u8vec2(vui, vui2);+ store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d);+ store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d);#else- const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5;- store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,- kvalues_mxfp4[vui2 & 0xF] * d));- store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,- kvalues_mxfp4[vui2 >> 4] * d));+ const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5;+ store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,+ kvalues_mxfp4[vui2 & 0xF] * d));+ store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,+ kvalues_mxfp4[vui2 >> 4] * d));#endif+#else+ if (MmTypeA == GGML_TYPE_Q4_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 4;++ const uint ib = idx / 4;+ const uint iqs = idx & 0x03;++ const float d = float(a_q4_0_p16.data[ib].d);+ const uint vui = uint(a_q4_0_p16.data[ib].qs[2*iqs]) | (uint(a_q4_0_p16.data[ib].qs[2*iqs + 1]) << 16);+ const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d;+ const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d;++ store_a(col, k_pair, FLOAT_TYPEV2(v0.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy));+ store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw));+ } else if (MmTypeA == GGML_TYPE_Q4_1) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 4;++ const uint ib = idx / 4;+ const uint iqs = idx & 0x03;++ const vec2 dm = vec2(a_q4_1_p32.data[ib].dm);+ const uint vui = a_q4_1_p32.data[ib].qs[iqs];+ const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y;+ const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y;++ store_a(col, k_pair, FLOAT_TYPEV2(v0.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy));+ store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw));+ } else if (MmTypeA == GGML_TYPE_Q5_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 4;++ const uint ib = idx / 8;+ const uint iqs = idx & 0x07;++ const float d = float(a_q5_0_p16.data[ib].d);+ const uint uint_qh = uint(a_q5_0_p16.data[ib].qh[1]) << 16 | uint(a_q5_0_p16.data[ib].qh[0]);+ const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10);+ const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10);++ const uint vui = uint(a_q5_0_p16.data[ib].qs[iqs]);+ const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d;++ store_a(col, k_pair, FLOAT_TYPEV2(v.xz));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v.yw));+ } else if (MmTypeA == GGML_TYPE_Q5_1) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 4;++ const uint ib = idx / 4;+ const uint iqs = idx & 0x03;++ const vec2 dm = vec2(a_q5_1_p32.data[ib].dm);+ const uint uint_qh = a_q5_1_p32.data[ib].qh;+ const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10);+ const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10);+ const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10);+ const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10);++ const uint vui = a_q5_1_p32.data[ib].qs[iqs];+ const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y;+ const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y;++ store_a(col, k_pair, FLOAT_TYPEV2(v0.xz));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz));+ store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw));+ store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw));+ } else if (MmTypeA == GGML_TYPE_Q8_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 8;+ const uint iqs = idx & 0x07;++ const float d = float(a_q8_0_p16.data[ib].d);+ const i8vec2 v0 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs])).xy; // vec4 used due to #12147+ const i8vec2 v1 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs + 1])).xy;+ const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;++ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));+ } else if (MmTypeA == GGML_TYPE_Q1_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 16;+ const uint iqs = idx & 0xfu;++ const float d = float(a_q1_0.data[ib].d);+ const uint bits = uint(a_q1_0.data[ib].qs[iqs]);++ store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d));+ store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d));+ store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d));+ store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d));+ } else if (MmTypeA == GGML_TYPE_Q2_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 16;+ const uint iqs = idx & 0xfu;++ const FLOAT_TYPE d = FLOAT_TYPE(a_q2_0.data[ib].d);+ const uint bits = uint(a_q2_0.data[ib].qs[iqs]);++ store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f)));+ store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f)));+ } else if (MmTypeA == GGML_TYPE_Q2_K) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 64; // 4 values per idx+ const uint iqs = (idx % 64) * 2; // 0,2,4..126++ const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15+ const uint scalesi = iqs / 8; // 0..15+ const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6++ const vec4 qs = vec4(unpack8((a_q2_k_p32.data[ib].qs[qsi / 2] >> qsshift) & 0x03030303));+ const uint scales = a_q2_k.data[ib].scales[scalesi];+ const vec2 dm = vec2(a_q2_k.data[ib].dm);++ const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4);++ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw));+ } else if (MmTypeA == GGML_TYPE_Q3_K) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 128; // 2 values per idx+ const uint iqs = idx % 128; // 0..127++ const uint n = iqs / 64; // 0,1+ const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62+ const uint hmi = (iqs % 16) * 2; // 0,2,4..30+ const uint j = (iqs % 64) / 4; // 0..3+ const uint is = iqs / 8; // 0..15+ const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3+ const uint qsshift = halfsplit * 2; // 0,2,4,6++ const int8_t us = int8_t(((a_q3_k.data[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF)+ | (((a_q3_k.data[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4));+ const float dl = float(a_q3_k.data[ib].d) * float(us - 32);++ const vec2 qs = vec2(unpack8((uint(a_q3_k_p16.data[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy);+ const vec2 hm = vec2(unpack8(((uint(a_q3_k_p16.data[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy);++ store_a(col, k_pair, FLOAT_TYPEV2(dl * (qs.x - hm.x),+ dl * (qs.y - hm.y)));++ } else if (MmTypeA == GGML_TYPE_Q4_K) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 64; // 4 values per idx+ const uint iqs = (idx % 64) * 2; // 0,2,4..126++ const uint n = iqs / 32; // 0,1,2,3+ const uint b = (iqs % 32) / 16; // 0,1+ const uint is = 2 * n + b; // 0..7+ const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126++ const vec2 loadd = vec2(a_q4_k.data[ib].dm);++ const uvec3 scales = uvec3(a_q4_k_p32.data[ib].scales[0],+ a_q4_k_p32.data[ib].scales[1],+ a_q4_k_p32.data[ib].scales[2]);+ const uint scalesoffs = (is & 3) * 8;++ const uint scidx0 = (is < 4) ? 0 : 2;+ const uint scidxshift0 = scalesoffs;+ const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;+ const uint mbidx0 = (is < 4) ? 1 : 2;+ const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;+ const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;++ const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));+ const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));++ const float d = loadd.x * sc;+ const float m = -loadd.y * mbyte;++ const vec4 q = vec4(unpack8((a_q4_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F));++ store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m)));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m)));+ } else if (MmTypeA == GGML_TYPE_Q5_K) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 64; // 4 values per idx+ const uint iqs = (idx % 64) * 2; // 0,2,4..126++ const uint n = iqs / 32; // 0,1,2,3+ const uint b = (iqs % 32) / 16; // 0,1+ const uint is = 2 * n + b; // 0..7+ const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126+ const uint qhi = (iqs % 16) * 2; // 0,2,4..30++ const vec2 loadd = vec2(a_q5_k.data[ib].dm);++ const uvec3 scales = uvec3(a_q5_k_p32.data[ib].scales[0],+ a_q5_k_p32.data[ib].scales[1],+ a_q5_k_p32.data[ib].scales[2]);+ const uint scalesoffs = (is & 3) * 8;++ const uint scidx0 = (is < 4) ? 0 : 2;+ const uint scidxshift0 = scalesoffs;+ const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;+ const uint mbidx0 = (is < 4) ? 1 : 2;+ const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;+ const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;++ const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));+ const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));++ const float d = loadd.x * sc;+ const float m = -loadd.y * mbyte;++ const uint qs = (a_q5_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F;+ const uint qh = ((a_q5_k_p32.data[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4;+ const vec4 q = vec4(unpack8(qs | qh));++ store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m)));+ store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m)));+ } else if (MmTypeA == GGML_TYPE_Q6_K) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;+ const uint k_pair = row * mm_load_vec_a() / 2;++ const uint ib = idx / 128; // 2 values per idx+ const uint iqs = idx % 128; // 0..127++ const uint n = iqs / 64; // 0,1+ const uint b = ((iqs % 64) / 32) * 4; // 0,4+ const uint is_b = (iqs % 16) / 8; // 0,1+ const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6+ const uint is = 8 * n + qhshift + is_b; // 0..15+ const uint qsi = n * 32 + (iqs % 32); // 0..63+ const uint qhi = n * 16 + (iqs % 16); // 0..31++ const float dscale = float(a_q6_k.data[ib].d) * float(a_q6_k.data[ib].scales[is]);++ const uint ql = (uint(a_q6_k_p16.data[ib].ql[qsi]) >> b) & 0x0F0F;+ const uint qh = (uint(a_q6_k_p16.data[ib].qh[qhi]) >> qhshift) & 0x0303;+ const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale;++ store_a(col, k_pair, FLOAT_TYPEV2(q.x, q.y));+ } else if (MmTypeA == GGML_TYPE_TQ1_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;++ const uint ib = idx / 128; // 2 values per idx+ const uint iqs = (idx % 128) * 2; // elem 0,2,4..254++ const float d = float(a_tq1_0.data[ib].d);+ vec2 v;+ for (uint kk = 0u; kk < 2u; ++kk) {+ const uint e = iqs + kk;+ const uint bidx = tq1_0_byte_of(e);+ const uint qbyte = uint(bidx < 48u ? a_tq1_0.data[ib].qs[bidx]+ : a_tq1_0.data[ib].qh[bidx - 48u]);+ v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0);+ }++ const uint k_pair = row * mm_load_vec_a() / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ } else if (MmTypeA == GGML_TYPE_TQ2_0) {+ const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row;++ const uint ib = idx / 128; // 2 values per idx+ const uint iqs = (idx % 128) * 2; // elem 0,2,4..254++ const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start+ const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6++ const uvec2 qs = uvec2(a_tq2_0.data[ib].qs[qsi], a_tq2_0.data[ib].qs[qsi + 1]);+ const float d = float(a_tq2_0.data[ib].d);++ const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0);++ const uint k_pair = row * mm_load_vec_a() / 2;+ store_a(col, k_pair, FLOAT_TYPEV2(v.xy));+ }#endif}#if !defined(MUL_MAT_ID)void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) {#if LOAD_VEC_B == 8- if (ALIGNED != 0) {- // Not supported for b_type bf16 because bf16mat2x4 does not exist- const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;- FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);- buf_b[buf_idx + 0] = bb[0].xy;- buf_b[buf_idx + 1] = bb[0].zw;- buf_b[buf_idx + 2] = bb[1].xy;- buf_b[buf_idx + 3] = bb[1].zw;- return;- }+ if (ALIGNED != 0) {+ // Not supported for b_type bf16 because bf16mat2x4 does not exist+ const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;+ FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);+ buf_b[buf_idx + 0] = bb[0].xy;+ buf_b[buf_idx + 1] = bb[0].zw;+ buf_b[buf_idx + 2] = bb[1].xy;+ buf_b[buf_idx + 3] = bb[1].zw;+ return;+ }#elif LOAD_VEC_B == 4- if (ALIGNED != 0) {- const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;+ if (ALIGNED != 0) {+ const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;#if defined(DATA_B_BF16)- FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));#else- FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);#endif- buf_b[buf_idx + 0] = bb.xy;- buf_b[buf_idx + 1] = bb.zw;- return;- }+ buf_b[buf_idx + 0] = bb.xy;+ buf_b[buf_idx + 1] = bb.zw;+ return;+ }#endif- const uint idx = pos_b + col * p.stride_b + row * 2;- const uint buf_idx = col * SHMEM_STRIDE + row;- if (idx_n < p.N && block + row * 2 + 1 < end_k) {- buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),- TO_FLOAT_TYPE(data_b_scalar[idx + 1]));- } else if (idx_n < p.N && block + row * 2 < end_k) {- buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);- } else {- buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);- }+ const uint idx = pos_b + col * p.stride_b + row * 2;+ const uint buf_idx = col * SHMEM_STRIDE + row;+ if (idx_n < p.N && block + row * 2 + 1 < end_k) {+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),+ TO_FLOAT_TYPE(data_b_scalar[idx + 1]));+ } else if (idx_n < p.N && block + row * 2 < end_k) {+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);+ } else {+ buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);+ }}#elsevoid load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint ic, const uint _ne1, const uint block, const uint end_k) {#if LOAD_VEC_B == 8- if (ALIGNED != 0) {- // Not supported for b_type bf16 because bf16mat2x4 does not exist- const u16vec2 row_idx = row_ids[col];- const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;- FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);- buf_b[buf_idx + 0] = bb[0].xy;- buf_b[buf_idx + 1] = bb[0].zw;- buf_b[buf_idx + 2] = bb[1].xy;- buf_b[buf_idx + 3] = bb[1].zw;- return;- }+ if (ALIGNED != 0) {+ // Not supported for b_type bf16 because bf16mat2x4 does not exist+ const u16vec2 row_idx = row_ids[col];+ const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;+ FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);+ buf_b[buf_idx + 0] = bb[0].xy;+ buf_b[buf_idx + 1] = bb[0].zw;+ buf_b[buf_idx + 2] = bb[1].xy;+ buf_b[buf_idx + 3] = bb[1].zw;+ return;+ }#elif LOAD_VEC_B == 4- if (ALIGNED != 0) {- const u16vec2 row_idx = row_ids[col];- const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;- const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;+ if (ALIGNED != 0) {+ const u16vec2 row_idx = row_ids[col];+ const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;#if defined(DATA_B_BF16)- FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));#else- FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);#endif- buf_b[buf_idx + 0] = bb.xy;- buf_b[buf_idx + 1] = bb.zw;- return;- }+ buf_b[buf_idx + 0] = bb.xy;+ buf_b[buf_idx + 1] = bb.zw;+ return;+ }#endif- const uint row_i = ic * BN + col;- const uint buf_idx = col * SHMEM_STRIDE + row;- if (row_i < _ne1 && block + row * 2 + 1 < end_k) {- const u16vec2 row_idx = row_ids[col];- const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;- buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),- TO_FLOAT_TYPE(data_b_scalar[idx + 1]));- } else if (row_i < _ne1 && block + row * 2 < end_k) {- const u16vec2 row_idx = row_ids[col];- const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;- buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);- } else {- buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);- }+ const uint row_i = ic * BN + col;+ const uint buf_idx = col * SHMEM_STRIDE + row;+ if (row_i < _ne1 && block + row * 2 + 1 < end_k) {+ const u16vec2 row_idx = row_ids[col];+ const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),+ TO_FLOAT_TYPE(data_b_scalar[idx + 1]));+ } else if (row_i < _ne1 && block + row * 2 < end_k) {+ const u16vec2 row_idx = row_ids[col];+ const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);+ } else {+ buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);+ }}#endifdiff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glslindex a19c7f2f4e..21d601d0eb 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl@@ -955,6 +955,7 @@ shared uint16_t iq1s_grid[2048];shared uint32_t iq1s_grid_gpu[2048];#endif+#if defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -978,6 +979,17 @@ void init_iq_shmem(uvec3 wgsize)barrier();}#endif+#endif++#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S)+#if defined(DATA_A_IQ2_S)+shared uvec2 iq2s_grid[1024];+#elif defined(DATA_A_IQ2_XS)+shared uvec2 iq2xs_grid[512];+#else+shared uvec2 iq2xxs_grid[256];+#endif+#endif#define QUANT_K_IQ2_XXS 256#define QUANT_R_IQ2_XXS 1@@ -1063,8 +1075,7 @@ const uvec2[256] iq2xxs_grid_const = {uvec2(0x08080808, 0x2b2b082b), uvec2(0x08192b08, 0x2b2b1908), uvec2(0x19190808, 0x2b2b2b08), uvec2(0x08081908, 0x2b2b2b19)};-shared uvec2 iq2xxs_grid[256];-+#if defined(DATA_A_IQ2_XXS)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -1076,12 +1087,15 @@ void init_iq_shmem(uvec3 wgsize)}barrier();}+#endif+#if defined(DATA_A_IQ2_XXS)#define QUANT_K QUANT_K_IQ2_XXS#define QUANT_R QUANT_R_IQ2_XXS#define A_TYPE block_iq2_xxs#define A_TYPE_PACKED16 block_iq2_xxs_packed16#endif+#endif#define QUANT_K_IQ2_XS 256#define QUANT_R_IQ2_XS 1@@ -1233,8 +1247,7 @@ const uvec2 iq2xs_grid_const[512] = {uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x082b2b2b, 0x2b2b2b2b), uvec2(0x2b190819, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b),};-shared uvec2 iq2xs_grid[512];-+#if defined(DATA_A_IQ2_XS)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -1246,12 +1259,15 @@ void init_iq_shmem(uvec3 wgsize)}barrier();}+#endif+#if defined(DATA_A_IQ2_XS)#define QUANT_K QUANT_K_IQ2_XS#define QUANT_R QUANT_R_IQ2_XS#define A_TYPE block_iq2_xs#define A_TYPE_PACKED16 block_iq2_xs_packed16#endif+#endif#define QUANT_K_IQ2_S 256#define QUANT_R_IQ2_S 1@@ -1533,8 +1549,7 @@ const uvec2 iq2s_grid_const[1024] = {uvec2(0x082b082b, 0x2b2b2b2b), uvec2(0x082b2b08, 0x2b2b2b2b), uvec2(0x2b082b08, 0x2b2b2b2b), uvec2(0x2b2b2b2b, 0x2b2b2b2b)};-shared uvec2 iq2s_grid[1024];-+#if defined(DATA_A_IQ2_S)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -1546,12 +1561,23 @@ void init_iq_shmem(uvec3 wgsize)}barrier();}+#endif+#if defined(DATA_A_IQ2_S)#define QUANT_K QUANT_K_IQ2_S#define QUANT_R QUANT_R_IQ2_S#define A_TYPE block_iq2_s#define A_TYPE_PACKED16 block_iq2_s_packed16#endif+#endif++#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S)+#if defined(DATA_A_IQ3_S)+shared uint32_t iq3s_grid[512];+#else+shared uint32_t iq3xxs_grid[256];+#endif+#endif#define QUANT_K_IQ3_XXS 256#define QUANT_R_IQ3_XXS 1@@ -1605,8 +1631,7 @@ const uint32_t iq3xxs_grid_const[256] = {0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,};-shared uint32_t iq3xxs_grid[256];-+#if defined(DATA_A_IQ3_XXS)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -1618,12 +1643,15 @@ void init_iq_shmem(uvec3 wgsize)}barrier();}+#endif+#if defined(DATA_A_IQ3_XXS)#define QUANT_K QUANT_K_IQ3_XXS#define QUANT_R QUANT_R_IQ3_XXS#define A_TYPE block_iq3_xxs#define A_TYPE_PACKED16 block_iq3_xxs_packed16#endif+#endif#define QUANT_K_IQ3_S 256#define QUANT_R_IQ3_S 1@@ -1715,8 +1743,7 @@ const uint32_t iq3s_grid_const[512] = {0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,};-shared uint32_t iq3s_grid[512];-+#if defined(DATA_A_IQ3_S)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -1728,12 +1755,15 @@ void init_iq_shmem(uvec3 wgsize)}barrier();}+#endif+#if defined(DATA_A_IQ3_S)#define QUANT_K QUANT_K_IQ3_S#define QUANT_R QUANT_R_IQ3_S#define A_TYPE block_iq3_s#define A_TYPE_PACKED16 block_iq3_s_packed16#endif+#endif#define QUANT_K_IQ4_XS 256#define QUANT_R_IQ4_XS 1@@ -1847,6 +1877,7 @@ const int8_t kvalues_iq4nl_const[16] = {shared FLOAT_TYPE kvalues_iq4nl[16];+#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){@@ -1857,6 +1888,7 @@ void init_iq_shmem(uvec3 wgsize)barrier();}#endif+#endif#if defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4)#if !defined(USE_OCP_FP4)@@ -1886,7 +1918,7 @@ float ue4m3_to_fp32_build(uint u) {}#endif-#if !defined(USE_OCP_FP4)+#if (defined(DATA_A_MXFP4) || defined(DATA_A_NVFP4)) && !defined(USE_OCP_FP4)#define NEEDS_INIT_IQ_SHMEMvoid init_iq_shmem(uvec3 wgsize){diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex ea4851b7ab..30fe0884e5 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -246,6 +246,17 @@ bool is_iq_quant(const std::string& type_name) {return string_starts_with(type_name, "iq");}+bool is_lut_quant(const std::string& type_name) {+ return is_iq_quant(type_name) || type_name == "mxfp4" || type_name == "nvfp4";+}++std::string lut_load_vec_a(const std::string& type_name) {+ if (type_name == "iq1_s" || type_name == "iq1_m" || type_name == "iq2_xxs" || type_name == "iq2_xs" || type_name == "iq2_s") {+ return "8";+ }+ return "4";+}+static const char path_separator = '/';std::string join_paths(const std::string& path1, const std::string& path2) {@@ -583,20 +594,28 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c}for (const auto& tname : type_names) {- std::string load_vec_quant = "2";- if ((tname == "q1_0") || (tname == "q4_0") || (tname == "q4_1") || (tname == "q5_1") || (tname == "iq1_s") || (tname == "iq1_m") || (tname == "iq2_xxs") || (tname == "iq2_xs") || (tname == "iq2_s"))- load_vec_quant = "8";- else if ((tname == "q2_0") || (tname == "q5_0") || (tname == "q8_0") || (tname == "q2_k") || (tname == "q4_k") || (tname == "q5_k") || (tname == "iq3_xxs") || (tname == "iq3_s") || (tname == "iq4_xs") || (tname == "iq4_nl") || (tname == "mxfp4") || (tname == "nvfp4"))- load_vec_quant = "4";-if (tname == "bf16") {continue;}- std::string data_a_key = "DATA_A_" + to_uppercase(tname);- // For aligned matmul loads- std::string load_vec_a = (coopmat2 || tname == "f32" || tname == "f16" || tname == "bf16") ? load_vec : load_vec_quant;+ // Float types keep per-type compilation (different accumulation loop structure)+ if (tname == "f32" || tname == "f16") {+ std::string data_a_key = "DATA_A_" + to_uppercase(tname);++ const std::map<std::string, std::string> float_type_dict = {+ {"FLOAT_TYPE", FLOAT_TYPE(1, tname)},+ {"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)},+ {"FLOAT_TYPEV4", FLOAT_TYPE(4, tname)},+ {"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)},+ };++ if (!coopmat2) {+ string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+ }+ continue;+ }+ std::string data_a_key = "DATA_A_" + to_uppercase(tname);const std::map<std::string, std::string> float_type_dict = {{"FLOAT_TYPE", FLOAT_TYPE(1, tname)},{"FLOAT_TYPEV2", FLOAT_TYPE(2, tname)},@@ -604,30 +623,52 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c{"FLOAT_TYPEV8", FLOAT_TYPE(8, tname)},};- // don't generate f32 variants for coopmat2- if (!coopmat2) {- string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)+ if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) {+ string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc);}+#endif- if (tname != "f16" && tname != "f32") {- string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);- }+ if (is_lut_quant(tname)) {+ std::string lva = lut_load_vec_a(tname);++ string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);-#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)- if ((coopmat || coopmat2) && (tname == "mxfp4" || tname == "nvfp4")) {if (!coopmat2) {- string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+ string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+ }++#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)+ if ((tname == "mxfp4" || tname == "nvfp4") && (coopmat || coopmat2)) {+ string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+ if (!coopmat2) {+ string_to_spv(shader_name + "_" + tname + "_f32_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+ }}- string_to_spv(shader_name + "_" + tname + "_f16_ocp" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"USE_OCP_FP4", "1"}, {"LOAD_VEC_A", load_vec_a}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);- }#endif+ continue;+ }-#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)- // Integer dot mmq performs better with f32 accumulators (different shader, skip for dot2)- if (!f16acc && !coopmat && !coopmat2 && !dot2 && (is_legacy_quant(tname) || is_k_quant(tname) || tname == "mxfp4")) {- string_to_spv(shader_name + "_" + tname + "_q8_1", "mul_mmq.comp", merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"D_TYPE", "float"},}), fp16, coopmat, coopmat2, f16acc);+ // dedicated shader needed due to regression on Ampere+ if (coopmat2 && (tname == "q4_k" || tname == "q5_k")) {+ string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);+ }+ }++ // Quant shader: one SPIR-V for all quant types, selected via MmTypeA spec constant+ {+ const std::map<std::string, std::string> quant_float_type_dict = {+ {"FLOAT_TYPE", FLOAT_TYPE(1, "q4_0")},+ {"FLOAT_TYPEV2", FLOAT_TYPE(2, "q4_0")},+ {"FLOAT_TYPEV4", FLOAT_TYPE(4, "q4_0")},+ {"FLOAT_TYPEV8", FLOAT_TYPE(8, "q4_0")},+ };++ string_to_spv(shader_name + "_quant_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);++ if (!coopmat2) {+ string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc);}-#endif}}