LLM inference in C/C++
C++ ggml
vulkan: add iq4_nl support back to FA (#24585)
Jeff Bolz81616410050c
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 5dcf4503bb..49fd6e0b42 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -3490,7 +3490,7 @@ struct vk_fa_tuning_params {};static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type);-static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16);+static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16);static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {@@ -3646,7 +3646,7 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_bool shape_ok = (f32acc && device->coopmat_support_16x16x16_f32acc) ||(!f32acc && device->coopmat_support_16x16x16_f16acc);const vk_fa_tuning_params params = get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc);- bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type);+ bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type, v_type);if (!shape_ok || !shmem_ok) {path = FA_SCALAR;@@ -3658,11 +3658,6 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_path = FA_SCALAR;}- // Q1_0 K/V is only implemented on coopmat2 (flash_attn_cm2); there is no scalar FA shader for it.- if ((k_type == GGML_TYPE_Q1_0 || v_type == GGML_TYPE_Q1_0) && device->coopmat2) {- path = FA_COOPMAT2;- }-switch (path) {case FA_SCALAR:return get_fa_tuning_params_scalar(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc);@@ -3904,16 +3899,27 @@ static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_devreturn 0; // If no matching configuration is found}-// Whether scalar flash attention will use the MMQ path for the given k_type.-static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type) {+// Whether scalar flash attention will use the MMQ path for the given K/V types.+static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ4_NL:+ return true;+ default:+ return false;+ }+}++static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) {#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)return device->integer_dot_product && device->subgroup_clustered &&+ !ggml_vk_fa_type_needs_shmem(v_type) &&(k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 ||k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 ||k_type == GGML_TYPE_Q8_0);#elseGGML_UNUSED(device);GGML_UNUSED(k_type);+ GGML_UNUSED(v_type);return false;#endif}@@ -4246,7 +4252,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {const bool fa_ds = fa.first.subgroup_size == 0;const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16;- const bool use_mmq = ggml_vk_fa_scalar_uses_mmq(device, fa.first.k_type);+ const bool use_mmq = ggml_vk_fa_scalar_uses_mmq(device, fa.first.k_type, fa.first.v_type);const void * spv_data = nullptr;size_t spv_size = 0;const char *name = nullptr;@@ -10380,7 +10386,6 @@ static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctxstatic bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {GGML_UNUSED(f32acc);- GGML_UNUSED(v_type);// Needs to be kept up to date on shader changesconst uint32_t wg_size = params.workgroup_size;const uint32_t Br = params.block_rows;@@ -10389,13 +10394,15 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, con// BF16 uses the fp32 shader (FLOAT_TYPE=float)const uint32_t float_type_size = (device->fp16 && k_type != GGML_TYPE_BF16) ? sizeof(ggml_fp16_t) : sizeof(float);- const bool mmq = ggml_vk_fa_scalar_uses_mmq(device, k_type);+ const bool mmq = ggml_vk_fa_scalar_uses_mmq(device, k_type, v_type);// tmpsh is overestimated slightlyconst uint32_t tmpsh = wg_size * sizeof(float);const uint32_t tmpshv4 = wg_size * 4 * float_type_size;const uint32_t masksh = Bc * (Br + 1) * float_type_size;+ // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated.+ const uint32_t iq_shmem = 16 * float_type_size;uint32_t Qf, kvsh, kblocksh_size;if (mmq) {@@ -10420,7 +10427,7 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, conkblocksh_size = 0;}- const uint32_t total_size = tmpsh + tmpshv4 + masksh + Qf + kvsh + kblocksh_size;+ const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size;const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;VK_LOG_DEBUG("ggml_vk_flash_attn_scalar_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", mmq=" << mmq << ", total_size=" << total_size << ", supported=" << supported);@@ -10428,7 +10435,8 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, conreturn supported;}-static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type) {+static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {+ GGML_UNUSED(v_type);// Needs to be kept up to date on shader changesconst uint32_t Br = params.block_rows;const uint32_t Bc = params.block_cols;@@ -10444,6 +10452,8 @@ static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, coconst uint32_t f16vec4 = 8;const uint32_t tmpsh = (Bc / MatBc) * sizeof(float);+ // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated.+ const uint32_t iq_shmem = 16 * sizeof(ggml_fp16_t);const uint32_t qstride = hsk_pad / 4 + 2;const uint32_t Qf = Br * qstride * f16vec4;@@ -10465,7 +10475,7 @@ static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, coconst uint32_t slope = Br * acctype;- const uint32_t total_size = tmpsh + Qf + Psh + sfsh + ksh + pvsh + slope;+ const uint32_t total_size = tmpsh + iq_shmem + Qf + Psh + sfsh + ksh + pvsh + slope;const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;VK_LOG_DEBUG("ggml_vk_flash_attn_coopmat_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", f32acc=" << f32acc << ", total_size=" << total_size << ", supported=" << supported);@@ -17617,7 +17627,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmif (op->src[3] && op->src[3]->type != GGML_TYPE_F16) {return false;}- auto fa_kv_ok = [coopmat2](ggml_type t) {+ auto fa_kv_ok = [](ggml_type t) {switch (t) {case GGML_TYPE_F32:case GGML_TYPE_F16:@@ -17627,9 +17637,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_TYPE_Q5_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q4_0:+ case GGML_TYPE_IQ4_NL:return true;- case GGML_TYPE_Q1_0:- return coopmat2;default:return false;}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.compindex 3192130ccf..6c264c7861 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp@@ -80,7 +80,9 @@ shared vec4 occupancy_limiter[LIMIT_OCCUPANCY_SHMEM > 0 ? LIMIT_OCCUPANCY_SHMEMvoid main() {#ifdef NEEDS_INIT_IQ_SHMEM- init_iq_shmem(gl_WorkGroupSize);+ if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) {+ init_iq_shmem(gl_WorkGroupSize);+ }#endifinit_indices();diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glslindex 66dcf61021..3c64f91dad 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl@@ -97,8 +97,8 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];};#define FA_TYPE_Q5_0 6u#define FA_TYPE_Q5_1 7u#define FA_TYPE_Q8_0 8u+#define FA_TYPE_IQ4_NL 20u#define FA_TYPE_BF16 30u-#define FA_TYPE_Q1_0 41u#if defined(BFLOAT16)#define O_TYPE float@@ -120,8 +120,8 @@ uint fa_block_elems(uint ty) {case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0);case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1);case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0);+ case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL);case FA_TYPE_BF16: return 1u;- case FA_TYPE_Q1_0: return uint(QUANT_K_Q1_0); // cm2-only, harmless elsewheredefault: return 1u;}}@@ -140,6 +140,13 @@ uint fa_quant_r_mmq(uint ty) {}}+bool fa_type_needs_shmem(uint ty) {+ switch (ty) {+ case FA_TYPE_IQ4_NL: return true;+ default: return false;+ }+}+// These can't be `const` globals because GLSL forbids function calls in global// const initializers, even when the spec constants would let the driver fold// them. Macros expand at the use site and fold after specialization.diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.compindex 16178e5770..057ed739aa 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp@@ -64,7 +64,9 @@ shared ACC_TYPE slope[Br];void main() {#ifdef NEEDS_INIT_IQ_SHMEM- init_iq_shmem(gl_WorkGroupSize);+ if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) {+ init_iq_shmem(gl_WorkGroupSize);+ }#endifinit_indices();diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.compindex b9c03fe499..3174111530 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp@@ -46,7 +46,7 @@ float16_t faDecodeK(const decodeBufFA_K bl_in, const uint blockCoords[2], constcase FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);default: return float16_t(0);}}@@ -59,7 +59,7 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], constcase FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case FA_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);default: return float16_t(0);}}@@ -67,26 +67,26 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const// V=4 vector decode for K/V; dispatches to per-format _v decoders.f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {switch (FaTypeK) {- case 0u: return f16vec4(decodeBufF32(bl_in).block);- case 2u: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);- case 3u: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);- case 6u: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);- case 7u: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);- case 8u: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case 41u: return dequantFuncQ1_0_v(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block);+ case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);default: return f16vec4(0);}}f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) {switch (FaTypeV) {- case 0u: return f16vec4(decodeBufF32(bl_in).block);- case 2u: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);- case 3u: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);- case 6u: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);- case 7u: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);- case 8u: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);- case 41u: return dequantFuncQ1_0_v(decodeBufQ1_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block);+ case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock);+ case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock);default: return f16vec4(0);}}@@ -169,6 +169,12 @@ ACC_TYPE perElemOpNonGqaSplitKStoreCol0(const in uint32_t r, const in uint32_t c}void main() {+#ifdef NEEDS_INIT_IQ_SHMEM+ if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) {+ init_iq_shmem(gl_WorkGroupSize);+ }+#endif+init_indices();tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutQ = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);@@ -302,7 +308,7 @@ void main() {coopmat<FLOAT_TYPE, gl_ScopeWorkgroup, HSK_pad, Bc, gl_MatrixUseB> K_T;uint32_t k_offset = ik2*p.nb12 + ik3*p.nb13;- // F16: bs_k==1 (direct load). F32: bs_k==4 (vec4 / dequantFuncF32). Q4/Q8 family: bs_k==32. Q1_0: bs_k==128.+ // F16: bs_k==1 (direct load). F32: bs_k==4 (vec4 / dequantFuncF32). Quantized types: bs_k==32.#if defined(BFLOAT16)coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose);#elsediff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glslindex 8704479d96..8ba4725f33 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl@@ -27,6 +27,8 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1 { block_q5_1_packed16 data[];layout (binding = 2) readonly buffer V_PACKED_Q5_1 { block_q5_1_packed16 data[]; } v_packed_q5_1;layout (binding = 1) readonly buffer K_PACKED_Q8_0 { block_q8_0_packed16 data[]; } k_packed_q8_0;layout (binding = 2) readonly buffer V_PACKED_Q8_0 { block_q8_0_packed16 data[]; } v_packed_q8_0;+layout (binding = 1) readonly buffer K_PACKED_IQ4_NL { block_iq4_nl_packed16 data[]; } k_packed_iq4_nl;+layout (binding = 2) readonly buffer V_PACKED_IQ4_NL { block_iq4_nl_packed16 data[]; } v_packed_iq4_nl;layout (binding = 1) readonly buffer K_PACKED_BF16 { u16vec4 data[]; } k_packed_bf16;layout (binding = 2) readonly buffer V_PACKED_BF16 { u16vec4 data[]; } v_packed_bf16;@@ -102,6 +104,17 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1_P32 { block_q5_1_packed32 datreturn FLOAT_TYPE(BUF.data[a_offset + ib].d) * FLOAT_TYPEV4(v0.x, v0.y, v1.x, v1.y); \}+#define FA_DEQUANT4_IQ4_NL(BUF) { \+ const uint shift = (iqs & 0x10) >> 2; \+ const uint qs_i = (iqs & 0xC) >> 1; \+ const uint qsw = uint(BUF.data[a_offset + ib].qs[qs_i]) \+ | (uint(BUF.data[a_offset + ib].qs[qs_i + 1u]) << 16); \+ const FLOAT_TYPE d = FLOAT_TYPE(BUF.data[a_offset + ib].d); \+ const u8vec4 q = unpack8((qsw >> shift) & 0x0F0F0F0Fu); \+ return d * FLOAT_TYPEV4(kvalues_iq4nl[q.x], kvalues_iq4nl[q.y], \+ kvalues_iq4nl[q.z], kvalues_iq4nl[q.w]); \+}+#define FA_DEQUANT4_BF16(BUF) \return FLOAT_TYPEV4(bf16_to_fp32(uvec4(BUF.data[(a_offset + ib) / 4])));@@ -114,6 +127,7 @@ FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) {case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0)case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1)case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0)+ case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl)case FA_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16)}} else {@@ -124,6 +138,7 @@ FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) {case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0)case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1)case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0)+ case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl)case FA_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16)}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex 58d347bc54..3e6b39576e 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -673,6 +673,8 @@ void process_shaders() {fa_base_dict["ACC_TYPE"] = fp16 && f16acc ? "float16_t" : "float";fa_base_dict["ACC_TYPEV2"] = fp16 && f16acc ? "f16vec2" : "vec2";fa_base_dict["ACC_TYPEV4"] = fp16 && f16acc ? "f16vec4" : "vec4";+ // Compile IQ4_NL support into all FA variants so its shared LUT is available when K or V uses it.+ fa_base_dict["DATA_A_IQ4_NL"] = "1";if (fp16 && f16acc) {fa_base_dict["ACC_TYPE_MAX"] = "float16_t(65504.0)";}