Skip to content

LLM inference in C/C++

C++ ggml

vulkan: FA MMQ should use fp32 for Q quantization calculations (#27413)

Jeff Bolz78ec4c378031
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
index 6c264c7861..0c1b6d0673 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp
@@ -121,13 +121,13 @@ void main() {
const uint buf_ib = r * qf_stride + d / 8;
const uint buf_iqs = d % 8;
- FLOAT_TYPEV4 vals = is_in_bounds ? FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale) : FLOAT_TYPEV4(0.0f);
- const FLOAT_TYPEV4 abs_vals = abs(vals);
+ vec4 vals = is_in_bounds ? data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale : vec4(0.0f);
+ const vec4 abs_vals = abs(vals);
- const FLOAT_TYPE thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
- const FLOAT_TYPE amax = subgroupClusteredMax(thread_max, 8);
- const FLOAT_TYPE qd = amax / FLOAT_TYPE(127.0);
- const FLOAT_TYPE qd_inv = qd != FLOAT_TYPE(0.0) ? FLOAT_TYPE(1.0) / qd : FLOAT_TYPE(0.0);
+ const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
+ const float amax = subgroupClusteredMax(thread_max, 8);
+ const float qd = amax / 127.0f;
+ const float qd_inv = qd != 0.0f ? 1.0f / qd : 0.0f;
vals = round(vals * qd_inv);
Qf[buf_ib].qs[buf_iqs] = pack32(i8vec4(vals));
@@ -136,11 +136,11 @@ void main() {
// the row-sum scaled by qd, used in k_dot_correction.
if (FaTypeK == FA_TYPE_Q8_0) {
if (buf_iqs == 0) {
- Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0);
+ Qf[buf_ib].ds = FLOAT_TYPEV2(qd, 0.0f);
}
} else {
- const FLOAT_TYPE thread_sum = vals.x + vals.y + vals.z + vals.w;
- const FLOAT_TYPE sum = subgroupClusteredAdd(thread_sum, 8);
+ const float thread_sum = vals.x + vals.y + vals.z + vals.w;
+ const float sum = subgroupClusteredAdd(thread_sum, 8);
if (buf_iqs == 0) {
Qf[buf_ib].ds = FLOAT_TYPEV2(qd, sum * qd);