Skip to content

Distribute and run LLMs with a single file.

10.6 KB
Raw Blame
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
--- a/llama.cpp/ggml/src/ggml-cpu/ops.cpp
+++ b/llama.cpp/ggml/src/ggml-cpu/ops.cpp
@@ -12,6 +12,18 @@
 #include <cfloat>
 #include <cmath>
 
+#if GGML_USE_LLAMAFILE
+// [jart] llamafile #975: forward declarations for the optimized AVX-512
+// flash-attention helpers implemented in llamafile/fa_helpers_* and
+// fa_simd_gemm_*. extern "C" because the implementations are exported
+// as C symbols from llamafile/sgemm.cpp; can't use llamafile/sgemm.h
+// directly because the submodule includes a shadowing upstream sgemm.h.
+extern "C" bool llamafile_fa_vec_dot_f16(int, float *, const void *, const void *);
+extern "C" bool llamafile_fa_fp16_to_fp32_row(const void *, float *, int64_t);
+extern "C" bool llamafile_fa_simd_gemm(float *, const float *, const float *,
+                                        int, int, int);
+#endif
+
 // ggml_compute_forward_dup
 
 static void ggml_compute_forward_dup_same_cont(
@@ -8897,6 +8909,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
     GGML_ASSERT((                            q_to_vec_dot) && "fattn: unsupported K-type");
     GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float  ) && "fattn: unsupported V-type");
 
+#if GGML_USE_LLAMAFILE
+    // [jart] llamafile #975: see file-scope extern "C" declarations of
+    // llamafile_fa_vec_dot_f16 / llamafile_fa_fp16_to_fp32_row above.
+    // Their multi-arch implementations live in llamafile/fa_helpers_*
+    // and are runtime-dispatched via llamafile/sgemm.cpp.
+    const bool kq_is_f16 = (k->type == GGML_TYPE_F16 && k_vec_dot_type == GGML_TYPE_F16);
+    const bool v_is_f16  = (v->type == GGML_TYPE_F16);
+#endif
+
     int ith = params->ith;
 
     for (int ir = ir0; ir < ir1; ++ir) {
@@ -8916,10 +8937,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
         ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator
         ggml_fp16_t * Q_q   = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16
 
-        if (v->type == GGML_TYPE_F16) {
-            memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));
-        } else {
+        // [jart] llamafile #975: when V is f16 and the hardware lacks
+        // native f16 FMA, accumulate VKQ in f32 instead of f16. The f16
+        // accumulator path round-trips f16<->f32 every KV iteration via
+        // _mm512_cvtph_ps / _mm512_cvtps_ph (see simd-mappings.h
+        // GGML_F32Cx16_*), dominating the FA inner loop on Xeon Gold /
+        // Skylake-X / Ice Lake / Zen 4 without AVX-512-FP16. Hardware
+        // with native f16 FMA keeps the existing fast path.
+#if defined(__AVX512FP16__) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || defined(__riscv_zvfh)
+        const bool use_f32_accum = (v->type != GGML_TYPE_F16);
+#else
+        const bool use_f32_accum = true;
+#endif
+        if (use_f32_accum) {
             memset(VKQ32, 0, DV*sizeof(float));
+        } else {
+            memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));
         }
 
         const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;
@@ -8948,7 +8981,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
             float s; // KQ value
 
             const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
+#if GGML_USE_LLAMAFILE
+            if (!kq_is_f16 || !llamafile_fa_vec_dot_f16(DK, &s, k_data, Q_q)) {
+                kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
+            }
+#else
             kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
+#endif
 
             s = s*scale; // scale KQ value
 
@@ -8965,21 +9004,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
 
             const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
 
-            if (v->type == GGML_TYPE_F16) {
+            if (use_f32_accum) {
                 if (s > M) {
                     // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
                     M = s;
                     ms = expf(Mold - M);
 
                     // V = V*expf(Mold - M)
-                    ggml_vec_scale_f16(DV, VKQ16, ms);
+                    ggml_vec_scale_f32(DV, VKQ32, ms);
                 } else {
                     // no new maximum, ms == 1.0f, vs != 1.0f
                     vs = expf(s - M);
                 }
 
                 // V += v*expf(s - M)
-                ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);
+                if (v_to_float) {
+                    // f16/quant -> f32 (covers GGML_TYPE_F16 too once
+                    // we're in the f32 accumulator path; v_to_float is
+                    // ggml_fp16_to_fp32_row for V=f16)
+#if GGML_USE_LLAMAFILE
+                    if (!v_is_f16 || !llamafile_fa_fp16_to_fp32_row(v_data, V32, DV)) {
+                        v_to_float(v_data, V32, DV);
+                    }
+#else
+                    v_to_float(v_data, V32, DV);
+#endif
+                    ggml_vec_mad_f32(DV, VKQ32, V32, vs);
+                } else {
+                    // V is F32
+                    ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);
+                }
             } else {
                 if (s > M) {
                     // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
@@ -8987,26 +9041,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
                     ms = expf(Mold - M);
 
                     // V = V*expf(Mold - M)
-                    ggml_vec_scale_f32(DV, VKQ32, ms);
+                    ggml_vec_scale_f16(DV, VKQ16, ms);
                 } else {
                     // no new maximum, ms == 1.0f, vs != 1.0f
                     vs = expf(s - M);
                 }
 
                 // V += v*expf(s - M)
-                if (v_to_float) {
-                    v_to_float(v_data, V32, DV);
-                    ggml_vec_mad_f32(DV, VKQ32, V32, vs);
-                } else {
-                    // V is F32
-                    ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);
-                }
+                ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);
             }
 
             S = S*ms + vs; // scale and increment sum with partial sum
         }
 
-        if (v->type == GGML_TYPE_F16) {
+        if (!use_f32_accum) {
             for (int64_t d = 0; d < DV; ++d) {
                 VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);
             }
@@ -9227,8 +9275,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
                 const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
                 if (kv_type == GGML_TYPE_F16) {
                     const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
-                    for (int64_t dk = 0; dk < DK; dk++) {
-                        K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);
+#if GGML_USE_LLAMAFILE
+                    // [jart] llamafile #975: dequant the K row contiguously
+                    // via SIMD (AVX-512 cvtph_ps), then scalar-transpose into
+                    // K_f32. ~2x faster than the upstream per-element table
+                    // lookup. Falls back to upstream's scalar loop on CPUs
+                    // without our optimized helper. DK up to 1024 covers all
+                    // realistic attention head dims (Qwen3=128, DeepSeek-MLA=576).
+                    alignas(64) float k_scratch[1024];
+                    if (DK <= 1024 && llamafile_fa_fp16_to_fp32_row(k_f16, k_scratch, DK)) {
+                        for (int64_t dk = 0; dk < DK; dk++) {
+                            K_f32[dk * KV_TILE_SZ + tk] = k_scratch[dk];
+                        }
+                    } else
+#endif
+                    {
+                        for (int64_t dk = 0; dk < DK; dk++) {
+                            K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);
+                        }
                     }
                 } else {
                     const float * k_f32_src = (const float *)k_data;
@@ -9238,7 +9302,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
                 }
             }
             memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
+#if GGML_USE_LLAMAFILE
+            // [jart] llamafile #975: prefer our AVX-512 simd_gemm wrapper
+            // (4x4 zmm tile, ~2.7x the FMA throughput of ops.cpp's
+            // default AVX2 6x2 ymm tile). Falls back to upstream's
+            // header-inlined simd_gemm on CPUs without AVX-512F.
+            if (!llamafile_fa_simd_gemm(KQ, (const float *)Q_q, K_f32,
+                                         Q_TILE_SZ, DK, KV_TILE_SZ)) {
+                simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);
+            }
+#else
             simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);
+#endif
             ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);
 
             if (logit_softcap != 0.0f) {
@@ -9291,7 +9366,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
             for (int tk = 0; tk < kv_tile; tk++) {
                 const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
                 if (kv_type == GGML_TYPE_F16) {
+#if GGML_USE_LLAMAFILE
+                    // [jart] llamafile #975: route through optimized AVX-512
+                    // dequant when available; fall back to upstream's helper.
+                    if (!llamafile_fa_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV)) {
+                        ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
+                    }
+#else
                     ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
+#endif
                 } else {
                     memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
                 }
@@ -9301,7 +9384,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
                     memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));
                 }
             }
+#if GGML_USE_LLAMAFILE
+            // [jart] llamafile #975: same AVX-512 simd_gemm wrapper as the
+            // KQ matmul above (4x4 zmm tile vs default AVX2 6x2).
+            if (!llamafile_fa_simd_gemm(VKQ32, KQ, V32,
+                                         Q_TILE_SZ, KV_TILE_SZ, DV)) {
+                simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);
+            }
+#else
             simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);
+#endif
         }
 
         // sinks (apply only to valid rows in the tile)