Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
--- a/llama.cpp/ggml/src/ggml-cpu/ops.cpp
+++ b/llama.cpp/ggml/src/ggml-cpu/ops.cpp
@@ -12,6 +12,18 @@
#include <cfloat>
#include <cmath>
+#if GGML_USE_LLAMAFILE
+// [jart] llamafile #975: forward declarations for the optimized AVX-512
+// flash-attention helpers implemented in llamafile/fa_helpers_* and
+// fa_simd_gemm_*. extern "C" because the implementations are exported
+// as C symbols from llamafile/sgemm.cpp; can't use llamafile/sgemm.h
+// directly because the submodule includes a shadowing upstream sgemm.h.
+extern "C" bool llamafile_fa_vec_dot_f16(int, float *, const void *, const void *);
+extern "C" bool llamafile_fa_fp16_to_fp32_row(const void *, float *, int64_t);
+extern "C" bool llamafile_fa_simd_gemm(float *, const float *, const float *,
+ int, int, int);
+#endif
+
// ggml_compute_forward_dup
static void ggml_compute_forward_dup_same_cont(
@@ -8897,6 +8909,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
GGML_ASSERT(( q_to_vec_dot) && "fattn: unsupported K-type");
GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && "fattn: unsupported V-type");
+#if GGML_USE_LLAMAFILE
+ // [jart] llamafile #975: see file-scope extern "C" declarations of
+ // llamafile_fa_vec_dot_f16 / llamafile_fa_fp16_to_fp32_row above.
+ // Their multi-arch implementations live in llamafile/fa_helpers_*
+ // and are runtime-dispatched via llamafile/sgemm.cpp.
+ const bool kq_is_f16 = (k->type == GGML_TYPE_F16 && k_vec_dot_type == GGML_TYPE_F16);
+ const bool v_is_f16 = (v->type == GGML_TYPE_F16);
+#endif
+
int ith = params->ith;
for (int ir = ir0; ir < ir1; ++ir) {
@@ -8916,10 +8937,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator
ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16
- if (v->type == GGML_TYPE_F16) {
- memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));
- } else {
+ // [jart] llamafile #975: when V is f16 and the hardware lacks
+ // native f16 FMA, accumulate VKQ in f32 instead of f16. The f16
+ // accumulator path round-trips f16<->f32 every KV iteration via
+ // _mm512_cvtph_ps / _mm512_cvtps_ph (see simd-mappings.h
+ // GGML_F32Cx16_*), dominating the FA inner loop on Xeon Gold /
+ // Skylake-X / Ice Lake / Zen 4 without AVX-512-FP16. Hardware
+ // with native f16 FMA keeps the existing fast path.
+#if defined(__AVX512FP16__) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || defined(__riscv_zvfh)
+ const bool use_f32_accum = (v->type != GGML_TYPE_F16);
+#else
+ const bool use_f32_accum = true;
+#endif
+ if (use_f32_accum) {
memset(VKQ32, 0, DV*sizeof(float));
+ } else {
+ memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));
}
const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;
@@ -8948,7 +8981,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
float s; // KQ value
const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
+#if GGML_USE_LLAMAFILE
+ if (!kq_is_f16 || !llamafile_fa_vec_dot_f16(DK, &s, k_data, Q_q)) {
+ kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
+ }
+#else
kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
+#endif
s = s*scale; // scale KQ value
@@ -8965,21 +9004,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
- if (v->type == GGML_TYPE_F16) {
+ if (use_f32_accum) {
if (s > M) {
// s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
M = s;
ms = expf(Mold - M);
// V = V*expf(Mold - M)
- ggml_vec_scale_f16(DV, VKQ16, ms);
+ ggml_vec_scale_f32(DV, VKQ32, ms);
} else {
// no new maximum, ms == 1.0f, vs != 1.0f
vs = expf(s - M);
}
// V += v*expf(s - M)
- ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);
+ if (v_to_float) {
+ // f16/quant -> f32 (covers GGML_TYPE_F16 too once
+ // we're in the f32 accumulator path; v_to_float is
+ // ggml_fp16_to_fp32_row for V=f16)
+#if GGML_USE_LLAMAFILE
+ if (!v_is_f16 || !llamafile_fa_fp16_to_fp32_row(v_data, V32, DV)) {
+ v_to_float(v_data, V32, DV);
+ }
+#else
+ v_to_float(v_data, V32, DV);
+#endif
+ ggml_vec_mad_f32(DV, VKQ32, V32, vs);
+ } else {
+ // V is F32
+ ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);
+ }
} else {
if (s > M) {
// s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
@@ -8987,26 +9041,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
ms = expf(Mold - M);
// V = V*expf(Mold - M)
- ggml_vec_scale_f32(DV, VKQ32, ms);
+ ggml_vec_scale_f16(DV, VKQ16, ms);
} else {
// no new maximum, ms == 1.0f, vs != 1.0f
vs = expf(s - M);
}
// V += v*expf(s - M)
- if (v_to_float) {
- v_to_float(v_data, V32, DV);
- ggml_vec_mad_f32(DV, VKQ32, V32, vs);
- } else {
- // V is F32
- ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);
- }
+ ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);
}
S = S*ms + vs; // scale and increment sum with partial sum
}
- if (v->type == GGML_TYPE_F16) {
+ if (!use_f32_accum) {
for (int64_t d = 0; d < DV; ++d) {
VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);
}
@@ -9227,8 +9275,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
if (kv_type == GGML_TYPE_F16) {
const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
- for (int64_t dk = 0; dk < DK; dk++) {
- K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);
+#if GGML_USE_LLAMAFILE
+ // [jart] llamafile #975: dequant the K row contiguously
+ // via SIMD (AVX-512 cvtph_ps), then scalar-transpose into
+ // K_f32. ~2x faster than the upstream per-element table
+ // lookup. Falls back to upstream's scalar loop on CPUs
+ // without our optimized helper. DK up to 1024 covers all
+ // realistic attention head dims (Qwen3=128, DeepSeek-MLA=576).
+ alignas(64) float k_scratch[1024];
+ if (DK <= 1024 && llamafile_fa_fp16_to_fp32_row(k_f16, k_scratch, DK)) {
+ for (int64_t dk = 0; dk < DK; dk++) {
+ K_f32[dk * KV_TILE_SZ + tk] = k_scratch[dk];
+ }
+ } else
+#endif
+ {
+ for (int64_t dk = 0; dk < DK; dk++) {
+ K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);
+ }
}
} else {
const float * k_f32_src = (const float *)k_data;
@@ -9238,7 +9302,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
}
}
memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
+#if GGML_USE_LLAMAFILE
+ // [jart] llamafile #975: prefer our AVX-512 simd_gemm wrapper
+ // (4x4 zmm tile, ~2.7x the FMA throughput of ops.cpp's
+ // default AVX2 6x2 ymm tile). Falls back to upstream's
+ // header-inlined simd_gemm on CPUs without AVX-512F.
+ if (!llamafile_fa_simd_gemm(KQ, (const float *)Q_q, K_f32,
+ Q_TILE_SZ, DK, KV_TILE_SZ)) {
+ simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);
+ }
+#else
simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);
+#endif
ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);
if (logit_softcap != 0.0f) {
@@ -9291,7 +9366,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
for (int tk = 0; tk < kv_tile; tk++) {
const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
if (kv_type == GGML_TYPE_F16) {
+#if GGML_USE_LLAMAFILE
+ // [jart] llamafile #975: route through optimized AVX-512
+ // dequant when available; fall back to upstream's helper.
+ if (!llamafile_fa_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV)) {
+ ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
+ }
+#else
ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
+#endif
} else {
memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
}
@@ -9301,7 +9384,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));
}
}
+#if GGML_USE_LLAMAFILE
+ // [jart] llamafile #975: same AVX-512 simd_gemm wrapper as the
+ // KQ matmul above (4x4 zmm tile vs default AVX2 6x2).
+ if (!llamafile_fa_simd_gemm(VKQ32, KQ, V32,
+ Q_TILE_SZ, KV_TILE_SZ, DV)) {
+ simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);
+ }
+#else
simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);
+#endif
}
// sinks (apply only to valid rows in the tile)