Skip to content

Distribute and run LLMs with a single file.

943 B
Raw Blame
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
--- a/llama.cpp/src/llama-context.cpp
+++ b/llama.cpp/src/llama-context.cpp
@@ -657,6 +657,16 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
         }
     };
 
+    if (cparams.auto_fa && model.devices.empty()) {
+        // llamafile: the CPU Flash Attention path (f16 mad/dot) is significantly
+        // slower than the non-FA path on x86 (see mozilla-ai/llamafile#975).
+        // When no GPU devices are present, default -fa auto to OFF. Users who
+        // want FA for memory savings on long contexts can still pass -fa on.
+        cparams.flash_attn = false;
+        cparams.auto_fa    = false;
+        LLAMA_LOG_INFO("%s: Flash Attention was auto, set to disabled (CPU-only)\n", __func__);
+    }
+
     if (cparams.auto_fa) {
         resolve(llm_fused_op_flash_attn_probe, cparams.flash_attn);
         cparams.auto_fa = false;