Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
--- a/llama.cpp/src/llama-context.cpp
+++ b/llama.cpp/src/llama-context.cpp
@@ -657,6 +657,16 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
}
};
+ if (cparams.auto_fa && model.devices.empty()) {
+ // llamafile: the CPU Flash Attention path (f16 mad/dot) is significantly
+ // slower than the non-FA path on x86 (see mozilla-ai/llamafile#975).
+ // When no GPU devices are present, default -fa auto to OFF. Users who
+ // want FA for memory savings on long contexts can still pass -fa on.
+ cparams.flash_attn = false;
+ cparams.auto_fa = false;
+ LLAMA_LOG_INFO("%s: Flash Attention was auto, set to disabled (CPU-only)\n", __func__);
+ }
+
if (cparams.auto_fa) {
resolve(llm_fused_op_flash_attn_probe, cparams.flash_attn);
cparams.auto_fa = false;