Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llm: Remove unneeded warning with flash attention enabled
Jesse Gross71cb86af3e8b
diff --git a/fs/ggml/ggml.go b/fs/ggml/ggml.goindex 56ad420e5..57476a9a8 100644--- a/fs/ggml/ggml.go+++ b/fs/ggml/ggml.go@@ -864,12 +864,16 @@ func (llm GGML) VisionGraphSize() (weights, graphSize uint64) {// SupportsKVCacheType checks if the requested cache type is supportedfunc (f GGML) SupportsKVCacheType(cacheType string) bool {+ if cacheType == "" || cacheType == "f16" {+ return true+ }+if arch := f.KV().Architecture(); slices.Contains([]string{"gptoss", "gpt-oss"}, arch) {// gpt-oss uses attention with sinks which does not support quantized cache types- slog.Warn("model only supports non-quantized cache types ", "mode", arch)- return cacheType == "f16"+ slog.Warn("model only supports non-quantized cache types", "model", arch)+ return false}- return slices.Contains([]string{"f16", "q8_0", "q4_0"}, cacheType)+ return slices.Contains([]string{"q8_0", "q4_0"}, cacheType)}// SupportsFlashAttention checks if the model supports flash attentiondiff --git a/llm/memory.go b/llm/memory.goindex ce128eb58..7a87b28fe 100644--- a/llm/memory.go+++ b/llm/memory.go@@ -202,7 +202,7 @@ func estimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinvar kvct stringif useFlashAttention {requested := strings.ToLower(envconfig.KvCacheType())- if requested != "" && f.SupportsKVCacheType(requested) {+ if f.SupportsKVCacheType(requested) {kvct = requested}}diff --git a/llm/server.go b/llm/server.goindex 4740a1fd4..a22ae9722 100644--- a/llm/server.go+++ b/llm/server.go@@ -221,7 +221,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a// Flash Attention also supports kv cache quantization// Enable if the requested and kv cache type is supported by the model- if kvct != "" && f.SupportsKVCacheType(kvct) {+ if f.SupportsKVCacheType(kvct) {loadRequest.KvCacheType = kvct} else {slog.Warn("kv cache type not supported by model", "type", kvct)