Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

gpt-oss: disable quantized kv cache (#11929)

Michael Yangdf335aac09cd
diff --git a/fs/ggml/ggml.go b/fs/ggml/ggml.go
index 1fef74524..a739e99ba 100644
--- a/fs/ggml/ggml.go
+++ b/fs/ggml/ggml.go
@@ -752,6 +752,11 @@ func (llm GGML) VisionGraphSize() (weights, graphSize uint64) {
// SupportsKVCacheType checks if the requested cache type is supported
func (f GGML) SupportsKVCacheType(cacheType string) bool {
+ if arch := f.KV().Architecture(); slices.Contains([]string{"gptoss", "gpt-oss"}, arch) {
+ // gpt-oss uses attention with sinks which does not support quantized cache types
+ slog.Warn("model only supports non-quantized cache types ", "mode", arch)
+ return cacheType == "f16"
+ }
return slices.Contains([]string{"f16", "q8_0", "q4_0"}, cacheType)
}