Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mllama cross attention
Michael Yang8c238e70abe7
diff --git a/llm/ggml.go b/llm/ggml.goindex e857d4b89..9cf9172e8 100644--- a/llm/ggml.go+++ b/llm/ggml.go@@ -400,6 +400,30 @@ func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload ui4*batch*(1+2*embedding+context*(1+heads))+embedding*(6*context*headsKV/heads+embedding*9/16),)}+ case "mllama":+ var visionTokens, tiles uint64 = 1601, 4++ fullOffload = max(+ 4*batch*(2+3*embedding+embeddingHeadsK*heads+context*(1+heads)),+ // vocab graph+ 4*batch*(embedding+vocab),+ )++ var ropeFreqsCount uint64+ if ropeFreqs, ok := llm.Tensors().Layers()["rope_freqs"]; ok {+ if ropeFreqsWeights, ok := ropeFreqs["weights"]; ok {+ ropeFreqsCount = ropeFreqsWeights.parameters()+ }+ }++ partialOffload = max(+ 4*(batch*+ (2*embedding+1+context*(1+heads)+embeddingHeadsK*heads)++ ropeFreqsCount++ embeddingHeadsK*context*headsKV),+ // vocab graph+ 4*batch*(embedding+vocab)+embedding*vocab*105/128,+ )case "gemma", "gemma2":fullOffload = max(4*batch*(embedding+vocab),