Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama3.1 memory

Michael Yang2003d601594c
diff --git a/llm/ggml.go b/llm/ggml.go
index d7f2eef7c..bde9c3473 100644
--- a/llm/ggml.go
+++ b/llm/ggml.go
@@ -344,11 +344,13 @@ func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload ui
switch llm.KV().Architecture() {
case "llama":
- fullOffload = 4 * batch * (1 + 4*embedding + context*(1+heads))
+ fullOffload = max(
+ 4*batch*(1+4*embedding+context*(1+heads)),
+ 4*batch*(embedding+vocab),
+ )
partialOffload = 4 * batch * embedding
partialOffload += max(
- // 4*batch*(4+6*embedding+context*(2*heads)+llm.KV().GQA()),
4*batch*(1+embedding+max(context, embedding))+embedding*embedding*9/16+4*context*(batch*heads+embeddingHeads*headsKV),
4*batch*(embedding+vocab)+embedding*vocab*105/128,
)