Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llm: Estimate projector memory correctly for Ollama engine
Jesse Grossd75557747357
diff --git a/llm/memory.go b/llm/memory.goindex 464723302..32774d28a 100644--- a/llm/memory.go+++ b/llm/memory.go@@ -85,8 +85,11 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinvar graphOffload uint64// Projectors loaded into GPU0 only- var projectorWeights uint64- var projectorGraph uint64+ var llamaEngineProjectorWeights uint64++ // Projectors loaded with output layer+ var ollamaEngineProjectorWeights uint64+ var ollamaEngineProjectorGraph uint64// Conditional output size on GPU 0var memoryLayerOutput uint64@@ -111,14 +114,14 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinslog.Debug("evaluating", "library", gpus[0].Library, "gpu_count", len(gpus), "available", availableList)for _, projector := range projectors {- weight := projectorMemoryRequirements(projector)- projectorWeights += weight+ llamaEngineProjectorWeights += projectorMemoryRequirements(projector)// multimodal models require at least 2048 contextopts.NumCtx = max(opts.NumCtx, 2048)}- if projectorWeights == 0 && projectorGraph == 0 {- projectorWeights, projectorGraph = f.VisionGraphSize()+ if llamaEngineProjectorWeights == 0 {+ ollamaEngineProjectorWeights, ollamaEngineProjectorGraph = f.VisionGraphSize()+ opts.NumCtx = max(opts.NumCtx, 2048)}layers := f.Tensors().GroupLayers()@@ -163,6 +166,7 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []stringraphFullOffload = graphPartialOffload}+ // Output layer handled at the end if we have spaceif layer, ok := layers["output_norm"]; ok {memoryLayerOutput += layer.Size()}@@ -172,8 +176,7 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinmemoryLayerOutput += layer.Size()}- // Output layer handled at the end if we have space- gpuZeroOverhead := projectorWeights + projectorGraph+ gpuZeroOverhead := llamaEngineProjectorWeights// Reduce set of GPUs to only those that have sufficient space to fit overhead and at least one layervar layerCount int@@ -258,13 +261,14 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin}// Determine if we need to consider output then find where it fits- if memoryLayerOutput > 0 {+ memoryLastLayer := memoryLayerOutput + ollamaEngineProjectorWeights + ollamaEngineProjectorGraph+ if memoryLastLayer > 0 {if opts.NumGPU < 0 || layerCount < opts.NumGPU {for j := len(gpusWithSpace); j > 0; j-- {g := gpusWithSpace[layerCount%j]used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)- if g.g.FreeMemory > overhead+used+memoryLayerOutput {- gpuAllocations[g.i] += memoryLayerOutput+ if g.g.FreeMemory > overhead+used+memoryLastLayer {+ gpuAllocations[g.i] += memoryLastLayerlayerCounts[g.i]++layerCount++break@@ -274,7 +278,7 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinif layerCount < int(f.KV().BlockCount())+1 {fullyLoaded = false- overflow += memoryLayerOutput+ overflow += memoryLastLayer}}@@ -332,8 +336,8 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strinmemoryLayerOutput: memoryLayerOutput,graphFullOffload: graphFullOffload,graphPartialOffload: graphPartialOffload,- projectorWeights: projectorWeights,- projectorGraph: projectorGraph,+ projectorWeights: llamaEngineProjectorWeights + ollamaEngineProjectorWeights,+ projectorGraph: ollamaEngineProjectorGraph,}if gpus[0].Library == "cpu" {