Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llm: Consistently track unassigned model data

Jesse Grossa2cc8571c5b2
diff --git a/llm/memory.go b/llm/memory.go
index b5a8dd5c6..464723302 100644
--- a/llm/memory.go
+++ b/llm/memory.go
@@ -216,6 +216,8 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
if len(gpusWithSpace) > 0 {
gpuZeroID = gpusWithSpace[0].i
gpuAllocations[gpuZeroID] += gpuZeroOverhead
+ } else {
+ overflow += gpuZeroOverhead
}
// For all the layers, find where they can fit on the GPU(s)
@@ -256,15 +258,17 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
}
// Determine if we need to consider output then find where it fits
- if memoryLayerOutput > 0 && (opts.NumGPU < 0 || layerCount < opts.NumGPU) {
- for j := len(gpusWithSpace); j > 0; j-- {
- g := gpusWithSpace[layerCount%j]
- used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)
- if g.g.FreeMemory > overhead+used+memoryLayerOutput {
- gpuAllocations[g.i] += memoryLayerOutput
- layerCounts[g.i]++
- layerCount++
- break
+ if memoryLayerOutput > 0 {
+ if opts.NumGPU < 0 || layerCount < opts.NumGPU {
+ for j := len(gpusWithSpace); j > 0; j-- {
+ g := gpusWithSpace[layerCount%j]
+ used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)
+ if g.g.FreeMemory > overhead+used+memoryLayerOutput {
+ gpuAllocations[g.i] += memoryLayerOutput
+ layerCounts[g.i]++
+ layerCount++
+ break
+ }
}
}