Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llm: Don't always evict models in CPU-only mode
Jesse Gross073fa31df501
diff --git a/llm/memory.go b/llm/memory.goindex ee4be7419..d8ae5e44a 100644--- a/llm/memory.go+++ b/llm/memory.go@@ -30,7 +30,7 @@ func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []strin// Try to pack into as few GPUs as possible, starting from 1 GPUfor numGPUs := 1; numGPUs <= len(sgl); numGPUs++ {gpuSubset := sgl[:numGPUs]- ok, estimatedVRAM := PredictServerFit(gpuSubset, f, adapters, projectors, opts, numParallel)+ ok, estimatedVRAM := predictServerFit(gpuSubset, f, adapters, projectors, opts, numParallel)if ok {slog.Info("new model will fit in available VRAM across minimum required GPUs, loading",@@ -48,7 +48,7 @@ func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []strin// - try subsets of GPUs instead of just falling back to 1 or all in a family// Now try all the GPUS (OLLAMA_SCHED_SPREAD is set)- if ok, estimatedVRAM := PredictServerFit(sgl, f, adapters, projectors, opts, numParallel); ok {+ if ok, estimatedVRAM := predictServerFit(sgl, f, adapters, projectors, opts, numParallel); ok {slog.Info("new model will fit in available VRAM, loading","model", modelPath,"library", sgl[0].Library,@@ -71,7 +71,7 @@ func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []svar bestEstimate uint64var bestFit intfor i, gl := range byLibrary {- _, estimatedVRAM := PredictServerFit(gl, f, adapters, projectors, opts, numParallel)+ _, estimatedVRAM := predictServerFit(gl, f, adapters, projectors, opts, numParallel)if estimatedVRAM > bestEstimate {bestEstimate = estimatedVRAMbestFit = i@@ -81,7 +81,7 @@ func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []s}// This algorithm looks for a complete fit to determine if we need to unload other models-func PredictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64) {+func predictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64) {// Split up the GPUs by type and try themvar estimatedVRAM uint64for _, gpus := range allGpus.ByLibrary() {@@ -97,6 +97,10 @@ func PredictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projreturn true, estimatedVRAM}}++ if len(gpus) == 1 && gpus[0].Library == "cpu" && estimate.TotalSize <= gpus[0].FreeMemory {+ return true, estimatedVRAM+ }}return false, estimatedVRAM}diff --git a/llm/server.go b/llm/server.goindex ecdaa90e9..b05e9b82d 100644--- a/llm/server.go+++ b/llm/server.go@@ -492,6 +492,7 @@ func (s *llamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requiif !requireFull {g = pickBestPartialFitByLibrary(s.ggml, []string{s.loadRequest.ProjectorPath}, s.loadRequest.LoraPath, s.options, gpus, s.numParallel)} else {+ slog.Info("model requires more memory than is currently available, evicting a model to make space", "estimate", s.estimate)return ErrLoadRequiredFull}}@@ -524,10 +525,6 @@ func (s *llamaServer) Load(ctx context.Context, gpus discover.GpuInfoList, requi}}- if requireFull && len(gpus) == 1 && gpus[0].Library == "cpu" && s.estimate.TotalSize > gpus[0].FreeMemory {- return ErrLoadRequiredFull- }-slog.Info("offload", "", s.estimate)s.gpus = gpus