Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

Merge pull request #4218 from dhiltgen/auto_parallel

Daniel Hiltgen3518aaef3318
diff --cc server/sched.go
index 0084b533b,de8c9d281..87da1db47
--- a/server/sched.go
+++ b/server/sched.go
@@@ -143,8 -165,28 +165,28 @@@ func (s *Scheduler) processPending(ctx
gpus = s.getGpuFn()
}
+ if envconfig.MaxRunners <= 0 {
+ // No user specified MaxRunners, so figure out what automatic setting to use
+ // If all GPUs have reliable free memory reporting, defaultModelsPerGPU * the number of GPUs
+ // if any GPU has unreliable free memory reporting, 1x the number of GPUs
+ allReliable := true
+ for _, gpu := range gpus {
+ if gpu.UnreliableFreeMemory {
+ allReliable = false
+ break
+ }
+ }
+ if allReliable {
+ envconfig.MaxRunners = defaultModelsPerGPU * len(gpus)
+ slog.Debug("updating default concurrency", "OLLAMA_MAX_LOADED_MODELS", envconfig.MaxRunners, "gpu_count", len(gpus))
+ } else {
+ slog.Info("one or more GPUs detected that are unable to accurately report free memory - disabling default concurrency")
+ envconfig.MaxRunners = len(gpus)
+ }
+ }
+
// Load model for fitting
- ggml, err := llm.LoadModel(pending.model.ModelPath)
+ ggml, err := llm.LoadModel(pending.model.ModelPath, 0)
if err != nil {
pending.errCh <- err
break