Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Merge pull request #4218 from dhiltgen/auto_parallel
Daniel Hiltgen3518aaef3318
diff --cc server/sched.goindex 0084b533b,de8c9d281..87da1db47--- a/server/sched.go+++ b/server/sched.go@@@ -143,8 -165,28 +165,28 @@@ func (s *Scheduler) processPending(ctxgpus = s.getGpuFn()}+ if envconfig.MaxRunners <= 0 {+ // No user specified MaxRunners, so figure out what automatic setting to use+ // If all GPUs have reliable free memory reporting, defaultModelsPerGPU * the number of GPUs+ // if any GPU has unreliable free memory reporting, 1x the number of GPUs+ allReliable := true+ for _, gpu := range gpus {+ if gpu.UnreliableFreeMemory {+ allReliable = false+ break+ }+ }+ if allReliable {+ envconfig.MaxRunners = defaultModelsPerGPU * len(gpus)+ slog.Debug("updating default concurrency", "OLLAMA_MAX_LOADED_MODELS", envconfig.MaxRunners, "gpu_count", len(gpus))+ } else {+ slog.Info("one or more GPUs detected that are unable to accurately report free memory - disabling default concurrency")+ envconfig.MaxRunners = len(gpus)+ }+ }+// Load model for fitting- ggml, err := llm.LoadModel(pending.model.ModelPath)+ ggml, err := llm.LoadModel(pending.model.ModelPath, 0)if err != nil {pending.errCh <- errbreak