Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Fix "Stopping..." scheduler hang (#10487)
Daniel Hiltgen415c8fcc3dee
diff --git a/server/sched.go b/server/sched.goindex 883540cea..8513fbf4b 100644--- a/server/sched.go+++ b/server/sched.go@@ -441,10 +441,9 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLisestimatedVRAM: llama.EstimatedVRAM(),estimatedTotal: llama.EstimatedTotal(),loading: true,- refCount: 1,}runner.numParallel = numParallel- runner.refMu.Lock()+ runner.refMu.Lock() // hold lock until running or aborteds.loadedMu.Lock()s.loaded[req.model.ModelPath] = runner@@ -455,13 +454,13 @@ func (s *Scheduler) load(req *LlmRequest, f *ggml.GGML, gpus discover.GpuInfoLisdefer runner.refMu.Unlock()if err = llama.WaitUntilRunning(req.ctx); err != nil {slog.Error("error loading llama server", "error", err)- runner.refCount--req.errCh <- errslog.Debug("triggering expiration for failed load", "model", runner.modelPath)s.expiredCh <- runnerreturn}slog.Debug("finished setting up runner", "model", req.model.ModelPath)+ runner.refCount++runner.loading = falsego func() {<-req.ctx.Done()@@ -479,7 +478,12 @@ func (s *Scheduler) updateFreeSpace(allGpus discover.GpuInfoList) {}predMap := map[predKey]uint64{} // Sum up the total predicted usage per GPU for all runnerss.loadedMu.Lock()+ runners := make([]*runnerRef, 0, len(s.loaded))for _, r := range s.loaded {+ runners = append(runners, r)+ }+ s.loadedMu.Unlock()+ for _, r := range runners {r.refMu.Lock()if r.llama != nil {for _, gpu := range allGpus {@@ -490,7 +494,6 @@ func (s *Scheduler) updateFreeSpace(allGpus discover.GpuInfoList) {}r.refMu.Unlock()}- s.loadedMu.Unlock()// Now that we've summed up all the GPU usage predictions across all the loaded runners, update the gpu listfor i := range allGpus {@@ -537,10 +540,8 @@ func (s *Scheduler) filterGPUsWithoutLoadingModels(allGpus discover.GpuInfoList)// TODO consolidate sched_types.gotype runnerRef struct {- refMu sync.Mutex- // refCond sync.Cond // Signaled on transition from 1 -> 0 refCount+ refMu sync.MutexrefCount uint // prevent unloading if > 0- // unloading bool // set to true when we are trying to unload the runnerllama llm.LlamaServerloading bool // True only during initial load, then false forever@@ -811,8 +812,8 @@ func (s *Scheduler) unloadAllRunners() {func (s *Scheduler) expireRunner(model *Model) {s.loadedMu.Lock()- defer s.loadedMu.Unlock()runner, ok := s.loaded[model.ModelPath]+ s.loadedMu.Unlock()if ok {runner.refMu.Lock()runner.expiresAt = time.Now()