Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llm: reserve required number of slots for embeddings (#6219)
Jeffrey Morgande4fc297732c
diff --git a/llm/server.go b/llm/server.goindex 152b7582f..41736068c 100644--- a/llm/server.go+++ b/llm/server.go@@ -44,11 +44,12 @@ type LlamaServer interface {// llmServer is an instance of the llama.cpp servertype llmServer struct {- port int- cmd *exec.Cmd- done chan error // Channel to signal when the process exits- status *StatusWriter- options api.Options+ port int+ cmd *exec.Cmd+ done chan error // Channel to signal when the process exits+ status *StatusWriter+ options api.Options+ numParallel intestimate MemoryEstimatetotalLayers uint64@@ -343,6 +344,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, prstatus: NewStatusWriter(os.Stderr),options: opts,estimate: estimate,+ numParallel: numParallel,sem: semaphore.NewWeighted(int64(numParallel)),totalLayers: ggml.KV().BlockCount() + 1,gpus: gpus,@@ -890,11 +892,14 @@ type EmbedResponse struct {}func (s *llmServer) Embed(ctx context.Context, input []string) (*EmbedResponse, error) {- if err := s.sem.Acquire(ctx, 1); err != nil {+ // each input will use a slot, so we need to acquire the semaphore for+ // the number of inputs up to numParallel+ slots := int64(min(len(input), s.numParallel))+ if err := s.sem.Acquire(ctx, slots); err != nil {slog.Error("Failed to acquire semaphore", "error", err)return nil, err}- defer s.sem.Release(1)+ defer s.sem.Release(slots)// Make sure the server is readystatus, err := s.getServerStatusRetry(ctx)