Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
int
Michael Yang0f1910129f0a
diff --git a/envconfig/config.go b/envconfig/config.goindex 34cc4dac5..01abea421 100644--- a/envconfig/config.go+++ b/envconfig/config.go@@ -213,13 +213,22 @@ func RunnersDir() (p string) {return p}+func Int(k string, n int) func() int {+ return func() int {+ if s := getenv(k); s != "" {+ if n, err := strconv.ParseInt(s, 10, 64); err == nil && n >= 0 {+ return int(n)+ }+ }++ return n+ }+}+var (- // Set via OLLAMA_MAX_LOADED_MODELS in the environment- MaxRunners int- // Set via OLLAMA_MAX_QUEUE in the environment- MaxQueuedRequests int- // Set via OLLAMA_NUM_PARALLEL in the environment- NumParallel int+ NumParallel = Int("OLLAMA_NUM_PARALLEL", 0)+ MaxRunners = Int("OLLAMA_MAX_LOADED_MODELS", 0)+ MaxQueue = Int("OLLAMA_MAX_QUEUE", 512))type EnvVar struct {@@ -235,12 +244,12 @@ func AsMap() map[string]EnvVar {"OLLAMA_HOST": {"OLLAMA_HOST", Host(), "IP Address for the ollama server (default 127.0.0.1:11434)"},"OLLAMA_KEEP_ALIVE": {"OLLAMA_KEEP_ALIVE", KeepAlive(), "The duration that models stay loaded in memory (default \"5m\")"},"OLLAMA_LLM_LIBRARY": {"OLLAMA_LLM_LIBRARY", LLMLibrary(), "Set LLM library to bypass autodetection"},- "OLLAMA_MAX_LOADED_MODELS": {"OLLAMA_MAX_LOADED_MODELS", MaxRunners, "Maximum number of loaded models per GPU"},- "OLLAMA_MAX_QUEUE": {"OLLAMA_MAX_QUEUE", MaxQueuedRequests, "Maximum number of queued requests"},+ "OLLAMA_MAX_LOADED_MODELS": {"OLLAMA_MAX_LOADED_MODELS", MaxRunners(), "Maximum number of loaded models per GPU"},+ "OLLAMA_MAX_QUEUE": {"OLLAMA_MAX_QUEUE", MaxQueue(), "Maximum number of queued requests"},"OLLAMA_MODELS": {"OLLAMA_MODELS", Models(), "The path to the models directory"},"OLLAMA_NOHISTORY": {"OLLAMA_NOHISTORY", NoHistory(), "Do not preserve readline history"},"OLLAMA_NOPRUNE": {"OLLAMA_NOPRUNE", NoPrune(), "Do not prune model blobs on startup"},- "OLLAMA_NUM_PARALLEL": {"OLLAMA_NUM_PARALLEL", NumParallel, "Maximum number of parallel requests"},+ "OLLAMA_NUM_PARALLEL": {"OLLAMA_NUM_PARALLEL", NumParallel(), "Maximum number of parallel requests"},"OLLAMA_ORIGINS": {"OLLAMA_ORIGINS", Origins(), "A comma separated list of allowed origins"},"OLLAMA_RUNNERS_DIR": {"OLLAMA_RUNNERS_DIR", RunnersDir(), "Location for runners"},"OLLAMA_SCHED_SPREAD": {"OLLAMA_SCHED_SPREAD", SchedSpread(), "Always schedule model across all GPUs"},@@ -269,42 +278,3 @@ func Values() map[string]string {func getenv(key string) string {return strings.Trim(os.Getenv(key), "\"' ")}--func init() {- // default values- NumParallel = 0 // Autoselect- MaxRunners = 0 // Autoselect- MaxQueuedRequests = 512-- LoadConfig()-}--func LoadConfig() {- if onp := getenv("OLLAMA_NUM_PARALLEL"); onp != "" {- val, err := strconv.Atoi(onp)- if err != nil {- slog.Error("invalid setting, ignoring", "OLLAMA_NUM_PARALLEL", onp, "error", err)- } else {- NumParallel = val- }- }-- maxRunners := getenv("OLLAMA_MAX_LOADED_MODELS")- if maxRunners != "" {- m, err := strconv.Atoi(maxRunners)- if err != nil {- slog.Error("invalid setting, ignoring", "OLLAMA_MAX_LOADED_MODELS", maxRunners, "error", err)- } else {- MaxRunners = m- }- }-- if onp := os.Getenv("OLLAMA_MAX_QUEUE"); onp != "" {- p, err := strconv.Atoi(onp)- if err != nil || p <= 0 {- slog.Error("invalid setting, ignoring", "OLLAMA_MAX_QUEUE", onp, "error", err)- } else {- MaxQueuedRequests = p- }- }-}diff --git a/integration/basic_test.go b/integration/basic_test.goindex 6e632a1ce..8e35b5c5b 100644--- a/integration/basic_test.go+++ b/integration/basic_test.go@@ -45,14 +45,7 @@ func TestUnicodeModelDir(t *testing.T) {defer os.RemoveAll(modelDir)slog.Info("unicode", "OLLAMA_MODELS", modelDir)- oldModelsDir := os.Getenv("OLLAMA_MODELS")- if oldModelsDir == "" {- defer os.Unsetenv("OLLAMA_MODELS")- } else {- defer os.Setenv("OLLAMA_MODELS", oldModelsDir)- }- err = os.Setenv("OLLAMA_MODELS", modelDir)- require.NoError(t, err)+ t.Setenv("OLLAMA_MODELS", modelDir)ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)defer cancel()diff --git a/integration/max_queue_test.go b/integration/max_queue_test.goindex dfa5eae0c..b06197e1f 100644--- a/integration/max_queue_test.go+++ b/integration/max_queue_test.go@@ -5,7 +5,6 @@ package integrationimport ("context""errors"- "fmt""log/slog""os""strconv"@@ -14,8 +13,10 @@ import ("testing""time"- "github.com/ollama/ollama/api""github.com/stretchr/testify/require"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/envconfig")func TestMaxQueue(t *testing.T) {@@ -27,13 +28,10 @@ func TestMaxQueue(t *testing.T) {// Note: This test can be quite slow when running in CPU mode, so keep the threadCount low unless your on GPU// Also note that by default Darwin can't sustain > ~128 connections without adjusting limitsthreadCount := 32- mq := os.Getenv("OLLAMA_MAX_QUEUE")- if mq != "" {- var err error- threadCount, err = strconv.Atoi(mq)- require.NoError(t, err)+ if maxQueue := envconfig.MaxQueue(); maxQueue != 0 {+ threadCount = maxQueue} else {- os.Setenv("OLLAMA_MAX_QUEUE", fmt.Sprintf("%d", threadCount))+ t.Setenv("OLLAMA_MAX_QUEUE", strconv.Itoa(threadCount))}req := api.GenerateRequest{diff --git a/server/sched.go b/server/sched.goindex ad40c4ef0..610a2c50f 100644--- a/server/sched.go+++ b/server/sched.go@@ -5,9 +5,11 @@ import ("errors""fmt""log/slog"+ "os""reflect""runtime""sort"+ "strconv""strings""sync""time"@@ -59,11 +61,12 @@ var defaultParallel = 4var ErrMaxQueue = fmt.Errorf("server busy, please try again. maximum pending requests exceeded")func InitScheduler(ctx context.Context) *Scheduler {+ maxQueue := envconfig.MaxQueue()sched := &Scheduler{- pendingReqCh: make(chan *LlmRequest, envconfig.MaxQueuedRequests),- finishedReqCh: make(chan *LlmRequest, envconfig.MaxQueuedRequests),- expiredCh: make(chan *runnerRef, envconfig.MaxQueuedRequests),- unloadedCh: make(chan interface{}, envconfig.MaxQueuedRequests),+ pendingReqCh: make(chan *LlmRequest, maxQueue),+ finishedReqCh: make(chan *LlmRequest, maxQueue),+ expiredCh: make(chan *runnerRef, maxQueue),+ unloadedCh: make(chan interface{}, maxQueue),loaded: make(map[string]*runnerRef),newServerFn: llm.NewLlamaServer,getGpuFn: gpu.GetGPUInfo,@@ -126,7 +129,7 @@ func (s *Scheduler) processPending(ctx context.Context) {slog.Debug("pending request cancelled or timed out, skipping scheduling")continue}- numParallel := envconfig.NumParallel+ numParallel := envconfig.NumParallel()// TODO (jmorganca): multimodal models don't support parallel yet// see https://github.com/ollama/ollama/issues/4165if len(pending.model.ProjectorPaths) > 0 && numParallel != 1 {@@ -148,7 +151,7 @@ func (s *Scheduler) processPending(ctx context.Context) {pending.useLoadedRunner(runner, s.finishedReqCh)break}- } else if envconfig.MaxRunners > 0 && loadedCount >= envconfig.MaxRunners {+ } else if envconfig.MaxRunners() > 0 && loadedCount >= envconfig.MaxRunners() {slog.Debug("max runners achieved, unloading one to make room", "runner_count", loadedCount)runnerToExpire = s.findRunnerToUnload()} else {@@ -161,7 +164,7 @@ func (s *Scheduler) processPending(ctx context.Context) {gpus = s.getGpuFn()}- if envconfig.MaxRunners <= 0 {+ if envconfig.MaxRunners() <= 0 {// No user specified MaxRunners, so figure out what automatic setting to use// If all GPUs have reliable free memory reporting, defaultModelsPerGPU * the number of GPUs// if any GPU has unreliable free memory reporting, 1x the number of GPUs@@ -173,11 +176,13 @@ func (s *Scheduler) processPending(ctx context.Context) {}}if allReliable {- envconfig.MaxRunners = defaultModelsPerGPU * len(gpus)+ // HACK+ os.Setenv("OLLAMA_MAX_LOADED_MODELS", strconv.Itoa(defaultModelsPerGPU*len(gpus)))slog.Debug("updating default concurrency", "OLLAMA_MAX_LOADED_MODELS", envconfig.MaxRunners, "gpu_count", len(gpus))} else {+ // HACK+ os.Setenv("OLLAMA_MAX_LOADED_MODELS", strconv.Itoa(len(gpus)))slog.Info("one or more GPUs detected that are unable to accurately report free memory - disabling default concurrency")- envconfig.MaxRunners = len(gpus)}}diff --git a/server/sched_test.go b/server/sched_test.goindex 9ddd1fabe..3166ff66c 100644--- a/server/sched_test.go+++ b/server/sched_test.go@@ -12,7 +12,6 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/app/lifecycle"- "github.com/ollama/ollama/envconfig""github.com/ollama/ollama/format""github.com/ollama/ollama/gpu""github.com/ollama/ollama/llm"@@ -272,7 +271,7 @@ func TestRequestsMultipleLoadedModels(t *testing.T) {c.req.opts.NumGPU = 0 // CPU load, will be allowedd := newScenarioRequest(t, ctx, "ollama-model-3c", 30, nil) // Needs prior unloaded- envconfig.MaxRunners = 1+ t.Setenv("OLLAMA_MAX_LOADED_MODELS", "1")s.newServerFn = a.newServerslog.Info("a")s.pendingReqCh <- a.req@@ -291,7 +290,7 @@ func TestRequestsMultipleLoadedModels(t *testing.T) {require.Len(t, s.loaded, 1)s.loadedMu.Unlock()- envconfig.MaxRunners = 0+ t.Setenv("OLLAMA_MAX_LOADED_MODELS", "0")s.newServerFn = b.newServerslog.Info("b")s.pendingReqCh <- b.req@@ -362,7 +361,7 @@ func TestGetRunner(t *testing.T) {a := newScenarioRequest(t, ctx, "ollama-model-1a", 10, &api.Duration{Duration: 2 * time.Millisecond})b := newScenarioRequest(t, ctx, "ollama-model-1b", 10, &api.Duration{Duration: 2 * time.Millisecond})c := newScenarioRequest(t, ctx, "ollama-model-1c", 10, &api.Duration{Duration: 2 * time.Millisecond})- envconfig.MaxQueuedRequests = 1+ t.Setenv("OLLAMA_MAX_QUEUE", "1")s := InitScheduler(ctx)s.getGpuFn = getGpuFns.getCpuFn = getCpuFn