Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
comments
Michael Yang85d9d73a7253
diff --git a/envconfig/config.go b/envconfig/config.goindex 01abea421..b82b773d4 100644--- a/envconfig/config.go+++ b/envconfig/config.go@@ -1,7 +1,6 @@package envconfigimport (- "errors""fmt""log/slog""math"@@ -15,15 +14,12 @@ import ("time")-var ErrInvalidHostPort = errors.New("invalid port specified in OLLAMA_HOST")-// Host returns the scheme and host. Host can be configured via the OLLAMA_HOST environment variable.// Default is scheme "http" and host "127.0.0.1:11434"func Host() *url.URL {defaultPort := "11434"- s := os.Getenv("OLLAMA_HOST")- s = strings.TrimSpace(strings.Trim(strings.TrimSpace(s), "\"'"))+ s := strings.TrimSpace(Var("OLLAMA_HOST"))scheme, hostport, ok := strings.Cut(s, "://")switch {case !ok:@@ -48,6 +44,7 @@ func Host() *url.URL {}if n, err := strconv.ParseInt(port, 10, 32); err != nil || n > 65535 || n < 0 {+ slog.Warn("invalid port, using default", "port", port, "default", defaultPort)return &url.URL{Scheme: scheme,Host: net.JoinHostPort(host, defaultPort),@@ -62,7 +59,7 @@ func Host() *url.URL {// Origins returns a list of allowed origins. Origins can be configured via the OLLAMA_ORIGINS environment variable.func Origins() (origins []string) {- if s := getenv("OLLAMA_ORIGINS"); s != "" {+ if s := Var("OLLAMA_ORIGINS"); s != "" {origins = strings.Split(s, ",")}@@ -87,7 +84,7 @@ func Origins() (origins []string) {// Models returns the path to the models directory. Models directory can be configured via the OLLAMA_MODELS environment variable.// Default is $HOME/.ollama/modelsfunc Models() string {- if s, ok := os.LookupEnv("OLLAMA_MODELS"); ok {+ if s := Var("OLLAMA_MODELS"); s != "" {return s}@@ -104,7 +101,7 @@ func Models() string {// Default is 5 minutes.func KeepAlive() (keepAlive time.Duration) {keepAlive = 5 * time.Minute- if s := os.Getenv("OLLAMA_KEEP_ALIVE"); s != "" {+ if s := Var("OLLAMA_KEEP_ALIVE"); s != "" {if d, err := time.ParseDuration(s); err == nil {keepAlive = d} else if n, err := strconv.ParseInt(s, 10, 64); err == nil {@@ -121,7 +118,7 @@ func KeepAlive() (keepAlive time.Duration) {func Bool(k string) func() bool {return func() bool {- if s := getenv(k); s != "" {+ if s := Var(k); s != "" {b, err := strconv.ParseBool(s)if err != nil {return true@@ -151,7 +148,7 @@ var (func String(s string) func() string {return func() string {- return getenv(s)+ return Var(s)}}@@ -167,7 +164,7 @@ var ()func RunnersDir() (p string) {- if p := getenv("OLLAMA_RUNNERS_DIR"); p != "" {+ if p := Var("OLLAMA_RUNNERS_DIR"); p != "" {return p}@@ -213,22 +210,29 @@ func RunnersDir() (p string) {return p}-func Int(k string, n int) func() int {- return func() int {- if s := getenv(k); s != "" {- if n, err := strconv.ParseInt(s, 10, 64); err == nil && n >= 0 {- return int(n)+func Uint(key string, defaultValue uint) func() uint {+ return func() uint {+ if s := Var(key); s != "" {+ if n, err := strconv.ParseUint(s, 10, 64); err != nil {+ slog.Warn("invalid environment variable, using default", "key", key, "value", s, "default", defaultValue)+ } else {+ return uint(n)}}- return n+ return defaultValue}}var (- NumParallel = Int("OLLAMA_NUM_PARALLEL", 0)- MaxRunners = Int("OLLAMA_MAX_LOADED_MODELS", 0)- MaxQueue = Int("OLLAMA_MAX_QUEUE", 512)+ // NumParallel sets the number of parallel model requests. NumParallel can be configured via the OLLAMA_NUM_PARALLEL environment variable.+ NumParallel = Uint("OLLAMA_NUM_PARALLEL", 0)+ // MaxRunners sets the maximum number of loaded models. MaxRunners can be configured via the OLLAMA_MAX_LOADED_MODELS environment variable.+ MaxRunners = Uint("OLLAMA_MAX_LOADED_MODELS", 0)+ // MaxQueue sets the maximum number of queued requests. MaxQueue can be configured via the OLLAMA_MAX_QUEUE environment variable.+ MaxQueue = Uint("OLLAMA_MAX_QUEUE", 512)+ // MaxVRAM sets a maximum VRAM override in bytes. MaxVRAM can be configured via the OLLAMA_MAX_VRAM environment variable.+ MaxVRAM = Uint("OLLAMA_MAX_VRAM", 0))type EnvVar struct {@@ -274,7 +278,7 @@ func Values() map[string]string {return vals}-// getenv returns an environment variable stripped of leading and trailing quotes or spaces-func getenv(key string) string {- return strings.Trim(os.Getenv(key), "\"' ")+// Var returns an environment variable stripped of leading and trailing quotes or spaces+func Var(key string) string {+ return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")}diff --git a/envconfig/config_test.go b/envconfig/config_test.goindex 977298aaf..92a500f15 100644--- a/envconfig/config_test.go+++ b/envconfig/config_test.go@@ -30,6 +30,10 @@ func TestHost(t *testing.T) {"extra quotes": {"\"1.2.3.4\"", "1.2.3.4:11434"},"extra space+quotes": {" \" 1.2.3.4 \" ", "1.2.3.4:11434"},"extra single quotes": {"'1.2.3.4'", "1.2.3.4:11434"},+ "http": {"http://1.2.3.4", "1.2.3.4:80"},+ "http port": {"http://1.2.3.4:4321", "1.2.3.4:4321"},+ "https": {"https://1.2.3.4", "1.2.3.4:443"},+ "https port": {"https://1.2.3.4:4321", "1.2.3.4:4321"},}for name, tt := range cases {@@ -133,24 +137,45 @@ func TestOrigins(t *testing.T) {}func TestBool(t *testing.T) {- cases := map[string]struct {- value string- expect bool- }{- "empty": {"", false},- "true": {"true", true},- "false": {"false", false},- "1": {"1", true},- "0": {"0", false},- "random": {"random", true},- "something": {"something", true},+ cases := map[string]bool{+ "": false,+ "true": true,+ "false": false,+ "1": true,+ "0": false,+ // invalid values+ "random": true,+ "something": true,}- for name, tt := range cases {- t.Run(name, func(t *testing.T) {- t.Setenv("OLLAMA_BOOL", tt.value)- if b := Bool("OLLAMA_BOOL"); b() != tt.expect {- t.Errorf("%s: expected %t, got %t", name, tt.expect, b())+ for k, v := range cases {+ t.Run(k, func(t *testing.T) {+ t.Setenv("OLLAMA_BOOL", k)+ if b := Bool("OLLAMA_BOOL")(); b != v {+ t.Errorf("%s: expected %t, got %t", k, v, b)+ }+ })+ }+}++func TestUint(t *testing.T) {+ cases := map[string]uint{+ "0": 0,+ "1": 1,+ "1337": 1337,+ // default values+ "": 11434,+ "-1": 11434,+ "0o10": 11434,+ "0x10": 11434,+ "string": 11434,+ }++ for k, v := range cases {+ t.Run(k, func(t *testing.T) {+ t.Setenv("OLLAMA_UINT", k)+ if i := Uint("OLLAMA_UINT", 11434)(); i != v {+ t.Errorf("%s: expected %d, got %d", k, v, i)}})}@@ -188,3 +213,23 @@ func TestKeepAlive(t *testing.T) {})}}++func TestVar(t *testing.T) {+ cases := map[string]string{+ "value": "value",+ " value ": "value",+ " 'value' ": "value",+ ` "value" `: "value",+ " ' value ' ": " value ",+ ` " value " `: " value ",+ }++ for k, v := range cases {+ t.Run(k, func(t *testing.T) {+ t.Setenv("OLLAMA_VAR", k)+ if s := Var("OLLAMA_VAR"); s != v {+ t.Errorf("%s: expected %q, got %q", k, v, s)+ }+ })+ }+}diff --git a/server/sched.go b/server/sched.goindex 610a2c50f..ce2945d8e 100644--- a/server/sched.go+++ b/server/sched.go@@ -129,7 +129,7 @@ func (s *Scheduler) processPending(ctx context.Context) {slog.Debug("pending request cancelled or timed out, skipping scheduling")continue}- numParallel := envconfig.NumParallel()+ numParallel := int(envconfig.NumParallel())// TODO (jmorganca): multimodal models don't support parallel yet// see https://github.com/ollama/ollama/issues/4165if len(pending.model.ProjectorPaths) > 0 && numParallel != 1 {@@ -151,7 +151,7 @@ func (s *Scheduler) processPending(ctx context.Context) {pending.useLoadedRunner(runner, s.finishedReqCh)break}- } else if envconfig.MaxRunners() > 0 && loadedCount >= envconfig.MaxRunners() {+ } else if envconfig.MaxRunners() > 0 && loadedCount >= int(envconfig.MaxRunners()) {slog.Debug("max runners achieved, unloading one to make room", "runner_count", loadedCount)runnerToExpire = s.findRunnerToUnload()} else {