Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

Reduce startup model hydration (#16215)

Daniel Hiltgen4b2d529966f5
diff --git a/api/types.go b/api/types.go
index 51ddafb86..a46d3edb1 100644
--- a/api/types.go
+++ b/api/types.go
@@ -824,14 +824,15 @@ type ProcessResponse struct {
// ListModelResponse is a single model description in [ListResponse].
type ListModelResponse struct {
- Name string `json:"name"`
- Model string `json:"model"`
- RemoteModel string `json:"remote_model,omitempty"`
- RemoteHost string `json:"remote_host,omitempty"`
- ModifiedAt time.Time `json:"modified_at"`
- Size int64 `json:"size"`
- Digest string `json:"digest"`
- Details ModelDetails `json:"details,omitempty"`
+ Name string `json:"name"`
+ Model string `json:"model"`
+ RemoteModel string `json:"remote_model,omitempty"`
+ RemoteHost string `json:"remote_host,omitempty"`
+ ModifiedAt time.Time `json:"modified_at"`
+ Size int64 `json:"size"`
+ Digest string `json:"digest"`
+ Details ModelDetails `json:"details,omitempty"`
+ Capabilities []model.Capability `json:"capabilities,omitempty"`
}
// ProcessModelResponse is a single model description in [ProcessResponse].
@@ -924,6 +925,8 @@ type ModelDetails struct {
Families []string `json:"families"`
ParameterSize string `json:"parameter_size"`
QuantizationLevel string `json:"quantization_level"`
+ ContextLength int `json:"context_length,omitempty"`
+ EmbeddingLength int `json:"embedding_length,omitempty"`
}
// UserResponse provides information about a user.
diff --git a/cmd/launch/integrations_test.go b/cmd/launch/integrations_test.go
index 7e7879bba..f1ef59ee5 100644
--- a/cmd/launch/integrations_test.go
+++ b/cmd/launch/integrations_test.go
@@ -481,11 +481,11 @@ func TestBuildModelList_ExistingRecommendedMarked(t *testing.T) {
func TestBuildModelList_PreservesRecommendationRequiredPlanForExistingCloudModel(t *testing.T) {
recommendations := []ModelItem{
{
- Name: "glm-5:cloud",
- Description: "Reasoning and code generation",
- Recommended: true,
- RequiredPlan: "pro",
- ContextLength: 202_752,
+ Name: "glm-5:cloud",
+ Description: "Reasoning and code generation",
+ Recommended: true,
+ RequiredPlan: "pro",
+ Details: api.ModelDetails{ContextLength: 202_752},
},
}
existing := []modelInfo{{Name: "glm-5:cloud", Remote: true}}
diff --git a/cmd/launch/launch.go b/cmd/launch/launch.go
index 0cb81c14e..9311ca393 100644
--- a/cmd/launch/launch.go
+++ b/cmd/launch/launch.go
@@ -12,6 +12,7 @@ import (
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/cmd/config"
+ modelpkg "github.com/ollama/ollama/types/model"
"github.com/spf13/cobra"
"golang.org/x/term"
)
@@ -240,9 +241,12 @@ type SupportedIntegration interface {
}
type modelInfo struct {
- Name string
- Remote bool
- ToolCapable bool
+ Name string
+ Remote bool
+ ToolCapable bool
+ Capabilities []modelpkg.Capability
+ Size int64
+ Details api.ModelDetails
}
// ModelInfo re-exports launcher model inventory details for callers.
@@ -254,9 +258,12 @@ type ModelItem struct {
Description string
Recommended bool
VRAMBytes int64
- ContextLength int
MaxOutputTokens int
RequiredPlan string
+ ToolCapable bool
+ Capabilities []modelpkg.Capability
+ Size int64
+ Details api.ModelDetails
}
// SelectionItem represents a model row after launch has derived selector-only UI state.
@@ -1179,9 +1186,11 @@ func (c *launcherClient) requestRecommendations(ctx context.Context) ([]ModelIte
Description: description,
Recommended: true,
VRAMBytes: rec.VRAMBytes,
- ContextLength: rec.ContextLength,
MaxOutputTokens: rec.MaxOutputTokens,
RequiredPlan: strings.TrimSpace(rec.RequiredPlan),
+ Details: api.ModelDetails{
+ ContextLength: rec.ContextLength,
+ },
})
}
@@ -1367,8 +1376,12 @@ func (c *launcherClient) loadModelInventoryOnce(ctx context.Context) error {
c.modelInventory = c.modelInventory[:0]
for _, model := range resp.Models {
c.modelInventory = append(c.modelInventory, ModelInfo{
- Name: model.Name,
- Remote: model.RemoteModel != "",
+ Name: model.Name,
+ Remote: model.RemoteModel != "",
+ ToolCapable: slices.Contains(model.Capabilities, modelpkg.CapabilityTools),
+ Capabilities: slices.Clone(model.Capabilities),
+ Size: model.Size,
+ Details: model.Details,
})
}
diff --git a/cmd/launch/launch_test.go b/cmd/launch/launch_test.go
index c0903aaf9..c813ce2c9 100644
--- a/cmd/launch/launch_test.go
+++ b/cmd/launch/launch_test.go
@@ -11,6 +11,7 @@ import (
"runtime"
"slices"
"strings"
+ "sync/atomic"
"testing"
"time"
@@ -1462,6 +1463,53 @@ func TestBuildLauncherState_ToleratesInventoryFailure(t *testing.T) {
}
}
+func TestBuildLauncherState_UsesTagsInventoryWithoutShow(t *testing.T) {
+ tmpDir := t.TempDir()
+ setLaunchTestHome(t, tmpDir)
+
+ if err := config.SetLastModel("llama3.2"); err != nil {
+ t.Fatalf("failed to seed last model: %v", err)
+ }
+ if err := config.SaveIntegration("codex", []string{"qwen3:8b"}); err != nil {
+ t.Fatalf("failed to seed codex config: %v", err)
+ }
+
+ var showCalls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api/tags":
+ fmt.Fprint(w, `{"models":[`+
+ `{"name":"llama3.2","capabilities":["completion","tools"],"context_length":131072,"size":3200000000},`+
+ `{"name":"qwen3:8b","capabilities":["completion","tools"],"context_length":65536,"size":4500000000}`+
+ `]}`)
+ case "/api/show":
+ showCalls.Add(1)
+ fmt.Fprint(w, `{"model_info":{"general.context_length":131072}}`)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer srv.Close()
+ t.Setenv("OLLAMA_HOST", srv.URL)
+
+ state, err := BuildLauncherState(context.Background())
+ if err != nil {
+ t.Fatalf("BuildLauncherState returned error: %v", err)
+ }
+ if !state.RunModelUsable {
+ t.Fatal("expected saved run model to be usable from tags inventory")
+ }
+ if state.Integrations["codex"].CurrentModel != "qwen3:8b" {
+ t.Fatalf("expected codex current model from saved config, got %q", state.Integrations["codex"].CurrentModel)
+ }
+ if !state.Integrations["codex"].ModelUsable {
+ t.Fatal("expected saved codex model to be usable from tags inventory")
+ }
+ if got := showCalls.Load(); got != 0 {
+ t.Fatalf("show calls = %d, want 0 for broad launcher state", got)
+ }
+}
+
func TestResolveRunModel_UsesSavedModelWithoutSelector(t *testing.T) {
tmpDir := t.TempDir()
setLaunchTestHome(t, tmpDir)
diff --git a/cmd/launch/models.go b/cmd/launch/models.go
index 2283af541..2114dfb8e 100644
--- a/cmd/launch/models.go
+++ b/cmd/launch/models.go
@@ -23,10 +23,10 @@ import (
)
var recommendedModels = []ModelItem{
- {Name: "kimi-k2.6:cloud", Description: "State-of-the-art coding, long-horizon execution, and multimodal agent swarm capability", Recommended: true, ContextLength: 262_144, MaxOutputTokens: 262_144},
- {Name: "qwen3.5:cloud", Description: "Reasoning, coding, and agentic tool use with vision", Recommended: true, ContextLength: 262_144, MaxOutputTokens: 32_768},
- {Name: "glm-5.1:cloud", Description: "Reasoning and code generation", Recommended: true, ContextLength: 202_752, MaxOutputTokens: 131_072},
- {Name: "minimax-m2.7:cloud", Description: "Fast, efficient coding and real-world productivity", Recommended: true, ContextLength: 204_800, MaxOutputTokens: 128_000},
+ {Name: "kimi-k2.6:cloud", Description: "State-of-the-art coding, long-horizon execution, and multimodal agent swarm capability", Recommended: true, Details: api.ModelDetails{ContextLength: 262_144}, MaxOutputTokens: 262_144},
+ {Name: "qwen3.5:cloud", Description: "Reasoning, coding, and agentic tool use with vision", Recommended: true, Details: api.ModelDetails{ContextLength: 262_144}, MaxOutputTokens: 32_768},
+ {Name: "glm-5.1:cloud", Description: "Reasoning and code generation", Recommended: true, Details: api.ModelDetails{ContextLength: 202_752}, MaxOutputTokens: 131_072},
+ {Name: "minimax-m2.7:cloud", Description: "Fast, efficient coding and real-world productivity", Recommended: true, Details: api.ModelDetails{ContextLength: 204_800}, MaxOutputTokens: 128_000},
{Name: "gemma4", Description: "Reasoning and code generation locally", Recommended: true, VRAMBytes: 12 * format.GigaByte},
{Name: "qwen3.5", Description: "Reasoning, coding, and visual understanding locally", Recommended: true, VRAMBytes: 14 * format.GigaByte},
}
@@ -115,7 +115,7 @@ func setDynamicCloudModelLimits(limits map[string]cloudModelLimit) {
func cloudModelLimitsFromRecommendations(recommendations []ModelItem) map[string]cloudModelLimit {
limits := make(map[string]cloudModelLimit, len(recommendations))
for _, rec := range recommendations {
- if !isCloudModelName(rec.Name) || rec.ContextLength <= 0 || rec.MaxOutputTokens <= 0 {
+ if !isCloudModelName(rec.Name) || rec.Details.ContextLength <= 0 || rec.MaxOutputTokens <= 0 {
continue
}
base, stripped := modelref.StripCloudSourceTag(rec.Name)
@@ -123,7 +123,7 @@ func cloudModelLimitsFromRecommendations(recommendations []ModelItem) map[string
continue
}
limits[base] = cloudModelLimit{
- Context: rec.ContextLength,
+ Context: rec.Details.ContextLength,
Output: rec.MaxOutputTokens,
}
}
@@ -365,11 +365,11 @@ func buildModelListWithRecommendations(existing []modelInfo, recommendations []M
}
displayName := strings.TrimSuffix(m.Name, ":latest")
existingModels[displayName] = true
- item := ModelItem{Name: displayName, Recommended: recommended[displayName], Description: recDesc[displayName]}
if rec, ok := recByName[displayName]; ok {
- item = copyModelRecommendationFields(displayName, rec)
+ items = append(items, modelItemFromInventory(displayName, m, copyModelRecommendationFields(displayName, rec)))
+ } else {
+ items = append(items, modelItemFromInventory(displayName, m, ModelItem{Name: displayName, Recommended: recommended[displayName], Description: recDesc[displayName]}))
}
- items = append(items, item)
}
for _, rec := range recommendations {
@@ -483,6 +483,15 @@ func copyModelRecommendationFields(name string, rec ModelItem) ModelItem {
return rec
}
+func modelItemFromInventory(name string, info modelInfo, item ModelItem) ModelItem {
+ item.Name = name
+ item.ToolCapable = info.ToolCapable
+ item.Capabilities = slices.Clone(info.Capabilities)
+ item.Size = info.Size
+ item.Details = info.Details
+ return item
+}
+
// isCloudModelName reports whether the model name has an explicit cloud source.
func isCloudModelName(name string) bool {
return modelref.HasExplicitCloudSource(name)
diff --git a/cmd/launch/models_test.go b/cmd/launch/models_test.go
new file mode 100644
index 000000000..30abf4091
--- /dev/null
+++ b/cmd/launch/models_test.go
@@ -0,0 +1,83 @@
+package launch
+
+import (
+ "testing"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/format"
+ modelpkg "github.com/ollama/ollama/types/model"
+)
+
+func TestBuildModelList_UsesInventoryMetadataForInstalledModels(t *testing.T) {
+ existing := []modelInfo{
+ {
+ Name: "custom-tools:latest",
+ ToolCapable: true,
+ Capabilities: []modelpkg.Capability{modelpkg.CapabilityCompletion, modelpkg.CapabilityTools, modelpkg.CapabilityThinking},
+ Size: 7500 * format.MegaByte,
+ Details: api.ModelDetails{
+ ParameterSize: "8B",
+ QuantizationLevel: "Q4_K_M",
+ ContextLength: 131_072,
+ EmbeddingLength: 4096,
+ },
+ },
+ }
+
+ items, _, _, _ := buildModelList(existing, nil, "")
+ var got ModelItem
+ for _, item := range items {
+ if item.Name == "custom-tools" {
+ got = item
+ break
+ }
+ }
+ if got.Name == "" {
+ t.Fatal("custom-tools not found in items")
+ }
+ if !got.ToolCapable {
+ t.Fatal("expected installed model to preserve tool capability from tags metadata")
+ }
+ if got.Details.ContextLength != 131_072 {
+ t.Fatalf("Details.ContextLength = %d, want 131072", got.Details.ContextLength)
+ }
+ if got.Size != 7500*format.MegaByte {
+ t.Fatalf("Size = %d, want %d", got.Size, 7500*format.MegaByte)
+ }
+ if got.Description != "" {
+ t.Fatalf("Description = %q, want empty for installed model without recommendation copy", got.Description)
+ }
+}
+
+func TestBuildModelList_InstalledRecommendedPreservesRecommendationAndMetadata(t *testing.T) {
+ existing := []modelInfo{
+ {
+ Name: "qwen3.5",
+ ToolCapable: true,
+ Capabilities: []modelpkg.Capability{modelpkg.CapabilityCompletion, modelpkg.CapabilityTools, modelpkg.CapabilityVision},
+ Size: 14 * format.GigaByte,
+ Details: api.ModelDetails{ContextLength: 262_144},
+ },
+ }
+
+ items, _, _, _ := buildModelList(existing, nil, "")
+ var got ModelItem
+ for _, item := range items {
+ if item.Name == "qwen3.5" {
+ got = item
+ break
+ }
+ }
+ if got.Name == "" {
+ t.Fatal("qwen3.5 not found in items")
+ }
+ if !got.Recommended || !got.ToolCapable {
+ t.Fatalf("recommended/tool metadata = %v/%v, want true/true", got.Recommended, got.ToolCapable)
+ }
+ if got.Details.ContextLength != 262_144 {
+ t.Fatalf("Details.ContextLength = %d, want 262144", got.Details.ContextLength)
+ }
+ if got.Description != "Reasoning, coding, and visual understanding locally" {
+ t.Fatalf("Description = %q, want recommendation description", got.Description)
+ }
+}
diff --git a/server/create.go b/server/create.go
index 49ca901e6..a6600a42f 100644
--- a/server/create.go
+++ b/server/create.go
@@ -271,6 +271,8 @@ func (s *Server) CreateHandler(c *gin.Context) {
}
}
+ s.refreshModelListCache(name)
+
ch <- api.ProgressResponse{Status: "success"}
}()
diff --git a/server/model_caches.go b/server/model_caches.go
index 720fa703d..6c09b477c 100644
--- a/server/model_caches.go
+++ b/server/model_caches.go
@@ -5,12 +5,14 @@ import "context"
type modelCaches struct {
recommendations *modelRecommendationsCache
show *modelShowCache
+ modelList *modelListCache
}
func newModelCaches() *modelCaches {
return &modelCaches{
recommendations: newModelRecommendationsCache(),
show: newModelShowCache(),
+ modelList: newModelListCache(),
}
}
@@ -24,4 +26,7 @@ func (c *modelCaches) Start(ctx context.Context) {
if c.show != nil {
c.show.Start(ctx)
}
+ if c.modelList != nil {
+ c.modelList.Start(ctx)
+ }
}
diff --git a/server/model_list_cache.go b/server/model_list_cache.go
new file mode 100644
index 000000000..8889ed63e
--- /dev/null
+++ b/server/model_list_cache.go
@@ -0,0 +1,824 @@
+package server
+
+import (
+ "bufio"
+ "cmp"
+ "context"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "os"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/model/parsers"
+ ollamatemplate "github.com/ollama/ollama/template"
+ "github.com/ollama/ollama/thinking"
+ "github.com/ollama/ollama/types/model"
+)
+
+type modelListSummary struct {
+ Model string
+ Name string
+ RemoteModel string
+ RemoteHost string
+ Size int64
+ Digest string
+ ModifiedAt time.Time
+ Details api.ModelDetails
+ Capabilities []model.Capability
+}
+
+type modelListCacheEntry struct {
+ Digest string
+ Summary modelListSummary
+}
+
+type modelListCache struct {
+ mu sync.RWMutex
+
+ entries map[string]modelListCacheEntry
+
+ once sync.Once
+ readyOnce sync.Once
+ ready chan struct{}
+ hydrateErr error
+ build func(model.Name, *manifest.Manifest) (modelListSummary, error)
+}
+
+func newModelListCache() *modelListCache {
+ return &modelListCache{
+ entries: make(map[string]modelListCacheEntry),
+ ready: make(chan struct{}),
+ build: buildModelListSummary,
+ }
+}
+
+func (c *modelListCache) Start(ctx context.Context) {
+ if c == nil {
+ return
+ }
+
+ c.once.Do(func() {
+ slog.Debug("starting model list cache")
+ go func() {
+ err := c.hydrate(ctx)
+ c.markReady(err)
+ if err != nil {
+ if ctx != nil && ctx.Err() != nil {
+ return
+ }
+ slog.Warn("model list cache hydration failed", "error", err)
+ }
+ }()
+ })
+}
+
+func (c *modelListCache) hydrate(ctx context.Context) error {
+ start := time.Now()
+
+ manifests, err := manifest.Manifests(true)
+ if err != nil {
+ return err
+ }
+
+ var hydrated, failed int
+ for name, mf := range manifests {
+ if ctx != nil {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ }
+
+ summary, err := c.build(name, mf)
+ if err != nil {
+ failed++
+ slog.Warn("failed to hydrate model list cache", "model", name.String(), "error", err)
+ continue
+ }
+
+ c.set(name, mf.Digest(), summary)
+ hydrated++
+ }
+
+ slog.Info("model list cache hydration complete", "models", hydrated, "failures", failed, "elapsed", time.Since(start))
+ return nil
+}
+
+func (c *modelListCache) markReady(err error) {
+ c.mu.Lock()
+ c.hydrateErr = err
+ c.mu.Unlock()
+
+ c.readyOnce.Do(func() {
+ close(c.ready)
+ })
+}
+
+func (c *modelListCache) Wait(ctx context.Context) error {
+ if c == nil {
+ return nil
+ }
+ if ctx == nil {
+ ctx = context.Background()
+ }
+
+ select {
+ case <-c.ready:
+ c.mu.RLock()
+ err := c.hydrateErr
+ c.mu.RUnlock()
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func (c *modelListCache) List(ctx context.Context) ([]api.ListModelResponse, error) {
+ if err := c.Wait(ctx); err != nil {
+ return nil, err
+ }
+ if err := c.syncManifests(ctx); err != nil {
+ return nil, err
+ }
+
+ c.mu.RLock()
+ models := make([]api.ListModelResponse, 0, len(c.entries))
+ for _, entry := range c.entries {
+ models = append(models, entry.Summary.ListModelResponse())
+ }
+ c.mu.RUnlock()
+
+ sortListModelResponses(models)
+ return models, nil
+}
+
+func (c *modelListCache) syncManifests(ctx context.Context) error {
+ manifests, err := manifest.Manifests(true)
+ if err != nil {
+ return err
+ }
+
+ c.mu.RLock()
+ current := make(map[string]string, len(c.entries))
+ for name, entry := range c.entries {
+ current[name] = entry.Digest
+ }
+ c.mu.RUnlock()
+
+ type update struct {
+ name model.Name
+ digest string
+ summary modelListSummary
+ }
+
+ seen := make(map[string]struct{}, len(manifests))
+ stale := make(map[string]struct{})
+ var updates []update
+ for name, mf := range manifests {
+ if ctx != nil {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ }
+
+ key := name.String()
+ digest := mf.Digest()
+ seen[key] = struct{}{}
+ if current[key] == digest {
+ continue
+ }
+
+ summary, err := c.build(name, mf)
+ if err != nil {
+ slog.Warn("failed to refresh model list cache", "model", key, "error", err)
+ if _, ok := current[key]; ok {
+ stale[key] = struct{}{}
+ }
+ continue
+ }
+ updates = append(updates, update{name: name, digest: digest, summary: summary})
+ }
+
+ c.mu.Lock()
+ for name := range c.entries {
+ if _, ok := seen[name]; !ok {
+ delete(c.entries, name)
+ continue
+ }
+ if _, ok := stale[name]; ok {
+ delete(c.entries, name)
+ }
+ }
+ for _, update := range updates {
+ c.entries[update.name.String()] = modelListCacheEntry{
+ Digest: update.digest,
+ Summary: cloneModelListSummary(update.summary),
+ }
+ }
+ c.mu.Unlock()
+
+ return nil
+}
+
+func (c *modelListCache) RefreshModel(name model.Name) error {
+ if c == nil {
+ return nil
+ }
+
+ if !name.IsFullyQualified() {
+ var err error
+ name, err = getExistingName(name)
+ if err != nil {
+ return err
+ }
+ }
+
+ mf, err := manifest.ParseNamedManifest(name)
+ if err != nil {
+ c.DeleteModel(name)
+ return err
+ }
+
+ summary, err := c.build(name, mf)
+ if err != nil {
+ c.DeleteModel(name)
+ return err
+ }
+
+ c.set(name, mf.Digest(), summary)
+ return nil
+}
+
+func (c *modelListCache) DeleteModel(name model.Name) {
+ if c == nil {
+ return
+ }
+
+ c.mu.Lock()
+ delete(c.entries, name.String())
+ c.mu.Unlock()
+}
+
+func (c *modelListCache) Get(name model.Name) (modelListSummary, bool) {
+ if c == nil {
+ return modelListSummary{}, false
+ }
+
+ if !name.IsFullyQualified() {
+ if existing, err := getExistingName(name); err == nil {
+ name = existing
+ }
+ }
+
+ c.mu.RLock()
+ entry, ok := c.entries[name.String()]
+ c.mu.RUnlock()
+ if !ok {
+ return modelListSummary{}, false
+ }
+
+ return cloneModelListSummary(entry.Summary), true
+}
+
+func (c *modelListCache) Len() int {
+ if c == nil {
+ return 0
+ }
+
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ return len(c.entries)
+}
+
+func (c *modelListCache) set(name model.Name, digest string, summary modelListSummary) {
+ c.mu.Lock()
+ c.entries[name.String()] = modelListCacheEntry{
+ Digest: digest,
+ Summary: cloneModelListSummary(summary),
+ }
+ c.mu.Unlock()
+}
+
+func buildModelListSummary(name model.Name, mf *manifest.Manifest) (modelListSummary, error) {
+ cfg, err := readModelListConfig(mf)
+ if err != nil {
+ return modelListSummary{}, err
+ }
+
+ var modified time.Time
+ if fi := mf.FileInfo(); fi != nil {
+ modified = fi.ModTime()
+ }
+
+ summary := modelListSummary{
+ Model: name.DisplayShortest(),
+ Name: name.DisplayShortest(),
+ RemoteModel: cfg.RemoteModel,
+ RemoteHost: cfg.RemoteHost,
+ Size: mf.Size(),
+ Digest: mf.Digest(),
+ ModifiedAt: modified,
+ Details: api.ModelDetails{
+ Format: cfg.ModelFormat,
+ Family: cfg.ModelFamily,
+ Families: append([]string(nil), cfg.ModelFamilies...),
+ ParameterSize: cfg.ModelType,
+ QuantizationLevel: cfg.FileType,
+ ContextLength: cfg.ContextLen,
+ EmbeddingLength: cfg.EmbedLen,
+ },
+ }
+
+ modelPath, projectorCount, tmpl, err := readModelListLayers(mf, &summary)
+ if err != nil {
+ return modelListSummary{}, err
+ }
+
+ if cfg.RemoteHost == "" && cfg.RemoteModel == "" && modelPath != "" {
+ info, err := readModelListGGUF(modelPath)
+ if err != nil {
+ slog.Debug("failed to read gguf model metadata", "model", name.String(), "error", err)
+ } else {
+ summary.Capabilities = appendModelListCapabilities(summary.Capabilities, info.Capabilities...)
+ if summary.Details.ContextLength == 0 {
+ summary.Details.ContextLength = info.ContextLength
+ }
+ if summary.Details.EmbeddingLength == 0 {
+ summary.Details.EmbeddingLength = info.EmbeddingLength
+ }
+ }
+ }
+
+ for _, c := range cfg.Capabilities {
+ summary.Capabilities = appendModelListCapability(summary.Capabilities, model.Capability(c))
+ }
+
+ builtinParser := parsers.ParserForName(cfg.Parser)
+ if tmpl != nil {
+ vars, err := tmpl.Vars()
+ if err != nil {
+ slog.Warn("model template contains errors", "model", name.String(), "error", err)
+ }
+ if slices.Contains(vars, "tools") || (builtinParser != nil && builtinParser.HasToolSupport()) {
+ summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityTools)
+ }
+ if slices.Contains(vars, "suffix") {
+ summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityInsert)
+ }
+
+ openingTag, closingTag := thinking.InferTags(tmpl.Template)
+ hasTags := openingTag != "" && closingTag != ""
+ isGptoss := slices.Contains([]string{"gptoss", "gpt-oss"}, cfg.ModelFamily)
+ if !slices.Contains(summary.Capabilities, model.CapabilityThinking) &&
+ (hasTags || isGptoss || (builtinParser != nil && builtinParser.HasThinkingSupport())) {
+ summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityThinking)
+ }
+ }
+
+ if projectorCount > 0 {
+ summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityVision)
+ }
+
+ if cfg.ModelFormat == "safetensors" && isGemma4Renderer(cfg.Renderer) {
+ summary.Capabilities = slices.DeleteFunc(summary.Capabilities, func(c model.Capability) bool {
+ return c == model.CapabilityVision || c == model.CapabilityAudio
+ })
+ }
+
+ return summary, nil
+}
+
+func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {
+ var cfg model.ConfigV2
+ if mf == nil || mf.Config.Digest == "" {
+ return cfg, nil
+ }
+
+ f, err := mf.Config.Open()
+ if err != nil {
+ return cfg, err
+ }
+ defer f.Close()
+
+ if err := json.NewDecoder(f).Decode(&cfg); err != nil {
+ return cfg, err
+ }
+
+ return cfg, nil
+}
+
+func readModelListLayers(mf *manifest.Manifest, summary *modelListSummary) (string, int, *ollamatemplate.Template, error) {
+ var modelPath string
+ var projectorCount int
+ tmpl := ollamatemplate.DefaultTemplate
+
+ for _, layer := range mf.Layers {
+ switch layer.MediaType {
+ case "application/vnd.ollama.image.model":
+ filename, err := manifest.BlobsPath(layer.Digest)
+ if err != nil {
+ return "", 0, nil, err
+ }
+ modelPath = filename
+ summary.Details.ParentModel = layer.From
+ case "application/vnd.ollama.image.projector":
+ projectorCount++
+ case "application/vnd.ollama.image.prompt",
+ "application/vnd.ollama.image.template":
+ filename, err := manifest.BlobsPath(layer.Digest)
+ if err != nil {
+ return "", 0, nil, err
+ }
+ bts, err := os.ReadFile(filename)
+ if err != nil {
+ return "", 0, nil, err
+ }
+
+ tmpl, err = ollamatemplate.Parse(string(bts))
+ if err != nil {
+ return "", 0, nil, err
+ }
+ }
+ }
+
+ return modelPath, projectorCount, tmpl, nil
+}
+
+type modelListGGUF struct {
+ Capabilities []model.Capability
+ ContextLength int
+ EmbeddingLength int
+}
+
+const (
+ modelListGGUFMagicLE = 0x46554747
+ modelListGGUFMagicBE = 0x47475546
+)
+
+const (
+ modelListGGUFTypeUint8 uint32 = iota
+ modelListGGUFTypeInt8
+ modelListGGUFTypeUint16
+ modelListGGUFTypeInt16
+ modelListGGUFTypeUint32
+ modelListGGUFTypeInt32
+ modelListGGUFTypeFloat32
+ modelListGGUFTypeBool
+ modelListGGUFTypeString
+ modelListGGUFTypeArray
+ modelListGGUFTypeUint64
+ modelListGGUFTypeInt64
+ modelListGGUFTypeFloat64
+)
+
+// readModelListGGUF scans only the small GGUF header values launch needs
+// and stops before tokenizer arrays. Using gguf.File.KeyValue for missing keys
+// can otherwise advance through large arrays just to discover absence.
+func readModelListGGUF(path string) (modelListGGUF, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return modelListGGUF{}, err
+ }
+ defer f.Close()
+
+ r := bufio.NewReaderSize(f, 32<<10)
+ var magic uint32
+ if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
+ return modelListGGUF{}, err
+ }
+
+ var byteOrder binary.ByteOrder = binary.LittleEndian
+ switch magic {
+ case modelListGGUFMagicLE:
+ case modelListGGUFMagicBE:
+ byteOrder = binary.BigEndian
+ default:
+ return modelListGGUF{}, fmt.Errorf("invalid file magic")
+ }
+
+ var version uint32
+ if err := binary.Read(r, byteOrder, &version); err != nil {
+ return modelListGGUF{}, err
+ }
+
+ var numKV uint64
+ switch version {
+ case 1:
+ var header struct {
+ NumTensor uint32
+ NumKV uint32
+ }
+ if err := binary.Read(r, byteOrder, &header); err != nil {
+ return modelListGGUF{}, err
+ }
+ numKV = uint64(header.NumKV)
+ default:
+ var header struct {
+ NumTensor uint64
+ NumKV uint64
+ }
+ if err := binary.Read(r, byteOrder, &header); err != nil {
+ return modelListGGUF{}, err
+ }
+ numKV = header.NumKV
+ }
+
+ info := modelListGGUF{}
+ var architecture string
+ var hasPoolingType bool
+
+ for range numKV {
+ key, err := readModelListGGUFString(r, byteOrder, version)
+ if err != nil {
+ return modelListGGUF{}, err
+ }
+
+ var valueType uint32
+ if err := binary.Read(r, byteOrder, &valueType); err != nil {
+ return modelListGGUF{}, err
+ }
+
+ if key == "general.architecture" {
+ value, err := readModelListGGUFStringValue(r, byteOrder, version, valueType)
+ if err != nil {
+ return modelListGGUF{}, err
+ }
+ architecture = value
+ continue
+ }
+
+ if architecture != "" && strings.HasPrefix(key, "tokenizer.") {
+ break
+ }
+
+ if architecture != "" && strings.HasPrefix(key, architecture+".") {
+ switch strings.TrimPrefix(key, architecture+".") {
+ case "pooling_type":
+ hasPoolingType = true
+ case "vision.block_count":
+ info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityVision)
+ case "audio.block_count":
+ info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityAudio)
+ case "context_length":
+ value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
+ if err != nil {
+ return modelListGGUF{}, err
+ }
+ info.ContextLength = value
+ continue
+ case "embedding_length":
+ value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
+ if err != nil {
+ return modelListGGUF{}, err
+ }
+ info.EmbeddingLength = value
+ continue
+ }
+ }
+
+ if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
+ return modelListGGUF{}, err
+ }
+ }
+
+ if hasPoolingType {
+ info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityEmbedding)
+ } else {
+ info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityCompletion)
+ }
+
+ return info, nil
+}
+
+func readModelListGGUFStringValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (string, error) {
+ if valueType != modelListGGUFTypeString {
+ if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
+ return "", err
+ }
+ return "", fmt.Errorf("unexpected gguf string type %d", valueType)
+ }
+ return readModelListGGUFString(r, byteOrder, version)
+}
+
+func readModelListGGUFIntValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (int, error) {
+ switch valueType {
+ case modelListGGUFTypeUint8:
+ var value uint8
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeInt8:
+ var value int8
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeUint16:
+ var value uint16
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeInt16:
+ var value int16
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeUint32:
+ var value uint32
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeInt32:
+ var value int32
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeUint64:
+ var value uint64
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ case modelListGGUFTypeInt64:
+ var value int64
+ if err := binary.Read(r, byteOrder, &value); err != nil {
+ return 0, err
+ }
+ return int(value), nil
+ default:
+ if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
+ return 0, err
+ }
+ return 0, fmt.Errorf("unexpected gguf integer type %d", valueType)
+ }
+}
+
+func skipModelListGGUFValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) error {
+ switch valueType {
+ case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:
+ return discardModelListGGUFBytes(r, 1)
+ case modelListGGUFTypeUint16, modelListGGUFTypeInt16:
+ return discardModelListGGUFBytes(r, 2)
+ case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:
+ return discardModelListGGUFBytes(r, 4)
+ case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:
+ return discardModelListGGUFBytes(r, 8)
+ case modelListGGUFTypeString:
+ return skipModelListGGUFString(r, byteOrder, version)
+ case modelListGGUFTypeArray:
+ var arrayType uint32
+ if err := binary.Read(r, byteOrder, &arrayType); err != nil {
+ return err
+ }
+ var count uint64
+ if err := binary.Read(r, byteOrder, &count); err != nil {
+ return err
+ }
+ return skipModelListGGUFArray(r, byteOrder, version, arrayType, count)
+ default:
+ return fmt.Errorf("unsupported gguf value type %d", valueType)
+ }
+}
+
+func skipModelListGGUFArray(r io.Reader, byteOrder binary.ByteOrder, version uint32, arrayType uint32, count uint64) error {
+ var size uint64
+ switch arrayType {
+ case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:
+ size = 1
+ case modelListGGUFTypeUint16, modelListGGUFTypeInt16:
+ size = 2
+ case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:
+ size = 4
+ case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:
+ size = 8
+ case modelListGGUFTypeString:
+ for range count {
+ if err := skipModelListGGUFString(r, byteOrder, version); err != nil {
+ return err
+ }
+ }
+ return nil
+ default:
+ return fmt.Errorf("unsupported gguf array type %d", arrayType)
+ }
+ return discardModelListGGUFBytes(r, int64(count*size))
+}
+
+func readModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) (string, error) {
+ var length uint64
+ if err := binary.Read(r, byteOrder, &length); err != nil {
+ return "", err
+ }
+
+ if length == 0 {
+ return "", nil
+ }
+
+ bts := make([]byte, length)
+ if _, err := io.ReadFull(r, bts); err != nil {
+ return "", err
+ }
+ if version == 1 && bts[len(bts)-1] == 0 {
+ bts = bts[:len(bts)-1]
+ }
+ return string(bts), nil
+}
+
+func skipModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) error {
+ var length uint64
+ if err := binary.Read(r, byteOrder, &length); err != nil {
+ return err
+ }
+ return discardModelListGGUFBytes(r, int64(length))
+}
+
+func discardModelListGGUFBytes(r io.Reader, n int64) error {
+ if n <= 0 {
+ return nil
+ }
+ _, err := io.CopyN(io.Discard, r, n)
+ return err
+}
+
+func appendModelListCapabilities(capabilities []model.Capability, values ...model.Capability) []model.Capability {
+ for _, capability := range values {
+ capabilities = appendModelListCapability(capabilities, capability)
+ }
+ return capabilities
+}
+
+func appendModelListCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {
+ if capability == "" || slices.Contains(capabilities, capability) {
+ return capabilities
+ }
+ return append(capabilities, capability)
+}
+
+func cloneModelListSummary(summary modelListSummary) modelListSummary {
+ summary.Details.Families = append([]string(nil), summary.Details.Families...)
+ summary.Capabilities = append([]model.Capability(nil), summary.Capabilities...)
+ return summary
+}
+
+func (s modelListSummary) ListModelResponse() api.ListModelResponse {
+ resp := api.ListModelResponse{
+ Model: s.Model,
+ Name: s.Name,
+ RemoteModel: s.RemoteModel,
+ RemoteHost: s.RemoteHost,
+ Size: s.Size,
+ Digest: s.Digest,
+ ModifiedAt: s.ModifiedAt,
+ Details: api.ModelDetails{
+ ParentModel: s.Details.ParentModel,
+ Format: s.Details.Format,
+ Family: s.Details.Family,
+ Families: append([]string(nil), s.Details.Families...),
+ ParameterSize: s.Details.ParameterSize,
+ QuantizationLevel: s.Details.QuantizationLevel,
+ ContextLength: s.Details.ContextLength,
+ EmbeddingLength: s.Details.EmbeddingLength,
+ },
+ }
+
+ resp.Capabilities = append([]model.Capability(nil), s.Capabilities...)
+
+ return resp
+}
+
+func sortListModelResponses(models []api.ListModelResponse) {
+ slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
+ // Preserve the existing /api/tags order: most recently modified first.
+ return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
+ })
+}
+
+func (s *Server) refreshModelListCache(name model.Name) {
+ if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {
+ return
+ }
+
+ if err := s.modelCaches.modelList.RefreshModel(name); err != nil {
+ slog.Warn("failed to refresh model list cache", "model", name.String(), "error", err)
+ }
+}
+
+func (s *Server) deleteModelListCache(name model.Name) {
+ if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {
+ return
+ }
+
+ s.modelCaches.modelList.DeleteModel(name)
+}
diff --git a/server/model_list_cache_test.go b/server/model_list_cache_test.go
new file mode 100644
index 000000000..1f40c7b3e
--- /dev/null
+++ b/server/model_list_cache_test.go
@@ -0,0 +1,239 @@
+package server
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "slices"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/types/model"
+)
+
+func TestModelListCacheHydratesSummary(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListCacheModel(t, "list-cache", map[string]any{
+ "test.context_length": uint32(4096),
+ "test.embedding_length": uint32(384),
+ }, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")
+
+ cache := newModelListCache()
+ if err := cache.hydrate(context.Background()); err != nil {
+ t.Fatalf("hydrate failed: %v", err)
+ }
+
+ summary, ok := cache.Get(model.ParseName("list-cache"))
+ if !ok {
+ t.Fatal("list summary missing")
+ }
+
+ if summary.Model != "list-cache:latest" || summary.Name != "list-cache:latest" {
+ t.Fatalf("summary model/name = %q/%q, want list-cache:latest", summary.Model, summary.Name)
+ }
+ if summary.Digest == "" {
+ t.Fatal("summary digest is empty")
+ }
+ if summary.Size == 0 {
+ t.Fatal("summary size is zero")
+ }
+ if summary.Details.Family != "test" || summary.Details.Format != "gguf" {
+ t.Fatalf("summary details = %+v, want gguf/test", summary.Details)
+ }
+ if summary.Details.ContextLength != 4096 {
+ t.Fatalf("context length = %d, want 4096", summary.Details.ContextLength)
+ }
+ if summary.Details.EmbeddingLength != 384 {
+ t.Fatalf("embedding length = %d, want 384", summary.Details.EmbeddingLength)
+ }
+
+ for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {
+ if !slices.Contains(summary.Capabilities, capability) {
+ t.Fatalf("capabilities = %v, want %s", summary.Capabilities, capability)
+ }
+ }
+
+ listModel := summary.ListModelResponse()
+ if !slices.Contains(listModel.Capabilities, model.CapabilityTools) ||
+ listModel.Details.ContextLength != 4096 ||
+ listModel.Details.EmbeddingLength != 384 {
+ t.Fatalf("list response = %+v, want capabilities/context/embedding", listModel)
+ }
+}
+
+func TestModelListCacheRefreshUpdatesEntry(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListCacheModel(t, "list-refresh", map[string]any{"test.context_length": uint32(1024)}, "")
+
+ cache := newModelListCache()
+ if err := cache.hydrate(context.Background()); err != nil {
+ t.Fatalf("hydrate failed: %v", err)
+ }
+
+ name := model.ParseName("list-refresh")
+ first, ok := cache.Get(name)
+ if !ok {
+ t.Fatal("list summary missing")
+ }
+
+ changeShowCacheManifest(t, "list-refresh")
+ if err := cache.RefreshModel(name); err != nil {
+ t.Fatalf("refresh failed: %v", err)
+ }
+
+ refreshed, ok := cache.Get(name)
+ if !ok {
+ t.Fatal("refreshed list summary missing")
+ }
+ if refreshed.Digest == first.Digest {
+ t.Fatalf("digest did not change after refresh: %s", refreshed.Digest)
+ }
+ if cache.Len() != 1 {
+ t.Fatalf("cache entries = %d, want 1", cache.Len())
+ }
+}
+
+func TestModelListCacheMutationHooks(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+
+ cache := newModelListCache()
+ s := Server{modelCaches: &modelCaches{modelList: cache}}
+
+ _, digest := createBinFile(t, map[string]any{"test.context_length": uint32(2048)}, nil)
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: "list-hooks",
+ Files: map[string]string{"model.gguf": digest},
+ Stream: &stream,
+ })
+ if w.Code != http.StatusOK {
+ t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+
+ if _, ok := cache.Get(model.ParseName("list-hooks")); !ok {
+ t.Fatal("create did not refresh model list cache")
+ }
+
+ w = createRequest(t, s.CopyHandler, api.CopyRequest{
+ Source: "list-hooks",
+ Destination: "list-hooks-copy",
+ })
+ if w.Code != http.StatusOK {
+ t.Fatalf("copy model status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+ if _, ok := cache.Get(model.ParseName("list-hooks-copy")); !ok {
+ t.Fatal("copy did not refresh model list cache")
+ }
+
+ w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: "list-hooks-copy"})
+ if w.Code != http.StatusOK {
+ t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+ if _, ok := cache.Get(model.ParseName("list-hooks-copy")); ok {
+ t.Fatal("delete did not remove model list cache entry")
+ }
+}
+
+func TestModelListCacheSyncsManifestChanges(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListCacheModel(t, "list-sync-a", map[string]any{"test.context_length": uint32(1024)}, "")
+
+ cache := newModelListCache()
+ cache.Start(context.Background())
+ if err := cache.Wait(context.Background()); err != nil {
+ t.Fatalf("wait failed: %v", err)
+ }
+
+ createListCacheModel(t, "list-sync-b", map[string]any{"test.context_length": uint32(2048)}, "")
+ models, err := cache.List(context.Background())
+ if err != nil {
+ t.Fatalf("list failed: %v", err)
+ }
+
+ names := make([]string, 0, len(models))
+ for _, m := range models {
+ names = append(names, m.Name)
+ }
+ for _, want := range []string{"list-sync-a:latest", "list-sync-b:latest"} {
+ if !slices.Contains(names, want) {
+ t.Fatalf("names = %v, want %s", names, want)
+ }
+ }
+
+ var other Server
+ w := createRequest(t, other.DeleteHandler, api.DeleteRequest{Model: "list-sync-a"})
+ if w.Code != http.StatusOK {
+ t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+
+ models, err = cache.List(context.Background())
+ if err != nil {
+ t.Fatalf("list after delete failed: %v", err)
+ }
+ names = names[:0]
+ for _, m := range models {
+ names = append(names, m.Name)
+ }
+ if slices.Contains(names, "list-sync-a:latest") || !slices.Contains(names, "list-sync-b:latest") {
+ t.Fatalf("names after delete = %v, want only list-sync-b", names)
+ }
+}
+
+func TestModelListCacheSyncDropsStaleEntryOnRefreshFailure(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListCacheModel(t, "list-stale", map[string]any{"test.context_length": uint32(1024)}, "")
+
+ cache := newModelListCache()
+ cache.Start(context.Background())
+ if err := cache.Wait(context.Background()); err != nil {
+ t.Fatalf("wait failed: %v", err)
+ }
+
+ name := model.ParseName("list-stale")
+ if _, ok := cache.Get(name); !ok {
+ t.Fatal("list summary missing")
+ }
+
+ changeShowCacheManifest(t, "list-stale")
+ cache.build = func(model.Name, *manifest.Manifest) (modelListSummary, error) {
+ return modelListSummary{}, errors.New("refresh failed")
+ }
+
+ models, err := cache.List(context.Background())
+ if err != nil {
+ t.Fatalf("list failed: %v", err)
+ }
+ if len(models) != 0 {
+ t.Fatalf("models = %+v, want stale entry removed", models)
+ }
+ if _, ok := cache.Get(name); ok {
+ t.Fatal("stale entry remained in cache after refresh failure")
+ }
+}
+
+func createListCacheModel(t *testing.T, name string, kv map[string]any, tmpl string) {
+ t.Helper()
+ _, digest := createBinFile(t, kv, nil)
+
+ req := api.CreateRequest{
+ Model: name,
+ Files: map[string]string{"model.gguf": digest},
+ Stream: &stream,
+ }
+ if tmpl != "" {
+ req.Template = tmpl
+ }
+
+ var s Server
+ w := createRequest(t, s.CreateHandler, req)
+ if w.Code != http.StatusOK {
+ t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+}
diff --git a/server/model_show_cache.go b/server/model_show_cache.go
index b32e025e8..bab5f241f 100644
--- a/server/model_show_cache.go
+++ b/server/model_show_cache.go
@@ -26,13 +26,15 @@ import (
The /api/show cache stores full api.ShowResponse values because callers use
more than capabilities: launch flows also need context length, embeddings
metadata, quantization details, remote metadata, and model-specific fields.
+TODO(parthsareen): Consider removing show cache if /api/tags grows to cover
+the remaining callers.
-Local model entries are stored by canonical model name and verbose flag, with
-the manifest digest recorded in the entry. The manifest digest is the freshness
-boundary: if the model content changes, the digest changes, so the previous
-response is replaced instead of accumulating under an old digest key. Requests
-with System or Options overlays bypass the cache because those overlays mutate
-the effective show response.
+Local model entries are stored lazily by canonical model name and verbose flag,
+with the manifest digest recorded in the entry. The manifest digest is the
+freshness boundary: if the model content changes, the digest changes, so the
+previous response is replaced instead of accumulating under an old digest key.
+Requests with System or Options overlays bypass the cache because those overlays
+mutate the effective show response.
Cloud model entries are keyed by normalized cloud base model name and verbose.
They use stale-while-revalidate behavior: a warm read returns the cached
@@ -42,10 +44,11 @@ in separate maps, so a local "qwen3.5" and an explicit "qwen3.5:cloud" cannot
collide. The cloud suffix is request routing intent; api.ShowResponse does not
carry a model-name field to reconstruct on the way out.
-The cache is process-local. Startup hydration runs asynchronously from the
-current local manifests and cloud tags; no show responses are written to or read
-from ~/.ollama/cache/show. That keeps cache lifetime tied to the server process
-and avoids snapshot freshness and invalidation cases for this iteration.
+The cache is process-local. Cloud startup hydration runs asynchronously from
+cloud tags, while local show responses are populated on demand. No show
+responses are written to or read from ~/.ollama/cache/show. That keeps cache
+lifetime tied to the server process and avoids snapshot freshness and
+invalidation cases for this iteration.
*/
const (
@@ -117,9 +120,9 @@ func modelShowCacheable(req api.ShowRequest) bool {
return req.System == "" && len(req.Options) == 0
}
-// Start kicks off non-blocking startup hydration. The cache remains
-// process-local; warm entries appear as the background local and cloud scans
-// populate the maps.
+// Start kicks off non-blocking startup hydration for cloud entries. Local show
+// responses stay lazy because even non-verbose show must load GGUF metadata that
+// is expensive for large model stores.
func (c *modelShowCache) Start(ctx context.Context) {
c.once.Do(func() {
slog.Debug("starting model show cache")
@@ -127,34 +130,18 @@ func (c *modelShowCache) Start(ctx context.Context) {
})
}
-// runStartup hydrates local and cloud caches concurrently. It is only called in
-// a goroutine from Start, so manifest scans and cloud requests cannot delay the
-// listener from accepting traffic.
+// runStartup hydrates the cloud cache. It is only called in a goroutine from
+// Start, so cloud requests cannot delay the listener from accepting traffic.
func (c *modelShowCache) runStartup(ctx context.Context) {
- var wg sync.WaitGroup
- wg.Add(2)
-
- go func() {
- defer wg.Done()
- if err := c.hydrateLocal(ctx); err != nil && !errors.Is(err, context.Canceled) {
- slog.Warn("model show local cache hydration failed", "error", err)
+ if err := c.hydrateCloud(ctx); err != nil {
+ switch {
+ case errors.Is(err, context.Canceled):
+ case errors.Is(err, errModelShowNoCloud):
+ slog.Debug("skipping model show cloud cache hydration because cloud is disabled")
+ default:
+ slog.Warn("model show cloud cache hydration failed", "error", err)
}
- }()
-
- go func() {
- defer wg.Done()
- if err := c.hydrateCloud(ctx); err != nil {
- switch {
- case errors.Is(err, context.Canceled):
- case errors.Is(err, errModelShowNoCloud):
- slog.Debug("skipping model show cloud cache hydration because cloud is disabled")
- default:
- slog.Warn("model show cloud cache hydration failed", "error", err)
- }
- }
- }()
-
- wg.Wait()
+ }
}
// GetLocal returns a cached local show response when the current manifest
diff --git a/server/model_show_cache_test.go b/server/model_show_cache_test.go
index 8b8175c66..04bab53ba 100644
--- a/server/model_show_cache_test.go
+++ b/server/model_show_cache_test.go
@@ -148,6 +148,25 @@ func TestModelShowCacheLocalHydrationSkipsUnchangedInMemory(t *testing.T) {
}
}
+func TestModelShowCacheStartupSkipsLocalHydration(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ t.Setenv("OLLAMA_NO_CLOUD", "1")
+ createShowCacheModel(t, "show-cache-startup", map[string]any{"test.context_length": uint32(1024)})
+
+ cache := newModelShowCache()
+ cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) {
+ t.Fatalf("startup should not hydrate local show cache, got request: %+v", req)
+ return nil, nil
+ }
+
+ cache.runStartup(context.Background())
+
+ if len(cache.local) != 0 {
+ t.Fatalf("local cache entries = %d, want 0", len(cache.local))
+ }
+}
+
func TestModelShowCacheBypassesSystemAndOptionsOverlays(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
diff --git a/server/routes.go b/server/routes.go
index 042ac4337..1acd61619 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -969,7 +969,10 @@ func (s *Server) PullHandler(c *gin.Context) {
if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
ch <- gin.H{"error": err.Error()}
+ return
}
+
+ s.refreshModelListCache(name)
}()
if req.Stream != nil && !*req.Stream {
@@ -1108,6 +1111,8 @@ func (s *Server) DeleteHandler(c *gin.Context) {
return
}
+ s.deleteModelListCache(n)
+
if err := m.RemoveLayers(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@@ -1431,54 +1436,17 @@ func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) {
}
func (s *Server) ListHandler(c *gin.Context) {
- ms, err := manifest.Manifests(true)
- if err != nil {
- c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ if s.modelCaches == nil || s.modelCaches.modelList == nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "model list cache unavailable"})
return
}
- models := []api.ListModelResponse{}
- for n, m := range ms {
- var cf model.ConfigV2
-
- if m.Config.Digest != "" {
- f, err := m.Config.Open()
- if err != nil {
- slog.Warn("bad manifest filepath", "name", n, "error", err)
- continue
- }
- defer f.Close()
-
- if err := json.NewDecoder(f).Decode(&cf); err != nil {
- slog.Warn("bad manifest config", "name", n, "error", err)
- continue
- }
- }
-
- // tag should never be masked
- models = append(models, api.ListModelResponse{
- Model: n.DisplayShortest(),
- Name: n.DisplayShortest(),
- RemoteModel: cf.RemoteModel,
- RemoteHost: cf.RemoteHost,
- Size: m.Size(),
- Digest: m.Digest(),
- ModifiedAt: m.FileInfo().ModTime(),
- Details: api.ModelDetails{
- Format: cf.ModelFormat,
- Family: cf.ModelFamily,
- Families: cf.ModelFamilies,
- ParameterSize: cf.ModelType,
- QuantizationLevel: cf.FileType,
- },
- })
+ models, err := s.modelCaches.modelList.List(c.Request.Context())
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
}
- slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
- // most recently modified first
- return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
- })
-
c.JSON(http.StatusOK, api.ListResponse{Models: models})
}
@@ -1518,6 +1486,8 @@ func (s *Server) CopyHandler(c *gin.Context) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
} else if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ } else {
+ s.refreshModelListCache(dst)
}
}
diff --git a/server/routes_list_test.go b/server/routes_list_test.go
index f6e899ad7..e18f27e61 100644
--- a/server/routes_list_test.go
+++ b/server/routes_list_test.go
@@ -1,6 +1,7 @@
package server
import (
+ "context"
"encoding/json"
"net/http"
"slices"
@@ -28,7 +29,12 @@ func TestList(t *testing.T) {
"myhost/mynamespace/lips:code",
}
- var s Server
+ s := Server{modelCaches: &modelCaches{modelList: newModelListCache()}}
+ s.modelCaches.modelList.Start(context.Background())
+ if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
for _, n := range expectNames {
_, digest := createBinFile(t, nil, nil)
@@ -63,4 +69,10 @@ func TestList(t *testing.T) {
if !slices.Equal(actualNames, expectNames) {
t.Fatalf("expected slices to be equal %v", actualNames)
}
+
+ for _, m := range resp.Models {
+ if !slices.Contains(m.Capabilities, "completion") {
+ t.Fatalf("capabilities for %q = %v, want completion", m.Name, m.Capabilities)
+ }
+ }
}
diff --git a/server/routes_test.go b/server/routes_test.go
index 8ffe8a510..036e61b3d 100644
--- a/server/routes_test.go
+++ b/server/routes_test.go
@@ -93,6 +93,9 @@ func (t *panicTransport) RoundTrip(r *http.Request) (*http.Response, error) {
var panicOnRoundTrip = &http.Client{Transport: &panicTransport{}}
func TestRoutes(t *testing.T) {
+ modelsDir := t.TempDir()
+ t.Setenv("OLLAMA_MODELS", modelsDir)
+
type testCase struct {
Name string
Method string
@@ -101,6 +104,12 @@ func TestRoutes(t *testing.T) {
Expected func(t *testing.T, resp *http.Response)
}
+ s := &Server{modelCaches: &modelCaches{modelList: newModelListCache()}}
+ s.modelCaches.modelList.Start(context.Background())
+ if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
createTestModel := func(t *testing.T, name string) {
t.Helper()
@@ -138,6 +147,7 @@ func TestRoutes(t *testing.T) {
if err := createModel(r, modelName, baseLayers, config, fn); err != nil {
t.Fatal(err)
}
+ s.refreshModelListCache(modelName)
}
testCases := []testCase{
@@ -496,9 +506,6 @@ func TestRoutes(t *testing.T) {
},
}
- modelsDir := t.TempDir()
- t.Setenv("OLLAMA_MODELS", modelsDir)
-
rc := &ollama.Registry{
// This is a temporary measure to allow us to move forward,
// surfacing any code contacting ollama.com we do not intended
@@ -514,7 +521,6 @@ func TestRoutes(t *testing.T) {
HTTPClient: panicOnRoundTrip,
}
- s := &Server{}
router, err := s.GenerateRoutes(rc)
if err != nil {
t.Fatalf("failed to generate routes: %v", err)
diff --git a/server/sched_test.go b/server/sched_test.go
index f40dc117f..450372ec6 100644
--- a/server/sched_test.go
+++ b/server/sched_test.go
@@ -536,7 +536,7 @@ func TestSchedGetRunnerReusesSameDigestWhenModelPathEmpty(t *testing.T) {
}
func TestSchedExpireRunner(t *testing.T) {
- ctx, done := context.WithTimeout(t.Context(), 20*time.Millisecond)
+ ctx, done := context.WithCancel(t.Context())
defer done()
s := InitScheduler(ctx)
s.waitForRecovery = 10 * time.Millisecond
@@ -556,8 +556,11 @@ func TestSchedExpireRunner(t *testing.T) {
{Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},
})
+ reqCtx, cancelReq := context.WithCancel(ctx)
+ defer cancelReq()
+
req := &LlmRequest{
- ctx: ctx,
+ ctx: reqCtx,
model: &Model{ModelPath: modelPath},
opts: api.DefaultOptions(),
successCh: make(chan *runnerRef, 1),
@@ -587,16 +590,33 @@ func TestSchedExpireRunner(t *testing.T) {
s.loadedMu.Unlock()
}
+ completedDone := make(chan struct{})
+ go func() {
+ defer close(completedDone)
+ s.processCompleted(ctx)
+ }()
+
s.expireRunner(&Model{ModelPath: modelPath})
+ cancelReq()
- s.finishedReqCh <- req
- s.processCompleted(ctx)
+ select {
+ case <-s.unloadedCh:
+ case <-time.After(5 * time.Second):
+ t.Fatal("expected model to be unloaded")
+ }
s.loadedMu.Lock()
if len(s.loaded) != 0 {
t.Fatalf("expected model to be unloaded")
}
s.loadedMu.Unlock()
+
+ done()
+ select {
+ case <-completedDone:
+ case <-time.After(5 * time.Second):
+ t.Fatal("expected completed loop to stop")
+ }
}
// TODO - add one scenario that triggers the bogus finished event with positive ref count