Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
server: cache show responses (#15967)
Parth Sareend319227df012
diff --git a/server/model_caches.go b/server/model_caches.gonew file mode 100644index 000000000..720fa703d--- /dev/null+++ b/server/model_caches.go@@ -0,0 +1,27 @@+package server++import "context"++type modelCaches struct {+ recommendations *modelRecommendationsCache+ show *modelShowCache+}++func newModelCaches() *modelCaches {+ return &modelCaches{+ recommendations: newModelRecommendationsCache(),+ show: newModelShowCache(),+ }+}++func (c *modelCaches) Start(ctx context.Context) {+ if c == nil {+ return+ }+ if c.recommendations != nil {+ c.recommendations.Start(ctx)+ }+ if c.show != nil {+ c.show.Start(ctx)+ }+}diff --git a/server/model_recommendations_test.go b/server/model_recommendations_test.goindex 346462fc4..1c7b389ef 100644--- a/server/model_recommendations_test.go+++ b/server/model_recommendations_test.go@@ -306,7 +306,7 @@ func TestModelRecommendationsHandlerUsesCache(t *testing.T) {ctx, _ := gin.CreateTestContext(w)ctx.Request = httptest.NewRequest(http.MethodGet, "/api/experimental/model-recommendations", nil)- s := &Server{modelRecommendations: cache}+ s := &Server{modelCaches: &modelCaches{recommendations: cache}}s.ModelRecommendationsExperimentalHandler(ctx)if w.Code != http.StatusOK {@@ -326,7 +326,7 @@ func TestModelRecommendationsRouteRegistration(t *testing.T) {cache := newModelRecommendationsCache()cache.set([]api.ModelRecommendation{{Model: "route-model", Description: "route description"}})- s := &Server{modelRecommendations: cache}+ s := &Server{modelCaches: &modelCaches{recommendations: cache}}router, err := s.GenerateRoutes(nil)if err != nil {diff --git a/server/model_show_cache.go b/server/model_show_cache.gonew file mode 100644index 000000000..b32e025e8--- /dev/null+++ b/server/model_show_cache.go@@ -0,0 +1,705 @@+package server++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "io"+ "log/slog"+ "net/http"+ "net/url"+ "slices"+ "strings"+ "sync"+ "time"++ "github.com/ollama/ollama/api"+ internalcloud "github.com/ollama/ollama/internal/cloud"+ "github.com/ollama/ollama/internal/modelref"+ "github.com/ollama/ollama/manifest"+ "github.com/ollama/ollama/types/model"+ "github.com/ollama/ollama/version"+)++/*+The /api/show cache stores full api.ShowResponse values because callers use+more than capabilities: launch flows also need context length, embeddings+metadata, quantization details, remote metadata, and model-specific fields.++Local model entries are stored by canonical model name and verbose flag, with+the manifest digest recorded in the entry. The manifest digest is the freshness+boundary: if the model content changes, the digest changes, so the previous+response is replaced instead of accumulating under an old digest key. Requests+with System or Options overlays bypass the cache because those overlays mutate+the effective show response.++Cloud model entries are keyed by normalized cloud base model name and verbose.+They use stale-while-revalidate behavior: a warm read returns the cached+response immediately and starts a throttled background refresh for that model.+Cold cloud reads preserve existing proxy behavior. Local and cloud entries live+in separate maps, so a local "qwen3.5" and an explicit "qwen3.5:cloud" cannot+collide. The cloud suffix is request routing intent; api.ShowResponse does not+carry a model-name field to reconstruct on the way out.++The cache is process-local. Startup hydration runs asynchronously from the+current local manifests and cloud tags; no show responses are written to or read+from ~/.ollama/cache/show. That keeps cache lifetime tied to the server process+and avoids snapshot freshness and invalidation cases for this iteration.+*/++const (+ modelShowCloudFetchTimeout = 3 * time.Second+ modelShowCloudReadRefreshCooldown = 5 * time.Second+ modelShowCloudHydrationConcurrency = 4+)++var errModelShowNoCloud = errors.New("cloud disabled")++// modelShowCache owns process-local show response caches for local and cloud+// models. All cached responses are cloned at read/write boundaries so+// handler-specific mutations, such as user-agent compatibility tweaks, cannot+// leak back into the cache.+type modelShowCache struct {+ mu sync.RWMutex++ local map[modelShowLocalKey]modelShowLocalEntry+ cloud map[modelShowCloudKey]*api.ShowResponse++ cloudRefreshing map[modelShowCloudKey]bool+ cloudNextReadRefreshAfter map[modelShowCloudKey]time.Time++ once sync.Once+ client *http.Client+ getModelInfo func(api.ShowRequest) (*api.ShowResponse, error)+}++// modelShowLocalKey describes the local cache slot for a model response. The+// manifest digest is stored in the entry instead of the key so a pulled or+// recreated model overwrites the previous response for the same model/verbose+// variant instead of leaving stale digest-keyed entries behind.+//+// Deleted models are not eagerly pruned from this process-local cache. Manifest+// resolution happens before local cache lookup, so stale delete entries are not+// served and disappear on process restart.+type modelShowLocalKey struct {+ Model string+ Verbose bool+}++type modelShowLocalEntry struct {+ Digest string+ Response *api.ShowResponse+}++// modelShowCloudKey intentionally excludes any local digest because cloud+// models are refreshed through SWR and normalized by cloud base model name.+type modelShowCloudKey struct {+ Model string+ Verbose bool+}++func newModelShowCache() *modelShowCache {+ return &modelShowCache{+ local: make(map[modelShowLocalKey]modelShowLocalEntry),+ cloud: make(map[modelShowCloudKey]*api.ShowResponse),+ cloudRefreshing: make(map[modelShowCloudKey]bool),+ cloudNextReadRefreshAfter: make(map[modelShowCloudKey]time.Time),+ client: http.DefaultClient,+ getModelInfo: GetModelInfo,+ }+}++// modelShowCacheable returns whether a request can use the shared show cache.+// System and Options overlays are request-specific response variants, so v1+// bypasses caching for those rather than expanding the key space.+func modelShowCacheable(req api.ShowRequest) bool {+ return req.System == "" && len(req.Options) == 0+}++// Start kicks off non-blocking startup hydration. The cache remains+// process-local; warm entries appear as the background local and cloud scans+// populate the maps.+func (c *modelShowCache) Start(ctx context.Context) {+ c.once.Do(func() {+ slog.Debug("starting model show cache")+ go c.runStartup(ctx)+ })+}++// runStartup hydrates local and cloud caches concurrently. It is only called in+// a goroutine from Start, so manifest scans and cloud requests cannot delay the+// listener from accepting traffic.+func (c *modelShowCache) runStartup(ctx context.Context) {+ var wg sync.WaitGroup+ wg.Add(2)++ go func() {+ defer wg.Done()+ if err := c.hydrateLocal(ctx); err != nil && !errors.Is(err, context.Canceled) {+ slog.Warn("model show local cache hydration failed", "error", err)+ }+ }()++ go func() {+ defer wg.Done()+ if err := c.hydrateCloud(ctx); err != nil {+ switch {+ case errors.Is(err, context.Canceled):+ case errors.Is(err, errModelShowNoCloud):+ slog.Debug("skipping model show cloud cache hydration because cloud is disabled")+ default:+ slog.Warn("model show cloud cache hydration failed", "error", err)+ }+ }+ }()++ wg.Wait()+}++// GetLocal returns a cached local show response when the current manifest+// digest matches. On a miss, it falls back to GetModelInfo, stores non-remote+// local responses, and returns a clone to the caller.+func (c *modelShowCache) GetLocal(req api.ShowRequest) (*api.ShowResponse, error) {+ key, digest, err := modelShowLocalKeyForRequest(req)+ if err != nil {+ return nil, err+ }++ if resp, ok := c.getLocal(key, digest); ok {+ return resp, nil+ }++ req.Model = key.Model+ resp, err := c.getModelInfo(req)+ if err != nil {+ return nil, err+ }++ if resp.RemoteHost == "" {+ c.setLocal(key, digest, resp)+ }++ return cloneShowResponse(resp), nil+}++// GetCloudSWR returns a cached cloud show response and triggers a throttled+// background refresh. The boolean is false on a cold miss so callers can+// preserve existing synchronous proxy behavior.+func (c *modelShowCache) GetCloudSWR(ctx context.Context, req api.ShowRequest) (*api.ShowResponse, bool) {+ key := modelShowCloudKeyForModel(req.Model, req.Verbose)+ resp, ok := c.getCloud(key)+ if !ok {+ return nil, false+ }++ c.triggerCloudRefreshOnRead(ctx, key)+ return resp, true+}++func (c *modelShowCache) getLocal(key modelShowLocalKey, digest string) (*api.ShowResponse, bool) {+ c.mu.RLock()+ entry, ok := c.local[key]+ c.mu.RUnlock()+ if !ok || entry.Digest != digest || entry.Response == nil {+ return nil, false+ }+ return cloneShowResponse(entry.Response), true+}++func (c *modelShowCache) setLocal(key modelShowLocalKey, digest string, resp *api.ShowResponse) {+ c.mu.Lock()+ c.local[key] = modelShowLocalEntry{+ Digest: digest,+ Response: cloneShowResponse(resp),+ }+ c.mu.Unlock()+}++func (c *modelShowCache) hasLocal(key modelShowLocalKey, digest string) bool {+ c.mu.RLock()+ entry, ok := c.local[key]+ c.mu.RUnlock()+ return ok && entry.Digest == digest && entry.Response != nil+}++func (c *modelShowCache) getCloud(key modelShowCloudKey) (*api.ShowResponse, bool) {+ c.mu.RLock()+ resp, ok := c.cloud[key]+ c.mu.RUnlock()+ if !ok || resp == nil {+ return nil, false+ }+ return cloneShowResponse(resp), true+}++func (c *modelShowCache) setCloud(key modelShowCloudKey, resp *api.ShowResponse) {+ c.mu.Lock()+ c.cloud[key] = cloneShowResponse(resp)+ c.mu.Unlock()+}++func (c *modelShowCache) beginCloudReadRefresh(key modelShowCloudKey) bool {+ c.mu.Lock()+ defer c.mu.Unlock()++ now := time.Now()+ if c.cloudRefreshing[key] || now.Before(c.cloudNextReadRefreshAfter[key]) {+ return false+ }++ c.cloudRefreshing[key] = true+ return true+}++func (c *modelShowCache) endCloudReadRefresh(key modelShowCloudKey) {+ c.mu.Lock()+ c.cloudRefreshing[key] = false+ c.cloudNextReadRefreshAfter[key] = time.Now().Add(modelShowCloudReadRefreshCooldown)+ c.mu.Unlock()+}++// triggerCloudRefreshOnRead starts the revalidation side of SWR. The refresh+// uses context.WithoutCancel so a completed client request does not cancel the+// cache update it initiated.+func (c *modelShowCache) triggerCloudRefreshOnRead(ctx context.Context, key modelShowCloudKey) {+ if !c.beginCloudReadRefresh(key) {+ return+ }+ if ctx == nil {+ ctx = context.Background()+ }+ ctx = context.WithoutCancel(ctx)++ slog.Debug("triggering model show cloud refresh on read", "model", key.Model, "verbose", key.Verbose)+ go func() {+ defer c.endCloudReadRefresh(key)++ if err := c.refreshCloud(ctx, key); err != nil {+ switch {+ case errors.Is(err, errModelShowNoCloud):+ slog.Debug("skipping model show cloud read refresh because cloud is disabled", "model", key.Model)+ default:+ slog.Warn("model show cloud read refresh failed", "model", key.Model, "error", err)+ }+ }+ }()+}++// refreshCloud fetches and stores one cloud show response. Refresh failures are+// returned without touching the existing cached entry, which preserves stale+// data for future reads.+func (c *modelShowCache) refreshCloud(ctx context.Context, key modelShowCloudKey) error {+ if disabled, _ := internalcloud.Status(); disabled {+ return errModelShowNoCloud+ }++ resp, err := c.fetchCloudShow(ctx, key.Model, key.Verbose)+ if err != nil {+ return err+ }++ c.setCloud(key, resp)+ return nil+}++// hydrateLocal scans manifests at startup and refreshes only entries missing+// for the current digest. It hydrates non-verbose responses only, avoiding an+// expensive tensor walk for users who have never asked for verbose show data.+func (c *modelShowCache) hydrateLocal(ctx context.Context) error {+ manifests, err := manifest.Manifests(true)+ if err != nil {+ return err+ }++ for name, mf := range manifests {+ if err := ctx.Err(); err != nil {+ return err+ }++ if modelShowManifestIsRemote(mf) {+ continue+ }++ modelName := name.String()+ digest := mf.Digest()+ key := modelShowLocalKey{+ Model: modelName,+ Verbose: false,+ }+ if c.hasLocal(key, digest) {+ continue+ }++ resp, err := c.getModelInfo(api.ShowRequest{Model: modelName})+ if err != nil {+ slog.Warn("failed to hydrate local model show cache", "model", modelName, "error", err)+ continue+ }+ if resp.RemoteHost != "" {+ continue+ }++ c.setLocal(key, digest, resp)+ }+ return nil+}++// hydrateCloud refreshes cloud show entries by listing cloud tags and fetching+// /api/show for each returned model with bounded concurrency. Per-model show+// failures are logged and skipped so one bad cloud entry does not prevent the+// rest of the cache from warming.+func (c *modelShowCache) hydrateCloud(ctx context.Context) error {+ if disabled, _ := internalcloud.Status(); disabled {+ return errModelShowNoCloud+ }++ models, err := c.fetchCloudTags(ctx)+ if err != nil {+ return err+ }++ jobs := make(chan string)+ var wg sync.WaitGroup++ worker := func() {+ defer wg.Done()+ for modelName := range jobs {+ if ctx.Err() != nil {+ continue+ }++ key := modelShowCloudKeyForModel(modelName, false)+ resp, err := c.fetchCloudShow(ctx, key.Model, key.Verbose)+ if err != nil {+ slog.Warn("failed to hydrate cloud model show cache", "model", key.Model, "error", err)+ continue+ }++ c.setCloud(key, resp)+ }+ }++ workers := min(modelShowCloudHydrationConcurrency, max(1, len(models)))+ for range workers {+ wg.Add(1)+ go worker()+ }++sendLoop:+ for _, modelName := range models {+ select {+ case <-ctx.Done():+ break sendLoop+ case jobs <- modelName:+ }+ }+ close(jobs)+ wg.Wait()++ if err := ctx.Err(); err != nil {+ return err+ }++ return nil+}++// fetchCloudTags returns de-duplicated cloud model names normalized to their+// show-cache key form. It accepts either ListModelResponse.Model or the legacy+// Name field because /api/tags responses may contain both.+func (c *modelShowCache) fetchCloudTags(ctx context.Context) ([]string, error) {+ var payload api.ListResponse+ if err := c.doCloudJSON(ctx, http.MethodGet, "/api/tags", nil, &payload); err != nil {+ return nil, err+ }++ seen := make(map[string]struct{}, len(payload.Models))+ models := make([]string, 0, len(payload.Models))+ for _, item := range payload.Models {+ name := strings.TrimSpace(item.Model)+ if name == "" {+ name = strings.TrimSpace(item.Name)+ }+ name = modelShowNormalizeCloudModel(name)+ if name == "" {+ continue+ }+ if _, ok := seen[name]; ok {+ continue+ }+ seen[name] = struct{}{}+ models = append(models, name)+ }++ return models, nil+}++func (c *modelShowCache) fetchCloudShow(ctx context.Context, modelName string, verbose bool) (*api.ShowResponse, error) {+ payload := api.ShowRequest{+ Model: modelShowNormalizeCloudModel(modelName),+ Verbose: verbose,+ }++ var resp api.ShowResponse+ if err := c.doCloudJSON(ctx, http.MethodPost, "/api/show", payload, &resp); err != nil {+ return nil, err+ }++ if resp.ModelInfo == nil {+ resp.ModelInfo = map[string]any{}+ }+ return &resp, nil+}++// doCloudJSON is the cache's direct cloud client. It mirrors the cloud proxy's+// signing and client-version behavior but uses an internal timeout because+// hydration and refreshes must not hang indefinitely.+func (c *modelShowCache) doCloudJSON(ctx context.Context, method, path string, payload any, out any) error {+ reqCtx, cancel := context.WithTimeout(ctx, modelShowCloudFetchTimeout)+ defer cancel()++ baseURL, err := url.Parse(cloudProxyBaseURL)+ if err != nil {+ return err+ }+ targetURL := baseURL.ResolveReference(&url.URL{Path: path})++ var body io.Reader+ if payload != nil {+ data, err := json.Marshal(payload)+ if err != nil {+ return err+ }+ body = bytes.NewReader(data)+ }++ req, err := http.NewRequestWithContext(reqCtx, method, targetURL.String(), body)+ if err != nil {+ return err+ }+ req.Header.Set("Accept", "application/json")+ if payload != nil {+ req.Header.Set("Content-Type", "application/json")+ }+ if clientVersion := strings.TrimSpace(version.Version); clientVersion != "" {+ req.Header.Set(cloudProxyClientVersionHeader, clientVersion)+ }++ if err := cloudProxySignRequest(req.Context(), req); err != nil {+ return err+ }++ resp, err := c.client.Do(req)+ if err != nil {+ return err+ }+ defer resp.Body.Close()++ data, err := io.ReadAll(resp.Body)+ if err != nil {+ return err+ }+ if resp.StatusCode >= http.StatusBadRequest {+ return modelShowStatusError(resp, data)+ }++ if out == nil {+ return nil+ }+ return json.Unmarshal(data, out)+}++// modelShowStatusError preserves the important error shape from cloud+// responses, including AuthorizationError for 401s and StatusError otherwise.+func modelShowStatusError(resp *http.Response, body []byte) error {+ if resp.StatusCode == http.StatusUnauthorized {+ err := api.AuthorizationError{+ StatusCode: resp.StatusCode,+ Status: resp.Status,+ }+ _ = json.Unmarshal(body, &err)+ if err.Status == "" {+ err.Status = resp.Status+ }+ return err+ }++ statusErr := api.StatusError{+ StatusCode: resp.StatusCode,+ Status: resp.Status,+ }+ if err := json.Unmarshal(body, &statusErr); err != nil || statusErr.ErrorMessage == "" {+ statusErr.ErrorMessage = strings.TrimSpace(string(body))+ }+ return statusErr+}++// modelShowLocalKeyForRequest normalizes a local show request to the canonical+// on-disk model name and returns the current manifest digest used to validate+// the cached entry.+func modelShowLocalKeyForRequest(req api.ShowRequest) (modelShowLocalKey, string, error) {+ name := model.ParseName(req.Model)+ if !name.IsValid() {+ return modelShowLocalKey{}, "", model.Unqualified(name)+ }+ name, err := getExistingName(name)+ if err != nil {+ return modelShowLocalKey{}, "", err+ }++ mf, err := manifest.ParseNamedManifest(name)+ if err != nil {+ return modelShowLocalKey{}, "", err+ }++ return modelShowLocalKey{+ Model: name.String(),+ Verbose: req.Verbose,+ }, mf.Digest(), nil+}++func modelShowCloudKeyForModel(modelName string, verbose bool) modelShowCloudKey {+ return modelShowCloudKey{+ Model: modelShowNormalizeCloudModel(modelName),+ Verbose: verbose,+ }+}++// modelShowNormalizeCloudModel strips explicit cloud source syntax, including+// legacy "-cloud" tags, so :cloud and -cloud forms share a cache entry.+func modelShowNormalizeCloudModel(modelName string) string {+ modelName = strings.TrimSpace(modelName)+ if modelName == "" {+ return ""+ }+ if base, stripped := modelref.StripCloudSourceTag(modelName); stripped {+ return strings.TrimSpace(base)+ }+ return modelName+}++// modelShowManifestIsRemote checks whether a manifest represents a local stub+// for a remote model. Startup hydration skips these so the local content cache+// does not store entries whose freshness is governed by cloud state.+func modelShowManifestIsRemote(mf *manifest.Manifest) bool {+ if mf == nil || mf.Config.Digest == "" {+ return false+ }++ f, err := mf.Config.Open()+ if err != nil {+ slog.Warn("failed to open manifest config while checking model show cache eligibility", "error", err)+ return false+ }+ defer f.Close()++ var cfg model.ConfigV2+ if err := json.NewDecoder(f).Decode(&cfg); err != nil {+ slog.Warn("failed to decode manifest config while checking model show cache eligibility", "error", err)+ return false+ }++ return cfg.RemoteHost != "" || cfg.RemoteModel != ""+}++// cloneShowResponse deep-copies mutable fields of api.ShowResponse before+// storing or returning cached entries. The response contains maps and slices,+// and some handlers mutate ModelInfo before writing JSON.+func cloneShowResponse(in *api.ShowResponse) *api.ShowResponse {+ if in == nil {+ return nil+ }++ out := *in+ out.Details.Families = slices.Clone(in.Details.Families)+ out.Messages = cloneMessages(in.Messages)+ out.Capabilities = slices.Clone(in.Capabilities)+ out.ModelInfo = cloneAnyMap(in.ModelInfo)+ out.ProjectorInfo = cloneAnyMap(in.ProjectorInfo)+ out.Tensors = cloneTensors(in.Tensors)+ return &out+}++func cloneMessages(in []api.Message) []api.Message {+ if in == nil {+ return nil+ }+ out := make([]api.Message, len(in))+ for i, msg := range in {+ out[i] = msg+ if msg.Images != nil {+ out[i].Images = make([]api.ImageData, len(msg.Images))+ for j, image := range msg.Images {+ out[i].Images[j] = slices.Clone(image)+ }+ }+ out[i].ToolCalls = slices.Clone(msg.ToolCalls)+ }+ return out+}++func cloneTensors(in []api.Tensor) []api.Tensor {+ if in == nil {+ return nil+ }+ out := make([]api.Tensor, len(in))+ for i, tensor := range in {+ out[i] = tensor+ out[i].Shape = slices.Clone(tensor.Shape)+ }+ return out+}++func cloneAnyMap(in map[string]any) map[string]any {+ if in == nil {+ return nil+ }+ out := make(map[string]any, len(in))+ for k, v := range in {+ out[k] = cloneAny(v)+ }+ return out+}++func cloneAny(v any) any {+ switch v := v.(type) {+ case map[string]any:+ return cloneAnyMap(v)+ case []any:+ out := make([]any, len(v))+ for i, item := range v {+ out[i] = cloneAny(item)+ }+ return out+ case []string:+ return slices.Clone(v)+ case []bool:+ return slices.Clone(v)+ case []int:+ return slices.Clone(v)+ case []int8:+ return slices.Clone(v)+ case []int16:+ return slices.Clone(v)+ case []int32:+ return slices.Clone(v)+ case []int64:+ return slices.Clone(v)+ case []uint:+ return slices.Clone(v)+ case []uint8:+ return slices.Clone(v)+ case []uint16:+ return slices.Clone(v)+ case []uint32:+ return slices.Clone(v)+ case []uint64:+ return slices.Clone(v)+ case []float32:+ return slices.Clone(v)+ case []float64:+ return slices.Clone(v)+ default:+ return v+ }+}diff --git a/server/model_show_cache_test.go b/server/model_show_cache_test.gonew file mode 100644index 000000000..8b8175c66--- /dev/null+++ b/server/model_show_cache_test.go@@ -0,0 +1,501 @@+package server++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "slices"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/gin-gonic/gin"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/envconfig"+ internalcloud "github.com/ollama/ollama/internal/cloud"+ "github.com/ollama/ollama/manifest"+ modelpkg "github.com/ollama/ollama/types/model"+)++func TestModelShowCacheLocalHitUsesManifestDigest(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createShowCacheModel(t, "show-cache-local", map[string]any{"test.context_length": uint32(1024)})++ cache := newModelShowCache()+ calls := 0+ cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) {+ calls+++ return showCacheTestResponse(calls, req.Verbose), nil+ }++ first, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-local"})+ if err != nil {+ t.Fatalf("first GetLocal failed: %v", err)+ }+ second, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-local"})+ if err != nil {+ t.Fatalf("second GetLocal failed: %v", err)+ }++ if calls != 1 {+ t.Fatalf("getModelInfo calls = %d, want 1", calls)+ }+ if first.ModelInfo["call"] != 1 || second.ModelInfo["call"] != 1 {+ t.Fatalf("cached call markers = %v / %v, want both 1", first.ModelInfo["call"], second.ModelInfo["call"])+ }+}++func TestModelShowCacheLocalManifestDigestChangeRefreshes(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createShowCacheModel(t, "show-cache-refresh", map[string]any{"test.context_length": uint32(1024)})++ cache := newModelShowCache()+ calls := 0+ cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) {+ calls+++ return showCacheTestResponse(calls, req.Verbose), nil+ }++ if _, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-refresh"}); err != nil {+ t.Fatalf("first GetLocal failed: %v", err)+ }+ changeShowCacheManifest(t, "show-cache-refresh")+ refreshed, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-refresh"})+ if err != nil {+ t.Fatalf("refreshed GetLocal failed: %v", err)+ }++ if calls != 2 {+ t.Fatalf("getModelInfo calls = %d, want 2", calls)+ }+ if refreshed.ModelInfo["call"] != 2 {+ t.Fatalf("refreshed call marker = %v, want 2", refreshed.ModelInfo["call"])+ }+ if len(cache.local) != 1 {+ t.Fatalf("local cache entries = %d, want 1", len(cache.local))+ }+}++func TestModelShowCacheLocalVerboseVariantsAreSeparate(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createShowCacheModel(t, "show-cache-verbose", map[string]any{"test.context_length": uint32(1024)})++ cache := newModelShowCache()+ calls := 0+ cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) {+ calls+++ return showCacheTestResponse(calls, req.Verbose), nil+ }++ plain, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-verbose"})+ if err != nil {+ t.Fatalf("plain GetLocal failed: %v", err)+ }+ verbose, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-verbose", Verbose: true})+ if err != nil {+ t.Fatalf("verbose GetLocal failed: %v", err)+ }+ plainAgain, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-verbose"})+ if err != nil {+ t.Fatalf("plain repeat GetLocal failed: %v", err)+ }++ if calls != 2 {+ t.Fatalf("getModelInfo calls = %d, want 2", calls)+ }+ if plain.ModelInfo["verbose"] != false || verbose.ModelInfo["verbose"] != true || plainAgain.ModelInfo["call"] != 1 {+ t.Fatalf("unexpected verbose cache markers: plain=%v verbose=%v plainAgainCall=%v", plain.ModelInfo, verbose.ModelInfo, plainAgain.ModelInfo["call"])+ }+}++func TestModelShowCacheLocalHydrationSkipsUnchangedInMemory(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createShowCacheModel(t, "show-cache-hydrate", map[string]any{"test.context_length": uint32(1024)})++ cache := newModelShowCache()+ calls := 0+ cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) {+ calls+++ return showCacheTestResponse(calls, req.Verbose), nil+ }++ if err := cache.hydrateLocal(context.Background()); err != nil {+ t.Fatalf("first hydrateLocal failed: %v", err)+ }+ if err := cache.hydrateLocal(context.Background()); err != nil {+ t.Fatalf("second hydrateLocal failed: %v", err)+ }+ resp, err := cache.GetLocal(api.ShowRequest{Model: "show-cache-hydrate"})+ if err != nil {+ t.Fatalf("GetLocal after hydration failed: %v", err)+ }++ if calls != 1 {+ t.Fatalf("getModelInfo calls after unchanged in-memory hydration = %d, want 1", calls)+ }+ if resp.ModelInfo["call"] != 1 {+ t.Fatalf("hydrated call marker = %v, want 1", resp.ModelInfo["call"])+ }+}++func TestModelShowCacheBypassesSystemAndOptionsOverlays(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createShowCacheModel(t, "show-cache-overlay", map[string]any{"test.context_length": uint32(1024)})++ cache := newModelShowCache()+ key, digest, err := modelShowLocalKeyForRequest(api.ShowRequest{Model: "show-cache-overlay"})+ if err != nil {+ t.Fatalf("local key failed: %v", err)+ }+ cache.setLocal(key, digest, &api.ShowResponse{System: "cached", ModelInfo: map[string]any{}})++ s := Server{modelCaches: &modelCaches{show: cache}}+ w := createRequest(t, s.ShowHandler, api.ShowRequest{+ Model: "show-cache-overlay",+ System: "overlay-system",+ })+ if w.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())+ }++ var resp api.ShowResponse+ if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {+ t.Fatalf("decode response failed: %v", err)+ }+ if resp.System != "overlay-system" {+ t.Fatalf("system = %q, want overlay-system", resp.System)+ }++ w = createRequest(t, s.ShowHandler, api.ShowRequest{+ Model: "show-cache-overlay",+ Options: map[string]any{"num_ctx": float64(8192)},+ })+ if w.Code != http.StatusOK {+ t.Fatalf("options overlay status = %d, want 200: %s", w.Code, w.Body.String())+ }+ if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {+ t.Fatalf("decode options response failed: %v", err)+ }+ if resp.System == "cached" {+ t.Fatalf("options overlay unexpectedly returned cached response")+ }+}++func TestModelShowCacheLocalAndCloudSameBaseDoNotCollide(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createShowCacheModel(t, "show-cache-dual", map[string]any{"test.context_length": uint32(1024)})++ cache := newModelShowCache()+ cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) {+ return &api.ShowResponse{+ Details: api.ModelDetails{Format: "local"},+ ModelInfo: map[string]any{},+ }, nil+ }+ cloudKey := modelShowCloudKeyForModel("show-cache-dual:cloud", false)+ cache.setCloud(cloudKey, &api.ShowResponse{+ Details: api.ModelDetails{Format: "cloud"},+ ModelInfo: map[string]any{},+ })+ cache.mu.Lock()+ cache.cloudNextReadRefreshAfter[cloudKey] = time.Now().Add(time.Hour)+ cache.mu.Unlock()++ s := Server{modelCaches: &modelCaches{show: cache}}++ w := createRequest(t, s.ShowHandler, api.ShowRequest{Model: "show-cache-dual:cloud"})+ if w.Code != http.StatusOK {+ t.Fatalf("cloud status = %d, want 200: %s", w.Code, w.Body.String())+ }+ var cloudResp api.ShowResponse+ if err := json.NewDecoder(w.Body).Decode(&cloudResp); err != nil {+ t.Fatalf("decode cloud response failed: %v", err)+ }+ if cloudResp.Details.Format != "cloud" {+ t.Fatalf("cloud format = %q, want cloud", cloudResp.Details.Format)+ }++ w = createRequest(t, s.ShowHandler, api.ShowRequest{Model: "show-cache-dual"})+ if w.Code != http.StatusOK {+ t.Fatalf("local status = %d, want 200: %s", w.Code, w.Body.String())+ }+ var localResp api.ShowResponse+ if err := json.NewDecoder(w.Body).Decode(&localResp); err != nil {+ t.Fatalf("decode local response failed: %v", err)+ }+ if localResp.Details.Format != "local" {+ t.Fatalf("local format = %q, want local", localResp.Details.Format)+ }+}++func TestModelShowCacheCloudWarmHitReturnsStaleAndRefreshes(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())++ refreshDone := make(chan struct{})+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/api/show" {+ t.Fatalf("unexpected upstream path %q", r.URL.Path)+ }+ defer close(refreshDone)+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"details":{"format":"updated"},"model_info":{"source":"fresh"}}`))+ }))+ defer upstream.Close()+ withCloudProxyBaseURL(t, upstream.URL)++ cache := newModelShowCache()+ cache.client = upstream.Client()+ cache.setCloud(modelShowCloudKeyForModel("kimi-k2.5:cloud", false), &api.ShowResponse{+ Details: api.ModelDetails{Format: "cached"},+ ModelInfo: map[string]any{"source": "stale"},+ })++ s := Server{modelCaches: &modelCaches{show: cache}}+ w := createRequest(t, s.ShowHandler, api.ShowRequest{Model: "kimi-k2.5:cloud"})+ if w.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())+ }++ var resp api.ShowResponse+ if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {+ t.Fatalf("decode response failed: %v", err)+ }+ if resp.Details.Format != "cached" {+ t.Fatalf("format = %q, want cached", resp.Details.Format)+ }++ select {+ case <-refreshDone:+ case <-time.After(2 * time.Second):+ t.Fatal("timed out waiting for cloud refresh")+ }+ waitForCondition(t, 2*time.Second, func() bool {+ resp, ok := cache.getCloud(modelShowCloudKeyForModel("kimi-k2.5:cloud", false))+ return ok && resp.Details.Format == "updated"+ })+}++func TestModelShowCacheCloudColdMissFallsBackToProxy(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())++ var capturedPath, capturedBody string+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ capturedPath = r.URL.Path+ body, _ := io.ReadAll(r.Body)+ capturedBody = string(body)+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"details":{"format":"cold"},"model_info":{}}`))+ }))+ defer upstream.Close()+ withCloudProxyBaseURL(t, upstream.URL)++ s := &Server{modelCaches: &modelCaches{show: newModelShowCache()}}+ router, err := s.GenerateRoutes(nil)+ if err != nil {+ t.Fatalf("GenerateRoutes failed: %v", err)+ }+ local := httptest.NewServer(router)+ defer local.Close()++ req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, local.URL+"/api/show", bytes.NewBufferString(`{"model":"kimi-k2.5:cloud"}`))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")+ resp, err := local.Client().Do(req)+ if err != nil {+ t.Fatal(err)+ }+ defer resp.Body.Close()++ if resp.StatusCode != http.StatusOK {+ body, _ := io.ReadAll(resp.Body)+ t.Fatalf("status = %d, want 200: %s", resp.StatusCode, string(body))+ }+ if capturedPath != "/api/show" {+ t.Fatalf("upstream path = %q, want /api/show", capturedPath)+ }+ if !strings.Contains(capturedBody, `"model":"kimi-k2.5"`) {+ t.Fatalf("expected normalized model in upstream body, got %q", capturedBody)+ }+}++func TestModelShowCacheCloudHydrationUsesTagsAndShow(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())++ var mu sync.Mutex+ var showModels []string+ var tagsCalled bool+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/tags":+ if r.Method != http.MethodGet {+ t.Fatalf("tags method = %s, want GET", r.Method)+ }+ mu.Lock()+ tagsCalled = true+ mu.Unlock()+ _, _ = w.Write([]byte(`{"models":[{"name":"alpha:cloud"},{"model":"beta"}]}`))+ case "/api/show":+ var req api.ShowRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Fatalf("decode show request failed: %v", err)+ }+ mu.Lock()+ showModels = append(showModels, req.Model)+ mu.Unlock()+ _ = json.NewEncoder(w).Encode(api.ShowResponse{+ Details: api.ModelDetails{Format: req.Model},+ ModelInfo: map[string]any{"model": req.Model},+ })+ default:+ t.Fatalf("unexpected upstream path %q", r.URL.Path)+ }+ }))+ defer upstream.Close()+ withCloudProxyBaseURL(t, upstream.URL)++ cache := newModelShowCache()+ cache.client = upstream.Client()+ if err := cache.hydrateCloud(context.Background()); err != nil {+ t.Fatalf("hydrateCloud failed: %v", err)+ }++ mu.Lock()+ gotTagsCalled := tagsCalled+ gotShowModels := slices.Clone(showModels)+ mu.Unlock()+ slices.Sort(gotShowModels)++ if !gotTagsCalled {+ t.Fatal("expected /api/tags to be called")+ }+ if !slices.Equal(gotShowModels, []string{"alpha", "beta"}) {+ t.Fatalf("show models = %v, want [alpha beta]", gotShowModels)+ }+ for _, modelName := range gotShowModels {+ resp, ok := cache.getCloud(modelShowCloudKeyForModel(modelName, false))+ if !ok {+ t.Fatalf("missing cached cloud show response for %s", modelName)+ }+ if resp.Details.Format != modelName {+ t.Fatalf("cached format for %s = %q", modelName, resp.Details.Format)+ }+ }+}++func TestModelShowCacheCloudKeyNormalizesSourceTags(t *testing.T) {+ tests := map[string]string{+ " kimi-k2.5:cloud ": "kimi-k2.5",+ "gpt-oss:20b-cloud": "gpt-oss:20b",+ "qwen3": "qwen3",+ }++ for input, want := range tests {+ if got := modelShowCloudKeyForModel(input, false).Model; got != want {+ t.Fatalf("cloud key model for %q = %q, want %q", input, got, want)+ }+ }+}++func TestModelShowCacheCloudDisabledDoesNotServeStale(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())++ t.Cleanup(envconfig.ReloadServerConfig)+ t.Setenv("OLLAMA_NO_CLOUD", "1")+ envconfig.ReloadServerConfig()++ cache := newModelShowCache()+ cache.setCloud(modelShowCloudKeyForModel("kimi-k2.5:cloud", false), &api.ShowResponse{+ Details: api.ModelDetails{Format: "cached"},+ ModelInfo: map[string]any{},+ })++ if err := cache.hydrateCloud(context.Background()); !errors.Is(err, errModelShowNoCloud) {+ t.Fatalf("hydrateCloud error = %v, want %v", err, errModelShowNoCloud)+ }++ s := Server{modelCaches: &modelCaches{show: cache}}+ w := createRequest(t, s.ShowHandler, api.ShowRequest{Model: "kimi-k2.5:cloud"})+ if w.Code != http.StatusForbidden {+ t.Fatalf("status = %d, want 403: %s", w.Code, w.Body.String())+ }+ if !strings.Contains(w.Body.String(), internalcloud.DisabledError(cloudErrRemoteModelDetailsUnavailable)) {+ t.Fatalf("unexpected disabled response: %s", w.Body.String())+ }+}++func createShowCacheModel(t *testing.T, name string, kv map[string]any) {+ t.Helper()+ _, digest := createBinFile(t, kv, nil)+ var s Server+ w := createRequest(t, s.CreateHandler, api.CreateRequest{+ Model: name,+ Files: map[string]string{"model.gguf": digest},+ Stream: &stream,+ })+ if w.Code != http.StatusOK {+ t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())+ }+}++func changeShowCacheManifest(t *testing.T, name string) {+ t.Helper()+ n, err := getExistingName(modelpkg.ParseName(name))+ if err != nil {+ t.Fatalf("get existing name: %v", err)+ }+ mf, err := manifest.ParseNamedManifest(n)+ if err != nil {+ t.Fatalf("parse manifest: %v", err)+ }+ layer, err := manifest.NewLayer(strings.NewReader("changed"), "application/vnd.ollama.image.system")+ if err != nil {+ t.Fatalf("new layer: %v", err)+ }+ layers := append([]manifest.Layer(nil), mf.Layers...)+ layers = append(layers, layer)+ if err := manifest.WriteManifest(n, mf.Config, layers); err != nil {+ t.Fatalf("write manifest: %v", err)+ }+}++func showCacheTestResponse(call int, verbose bool) *api.ShowResponse {+ return &api.ShowResponse{+ Details: api.ModelDetails{+ Format: "gguf",+ Family: "test",+ },+ Capabilities: []modelpkg.Capability{modelpkg.CapabilityCompletion},+ ModelInfo: map[string]any{+ "call": call,+ "verbose": verbose,+ },+ }+}++func withCloudProxyBaseURL(t *testing.T, url string) {+ t.Helper()+ original := cloudProxyBaseURL+ cloudProxyBaseURL = url+ t.Cleanup(func() {+ cloudProxyBaseURL = original+ })+}diff --git a/server/routes.go b/server/routes.goindex 32cd99cd0..646142d8b 100644--- a/server/routes.go+++ b/server/routes.go@@ -98,11 +98,11 @@ var useClient2 = experimentEnabled("client2")var mode string = gin.DebugModetype Server struct {- addr net.Addr- sched *Scheduler- defaultNumCtx int- requestLogger *inferenceRequestLogger- modelRecommendations *modelRecommendationsCache+ addr net.Addr+ sched *Scheduler+ defaultNumCtx int+ requestLogger *inferenceRequestLogger+ modelCaches *modelCaches}func init() {@@ -1143,13 +1143,33 @@ func (s *Server) ShowHandler(c *gin.Context) {if modelRef.Source == modelSourceCloud {req.Model = modelRef.Base+ if modelShowCacheable(req) && s.modelCaches != nil && s.modelCaches.show != nil {+ if disabled, _ := internalcloud.Status(); disabled {+ c.JSON(http.StatusForbidden, gin.H{"error": internalcloud.DisabledError(cloudErrRemoteModelDetailsUnavailable)})+ return+ }++ ctx := context.Background()+ if c.Request != nil {+ ctx = c.Request.Context()+ }+ if resp, ok := s.modelCaches.show.GetCloudSWR(ctx, req); ok {+ c.JSON(http.StatusOK, resp)+ return+ }+ }proxyCloudJSONRequest(c, req, cloudErrRemoteModelDetailsUnavailable)return}req.Model = modelRef.Base- resp, err := GetModelInfo(req)+ var resp *api.ShowResponse+ if modelShowCacheable(req) && s.modelCaches != nil && s.modelCaches.show != nil {+ resp, err = s.modelCaches.show.GetLocal(req)+ } else {+ resp, err = GetModelInfo(req)+ }if err != nil {var statusErr api.StatusErrorswitch {@@ -1762,12 +1782,12 @@ func (s *Server) GenerateRoutes(rc *ollama.Registry) (http.Handler, error) {func (s *Server) ModelRecommendationsExperimentalHandler(c *gin.Context) {recs := defaultModelRecommendationssource := "default"- if s.modelRecommendations != nil {+ if s.modelCaches != nil && s.modelCaches.recommendations != nil {ctx := context.Background()if c.Request != nil {ctx = c.Request.Context()}- recs = s.modelRecommendations.GetSWR(ctx)+ recs = s.modelCaches.recommendations.GetSWR(ctx)source = "cache"}@@ -1811,12 +1831,9 @@ func Serve(ln net.Listener) error {}}- // TODO(parthsareen): If we add more runtime caches, prefer introducing a- // small cache manager owned by Server (for shared start/stop/health wiring)- // instead of adding one top-level field per cache here in Serve.s := &Server{- addr: ln.Addr(),- modelRecommendations: newModelRecommendationsCache(),+ addr: ln.Addr(),+ modelCaches: newModelCaches(),}if err := s.initRequestLogging(); err != nil {return err@@ -1842,7 +1859,7 @@ func Serve(ln net.Listener) error {schedCtx, schedDone := context.WithCancel(ctx)sched := InitScheduler(schedCtx)s.sched = sched- s.modelRecommendations.Start(ctx)+ s.modelCaches.Start(ctx)slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))srvr := &http.Server{