Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

server: extract GGUF metadata and unify capabilities (#17858)

Daniel Hiltgenea8d65004aff
diff --git a/fs/gguf/gguf.go b/fs/gguf/gguf.go
index 4468a3140..d9bb876ec 100644
--- a/fs/gguf/gguf.go
+++ b/fs/gguf/gguf.go
@@ -51,12 +51,19 @@ type File struct {
bts []byte
}
-func Open(path string) (f *File, err error) {
- f = &File{bts: make([]byte, 4096)}
+func Open(path string) (_ *File, err error) {
+ f := &File{bts: make([]byte, 4096)}
f.file, err = os.Open(path)
if err != nil {
return nil, err
}
+ defer func() {
+ if err != nil {
+ if closeErr := f.close(); closeErr != nil {
+ err = errors.Join(err, closeErr)
+ }
+ }
+ }()
f.reader = newBufferedReader(f.file, 32<<10)
@@ -273,14 +280,14 @@ func readArrayData[T any](f *File, n uint64) (s []T, err error) {
return nil, err
}
- s = make([]T, size)
- for i := range size {
+ s = make([]T, 0, min(size, 4096))
+ for range size {
e, err := read[T](f)
if err != nil {
return nil, err
}
- s[i] = e
+ s = append(s, e)
}
return s, nil
@@ -292,14 +299,14 @@ func readArrayString(f *File, n uint64) (s []string, err error) {
return nil, err
}
- s = make([]string, size)
- for i := range size {
+ s = make([]string, 0, min(size, 4096))
+ for range size {
e, err := readString(f)
if err != nil {
return nil, err
}
- s[i] = e
+ s = append(s, e)
}
return s, nil
@@ -324,8 +331,19 @@ func maxInt64() uint64 {
}
func (f *File) Close() error {
- f.keyValues.stop()
- f.tensors.stop()
+ return f.close()
+}
+
+func (f *File) close() error {
+ if f.keyValues != nil {
+ f.keyValues.stop()
+ }
+ if f.tensors != nil {
+ f.tensors.stop()
+ }
+ if f.file == nil {
+ return nil
+ }
return f.file.Close()
}
diff --git a/fs/gguf/keyvalue.go b/fs/gguf/keyvalue.go
index 474201165..f2f2b7f24 100644
--- a/fs/gguf/keyvalue.go
+++ b/fs/gguf/keyvalue.go
@@ -18,6 +18,11 @@ type Value struct {
value any
}
+// Any returns Value as stored, without conversion. If it is not set, it returns nil.
+func (v Value) Any() any {
+ return v.value
+}
+
func value[T any](v Value, kinds ...reflect.Kind) (t T) {
vv := reflect.ValueOf(v.value)
if slices.Contains(kinds, vv.Kind()) {
diff --git a/manifest/manifest.go b/manifest/manifest.go
index c0277e9a5..d010ead73 100644
--- a/manifest/manifest.go
+++ b/manifest/manifest.go
@@ -71,10 +71,13 @@ func (m *Manifest) Remove() error {
return PruneDirectory(manifests)
}
-func (m *Manifest) RemoveLayers() error {
+// RemoveLayers deletes the layers no other manifest references. The digests it
+// removed are returned even alongside an error, so callers can clean up
+// anything derived from them.
+func (m *Manifest) RemoveLayers() ([]string, error) {
ms, err := Manifests(true)
if err != nil {
- return err
+ return nil, err
}
// Build set of digests still in use by other manifests
@@ -88,6 +91,7 @@ func (m *Manifest) RemoveLayers() error {
}
// Remove layers not used by any other manifest
+ var removed []string
for _, layer := range append(m.Layers, m.Config) {
if layer.Digest == "" {
continue
@@ -97,16 +101,17 @@ func (m *Manifest) RemoveLayers() error {
}
blob, err := BlobsPath(layer.Digest)
if err != nil {
- return err
+ return removed, err
}
if err := os.Remove(blob); os.IsNotExist(err) {
slog.Debug("layer does not exist", "digest", layer.Digest)
} else if err != nil {
- return err
+ return removed, err
}
+ removed = append(removed, layer.Digest)
}
- return nil
+ return removed, nil
}
func ParseNamedManifest(n model.Name) (*Manifest, error) {
diff --git a/manifest/paths.go b/manifest/paths.go
index 4451c81aa..df452794e 100644
--- a/manifest/paths.go
+++ b/manifest/paths.go
@@ -14,6 +14,18 @@ import (
var ErrInvalidDigestFormat = errors.New("invalid digest format")
+// a manifest spells a digest with ":", a filename with "-"
+var digestPattern = regexp.MustCompile(`^sha256[:-][0-9a-fA-F]{64}$`)
+
+// ValidateDigest reports whether digest names a blob.
+func ValidateDigest(digest string) error {
+ if !digestPattern.MatchString(digest) {
+ return ErrInvalidDigestFormat
+ }
+
+ return nil
+}
+
func Path() (string, error) {
path := filepath.Join(envconfig.Models(), "manifests")
if err := os.MkdirAll(path, 0o755); err != nil {
@@ -38,12 +50,11 @@ func PathForName(n model.Name) (string, error) {
}
func BlobsPath(digest string) (string, error) {
- // only accept actual sha256 digests
- pattern := "^sha256[:-][0-9a-fA-F]{64}$"
- re := regexp.MustCompile(pattern)
-
- if digest != "" && !re.MatchString(digest) {
- return "", ErrInvalidDigestFormat
+ // the empty digest names the blobs directory
+ if digest != "" {
+ if err := ValidateDigest(digest); err != nil {
+ return "", err
+ }
}
digest = strings.ReplaceAll(digest, ":", "-")
diff --git a/server/create.go b/server/create.go
index fddbe5ae0..f0ad013d1 100644
--- a/server/create.go
+++ b/server/create.go
@@ -337,13 +337,13 @@ func (s *Server) CreateHandler(c *gin.Context) {
}
if !envconfig.NoPrune() && oldManifest != nil {
- if err := oldManifest.RemoveLayers(); err != nil {
+ removed, err := oldManifest.RemoveLayers()
+ removeGGUFMetadata(removed...)
+ if err != nil {
ch <- gin.H{"error": err.Error()}
}
}
- s.refreshModelListCache(name)
-
ch <- api.ProgressResponse{Status: "success"}
}()
diff --git a/server/gguf_metadata.go b/server/gguf_metadata.go
new file mode 100644
index 000000000..118fb7445
--- /dev/null
+++ b/server/gguf_metadata.go
@@ -0,0 +1,338 @@
+package server
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "math"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/ollama/ollama/envconfig"
+ fsgguf "github.com/ollama/ollama/fs/gguf"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/version"
+)
+
+// A blob's metadata block is extracted once into a file beside the model
+// store, keyed by blob digest. Values are held verbatim rather than derived, so
+// reading them differently later needs no invalidation, and values left out are
+// recorded by key so an absent key stays distinct from an uncopied one.
+
+// Comfortably above per-layer arrays, which scale with block count, and far
+// below any tokenizer vocabulary.
+const ggufMetadataMaxArray = 4096
+
+type ggufMetadata struct {
+ OllamaVersion string `json:"ollama_version,omitempty"`
+
+ // Values keyed exactly as they appear in the file. Never omitempty: absent
+ // on load means the file is not ours, and an all-omitted file is still
+ // usable.
+ KV map[string]any `json:"kv"`
+ // Keys present in the file whose values were too large to copy.
+ Omitted []string `json:"omitted,omitempty"`
+}
+
+// Valid reports whether the key is present. Keys are architecture qualified the
+// same way fs/gguf does it.
+func (m ggufMetadata) Valid(key string) bool {
+ _, ok := m.lookup(key)
+ return ok
+}
+
+func (m ggufMetadata) String(key string) string {
+ v, _ := m.lookup(key)
+ s, _ := v.(string)
+ return s
+}
+
+func (m ggufMetadata) Int(key string) int64 {
+ n, ok := m.number(key)
+ if !ok {
+ return 0
+ }
+ i, err := n.Int64()
+ if err != nil {
+ return 0
+ }
+ return i
+}
+
+func (m ggufMetadata) number(key string) (json.Number, bool) {
+ v, _ := m.lookup(key)
+ n, ok := v.(json.Number)
+ return n, ok
+}
+
+// Keys returns every key the file carried, including omitted ones.
+func (m ggufMetadata) Keys() []string {
+ keys := make([]string, 0, len(m.KV)+len(m.Omitted))
+ for k := range m.KV {
+ keys = append(keys, k)
+ }
+ return append(keys, m.Omitted...)
+}
+
+func (m ggufMetadata) lookup(key string) (any, bool) {
+ if !strings.HasPrefix(key, "general.") && !strings.HasPrefix(key, "tokenizer.") {
+ arch, _ := m.KV["general.architecture"].(string)
+ key = arch + "." + key
+ }
+ v, ok := m.KV[key]
+ return v, ok
+}
+
+func ggufMetadataPath(digest string) (string, error) {
+ if err := manifest.ValidateDigest(digest); err != nil {
+ return "", fmt.Errorf("%w: %q", err, digest)
+ }
+ return filepath.Join(ggufMetadataDir(), strings.ReplaceAll(digest, ":", "-")+".json"), nil
+}
+
+func ggufMetadataDir() string {
+ return filepath.Join(envconfig.Models(), "metadata")
+}
+
+// readGGUFMetadata extracts the blob when there is no usable metadata file. Best
+// effort throughout: a bad file is re-extracted, a failed write is dropped.
+func readGGUFMetadata(digest string) (ggufMetadata, error) {
+ path, err := ggufMetadataPath(digest)
+ if err != nil {
+ return ggufMetadata{}, err
+ }
+ if md, ok := loadGGUFMetadata(path); ok {
+ return md, nil
+ }
+
+ blob, err := manifest.BlobsPath(digest)
+ if err != nil {
+ return ggufMetadata{}, err
+ }
+ md, err := extractGGUFMetadata(blob)
+ if err != nil {
+ return ggufMetadata{}, err
+ }
+
+ writeGGUFMetadata(path, blob, md)
+ return md, nil
+}
+
+func loadGGUFMetadata(path string) (ggufMetadata, bool) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ slog.Debug("could not read gguf metadata file", "path", path, "error", err)
+ }
+ return ggufMetadata{}, false
+ }
+
+ md, err := decodeGGUFMetadata(data)
+ if err != nil {
+ slog.Debug("ignoring unusable gguf metadata file", "path", path, "error", err)
+ return ggufMetadata{}, false
+ }
+ return md, true
+}
+
+func writeGGUFMetadata(path, blob string, md ggufMetadata) {
+ data, err := json.Marshal(md)
+ if err != nil {
+ slog.Debug("could not encode gguf metadata", "path", path, "error", err)
+ return
+ }
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ slog.Debug("could not create gguf metadata dir", "path", path, "error", err)
+ return
+ }
+
+ tmp, err := os.CreateTemp(filepath.Dir(path), ".gguf-metadata-*.tmp")
+ if err != nil {
+ slog.Debug("could not create gguf metadata temp file", "path", path, "error", err)
+ return
+ }
+ defer os.Remove(tmp.Name())
+
+ if _, err = tmp.Write(data); err == nil {
+ err = tmp.Sync()
+ }
+ if cerr := tmp.Close(); err == nil {
+ err = cerr
+ }
+ if err == nil {
+ // Can fail transiently on Windows while another process holds the path.
+ err = os.Rename(tmp.Name(), path)
+ }
+ if err == nil {
+ // Blob removal precedes metadata removal. Rechecking after publication
+ // ensures either this writer or a concurrent remover cleans up the file.
+ if _, statErr := os.Stat(blob); errors.Is(statErr, os.ErrNotExist) {
+ if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
+ slog.Debug("could not remove gguf metadata file", "path", path, "error", removeErr)
+ }
+ return
+ }
+ }
+ if err != nil {
+ slog.Debug("could not write gguf metadata file", "path", path, "error", err)
+ }
+}
+
+// removeGGUFMetadata drops metadata for deleted blobs. Pass only digests whose
+// last reference is gone.
+func removeGGUFMetadata(digests ...string) {
+ for _, digest := range digests {
+ path, err := ggufMetadataPath(digest)
+ if err != nil {
+ continue
+ }
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ slog.Debug("could not remove gguf metadata file", "path", path, "error", err)
+ }
+ }
+}
+
+func pruneGGUFMetadata() {
+ dir := ggufMetadataDir()
+ entries, err := os.ReadDir(dir)
+ if errors.Is(err, os.ErrNotExist) {
+ return
+ }
+ if err != nil {
+ slog.Debug("could not read gguf metadata dir", "path", dir, "error", err)
+ return
+ }
+
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.IsDir() || !strings.HasSuffix(name, ".json") {
+ continue
+ }
+
+ digest := strings.TrimSuffix(name, ".json")
+ blob, err := manifest.BlobsPath(digest)
+ if err != nil {
+ continue
+ }
+ if _, err := os.Stat(blob); err == nil || !errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ path := filepath.Join(dir, name)
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ slog.Debug("could not remove gguf metadata file", "path", path, "error", err)
+ }
+ }
+}
+
+// extractGGUFMetadata reads the whole block; proving a key absent means reaching
+// the end. Normalized through JSON so extracted and loaded metadata hold the
+// same value types.
+func extractGGUFMetadata(path string) (ggufMetadata, error) {
+ md, err := scanGGUFMetadata(path)
+ if err != nil {
+ return ggufMetadata{}, err
+ }
+ md.OllamaVersion = version.Version
+
+ data, err := json.Marshal(md)
+ if err != nil {
+ return ggufMetadata{}, err
+ }
+ return decodeGGUFMetadata(data)
+}
+
+func decodeGGUFMetadata(data []byte) (ggufMetadata, error) {
+ // UseNumber keeps integers exact; the default would widen them to float64.
+ dec := json.NewDecoder(bytes.NewReader(data))
+ dec.UseNumber()
+ var md ggufMetadata
+ if err := dec.Decode(&md); err != nil {
+ return ggufMetadata{}, err
+ }
+ if md.KV == nil {
+ return ggufMetadata{}, fmt.Errorf("no metadata")
+ }
+ return md, nil
+}
+
+func scanGGUFMetadata(path string) (ggufMetadata, error) {
+ f, err := fsgguf.Open(path)
+ if err != nil {
+ return ggufMetadata{}, err
+ }
+ defer f.Close()
+
+ md := ggufMetadata{KV: make(map[string]any)}
+ for _, kv := range f.KeyValues() {
+ if omitValue(kv.Any()) {
+ md.Omitted = append(md.Omitted, kv.Key)
+ continue
+ }
+ md.KV[kv.Key] = kv.Any()
+ }
+ if err := f.Err(); err != nil {
+ return ggufMetadata{}, err
+ }
+ return md, nil
+}
+
+// omitValue reports values left out of the metadata file: long arrays, which are the
+// tokenizer, and non-finite floats, which JSON cannot represent.
+func omitValue(v any) bool {
+ switch v := v.(type) {
+ case float32:
+ return !finite(float64(v))
+ case float64:
+ return !finite(v)
+ case []float32:
+ for _, f := range v {
+ if !finite(float64(f)) {
+ return true
+ }
+ }
+ case []float64:
+ for _, f := range v {
+ if !finite(f) {
+ return true
+ }
+ }
+ }
+ return oversized(v)
+}
+
+func finite(f float64) bool {
+ return !math.IsInf(f, 0) && !math.IsNaN(f)
+}
+
+func oversized(v any) bool {
+ switch v := v.(type) {
+ case []string:
+ return len(v) > ggufMetadataMaxArray
+ case []int8:
+ return len(v) > ggufMetadataMaxArray
+ case []int16:
+ return len(v) > ggufMetadataMaxArray
+ case []int32:
+ return len(v) > ggufMetadataMaxArray
+ case []int64:
+ return len(v) > ggufMetadataMaxArray
+ case []uint8:
+ return len(v) > ggufMetadataMaxArray
+ case []uint16:
+ return len(v) > ggufMetadataMaxArray
+ case []uint32:
+ return len(v) > ggufMetadataMaxArray
+ case []uint64:
+ return len(v) > ggufMetadataMaxArray
+ case []float32:
+ return len(v) > ggufMetadataMaxArray
+ case []float64:
+ return len(v) > ggufMetadataMaxArray
+ case []bool:
+ return len(v) > ggufMetadataMaxArray
+ }
+ return false
+}
diff --git a/server/gguf_metadata_test.go b/server/gguf_metadata_test.go
new file mode 100644
index 000000000..f62951e35
--- /dev/null
+++ b/server/gguf_metadata_test.go
@@ -0,0 +1,736 @@
+package server
+
+import (
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "math"
+ "net/http"
+ "os"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/envconfig"
+ "github.com/ollama/ollama/fs/ggml"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/types/model"
+)
+
+func TestGGUFMetadataExtraction(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ big := make([]string, 5000)
+ for i := range big {
+ big[i] = "tok"
+ }
+ perLayer := make([]int32, 64)
+ for i := range perLayer {
+ perLayer[i] = int32(i)
+ }
+
+ path, _ := createBinFile(t, ggml.KV{
+ "general.architecture": "bert",
+ "bert.attention.softcap": float32(math.Inf(-1)),
+ "general.file_type": uint32(2),
+ "bert.pooling_type": uint32(1),
+ "bert.context_length": uint32(512),
+ "bert.attention.head_count_kv": perLayer,
+ "tokenizer.chat_template": "{{ messages }}",
+ "tokenizer.ggml.tokens": big,
+ "tokenizer.ggml.merges": big,
+ }, nil)
+
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if got := md.String("general.architecture"); got != "bert" {
+ t.Errorf("architecture = %q", got)
+ }
+ // Architecture-prefixed lookup, and the key order in the file must not matter.
+ if !md.Valid("pooling_type") {
+ t.Error("pooling_type not found")
+ }
+ if got := md.Int("context_length"); got != 512 {
+ t.Errorf("context_length = %d, want 512", got)
+ }
+ if got := md.String("tokenizer.chat_template"); got != "{{ messages }}" {
+ t.Errorf("chat_template = %q", got)
+ }
+ if md.Valid("vision.block_count") {
+ t.Error("absent key reported present")
+ }
+
+ // A 64-element per-layer array is real metadata and must survive.
+ if v, ok := md.KV["bert.attention.head_count_kv"].([]any); !ok || len(v) != 64 {
+ t.Errorf("per-layer array = %#v, want 64 elements", md.KV["bert.attention.head_count_kv"])
+ }
+
+ // The tokenizer is recorded as omitted, not silently missing.
+ want := map[string]bool{"tokenizer.ggml.tokens": true, "tokenizer.ggml.merges": true}
+ for _, key := range md.Omitted {
+ delete(want, key)
+ }
+ if len(want) != 0 {
+ t.Errorf("not reported omitted: %v (omitted=%v)", want, md.Omitted)
+ }
+ if _, ok := md.KV["tokenizer.ggml.tokens"]; ok {
+ t.Error("oversized array was copied into the metadata file")
+ }
+
+ // JSON cannot represent an infinity, and gemma3n ships one. Dropping that
+ // key must not cost the rest of the file.
+ if !slices.Contains(md.Omitted, "bert.attention.softcap") {
+ t.Errorf("non-finite float not omitted; omitted=%v", md.Omitted)
+ }
+}
+
+func TestGGUFMetadataFileRoundTrip(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, digest := createBinFile(t, ggml.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(32),
+ }, nil)
+
+ first, err := readGGUFMetadata(digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := first.Int("block_count"); got != 32 {
+ t.Fatalf("block_count = %d, want 32", got)
+ }
+
+ // With the blob gone, only the metadata file can answer.
+ if err := removeBlob(t, digest); err != nil {
+ t.Fatal(err)
+ }
+ second, err := readGGUFMetadata(digest)
+ if err != nil {
+ t.Fatalf("metadata file read failed: %v", err)
+ }
+ if got := second.Int("block_count"); got != 32 {
+ t.Fatalf("block_count from metadata file = %d, want 32", got)
+ }
+
+ // And it goes away with the blob.
+ removeGGUFMetadata(digest)
+ path, err := ggufMetadataPath(digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(path); !os.IsNotExist(err) {
+ t.Fatalf("metadata file still present after removal: %v", err)
+ }
+}
+
+// The metadata file is only ever an optimization: anything unusable must be re-extracted.
+func TestGGUFMetadataUnusableFile(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, digest := createBinFile(t, ggml.KV{"general.architecture": "llama"}, nil)
+ if _, err := readGGUFMetadata(digest); err != nil {
+ t.Fatal(err)
+ }
+ path, err := ggufMetadataPath(digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, tt := range []struct {
+ name string
+ content []byte
+ }{
+ {"truncated", []byte(`{"kv":{"general.arch`)},
+ {"not json", []byte("\x00\x01\x02")},
+ {"empty", []byte{}},
+ {"wrong shape", []byte(`[1,2,3]`)},
+ {"no kv", []byte(`{"ollama_version":"1.2.3"}`)},
+ {"kv not an object", []byte(`{"kv":[]}`)},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := os.WriteFile(path, tt.content, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := loadGGUFMetadata(path); ok {
+ t.Fatal("unusable metadata file reported usable")
+ }
+ md, err := readGGUFMetadata(digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := md.String("general.architecture"); got != "llama" {
+ t.Fatalf("architecture = %q, want llama", got)
+ }
+ })
+ }
+}
+
+func TestGGUFMetadataRejectsMalformed(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ dir := t.TempDir()
+
+ for _, tt := range []struct {
+ name string
+ data []byte
+ }{
+ {"empty file", nil},
+ {"bad magic", []byte("NOPE")},
+ {"truncated header", []byte("GGUF\x03\x00\x00\x00")},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ path := filepath.Join(dir, tt.name)
+ if err := os.WriteFile(path, tt.data, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := extractGGUFMetadata(path); err == nil {
+ t.Fatal("expected an error")
+ }
+ })
+ }
+}
+
+// The omission rule decides what never reaches the metadata file, so every array type
+// has to agree on it. A miss here silently drops real metadata.
+func TestGGUFMetadataOmitRule(t *testing.T) {
+ short, long := 64, ggufMetadataMaxArray+1
+
+ for _, tt := range []struct {
+ name string
+ keep any
+ omit any
+ }{
+ {"string", make([]string, short), make([]string, long)},
+ {"int8", make([]int8, short), make([]int8, long)},
+ {"int16", make([]int16, short), make([]int16, long)},
+ {"int32", make([]int32, short), make([]int32, long)},
+ {"int64", make([]int64, short), make([]int64, long)},
+ {"uint8", make([]uint8, short), make([]uint8, long)},
+ {"uint16", make([]uint16, short), make([]uint16, long)},
+ {"uint32", make([]uint32, short), make([]uint32, long)},
+ {"uint64", make([]uint64, short), make([]uint64, long)},
+ {"float32", make([]float32, short), make([]float32, long)},
+ {"float64", make([]float64, short), make([]float64, long)},
+ {"bool", make([]bool, short), make([]bool, long)},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ if omitValue(tt.keep) {
+ t.Errorf("%d-element %s array omitted; real per-layer metadata would be lost", short, tt.name)
+ }
+ if !omitValue(tt.omit) {
+ t.Errorf("%d-element %s array kept; the tokenizer would be copied", long, tt.name)
+ }
+ })
+ }
+
+ // Scalars are always kept, except the floats JSON cannot encode.
+ for _, keep := range []any{"s", int32(0), uint32(0), float32(1.5), float64(1.5), true} {
+ if omitValue(keep) {
+ t.Errorf("scalar %#v omitted", keep)
+ }
+ }
+ for _, omit := range []any{
+ float32(math.Inf(1)), float32(math.Inf(-1)), float64(math.NaN()),
+ []float32{1, float32(math.Inf(1))},
+ []float64{math.NaN()},
+ } {
+ if !omitValue(omit) {
+ t.Errorf("non-finite %#v kept; json.Marshal would fail the whole file", omit)
+ }
+ }
+}
+
+func TestGGUFMetadataKeysIncludesOmitted(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ big := make([]string, ggufMetadataMaxArray+1)
+ path, _ := createBinFile(t, ggml.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(4),
+ "tokenizer.ggml.tokens": big,
+ }, nil)
+
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ keys := md.Keys()
+ for _, want := range []string{"general.architecture", "llama.block_count", "tokenizer.ggml.tokens"} {
+ if !slices.Contains(keys, want) {
+ t.Errorf("Keys() = %v, missing %s", keys, want)
+ }
+ }
+}
+
+// Projector audio detection reads through Keys(), including prefixed forms.
+func TestProjectorAudioFromMetadata(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ for _, tt := range []struct {
+ name string
+ kv ggml.KV
+ audio bool
+ }{
+ {"bare key", ggml.KV{"general.architecture": "clip", "has_audio_encoder": true}, true},
+ {"prefixed key", ggml.KV{"general.architecture": "clip", "clip.has_audio_encoder": true}, true},
+ {"present but false", ggml.KV{"general.architecture": "clip", "has_audio_encoder": false}, false},
+ {"absent", ggml.KV{"general.architecture": "clip"}, false},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ path, _ := createBinFile(t, tt.kv, nil)
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := projectorHasAudio(md); got != tt.audio {
+ t.Errorf("projectorHasAudio = %v, want %v", got, tt.audio)
+ }
+ })
+ }
+
+ path, _ := createBinFile(t, ggml.KV{
+ "general.architecture": "clip",
+ "has_audio_encoder": true,
+ "clip.vision.projector_type": "gemma3nv",
+ }, nil)
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !projectorSuppressesAudioCapability(md) {
+ t.Error("gemma3nv projector should suppress audio")
+ }
+}
+
+func TestGGUFMetadataPathRejectsBadDigest(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+ hex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+
+ for _, digest := range []string{"", "sha256-short", "md5:" + hex, "../escape", "sha256:" + hex + "x"} {
+ if _, err := ggufMetadataPath(digest); err == nil {
+ t.Errorf("ggufMetadataPath(%q) accepted", digest)
+ }
+ }
+ for _, digest := range []string{"sha256:" + hex, "sha256-" + hex} {
+ if _, err := ggufMetadataPath(digest); err != nil {
+ t.Errorf("ggufMetadataPath(%q) rejected: %v", digest, err)
+ }
+ }
+}
+
+// A file can carry no values at all, and every value it does carry can be one
+// the omission rule drops. Either way what is left still says which keys the
+// file had, so it has to be storable: treating it as corrupt would rescan the
+// blob on every request forever.
+func TestGGUFMetadataWithNothingToCopy(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ path := filepath.Join(t.TempDir(), "empty.gguf")
+ if err := os.WriteFile(path, ggufBytes(0, nil), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ t.Fatalf("a file with no metadata is still a file: %v", err)
+ }
+ if len(md.KV) != 0 {
+ t.Errorf("KV = %v, want empty", md.KV)
+ }
+
+ encoded, err := json.Marshal(md)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := decodeGGUFMetadata(encoded); err != nil {
+ t.Errorf("written metadata cannot be read back, so it would be rescanned forever: %v", err)
+ }
+}
+
+const allocBudget = 1 << 20
+
+func ggufBytes(kvCount uint64, extra []byte) []byte {
+ b := make([]byte, 0, 24+len(extra))
+ b = binary.LittleEndian.AppendUint32(b, 0x46554747) // GGUF
+ b = binary.LittleEndian.AppendUint32(b, 3) // version
+ b = binary.LittleEndian.AppendUint64(b, 0) // tensor count
+ b = binary.LittleEndian.AppendUint64(b, kvCount) // kv count
+ return append(b, extra...)
+}
+
+func declaredGGUF(t *testing.T, kvCount uint64, extra []byte) string {
+ t.Helper()
+ b := ggufBytes(kvCount, extra)
+
+ path := filepath.Join(t.TempDir(), "declared.gguf")
+ if err := os.WriteFile(path, b, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("file is %d bytes, declares %d KV entries", len(b), kvCount)
+ return path
+}
+
+func allocDelta(t *testing.T, fn func()) uint64 {
+ t.Helper()
+ var before, after runtime.MemStats
+ runtime.GC()
+ runtime.ReadMemStats(&before)
+ fn()
+ runtime.ReadMemStats(&after)
+ return after.TotalAlloc - before.TotalAlloc
+}
+
+func TestGGUFMetadataBoundsDeclaredKVCount(t *testing.T) {
+ path := declaredGGUF(t, 1<<20, nil)
+ got := allocDelta(t, func() {
+ if _, err := extractGGUFMetadata(path); err == nil {
+ t.Error("expected an error from a 24-byte file")
+ }
+ })
+ t.Logf("allocated %d bytes before failing", got)
+ if got > allocBudget {
+ t.Errorf("allocated %d bytes from a 24-byte file, want at most %d", got, allocBudget)
+ }
+}
+
+// Cover every array element type because strings and fixed-width values use
+// separate readers.
+func TestGGUFMetadataBoundsDeclaredArray(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ elemType uint32
+ }{
+ {"uint8", 0},
+ {"int8", 1},
+ {"uint16", 2},
+ {"int16", 3},
+ {"uint32", 4},
+ {"int32", 5},
+ {"float32", 6},
+ {"bool", 7},
+ {"string", 8},
+ {"uint64", 10},
+ {"int64", 11},
+ {"float64", 12},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ var extra []byte
+ extra = binary.LittleEndian.AppendUint64(extra, 3) // key length
+ extra = append(extra, "big"...)
+ extra = binary.LittleEndian.AppendUint32(extra, 9) // type: array
+ extra = binary.LittleEndian.AppendUint32(extra, tt.elemType)
+ extra = binary.LittleEndian.AppendUint64(extra, 8_000_000) // element count
+
+ path := declaredGGUF(t, 1, extra)
+ got := allocDelta(t, func() {
+ if _, err := extractGGUFMetadata(path); err == nil {
+ t.Error("expected an error")
+ }
+ })
+ if got > allocBudget {
+ t.Errorf("allocated %d bytes for a declared array of 8000000, want at most %d", got, allocBudget)
+ }
+ })
+ }
+}
+
+func removeBlob(t *testing.T, digest string) error {
+ t.Helper()
+ path, err := manifest.BlobsPath(digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return os.Remove(path)
+}
+
+func metadataCount(t *testing.T) int {
+ t.Helper()
+ entries, err := os.ReadDir(filepath.Join(envconfig.Models(), "metadata"))
+ if err != nil {
+ return 0
+ }
+ return len(entries)
+}
+
+// createModelFromBlob and deleteModelNamed drive the handlers rather than
+// writing manifests, so the metadata hooks under test are the ones the server
+// actually runs.
+func createModelFromBlob(t *testing.T, name, digest, tmpl string) {
+ t.Helper()
+ var s Server
+ w := createRequest(t, s.CreateHandler, api.CreateRequest{
+ Model: name, Files: map[string]string{"model.gguf": digest},
+ Template: tmpl, Stream: &stream,
+ })
+ if w.Code != http.StatusOK {
+ t.Fatalf("create %s: %d %s", name, w.Code, w.Body.String())
+ }
+}
+
+func deleteModelNamed(t *testing.T, name string) {
+ t.Helper()
+ var s Server
+ w := createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: name})
+ if w.Code != http.StatusOK {
+ t.Fatalf("delete %s: %d %s", name, w.Code, w.Body.String())
+ }
+}
+
+// Metadata is derived from a blob, so it lives exactly as long as the blob: it
+// survives while any manifest still references it and goes with the last one.
+func TestGGUFMetadataRemovedWithLastReference(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, digest := createBinFile(t, ggml.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(1),
+ }, nil)
+
+ createModelFromBlob(t, "alias-one", digest, "")
+ createModelFromBlob(t, "alias-two", digest, "")
+
+ if _, err := readGGUFMetadata(digest); err != nil {
+ t.Fatal(err)
+ }
+ if got := metadataCount(t); got != 1 {
+ t.Fatalf("metadata files = %d, want 1 (both aliases share one blob)", got)
+ }
+
+ deleteModelNamed(t, "alias-one")
+ if got := metadataCount(t); got != 1 {
+ t.Fatalf("metadata files after removing one alias = %d, want 1", got)
+ }
+
+ deleteModelNamed(t, "alias-two")
+ if got := metadataCount(t); got != 0 {
+ t.Fatalf("metadata files after removing the last reference = %d, want 0", got)
+ }
+}
+
+// Recreating a model over the same name replaces its blob; the old blob's
+// metadata must go with it.
+func TestGGUFMetadataRemovedOnReplacement(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, first := createBinFile(t, ggml.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(1),
+ }, nil)
+ createModelFromBlob(t, "replace-me", first, "")
+ if _, err := readGGUFMetadata(first); err != nil {
+ t.Fatal(err)
+ }
+
+ _, second := createBinFile(t, ggml.KV{
+ "general.architecture": "llama",
+ "llama.block_count": uint32(2),
+ }, nil)
+ createModelFromBlob(t, "replace-me", second, "")
+ if _, err := readGGUFMetadata(second); err != nil {
+ t.Fatal(err)
+ }
+
+ if got := metadataCount(t); got != 1 {
+ t.Fatalf("metadata files after replacement = %d, want 1 (the old blob's should be gone)", got)
+ }
+ old, err := ggufMetadataPath(first)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(old); !os.IsNotExist(err) {
+ t.Errorf("metadata for the replaced blob still present: %v", err)
+ }
+}
+
+// A blob missing from disk is still a blob whose metadata has to go. It no
+// longer appears in the blob scan, so pruning has to find its metadata directly.
+func TestGGUFMetadataRemovedForMissingBlob(t *testing.T) {
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, orphan := createBinFile(t, ggml.KV{"general.architecture": "llama"}, nil)
+ if _, err := readGGUFMetadata(orphan); err != nil {
+ t.Fatal(err)
+ }
+ if err := removeBlob(t, orphan); err != nil {
+ t.Fatal(err)
+ }
+
+ _, live := createBinFile(t, ggml.KV{"general.architecture": "bert"}, nil)
+ if _, err := readGGUFMetadata(live); err != nil {
+ t.Fatal(err)
+ }
+ if got := metadataCount(t); got != 2 {
+ t.Fatalf("metadata files before the sweep = %d, want 2", got)
+ }
+
+ if err := PruneLayers(); err != nil {
+ t.Fatal(err)
+ }
+ if got := metadataCount(t); got != 1 {
+ t.Errorf("metadata files = %d, want 1", got)
+ }
+ path, err := ggufMetadataPath(live)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(path); err != nil {
+ t.Errorf("metadata for existing blob was removed: %v", err)
+ }
+}
+
+func TestGGUFMetadataNotPublishedAfterDelete(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, digest := createBinFile(t, ggml.KV{
+ "general.architecture": "llama",
+ "general.description": strings.Repeat("x", 16<<20),
+ }, nil)
+ createModelFromBlob(t, "delete-during-load", digest, "")
+
+ loaded := make(chan error, 1)
+ go func() {
+ _, err := GetModel("delete-during-load")
+ loaded <- err
+ }()
+
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ entries, err := os.ReadDir(ggufMetadataDir())
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ t.Fatal(err)
+ }
+ if slices.ContainsFunc(entries, func(entry os.DirEntry) bool {
+ return strings.HasPrefix(entry.Name(), ".gguf-metadata-")
+ }) {
+ break
+ }
+ select {
+ case err := <-loaded:
+ t.Fatalf("GetModel completed before its metadata write could overlap deletion: %v", err)
+ default:
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("timed out waiting for metadata write")
+ }
+ runtime.Gosched()
+ }
+
+ deleteModelNamed(t, "delete-during-load")
+ select {
+ case err := <-loaded:
+ if err != nil {
+ t.Fatalf("GetModel: %v", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("timed out waiting for GetModel")
+ }
+
+ path, err := ggufMetadataPath(digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("metadata published after model deletion: %v", err)
+ }
+}
+
+// The point of the metadata file is that loading a model no longer reads its
+// blob. With the blob deleted, GetModel must still describe the model exactly
+// as before - which is also the guard against anything reintroducing a
+// per-request read of the model file.
+func TestGetModelReadsNoBlob(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+ _, digest := createBinFile(t, ggml.KV{
+ "general.architecture": "bert",
+ "bert.pooling_type": uint32(1),
+ "bert.context_length": uint32(512),
+ "bert.embedding_length": uint32(384),
+ }, nil)
+ createModelFromBlob(t, "embedder", digest, "")
+
+ want, err := GetModel("embedder")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Contains(want.Capabilities(), model.CapabilityEmbedding) {
+ t.Fatalf("capabilities = %v, want embedding", want.Capabilities())
+ }
+
+ if err := removeBlob(t, digest); err != nil {
+ t.Fatal(err)
+ }
+ got, err := GetModel("embedder")
+ if err != nil {
+ t.Fatalf("GetModel needed the blob: %v", err)
+ }
+ if !slices.Equal(got.Capabilities(), want.Capabilities()) {
+ t.Errorf("capabilities without the blob = %v, want %v", got.Capabilities(), want.Capabilities())
+ }
+ if got.metadata.Int("context_length") != 512 {
+ t.Errorf("context_length = %d, want 512", got.metadata.Int("context_length"))
+ }
+}
+
+// Extraction either fails or produces metadata that survives a metadata-file
+// round trip unchanged.
+func FuzzGGUFMetadata(f *testing.F) {
+ f.Add(ggufBytes(0, nil))
+ f.Add(ggufBytes(1<<20, nil))
+
+ var str []byte
+ str = binary.LittleEndian.AppendUint64(str, 20)
+ str = append(str, "general.architecture"...)
+ str = binary.LittleEndian.AppendUint32(str, 8) // type: string
+ str = binary.LittleEndian.AppendUint64(str, 5)
+ str = append(str, "llama"...)
+ f.Add(ggufBytes(1, str))
+
+ var arr []byte
+ arr = binary.LittleEndian.AppendUint64(arr, 3)
+ arr = append(arr, "big"...)
+ arr = binary.LittleEndian.AppendUint32(arr, 9) // type: array
+ arr = binary.LittleEndian.AppendUint32(arr, 8) // element type: string
+ arr = binary.LittleEndian.AppendUint64(arr, 8_000_000)
+ f.Add(ggufBytes(1, arr))
+
+ path := filepath.Join(f.TempDir(), "fuzz.gguf")
+ f.Fuzz(func(t *testing.T, data []byte) {
+ if err := os.WriteFile(path, data, 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ return
+ }
+ if md.KV == nil {
+ t.Fatal("extraction reported success with no metadata")
+ }
+
+ encoded, err := json.Marshal(md)
+ if err != nil {
+ t.Fatalf("extracted metadata cannot be written: %v", err)
+ }
+ loaded, err := decodeGGUFMetadata(encoded)
+ if err != nil {
+ t.Fatalf("extracted metadata cannot be read back: %v", err)
+ }
+ if !reflect.DeepEqual(md, loaded) {
+ t.Errorf("metadata changed on the trip through a file:\n%#v\n%#v", md, loaded)
+ }
+ })
+}
diff --git a/server/images.go b/server/images.go
index 9be4ce51d..435030590 100644
--- a/server/images.go
+++ b/server/images.go
@@ -10,7 +10,6 @@ import (
"io"
"log"
"log/slog"
- "math"
"net"
"net/http"
"net/url"
@@ -24,7 +23,6 @@ import (
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/envconfig"
- "github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/model/parsers"
"github.com/ollama/ollama/parser"
@@ -83,8 +81,10 @@ type Model struct {
Template *template.Template
- capabilities []model.Capability
- capabilitiesCached bool
+ // Metadata of the model blob and of each projector, read from their
+ // metadata files when the model is loaded.
+ metadata ggufMetadata
+ projectorMetadata []ggufMetadata
}
func (m *Model) IsMLX() bool {
@@ -95,47 +95,33 @@ func (m *Model) isGGUF() bool {
return m.Config.ModelFormat == "" || m.Config.ModelFormat == "gguf"
}
-func generationDefaultsFromGGUF(f *gguf.File) model.GenerationDefaults {
+func generationDefaultsFromMetadata(md ggufMetadata) model.GenerationDefaults {
return model.ParseGGUFGenerationDefaults(
func(key string) (int64, bool) {
- return ggufIntGenerationDefault(f.KeyValue(key))
+ n, ok := md.number(key)
+ if !ok {
+ return 0, false
+ }
+ if value, err := n.Int64(); err == nil {
+ return value, true
+ }
+ value, err := n.Float64()
+ if err != nil {
+ return 0, false
+ }
+ return int64(value), true
},
func(key string) (float64, bool) {
- return ggufFloatGenerationDefault(f.KeyValue(key))
+ n, ok := md.number(key)
+ if !ok {
+ return 0, false
+ }
+ value, err := n.Float64()
+ return value, err == nil
},
)
}
-func ggufIntGenerationDefault(kv gguf.KeyValue) (int64, bool) {
- if value, ok := kv.IntOK(); ok {
- return value, true
- }
- if value, ok := kv.UintOK(); ok {
- if value > math.MaxInt64 {
- return 0, false
- }
- return int64(value), true
- }
- if value, ok := kv.FloatOK(); ok {
- // Match api.Options.FromMap; rounding may be better for near-integers.
- return int64(value), true
- }
- return 0, false
-}
-
-func ggufFloatGenerationDefault(kv gguf.KeyValue) (float64, bool) {
- if value, ok := kv.FloatOK(); ok {
- return value, true
- }
- if value, ok := kv.IntOK(); ok {
- return float64(value), true
- }
- if value, ok := kv.UintOK(); ok {
- return float64(value), true
- }
- return 0, false
-}
-
func appendCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {
if slices.Contains(capabilities, capability) {
return capabilities
@@ -153,11 +139,7 @@ const (
// Capabilities returns the capabilities that the model supports
func (m *Model) Capabilities() []model.Capability {
- if m.capabilitiesCached {
- return slices.Clone(m.capabilities)
- }
-
- capabilities := m.capabilitiesForTemplate(templateCapabilitySelected, nil)
+ capabilities := m.capabilitiesForTemplate(templateCapabilitySelected)
if len(capabilities) == 0 {
slog.Warn("unknown capabilities for model", "model", m.Name)
}
@@ -165,12 +147,12 @@ func (m *Model) Capabilities() []model.Capability {
return capabilities
}
-func (m *Model) capabilitiesForTemplate(source templateCapabilitySource, f *gguf.File) []model.Capability {
+func (m *Model) capabilitiesForTemplate(source templateCapabilitySource) []model.Capability {
capabilities := []model.Capability{}
var modelArch string
capabilities = m.configCapabilities(capabilities)
- capabilities, modelArch = m.ggufCapabilities(capabilities, source, f)
+ capabilities, modelArch = m.ggufCapabilities(capabilities, source)
capabilities = m.projectorCapabilities(capabilities)
capabilities = m.templateCapabilities(capabilities, source)
capabilities = m.parserCapabilities(capabilities)
@@ -187,44 +169,33 @@ func (m *Model) configCapabilities(capabilities []model.Capability) []model.Capa
return capabilities
}
-func (m *Model) ggufCapabilities(capabilities []model.Capability, source templateCapabilitySource, f *gguf.File) ([]model.Capability, string) {
+func (m *Model) ggufCapabilities(capabilities []model.Capability, source templateCapabilitySource) ([]model.Capability, string) {
if m.ModelPath == "" || !m.isGGUF() {
return capabilities, ""
}
- if f == nil {
- var err error
- f, err = gguf.Open(m.ModelPath)
- if err != nil {
- slog.Error("couldn't open model file", "error", err)
- return capabilities, ""
- }
- defer f.Close()
- }
-
- modelArch := f.KeyValue("general.architecture").String()
switch source {
case templateCapabilitySelected:
if !usesOllamaRenderedChat(m) {
- capabilities = chatTemplateCapabilities(capabilities, f.KeyValue("tokenizer.chat_template").String())
+ capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template"))
}
case templateCapabilityChat:
- capabilities = chatTemplateCapabilities(capabilities, f.KeyValue("tokenizer.chat_template").String())
+ capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template"))
}
- if f.KeyValue("pooling_type").Valid() {
+ if m.metadata.Valid("pooling_type") {
capabilities = appendCapability(capabilities, model.CapabilityEmbedding)
} else {
// If no embedding is specified, we assume the model supports completion.
capabilities = appendCapability(capabilities, model.CapabilityCompletion)
}
- if f.KeyValue("vision.block_count").Valid() {
+ if m.metadata.Valid("vision.block_count") {
capabilities = appendCapability(capabilities, model.CapabilityVision)
}
- if f.KeyValue("audio.block_count").Valid() {
+ if m.metadata.Valid("audio.block_count") {
capabilities = appendCapability(capabilities, model.CapabilityAudio)
}
- return capabilities, modelArch
+ return capabilities, m.metadata.String("general.architecture")
}
func chatTemplateCapabilities(capabilities []model.Capability, chatTemplate string) []model.Capability {
@@ -390,28 +361,17 @@ func capabilityLogValue(present bool, capabilities []model.Capability) any {
}
func (m *Model) templateSelectionCapabilities(usesHarmony bool) (goTemplate, chatTemplate, harmony, rendererParser []model.Capability) {
- var f *gguf.File
- if m.ModelPath != "" && m.isGGUF() {
- var err error
- f, err = gguf.Open(m.ModelPath)
- if err != nil {
- slog.Error("couldn't open model file", "error", err)
- } else {
- defer f.Close()
- }
- }
-
if m.HasGoTemplate {
- goTemplate = m.capabilitiesForTemplate(templateCapabilityGo, f)
+ goTemplate = m.capabilitiesForTemplate(templateCapabilityGo)
}
if m.HasChatTemplate {
- chatTemplate = m.capabilitiesForTemplate(templateCapabilityChat, f)
+ chatTemplate = m.capabilitiesForTemplate(templateCapabilityChat)
}
if usesHarmony {
- harmony = m.capabilitiesForTemplate(templateCapabilitySelected, f)
+ harmony = m.capabilitiesForTemplate(templateCapabilitySelected)
}
if m.Config.Renderer != "" || m.Config.Parser != "" {
- rendererParser = m.capabilitiesForTemplate(templateCapabilitySelected, f)
+ rendererParser = m.capabilitiesForTemplate(templateCapabilitySelected)
}
return goTemplate, chatTemplate, harmony, rendererParser
@@ -439,16 +399,10 @@ func (m *Model) projectorCapabilities(capabilities []model.Capability) []model.C
}
capabilities = appendCapability(capabilities, model.CapabilityVision)
- for _, projectorPath := range m.ProjectorPaths {
- f, err := gguf.Open(projectorPath)
- if err != nil {
- slog.Error("couldn't open projector file", "error", err)
- continue
- }
- if projectorHasAudio(f) && !projectorSuppressesAudioCapability(f) {
+ for _, md := range m.projectorMetadata {
+ if projectorHasAudio(md) && !projectorSuppressesAudioCapability(md) {
capabilities = appendCapability(capabilities, model.CapabilityAudio)
}
- f.Close()
}
return capabilities
@@ -551,22 +505,21 @@ func isNemotron3NanoSafetensorsConfig(cfg model.ConfigV2) bool {
slices.Contains(cfg.ModelFamilies, "nemotron_h_omni"))
}
-func projectorHasAudio(f *gguf.File) bool {
- if f.KeyValue("has_audio_encoder").Bool() {
- return true
- }
-
- for _, kv := range f.KeyValues() {
- if strings.HasSuffix(kv.Key, ".has_audio_encoder") && kv.Bool() {
- return true
+func projectorHasAudio(md ggufMetadata) bool {
+ // read directly: Keys reports qualified keys, the accessors qualify theirs
+ for _, key := range md.Keys() {
+ if key == "has_audio_encoder" || strings.HasSuffix(key, ".has_audio_encoder") {
+ if b, ok := md.KV[key].(bool); ok && b {
+ return true
+ }
}
}
return false
}
-func projectorSuppressesAudioCapability(f *gguf.File) bool {
- switch f.KeyValue("vision.projector_type").String() {
+func projectorSuppressesAudioCapability(md ggufMetadata) bool {
+ switch md.String("vision.projector_type") {
case "gemma3nv":
return true
}
@@ -745,6 +698,13 @@ func GetModel(name string) (*Model, error) {
modelHasPooling := false
ggufChatTemplate := ""
for _, layer := range mf.Layers {
+ // Nothing below reads a tensor layer, and resolving a path costs a
+ // syscall each. Named rather than allowlisting the types below, so a new
+ // layer type is slower here instead of silently unread.
+ if layer.MediaType == manifest.MediaTypeImageTensor {
+ continue
+ }
+
filename, err := manifest.BlobsPath(layer.Digest)
if err != nil {
return nil, err
@@ -755,16 +715,16 @@ func GetModel(name string) (*Model, error) {
m.ModelPath = filename
m.ParentModel = layer.From
if m.isGGUF() {
- f, err := gguf.Open(filename)
+ md, err := readGGUFMetadata(layer.Digest)
if err != nil {
- slog.Error("couldn't open model file", "error", err)
+ slog.Error("couldn't read model metadata", "error", err)
break
}
- ggufChatTemplate = f.KeyValue("tokenizer.chat_template").String()
+ m.metadata = md
+ ggufChatTemplate = md.String("tokenizer.chat_template")
m.HasChatTemplate = ggufChatTemplate != ""
- modelHasPooling = f.KeyValue("pooling_type").Valid()
- m.GenerationDefaults = generationDefaultsFromGGUF(f)
- f.Close()
+ modelHasPooling = md.Valid("pooling_type")
+ m.GenerationDefaults = generationDefaultsFromMetadata(md)
}
case manifest.MediaTypeImageDraft:
m.DraftPath = filename
@@ -776,6 +736,11 @@ func GetModel(name string) (*Model, error) {
m.AdapterPaths = append(m.AdapterPaths, filename)
case "application/vnd.ollama.image.projector":
m.ProjectorPaths = append(m.ProjectorPaths, filename)
+ if md, err := readGGUFMetadata(layer.Digest); err != nil {
+ slog.Error("couldn't read projector metadata", "error", err)
+ } else {
+ m.projectorMetadata = append(m.projectorMetadata, md)
+ }
case "application/vnd.ollama.image.prompt",
"application/vnd.ollama.image.template":
m.HasGoTemplate = true
@@ -900,10 +865,11 @@ func deleteUnusedLayers(deleteMap map[string]struct{}) error {
slog.Info(fmt.Sprintf("couldn't get file path for '%s': %v", k, err))
continue
}
- if err := os.Remove(fp); err != nil {
+ if err := os.Remove(fp); err != nil && !errors.Is(err, os.ErrNotExist) {
slog.Info(fmt.Sprintf("couldn't remove file '%s': %v", fp, err))
continue
}
+ removeGGUFMetadata(k)
}
return nil
@@ -960,6 +926,7 @@ func PruneLayers() error {
slog.Error(fmt.Sprintf("couldn't remove unused layers: %v", err))
return nil
}
+ pruneGGUFMetadata()
slog.Info(fmt.Sprintf("total unused blobs removed: %d", len(deleteMap)))
@@ -1122,6 +1089,7 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
if err := os.Remove(fp); err != nil {
slog.Info(fmt.Sprintf("couldn't remove file with digest mismatch '%s': %v", fp, err))
}
+ removeGGUFMetadata(layer.Digest)
}
return err
}
diff --git a/server/images_test.go b/server/images_test.go
index fd521dcbd..a0302bbee 100644
--- a/server/images_test.go
+++ b/server/images_test.go
@@ -14,7 +14,6 @@ import (
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/fs/ggml"
- fsgguf "github.com/ollama/ollama/fs/gguf"
"github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/template"
"github.com/ollama/ollama/types/model"
@@ -61,7 +60,7 @@ func TestPruneLayersSkipsRecentOrphans(t *testing.T) {
}
}
-func TestGenerationDefaultsFromGGUF(t *testing.T) {
+func TestGenerationDefaultsFromMetadata(t *testing.T) {
file, err := os.CreateTemp(t.TempDir(), "model-*.gguf")
if err != nil {
t.Fatal(err)
@@ -87,13 +86,12 @@ func TestGenerationDefaultsFromGGUF(t *testing.T) {
t.Fatal(err)
}
- f, err := fsgguf.Open(file.Name())
+ md, err := extractGGUFMetadata(file.Name())
if err != nil {
t.Fatal(err)
}
- defer f.Close()
- defaults := generationDefaultsFromGGUF(f)
+ defaults := generationDefaultsFromMetadata(md)
check := func(key string, want any) {
t.Helper()
if got := defaults[key]; got != want {
@@ -104,10 +102,10 @@ func TestGenerationDefaultsFromGGUF(t *testing.T) {
check("top_k", int64(40))
check("top_p", float64(1))
check("min_p", float64(0))
- check("typical_p", float64(float32(0.95)))
+ check("typical_p", float64(0.95))
check("temperature", float64(1))
check("repeat_last_n", int64(64))
- check("repeat_penalty", float64(float32(1.05)))
+ check("repeat_penalty", float64(1.05))
check("frequency_penalty", float64(0))
check("presence_penalty", float64(0))
if _, ok := defaults["mirostat_tau"]; ok {
@@ -354,6 +352,28 @@ func writeTestModelManifest(t *testing.T, name, digest, tmpl string) {
}
}
+// loadTestMetadata fills in what GetModel would have read from the metadata
+// files, so
+// hand-built models resolve capabilities the same way loaded ones do.
+func loadTestMetadata(t *testing.T, m *Model) {
+ t.Helper()
+ if m.ModelPath != "" {
+ md, err := extractGGUFMetadata(m.ModelPath)
+ if err != nil {
+ t.Fatalf("metadata for %s: %v", m.ModelPath, err)
+ }
+ m.metadata = md
+ }
+ m.projectorMetadata = nil
+ for _, path := range m.ProjectorPaths {
+ md, err := extractGGUFMetadata(path)
+ if err != nil {
+ t.Fatalf("projector metadata for %s: %v", path, err)
+ }
+ m.projectorMetadata = append(m.projectorMetadata, md)
+ }
+}
+
func TestModelCapabilities(t *testing.T) {
// Create completion model (llama architecture without vision)
completionModelPath, _ := createBinFile(t, ggml.KV{
@@ -618,6 +638,8 @@ func TestModelCapabilities(t *testing.T) {
for _, tt := range testModels {
t.Run(tt.name, func(t *testing.T) {
+ loadTestMetadata(t, &tt.model)
+
// Test Capabilities method
caps := tt.model.Capabilities()
if !compareCapabilities(caps, tt.expectedCaps) {
@@ -748,6 +770,8 @@ func TestModelCheckCapabilities(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+ loadTestMetadata(t, &tt.model)
+
// Test CheckCapabilities method
err := tt.model.CheckCapabilities(tt.checkCaps...)
if tt.expectedErrMsg == "" {
diff --git a/server/model_caches.go b/server/model_caches.go
index 06f9e52b9..720fa703d 100644
--- a/server/model_caches.go
+++ b/server/model_caches.go
@@ -5,16 +5,12 @@ import "context"
type modelCaches struct {
recommendations *modelRecommendationsCache
show *modelShowCache
- modelList *modelListCache
- inference *inferenceModelCache
}
func newModelCaches() *modelCaches {
return &modelCaches{
recommendations: newModelRecommendationsCache(),
show: newModelShowCache(),
- modelList: newModelListCache(),
- inference: newInferenceModelCache(),
}
}
@@ -28,7 +24,4 @@ func (c *modelCaches) Start(ctx context.Context) {
if c.show != nil {
c.show.Start(ctx)
}
- if c.modelList != nil {
- c.modelList.Start(ctx)
- }
}
diff --git a/server/model_inference_cache.go b/server/model_inference_cache.go
deleted file mode 100644
index a5edcbd44..000000000
--- a/server/model_inference_cache.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package server
-
-import (
- "maps"
- "slices"
- "strconv"
- "sync"
-
- "github.com/ollama/ollama/envconfig"
- "github.com/ollama/ollama/manifest"
- "github.com/ollama/ollama/types/model"
- "golang.org/x/sync/singleflight"
-)
-
-// inferenceModelCache stores fully resolved model metadata and capabilities.
-// Model blobs are content-addressed, and the manifest digest is the freshness
-// boundary, so cached entries remain valid until the model is recreated or
-// pulled with different content.
-type inferenceModelCache struct {
- mu sync.RWMutex
- entries map[inferenceModelCacheKey]inferenceModelCacheEntry
- loads singleflight.Group
-
- loadModel func(string) (*Model, error)
-}
-
-type inferenceModelCacheKey struct {
- name string
- goTemplate bool
- goTemplateSet bool
-}
-
-type inferenceModelCacheEntry struct {
- digest string
- model *Model
-}
-
-func newInferenceModelCache() *inferenceModelCache {
- return &inferenceModelCache{
- entries: make(map[inferenceModelCacheKey]inferenceModelCacheEntry),
- loadModel: GetModel,
- }
-}
-
-func (c *inferenceModelCache) Get(name string) (*Model, error) {
- n := model.ParseName(name)
- mf, err := manifest.ParseNamedManifest(n)
- if err != nil {
- return nil, err
- }
-
- key := inferenceModelCacheKey{
- name: n.String(),
- goTemplate: envconfig.GoTemplate(true),
- goTemplateSet: goTemplateEnvSet(),
- }
- digest := mf.Digest()
-
- c.mu.RLock()
- entry, ok := c.entries[key]
- c.mu.RUnlock()
- if ok && entry.digest == digest {
- return cloneInferenceModel(entry.model), nil
- }
-
- loadKey := key.name + "\x00" + digest + "\x00" + strconv.FormatBool(key.goTemplate) + "\x00" + strconv.FormatBool(key.goTemplateSet)
- v, err, _ := c.loads.Do(loadKey, func() (any, error) {
- c.mu.RLock()
- entry, ok := c.entries[key]
- c.mu.RUnlock()
- if ok && entry.digest == digest {
- return entry.model, nil
- }
-
- m, err := c.loadModel(name)
- if err != nil {
- return nil, err
- }
- m.capabilities = m.Capabilities()
- m.capabilitiesCached = true
-
- c.mu.Lock()
- c.entries[key] = inferenceModelCacheEntry{digest: m.Digest, model: m}
- c.mu.Unlock()
- return m, nil
- })
- if err != nil {
- return nil, err
- }
-
- return cloneInferenceModel(v.(*Model)), nil
-}
-
-func cloneInferenceModel(src *Model) *Model {
- if src == nil {
- return nil
- }
-
- dst := *src
- dst.Config.ModelFamilies = slices.Clone(src.Config.ModelFamilies)
- dst.Config.Capabilities = slices.Clone(src.Config.Capabilities)
- if src.Config.Draft != nil {
- draft := *src.Config.Draft
- dst.Config.Draft = &draft
- }
- dst.AdapterPaths = slices.Clone(src.AdapterPaths)
- dst.ProjectorPaths = slices.Clone(src.ProjectorPaths)
- dst.License = slices.Clone(src.License)
- dst.Options = maps.Clone(src.Options)
- dst.Messages = slices.Clone(src.Messages)
- dst.capabilities = slices.Clone(src.capabilities)
-
- return &dst
-}
-
-func (s *Server) getModel(name string) (*Model, error) {
- if s != nil && s.modelCaches != nil && s.modelCaches.inference != nil {
- return s.modelCaches.inference.Get(name)
- }
- return GetModel(name)
-}
diff --git a/server/model_inference_cache_test.go b/server/model_inference_cache_test.go
deleted file mode 100644
index fb53202e2..000000000
--- a/server/model_inference_cache_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package server
-
-import (
- "slices"
- "sync"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/ollama/ollama/fs/ggml"
- "github.com/ollama/ollama/types/model"
-)
-
-func TestInferenceModelCache(t *testing.T) {
- t.Setenv("OLLAMA_MODELS", t.TempDir())
- t.Setenv("OLLAMA_GO_TEMPLATE", "")
-
- _, completionDigest := createBinFile(t, ggml.KV{
- "general.architecture": "llama",
- }, nil)
- writeTestModelManifest(t, "inference-cache", completionDigest, "{{ .Prompt }}")
-
- cache := newInferenceModelCache()
- loadCount := 0
- cache.loadModel = func(name string) (*Model, error) {
- loadCount++
- return GetModel(name)
- }
-
- first, err := cache.Get("inference-cache")
- if err != nil {
- t.Fatal(err)
- }
- if loadCount != 1 {
- t.Fatalf("load count = %d, want 1", loadCount)
- }
- if !first.capabilitiesCached {
- t.Fatal("capabilities were not cached")
- }
- if got := first.Capabilities(); !slices.Contains(got, model.CapabilityCompletion) {
- t.Fatalf("capabilities = %v, want completion", got)
- }
-
- // Returned models are request-local clones. Mutating one must not alter the
- // cached model or a later request.
- first.Config.Parser = "mutated"
- first.Config.ModelFamilies[0] = "mutated"
- first.capabilities[0] = model.CapabilityImage
-
- second, err := cache.Get("inference-cache")
- if err != nil {
- t.Fatal(err)
- }
- if loadCount != 1 {
- t.Fatalf("cache hit load count = %d, want 1", loadCount)
- }
- if second.Config.Parser == "mutated" || slices.Contains(second.Config.ModelFamilies, "mutated") {
- t.Fatalf("cached model was mutated: %#v", second.Config)
- }
- if got := second.Capabilities(); !slices.Contains(got, model.CapabilityCompletion) || slices.Contains(got, model.CapabilityImage) {
- t.Fatalf("cached capabilities = %v, want completion only", got)
- }
-
- // Recreating the manifest changes its digest and invalidates the entry.
- _, embeddingDigest := createBinFile(t, ggml.KV{
- "general.architecture": "bert",
- "bert.pooling_type": uint32(1),
- }, nil)
- writeTestModelManifest(t, "inference-cache", embeddingDigest, "{{ .Prompt }}")
-
- third, err := cache.Get("inference-cache")
- if err != nil {
- t.Fatal(err)
- }
- if loadCount != 2 {
- t.Fatalf("invalidated load count = %d, want 2", loadCount)
- }
- if got := third.Capabilities(); !slices.Contains(got, model.CapabilityEmbedding) || slices.Contains(got, model.CapabilityCompletion) {
- t.Fatalf("refreshed capabilities = %v, want embedding only", got)
- }
-}
-
-func TestInferenceModelCacheConcurrentMiss(t *testing.T) {
- t.Setenv("OLLAMA_MODELS", t.TempDir())
- t.Setenv("OLLAMA_GO_TEMPLATE", "")
-
- _, digest := createBinFile(t, ggml.KV{
- "general.architecture": "llama",
- }, nil)
- writeTestModelManifest(t, "inference-cache-concurrent", digest, "{{ .Prompt }}")
-
- cache := newInferenceModelCache()
- var loadCount atomic.Int32
- cache.loadModel = func(name string) (*Model, error) {
- loadCount.Add(1)
- time.Sleep(10 * time.Millisecond)
- return GetModel(name)
- }
-
- var wg sync.WaitGroup
- errs := make(chan error, 8)
- for range 8 {
- wg.Add(1)
- go func() {
- defer wg.Done()
- _, err := cache.Get("inference-cache-concurrent")
- errs <- err
- }()
- }
- wg.Wait()
- close(errs)
-
- for err := range errs {
- if err != nil {
- t.Fatal(err)
- }
- }
- if got := loadCount.Load(); got != 1 {
- t.Fatalf("load count = %d, want 1", got)
- }
-}
diff --git a/server/model_list.go b/server/model_list.go
new file mode 100644
index 000000000..237e1b63c
--- /dev/null
+++ b/server/model_list.go
@@ -0,0 +1,133 @@
+package server
+
+import (
+ "cmp"
+ "context"
+ "encoding/json"
+ "log/slog"
+ "slices"
+ "time"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/fs/ggml"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/types/model"
+)
+
+// listModels builds /api/tags from the manifests and the per-blob metadata
+// files, extracting for any blob that has none yet.
+func listModels(ctx context.Context) ([]api.ListModelResponse, error) {
+ manifests, err := manifest.Manifests(true)
+ if err != nil {
+ return nil, err
+ }
+
+ models := make([]api.ListModelResponse, 0, len(manifests))
+ for name, mf := range manifests {
+ if ctx != nil {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ }
+
+ summary, err := describeModel(name, mf)
+ if err != nil {
+ slog.Warn("failed to describe model", "model", name.String(), "error", err)
+ continue
+ }
+ models = append(models, summary)
+ }
+
+ sortListModelResponses(models)
+ return models, nil
+}
+
+// describeModel describes one model for /api/tags. Capabilities come from the
+// same Model.Capabilities() the inference path uses, so the two cannot drift.
+func describeModel(name model.Name, mf *manifest.Manifest) (api.ListModelResponse, error) {
+ cfg, err := readModelListConfig(mf)
+ if err != nil {
+ return api.ListModelResponse{}, err
+ }
+
+ var modified time.Time
+ if fi := mf.FileInfo(); fi != nil {
+ modified = fi.ModTime()
+ }
+
+ summary := api.ListModelResponse{
+ Model: name.DisplayShortest(),
+ Name: name.DisplayShortest(),
+ RemoteModel: cfg.RemoteModel,
+ RemoteHost: cfg.RemoteHost,
+ Size: mf.Size(),
+ Digest: mf.Digest(),
+ ModifiedAt: modified,
+ Details: api.ModelDetails{
+ Format: cfg.ModelFormat,
+ Family: cfg.ModelFamily,
+ Families: append([]string(nil), cfg.ModelFamilies...),
+ ParameterSize: cfg.ModelType,
+ QuantizationLevel: cfg.FileType,
+ ContextLength: cfg.ContextLen,
+ EmbeddingLength: cfg.EmbedLen,
+ },
+ }
+
+ m, err := GetModel(name.String())
+ if err != nil {
+ // A model that will not load is the one a user most needs to see, in
+ // order to remove it. Report what the manifest says.
+ slog.Warn("could not load model to describe it", "model", name.String(), "error", err)
+ return summary, nil
+ }
+ summary.Details.ParentModel = m.ParentModel
+ summary.Capabilities = m.Capabilities()
+
+ if m.ModelPath != "" && m.isGGUF() {
+ if summary.Details.ContextLength == 0 {
+ summary.Details.ContextLength = int(m.metadata.Int("context_length"))
+ }
+ if summary.Details.EmbeddingLength == 0 {
+ summary.Details.EmbeddingLength = int(m.metadata.Int("embedding_length"))
+ }
+ if m.metadata.Valid("general.file_type") {
+ fileType := ggml.FileType(m.metadata.Int("general.file_type")).String()
+ if isUnknownQuantization(summary.Details.QuantizationLevel) && !isUnknownQuantization(fileType) {
+ summary.Details.QuantizationLevel = fileType
+ }
+ }
+ }
+
+ return summary, nil
+}
+
+func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {
+ var cfg model.ConfigV2
+ if mf == nil || mf.Config.Digest == "" {
+ return cfg, nil
+ }
+
+ f, err := mf.Config.Open()
+ if err != nil {
+ return cfg, err
+ }
+ defer f.Close()
+
+ if err := json.NewDecoder(f).Decode(&cfg); err != nil {
+ return cfg, err
+ }
+
+ return cfg, nil
+}
+
+func isUnknownQuantization(quantization string) bool {
+ return quantization == "" || quantization == "unknown"
+}
+
+func sortListModelResponses(models []api.ListModelResponse) {
+ slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
+ // Preserve the existing /api/tags order: most recently modified first.
+ return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
+ })
+}
diff --git a/server/model_list_cache.go b/server/model_list_cache.go
deleted file mode 100644
index 9017aae70..000000000
--- a/server/model_list_cache.go
+++ /dev/null
@@ -1,888 +0,0 @@
-package server
-
-import (
- "bufio"
- "cmp"
- "context"
- "encoding/binary"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
- "os"
- "slices"
- "strings"
- "sync"
- "time"
-
- "github.com/ollama/ollama/api"
- "github.com/ollama/ollama/fs/ggml"
- fsgguf "github.com/ollama/ollama/fs/gguf"
- "github.com/ollama/ollama/manifest"
- "github.com/ollama/ollama/model/parsers"
- ollamatemplate "github.com/ollama/ollama/template"
- "github.com/ollama/ollama/thinking"
- "github.com/ollama/ollama/types/model"
-)
-
-type modelListSummary struct {
- Model string
- Name string
- RemoteModel string
- RemoteHost string
- Size int64
- Digest string
- ModifiedAt time.Time
- Details api.ModelDetails
- Capabilities []model.Capability
-}
-
-type modelListCacheEntry struct {
- Digest string
- Summary modelListSummary
-}
-
-type modelListCache struct {
- mu sync.RWMutex
-
- entries map[string]modelListCacheEntry
-
- once sync.Once
- readyOnce sync.Once
- ready chan struct{}
- hydrateErr error
- build func(model.Name, *manifest.Manifest) (modelListSummary, error)
-}
-
-func newModelListCache() *modelListCache {
- return &modelListCache{
- entries: make(map[string]modelListCacheEntry),
- ready: make(chan struct{}),
- build: buildModelListSummary,
- }
-}
-
-func (c *modelListCache) Start(ctx context.Context) {
- if c == nil {
- return
- }
-
- c.once.Do(func() {
- slog.Debug("starting model list cache")
- go func() {
- err := c.hydrate(ctx)
- c.markReady(err)
- if err != nil {
- if ctx != nil && ctx.Err() != nil {
- return
- }
- slog.Warn("model list cache hydration failed", "error", err)
- }
- }()
- })
-}
-
-func (c *modelListCache) hydrate(ctx context.Context) error {
- start := time.Now()
-
- manifests, err := manifest.Manifests(true)
- if err != nil {
- return err
- }
-
- var hydrated, failed int
- for name, mf := range manifests {
- if ctx != nil {
- if err := ctx.Err(); err != nil {
- return err
- }
- }
-
- summary, err := c.build(name, mf)
- if err != nil {
- failed++
- slog.Warn("failed to hydrate model list cache", "model", name.String(), "error", err)
- continue
- }
-
- c.set(name, mf.Digest(), summary)
- hydrated++
- }
-
- slog.Info("model list cache hydration complete", "models", hydrated, "failures", failed, "elapsed", time.Since(start))
- return nil
-}
-
-func (c *modelListCache) markReady(err error) {
- c.mu.Lock()
- c.hydrateErr = err
- c.mu.Unlock()
-
- c.readyOnce.Do(func() {
- close(c.ready)
- })
-}
-
-func (c *modelListCache) Wait(ctx context.Context) error {
- if c == nil {
- return nil
- }
- if ctx == nil {
- ctx = context.Background()
- }
-
- select {
- case <-c.ready:
- c.mu.RLock()
- err := c.hydrateErr
- c.mu.RUnlock()
- return err
- case <-ctx.Done():
- return ctx.Err()
- }
-}
-
-func (c *modelListCache) List(ctx context.Context) ([]api.ListModelResponse, error) {
- if err := c.Wait(ctx); err != nil {
- return nil, err
- }
- if err := c.syncManifests(ctx); err != nil {
- return nil, err
- }
-
- c.mu.RLock()
- models := make([]api.ListModelResponse, 0, len(c.entries))
- for _, entry := range c.entries {
- models = append(models, entry.Summary.ListModelResponse())
- }
- c.mu.RUnlock()
-
- sortListModelResponses(models)
- return models, nil
-}
-
-func (c *modelListCache) syncManifests(ctx context.Context) error {
- manifests, err := manifest.Manifests(true)
- if err != nil {
- return err
- }
-
- c.mu.RLock()
- current := make(map[string]string, len(c.entries))
- for name, entry := range c.entries {
- current[name] = entry.Digest
- }
- c.mu.RUnlock()
-
- type update struct {
- name model.Name
- digest string
- summary modelListSummary
- }
-
- seen := make(map[string]struct{}, len(manifests))
- stale := make(map[string]struct{})
- var updates []update
- for name, mf := range manifests {
- if ctx != nil {
- if err := ctx.Err(); err != nil {
- return err
- }
- }
-
- key := name.String()
- digest := mf.Digest()
- seen[key] = struct{}{}
- if current[key] == digest {
- continue
- }
-
- summary, err := c.build(name, mf)
- if err != nil {
- slog.Warn("failed to refresh model list cache", "model", key, "error", err)
- if _, ok := current[key]; ok {
- stale[key] = struct{}{}
- }
- continue
- }
- updates = append(updates, update{name: name, digest: digest, summary: summary})
- }
-
- c.mu.Lock()
- for name := range c.entries {
- if _, ok := seen[name]; !ok {
- delete(c.entries, name)
- continue
- }
- if _, ok := stale[name]; ok {
- delete(c.entries, name)
- }
- }
- for _, update := range updates {
- c.entries[update.name.String()] = modelListCacheEntry{
- Digest: update.digest,
- Summary: cloneModelListSummary(update.summary),
- }
- }
- c.mu.Unlock()
-
- return nil
-}
-
-func (c *modelListCache) RefreshModel(name model.Name) error {
- if c == nil {
- return nil
- }
-
- if !name.IsFullyQualified() {
- var err error
- name, err = getExistingName(name)
- if err != nil {
- return err
- }
- }
-
- mf, err := manifest.ParseNamedManifest(name)
- if err != nil {
- c.DeleteModel(name)
- return err
- }
-
- summary, err := c.build(name, mf)
- if err != nil {
- c.DeleteModel(name)
- return err
- }
-
- c.set(name, mf.Digest(), summary)
- return nil
-}
-
-func (c *modelListCache) DeleteModel(name model.Name) {
- if c == nil {
- return
- }
-
- c.mu.Lock()
- delete(c.entries, name.String())
- c.mu.Unlock()
-}
-
-func (c *modelListCache) Get(name model.Name) (modelListSummary, bool) {
- if c == nil {
- return modelListSummary{}, false
- }
-
- if !name.IsFullyQualified() {
- if existing, err := getExistingName(name); err == nil {
- name = existing
- }
- }
-
- c.mu.RLock()
- entry, ok := c.entries[name.String()]
- c.mu.RUnlock()
- if !ok {
- return modelListSummary{}, false
- }
-
- return cloneModelListSummary(entry.Summary), true
-}
-
-func (c *modelListCache) Len() int {
- if c == nil {
- return 0
- }
-
- c.mu.RLock()
- defer c.mu.RUnlock()
- return len(c.entries)
-}
-
-func (c *modelListCache) set(name model.Name, digest string, summary modelListSummary) {
- c.mu.Lock()
- c.entries[name.String()] = modelListCacheEntry{
- Digest: digest,
- Summary: cloneModelListSummary(summary),
- }
- c.mu.Unlock()
-}
-
-func buildModelListSummary(name model.Name, mf *manifest.Manifest) (modelListSummary, error) {
- cfg, err := readModelListConfig(mf)
- if err != nil {
- return modelListSummary{}, err
- }
-
- var modified time.Time
- if fi := mf.FileInfo(); fi != nil {
- modified = fi.ModTime()
- }
-
- summary := modelListSummary{
- Model: name.DisplayShortest(),
- Name: name.DisplayShortest(),
- RemoteModel: cfg.RemoteModel,
- RemoteHost: cfg.RemoteHost,
- Size: mf.Size(),
- Digest: mf.Digest(),
- ModifiedAt: modified,
- Details: api.ModelDetails{
- Format: cfg.ModelFormat,
- Family: cfg.ModelFamily,
- Families: append([]string(nil), cfg.ModelFamilies...),
- ParameterSize: cfg.ModelType,
- QuantizationLevel: cfg.FileType,
- ContextLength: cfg.ContextLen,
- EmbeddingLength: cfg.EmbedLen,
- },
- }
-
- modelPath, projectorCount, tmpl, err := readModelListLayers(mf, &summary)
- if err != nil {
- return modelListSummary{}, err
- }
-
- if cfg.RemoteHost == "" && cfg.RemoteModel == "" && modelPath != "" {
- info, err := readModelListGGUF(modelPath)
- if err != nil {
- slog.Debug("failed to read gguf model metadata", "model", name.String(), "error", err)
- } else {
- summary.Capabilities = appendModelListCapabilities(summary.Capabilities, info.Capabilities...)
- if summary.Details.ContextLength == 0 {
- summary.Details.ContextLength = info.ContextLength
- }
- if summary.Details.EmbeddingLength == 0 {
- summary.Details.EmbeddingLength = info.EmbeddingLength
- }
- if isUnknownQuantization(summary.Details.QuantizationLevel) && !isUnknownQuantization(info.FileType) {
- summary.Details.QuantizationLevel = info.FileType
- }
- }
- }
-
- for _, c := range cfg.Capabilities {
- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.Capability(c))
- }
-
- builtinParser := parsers.ParserForName(cfg.Parser)
- if tmpl != nil {
- vars, err := tmpl.Vars()
- if err != nil {
- slog.Warn("model template contains errors", "model", name.String(), "error", err)
- }
- if slices.Contains(vars, "tools") || (builtinParser != nil && builtinParser.HasToolSupport()) {
- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityTools)
- }
- if slices.Contains(vars, "suffix") {
- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityInsert)
- }
-
- openingTag, closingTag := thinking.InferTags(tmpl.Template)
- hasTags := openingTag != "" && closingTag != ""
- isGptoss := slices.Contains([]string{"gptoss", "gpt-oss"}, cfg.ModelFamily)
- if !slices.Contains(summary.Capabilities, model.CapabilityThinking) &&
- (hasTags || isGptoss || (builtinParser != nil && builtinParser.HasThinkingSupport())) {
- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityThinking)
- }
- }
-
- if projectorCount > 0 {
- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityVision)
- }
-
- summary.Capabilities = filterUnsupportedModelListCapabilities(summary.Capabilities, cfg)
-
- return summary, nil
-}
-
-func filterUnsupportedModelListCapabilities(capabilities []model.Capability, cfg model.ConfigV2) []model.Capability {
- if cfg.ModelFormat == "safetensors" && isNemotron3NanoSafetensorsConfig(cfg) {
- capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool {
- return c == model.CapabilityVision || c == model.CapabilityAudio
- })
- }
- // Mirrors suppressAudioCapability in images.go so /api/tags and /api/show
- // agree for safetensors models whose MLX runner serves vision but not audio.
- if cfg.ModelFormat == "safetensors" && cfg.Renderer == "glimmer" {
- capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool {
- return c == model.CapabilityAudio
- })
- }
-
- return capabilities
-}
-
-func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {
- var cfg model.ConfigV2
- if mf == nil || mf.Config.Digest == "" {
- return cfg, nil
- }
-
- f, err := mf.Config.Open()
- if err != nil {
- return cfg, err
- }
- defer f.Close()
-
- if err := json.NewDecoder(f).Decode(&cfg); err != nil {
- return cfg, err
- }
-
- return cfg, nil
-}
-
-func readModelListLayers(mf *manifest.Manifest, summary *modelListSummary) (string, int, *ollamatemplate.Template, error) {
- var modelPath string
- var projectorCount int
- tmpl := ollamatemplate.DefaultTemplate
-
- for _, layer := range mf.Layers {
- switch layer.MediaType {
- case "application/vnd.ollama.image.model":
- filename, err := manifest.BlobsPath(layer.Digest)
- if err != nil {
- return "", 0, nil, err
- }
- modelPath = filename
- summary.Details.ParentModel = layer.From
- case "application/vnd.ollama.image.projector":
- projectorCount++
- case "application/vnd.ollama.image.prompt",
- "application/vnd.ollama.image.template":
- filename, err := manifest.BlobsPath(layer.Digest)
- if err != nil {
- return "", 0, nil, err
- }
- bts, err := os.ReadFile(filename)
- if err != nil {
- return "", 0, nil, err
- }
-
- tmpl, err = ollamatemplate.Parse(string(bts))
- if err != nil {
- return "", 0, nil, err
- }
- }
- }
-
- return modelPath, projectorCount, tmpl, nil
-}
-
-type modelListGGUF struct {
- Capabilities []model.Capability
- ContextLength int
- EmbeddingLength int
- FileType string
-}
-
-const (
- modelListGGUFMagicLE = 0x46554747
- modelListGGUFMagicBE = 0x47475546
-)
-
-const (
- modelListGGUFTypeUint8 uint32 = iota
- modelListGGUFTypeInt8
- modelListGGUFTypeUint16
- modelListGGUFTypeInt16
- modelListGGUFTypeUint32
- modelListGGUFTypeInt32
- modelListGGUFTypeFloat32
- modelListGGUFTypeBool
- modelListGGUFTypeString
- modelListGGUFTypeArray
- modelListGGUFTypeUint64
- modelListGGUFTypeInt64
- modelListGGUFTypeFloat64
-)
-
-// readModelListGGUF scans only the small GGUF header values launch needs
-// and stops before tokenizer arrays. Using gguf.File.KeyValue for missing keys
-// can otherwise advance through large arrays just to discover absence.
-func readModelListGGUF(path string) (modelListGGUF, error) {
- f, err := os.Open(path)
- if err != nil {
- return modelListGGUF{}, err
- }
- defer f.Close()
-
- r := bufio.NewReaderSize(f, 32<<10)
- var magic uint32
- if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
- return modelListGGUF{}, err
- }
-
- var byteOrder binary.ByteOrder = binary.LittleEndian
- switch magic {
- case modelListGGUFMagicLE:
- case modelListGGUFMagicBE:
- byteOrder = binary.BigEndian
- default:
- return modelListGGUF{}, fmt.Errorf("invalid file magic")
- }
-
- var version uint32
- if err := binary.Read(r, byteOrder, &version); err != nil {
- return modelListGGUF{}, err
- }
-
- var numKV uint64
- switch version {
- case 1:
- var header struct {
- NumTensor uint32
- NumKV uint32
- }
- if err := binary.Read(r, byteOrder, &header); err != nil {
- return modelListGGUF{}, err
- }
- numKV = uint64(header.NumKV)
- default:
- var header struct {
- NumTensor uint64
- NumKV uint64
- }
- if err := binary.Read(r, byteOrder, &header); err != nil {
- return modelListGGUF{}, err
- }
- numKV = header.NumKV
- }
-
- info := modelListGGUF{}
- var architecture string
- var hasPoolingType bool
-
- for range numKV {
- key, err := readModelListGGUFString(r, byteOrder, version)
- if err != nil {
- return modelListGGUF{}, err
- }
-
- var valueType uint32
- if err := binary.Read(r, byteOrder, &valueType); err != nil {
- return modelListGGUF{}, err
- }
-
- if key == "general.architecture" {
- value, err := readModelListGGUFStringValue(r, byteOrder, version, valueType)
- if err != nil {
- return modelListGGUF{}, err
- }
- architecture = value
- continue
- }
-
- if key == "general.file_type" {
- value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
- if err != nil {
- return modelListGGUF{}, err
- }
- info.FileType = ggml.FileType(value).String()
- continue
- }
-
- if architecture != "" && strings.HasPrefix(key, "tokenizer.") {
- break
- }
-
- if architecture != "" && strings.HasPrefix(key, architecture+".") {
- switch strings.TrimPrefix(key, architecture+".") {
- case "pooling_type":
- hasPoolingType = true
- case "vision.block_count":
- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityVision)
- case "audio.block_count":
- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityAudio)
- case "context_length":
- value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
- if err != nil {
- return modelListGGUF{}, err
- }
- info.ContextLength = value
- continue
- case "embedding_length":
- value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)
- if err != nil {
- return modelListGGUF{}, err
- }
- info.EmbeddingLength = value
- continue
- }
- }
-
- if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
- return modelListGGUF{}, err
- }
- }
-
- if hasPoolingType {
- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityEmbedding)
- } else {
- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityCompletion)
- }
-
- return info, nil
-}
-
-func readModelListGGUFStringValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (string, error) {
- if valueType != modelListGGUFTypeString {
- if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
- return "", err
- }
- return "", fmt.Errorf("unexpected gguf string type %d", valueType)
- }
- return readModelListGGUFString(r, byteOrder, version)
-}
-
-func readModelListGGUFIntValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (int, error) {
- switch valueType {
- case modelListGGUFTypeUint8:
- var value uint8
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeInt8:
- var value int8
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeUint16:
- var value uint16
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeInt16:
- var value int16
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeUint32:
- var value uint32
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeInt32:
- var value int32
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeUint64:
- var value uint64
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- case modelListGGUFTypeInt64:
- var value int64
- if err := binary.Read(r, byteOrder, &value); err != nil {
- return 0, err
- }
- return int(value), nil
- default:
- if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {
- return 0, err
- }
- return 0, fmt.Errorf("unexpected gguf integer type %d", valueType)
- }
-}
-
-func skipModelListGGUFValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) error {
- switch valueType {
- case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:
- return discardModelListGGUFBytes(r, 1)
- case modelListGGUFTypeUint16, modelListGGUFTypeInt16:
- return discardModelListGGUFBytes(r, 2)
- case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:
- return discardModelListGGUFBytes(r, 4)
- case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:
- return discardModelListGGUFBytes(r, 8)
- case modelListGGUFTypeString:
- return skipModelListGGUFString(r, byteOrder, version)
- case modelListGGUFTypeArray:
- var arrayType uint32
- if err := binary.Read(r, byteOrder, &arrayType); err != nil {
- return err
- }
- var count uint64
- if err := binary.Read(r, byteOrder, &count); err != nil {
- return err
- }
- return skipModelListGGUFArray(r, byteOrder, version, arrayType, count)
- default:
- return fmt.Errorf("unsupported gguf value type %d", valueType)
- }
-}
-
-func skipModelListGGUFArray(r io.Reader, byteOrder binary.ByteOrder, version uint32, arrayType uint32, count uint64) error {
- if _, err := checkedModelListGGUFLength(count, "array size", fsgguf.MaxArraySize); err != nil {
- return err
- }
-
- var size uint64
- switch arrayType {
- case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:
- size = 1
- case modelListGGUFTypeUint16, modelListGGUFTypeInt16:
- size = 2
- case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:
- size = 4
- case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:
- size = 8
- case modelListGGUFTypeString:
- for range count {
- if err := skipModelListGGUFString(r, byteOrder, version); err != nil {
- return err
- }
- }
- return nil
- default:
- return fmt.Errorf("unsupported gguf array type %d", arrayType)
- }
-
- if count > uint64(maxModelListGGUFInt64())/size {
- return fmt.Errorf("gguf array byte length %d*%d exceeds maximum %d", count, size, maxModelListGGUFInt64())
- }
- return discardModelListGGUFBytes(r, int64(count*size))
-}
-
-func readModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) (string, error) {
- var length uint64
- if err := binary.Read(r, byteOrder, &length); err != nil {
- return "", err
- }
-
- n, err := checkedModelListGGUFLength(length, "string", fsgguf.MaxStringLength)
- if err != nil {
- return "", err
- }
-
- bts := make([]byte, n)
- if _, err := io.ReadFull(r, bts); err != nil {
- return "", err
- }
- if version == 1 && len(bts) > 0 && bts[len(bts)-1] == 0 {
- bts = bts[:len(bts)-1]
- }
- return string(bts), nil
-}
-
-func skipModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) error {
- var length uint64
- if err := binary.Read(r, byteOrder, &length); err != nil {
- return err
- }
-
- n, err := checkedModelListGGUFLength(length, "string", fsgguf.MaxStringLength)
- if err != nil {
- return err
- }
- return discardModelListGGUFBytes(r, int64(n))
-}
-
-func discardModelListGGUFBytes(r io.Reader, n int64) error {
- if n <= 0 {
- return nil
- }
- _, err := io.CopyN(io.Discard, r, n)
- return err
-}
-
-func checkedModelListGGUFLength(n uint64, kind string, max uint64) (int, error) {
- if n > uint64(maxModelListGGUFInt()) {
- return 0, fmt.Errorf("gguf %s %d exceeds maximum %d", kind, n, maxModelListGGUFInt())
- }
- if n > max {
- return 0, fmt.Errorf("gguf %s %d exceeds maximum %d", kind, n, max)
- }
- return int(n), nil
-}
-
-func maxModelListGGUFInt() int {
- return int(^uint(0) >> 1)
-}
-
-func maxModelListGGUFInt64() int64 {
- return 1<<63 - 1
-}
-
-func appendModelListCapabilities(capabilities []model.Capability, values ...model.Capability) []model.Capability {
- for _, capability := range values {
- capabilities = appendModelListCapability(capabilities, capability)
- }
- return capabilities
-}
-
-func appendModelListCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {
- if capability == "" || slices.Contains(capabilities, capability) {
- return capabilities
- }
- return append(capabilities, capability)
-}
-
-func isUnknownQuantization(quantization string) bool {
- return quantization == "" || quantization == "unknown"
-}
-
-func cloneModelListSummary(summary modelListSummary) modelListSummary {
- summary.Details.Families = append([]string(nil), summary.Details.Families...)
- summary.Capabilities = append([]model.Capability(nil), summary.Capabilities...)
- return summary
-}
-
-func (s modelListSummary) ListModelResponse() api.ListModelResponse {
- resp := api.ListModelResponse{
- Model: s.Model,
- Name: s.Name,
- RemoteModel: s.RemoteModel,
- RemoteHost: s.RemoteHost,
- Size: s.Size,
- Digest: s.Digest,
- ModifiedAt: s.ModifiedAt,
- Details: api.ModelDetails{
- ParentModel: s.Details.ParentModel,
- Format: s.Details.Format,
- Family: s.Details.Family,
- Families: append([]string(nil), s.Details.Families...),
- ParameterSize: s.Details.ParameterSize,
- QuantizationLevel: s.Details.QuantizationLevel,
- ContextLength: s.Details.ContextLength,
- EmbeddingLength: s.Details.EmbeddingLength,
- },
- }
-
- resp.Capabilities = append([]model.Capability(nil), s.Capabilities...)
-
- return resp
-}
-
-func sortListModelResponses(models []api.ListModelResponse) {
- slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
- // Preserve the existing /api/tags order: most recently modified first.
- return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
- })
-}
-
-func (s *Server) refreshModelListCache(name model.Name) {
- if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {
- return
- }
-
- if err := s.modelCaches.modelList.RefreshModel(name); err != nil {
- slog.Warn("failed to refresh model list cache", "model", name.String(), "error", err)
- }
-}
-
-func (s *Server) deleteModelListCache(name model.Name) {
- if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {
- return
- }
-
- s.modelCaches.modelList.DeleteModel(name)
-}
diff --git a/server/model_list_cache_test.go b/server/model_list_cache_test.go
deleted file mode 100644
index f9fb960cd..000000000
--- a/server/model_list_cache_test.go
+++ /dev/null
@@ -1,371 +0,0 @@
-package server
-
-import (
- "bytes"
- "context"
- "encoding/binary"
- "errors"
- "net/http"
- "os"
- "slices"
- "strings"
- "testing"
-
- "github.com/gin-gonic/gin"
-
- "github.com/ollama/ollama/api"
- fsgguf "github.com/ollama/ollama/fs/gguf"
- "github.com/ollama/ollama/manifest"
- "github.com/ollama/ollama/types/model"
-)
-
-func TestModelListCacheHydratesSummary(t *testing.T) {
- gin.SetMode(gin.TestMode)
- setTestHome(t, t.TempDir())
- createListCacheModel(t, "list-cache", map[string]any{
- "test.context_length": uint32(4096),
- "test.embedding_length": uint32(384),
- }, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")
-
- cache := newModelListCache()
- if err := cache.hydrate(context.Background()); err != nil {
- t.Fatalf("hydrate failed: %v", err)
- }
-
- summary, ok := cache.Get(model.ParseName("list-cache"))
- if !ok {
- t.Fatal("list summary missing")
- }
-
- if summary.Model != "list-cache:latest" || summary.Name != "list-cache:latest" {
- t.Fatalf("summary model/name = %q/%q, want list-cache:latest", summary.Model, summary.Name)
- }
- if summary.Digest == "" {
- t.Fatal("summary digest is empty")
- }
- if summary.Size == 0 {
- t.Fatal("summary size is zero")
- }
- if summary.Details.Family != "test" || summary.Details.Format != "gguf" {
- t.Fatalf("summary details = %+v, want gguf/test", summary.Details)
- }
- if summary.Details.ContextLength != 4096 {
- t.Fatalf("context length = %d, want 4096", summary.Details.ContextLength)
- }
- if summary.Details.EmbeddingLength != 384 {
- t.Fatalf("embedding length = %d, want 384", summary.Details.EmbeddingLength)
- }
-
- for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {
- if !slices.Contains(summary.Capabilities, capability) {
- t.Fatalf("capabilities = %v, want %s", summary.Capabilities, capability)
- }
- }
-
- listModel := summary.ListModelResponse()
- if !slices.Contains(listModel.Capabilities, model.CapabilityTools) ||
- listModel.Details.ContextLength != 4096 ||
- listModel.Details.EmbeddingLength != 384 {
- t.Fatalf("list response = %+v, want capabilities/context/embedding", listModel)
- }
-}
-
-func TestModelListCacheSuppressesNemotronSafetensorsMedia(t *testing.T) {
- caps := []model.Capability{
- model.CapabilityCompletion,
- model.CapabilityTools,
- model.CapabilityThinking,
- model.CapabilityVision,
- model.CapabilityAudio,
- }
- got := filterUnsupportedModelListCapabilities(caps, model.ConfigV2{
- ModelFormat: "safetensors",
- Renderer: "nemotron-3-nano",
- Parser: "nemotron-3-nano",
- })
-
- for _, capability := range []model.Capability{
- model.CapabilityCompletion,
- model.CapabilityTools,
- model.CapabilityThinking,
- } {
- if !slices.Contains(got, capability) {
- t.Fatalf("capabilities = %v, want %s", got, capability)
- }
- }
- for _, capability := range []model.Capability{model.CapabilityVision, model.CapabilityAudio} {
- if slices.Contains(got, capability) {
- t.Fatalf("capabilities = %v, did not expect %s", got, capability)
- }
- }
-}
-
-func TestModelListCacheRefreshUpdatesEntry(t *testing.T) {
- gin.SetMode(gin.TestMode)
- setTestHome(t, t.TempDir())
- createListCacheModel(t, "list-refresh", map[string]any{"test.context_length": uint32(1024)}, "")
-
- cache := newModelListCache()
- if err := cache.hydrate(context.Background()); err != nil {
- t.Fatalf("hydrate failed: %v", err)
- }
-
- name := model.ParseName("list-refresh")
- first, ok := cache.Get(name)
- if !ok {
- t.Fatal("list summary missing")
- }
-
- changeShowCacheManifest(t, "list-refresh")
- if err := cache.RefreshModel(name); err != nil {
- t.Fatalf("refresh failed: %v", err)
- }
-
- refreshed, ok := cache.Get(name)
- if !ok {
- t.Fatal("refreshed list summary missing")
- }
- if refreshed.Digest == first.Digest {
- t.Fatalf("digest did not change after refresh: %s", refreshed.Digest)
- }
- if cache.Len() != 1 {
- t.Fatalf("cache entries = %d, want 1", cache.Len())
- }
-}
-
-func TestModelListCacheMutationHooks(t *testing.T) {
- gin.SetMode(gin.TestMode)
- setTestHome(t, t.TempDir())
-
- cache := newModelListCache()
- s := Server{modelCaches: &modelCaches{modelList: cache}}
-
- _, digest := createBinFile(t, map[string]any{"test.context_length": uint32(2048)}, nil)
- w := createRequest(t, s.CreateHandler, api.CreateRequest{
- Model: "list-hooks",
- Files: map[string]string{"model.gguf": digest},
- Stream: &stream,
- })
- if w.Code != http.StatusOK {
- t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())
- }
-
- if _, ok := cache.Get(model.ParseName("list-hooks")); !ok {
- t.Fatal("create did not refresh model list cache")
- }
-
- w = createRequest(t, s.CopyHandler, api.CopyRequest{
- Source: "list-hooks",
- Destination: "list-hooks-copy",
- })
- if w.Code != http.StatusOK {
- t.Fatalf("copy model status = %d, want 200: %s", w.Code, w.Body.String())
- }
- if _, ok := cache.Get(model.ParseName("list-hooks-copy")); !ok {
- t.Fatal("copy did not refresh model list cache")
- }
-
- w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: "list-hooks-copy"})
- if w.Code != http.StatusOK {
- t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())
- }
- if _, ok := cache.Get(model.ParseName("list-hooks-copy")); ok {
- t.Fatal("delete did not remove model list cache entry")
- }
-}
-
-func TestModelListCacheSyncsManifestChanges(t *testing.T) {
- gin.SetMode(gin.TestMode)
- setTestHome(t, t.TempDir())
- createListCacheModel(t, "list-sync-a", map[string]any{"test.context_length": uint32(1024)}, "")
-
- cache := newModelListCache()
- cache.Start(context.Background())
- if err := cache.Wait(context.Background()); err != nil {
- t.Fatalf("wait failed: %v", err)
- }
-
- createListCacheModel(t, "list-sync-b", map[string]any{"test.context_length": uint32(2048)}, "")
- models, err := cache.List(context.Background())
- if err != nil {
- t.Fatalf("list failed: %v", err)
- }
-
- names := make([]string, 0, len(models))
- for _, m := range models {
- names = append(names, m.Name)
- }
- for _, want := range []string{"list-sync-a:latest", "list-sync-b:latest"} {
- if !slices.Contains(names, want) {
- t.Fatalf("names = %v, want %s", names, want)
- }
- }
-
- var other Server
- w := createRequest(t, other.DeleteHandler, api.DeleteRequest{Model: "list-sync-a"})
- if w.Code != http.StatusOK {
- t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())
- }
-
- models, err = cache.List(context.Background())
- if err != nil {
- t.Fatalf("list after delete failed: %v", err)
- }
- names = names[:0]
- for _, m := range models {
- names = append(names, m.Name)
- }
- if slices.Contains(names, "list-sync-a:latest") || !slices.Contains(names, "list-sync-b:latest") {
- t.Fatalf("names after delete = %v, want only list-sync-b", names)
- }
-}
-
-func TestModelListCacheSyncDropsStaleEntryOnRefreshFailure(t *testing.T) {
- gin.SetMode(gin.TestMode)
- setTestHome(t, t.TempDir())
- createListCacheModel(t, "list-stale", map[string]any{"test.context_length": uint32(1024)}, "")
-
- cache := newModelListCache()
- cache.Start(context.Background())
- if err := cache.Wait(context.Background()); err != nil {
- t.Fatalf("wait failed: %v", err)
- }
-
- name := model.ParseName("list-stale")
- if _, ok := cache.Get(name); !ok {
- t.Fatal("list summary missing")
- }
-
- changeShowCacheManifest(t, "list-stale")
- cache.build = func(model.Name, *manifest.Manifest) (modelListSummary, error) {
- return modelListSummary{}, errors.New("refresh failed")
- }
-
- models, err := cache.List(context.Background())
- if err != nil {
- t.Fatalf("list failed: %v", err)
- }
- if len(models) != 0 {
- t.Fatalf("models = %+v, want stale entry removed", models)
- }
- if _, ok := cache.Get(name); ok {
- t.Fatal("stale entry remained in cache after refresh failure")
- }
-}
-
-func TestReadModelListGGUFRejectsMalformedMetadata(t *testing.T) {
- cases := []struct {
- name string
- data []byte
- want string
- }{
- {
- name: "oversized key string",
- data: modelListGGUFTestFile(func(b *bytes.Buffer) {
- writeModelListGGUFHeader(t, b, 1)
- writeModelListGGUFUint64(t, b, fsgguf.MaxStringLength+1)
- }),
- want: "string",
- },
- {
- name: "oversized skipped string",
- data: modelListGGUFTestFile(func(b *bytes.Buffer) {
- writeModelListGGUFHeader(t, b, 1)
- writeModelListGGUFString(t, b, "unused")
- writeModelListGGUFUint32(t, b, modelListGGUFTypeString)
- writeModelListGGUFUint64(t, b, fsgguf.MaxStringLength+1)
- }),
- want: "string",
- },
- {
- name: "oversized skipped array",
- data: modelListGGUFTestFile(func(b *bytes.Buffer) {
- writeModelListGGUFHeader(t, b, 1)
- writeModelListGGUFString(t, b, "unused")
- writeModelListGGUFUint32(t, b, modelListGGUFTypeArray)
- writeModelListGGUFUint32(t, b, modelListGGUFTypeUint8)
- writeModelListGGUFUint64(t, b, fsgguf.MaxArraySize+1)
- }),
- want: "array size",
- },
- }
-
- for _, tt := range cases {
- t.Run(tt.name, func(t *testing.T) {
- defer func() {
- if r := recover(); r != nil {
- t.Fatalf("readModelListGGUF panicked: %v", r)
- }
- }()
-
- path := t.TempDir() + "/model.gguf"
- if err := os.WriteFile(path, tt.data, 0o600); err != nil {
- t.Fatal(err)
- }
-
- _, err := readModelListGGUF(path)
- if err == nil {
- t.Fatal("expected error")
- }
- if !strings.Contains(err.Error(), tt.want) {
- t.Fatalf("error = %v, want substring %q", err, tt.want)
- }
- })
- }
-}
-
-func createListCacheModel(t *testing.T, name string, kv map[string]any, tmpl string) {
- t.Helper()
- _, digest := createBinFile(t, kv, nil)
-
- req := api.CreateRequest{
- Model: name,
- Files: map[string]string{"model.gguf": digest},
- Stream: &stream,
- }
- if tmpl != "" {
- req.Template = tmpl
- }
-
- var s Server
- w := createRequest(t, s.CreateHandler, req)
- if w.Code != http.StatusOK {
- t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())
- }
-}
-
-func modelListGGUFTestFile(fn func(*bytes.Buffer)) []byte {
- var b bytes.Buffer
- fn(&b)
- return b.Bytes()
-}
-
-func writeModelListGGUFHeader(t *testing.T, b *bytes.Buffer, numKV uint64) {
- t.Helper()
- writeModelListGGUFUint32(t, b, modelListGGUFMagicLE)
- writeModelListGGUFUint32(t, b, 3)
- writeModelListGGUFUint64(t, b, 0)
- writeModelListGGUFUint64(t, b, numKV)
-}
-
-func writeModelListGGUFString(t *testing.T, b *bytes.Buffer, s string) {
- t.Helper()
- writeModelListGGUFUint64(t, b, uint64(len(s)))
- if _, err := b.WriteString(s); err != nil {
- t.Fatal(err)
- }
-}
-
-func writeModelListGGUFUint32(t *testing.T, b *bytes.Buffer, v uint32) {
- t.Helper()
- if err := binary.Write(b, binary.LittleEndian, v); err != nil {
- t.Fatal(err)
- }
-}
-
-func writeModelListGGUFUint64(t *testing.T, b *bytes.Buffer, v uint64) {
- t.Helper()
- if err := binary.Write(b, binary.LittleEndian, v); err != nil {
- t.Fatal(err)
- }
-}
diff --git a/server/model_list_test.go b/server/model_list_test.go
new file mode 100644
index 000000000..44fd83036
--- /dev/null
+++ b/server/model_list_test.go
@@ -0,0 +1,168 @@
+package server
+
+import (
+ "context"
+ "os"
+ "slices"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/manifest"
+ "github.com/ollama/ollama/types/model"
+)
+
+func listedModel(t *testing.T, name string) api.ListModelResponse {
+ t.Helper()
+ models, err := listModels(context.Background())
+ if err != nil {
+ t.Fatalf("listModels failed: %v", err)
+ }
+ for _, m := range models {
+ if m.Name == name {
+ return m
+ }
+ }
+ t.Fatalf("%s not listed; got %v", name, models)
+ return api.ListModelResponse{}
+}
+
+func TestListModelsDescribesModel(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListedModelFromKV(t, "list-describe", map[string]any{
+ "test.context_length": uint32(4096),
+ "test.embedding_length": uint32(384),
+ }, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")
+
+ got := listedModel(t, "list-describe:latest")
+
+ if got.Model != "list-describe:latest" {
+ t.Errorf("model = %q", got.Model)
+ }
+ if got.Digest == "" || got.Size == 0 {
+ t.Errorf("digest = %q size = %d, want both set", got.Digest, got.Size)
+ }
+ if got.Details.Family != "test" || got.Details.Format != "gguf" {
+ t.Errorf("details = %+v, want gguf/test", got.Details)
+ }
+ if got.Details.ContextLength != 4096 {
+ t.Errorf("context length = %d, want 4096", got.Details.ContextLength)
+ }
+ if got.Details.EmbeddingLength != 384 {
+ t.Errorf("embedding length = %d, want 384", got.Details.EmbeddingLength)
+ }
+ // The test GGUF has no general.file_type; the list must not guess F32
+ // (FileType 0) from a missing key.
+ if !isUnknownQuantization(got.Details.QuantizationLevel) {
+ t.Errorf("quantization = %q, want unknown (no general.file_type in GGUF)", got.Details.QuantizationLevel)
+ }
+ for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {
+ if !slices.Contains(got.Capabilities, capability) {
+ t.Errorf("capabilities = %v, want %s", got.Capabilities, capability)
+ }
+ }
+}
+
+// Listing reads the manifests every time, so a model created or deleted after
+// the last call shows up without anything needing to be told about it.
+func TestListModelsFollowsManifestChanges(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListedModelFromKV(t, "list-follow-a", map[string]any{"test.context_length": uint32(1024)}, "")
+
+ listedModel(t, "list-follow-a:latest")
+
+ createListedModelFromKV(t, "list-follow-b", map[string]any{"test.context_length": uint32(2048)}, "")
+ listedModel(t, "list-follow-a:latest")
+ listedModel(t, "list-follow-b:latest")
+
+ deleteModelNamed(t, "list-follow-a")
+
+ models, err := listModels(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+ names := make([]string, 0, len(models))
+ for _, m := range models {
+ names = append(names, m.Name)
+ }
+ if slices.Contains(names, "list-follow-a:latest") || !slices.Contains(names, "list-follow-b:latest") {
+ t.Fatalf("names after delete = %v, want only list-follow-b", names)
+ }
+}
+
+func TestCapabilitiesSuppressNemotronSafetensorsMedia(t *testing.T) {
+ caps := []model.Capability{
+ model.CapabilityCompletion,
+ model.CapabilityTools,
+ model.CapabilityThinking,
+ model.CapabilityVision,
+ model.CapabilityAudio,
+ }
+ got := (&Model{Config: model.ConfigV2{
+ ModelFormat: "safetensors",
+ Renderer: "nemotron-3-nano",
+ Parser: "nemotron-3-nano",
+ }}).filterUnsupportedCapabilities(caps, "")
+
+ for _, capability := range []model.Capability{
+ model.CapabilityCompletion,
+ model.CapabilityTools,
+ model.CapabilityThinking,
+ } {
+ if !slices.Contains(got, capability) {
+ t.Errorf("capabilities = %v, want %s", got, capability)
+ }
+ }
+ for _, capability := range []model.Capability{model.CapabilityVision, model.CapabilityAudio} {
+ if slices.Contains(got, capability) {
+ t.Errorf("capabilities = %v, did not expect %s", got, capability)
+ }
+ }
+}
+
+// A model whose layers no longer parse must still be listed: it is the one the
+// user most needs to see, in order to remove it.
+func TestListModelsKeepsUnloadableModel(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ setTestHome(t, t.TempDir())
+ createListedModelFromKV(t, "broken", map[string]any{"test.context_length": uint32(1024)}, "{{ .Prompt }}")
+
+ mf, err := manifest.ParseNamedManifest(model.ParseName("broken"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var corrupted bool
+ for _, layer := range mf.Layers {
+ if layer.MediaType != "application/vnd.ollama.image.template" {
+ continue
+ }
+ path, err := manifest.BlobsPath(layer.Digest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte("{{ if }"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ corrupted = true
+ }
+ if !corrupted {
+ t.Fatal("no template layer to corrupt")
+ }
+ if _, err := GetModel("broken"); err == nil {
+ t.Fatal("model still loads, so the listing is not being asked the question")
+ }
+
+ got := listedModel(t, "broken:latest")
+ if got.Details.Family != "test" {
+ t.Errorf("details = %+v, want the manifest config's family", got.Details)
+ }
+}
+
+func createListedModelFromKV(t *testing.T, name string, kv map[string]any, tmpl string) {
+ t.Helper()
+ _, digest := createBinFile(t, kv, nil)
+ createModelFromBlob(t, name, digest, tmpl)
+}
diff --git a/server/routes.go b/server/routes.go
index 9b73139a1..177a7441f 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -286,7 +286,7 @@ func (s *Server) GenerateHandler(c *gin.Context) {
return
}
- m, err := s.getModel(name.String())
+ m, err := GetModel(name.String())
if err != nil {
switch {
case errors.Is(err, fs.ErrNotExist):
@@ -850,7 +850,7 @@ func (s *Server) EmbedHandler(c *gin.Context) {
return
}
- m, err := s.getModel(name.String())
+ m, err := GetModel(name.String())
if err != nil {
handleScheduleError(c, req.Model, err)
return
@@ -1068,7 +1068,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
name := modelRef.Name
- m, err := s.getModel(name.String())
+ m, err := GetModel(name.String())
if err != nil {
handleScheduleError(c, req.Model, err)
return
@@ -1153,8 +1153,6 @@ func (s *Server) PullHandler(c *gin.Context) {
ch <- gin.H{"error": err.Error()}
return
}
-
- s.refreshModelListCache(name)
}()
if req.Stream != nil && !*req.Stream {
@@ -1294,9 +1292,9 @@ func (s *Server) DeleteHandler(c *gin.Context) {
return
}
- s.deleteModelListCache(n)
-
- if err := m.RemoveLayers(); err != nil {
+ removed, err := m.RemoveLayers()
+ removeGGUFMetadata(removed...)
+ if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
@@ -1635,12 +1633,7 @@ func selectedModelTemplate(m *Model, kv ggml.KV) string {
}
func (s *Server) ListHandler(c *gin.Context) {
- if s.modelCaches == nil || s.modelCaches.modelList == nil {
- c.JSON(http.StatusInternalServerError, gin.H{"error": "model list cache unavailable"})
- return
- }
-
- models, err := s.modelCaches.modelList.List(c.Request.Context())
+ models, err := listModels(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@@ -1685,8 +1678,6 @@ func (s *Server) CopyHandler(c *gin.Context) {
c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
} else if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
- } else {
- s.refreshModelListCache(dst)
}
}
@@ -2492,7 +2483,7 @@ func (s *Server) ChatHandler(c *gin.Context) {
return
}
- m, err := s.getModel(name.String())
+ m, err := GetModel(name.String())
if err != nil {
switch {
case os.IsNotExist(err):
diff --git a/server/routes_list_test.go b/server/routes_list_test.go
index eb409c8b0..9417c6292 100644
--- a/server/routes_list_test.go
+++ b/server/routes_list_test.go
@@ -1,7 +1,6 @@
package server
import (
- "context"
"encoding/json"
"io"
"net/http"
@@ -36,11 +35,7 @@ func TestList(t *testing.T) {
"myhost/mynamespace/lips:code",
}
- s := Server{modelCaches: &modelCaches{modelList: newModelListCache()}}
- s.modelCaches.modelList.Start(context.Background())
- if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {
- t.Fatal(err)
- }
+ var s Server
for _, n := range expectNames {
_, digest := createBinFile(t, nil, nil)
@@ -88,12 +83,7 @@ func TestOpenAIListMatchesTagsModels(t *testing.T) {
gin.SetMode(gin.TestMode)
setTestHome(t, t.TempDir())
- cache := newModelListCache()
- s := Server{modelCaches: &modelCaches{modelList: cache}}
- cache.Start(context.Background())
- if err := cache.Wait(context.Background()); err != nil {
- t.Fatal(err)
- }
+ var s Server
createModel := func(name string) {
t.Helper()
@@ -123,9 +113,6 @@ func TestOpenAIListMatchesTagsModels(t *testing.T) {
if err := os.Chtimes(path, modified, modified); err != nil {
t.Fatalf("set manifest time for %s: %v", name, err)
}
- if err := cache.RefreshModel(parsed); err != nil {
- t.Fatalf("refresh %s: %v", name, err)
- }
}
older := time.Unix(1000, 0).UTC()
diff --git a/server/routes_test.go b/server/routes_test.go
index 8164454e9..44639f6f0 100644
--- a/server/routes_test.go
+++ b/server/routes_test.go
@@ -95,11 +95,7 @@ func TestRoutes(t *testing.T) {
Expected func(t *testing.T, resp *http.Response)
}
- s := &Server{modelCaches: &modelCaches{modelList: newModelListCache()}}
- s.modelCaches.modelList.Start(context.Background())
- if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {
- t.Fatal(err)
- }
+ s := &Server{}
createTestModel := func(t *testing.T, name string) {
t.Helper()
@@ -135,7 +131,6 @@ func TestRoutes(t *testing.T) {
if err := createModel(r, modelName, baseLayers, config, fn); err != nil {
t.Fatal(err)
}
- s.refreshModelListCache(modelName)
}
testCases := []testCase{