Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
server: extract GGUF metadata and unify capabilities (#17858)
Daniel Hiltgenea8d65004aff
diff --git a/fs/gguf/gguf.go b/fs/gguf/gguf.goindex 4468a3140..d9bb876ec 100644--- a/fs/gguf/gguf.go+++ b/fs/gguf/gguf.go@@ -51,12 +51,19 @@ type File struct {bts []byte}-func Open(path string) (f *File, err error) {- f = &File{bts: make([]byte, 4096)}+func Open(path string) (_ *File, err error) {+ f := &File{bts: make([]byte, 4096)}f.file, err = os.Open(path)if err != nil {return nil, err}+ defer func() {+ if err != nil {+ if closeErr := f.close(); closeErr != nil {+ err = errors.Join(err, closeErr)+ }+ }+ }()f.reader = newBufferedReader(f.file, 32<<10)@@ -273,14 +280,14 @@ func readArrayData[T any](f *File, n uint64) (s []T, err error) {return nil, err}- s = make([]T, size)- for i := range size {+ s = make([]T, 0, min(size, 4096))+ for range size {e, err := read[T](f)if err != nil {return nil, err}- s[i] = e+ s = append(s, e)}return s, nil@@ -292,14 +299,14 @@ func readArrayString(f *File, n uint64) (s []string, err error) {return nil, err}- s = make([]string, size)- for i := range size {+ s = make([]string, 0, min(size, 4096))+ for range size {e, err := readString(f)if err != nil {return nil, err}- s[i] = e+ s = append(s, e)}return s, nil@@ -324,8 +331,19 @@ func maxInt64() uint64 {}func (f *File) Close() error {- f.keyValues.stop()- f.tensors.stop()+ return f.close()+}++func (f *File) close() error {+ if f.keyValues != nil {+ f.keyValues.stop()+ }+ if f.tensors != nil {+ f.tensors.stop()+ }+ if f.file == nil {+ return nil+ }return f.file.Close()}diff --git a/fs/gguf/keyvalue.go b/fs/gguf/keyvalue.goindex 474201165..f2f2b7f24 100644--- a/fs/gguf/keyvalue.go+++ b/fs/gguf/keyvalue.go@@ -18,6 +18,11 @@ type Value struct {value any}+// Any returns Value as stored, without conversion. If it is not set, it returns nil.+func (v Value) Any() any {+ return v.value+}+func value[T any](v Value, kinds ...reflect.Kind) (t T) {vv := reflect.ValueOf(v.value)if slices.Contains(kinds, vv.Kind()) {diff --git a/manifest/manifest.go b/manifest/manifest.goindex c0277e9a5..d010ead73 100644--- a/manifest/manifest.go+++ b/manifest/manifest.go@@ -71,10 +71,13 @@ func (m *Manifest) Remove() error {return PruneDirectory(manifests)}-func (m *Manifest) RemoveLayers() error {+// RemoveLayers deletes the layers no other manifest references. The digests it+// removed are returned even alongside an error, so callers can clean up+// anything derived from them.+func (m *Manifest) RemoveLayers() ([]string, error) {ms, err := Manifests(true)if err != nil {- return err+ return nil, err}// Build set of digests still in use by other manifests@@ -88,6 +91,7 @@ func (m *Manifest) RemoveLayers() error {}// Remove layers not used by any other manifest+ var removed []stringfor _, layer := range append(m.Layers, m.Config) {if layer.Digest == "" {continue@@ -97,16 +101,17 @@ func (m *Manifest) RemoveLayers() error {}blob, err := BlobsPath(layer.Digest)if err != nil {- return err+ return removed, err}if err := os.Remove(blob); os.IsNotExist(err) {slog.Debug("layer does not exist", "digest", layer.Digest)} else if err != nil {- return err+ return removed, err}+ removed = append(removed, layer.Digest)}- return nil+ return removed, nil}func ParseNamedManifest(n model.Name) (*Manifest, error) {diff --git a/manifest/paths.go b/manifest/paths.goindex 4451c81aa..df452794e 100644--- a/manifest/paths.go+++ b/manifest/paths.go@@ -14,6 +14,18 @@ import (var ErrInvalidDigestFormat = errors.New("invalid digest format")+// a manifest spells a digest with ":", a filename with "-"+var digestPattern = regexp.MustCompile(`^sha256[:-][0-9a-fA-F]{64}$`)++// ValidateDigest reports whether digest names a blob.+func ValidateDigest(digest string) error {+ if !digestPattern.MatchString(digest) {+ return ErrInvalidDigestFormat+ }++ return nil+}+func Path() (string, error) {path := filepath.Join(envconfig.Models(), "manifests")if err := os.MkdirAll(path, 0o755); err != nil {@@ -38,12 +50,11 @@ func PathForName(n model.Name) (string, error) {}func BlobsPath(digest string) (string, error) {- // only accept actual sha256 digests- pattern := "^sha256[:-][0-9a-fA-F]{64}$"- re := regexp.MustCompile(pattern)-- if digest != "" && !re.MatchString(digest) {- return "", ErrInvalidDigestFormat+ // the empty digest names the blobs directory+ if digest != "" {+ if err := ValidateDigest(digest); err != nil {+ return "", err+ }}digest = strings.ReplaceAll(digest, ":", "-")diff --git a/server/create.go b/server/create.goindex fddbe5ae0..f0ad013d1 100644--- a/server/create.go+++ b/server/create.go@@ -337,13 +337,13 @@ func (s *Server) CreateHandler(c *gin.Context) {}if !envconfig.NoPrune() && oldManifest != nil {- if err := oldManifest.RemoveLayers(); err != nil {+ removed, err := oldManifest.RemoveLayers()+ removeGGUFMetadata(removed...)+ if err != nil {ch <- gin.H{"error": err.Error()}}}- s.refreshModelListCache(name)-ch <- api.ProgressResponse{Status: "success"}}()diff --git a/server/gguf_metadata.go b/server/gguf_metadata.gonew file mode 100644index 000000000..118fb7445--- /dev/null+++ b/server/gguf_metadata.go@@ -0,0 +1,338 @@+package server++import (+ "bytes"+ "encoding/json"+ "errors"+ "fmt"+ "log/slog"+ "math"+ "os"+ "path/filepath"+ "strings"++ "github.com/ollama/ollama/envconfig"+ fsgguf "github.com/ollama/ollama/fs/gguf"+ "github.com/ollama/ollama/manifest"+ "github.com/ollama/ollama/version"+)++// A blob's metadata block is extracted once into a file beside the model+// store, keyed by blob digest. Values are held verbatim rather than derived, so+// reading them differently later needs no invalidation, and values left out are+// recorded by key so an absent key stays distinct from an uncopied one.++// Comfortably above per-layer arrays, which scale with block count, and far+// below any tokenizer vocabulary.+const ggufMetadataMaxArray = 4096++type ggufMetadata struct {+ OllamaVersion string `json:"ollama_version,omitempty"`++ // Values keyed exactly as they appear in the file. Never omitempty: absent+ // on load means the file is not ours, and an all-omitted file is still+ // usable.+ KV map[string]any `json:"kv"`+ // Keys present in the file whose values were too large to copy.+ Omitted []string `json:"omitted,omitempty"`+}++// Valid reports whether the key is present. Keys are architecture qualified the+// same way fs/gguf does it.+func (m ggufMetadata) Valid(key string) bool {+ _, ok := m.lookup(key)+ return ok+}++func (m ggufMetadata) String(key string) string {+ v, _ := m.lookup(key)+ s, _ := v.(string)+ return s+}++func (m ggufMetadata) Int(key string) int64 {+ n, ok := m.number(key)+ if !ok {+ return 0+ }+ i, err := n.Int64()+ if err != nil {+ return 0+ }+ return i+}++func (m ggufMetadata) number(key string) (json.Number, bool) {+ v, _ := m.lookup(key)+ n, ok := v.(json.Number)+ return n, ok+}++// Keys returns every key the file carried, including omitted ones.+func (m ggufMetadata) Keys() []string {+ keys := make([]string, 0, len(m.KV)+len(m.Omitted))+ for k := range m.KV {+ keys = append(keys, k)+ }+ return append(keys, m.Omitted...)+}++func (m ggufMetadata) lookup(key string) (any, bool) {+ if !strings.HasPrefix(key, "general.") && !strings.HasPrefix(key, "tokenizer.") {+ arch, _ := m.KV["general.architecture"].(string)+ key = arch + "." + key+ }+ v, ok := m.KV[key]+ return v, ok+}++func ggufMetadataPath(digest string) (string, error) {+ if err := manifest.ValidateDigest(digest); err != nil {+ return "", fmt.Errorf("%w: %q", err, digest)+ }+ return filepath.Join(ggufMetadataDir(), strings.ReplaceAll(digest, ":", "-")+".json"), nil+}++func ggufMetadataDir() string {+ return filepath.Join(envconfig.Models(), "metadata")+}++// readGGUFMetadata extracts the blob when there is no usable metadata file. Best+// effort throughout: a bad file is re-extracted, a failed write is dropped.+func readGGUFMetadata(digest string) (ggufMetadata, error) {+ path, err := ggufMetadataPath(digest)+ if err != nil {+ return ggufMetadata{}, err+ }+ if md, ok := loadGGUFMetadata(path); ok {+ return md, nil+ }++ blob, err := manifest.BlobsPath(digest)+ if err != nil {+ return ggufMetadata{}, err+ }+ md, err := extractGGUFMetadata(blob)+ if err != nil {+ return ggufMetadata{}, err+ }++ writeGGUFMetadata(path, blob, md)+ return md, nil+}++func loadGGUFMetadata(path string) (ggufMetadata, bool) {+ data, err := os.ReadFile(path)+ if err != nil {+ if !errors.Is(err, os.ErrNotExist) {+ slog.Debug("could not read gguf metadata file", "path", path, "error", err)+ }+ return ggufMetadata{}, false+ }++ md, err := decodeGGUFMetadata(data)+ if err != nil {+ slog.Debug("ignoring unusable gguf metadata file", "path", path, "error", err)+ return ggufMetadata{}, false+ }+ return md, true+}++func writeGGUFMetadata(path, blob string, md ggufMetadata) {+ data, err := json.Marshal(md)+ if err != nil {+ slog.Debug("could not encode gguf metadata", "path", path, "error", err)+ return+ }+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {+ slog.Debug("could not create gguf metadata dir", "path", path, "error", err)+ return+ }++ tmp, err := os.CreateTemp(filepath.Dir(path), ".gguf-metadata-*.tmp")+ if err != nil {+ slog.Debug("could not create gguf metadata temp file", "path", path, "error", err)+ return+ }+ defer os.Remove(tmp.Name())++ if _, err = tmp.Write(data); err == nil {+ err = tmp.Sync()+ }+ if cerr := tmp.Close(); err == nil {+ err = cerr+ }+ if err == nil {+ // Can fail transiently on Windows while another process holds the path.+ err = os.Rename(tmp.Name(), path)+ }+ if err == nil {+ // Blob removal precedes metadata removal. Rechecking after publication+ // ensures either this writer or a concurrent remover cleans up the file.+ if _, statErr := os.Stat(blob); errors.Is(statErr, os.ErrNotExist) {+ if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {+ slog.Debug("could not remove gguf metadata file", "path", path, "error", removeErr)+ }+ return+ }+ }+ if err != nil {+ slog.Debug("could not write gguf metadata file", "path", path, "error", err)+ }+}++// removeGGUFMetadata drops metadata for deleted blobs. Pass only digests whose+// last reference is gone.+func removeGGUFMetadata(digests ...string) {+ for _, digest := range digests {+ path, err := ggufMetadataPath(digest)+ if err != nil {+ continue+ }+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {+ slog.Debug("could not remove gguf metadata file", "path", path, "error", err)+ }+ }+}++func pruneGGUFMetadata() {+ dir := ggufMetadataDir()+ entries, err := os.ReadDir(dir)+ if errors.Is(err, os.ErrNotExist) {+ return+ }+ if err != nil {+ slog.Debug("could not read gguf metadata dir", "path", dir, "error", err)+ return+ }++ for _, entry := range entries {+ name := entry.Name()+ if entry.IsDir() || !strings.HasSuffix(name, ".json") {+ continue+ }++ digest := strings.TrimSuffix(name, ".json")+ blob, err := manifest.BlobsPath(digest)+ if err != nil {+ continue+ }+ if _, err := os.Stat(blob); err == nil || !errors.Is(err, os.ErrNotExist) {+ continue+ }+ path := filepath.Join(dir, name)+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {+ slog.Debug("could not remove gguf metadata file", "path", path, "error", err)+ }+ }+}++// extractGGUFMetadata reads the whole block; proving a key absent means reaching+// the end. Normalized through JSON so extracted and loaded metadata hold the+// same value types.+func extractGGUFMetadata(path string) (ggufMetadata, error) {+ md, err := scanGGUFMetadata(path)+ if err != nil {+ return ggufMetadata{}, err+ }+ md.OllamaVersion = version.Version++ data, err := json.Marshal(md)+ if err != nil {+ return ggufMetadata{}, err+ }+ return decodeGGUFMetadata(data)+}++func decodeGGUFMetadata(data []byte) (ggufMetadata, error) {+ // UseNumber keeps integers exact; the default would widen them to float64.+ dec := json.NewDecoder(bytes.NewReader(data))+ dec.UseNumber()+ var md ggufMetadata+ if err := dec.Decode(&md); err != nil {+ return ggufMetadata{}, err+ }+ if md.KV == nil {+ return ggufMetadata{}, fmt.Errorf("no metadata")+ }+ return md, nil+}++func scanGGUFMetadata(path string) (ggufMetadata, error) {+ f, err := fsgguf.Open(path)+ if err != nil {+ return ggufMetadata{}, err+ }+ defer f.Close()++ md := ggufMetadata{KV: make(map[string]any)}+ for _, kv := range f.KeyValues() {+ if omitValue(kv.Any()) {+ md.Omitted = append(md.Omitted, kv.Key)+ continue+ }+ md.KV[kv.Key] = kv.Any()+ }+ if err := f.Err(); err != nil {+ return ggufMetadata{}, err+ }+ return md, nil+}++// omitValue reports values left out of the metadata file: long arrays, which are the+// tokenizer, and non-finite floats, which JSON cannot represent.+func omitValue(v any) bool {+ switch v := v.(type) {+ case float32:+ return !finite(float64(v))+ case float64:+ return !finite(v)+ case []float32:+ for _, f := range v {+ if !finite(float64(f)) {+ return true+ }+ }+ case []float64:+ for _, f := range v {+ if !finite(f) {+ return true+ }+ }+ }+ return oversized(v)+}++func finite(f float64) bool {+ return !math.IsInf(f, 0) && !math.IsNaN(f)+}++func oversized(v any) bool {+ switch v := v.(type) {+ case []string:+ return len(v) > ggufMetadataMaxArray+ case []int8:+ return len(v) > ggufMetadataMaxArray+ case []int16:+ return len(v) > ggufMetadataMaxArray+ case []int32:+ return len(v) > ggufMetadataMaxArray+ case []int64:+ return len(v) > ggufMetadataMaxArray+ case []uint8:+ return len(v) > ggufMetadataMaxArray+ case []uint16:+ return len(v) > ggufMetadataMaxArray+ case []uint32:+ return len(v) > ggufMetadataMaxArray+ case []uint64:+ return len(v) > ggufMetadataMaxArray+ case []float32:+ return len(v) > ggufMetadataMaxArray+ case []float64:+ return len(v) > ggufMetadataMaxArray+ case []bool:+ return len(v) > ggufMetadataMaxArray+ }+ return false+}diff --git a/server/gguf_metadata_test.go b/server/gguf_metadata_test.gonew file mode 100644index 000000000..f62951e35--- /dev/null+++ b/server/gguf_metadata_test.go@@ -0,0 +1,736 @@+package server++import (+ "encoding/binary"+ "encoding/json"+ "errors"+ "math"+ "net/http"+ "os"+ "path/filepath"+ "reflect"+ "runtime"+ "slices"+ "strings"+ "testing"+ "time"++ "github.com/gin-gonic/gin"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/envconfig"+ "github.com/ollama/ollama/fs/ggml"+ "github.com/ollama/ollama/manifest"+ "github.com/ollama/ollama/types/model"+)++func TestGGUFMetadataExtraction(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ big := make([]string, 5000)+ for i := range big {+ big[i] = "tok"+ }+ perLayer := make([]int32, 64)+ for i := range perLayer {+ perLayer[i] = int32(i)+ }++ path, _ := createBinFile(t, ggml.KV{+ "general.architecture": "bert",+ "bert.attention.softcap": float32(math.Inf(-1)),+ "general.file_type": uint32(2),+ "bert.pooling_type": uint32(1),+ "bert.context_length": uint32(512),+ "bert.attention.head_count_kv": perLayer,+ "tokenizer.chat_template": "{{ messages }}",+ "tokenizer.ggml.tokens": big,+ "tokenizer.ggml.merges": big,+ }, nil)++ md, err := extractGGUFMetadata(path)+ if err != nil {+ t.Fatal(err)+ }++ if got := md.String("general.architecture"); got != "bert" {+ t.Errorf("architecture = %q", got)+ }+ // Architecture-prefixed lookup, and the key order in the file must not matter.+ if !md.Valid("pooling_type") {+ t.Error("pooling_type not found")+ }+ if got := md.Int("context_length"); got != 512 {+ t.Errorf("context_length = %d, want 512", got)+ }+ if got := md.String("tokenizer.chat_template"); got != "{{ messages }}" {+ t.Errorf("chat_template = %q", got)+ }+ if md.Valid("vision.block_count") {+ t.Error("absent key reported present")+ }++ // A 64-element per-layer array is real metadata and must survive.+ if v, ok := md.KV["bert.attention.head_count_kv"].([]any); !ok || len(v) != 64 {+ t.Errorf("per-layer array = %#v, want 64 elements", md.KV["bert.attention.head_count_kv"])+ }++ // The tokenizer is recorded as omitted, not silently missing.+ want := map[string]bool{"tokenizer.ggml.tokens": true, "tokenizer.ggml.merges": true}+ for _, key := range md.Omitted {+ delete(want, key)+ }+ if len(want) != 0 {+ t.Errorf("not reported omitted: %v (omitted=%v)", want, md.Omitted)+ }+ if _, ok := md.KV["tokenizer.ggml.tokens"]; ok {+ t.Error("oversized array was copied into the metadata file")+ }++ // JSON cannot represent an infinity, and gemma3n ships one. Dropping that+ // key must not cost the rest of the file.+ if !slices.Contains(md.Omitted, "bert.attention.softcap") {+ t.Errorf("non-finite float not omitted; omitted=%v", md.Omitted)+ }+}++func TestGGUFMetadataFileRoundTrip(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, digest := createBinFile(t, ggml.KV{+ "general.architecture": "llama",+ "llama.block_count": uint32(32),+ }, nil)++ first, err := readGGUFMetadata(digest)+ if err != nil {+ t.Fatal(err)+ }+ if got := first.Int("block_count"); got != 32 {+ t.Fatalf("block_count = %d, want 32", got)+ }++ // With the blob gone, only the metadata file can answer.+ if err := removeBlob(t, digest); err != nil {+ t.Fatal(err)+ }+ second, err := readGGUFMetadata(digest)+ if err != nil {+ t.Fatalf("metadata file read failed: %v", err)+ }+ if got := second.Int("block_count"); got != 32 {+ t.Fatalf("block_count from metadata file = %d, want 32", got)+ }++ // And it goes away with the blob.+ removeGGUFMetadata(digest)+ path, err := ggufMetadataPath(digest)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := os.Stat(path); !os.IsNotExist(err) {+ t.Fatalf("metadata file still present after removal: %v", err)+ }+}++// The metadata file is only ever an optimization: anything unusable must be re-extracted.+func TestGGUFMetadataUnusableFile(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, digest := createBinFile(t, ggml.KV{"general.architecture": "llama"}, nil)+ if _, err := readGGUFMetadata(digest); err != nil {+ t.Fatal(err)+ }+ path, err := ggufMetadataPath(digest)+ if err != nil {+ t.Fatal(err)+ }++ for _, tt := range []struct {+ name string+ content []byte+ }{+ {"truncated", []byte(`{"kv":{"general.arch`)},+ {"not json", []byte("\x00\x01\x02")},+ {"empty", []byte{}},+ {"wrong shape", []byte(`[1,2,3]`)},+ {"no kv", []byte(`{"ollama_version":"1.2.3"}`)},+ {"kv not an object", []byte(`{"kv":[]}`)},+ } {+ t.Run(tt.name, func(t *testing.T) {+ if err := os.WriteFile(path, tt.content, 0o644); err != nil {+ t.Fatal(err)+ }+ if _, ok := loadGGUFMetadata(path); ok {+ t.Fatal("unusable metadata file reported usable")+ }+ md, err := readGGUFMetadata(digest)+ if err != nil {+ t.Fatal(err)+ }+ if got := md.String("general.architecture"); got != "llama" {+ t.Fatalf("architecture = %q, want llama", got)+ }+ })+ }+}++func TestGGUFMetadataRejectsMalformed(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())+ dir := t.TempDir()++ for _, tt := range []struct {+ name string+ data []byte+ }{+ {"empty file", nil},+ {"bad magic", []byte("NOPE")},+ {"truncated header", []byte("GGUF\x03\x00\x00\x00")},+ } {+ t.Run(tt.name, func(t *testing.T) {+ path := filepath.Join(dir, tt.name)+ if err := os.WriteFile(path, tt.data, 0o600); err != nil {+ t.Fatal(err)+ }+ if _, err := extractGGUFMetadata(path); err == nil {+ t.Fatal("expected an error")+ }+ })+ }+}++// The omission rule decides what never reaches the metadata file, so every array type+// has to agree on it. A miss here silently drops real metadata.+func TestGGUFMetadataOmitRule(t *testing.T) {+ short, long := 64, ggufMetadataMaxArray+1++ for _, tt := range []struct {+ name string+ keep any+ omit any+ }{+ {"string", make([]string, short), make([]string, long)},+ {"int8", make([]int8, short), make([]int8, long)},+ {"int16", make([]int16, short), make([]int16, long)},+ {"int32", make([]int32, short), make([]int32, long)},+ {"int64", make([]int64, short), make([]int64, long)},+ {"uint8", make([]uint8, short), make([]uint8, long)},+ {"uint16", make([]uint16, short), make([]uint16, long)},+ {"uint32", make([]uint32, short), make([]uint32, long)},+ {"uint64", make([]uint64, short), make([]uint64, long)},+ {"float32", make([]float32, short), make([]float32, long)},+ {"float64", make([]float64, short), make([]float64, long)},+ {"bool", make([]bool, short), make([]bool, long)},+ } {+ t.Run(tt.name, func(t *testing.T) {+ if omitValue(tt.keep) {+ t.Errorf("%d-element %s array omitted; real per-layer metadata would be lost", short, tt.name)+ }+ if !omitValue(tt.omit) {+ t.Errorf("%d-element %s array kept; the tokenizer would be copied", long, tt.name)+ }+ })+ }++ // Scalars are always kept, except the floats JSON cannot encode.+ for _, keep := range []any{"s", int32(0), uint32(0), float32(1.5), float64(1.5), true} {+ if omitValue(keep) {+ t.Errorf("scalar %#v omitted", keep)+ }+ }+ for _, omit := range []any{+ float32(math.Inf(1)), float32(math.Inf(-1)), float64(math.NaN()),+ []float32{1, float32(math.Inf(1))},+ []float64{math.NaN()},+ } {+ if !omitValue(omit) {+ t.Errorf("non-finite %#v kept; json.Marshal would fail the whole file", omit)+ }+ }+}++func TestGGUFMetadataKeysIncludesOmitted(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ big := make([]string, ggufMetadataMaxArray+1)+ path, _ := createBinFile(t, ggml.KV{+ "general.architecture": "llama",+ "llama.block_count": uint32(4),+ "tokenizer.ggml.tokens": big,+ }, nil)++ md, err := extractGGUFMetadata(path)+ if err != nil {+ t.Fatal(err)+ }+ keys := md.Keys()+ for _, want := range []string{"general.architecture", "llama.block_count", "tokenizer.ggml.tokens"} {+ if !slices.Contains(keys, want) {+ t.Errorf("Keys() = %v, missing %s", keys, want)+ }+ }+}++// Projector audio detection reads through Keys(), including prefixed forms.+func TestProjectorAudioFromMetadata(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ for _, tt := range []struct {+ name string+ kv ggml.KV+ audio bool+ }{+ {"bare key", ggml.KV{"general.architecture": "clip", "has_audio_encoder": true}, true},+ {"prefixed key", ggml.KV{"general.architecture": "clip", "clip.has_audio_encoder": true}, true},+ {"present but false", ggml.KV{"general.architecture": "clip", "has_audio_encoder": false}, false},+ {"absent", ggml.KV{"general.architecture": "clip"}, false},+ } {+ t.Run(tt.name, func(t *testing.T) {+ path, _ := createBinFile(t, tt.kv, nil)+ md, err := extractGGUFMetadata(path)+ if err != nil {+ t.Fatal(err)+ }+ if got := projectorHasAudio(md); got != tt.audio {+ t.Errorf("projectorHasAudio = %v, want %v", got, tt.audio)+ }+ })+ }++ path, _ := createBinFile(t, ggml.KV{+ "general.architecture": "clip",+ "has_audio_encoder": true,+ "clip.vision.projector_type": "gemma3nv",+ }, nil)+ md, err := extractGGUFMetadata(path)+ if err != nil {+ t.Fatal(err)+ }+ if !projectorSuppressesAudioCapability(md) {+ t.Error("gemma3nv projector should suppress audio")+ }+}++func TestGGUFMetadataPathRejectsBadDigest(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())+ hex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"++ for _, digest := range []string{"", "sha256-short", "md5:" + hex, "../escape", "sha256:" + hex + "x"} {+ if _, err := ggufMetadataPath(digest); err == nil {+ t.Errorf("ggufMetadataPath(%q) accepted", digest)+ }+ }+ for _, digest := range []string{"sha256:" + hex, "sha256-" + hex} {+ if _, err := ggufMetadataPath(digest); err != nil {+ t.Errorf("ggufMetadataPath(%q) rejected: %v", digest, err)+ }+ }+}++// A file can carry no values at all, and every value it does carry can be one+// the omission rule drops. Either way what is left still says which keys the+// file had, so it has to be storable: treating it as corrupt would rescan the+// blob on every request forever.+func TestGGUFMetadataWithNothingToCopy(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ path := filepath.Join(t.TempDir(), "empty.gguf")+ if err := os.WriteFile(path, ggufBytes(0, nil), 0o600); err != nil {+ t.Fatal(err)+ }++ md, err := extractGGUFMetadata(path)+ if err != nil {+ t.Fatalf("a file with no metadata is still a file: %v", err)+ }+ if len(md.KV) != 0 {+ t.Errorf("KV = %v, want empty", md.KV)+ }++ encoded, err := json.Marshal(md)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := decodeGGUFMetadata(encoded); err != nil {+ t.Errorf("written metadata cannot be read back, so it would be rescanned forever: %v", err)+ }+}++const allocBudget = 1 << 20++func ggufBytes(kvCount uint64, extra []byte) []byte {+ b := make([]byte, 0, 24+len(extra))+ b = binary.LittleEndian.AppendUint32(b, 0x46554747) // GGUF+ b = binary.LittleEndian.AppendUint32(b, 3) // version+ b = binary.LittleEndian.AppendUint64(b, 0) // tensor count+ b = binary.LittleEndian.AppendUint64(b, kvCount) // kv count+ return append(b, extra...)+}++func declaredGGUF(t *testing.T, kvCount uint64, extra []byte) string {+ t.Helper()+ b := ggufBytes(kvCount, extra)++ path := filepath.Join(t.TempDir(), "declared.gguf")+ if err := os.WriteFile(path, b, 0o600); err != nil {+ t.Fatal(err)+ }+ t.Logf("file is %d bytes, declares %d KV entries", len(b), kvCount)+ return path+}++func allocDelta(t *testing.T, fn func()) uint64 {+ t.Helper()+ var before, after runtime.MemStats+ runtime.GC()+ runtime.ReadMemStats(&before)+ fn()+ runtime.ReadMemStats(&after)+ return after.TotalAlloc - before.TotalAlloc+}++func TestGGUFMetadataBoundsDeclaredKVCount(t *testing.T) {+ path := declaredGGUF(t, 1<<20, nil)+ got := allocDelta(t, func() {+ if _, err := extractGGUFMetadata(path); err == nil {+ t.Error("expected an error from a 24-byte file")+ }+ })+ t.Logf("allocated %d bytes before failing", got)+ if got > allocBudget {+ t.Errorf("allocated %d bytes from a 24-byte file, want at most %d", got, allocBudget)+ }+}++// Cover every array element type because strings and fixed-width values use+// separate readers.+func TestGGUFMetadataBoundsDeclaredArray(t *testing.T) {+ for _, tt := range []struct {+ name string+ elemType uint32+ }{+ {"uint8", 0},+ {"int8", 1},+ {"uint16", 2},+ {"int16", 3},+ {"uint32", 4},+ {"int32", 5},+ {"float32", 6},+ {"bool", 7},+ {"string", 8},+ {"uint64", 10},+ {"int64", 11},+ {"float64", 12},+ } {+ t.Run(tt.name, func(t *testing.T) {+ var extra []byte+ extra = binary.LittleEndian.AppendUint64(extra, 3) // key length+ extra = append(extra, "big"...)+ extra = binary.LittleEndian.AppendUint32(extra, 9) // type: array+ extra = binary.LittleEndian.AppendUint32(extra, tt.elemType)+ extra = binary.LittleEndian.AppendUint64(extra, 8_000_000) // element count++ path := declaredGGUF(t, 1, extra)+ got := allocDelta(t, func() {+ if _, err := extractGGUFMetadata(path); err == nil {+ t.Error("expected an error")+ }+ })+ if got > allocBudget {+ t.Errorf("allocated %d bytes for a declared array of 8000000, want at most %d", got, allocBudget)+ }+ })+ }+}++func removeBlob(t *testing.T, digest string) error {+ t.Helper()+ path, err := manifest.BlobsPath(digest)+ if err != nil {+ t.Fatal(err)+ }+ return os.Remove(path)+}++func metadataCount(t *testing.T) int {+ t.Helper()+ entries, err := os.ReadDir(filepath.Join(envconfig.Models(), "metadata"))+ if err != nil {+ return 0+ }+ return len(entries)+}++// createModelFromBlob and deleteModelNamed drive the handlers rather than+// writing manifests, so the metadata hooks under test are the ones the server+// actually runs.+func createModelFromBlob(t *testing.T, name, digest, tmpl string) {+ t.Helper()+ var s Server+ w := createRequest(t, s.CreateHandler, api.CreateRequest{+ Model: name, Files: map[string]string{"model.gguf": digest},+ Template: tmpl, Stream: &stream,+ })+ if w.Code != http.StatusOK {+ t.Fatalf("create %s: %d %s", name, w.Code, w.Body.String())+ }+}++func deleteModelNamed(t *testing.T, name string) {+ t.Helper()+ var s Server+ w := createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: name})+ if w.Code != http.StatusOK {+ t.Fatalf("delete %s: %d %s", name, w.Code, w.Body.String())+ }+}++// Metadata is derived from a blob, so it lives exactly as long as the blob: it+// survives while any manifest still references it and goes with the last one.+func TestGGUFMetadataRemovedWithLastReference(t *testing.T) {+ gin.SetMode(gin.TestMode)+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, digest := createBinFile(t, ggml.KV{+ "general.architecture": "llama",+ "llama.block_count": uint32(1),+ }, nil)++ createModelFromBlob(t, "alias-one", digest, "")+ createModelFromBlob(t, "alias-two", digest, "")++ if _, err := readGGUFMetadata(digest); err != nil {+ t.Fatal(err)+ }+ if got := metadataCount(t); got != 1 {+ t.Fatalf("metadata files = %d, want 1 (both aliases share one blob)", got)+ }++ deleteModelNamed(t, "alias-one")+ if got := metadataCount(t); got != 1 {+ t.Fatalf("metadata files after removing one alias = %d, want 1", got)+ }++ deleteModelNamed(t, "alias-two")+ if got := metadataCount(t); got != 0 {+ t.Fatalf("metadata files after removing the last reference = %d, want 0", got)+ }+}++// Recreating a model over the same name replaces its blob; the old blob's+// metadata must go with it.+func TestGGUFMetadataRemovedOnReplacement(t *testing.T) {+ gin.SetMode(gin.TestMode)+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, first := createBinFile(t, ggml.KV{+ "general.architecture": "llama",+ "llama.block_count": uint32(1),+ }, nil)+ createModelFromBlob(t, "replace-me", first, "")+ if _, err := readGGUFMetadata(first); err != nil {+ t.Fatal(err)+ }++ _, second := createBinFile(t, ggml.KV{+ "general.architecture": "llama",+ "llama.block_count": uint32(2),+ }, nil)+ createModelFromBlob(t, "replace-me", second, "")+ if _, err := readGGUFMetadata(second); err != nil {+ t.Fatal(err)+ }++ if got := metadataCount(t); got != 1 {+ t.Fatalf("metadata files after replacement = %d, want 1 (the old blob's should be gone)", got)+ }+ old, err := ggufMetadataPath(first)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := os.Stat(old); !os.IsNotExist(err) {+ t.Errorf("metadata for the replaced blob still present: %v", err)+ }+}++// A blob missing from disk is still a blob whose metadata has to go. It no+// longer appears in the blob scan, so pruning has to find its metadata directly.+func TestGGUFMetadataRemovedForMissingBlob(t *testing.T) {+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, orphan := createBinFile(t, ggml.KV{"general.architecture": "llama"}, nil)+ if _, err := readGGUFMetadata(orphan); err != nil {+ t.Fatal(err)+ }+ if err := removeBlob(t, orphan); err != nil {+ t.Fatal(err)+ }++ _, live := createBinFile(t, ggml.KV{"general.architecture": "bert"}, nil)+ if _, err := readGGUFMetadata(live); err != nil {+ t.Fatal(err)+ }+ if got := metadataCount(t); got != 2 {+ t.Fatalf("metadata files before the sweep = %d, want 2", got)+ }++ if err := PruneLayers(); err != nil {+ t.Fatal(err)+ }+ if got := metadataCount(t); got != 1 {+ t.Errorf("metadata files = %d, want 1", got)+ }+ path, err := ggufMetadataPath(live)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := os.Stat(path); err != nil {+ t.Errorf("metadata for existing blob was removed: %v", err)+ }+}++func TestGGUFMetadataNotPublishedAfterDelete(t *testing.T) {+ gin.SetMode(gin.TestMode)+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, digest := createBinFile(t, ggml.KV{+ "general.architecture": "llama",+ "general.description": strings.Repeat("x", 16<<20),+ }, nil)+ createModelFromBlob(t, "delete-during-load", digest, "")++ loaded := make(chan error, 1)+ go func() {+ _, err := GetModel("delete-during-load")+ loaded <- err+ }()++ deadline := time.Now().Add(5 * time.Second)+ for {+ entries, err := os.ReadDir(ggufMetadataDir())+ if err != nil && !errors.Is(err, os.ErrNotExist) {+ t.Fatal(err)+ }+ if slices.ContainsFunc(entries, func(entry os.DirEntry) bool {+ return strings.HasPrefix(entry.Name(), ".gguf-metadata-")+ }) {+ break+ }+ select {+ case err := <-loaded:+ t.Fatalf("GetModel completed before its metadata write could overlap deletion: %v", err)+ default:+ }+ if time.Now().After(deadline) {+ t.Fatal("timed out waiting for metadata write")+ }+ runtime.Gosched()+ }++ deleteModelNamed(t, "delete-during-load")+ select {+ case err := <-loaded:+ if err != nil {+ t.Fatalf("GetModel: %v", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("timed out waiting for GetModel")+ }++ path, err := ggufMetadataPath(digest)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {+ t.Fatalf("metadata published after model deletion: %v", err)+ }+}++// The point of the metadata file is that loading a model no longer reads its+// blob. With the blob deleted, GetModel must still describe the model exactly+// as before - which is also the guard against anything reintroducing a+// per-request read of the model file.+func TestGetModelReadsNoBlob(t *testing.T) {+ gin.SetMode(gin.TestMode)+ t.Setenv("OLLAMA_MODELS", t.TempDir())++ _, digest := createBinFile(t, ggml.KV{+ "general.architecture": "bert",+ "bert.pooling_type": uint32(1),+ "bert.context_length": uint32(512),+ "bert.embedding_length": uint32(384),+ }, nil)+ createModelFromBlob(t, "embedder", digest, "")++ want, err := GetModel("embedder")+ if err != nil {+ t.Fatal(err)+ }+ if !slices.Contains(want.Capabilities(), model.CapabilityEmbedding) {+ t.Fatalf("capabilities = %v, want embedding", want.Capabilities())+ }++ if err := removeBlob(t, digest); err != nil {+ t.Fatal(err)+ }+ got, err := GetModel("embedder")+ if err != nil {+ t.Fatalf("GetModel needed the blob: %v", err)+ }+ if !slices.Equal(got.Capabilities(), want.Capabilities()) {+ t.Errorf("capabilities without the blob = %v, want %v", got.Capabilities(), want.Capabilities())+ }+ if got.metadata.Int("context_length") != 512 {+ t.Errorf("context_length = %d, want 512", got.metadata.Int("context_length"))+ }+}++// Extraction either fails or produces metadata that survives a metadata-file+// round trip unchanged.+func FuzzGGUFMetadata(f *testing.F) {+ f.Add(ggufBytes(0, nil))+ f.Add(ggufBytes(1<<20, nil))++ var str []byte+ str = binary.LittleEndian.AppendUint64(str, 20)+ str = append(str, "general.architecture"...)+ str = binary.LittleEndian.AppendUint32(str, 8) // type: string+ str = binary.LittleEndian.AppendUint64(str, 5)+ str = append(str, "llama"...)+ f.Add(ggufBytes(1, str))++ var arr []byte+ arr = binary.LittleEndian.AppendUint64(arr, 3)+ arr = append(arr, "big"...)+ arr = binary.LittleEndian.AppendUint32(arr, 9) // type: array+ arr = binary.LittleEndian.AppendUint32(arr, 8) // element type: string+ arr = binary.LittleEndian.AppendUint64(arr, 8_000_000)+ f.Add(ggufBytes(1, arr))++ path := filepath.Join(f.TempDir(), "fuzz.gguf")+ f.Fuzz(func(t *testing.T, data []byte) {+ if err := os.WriteFile(path, data, 0o600); err != nil {+ t.Fatal(err)+ }++ md, err := extractGGUFMetadata(path)+ if err != nil {+ return+ }+ if md.KV == nil {+ t.Fatal("extraction reported success with no metadata")+ }++ encoded, err := json.Marshal(md)+ if err != nil {+ t.Fatalf("extracted metadata cannot be written: %v", err)+ }+ loaded, err := decodeGGUFMetadata(encoded)+ if err != nil {+ t.Fatalf("extracted metadata cannot be read back: %v", err)+ }+ if !reflect.DeepEqual(md, loaded) {+ t.Errorf("metadata changed on the trip through a file:\n%#v\n%#v", md, loaded)+ }+ })+}diff --git a/server/images.go b/server/images.goindex 9be4ce51d..435030590 100644--- a/server/images.go+++ b/server/images.go@@ -10,7 +10,6 @@ import ("io""log""log/slog"- "math""net""net/http""net/url"@@ -24,7 +23,6 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/envconfig"- "github.com/ollama/ollama/fs/gguf""github.com/ollama/ollama/manifest""github.com/ollama/ollama/model/parsers""github.com/ollama/ollama/parser"@@ -83,8 +81,10 @@ type Model struct {Template *template.Template- capabilities []model.Capability- capabilitiesCached bool+ // Metadata of the model blob and of each projector, read from their+ // metadata files when the model is loaded.+ metadata ggufMetadata+ projectorMetadata []ggufMetadata}func (m *Model) IsMLX() bool {@@ -95,47 +95,33 @@ func (m *Model) isGGUF() bool {return m.Config.ModelFormat == "" || m.Config.ModelFormat == "gguf"}-func generationDefaultsFromGGUF(f *gguf.File) model.GenerationDefaults {+func generationDefaultsFromMetadata(md ggufMetadata) model.GenerationDefaults {return model.ParseGGUFGenerationDefaults(func(key string) (int64, bool) {- return ggufIntGenerationDefault(f.KeyValue(key))+ n, ok := md.number(key)+ if !ok {+ return 0, false+ }+ if value, err := n.Int64(); err == nil {+ return value, true+ }+ value, err := n.Float64()+ if err != nil {+ return 0, false+ }+ return int64(value), true},func(key string) (float64, bool) {- return ggufFloatGenerationDefault(f.KeyValue(key))+ n, ok := md.number(key)+ if !ok {+ return 0, false+ }+ value, err := n.Float64()+ return value, err == nil},)}-func ggufIntGenerationDefault(kv gguf.KeyValue) (int64, bool) {- if value, ok := kv.IntOK(); ok {- return value, true- }- if value, ok := kv.UintOK(); ok {- if value > math.MaxInt64 {- return 0, false- }- return int64(value), true- }- if value, ok := kv.FloatOK(); ok {- // Match api.Options.FromMap; rounding may be better for near-integers.- return int64(value), true- }- return 0, false-}--func ggufFloatGenerationDefault(kv gguf.KeyValue) (float64, bool) {- if value, ok := kv.FloatOK(); ok {- return value, true- }- if value, ok := kv.IntOK(); ok {- return float64(value), true- }- if value, ok := kv.UintOK(); ok {- return float64(value), true- }- return 0, false-}-func appendCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {if slices.Contains(capabilities, capability) {return capabilities@@ -153,11 +139,7 @@ const (// Capabilities returns the capabilities that the model supportsfunc (m *Model) Capabilities() []model.Capability {- if m.capabilitiesCached {- return slices.Clone(m.capabilities)- }-- capabilities := m.capabilitiesForTemplate(templateCapabilitySelected, nil)+ capabilities := m.capabilitiesForTemplate(templateCapabilitySelected)if len(capabilities) == 0 {slog.Warn("unknown capabilities for model", "model", m.Name)}@@ -165,12 +147,12 @@ func (m *Model) Capabilities() []model.Capability {return capabilities}-func (m *Model) capabilitiesForTemplate(source templateCapabilitySource, f *gguf.File) []model.Capability {+func (m *Model) capabilitiesForTemplate(source templateCapabilitySource) []model.Capability {capabilities := []model.Capability{}var modelArch stringcapabilities = m.configCapabilities(capabilities)- capabilities, modelArch = m.ggufCapabilities(capabilities, source, f)+ capabilities, modelArch = m.ggufCapabilities(capabilities, source)capabilities = m.projectorCapabilities(capabilities)capabilities = m.templateCapabilities(capabilities, source)capabilities = m.parserCapabilities(capabilities)@@ -187,44 +169,33 @@ func (m *Model) configCapabilities(capabilities []model.Capability) []model.Capareturn capabilities}-func (m *Model) ggufCapabilities(capabilities []model.Capability, source templateCapabilitySource, f *gguf.File) ([]model.Capability, string) {+func (m *Model) ggufCapabilities(capabilities []model.Capability, source templateCapabilitySource) ([]model.Capability, string) {if m.ModelPath == "" || !m.isGGUF() {return capabilities, ""}- if f == nil {- var err error- f, err = gguf.Open(m.ModelPath)- if err != nil {- slog.Error("couldn't open model file", "error", err)- return capabilities, ""- }- defer f.Close()- }-- modelArch := f.KeyValue("general.architecture").String()switch source {case templateCapabilitySelected:if !usesOllamaRenderedChat(m) {- capabilities = chatTemplateCapabilities(capabilities, f.KeyValue("tokenizer.chat_template").String())+ capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template"))}case templateCapabilityChat:- capabilities = chatTemplateCapabilities(capabilities, f.KeyValue("tokenizer.chat_template").String())+ capabilities = chatTemplateCapabilities(capabilities, m.metadata.String("tokenizer.chat_template"))}- if f.KeyValue("pooling_type").Valid() {+ if m.metadata.Valid("pooling_type") {capabilities = appendCapability(capabilities, model.CapabilityEmbedding)} else {// If no embedding is specified, we assume the model supports completion.capabilities = appendCapability(capabilities, model.CapabilityCompletion)}- if f.KeyValue("vision.block_count").Valid() {+ if m.metadata.Valid("vision.block_count") {capabilities = appendCapability(capabilities, model.CapabilityVision)}- if f.KeyValue("audio.block_count").Valid() {+ if m.metadata.Valid("audio.block_count") {capabilities = appendCapability(capabilities, model.CapabilityAudio)}- return capabilities, modelArch+ return capabilities, m.metadata.String("general.architecture")}func chatTemplateCapabilities(capabilities []model.Capability, chatTemplate string) []model.Capability {@@ -390,28 +361,17 @@ func capabilityLogValue(present bool, capabilities []model.Capability) any {}func (m *Model) templateSelectionCapabilities(usesHarmony bool) (goTemplate, chatTemplate, harmony, rendererParser []model.Capability) {- var f *gguf.File- if m.ModelPath != "" && m.isGGUF() {- var err error- f, err = gguf.Open(m.ModelPath)- if err != nil {- slog.Error("couldn't open model file", "error", err)- } else {- defer f.Close()- }- }-if m.HasGoTemplate {- goTemplate = m.capabilitiesForTemplate(templateCapabilityGo, f)+ goTemplate = m.capabilitiesForTemplate(templateCapabilityGo)}if m.HasChatTemplate {- chatTemplate = m.capabilitiesForTemplate(templateCapabilityChat, f)+ chatTemplate = m.capabilitiesForTemplate(templateCapabilityChat)}if usesHarmony {- harmony = m.capabilitiesForTemplate(templateCapabilitySelected, f)+ harmony = m.capabilitiesForTemplate(templateCapabilitySelected)}if m.Config.Renderer != "" || m.Config.Parser != "" {- rendererParser = m.capabilitiesForTemplate(templateCapabilitySelected, f)+ rendererParser = m.capabilitiesForTemplate(templateCapabilitySelected)}return goTemplate, chatTemplate, harmony, rendererParser@@ -439,16 +399,10 @@ func (m *Model) projectorCapabilities(capabilities []model.Capability) []model.C}capabilities = appendCapability(capabilities, model.CapabilityVision)- for _, projectorPath := range m.ProjectorPaths {- f, err := gguf.Open(projectorPath)- if err != nil {- slog.Error("couldn't open projector file", "error", err)- continue- }- if projectorHasAudio(f) && !projectorSuppressesAudioCapability(f) {+ for _, md := range m.projectorMetadata {+ if projectorHasAudio(md) && !projectorSuppressesAudioCapability(md) {capabilities = appendCapability(capabilities, model.CapabilityAudio)}- f.Close()}return capabilities@@ -551,22 +505,21 @@ func isNemotron3NanoSafetensorsConfig(cfg model.ConfigV2) bool {slices.Contains(cfg.ModelFamilies, "nemotron_h_omni"))}-func projectorHasAudio(f *gguf.File) bool {- if f.KeyValue("has_audio_encoder").Bool() {- return true- }-- for _, kv := range f.KeyValues() {- if strings.HasSuffix(kv.Key, ".has_audio_encoder") && kv.Bool() {- return true+func projectorHasAudio(md ggufMetadata) bool {+ // read directly: Keys reports qualified keys, the accessors qualify theirs+ for _, key := range md.Keys() {+ if key == "has_audio_encoder" || strings.HasSuffix(key, ".has_audio_encoder") {+ if b, ok := md.KV[key].(bool); ok && b {+ return true+ }}}return false}-func projectorSuppressesAudioCapability(f *gguf.File) bool {- switch f.KeyValue("vision.projector_type").String() {+func projectorSuppressesAudioCapability(md ggufMetadata) bool {+ switch md.String("vision.projector_type") {case "gemma3nv":return true}@@ -745,6 +698,13 @@ func GetModel(name string) (*Model, error) {modelHasPooling := falseggufChatTemplate := ""for _, layer := range mf.Layers {+ // Nothing below reads a tensor layer, and resolving a path costs a+ // syscall each. Named rather than allowlisting the types below, so a new+ // layer type is slower here instead of silently unread.+ if layer.MediaType == manifest.MediaTypeImageTensor {+ continue+ }+filename, err := manifest.BlobsPath(layer.Digest)if err != nil {return nil, err@@ -755,16 +715,16 @@ func GetModel(name string) (*Model, error) {m.ModelPath = filenamem.ParentModel = layer.Fromif m.isGGUF() {- f, err := gguf.Open(filename)+ md, err := readGGUFMetadata(layer.Digest)if err != nil {- slog.Error("couldn't open model file", "error", err)+ slog.Error("couldn't read model metadata", "error", err)break}- ggufChatTemplate = f.KeyValue("tokenizer.chat_template").String()+ m.metadata = md+ ggufChatTemplate = md.String("tokenizer.chat_template")m.HasChatTemplate = ggufChatTemplate != ""- modelHasPooling = f.KeyValue("pooling_type").Valid()- m.GenerationDefaults = generationDefaultsFromGGUF(f)- f.Close()+ modelHasPooling = md.Valid("pooling_type")+ m.GenerationDefaults = generationDefaultsFromMetadata(md)}case manifest.MediaTypeImageDraft:m.DraftPath = filename@@ -776,6 +736,11 @@ func GetModel(name string) (*Model, error) {m.AdapterPaths = append(m.AdapterPaths, filename)case "application/vnd.ollama.image.projector":m.ProjectorPaths = append(m.ProjectorPaths, filename)+ if md, err := readGGUFMetadata(layer.Digest); err != nil {+ slog.Error("couldn't read projector metadata", "error", err)+ } else {+ m.projectorMetadata = append(m.projectorMetadata, md)+ }case "application/vnd.ollama.image.prompt","application/vnd.ollama.image.template":m.HasGoTemplate = true@@ -900,10 +865,11 @@ func deleteUnusedLayers(deleteMap map[string]struct{}) error {slog.Info(fmt.Sprintf("couldn't get file path for '%s': %v", k, err))continue}- if err := os.Remove(fp); err != nil {+ if err := os.Remove(fp); err != nil && !errors.Is(err, os.ErrNotExist) {slog.Info(fmt.Sprintf("couldn't remove file '%s': %v", fp, err))continue}+ removeGGUFMetadata(k)}return nil@@ -960,6 +926,7 @@ func PruneLayers() error {slog.Error(fmt.Sprintf("couldn't remove unused layers: %v", err))return nil}+ pruneGGUFMetadata()slog.Info(fmt.Sprintf("total unused blobs removed: %d", len(deleteMap)))@@ -1122,6 +1089,7 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fuif err := os.Remove(fp); err != nil {slog.Info(fmt.Sprintf("couldn't remove file with digest mismatch '%s': %v", fp, err))}+ removeGGUFMetadata(layer.Digest)}return err}diff --git a/server/images_test.go b/server/images_test.goindex fd521dcbd..a0302bbee 100644--- a/server/images_test.go+++ b/server/images_test.go@@ -14,7 +14,6 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/fs/ggml"- fsgguf "github.com/ollama/ollama/fs/gguf""github.com/ollama/ollama/manifest""github.com/ollama/ollama/template""github.com/ollama/ollama/types/model"@@ -61,7 +60,7 @@ func TestPruneLayersSkipsRecentOrphans(t *testing.T) {}}-func TestGenerationDefaultsFromGGUF(t *testing.T) {+func TestGenerationDefaultsFromMetadata(t *testing.T) {file, err := os.CreateTemp(t.TempDir(), "model-*.gguf")if err != nil {t.Fatal(err)@@ -87,13 +86,12 @@ func TestGenerationDefaultsFromGGUF(t *testing.T) {t.Fatal(err)}- f, err := fsgguf.Open(file.Name())+ md, err := extractGGUFMetadata(file.Name())if err != nil {t.Fatal(err)}- defer f.Close()- defaults := generationDefaultsFromGGUF(f)+ defaults := generationDefaultsFromMetadata(md)check := func(key string, want any) {t.Helper()if got := defaults[key]; got != want {@@ -104,10 +102,10 @@ func TestGenerationDefaultsFromGGUF(t *testing.T) {check("top_k", int64(40))check("top_p", float64(1))check("min_p", float64(0))- check("typical_p", float64(float32(0.95)))+ check("typical_p", float64(0.95))check("temperature", float64(1))check("repeat_last_n", int64(64))- check("repeat_penalty", float64(float32(1.05)))+ check("repeat_penalty", float64(1.05))check("frequency_penalty", float64(0))check("presence_penalty", float64(0))if _, ok := defaults["mirostat_tau"]; ok {@@ -354,6 +352,28 @@ func writeTestModelManifest(t *testing.T, name, digest, tmpl string) {}}+// loadTestMetadata fills in what GetModel would have read from the metadata+// files, so+// hand-built models resolve capabilities the same way loaded ones do.+func loadTestMetadata(t *testing.T, m *Model) {+ t.Helper()+ if m.ModelPath != "" {+ md, err := extractGGUFMetadata(m.ModelPath)+ if err != nil {+ t.Fatalf("metadata for %s: %v", m.ModelPath, err)+ }+ m.metadata = md+ }+ m.projectorMetadata = nil+ for _, path := range m.ProjectorPaths {+ md, err := extractGGUFMetadata(path)+ if err != nil {+ t.Fatalf("projector metadata for %s: %v", path, err)+ }+ m.projectorMetadata = append(m.projectorMetadata, md)+ }+}+func TestModelCapabilities(t *testing.T) {// Create completion model (llama architecture without vision)completionModelPath, _ := createBinFile(t, ggml.KV{@@ -618,6 +638,8 @@ func TestModelCapabilities(t *testing.T) {for _, tt := range testModels {t.Run(tt.name, func(t *testing.T) {+ loadTestMetadata(t, &tt.model)+// Test Capabilities methodcaps := tt.model.Capabilities()if !compareCapabilities(caps, tt.expectedCaps) {@@ -748,6 +770,8 @@ func TestModelCheckCapabilities(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {+ loadTestMetadata(t, &tt.model)+// Test CheckCapabilities methoderr := tt.model.CheckCapabilities(tt.checkCaps...)if tt.expectedErrMsg == "" {diff --git a/server/model_caches.go b/server/model_caches.goindex 06f9e52b9..720fa703d 100644--- a/server/model_caches.go+++ b/server/model_caches.go@@ -5,16 +5,12 @@ import "context"type modelCaches struct {recommendations *modelRecommendationsCacheshow *modelShowCache- modelList *modelListCache- inference *inferenceModelCache}func newModelCaches() *modelCaches {return &modelCaches{recommendations: newModelRecommendationsCache(),show: newModelShowCache(),- modelList: newModelListCache(),- inference: newInferenceModelCache(),}}@@ -28,7 +24,4 @@ func (c *modelCaches) Start(ctx context.Context) {if c.show != nil {c.show.Start(ctx)}- if c.modelList != nil {- c.modelList.Start(ctx)- }}diff --git a/server/model_inference_cache.go b/server/model_inference_cache.godeleted file mode 100644index a5edcbd44..000000000--- a/server/model_inference_cache.go+++ /dev/null@@ -1,121 +0,0 @@-package server--import (- "maps"- "slices"- "strconv"- "sync"-- "github.com/ollama/ollama/envconfig"- "github.com/ollama/ollama/manifest"- "github.com/ollama/ollama/types/model"- "golang.org/x/sync/singleflight"-)--// inferenceModelCache stores fully resolved model metadata and capabilities.-// Model blobs are content-addressed, and the manifest digest is the freshness-// boundary, so cached entries remain valid until the model is recreated or-// pulled with different content.-type inferenceModelCache struct {- mu sync.RWMutex- entries map[inferenceModelCacheKey]inferenceModelCacheEntry- loads singleflight.Group-- loadModel func(string) (*Model, error)-}--type inferenceModelCacheKey struct {- name string- goTemplate bool- goTemplateSet bool-}--type inferenceModelCacheEntry struct {- digest string- model *Model-}--func newInferenceModelCache() *inferenceModelCache {- return &inferenceModelCache{- entries: make(map[inferenceModelCacheKey]inferenceModelCacheEntry),- loadModel: GetModel,- }-}--func (c *inferenceModelCache) Get(name string) (*Model, error) {- n := model.ParseName(name)- mf, err := manifest.ParseNamedManifest(n)- if err != nil {- return nil, err- }-- key := inferenceModelCacheKey{- name: n.String(),- goTemplate: envconfig.GoTemplate(true),- goTemplateSet: goTemplateEnvSet(),- }- digest := mf.Digest()-- c.mu.RLock()- entry, ok := c.entries[key]- c.mu.RUnlock()- if ok && entry.digest == digest {- return cloneInferenceModel(entry.model), nil- }-- loadKey := key.name + "\x00" + digest + "\x00" + strconv.FormatBool(key.goTemplate) + "\x00" + strconv.FormatBool(key.goTemplateSet)- v, err, _ := c.loads.Do(loadKey, func() (any, error) {- c.mu.RLock()- entry, ok := c.entries[key]- c.mu.RUnlock()- if ok && entry.digest == digest {- return entry.model, nil- }-- m, err := c.loadModel(name)- if err != nil {- return nil, err- }- m.capabilities = m.Capabilities()- m.capabilitiesCached = true-- c.mu.Lock()- c.entries[key] = inferenceModelCacheEntry{digest: m.Digest, model: m}- c.mu.Unlock()- return m, nil- })- if err != nil {- return nil, err- }-- return cloneInferenceModel(v.(*Model)), nil-}--func cloneInferenceModel(src *Model) *Model {- if src == nil {- return nil- }-- dst := *src- dst.Config.ModelFamilies = slices.Clone(src.Config.ModelFamilies)- dst.Config.Capabilities = slices.Clone(src.Config.Capabilities)- if src.Config.Draft != nil {- draft := *src.Config.Draft- dst.Config.Draft = &draft- }- dst.AdapterPaths = slices.Clone(src.AdapterPaths)- dst.ProjectorPaths = slices.Clone(src.ProjectorPaths)- dst.License = slices.Clone(src.License)- dst.Options = maps.Clone(src.Options)- dst.Messages = slices.Clone(src.Messages)- dst.capabilities = slices.Clone(src.capabilities)-- return &dst-}--func (s *Server) getModel(name string) (*Model, error) {- if s != nil && s.modelCaches != nil && s.modelCaches.inference != nil {- return s.modelCaches.inference.Get(name)- }- return GetModel(name)-}diff --git a/server/model_inference_cache_test.go b/server/model_inference_cache_test.godeleted file mode 100644index fb53202e2..000000000--- a/server/model_inference_cache_test.go+++ /dev/null@@ -1,121 +0,0 @@-package server--import (- "slices"- "sync"- "sync/atomic"- "testing"- "time"-- "github.com/ollama/ollama/fs/ggml"- "github.com/ollama/ollama/types/model"-)--func TestInferenceModelCache(t *testing.T) {- t.Setenv("OLLAMA_MODELS", t.TempDir())- t.Setenv("OLLAMA_GO_TEMPLATE", "")-- _, completionDigest := createBinFile(t, ggml.KV{- "general.architecture": "llama",- }, nil)- writeTestModelManifest(t, "inference-cache", completionDigest, "{{ .Prompt }}")-- cache := newInferenceModelCache()- loadCount := 0- cache.loadModel = func(name string) (*Model, error) {- loadCount++- return GetModel(name)- }-- first, err := cache.Get("inference-cache")- if err != nil {- t.Fatal(err)- }- if loadCount != 1 {- t.Fatalf("load count = %d, want 1", loadCount)- }- if !first.capabilitiesCached {- t.Fatal("capabilities were not cached")- }- if got := first.Capabilities(); !slices.Contains(got, model.CapabilityCompletion) {- t.Fatalf("capabilities = %v, want completion", got)- }-- // Returned models are request-local clones. Mutating one must not alter the- // cached model or a later request.- first.Config.Parser = "mutated"- first.Config.ModelFamilies[0] = "mutated"- first.capabilities[0] = model.CapabilityImage-- second, err := cache.Get("inference-cache")- if err != nil {- t.Fatal(err)- }- if loadCount != 1 {- t.Fatalf("cache hit load count = %d, want 1", loadCount)- }- if second.Config.Parser == "mutated" || slices.Contains(second.Config.ModelFamilies, "mutated") {- t.Fatalf("cached model was mutated: %#v", second.Config)- }- if got := second.Capabilities(); !slices.Contains(got, model.CapabilityCompletion) || slices.Contains(got, model.CapabilityImage) {- t.Fatalf("cached capabilities = %v, want completion only", got)- }-- // Recreating the manifest changes its digest and invalidates the entry.- _, embeddingDigest := createBinFile(t, ggml.KV{- "general.architecture": "bert",- "bert.pooling_type": uint32(1),- }, nil)- writeTestModelManifest(t, "inference-cache", embeddingDigest, "{{ .Prompt }}")-- third, err := cache.Get("inference-cache")- if err != nil {- t.Fatal(err)- }- if loadCount != 2 {- t.Fatalf("invalidated load count = %d, want 2", loadCount)- }- if got := third.Capabilities(); !slices.Contains(got, model.CapabilityEmbedding) || slices.Contains(got, model.CapabilityCompletion) {- t.Fatalf("refreshed capabilities = %v, want embedding only", got)- }-}--func TestInferenceModelCacheConcurrentMiss(t *testing.T) {- t.Setenv("OLLAMA_MODELS", t.TempDir())- t.Setenv("OLLAMA_GO_TEMPLATE", "")-- _, digest := createBinFile(t, ggml.KV{- "general.architecture": "llama",- }, nil)- writeTestModelManifest(t, "inference-cache-concurrent", digest, "{{ .Prompt }}")-- cache := newInferenceModelCache()- var loadCount atomic.Int32- cache.loadModel = func(name string) (*Model, error) {- loadCount.Add(1)- time.Sleep(10 * time.Millisecond)- return GetModel(name)- }-- var wg sync.WaitGroup- errs := make(chan error, 8)- for range 8 {- wg.Add(1)- go func() {- defer wg.Done()- _, err := cache.Get("inference-cache-concurrent")- errs <- err- }()- }- wg.Wait()- close(errs)-- for err := range errs {- if err != nil {- t.Fatal(err)- }- }- if got := loadCount.Load(); got != 1 {- t.Fatalf("load count = %d, want 1", got)- }-}diff --git a/server/model_list.go b/server/model_list.gonew file mode 100644index 000000000..237e1b63c--- /dev/null+++ b/server/model_list.go@@ -0,0 +1,133 @@+package server++import (+ "cmp"+ "context"+ "encoding/json"+ "log/slog"+ "slices"+ "time"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/fs/ggml"+ "github.com/ollama/ollama/manifest"+ "github.com/ollama/ollama/types/model"+)++// listModels builds /api/tags from the manifests and the per-blob metadata+// files, extracting for any blob that has none yet.+func listModels(ctx context.Context) ([]api.ListModelResponse, error) {+ manifests, err := manifest.Manifests(true)+ if err != nil {+ return nil, err+ }++ models := make([]api.ListModelResponse, 0, len(manifests))+ for name, mf := range manifests {+ if ctx != nil {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ }++ summary, err := describeModel(name, mf)+ if err != nil {+ slog.Warn("failed to describe model", "model", name.String(), "error", err)+ continue+ }+ models = append(models, summary)+ }++ sortListModelResponses(models)+ return models, nil+}++// describeModel describes one model for /api/tags. Capabilities come from the+// same Model.Capabilities() the inference path uses, so the two cannot drift.+func describeModel(name model.Name, mf *manifest.Manifest) (api.ListModelResponse, error) {+ cfg, err := readModelListConfig(mf)+ if err != nil {+ return api.ListModelResponse{}, err+ }++ var modified time.Time+ if fi := mf.FileInfo(); fi != nil {+ modified = fi.ModTime()+ }++ summary := api.ListModelResponse{+ Model: name.DisplayShortest(),+ Name: name.DisplayShortest(),+ RemoteModel: cfg.RemoteModel,+ RemoteHost: cfg.RemoteHost,+ Size: mf.Size(),+ Digest: mf.Digest(),+ ModifiedAt: modified,+ Details: api.ModelDetails{+ Format: cfg.ModelFormat,+ Family: cfg.ModelFamily,+ Families: append([]string(nil), cfg.ModelFamilies...),+ ParameterSize: cfg.ModelType,+ QuantizationLevel: cfg.FileType,+ ContextLength: cfg.ContextLen,+ EmbeddingLength: cfg.EmbedLen,+ },+ }++ m, err := GetModel(name.String())+ if err != nil {+ // A model that will not load is the one a user most needs to see, in+ // order to remove it. Report what the manifest says.+ slog.Warn("could not load model to describe it", "model", name.String(), "error", err)+ return summary, nil+ }+ summary.Details.ParentModel = m.ParentModel+ summary.Capabilities = m.Capabilities()++ if m.ModelPath != "" && m.isGGUF() {+ if summary.Details.ContextLength == 0 {+ summary.Details.ContextLength = int(m.metadata.Int("context_length"))+ }+ if summary.Details.EmbeddingLength == 0 {+ summary.Details.EmbeddingLength = int(m.metadata.Int("embedding_length"))+ }+ if m.metadata.Valid("general.file_type") {+ fileType := ggml.FileType(m.metadata.Int("general.file_type")).String()+ if isUnknownQuantization(summary.Details.QuantizationLevel) && !isUnknownQuantization(fileType) {+ summary.Details.QuantizationLevel = fileType+ }+ }+ }++ return summary, nil+}++func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {+ var cfg model.ConfigV2+ if mf == nil || mf.Config.Digest == "" {+ return cfg, nil+ }++ f, err := mf.Config.Open()+ if err != nil {+ return cfg, err+ }+ defer f.Close()++ if err := json.NewDecoder(f).Decode(&cfg); err != nil {+ return cfg, err+ }++ return cfg, nil+}++func isUnknownQuantization(quantization string) bool {+ return quantization == "" || quantization == "unknown"+}++func sortListModelResponses(models []api.ListModelResponse) {+ slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {+ // Preserve the existing /api/tags order: most recently modified first.+ return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())+ })+}diff --git a/server/model_list_cache.go b/server/model_list_cache.godeleted file mode 100644index 9017aae70..000000000--- a/server/model_list_cache.go+++ /dev/null@@ -1,888 +0,0 @@-package server--import (- "bufio"- "cmp"- "context"- "encoding/binary"- "encoding/json"- "fmt"- "io"- "log/slog"- "os"- "slices"- "strings"- "sync"- "time"-- "github.com/ollama/ollama/api"- "github.com/ollama/ollama/fs/ggml"- fsgguf "github.com/ollama/ollama/fs/gguf"- "github.com/ollama/ollama/manifest"- "github.com/ollama/ollama/model/parsers"- ollamatemplate "github.com/ollama/ollama/template"- "github.com/ollama/ollama/thinking"- "github.com/ollama/ollama/types/model"-)--type modelListSummary struct {- Model string- Name string- RemoteModel string- RemoteHost string- Size int64- Digest string- ModifiedAt time.Time- Details api.ModelDetails- Capabilities []model.Capability-}--type modelListCacheEntry struct {- Digest string- Summary modelListSummary-}--type modelListCache struct {- mu sync.RWMutex-- entries map[string]modelListCacheEntry-- once sync.Once- readyOnce sync.Once- ready chan struct{}- hydrateErr error- build func(model.Name, *manifest.Manifest) (modelListSummary, error)-}--func newModelListCache() *modelListCache {- return &modelListCache{- entries: make(map[string]modelListCacheEntry),- ready: make(chan struct{}),- build: buildModelListSummary,- }-}--func (c *modelListCache) Start(ctx context.Context) {- if c == nil {- return- }-- c.once.Do(func() {- slog.Debug("starting model list cache")- go func() {- err := c.hydrate(ctx)- c.markReady(err)- if err != nil {- if ctx != nil && ctx.Err() != nil {- return- }- slog.Warn("model list cache hydration failed", "error", err)- }- }()- })-}--func (c *modelListCache) hydrate(ctx context.Context) error {- start := time.Now()-- manifests, err := manifest.Manifests(true)- if err != nil {- return err- }-- var hydrated, failed int- for name, mf := range manifests {- if ctx != nil {- if err := ctx.Err(); err != nil {- return err- }- }-- summary, err := c.build(name, mf)- if err != nil {- failed++- slog.Warn("failed to hydrate model list cache", "model", name.String(), "error", err)- continue- }-- c.set(name, mf.Digest(), summary)- hydrated++- }-- slog.Info("model list cache hydration complete", "models", hydrated, "failures", failed, "elapsed", time.Since(start))- return nil-}--func (c *modelListCache) markReady(err error) {- c.mu.Lock()- c.hydrateErr = err- c.mu.Unlock()-- c.readyOnce.Do(func() {- close(c.ready)- })-}--func (c *modelListCache) Wait(ctx context.Context) error {- if c == nil {- return nil- }- if ctx == nil {- ctx = context.Background()- }-- select {- case <-c.ready:- c.mu.RLock()- err := c.hydrateErr- c.mu.RUnlock()- return err- case <-ctx.Done():- return ctx.Err()- }-}--func (c *modelListCache) List(ctx context.Context) ([]api.ListModelResponse, error) {- if err := c.Wait(ctx); err != nil {- return nil, err- }- if err := c.syncManifests(ctx); err != nil {- return nil, err- }-- c.mu.RLock()- models := make([]api.ListModelResponse, 0, len(c.entries))- for _, entry := range c.entries {- models = append(models, entry.Summary.ListModelResponse())- }- c.mu.RUnlock()-- sortListModelResponses(models)- return models, nil-}--func (c *modelListCache) syncManifests(ctx context.Context) error {- manifests, err := manifest.Manifests(true)- if err != nil {- return err- }-- c.mu.RLock()- current := make(map[string]string, len(c.entries))- for name, entry := range c.entries {- current[name] = entry.Digest- }- c.mu.RUnlock()-- type update struct {- name model.Name- digest string- summary modelListSummary- }-- seen := make(map[string]struct{}, len(manifests))- stale := make(map[string]struct{})- var updates []update- for name, mf := range manifests {- if ctx != nil {- if err := ctx.Err(); err != nil {- return err- }- }-- key := name.String()- digest := mf.Digest()- seen[key] = struct{}{}- if current[key] == digest {- continue- }-- summary, err := c.build(name, mf)- if err != nil {- slog.Warn("failed to refresh model list cache", "model", key, "error", err)- if _, ok := current[key]; ok {- stale[key] = struct{}{}- }- continue- }- updates = append(updates, update{name: name, digest: digest, summary: summary})- }-- c.mu.Lock()- for name := range c.entries {- if _, ok := seen[name]; !ok {- delete(c.entries, name)- continue- }- if _, ok := stale[name]; ok {- delete(c.entries, name)- }- }- for _, update := range updates {- c.entries[update.name.String()] = modelListCacheEntry{- Digest: update.digest,- Summary: cloneModelListSummary(update.summary),- }- }- c.mu.Unlock()-- return nil-}--func (c *modelListCache) RefreshModel(name model.Name) error {- if c == nil {- return nil- }-- if !name.IsFullyQualified() {- var err error- name, err = getExistingName(name)- if err != nil {- return err- }- }-- mf, err := manifest.ParseNamedManifest(name)- if err != nil {- c.DeleteModel(name)- return err- }-- summary, err := c.build(name, mf)- if err != nil {- c.DeleteModel(name)- return err- }-- c.set(name, mf.Digest(), summary)- return nil-}--func (c *modelListCache) DeleteModel(name model.Name) {- if c == nil {- return- }-- c.mu.Lock()- delete(c.entries, name.String())- c.mu.Unlock()-}--func (c *modelListCache) Get(name model.Name) (modelListSummary, bool) {- if c == nil {- return modelListSummary{}, false- }-- if !name.IsFullyQualified() {- if existing, err := getExistingName(name); err == nil {- name = existing- }- }-- c.mu.RLock()- entry, ok := c.entries[name.String()]- c.mu.RUnlock()- if !ok {- return modelListSummary{}, false- }-- return cloneModelListSummary(entry.Summary), true-}--func (c *modelListCache) Len() int {- if c == nil {- return 0- }-- c.mu.RLock()- defer c.mu.RUnlock()- return len(c.entries)-}--func (c *modelListCache) set(name model.Name, digest string, summary modelListSummary) {- c.mu.Lock()- c.entries[name.String()] = modelListCacheEntry{- Digest: digest,- Summary: cloneModelListSummary(summary),- }- c.mu.Unlock()-}--func buildModelListSummary(name model.Name, mf *manifest.Manifest) (modelListSummary, error) {- cfg, err := readModelListConfig(mf)- if err != nil {- return modelListSummary{}, err- }-- var modified time.Time- if fi := mf.FileInfo(); fi != nil {- modified = fi.ModTime()- }-- summary := modelListSummary{- Model: name.DisplayShortest(),- Name: name.DisplayShortest(),- RemoteModel: cfg.RemoteModel,- RemoteHost: cfg.RemoteHost,- Size: mf.Size(),- Digest: mf.Digest(),- ModifiedAt: modified,- Details: api.ModelDetails{- Format: cfg.ModelFormat,- Family: cfg.ModelFamily,- Families: append([]string(nil), cfg.ModelFamilies...),- ParameterSize: cfg.ModelType,- QuantizationLevel: cfg.FileType,- ContextLength: cfg.ContextLen,- EmbeddingLength: cfg.EmbedLen,- },- }-- modelPath, projectorCount, tmpl, err := readModelListLayers(mf, &summary)- if err != nil {- return modelListSummary{}, err- }-- if cfg.RemoteHost == "" && cfg.RemoteModel == "" && modelPath != "" {- info, err := readModelListGGUF(modelPath)- if err != nil {- slog.Debug("failed to read gguf model metadata", "model", name.String(), "error", err)- } else {- summary.Capabilities = appendModelListCapabilities(summary.Capabilities, info.Capabilities...)- if summary.Details.ContextLength == 0 {- summary.Details.ContextLength = info.ContextLength- }- if summary.Details.EmbeddingLength == 0 {- summary.Details.EmbeddingLength = info.EmbeddingLength- }- if isUnknownQuantization(summary.Details.QuantizationLevel) && !isUnknownQuantization(info.FileType) {- summary.Details.QuantizationLevel = info.FileType- }- }- }-- for _, c := range cfg.Capabilities {- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.Capability(c))- }-- builtinParser := parsers.ParserForName(cfg.Parser)- if tmpl != nil {- vars, err := tmpl.Vars()- if err != nil {- slog.Warn("model template contains errors", "model", name.String(), "error", err)- }- if slices.Contains(vars, "tools") || (builtinParser != nil && builtinParser.HasToolSupport()) {- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityTools)- }- if slices.Contains(vars, "suffix") {- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityInsert)- }-- openingTag, closingTag := thinking.InferTags(tmpl.Template)- hasTags := openingTag != "" && closingTag != ""- isGptoss := slices.Contains([]string{"gptoss", "gpt-oss"}, cfg.ModelFamily)- if !slices.Contains(summary.Capabilities, model.CapabilityThinking) &&- (hasTags || isGptoss || (builtinParser != nil && builtinParser.HasThinkingSupport())) {- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityThinking)- }- }-- if projectorCount > 0 {- summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityVision)- }-- summary.Capabilities = filterUnsupportedModelListCapabilities(summary.Capabilities, cfg)-- return summary, nil-}--func filterUnsupportedModelListCapabilities(capabilities []model.Capability, cfg model.ConfigV2) []model.Capability {- if cfg.ModelFormat == "safetensors" && isNemotron3NanoSafetensorsConfig(cfg) {- capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool {- return c == model.CapabilityVision || c == model.CapabilityAudio- })- }- // Mirrors suppressAudioCapability in images.go so /api/tags and /api/show- // agree for safetensors models whose MLX runner serves vision but not audio.- if cfg.ModelFormat == "safetensors" && cfg.Renderer == "glimmer" {- capabilities = slices.DeleteFunc(capabilities, func(c model.Capability) bool {- return c == model.CapabilityAudio- })- }-- return capabilities-}--func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) {- var cfg model.ConfigV2- if mf == nil || mf.Config.Digest == "" {- return cfg, nil- }-- f, err := mf.Config.Open()- if err != nil {- return cfg, err- }- defer f.Close()-- if err := json.NewDecoder(f).Decode(&cfg); err != nil {- return cfg, err- }-- return cfg, nil-}--func readModelListLayers(mf *manifest.Manifest, summary *modelListSummary) (string, int, *ollamatemplate.Template, error) {- var modelPath string- var projectorCount int- tmpl := ollamatemplate.DefaultTemplate-- for _, layer := range mf.Layers {- switch layer.MediaType {- case "application/vnd.ollama.image.model":- filename, err := manifest.BlobsPath(layer.Digest)- if err != nil {- return "", 0, nil, err- }- modelPath = filename- summary.Details.ParentModel = layer.From- case "application/vnd.ollama.image.projector":- projectorCount++- case "application/vnd.ollama.image.prompt",- "application/vnd.ollama.image.template":- filename, err := manifest.BlobsPath(layer.Digest)- if err != nil {- return "", 0, nil, err- }- bts, err := os.ReadFile(filename)- if err != nil {- return "", 0, nil, err- }-- tmpl, err = ollamatemplate.Parse(string(bts))- if err != nil {- return "", 0, nil, err- }- }- }-- return modelPath, projectorCount, tmpl, nil-}--type modelListGGUF struct {- Capabilities []model.Capability- ContextLength int- EmbeddingLength int- FileType string-}--const (- modelListGGUFMagicLE = 0x46554747- modelListGGUFMagicBE = 0x47475546-)--const (- modelListGGUFTypeUint8 uint32 = iota- modelListGGUFTypeInt8- modelListGGUFTypeUint16- modelListGGUFTypeInt16- modelListGGUFTypeUint32- modelListGGUFTypeInt32- modelListGGUFTypeFloat32- modelListGGUFTypeBool- modelListGGUFTypeString- modelListGGUFTypeArray- modelListGGUFTypeUint64- modelListGGUFTypeInt64- modelListGGUFTypeFloat64-)--// readModelListGGUF scans only the small GGUF header values launch needs-// and stops before tokenizer arrays. Using gguf.File.KeyValue for missing keys-// can otherwise advance through large arrays just to discover absence.-func readModelListGGUF(path string) (modelListGGUF, error) {- f, err := os.Open(path)- if err != nil {- return modelListGGUF{}, err- }- defer f.Close()-- r := bufio.NewReaderSize(f, 32<<10)- var magic uint32- if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {- return modelListGGUF{}, err- }-- var byteOrder binary.ByteOrder = binary.LittleEndian- switch magic {- case modelListGGUFMagicLE:- case modelListGGUFMagicBE:- byteOrder = binary.BigEndian- default:- return modelListGGUF{}, fmt.Errorf("invalid file magic")- }-- var version uint32- if err := binary.Read(r, byteOrder, &version); err != nil {- return modelListGGUF{}, err- }-- var numKV uint64- switch version {- case 1:- var header struct {- NumTensor uint32- NumKV uint32- }- if err := binary.Read(r, byteOrder, &header); err != nil {- return modelListGGUF{}, err- }- numKV = uint64(header.NumKV)- default:- var header struct {- NumTensor uint64- NumKV uint64- }- if err := binary.Read(r, byteOrder, &header); err != nil {- return modelListGGUF{}, err- }- numKV = header.NumKV- }-- info := modelListGGUF{}- var architecture string- var hasPoolingType bool-- for range numKV {- key, err := readModelListGGUFString(r, byteOrder, version)- if err != nil {- return modelListGGUF{}, err- }-- var valueType uint32- if err := binary.Read(r, byteOrder, &valueType); err != nil {- return modelListGGUF{}, err- }-- if key == "general.architecture" {- value, err := readModelListGGUFStringValue(r, byteOrder, version, valueType)- if err != nil {- return modelListGGUF{}, err- }- architecture = value- continue- }-- if key == "general.file_type" {- value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)- if err != nil {- return modelListGGUF{}, err- }- info.FileType = ggml.FileType(value).String()- continue- }-- if architecture != "" && strings.HasPrefix(key, "tokenizer.") {- break- }-- if architecture != "" && strings.HasPrefix(key, architecture+".") {- switch strings.TrimPrefix(key, architecture+".") {- case "pooling_type":- hasPoolingType = true- case "vision.block_count":- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityVision)- case "audio.block_count":- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityAudio)- case "context_length":- value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)- if err != nil {- return modelListGGUF{}, err- }- info.ContextLength = value- continue- case "embedding_length":- value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType)- if err != nil {- return modelListGGUF{}, err- }- info.EmbeddingLength = value- continue- }- }-- if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {- return modelListGGUF{}, err- }- }-- if hasPoolingType {- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityEmbedding)- } else {- info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityCompletion)- }-- return info, nil-}--func readModelListGGUFStringValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (string, error) {- if valueType != modelListGGUFTypeString {- if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {- return "", err- }- return "", fmt.Errorf("unexpected gguf string type %d", valueType)- }- return readModelListGGUFString(r, byteOrder, version)-}--func readModelListGGUFIntValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (int, error) {- switch valueType {- case modelListGGUFTypeUint8:- var value uint8- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeInt8:- var value int8- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeUint16:- var value uint16- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeInt16:- var value int16- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeUint32:- var value uint32- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeInt32:- var value int32- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeUint64:- var value uint64- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- case modelListGGUFTypeInt64:- var value int64- if err := binary.Read(r, byteOrder, &value); err != nil {- return 0, err- }- return int(value), nil- default:- if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil {- return 0, err- }- return 0, fmt.Errorf("unexpected gguf integer type %d", valueType)- }-}--func skipModelListGGUFValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) error {- switch valueType {- case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:- return discardModelListGGUFBytes(r, 1)- case modelListGGUFTypeUint16, modelListGGUFTypeInt16:- return discardModelListGGUFBytes(r, 2)- case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:- return discardModelListGGUFBytes(r, 4)- case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:- return discardModelListGGUFBytes(r, 8)- case modelListGGUFTypeString:- return skipModelListGGUFString(r, byteOrder, version)- case modelListGGUFTypeArray:- var arrayType uint32- if err := binary.Read(r, byteOrder, &arrayType); err != nil {- return err- }- var count uint64- if err := binary.Read(r, byteOrder, &count); err != nil {- return err- }- return skipModelListGGUFArray(r, byteOrder, version, arrayType, count)- default:- return fmt.Errorf("unsupported gguf value type %d", valueType)- }-}--func skipModelListGGUFArray(r io.Reader, byteOrder binary.ByteOrder, version uint32, arrayType uint32, count uint64) error {- if _, err := checkedModelListGGUFLength(count, "array size", fsgguf.MaxArraySize); err != nil {- return err- }-- var size uint64- switch arrayType {- case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool:- size = 1- case modelListGGUFTypeUint16, modelListGGUFTypeInt16:- size = 2- case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32:- size = 4- case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64:- size = 8- case modelListGGUFTypeString:- for range count {- if err := skipModelListGGUFString(r, byteOrder, version); err != nil {- return err- }- }- return nil- default:- return fmt.Errorf("unsupported gguf array type %d", arrayType)- }-- if count > uint64(maxModelListGGUFInt64())/size {- return fmt.Errorf("gguf array byte length %d*%d exceeds maximum %d", count, size, maxModelListGGUFInt64())- }- return discardModelListGGUFBytes(r, int64(count*size))-}--func readModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) (string, error) {- var length uint64- if err := binary.Read(r, byteOrder, &length); err != nil {- return "", err- }-- n, err := checkedModelListGGUFLength(length, "string", fsgguf.MaxStringLength)- if err != nil {- return "", err- }-- bts := make([]byte, n)- if _, err := io.ReadFull(r, bts); err != nil {- return "", err- }- if version == 1 && len(bts) > 0 && bts[len(bts)-1] == 0 {- bts = bts[:len(bts)-1]- }- return string(bts), nil-}--func skipModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) error {- var length uint64- if err := binary.Read(r, byteOrder, &length); err != nil {- return err- }-- n, err := checkedModelListGGUFLength(length, "string", fsgguf.MaxStringLength)- if err != nil {- return err- }- return discardModelListGGUFBytes(r, int64(n))-}--func discardModelListGGUFBytes(r io.Reader, n int64) error {- if n <= 0 {- return nil- }- _, err := io.CopyN(io.Discard, r, n)- return err-}--func checkedModelListGGUFLength(n uint64, kind string, max uint64) (int, error) {- if n > uint64(maxModelListGGUFInt()) {- return 0, fmt.Errorf("gguf %s %d exceeds maximum %d", kind, n, maxModelListGGUFInt())- }- if n > max {- return 0, fmt.Errorf("gguf %s %d exceeds maximum %d", kind, n, max)- }- return int(n), nil-}--func maxModelListGGUFInt() int {- return int(^uint(0) >> 1)-}--func maxModelListGGUFInt64() int64 {- return 1<<63 - 1-}--func appendModelListCapabilities(capabilities []model.Capability, values ...model.Capability) []model.Capability {- for _, capability := range values {- capabilities = appendModelListCapability(capabilities, capability)- }- return capabilities-}--func appendModelListCapability(capabilities []model.Capability, capability model.Capability) []model.Capability {- if capability == "" || slices.Contains(capabilities, capability) {- return capabilities- }- return append(capabilities, capability)-}--func isUnknownQuantization(quantization string) bool {- return quantization == "" || quantization == "unknown"-}--func cloneModelListSummary(summary modelListSummary) modelListSummary {- summary.Details.Families = append([]string(nil), summary.Details.Families...)- summary.Capabilities = append([]model.Capability(nil), summary.Capabilities...)- return summary-}--func (s modelListSummary) ListModelResponse() api.ListModelResponse {- resp := api.ListModelResponse{- Model: s.Model,- Name: s.Name,- RemoteModel: s.RemoteModel,- RemoteHost: s.RemoteHost,- Size: s.Size,- Digest: s.Digest,- ModifiedAt: s.ModifiedAt,- Details: api.ModelDetails{- ParentModel: s.Details.ParentModel,- Format: s.Details.Format,- Family: s.Details.Family,- Families: append([]string(nil), s.Details.Families...),- ParameterSize: s.Details.ParameterSize,- QuantizationLevel: s.Details.QuantizationLevel,- ContextLength: s.Details.ContextLength,- EmbeddingLength: s.Details.EmbeddingLength,- },- }-- resp.Capabilities = append([]model.Capability(nil), s.Capabilities...)-- return resp-}--func sortListModelResponses(models []api.ListModelResponse) {- slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {- // Preserve the existing /api/tags order: most recently modified first.- return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())- })-}--func (s *Server) refreshModelListCache(name model.Name) {- if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {- return- }-- if err := s.modelCaches.modelList.RefreshModel(name); err != nil {- slog.Warn("failed to refresh model list cache", "model", name.String(), "error", err)- }-}--func (s *Server) deleteModelListCache(name model.Name) {- if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil {- return- }-- s.modelCaches.modelList.DeleteModel(name)-}diff --git a/server/model_list_cache_test.go b/server/model_list_cache_test.godeleted file mode 100644index f9fb960cd..000000000--- a/server/model_list_cache_test.go+++ /dev/null@@ -1,371 +0,0 @@-package server--import (- "bytes"- "context"- "encoding/binary"- "errors"- "net/http"- "os"- "slices"- "strings"- "testing"-- "github.com/gin-gonic/gin"-- "github.com/ollama/ollama/api"- fsgguf "github.com/ollama/ollama/fs/gguf"- "github.com/ollama/ollama/manifest"- "github.com/ollama/ollama/types/model"-)--func TestModelListCacheHydratesSummary(t *testing.T) {- gin.SetMode(gin.TestMode)- setTestHome(t, t.TempDir())- createListCacheModel(t, "list-cache", map[string]any{- "test.context_length": uint32(4096),- "test.embedding_length": uint32(384),- }, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")-- cache := newModelListCache()- if err := cache.hydrate(context.Background()); err != nil {- t.Fatalf("hydrate failed: %v", err)- }-- summary, ok := cache.Get(model.ParseName("list-cache"))- if !ok {- t.Fatal("list summary missing")- }-- if summary.Model != "list-cache:latest" || summary.Name != "list-cache:latest" {- t.Fatalf("summary model/name = %q/%q, want list-cache:latest", summary.Model, summary.Name)- }- if summary.Digest == "" {- t.Fatal("summary digest is empty")- }- if summary.Size == 0 {- t.Fatal("summary size is zero")- }- if summary.Details.Family != "test" || summary.Details.Format != "gguf" {- t.Fatalf("summary details = %+v, want gguf/test", summary.Details)- }- if summary.Details.ContextLength != 4096 {- t.Fatalf("context length = %d, want 4096", summary.Details.ContextLength)- }- if summary.Details.EmbeddingLength != 384 {- t.Fatalf("embedding length = %d, want 384", summary.Details.EmbeddingLength)- }-- for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {- if !slices.Contains(summary.Capabilities, capability) {- t.Fatalf("capabilities = %v, want %s", summary.Capabilities, capability)- }- }-- listModel := summary.ListModelResponse()- if !slices.Contains(listModel.Capabilities, model.CapabilityTools) ||- listModel.Details.ContextLength != 4096 ||- listModel.Details.EmbeddingLength != 384 {- t.Fatalf("list response = %+v, want capabilities/context/embedding", listModel)- }-}--func TestModelListCacheSuppressesNemotronSafetensorsMedia(t *testing.T) {- caps := []model.Capability{- model.CapabilityCompletion,- model.CapabilityTools,- model.CapabilityThinking,- model.CapabilityVision,- model.CapabilityAudio,- }- got := filterUnsupportedModelListCapabilities(caps, model.ConfigV2{- ModelFormat: "safetensors",- Renderer: "nemotron-3-nano",- Parser: "nemotron-3-nano",- })-- for _, capability := range []model.Capability{- model.CapabilityCompletion,- model.CapabilityTools,- model.CapabilityThinking,- } {- if !slices.Contains(got, capability) {- t.Fatalf("capabilities = %v, want %s", got, capability)- }- }- for _, capability := range []model.Capability{model.CapabilityVision, model.CapabilityAudio} {- if slices.Contains(got, capability) {- t.Fatalf("capabilities = %v, did not expect %s", got, capability)- }- }-}--func TestModelListCacheRefreshUpdatesEntry(t *testing.T) {- gin.SetMode(gin.TestMode)- setTestHome(t, t.TempDir())- createListCacheModel(t, "list-refresh", map[string]any{"test.context_length": uint32(1024)}, "")-- cache := newModelListCache()- if err := cache.hydrate(context.Background()); err != nil {- t.Fatalf("hydrate failed: %v", err)- }-- name := model.ParseName("list-refresh")- first, ok := cache.Get(name)- if !ok {- t.Fatal("list summary missing")- }-- changeShowCacheManifest(t, "list-refresh")- if err := cache.RefreshModel(name); err != nil {- t.Fatalf("refresh failed: %v", err)- }-- refreshed, ok := cache.Get(name)- if !ok {- t.Fatal("refreshed list summary missing")- }- if refreshed.Digest == first.Digest {- t.Fatalf("digest did not change after refresh: %s", refreshed.Digest)- }- if cache.Len() != 1 {- t.Fatalf("cache entries = %d, want 1", cache.Len())- }-}--func TestModelListCacheMutationHooks(t *testing.T) {- gin.SetMode(gin.TestMode)- setTestHome(t, t.TempDir())-- cache := newModelListCache()- s := Server{modelCaches: &modelCaches{modelList: cache}}-- _, digest := createBinFile(t, map[string]any{"test.context_length": uint32(2048)}, nil)- w := createRequest(t, s.CreateHandler, api.CreateRequest{- Model: "list-hooks",- Files: map[string]string{"model.gguf": digest},- Stream: &stream,- })- if w.Code != http.StatusOK {- t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())- }-- if _, ok := cache.Get(model.ParseName("list-hooks")); !ok {- t.Fatal("create did not refresh model list cache")- }-- w = createRequest(t, s.CopyHandler, api.CopyRequest{- Source: "list-hooks",- Destination: "list-hooks-copy",- })- if w.Code != http.StatusOK {- t.Fatalf("copy model status = %d, want 200: %s", w.Code, w.Body.String())- }- if _, ok := cache.Get(model.ParseName("list-hooks-copy")); !ok {- t.Fatal("copy did not refresh model list cache")- }-- w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: "list-hooks-copy"})- if w.Code != http.StatusOK {- t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())- }- if _, ok := cache.Get(model.ParseName("list-hooks-copy")); ok {- t.Fatal("delete did not remove model list cache entry")- }-}--func TestModelListCacheSyncsManifestChanges(t *testing.T) {- gin.SetMode(gin.TestMode)- setTestHome(t, t.TempDir())- createListCacheModel(t, "list-sync-a", map[string]any{"test.context_length": uint32(1024)}, "")-- cache := newModelListCache()- cache.Start(context.Background())- if err := cache.Wait(context.Background()); err != nil {- t.Fatalf("wait failed: %v", err)- }-- createListCacheModel(t, "list-sync-b", map[string]any{"test.context_length": uint32(2048)}, "")- models, err := cache.List(context.Background())- if err != nil {- t.Fatalf("list failed: %v", err)- }-- names := make([]string, 0, len(models))- for _, m := range models {- names = append(names, m.Name)- }- for _, want := range []string{"list-sync-a:latest", "list-sync-b:latest"} {- if !slices.Contains(names, want) {- t.Fatalf("names = %v, want %s", names, want)- }- }-- var other Server- w := createRequest(t, other.DeleteHandler, api.DeleteRequest{Model: "list-sync-a"})- if w.Code != http.StatusOK {- t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String())- }-- models, err = cache.List(context.Background())- if err != nil {- t.Fatalf("list after delete failed: %v", err)- }- names = names[:0]- for _, m := range models {- names = append(names, m.Name)- }- if slices.Contains(names, "list-sync-a:latest") || !slices.Contains(names, "list-sync-b:latest") {- t.Fatalf("names after delete = %v, want only list-sync-b", names)- }-}--func TestModelListCacheSyncDropsStaleEntryOnRefreshFailure(t *testing.T) {- gin.SetMode(gin.TestMode)- setTestHome(t, t.TempDir())- createListCacheModel(t, "list-stale", map[string]any{"test.context_length": uint32(1024)}, "")-- cache := newModelListCache()- cache.Start(context.Background())- if err := cache.Wait(context.Background()); err != nil {- t.Fatalf("wait failed: %v", err)- }-- name := model.ParseName("list-stale")- if _, ok := cache.Get(name); !ok {- t.Fatal("list summary missing")- }-- changeShowCacheManifest(t, "list-stale")- cache.build = func(model.Name, *manifest.Manifest) (modelListSummary, error) {- return modelListSummary{}, errors.New("refresh failed")- }-- models, err := cache.List(context.Background())- if err != nil {- t.Fatalf("list failed: %v", err)- }- if len(models) != 0 {- t.Fatalf("models = %+v, want stale entry removed", models)- }- if _, ok := cache.Get(name); ok {- t.Fatal("stale entry remained in cache after refresh failure")- }-}--func TestReadModelListGGUFRejectsMalformedMetadata(t *testing.T) {- cases := []struct {- name string- data []byte- want string- }{- {- name: "oversized key string",- data: modelListGGUFTestFile(func(b *bytes.Buffer) {- writeModelListGGUFHeader(t, b, 1)- writeModelListGGUFUint64(t, b, fsgguf.MaxStringLength+1)- }),- want: "string",- },- {- name: "oversized skipped string",- data: modelListGGUFTestFile(func(b *bytes.Buffer) {- writeModelListGGUFHeader(t, b, 1)- writeModelListGGUFString(t, b, "unused")- writeModelListGGUFUint32(t, b, modelListGGUFTypeString)- writeModelListGGUFUint64(t, b, fsgguf.MaxStringLength+1)- }),- want: "string",- },- {- name: "oversized skipped array",- data: modelListGGUFTestFile(func(b *bytes.Buffer) {- writeModelListGGUFHeader(t, b, 1)- writeModelListGGUFString(t, b, "unused")- writeModelListGGUFUint32(t, b, modelListGGUFTypeArray)- writeModelListGGUFUint32(t, b, modelListGGUFTypeUint8)- writeModelListGGUFUint64(t, b, fsgguf.MaxArraySize+1)- }),- want: "array size",- },- }-- for _, tt := range cases {- t.Run(tt.name, func(t *testing.T) {- defer func() {- if r := recover(); r != nil {- t.Fatalf("readModelListGGUF panicked: %v", r)- }- }()-- path := t.TempDir() + "/model.gguf"- if err := os.WriteFile(path, tt.data, 0o600); err != nil {- t.Fatal(err)- }-- _, err := readModelListGGUF(path)- if err == nil {- t.Fatal("expected error")- }- if !strings.Contains(err.Error(), tt.want) {- t.Fatalf("error = %v, want substring %q", err, tt.want)- }- })- }-}--func createListCacheModel(t *testing.T, name string, kv map[string]any, tmpl string) {- t.Helper()- _, digest := createBinFile(t, kv, nil)-- req := api.CreateRequest{- Model: name,- Files: map[string]string{"model.gguf": digest},- Stream: &stream,- }- if tmpl != "" {- req.Template = tmpl- }-- var s Server- w := createRequest(t, s.CreateHandler, req)- if w.Code != http.StatusOK {- t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String())- }-}--func modelListGGUFTestFile(fn func(*bytes.Buffer)) []byte {- var b bytes.Buffer- fn(&b)- return b.Bytes()-}--func writeModelListGGUFHeader(t *testing.T, b *bytes.Buffer, numKV uint64) {- t.Helper()- writeModelListGGUFUint32(t, b, modelListGGUFMagicLE)- writeModelListGGUFUint32(t, b, 3)- writeModelListGGUFUint64(t, b, 0)- writeModelListGGUFUint64(t, b, numKV)-}--func writeModelListGGUFString(t *testing.T, b *bytes.Buffer, s string) {- t.Helper()- writeModelListGGUFUint64(t, b, uint64(len(s)))- if _, err := b.WriteString(s); err != nil {- t.Fatal(err)- }-}--func writeModelListGGUFUint32(t *testing.T, b *bytes.Buffer, v uint32) {- t.Helper()- if err := binary.Write(b, binary.LittleEndian, v); err != nil {- t.Fatal(err)- }-}--func writeModelListGGUFUint64(t *testing.T, b *bytes.Buffer, v uint64) {- t.Helper()- if err := binary.Write(b, binary.LittleEndian, v); err != nil {- t.Fatal(err)- }-}diff --git a/server/model_list_test.go b/server/model_list_test.gonew file mode 100644index 000000000..44fd83036--- /dev/null+++ b/server/model_list_test.go@@ -0,0 +1,168 @@+package server++import (+ "context"+ "os"+ "slices"+ "testing"++ "github.com/gin-gonic/gin"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/manifest"+ "github.com/ollama/ollama/types/model"+)++func listedModel(t *testing.T, name string) api.ListModelResponse {+ t.Helper()+ models, err := listModels(context.Background())+ if err != nil {+ t.Fatalf("listModels failed: %v", err)+ }+ for _, m := range models {+ if m.Name == name {+ return m+ }+ }+ t.Fatalf("%s not listed; got %v", name, models)+ return api.ListModelResponse{}+}++func TestListModelsDescribesModel(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createListedModelFromKV(t, "list-describe", map[string]any{+ "test.context_length": uint32(4096),+ "test.embedding_length": uint32(384),+ }, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")++ got := listedModel(t, "list-describe:latest")++ if got.Model != "list-describe:latest" {+ t.Errorf("model = %q", got.Model)+ }+ if got.Digest == "" || got.Size == 0 {+ t.Errorf("digest = %q size = %d, want both set", got.Digest, got.Size)+ }+ if got.Details.Family != "test" || got.Details.Format != "gguf" {+ t.Errorf("details = %+v, want gguf/test", got.Details)+ }+ if got.Details.ContextLength != 4096 {+ t.Errorf("context length = %d, want 4096", got.Details.ContextLength)+ }+ if got.Details.EmbeddingLength != 384 {+ t.Errorf("embedding length = %d, want 384", got.Details.EmbeddingLength)+ }+ // The test GGUF has no general.file_type; the list must not guess F32+ // (FileType 0) from a missing key.+ if !isUnknownQuantization(got.Details.QuantizationLevel) {+ t.Errorf("quantization = %q, want unknown (no general.file_type in GGUF)", got.Details.QuantizationLevel)+ }+ for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} {+ if !slices.Contains(got.Capabilities, capability) {+ t.Errorf("capabilities = %v, want %s", got.Capabilities, capability)+ }+ }+}++// Listing reads the manifests every time, so a model created or deleted after+// the last call shows up without anything needing to be told about it.+func TestListModelsFollowsManifestChanges(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createListedModelFromKV(t, "list-follow-a", map[string]any{"test.context_length": uint32(1024)}, "")++ listedModel(t, "list-follow-a:latest")++ createListedModelFromKV(t, "list-follow-b", map[string]any{"test.context_length": uint32(2048)}, "")+ listedModel(t, "list-follow-a:latest")+ listedModel(t, "list-follow-b:latest")++ deleteModelNamed(t, "list-follow-a")++ models, err := listModels(context.Background())+ if err != nil {+ t.Fatal(err)+ }+ names := make([]string, 0, len(models))+ for _, m := range models {+ names = append(names, m.Name)+ }+ if slices.Contains(names, "list-follow-a:latest") || !slices.Contains(names, "list-follow-b:latest") {+ t.Fatalf("names after delete = %v, want only list-follow-b", names)+ }+}++func TestCapabilitiesSuppressNemotronSafetensorsMedia(t *testing.T) {+ caps := []model.Capability{+ model.CapabilityCompletion,+ model.CapabilityTools,+ model.CapabilityThinking,+ model.CapabilityVision,+ model.CapabilityAudio,+ }+ got := (&Model{Config: model.ConfigV2{+ ModelFormat: "safetensors",+ Renderer: "nemotron-3-nano",+ Parser: "nemotron-3-nano",+ }}).filterUnsupportedCapabilities(caps, "")++ for _, capability := range []model.Capability{+ model.CapabilityCompletion,+ model.CapabilityTools,+ model.CapabilityThinking,+ } {+ if !slices.Contains(got, capability) {+ t.Errorf("capabilities = %v, want %s", got, capability)+ }+ }+ for _, capability := range []model.Capability{model.CapabilityVision, model.CapabilityAudio} {+ if slices.Contains(got, capability) {+ t.Errorf("capabilities = %v, did not expect %s", got, capability)+ }+ }+}++// A model whose layers no longer parse must still be listed: it is the one the+// user most needs to see, in order to remove it.+func TestListModelsKeepsUnloadableModel(t *testing.T) {+ gin.SetMode(gin.TestMode)+ setTestHome(t, t.TempDir())+ createListedModelFromKV(t, "broken", map[string]any{"test.context_length": uint32(1024)}, "{{ .Prompt }}")++ mf, err := manifest.ParseNamedManifest(model.ParseName("broken"))+ if err != nil {+ t.Fatal(err)+ }+ var corrupted bool+ for _, layer := range mf.Layers {+ if layer.MediaType != "application/vnd.ollama.image.template" {+ continue+ }+ path, err := manifest.BlobsPath(layer.Digest)+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(path, []byte("{{ if }"), 0o644); err != nil {+ t.Fatal(err)+ }+ corrupted = true+ }+ if !corrupted {+ t.Fatal("no template layer to corrupt")+ }+ if _, err := GetModel("broken"); err == nil {+ t.Fatal("model still loads, so the listing is not being asked the question")+ }++ got := listedModel(t, "broken:latest")+ if got.Details.Family != "test" {+ t.Errorf("details = %+v, want the manifest config's family", got.Details)+ }+}++func createListedModelFromKV(t *testing.T, name string, kv map[string]any, tmpl string) {+ t.Helper()+ _, digest := createBinFile(t, kv, nil)+ createModelFromBlob(t, name, digest, tmpl)+}diff --git a/server/routes.go b/server/routes.goindex 9b73139a1..177a7441f 100644--- a/server/routes.go+++ b/server/routes.go@@ -286,7 +286,7 @@ func (s *Server) GenerateHandler(c *gin.Context) {return}- m, err := s.getModel(name.String())+ m, err := GetModel(name.String())if err != nil {switch {case errors.Is(err, fs.ErrNotExist):@@ -850,7 +850,7 @@ func (s *Server) EmbedHandler(c *gin.Context) {return}- m, err := s.getModel(name.String())+ m, err := GetModel(name.String())if err != nil {handleScheduleError(c, req.Model, err)return@@ -1068,7 +1068,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {name := modelRef.Name- m, err := s.getModel(name.String())+ m, err := GetModel(name.String())if err != nil {handleScheduleError(c, req.Model, err)return@@ -1153,8 +1153,6 @@ func (s *Server) PullHandler(c *gin.Context) {ch <- gin.H{"error": err.Error()}return}-- s.refreshModelListCache(name)}()if req.Stream != nil && !*req.Stream {@@ -1294,9 +1292,9 @@ func (s *Server) DeleteHandler(c *gin.Context) {return}- s.deleteModelListCache(n)-- if err := m.RemoveLayers(); err != nil {+ removed, err := m.RemoveLayers()+ removeGGUFMetadata(removed...)+ if err != nil {c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})return}@@ -1635,12 +1633,7 @@ func selectedModelTemplate(m *Model, kv ggml.KV) string {}func (s *Server) ListHandler(c *gin.Context) {- if s.modelCaches == nil || s.modelCaches.modelList == nil {- c.JSON(http.StatusInternalServerError, gin.H{"error": "model list cache unavailable"})- return- }-- models, err := s.modelCaches.modelList.List(c.Request.Context())+ models, err := listModels(c.Request.Context())if err != nil {c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})return@@ -1685,8 +1678,6 @@ func (s *Server) CopyHandler(c *gin.Context) {c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})} else if err != nil {c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})- } else {- s.refreshModelListCache(dst)}}@@ -2492,7 +2483,7 @@ func (s *Server) ChatHandler(c *gin.Context) {return}- m, err := s.getModel(name.String())+ m, err := GetModel(name.String())if err != nil {switch {case os.IsNotExist(err):diff --git a/server/routes_list_test.go b/server/routes_list_test.goindex eb409c8b0..9417c6292 100644--- a/server/routes_list_test.go+++ b/server/routes_list_test.go@@ -1,7 +1,6 @@package serverimport (- "context""encoding/json""io""net/http"@@ -36,11 +35,7 @@ func TestList(t *testing.T) {"myhost/mynamespace/lips:code",}- s := Server{modelCaches: &modelCaches{modelList: newModelListCache()}}- s.modelCaches.modelList.Start(context.Background())- if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {- t.Fatal(err)- }+ var s Serverfor _, n := range expectNames {_, digest := createBinFile(t, nil, nil)@@ -88,12 +83,7 @@ func TestOpenAIListMatchesTagsModels(t *testing.T) {gin.SetMode(gin.TestMode)setTestHome(t, t.TempDir())- cache := newModelListCache()- s := Server{modelCaches: &modelCaches{modelList: cache}}- cache.Start(context.Background())- if err := cache.Wait(context.Background()); err != nil {- t.Fatal(err)- }+ var s ServercreateModel := func(name string) {t.Helper()@@ -123,9 +113,6 @@ func TestOpenAIListMatchesTagsModels(t *testing.T) {if err := os.Chtimes(path, modified, modified); err != nil {t.Fatalf("set manifest time for %s: %v", name, err)}- if err := cache.RefreshModel(parsed); err != nil {- t.Fatalf("refresh %s: %v", name, err)- }}older := time.Unix(1000, 0).UTC()diff --git a/server/routes_test.go b/server/routes_test.goindex 8164454e9..44639f6f0 100644--- a/server/routes_test.go+++ b/server/routes_test.go@@ -95,11 +95,7 @@ func TestRoutes(t *testing.T) {Expected func(t *testing.T, resp *http.Response)}- s := &Server{modelCaches: &modelCaches{modelList: newModelListCache()}}- s.modelCaches.modelList.Start(context.Background())- if err := s.modelCaches.modelList.Wait(context.Background()); err != nil {- t.Fatal(err)- }+ s := &Server{}createTestModel := func(t *testing.T, name string) {t.Helper()@@ -135,7 +131,6 @@ func TestRoutes(t *testing.T) {if err := createModel(r, modelName, baseLayers, config, fn); err != nil {t.Fatal(err)}- s.refreshModelListCache(modelName)}testCases := []testCase{