Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Revert "feat: incremental gguf parser (#10822)" (#11114)
Jeffrey Morgana6e64fbdf28f
diff --git a/fs/gguf/gguf.go b/fs/gguf/gguf.godeleted file mode 100644index ebb9286f3..000000000--- a/fs/gguf/gguf.go+++ /dev/null@@ -1,347 +0,0 @@-package gguf--import (- "bytes"- "cmp"- "encoding/binary"- "errors"- "fmt"- "io"- "iter"- "os"- "slices"- "strings"-)--const (- typeUint8 uint32 = iota- typeInt8- typeUint16- typeInt16- typeUint32- typeInt32- typeFloat32- typeBool- typeString- typeArray- typeUint64- typeInt64- typeFloat64-)--var ErrUnsupported = errors.New("unsupported")--type File struct {- Magic [4]byte- Version uint32-- keyValues *lazy[KeyValue]- tensors *lazy[TensorInfo]- offset int64-- file *os.File- reader *bufferedReader- bts []byte-}--func Open(path string) (f *File, err error) {- f = &File{bts: make([]byte, 4096)}- f.file, err = os.Open(path)- if err != nil {- return nil, err- }-- f.reader = newBufferedReader(f.file, 32<<10)-- if err := binary.Read(f.reader, binary.LittleEndian, &f.Magic); err != nil {- return nil, err- }-- if bytes.Equal(f.Magic[:], []byte("gguf")) {- return nil, fmt.Errorf("%w file type %v", ErrUnsupported, f.Magic)- }-- if err := binary.Read(f.reader, binary.LittleEndian, &f.Version); err != nil {- return nil, err- }-- if f.Version != 3 {- return nil, fmt.Errorf("%w version %v", ErrUnsupported, f.Version)- }-- f.tensors, err = newLazy(f, f.readTensor)- if err != nil {- return nil, err- }-- f.tensors.successFunc = func() error {- offset := f.reader.offset-- alignment := cmp.Or(f.KeyValue("general.alignment").Int(), 32)- f.offset = offset + (alignment-offset%alignment)%alignment- return nil- }-- f.keyValues, err = newLazy(f, f.readKeyValue)- if err != nil {- return nil, err- }-- return f, nil-}--func (f *File) readTensor() (TensorInfo, error) {- name, err := readString(f)- if err != nil {- return TensorInfo{}, err- }-- dims, err := read[uint32](f)- if err != nil {- return TensorInfo{}, err- }-- shape := make([]uint64, dims)- for i := range dims {- shape[i], err = read[uint64](f)- if err != nil {- return TensorInfo{}, err- }- }-- type_, err := read[uint32](f)- if err != nil {- return TensorInfo{}, err- }-- offset, err := read[uint64](f)- if err != nil {- return TensorInfo{}, err- }-- return TensorInfo{- Name: name,- Offset: offset,- Shape: shape,- Type: TensorType(type_),- }, nil-}--func (f *File) readKeyValue() (KeyValue, error) {- key, err := readString(f)- if err != nil {- return KeyValue{}, err- }-- t, err := read[uint32](f)- if err != nil {- return KeyValue{}, err- }-- value, err := func() (any, error) {- switch t {- case typeUint8:- return read[uint8](f)- case typeInt8:- return read[int8](f)- case typeUint16:- return read[uint16](f)- case typeInt16:- return read[int16](f)- case typeUint32:- return read[uint32](f)- case typeInt32:- return read[int32](f)- case typeUint64:- return read[uint64](f)- case typeInt64:- return read[int64](f)- case typeFloat32:- return read[float32](f)- case typeFloat64:- return read[float64](f)- case typeBool:- return read[bool](f)- case typeString:- return readString(f)- case typeArray:- return readArray(f)- default:- return nil, fmt.Errorf("%w type %d", ErrUnsupported, t)- }- }()- if err != nil {- return KeyValue{}, err- }-- return KeyValue{- Key: key,- Value: Value{value},- }, nil-}--func read[T any](f *File) (t T, err error) {- err = binary.Read(f.reader, binary.LittleEndian, &t)- return t, err-}--func readString(f *File) (string, error) {- n, err := read[uint64](f)- if err != nil {- return "", err- }-- if int(n) > len(f.bts) {- f.bts = make([]byte, n)- }-- bts := f.bts[:n]- if _, err := io.ReadFull(f.reader, bts); err != nil {- return "", err- }- defer clear(bts)-- return string(bts), nil-}--func readArray(f *File) (any, error) {- t, err := read[uint32](f)- if err != nil {- return nil, err- }-- n, err := read[uint64](f)- if err != nil {- return nil, err- }-- switch t {- case typeUint8:- return readArrayData[uint8](f, n)- case typeInt8:- return readArrayData[int8](f, n)- case typeUint16:- return readArrayData[uint16](f, n)- case typeInt16:- return readArrayData[int16](f, n)- case typeUint32:- return readArrayData[uint32](f, n)- case typeInt32:- return readArrayData[int32](f, n)- case typeUint64:- return readArrayData[uint64](f, n)- case typeInt64:- return readArrayData[int64](f, n)- case typeFloat32:- return readArrayData[float32](f, n)- case typeFloat64:- return readArrayData[float64](f, n)- case typeBool:- return readArrayData[bool](f, n)- case typeString:- return readArrayString(f, n)- default:- return nil, fmt.Errorf("%w type %d", ErrUnsupported, t)- }-}--func readArrayData[T any](f *File, n uint64) (s []T, err error) {- s = make([]T, n)- for i := range n {- e, err := read[T](f)- if err != nil {- return nil, err- }-- s[i] = e- }-- return s, nil-}--func readArrayString(f *File, n uint64) (s []string, err error) {- s = make([]string, n)- for i := range n {- e, err := readString(f)- if err != nil {- return nil, err- }-- s[i] = e- }-- return s, nil-}--func (f *File) Close() error {- f.keyValues.stop()- f.tensors.stop()- return f.file.Close()-}--func (f *File) KeyValue(key string) KeyValue {- if !strings.HasPrefix(key, "general.") && !strings.HasPrefix(key, "tokenizer.") {- key = f.KeyValue("general.architecture").String() + "." + key- }-- if index := slices.IndexFunc(f.keyValues.values, func(kv KeyValue) bool {- return kv.Key == key- }); index >= 0 {- return f.keyValues.values[index]- }-- for keyValue, ok := f.keyValues.next(); ok; keyValue, ok = f.keyValues.next() {- if keyValue.Key == key {- return keyValue- }- }-- return KeyValue{}-}--func (f *File) NumKeyValues() int {- return int(f.keyValues.count)-}--func (f *File) KeyValues() iter.Seq2[int, KeyValue] {- return f.keyValues.All()-}--func (f *File) TensorInfo(name string) TensorInfo {- if index := slices.IndexFunc(f.tensors.values, func(t TensorInfo) bool {- return t.Name == name- }); index >= 0 {- return f.tensors.values[index]- }-- // fast-forward through key values if we haven't already- _ = f.keyValues.rest()- for tensor, ok := f.tensors.next(); ok; tensor, ok = f.tensors.next() {- if tensor.Name == name {- return tensor- }- }-- return TensorInfo{}-}--func (f *File) NumTensors() int {- return int(f.tensors.count)-}--func (f *File) TensorInfos() iter.Seq2[int, TensorInfo] {- // fast forward through key values if we haven't already- f.keyValues.rest()- return f.tensors.All()-}--func (f *File) TensorReader(name string) (TensorInfo, io.Reader, error) {- t := f.TensorInfo(name)- if t.NumBytes() == 0 {- return TensorInfo{}, nil, fmt.Errorf("tensor %s not found", name)- }-- // fast forward through tensor info if we haven't already- _ = f.tensors.rest()- return t, io.NewSectionReader(f.file, f.offset+int64(t.Offset), t.NumBytes()), nil-}diff --git a/fs/gguf/gguf_test.go b/fs/gguf/gguf_test.godeleted file mode 100644index eea28a480..000000000--- a/fs/gguf/gguf_test.go+++ /dev/null@@ -1,249 +0,0 @@-package gguf_test--import (- "bytes"- "os"- "strconv"- "strings"- "testing"-- "github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/ollama/ollama/fs/ggml"- "github.com/ollama/ollama/fs/gguf"-)--func createBinFile(tb testing.TB) string {- tb.Helper()- f, err := os.CreateTemp(tb.TempDir(), "")- if err != nil {- tb.Fatal(err)- }- defer f.Close()-- kv := ggml.KV{- "general.architecture": "llama",- "llama.block_count": uint32(8),- "llama.embedding_length": uint32(3),- "llama.attention.head_count": uint32(2),- "llama.attention.head_count_kv": uint32(2),- "llama.attention.key_length": uint32(3),- "llama.rope.dimension_count": uint32(4),- "llama.rope.freq_base": float32(10000.0),- "llama.rope.freq_scale": float32(1.0),- "llama.attention.layer_norm_rms_epsilon": float32(1e-6),- "tokenizer.ggml.eos_token_id": uint32(0),- "tokenizer.ggml.eos_token_ids": []int32{1, 2, 3},- "tokenizer.ggml.tokens": []string{"hello", "world"},- "tokenizer.ggml.scores": []float32{0, 1},- }-- tensors := []*ggml.Tensor{- {- Name: "token_embd.weight",- Kind: 0,- Shape: []uint64{2, 3},- WriterTo: bytes.NewBuffer(make([]byte, 4*2*3)),- },- {- Name: "output.weight",- Kind: 0,- Shape: []uint64{3, 2},- WriterTo: bytes.NewBuffer(make([]byte, 4*3*2)),- },- }-- for i := range 8 {- tensors = append(tensors, &ggml.Tensor{- Name: "blk." + strconv.Itoa(i) + ".attn_q.weight",- Kind: 0,- Shape: []uint64{3, 3},- WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)),- }, &ggml.Tensor{- Name: "blk." + strconv.Itoa(i) + ".attn_k.weight",- Kind: 0,- Shape: []uint64{3, 3},- WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)),- }, &ggml.Tensor{- Name: "blk." + strconv.Itoa(i) + ".attn_v.weight",- Kind: 0,- Shape: []uint64{3, 3},- WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)),- }, &ggml.Tensor{- Name: "blk." + strconv.Itoa(i) + ".attn_output.weight",- Kind: 0,- Shape: []uint64{3, 3},- WriterTo: bytes.NewBuffer(make([]byte, 4*3*3)),- })- }-- if err := ggml.WriteGGUF(f, kv, tensors); err != nil {- tb.Fatal(err)- }-- return f.Name()-}--func TestRead(t *testing.T) {- f, err := gguf.Open(createBinFile(t))- if err != nil {- t.Fatal(err)- }- defer f.Close()-- if got := f.KeyValue("does.not.exist").Valid(); got {- t.Errorf(`KeyValue("does.not.exist").Exists() = %v, want false`, got)- }-- if got := f.KeyValue("general.architecture").String(); got != "llama" {- t.Errorf(`KeyValue("general.architecture").String() = %q, want %q`, got, "llama")- }-- if got := f.TensorInfo("token_embd.weight"); got.Name != "token_embd.weight" {- t.Errorf(`TensorInfo("token_embd.weight").Name = %q, want %q`, got.Name, "token_embd.weight")- } else if diff := cmp.Diff(got.Shape, []uint64{2, 3}); diff != "" {- t.Errorf(`TensorInfo("token_embd.weight").Shape mismatch (-got +want):\n%s`, diff)- } else if got.Type != gguf.TensorTypeF32 {- t.Errorf(`TensorInfo("token_embd.weight").Type = %d, want %d`, got.Type, gguf.TensorTypeF32)- }-- if got := f.KeyValue("block_count").Uint(); got != 8 {- t.Errorf(`KeyValue("block_count").Uint() = %d, want %d`, got, 8)- }-- if diff := cmp.Diff(f.KeyValue("tokenizer.ggml.tokens").Strings(), []string{"hello", "world"}); diff != "" {- t.Errorf("KeyValue(\"tokenizer.ggml.tokens\").Strings() mismatch (-got +want):\n%s", diff)- }-- if diff := cmp.Diff(f.KeyValue("tokenizer.ggml.scores").Floats(), []float64{0, 1}); diff != "" {- t.Errorf("KeyValue(\"tokenizer.ggml.scores\").Ints() mismatch (-got +want):\n%s", diff)- }-- var kvs []string- for _, kv := range f.KeyValues() {- if !kv.Valid() {- t.Error("found invalid key-value pair:", kv)- }-- kvs = append(kvs, kv.Key)- }-- if len(kvs) != f.NumKeyValues() {- t.Errorf("iterated key count = %d, want %d", len(kvs), f.NumKeyValues())- }-- if diff := cmp.Diff(kvs, []string{- "general.architecture",- "llama.block_count",- "llama.embedding_length",- "llama.attention.head_count",- "llama.attention.head_count_kv",- "llama.attention.key_length",- "llama.rope.dimension_count",- "llama.rope.freq_base",- "llama.rope.freq_scale",- "llama.attention.layer_norm_rms_epsilon",- "tokenizer.ggml.eos_token_id",- "tokenizer.ggml.eos_token_ids",- "tokenizer.ggml.tokens",- "tokenizer.ggml.scores",- }, cmpopts.SortSlices(strings.Compare)); diff != "" {- t.Errorf("KeyValues() mismatch (-got +want):\n%s", diff)- }-- var tis []string- for _, ti := range f.TensorInfos() {- if !ti.Valid() {- t.Error("found invalid tensor info:", ti)- }-- tis = append(tis, ti.Name)- }-- if len(tis) != f.NumTensors() {- t.Errorf("iterated tensor count = %d, want %d", len(tis), f.NumTensors())- }-- if diff := cmp.Diff(tis, []string{- "token_embd.weight",- "output.weight",- "blk.0.attn_q.weight",- "blk.0.attn_k.weight",- "blk.0.attn_v.weight",- "blk.0.attn_output.weight",- "blk.1.attn_q.weight",- "blk.1.attn_k.weight",- "blk.1.attn_v.weight",- "blk.1.attn_output.weight",- "blk.2.attn_q.weight",- "blk.2.attn_k.weight",- "blk.2.attn_v.weight",- "blk.2.attn_output.weight",- "blk.3.attn_q.weight",- "blk.3.attn_k.weight",- "blk.3.attn_v.weight",- "blk.3.attn_output.weight",- "blk.4.attn_q.weight",- "blk.4.attn_k.weight",- "blk.4.attn_v.weight",- "blk.4.attn_output.weight",- "blk.5.attn_q.weight",- "blk.5.attn_k.weight",- "blk.5.attn_v.weight",- "blk.5.attn_output.weight",- "blk.6.attn_q.weight",- "blk.6.attn_k.weight",- "blk.6.attn_v.weight",- "blk.6.attn_output.weight",- "blk.7.attn_q.weight",- "blk.7.attn_k.weight",- "blk.7.attn_v.weight",- "blk.7.attn_output.weight",- }, cmpopts.SortSlices(strings.Compare)); diff != "" {- t.Errorf("TensorInfos() mismatch (-got +want):\n%s", diff)- }-- ti, r, err := f.TensorReader("output.weight")- if err != nil {- t.Fatalf(`TensorReader("output.weight") error: %v`, err)- }-- if ti.Name != "output.weight" {- t.Errorf(`TensorReader("output.weight").Name = %q, want %q`, ti.Name, "output.weight")- } else if diff := cmp.Diff(ti.Shape, []uint64{3, 2}); diff != "" {- t.Errorf(`TensorReader("output.weight").Shape mismatch (-got +want):\n%s`, diff)- } else if ti.Type != gguf.TensorTypeF32 {- t.Errorf(`TensorReader("output.weight").Type = %d, want %d`, ti.Type, gguf.TensorTypeF32)- }-- var b bytes.Buffer- if _, err := b.ReadFrom(r); err != nil {- t.Fatalf(`ReadFrom TensorReader("output.weight") error: %v`, err)- }-- if b.Len() != int(ti.NumBytes()) {- t.Errorf(`ReadFrom TensorReader("output.weight") length = %d, want %d`, b.Len(), ti.NumBytes())- }-}--func BenchmarkRead(b *testing.B) {- b.ReportAllocs()-- p := createBinFile(b)- for b.Loop() {- f, err := gguf.Open(p)- if err != nil {- b.Fatal(err)- }-- if got := f.KeyValue("general.architecture").String(); got != "llama" {- b.Errorf("got = %q, want %q", got, "llama")- }-- // Iterate through some tensors- for range f.TensorInfos() {- }-- f.Close()- }-}diff --git a/fs/gguf/keyvalue.go b/fs/gguf/keyvalue.godeleted file mode 100644index 5843326c1..000000000--- a/fs/gguf/keyvalue.go+++ /dev/null@@ -1,90 +0,0 @@-package gguf--import (- "reflect"- "slices"-)--type KeyValue struct {- Key string- Value-}--func (kv KeyValue) Valid() bool {- return kv.Key != "" && kv.Value.value != nil-}--type Value struct {- value any-}--func value[T any](v Value, kinds ...reflect.Kind) (t T) {- vv := reflect.ValueOf(v.value)- if slices.Contains(kinds, vv.Kind()) {- t = vv.Convert(reflect.TypeOf(t)).Interface().(T)- }- return-}--func values[T any](v Value, kinds ...reflect.Kind) (ts []T) {- switch vv := reflect.ValueOf(v.value); vv.Kind() {- case reflect.Slice:- if slices.Contains(kinds, vv.Type().Elem().Kind()) {- ts = make([]T, vv.Len())- for i := range vv.Len() {- ts[i] = vv.Index(i).Convert(reflect.TypeOf(ts[i])).Interface().(T)- }- }- }- return-}--// Int returns Value as a signed integer. If it is not a signed integer, it returns 0.-func (v Value) Int() int64 {- return value[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64)-}--// Ints returns Value as a signed integer slice. If it is not a signed integer slice, it returns nil.-func (v Value) Ints() (i64s []int64) {- return values[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64)-}--// Uint converts an unsigned integer value to uint64. If the value is not a unsigned integer, it returns 0.-func (v Value) Uint() uint64 {- return value[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64)-}--// Uints returns Value as a unsigned integer slice. If it is not a unsigned integer slice, it returns nil.-func (v Value) Uints() (u64s []uint64) {- return values[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64)-}--// Float returns Value as a float. If it is not a float, it returns 0.-func (v Value) Float() float64 {- return value[float64](v, reflect.Float32, reflect.Float64)-}--// Floats returns Value as a float slice. If it is not a float slice, it returns nil.-func (v Value) Floats() (f64s []float64) {- return values[float64](v, reflect.Float32, reflect.Float64)-}--// Bool returns Value as a boolean. If it is not a boolean, it returns false.-func (v Value) Bool() bool {- return value[bool](v, reflect.Bool)-}--// Bools returns Value as a boolean slice. If it is not a boolean slice, it returns nil.-func (v Value) Bools() (bools []bool) {- return values[bool](v, reflect.Bool)-}--// String returns Value as a string. If it is not a string, it returns an empty string.-func (v Value) String() string {- return value[string](v, reflect.String)-}--// Strings returns Value as a string slice. If it is not a string slice, it returns nil.-func (v Value) Strings() (strings []string) {- return values[string](v, reflect.String)-}diff --git a/fs/gguf/keyvalue_test.go b/fs/gguf/keyvalue_test.godeleted file mode 100644index 2caacc538..000000000--- a/fs/gguf/keyvalue_test.go+++ /dev/null@@ -1,208 +0,0 @@-package gguf--import (- "testing"-- "github.com/google/go-cmp/cmp"-)--func split(name string, values map[string][]any) (matched []any, unmatched []any) {- for key, value := range values {- if key == name {- matched = value- } else {- unmatched = append(unmatched, value...)- }- }- return-}--func TestValue(t *testing.T) {- values := map[string][]any{- "int64": {int(42), int8(42), int16(42), int32(42), int64(42)},- "uint64": {uint(42), uint8(42), uint16(42), uint32(42), uint64(42)},- "float64": {float32(42), float64(42)},- "string": {"42", "hello"},- "bool": {true, false},- }-- t.Run("int64", func(t *testing.T) {- matched, unmatched := split("int64", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if i64 := kv.Int(); i64 != 42 {- t.Errorf("expected 42, got %d", i64)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if i64 := kv.Int(); i64 != 0 {- t.Errorf("expected 42, got %d", i64)- }- }- })-- t.Run("uint64", func(t *testing.T) {- matched, unmatched := split("uint64", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if u64 := kv.Uint(); u64 != 42 {- t.Errorf("expected 42, got %d", u64)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if u64 := kv.Uint(); u64 != 0 {- t.Errorf("expected 42, got %d", u64)- }- }- })-- t.Run("float64", func(t *testing.T) {- matched, unmatched := split("float64", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if f64 := kv.Float(); f64 != 42 {- t.Errorf("expected 42, got %f", f64)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if f64 := kv.Float(); f64 != 0 {- t.Errorf("expected 42, got %f", f64)- }- }- })-- t.Run("string", func(t *testing.T) {- matched, unmatched := split("string", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if s := kv.String(); s != v {- t.Errorf("expected 42, got %s", s)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if s := kv.String(); s != "" {- t.Errorf("expected 42, got %s", s)- }- }- })-- t.Run("bool", func(t *testing.T) {- matched, unmatched := split("bool", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if b := kv.Bool(); b != v {- t.Errorf("expected true, got %v", b)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if b := kv.Bool(); b != false {- t.Errorf("expected false, got %v", b)- }- }- })-}--func TestValues(t *testing.T) {- values := map[string][]any{- "int64s": {[]int{42}, []int8{42}, []int16{42}, []int32{42}, []int64{42}},- "uint64s": {[]uint{42}, []uint8{42}, []uint16{42}, []uint32{42}, []uint64{42}},- "float64s": {[]float32{42}, []float64{42}},- "strings": {[]string{"42"}, []string{"hello"}},- "bools": {[]bool{true}, []bool{false}},- }-- t.Run("int64s", func(t *testing.T) {- matched, unmatched := split("int64s", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if diff := cmp.Diff(kv.Ints(), []int64{42}); diff != "" {- t.Errorf("diff: %s", diff)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if i64s := kv.Ints(); i64s != nil {- t.Errorf("expected nil, got %v", i64s)- }- }- })-- t.Run("uint64s", func(t *testing.T) {- matched, unmatched := split("uint64s", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if diff := cmp.Diff(kv.Uints(), []uint64{42}); diff != "" {- t.Errorf("diff: %s", diff)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if u64s := kv.Uints(); u64s != nil {- t.Errorf("expected nil, got %v", u64s)- }- }- })-- t.Run("float64s", func(t *testing.T) {- matched, unmatched := split("float64s", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if diff := cmp.Diff(kv.Floats(), []float64{42}); diff != "" {- t.Errorf("diff: %s", diff)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if f64s := kv.Floats(); f64s != nil {- t.Errorf("expected nil, got %v", f64s)- }- }- })-- t.Run("strings", func(t *testing.T) {- matched, unmatched := split("strings", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if diff := cmp.Diff(kv.Strings(), v); diff != "" {- t.Errorf("diff: %s", diff)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if s := kv.Strings(); s != nil {- t.Errorf("expected nil, got %v", s)- }- }- })-- t.Run("bools", func(t *testing.T) {- matched, unmatched := split("bools", values)- for _, v := range matched {- kv := KeyValue{"key", Value{v}}- if diff := cmp.Diff(kv.Bools(), v); diff != "" {- t.Errorf("diff: %s", diff)- }- }-- for _, v := range unmatched {- kv := KeyValue{"key", Value{v}}- if b := kv.Bools(); b != nil {- t.Errorf("expected nil, got %v", b)- }- }- })-}diff --git a/fs/gguf/lazy.go b/fs/gguf/lazy.godeleted file mode 100644index 16ab99093..000000000--- a/fs/gguf/lazy.go+++ /dev/null@@ -1,89 +0,0 @@-package gguf--import (- "encoding/binary"- "iter"- "log/slog"-)--type lazy[T any] struct {- count uint64- next func() (T, bool)- stop func()- values []T-- // successFunc is called when all values have been successfully read.- successFunc func() error-}--func newLazy[T any](f *File, fn func() (T, error)) (*lazy[T], error) {- it := lazy[T]{}- if err := binary.Read(f.reader, binary.LittleEndian, &it.count); err != nil {- return nil, err- }-- it.values = make([]T, 0)- it.next, it.stop = iter.Pull(func(yield func(T) bool) {- for i := range it.count {- t, err := fn()- if err != nil {- slog.Error("error reading tensor", "index", i, "error", err)- return- }-- it.values = append(it.values, t)- if !yield(t) {- break- }- }-- if it.successFunc != nil {- it.successFunc()- }- })-- return &it, nil-}--func (g *lazy[T]) Values() iter.Seq[T] {- return func(yield func(T) bool) {- for _, v := range g.All() {- if !yield(v) {- break- }- }- }-}--func (g *lazy[T]) All() iter.Seq2[int, T] {- return func(yield func(int, T) bool) {- for i := range int(g.count) {- if i < len(g.values) {- if !yield(i, g.values[i]) {- break- }- } else {- t, ok := g.next()- if !ok {- break- }-- if !yield(i, t) {- break- }- }- }- }-}--func (g *lazy[T]) rest() (collected bool) {- for {- _, ok := g.next()- collected = collected || ok- if !ok {- break- }- }-- return collected-}diff --git a/fs/gguf/reader.go b/fs/gguf/reader.godeleted file mode 100644index 0bd761840..000000000--- a/fs/gguf/reader.go+++ /dev/null@@ -1,23 +0,0 @@-package gguf--import (- "bufio"- "io"-)--type bufferedReader struct {- offset int64- *bufio.Reader-}--func newBufferedReader(rs io.ReadSeeker, size int) *bufferedReader {- return &bufferedReader{- Reader: bufio.NewReaderSize(rs, size),- }-}--func (rs *bufferedReader) Read(p []byte) (n int, err error) {- n, err = rs.Reader.Read(p)- rs.offset += int64(n)- return n, err-}diff --git a/fs/gguf/tensor.go b/fs/gguf/tensor.godeleted file mode 100644index 194c1d739..000000000--- a/fs/gguf/tensor.go+++ /dev/null@@ -1,288 +0,0 @@-package gguf--import (- "log/slog"- "strings"-)--type TensorInfo struct {- Name string- Offset uint64- Shape []uint64- Type TensorType-}--func (ti TensorInfo) Valid() bool {- return ti.Name != "" && ti.NumBytes() > 0-}--func (ti TensorInfo) NumValues() int64 {- var numItems int64 = 1- for _, dim := range ti.Shape {- numItems *= int64(dim)- }- return numItems-}--// NumBytes returns the number of bytes in the tensor.-func (ti TensorInfo) NumBytes() int64 {- return int64(float64(ti.NumValues()) * ti.Type.NumBytes())-}--func (ti TensorInfo) LogValue() slog.Value {- return slog.GroupValue(- slog.String("name", ti.Name),- slog.Int64("offset", int64(ti.Offset)),- slog.Any("shape", ti.Shape),- slog.Int64("num_values", ti.NumValues()),- slog.Int64("num_bytes", ti.NumBytes()),- slog.Any("type", ti.Type),- )-}--type TensorType uint32--const (- TensorTypeF32 TensorType = iota- TensorTypeF16- TensorTypeQ4_0- TensorTypeQ4_1-- // unexported // unused in gguf- tensorTypeQ4_2- tensorTypeQ4_3-- TensorTypeQ5_0- TensorTypeQ5_1- TensorTypeQ8_0- TensorTypeQ8_1- TensorTypeQ2_K- TensorTypeQ3_K- TensorTypeQ4_K- TensorTypeQ5_K- TensorTypeQ6_K- TensorTypeQ8_K-- // unexported // unquantizable by ollama- tensorTypeIQ2_XXS- tensorTypeIQ2_XS- tensorTypeIQ3_XXS- tensorTypeIQ1_S- tensorTypeIQ4_NL- tensorTypeIQ3_S- tensorTypeIQ2_S- tensorTypeIQ4_XS-- TensorTypeI8- TensorTypeI16- TensorTypeI32- TensorTypeI64- TensorTypeF64-- // unexported // unquantizable by ollama- tensorTypeIQ1_M-- TensorTypeBF16-- // unexported // unused in gguf- tensorTypeQ4_0_4_4- tensorTypeQ4_0_4_8- tensorTypeQ4_0_8_8-- // unexported // unquantizable by ollama- tensorTypeTQ1_0- tensorTypeTQ2_0-- // unexported // unused in gguf- tensorTypeIQ4_NL_4_4- tensorTypeIQ4_NL_4_8- tensorTypeIQ4_NL_8_8-)--func (tt TensorType) NumBytes() float64 {- return float64(tt.typeSize()) / float64(tt.blockSize())-}--func (tt TensorType) typeSize() int64 {- switch tt {- case TensorTypeF32:- return 4- case TensorTypeF16:- return 2- case TensorTypeQ4_0:- return 2 + tt.blockSize()/2- case TensorTypeQ4_1:- return 2 + 2 + tt.blockSize()/2- case TensorTypeQ5_0:- return 2 + 4 + tt.blockSize()/2- case TensorTypeQ5_1:- return 2 + 2 + 4 + tt.blockSize()/2- case TensorTypeQ8_0:- return 2 + tt.blockSize()- case TensorTypeQ8_1:- return 2 + 2 + tt.blockSize()- case TensorTypeQ2_K:- return tt.blockSize()/16 + tt.blockSize()/4 + 2 + 2- case TensorTypeQ3_K:- return tt.blockSize()/8 + tt.blockSize()/4 + 12 + 2- case TensorTypeQ4_K:- return 2 + 2 + 12 + tt.blockSize()/2- case TensorTypeQ5_K:- return 2 + 2 + 12 + tt.blockSize()/8 + tt.blockSize()/2- case TensorTypeQ6_K:- return tt.blockSize()/2 + tt.blockSize()/4 + tt.blockSize()/16 + 2- case TensorTypeQ8_K:- return 4 + tt.blockSize() + 2*tt.blockSize()/16- case tensorTypeIQ2_XXS:- return 2 + 2*tt.blockSize()/8- case tensorTypeIQ2_XS:- return 2 + 2*tt.blockSize()/8 + tt.blockSize()/32- case tensorTypeIQ3_XXS:- return 2 + tt.blockSize()/4 + tt.blockSize()/8- case tensorTypeIQ1_S:- return 2 + tt.blockSize()/8 + tt.blockSize()/16- case tensorTypeIQ4_NL:- return 2 + tt.blockSize()/2- case tensorTypeIQ3_S:- return 2 + tt.blockSize()/4 + tt.blockSize()/8 + tt.blockSize()/32 + 4- case tensorTypeIQ2_S:- return 2 + tt.blockSize()/4 + tt.blockSize()/16- case tensorTypeIQ4_XS:- return 2 + 2 + tt.blockSize()/2 + tt.blockSize()/64- case TensorTypeI8:- return 1- case TensorTypeI16:- return 2- case TensorTypeI32:- return 4- case TensorTypeI64:- return 8- case TensorTypeF64:- return 8- case tensorTypeIQ1_M:- return tt.blockSize()/8 + tt.blockSize()/16 + tt.blockSize()/32- case TensorTypeBF16:- return 2- default:- return 0- }-}--func (tt TensorType) blockSize() int64 {- switch tt {- case TensorTypeF32,- TensorTypeF16,- TensorTypeI8,- TensorTypeI16,- TensorTypeI32,- TensorTypeI64,- TensorTypeF64,- TensorTypeBF16:- return 1- case TensorTypeQ4_0,- TensorTypeQ4_1,- TensorTypeQ5_0,- TensorTypeQ5_1,- TensorTypeQ8_0,- TensorTypeQ8_1,- tensorTypeIQ4_NL:- return 32- default:- return 256- }-}--func (tt TensorType) String() string {- switch tt {- case TensorTypeF32:- return "f32"- case TensorTypeF16:- return "f16"- case TensorTypeQ4_0:- return "q4_0"- case TensorTypeQ4_1:- return "q4_1"- case tensorTypeQ4_2:- return "q4_2"- case tensorTypeQ4_3:- return "q4_3"- case TensorTypeQ5_0:- return "q5_0"- case TensorTypeQ5_1:- return "q5_1"- case TensorTypeQ8_0:- return "q8_0"- case TensorTypeQ8_1:- return "q8_1"- case TensorTypeQ2_K:- return "q2_k"- case TensorTypeQ3_K:- return "q3_k"- case TensorTypeQ4_K:- return "q4_k"- case TensorTypeQ5_K:- return "q5_k"- case TensorTypeQ6_K:- return "q6_k"- case TensorTypeQ8_K:- return "q8_k"- case tensorTypeIQ2_XXS:- return "iq2_xxs"- case tensorTypeIQ2_XS:- return "iq2_xs"- case tensorTypeIQ3_XXS:- return "iq3_xxs"- case tensorTypeIQ1_S:- return "iq1_s"- case tensorTypeIQ4_NL:- return "iq4_nl"- case tensorTypeIQ3_S:- return "iq3_s"- case tensorTypeIQ2_S:- return "iq2_s"- case tensorTypeIQ4_XS:- return "iq4_xs"- case TensorTypeI8:- return "i8"- case TensorTypeI16:- return "i16"- case TensorTypeI32:- return "i32"- case TensorTypeI64:- return "i64"- case TensorTypeF64:- return "f64"- case tensorTypeIQ1_M:- return "iq1_m"- case TensorTypeBF16:- return "bf16"- case tensorTypeQ4_0_4_4:- return "q4_0_4_4"- case tensorTypeQ4_0_4_8:- return "q4_0_4_8"- case tensorTypeQ4_0_8_8:- return "q4_0_8_8"- case tensorTypeTQ1_0:- return "tq1_0"- case tensorTypeTQ2_0:- return "tq2_0"- case tensorTypeIQ4_NL_4_4:- return "iq4_nl_4_4"- case tensorTypeIQ4_NL_4_8:- return "iq4_nl_4_8"- case tensorTypeIQ4_NL_8_8:- return "iq4_nl_8_8"- default:- return "unknown"- }-}--func (tt TensorType) LogValue() slog.Value {- return slog.GroupValue(- slog.Uint64("value", uint64(tt)),- slog.String("name", strings.ToUpper(tt.String())),- slog.Int64("size", tt.typeSize()),- slog.Int64("block_size", tt.blockSize()),- slog.Float64("num_bytes", tt.NumBytes()),- )-}diff --git a/go.mod b/go.modindex 6de5959be..283286b7d 100644--- a/go.mod+++ b/go.mod@@ -19,7 +19,7 @@ require (github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1github.com/dlclark/regexp2 v1.11.4github.com/emirpasic/gods/v2 v2.0.0-alpha- github.com/google/go-cmp v0.7.0+ github.com/google/go-cmp v0.6.0github.com/mattn/go-runewidth v0.0.14github.com/nlpodyssey/gopickle v0.3.0github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727cdiff --git a/go.sum b/go.sumindex c0ab53aab..5755616f6 100644--- a/go.sum+++ b/go.sum@@ -112,8 +112,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=diff --git a/server/images.go b/server/images.goindex 38505cc51..d6cceff4c 100644--- a/server/images.go+++ b/server/images.go@@ -23,7 +23,7 @@ import ("github.com/ollama/ollama/api""github.com/ollama/ollama/envconfig"- "github.com/ollama/ollama/fs/gguf"+ "github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/parser""github.com/ollama/ollama/template""github.com/ollama/ollama/thinking"@@ -73,18 +73,22 @@ func (m *Model) Capabilities() []model.Capability {capabilities := []model.Capability{}// Check for completion capability- f, err := gguf.Open(m.ModelPath)+ r, err := os.Open(m.ModelPath)if err == nil {- defer f.Close()-- if f.KeyValue("pooling_type").Valid() {- capabilities = append(capabilities, model.CapabilityEmbedding)+ defer r.Close()++ f, err := ggml.Decode(r, 1024)+ if err == nil {+ if _, ok := f.KV()[fmt.Sprintf("%s.pooling_type", f.KV().Architecture())]; ok {+ capabilities = append(capabilities, model.CapabilityEmbedding)+ } else {+ capabilities = append(capabilities, model.CapabilityCompletion)+ }+ if _, ok := f.KV()[fmt.Sprintf("%s.vision.block_count", f.KV().Architecture())]; ok {+ capabilities = append(capabilities, model.CapabilityVision)+ }} else {- // If no embedding is specified, we assume the model supports completion- capabilities = append(capabilities, model.CapabilityCompletion)- }- if f.KeyValue("vision.block_count").Valid() {- capabilities = append(capabilities, model.CapabilityVision)+ slog.Error("couldn't decode ggml", "error", err)}} else {slog.Error("couldn't open model file", "error", err)diff --git a/server/images_test.go b/server/images_test.goindex a2fba8d98..363b298e1 100644--- a/server/images_test.go+++ b/server/images_test.go@@ -1,42 +1,123 @@package serverimport (+ "bytes"+ "encoding/binary"+ "errors"+ "os"+ "path/filepath""strings""testing"- "github.com/ollama/ollama/fs/ggml""github.com/ollama/ollama/template""github.com/ollama/ollama/types/model")+// Constants for GGUF magic bytes and version+var (+ ggufMagic = []byte{0x47, 0x47, 0x55, 0x46} // "GGUF"+ ggufVer = uint32(3) // Version 3+)++// Helper function to create mock GGUF data+func createMockGGUFData(architecture string, vision bool) []byte {+ var buf bytes.Buffer++ // Write GGUF header+ buf.Write(ggufMagic)+ binary.Write(&buf, binary.LittleEndian, ggufVer)++ // Write tensor count (0 for our test)+ var numTensors uint64 = 0+ binary.Write(&buf, binary.LittleEndian, numTensors)++ // Calculate number of metadata entries+ numMetaEntries := uint64(1) // architecture entry+ if vision {+ numMetaEntries+++ }+ // Add embedding entry if architecture is "bert"+ if architecture == "bert" {+ numMetaEntries+++ }+ binary.Write(&buf, binary.LittleEndian, numMetaEntries)++ // Write architecture metadata+ archKey := "general.architecture"+ keyLen := uint64(len(archKey))+ binary.Write(&buf, binary.LittleEndian, keyLen)+ buf.WriteString(archKey)++ // String type (8)+ var strType uint32 = 8+ binary.Write(&buf, binary.LittleEndian, strType)++ // String length+ strLen := uint64(len(architecture))+ binary.Write(&buf, binary.LittleEndian, strLen)+ buf.WriteString(architecture)++ if vision {+ visionKey := architecture + ".vision.block_count"+ keyLen = uint64(len(visionKey))+ binary.Write(&buf, binary.LittleEndian, keyLen)+ buf.WriteString(visionKey)++ // uint32 type (4)+ var uint32Type uint32 = 4+ binary.Write(&buf, binary.LittleEndian, uint32Type)++ // uint32 value (1)+ var countVal uint32 = 1+ binary.Write(&buf, binary.LittleEndian, countVal)+ }+ // Write embedding metadata if architecture is "bert"+ if architecture == "bert" {+ poolKey := architecture + ".pooling_type"+ keyLen = uint64(len(poolKey))+ binary.Write(&buf, binary.LittleEndian, keyLen)+ buf.WriteString(poolKey)++ // uint32 type (4)+ var uint32Type uint32 = 4+ binary.Write(&buf, binary.LittleEndian, uint32Type)++ // uint32 value (1)+ var poolingVal uint32 = 1+ binary.Write(&buf, binary.LittleEndian, poolingVal)+ }++ return buf.Bytes()+}+func TestModelCapabilities(t *testing.T) {- // Create completion model (llama architecture without vision)- completionModelPath, _ := createBinFile(t, ggml.KV{- "general.architecture": "llama",- }, []*ggml.Tensor{})-- // Create vision model (llama architecture with vision block count)- visionModelPath, _ := createBinFile(t, ggml.KV{- "general.architecture": "llama",- "llama.vision.block_count": uint32(1),- }, []*ggml.Tensor{})-- // Create embedding model (bert architecture with pooling type)- embeddingModelPath, _ := createBinFile(t, ggml.KV{- "general.architecture": "bert",- "bert.pooling_type": uint32(1),- }, []*ggml.Tensor{})+ // Create a temporary directory for test files+ tempDir := t.TempDir()++ // Create different types of mock model files+ completionModelPath := filepath.Join(tempDir, "model.bin")+ visionModelPath := filepath.Join(tempDir, "vision_model.bin")+ embeddingModelPath := filepath.Join(tempDir, "embedding_model.bin")+ // Create a simple model file for tests that don't depend on GGUF content+ simpleModelPath := filepath.Join(tempDir, "simple_model.bin")++ if err := errors.Join(+ os.WriteFile(completionModelPath, createMockGGUFData("llama", false), 0o644),+ os.WriteFile(visionModelPath, createMockGGUFData("llama", true), 0o644),+ os.WriteFile(embeddingModelPath, createMockGGUFData("bert", false), 0o644),+ os.WriteFile(simpleModelPath, []byte("dummy model data"), 0o644),+ ); err != nil {+ t.Fatalf("Failed to create model files: %v", err)+ }toolsInsertTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")if err != nil {t.Fatalf("Failed to parse template: %v", err)}-chatTemplate, err := template.Parse("{{ .prompt }}")if err != nil {t.Fatalf("Failed to parse template: %v", err)}-toolsTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}")if err != nil {t.Fatalf("Failed to parse template: %v", err)@@ -64,13 +145,21 @@ func TestModelCapabilities(t *testing.T) {},expectedCaps: []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert},},+ {+ name: "model with tools and insert capability",+ model: Model{+ ModelPath: simpleModelPath,+ Template: toolsInsertTemplate,+ },+ expectedCaps: []model.Capability{model.CapabilityTools, model.CapabilityInsert},+ },{name: "model with tools capability",model: Model{- ModelPath: completionModelPath,+ ModelPath: simpleModelPath,Template: toolsTemplate,},- expectedCaps: []model.Capability{model.CapabilityCompletion, model.CapabilityTools},+ expectedCaps: []model.Capability{model.CapabilityTools},},{name: "model with vision capability",@@ -135,33 +224,29 @@ func TestModelCapabilities(t *testing.T) {}func TestModelCheckCapabilities(t *testing.T) {- // Create simple model file for tests that don't depend on GGUF content- completionModelPath, _ := createBinFile(t, ggml.KV{- "general.architecture": "llama",- }, []*ggml.Tensor{})-- // Create vision model (llama architecture with vision block count)- visionModelPath, _ := createBinFile(t, ggml.KV{- "general.architecture": "llama",- "llama.vision.block_count": uint32(1),- }, []*ggml.Tensor{})-- // Create embedding model (bert architecture with pooling type)- embeddingModelPath, _ := createBinFile(t, ggml.KV{- "general.architecture": "bert",- "bert.pooling_type": uint32(1),- }, []*ggml.Tensor{})+ // Create a temporary directory for test files+ tempDir := t.TempDir()++ visionModelPath := filepath.Join(tempDir, "vision_model.bin")+ simpleModelPath := filepath.Join(tempDir, "model.bin")+ embeddingModelPath := filepath.Join(tempDir, "embedding_model.bin")++ if err := errors.Join(+ os.WriteFile(simpleModelPath, []byte("dummy model data"), 0o644),+ os.WriteFile(visionModelPath, createMockGGUFData("llama", true), 0o644),+ os.WriteFile(embeddingModelPath, createMockGGUFData("bert", false), 0o644),+ ); err != nil {+ t.Fatalf("Failed to create model files: %v", err)+ }toolsInsertTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}")if err != nil {t.Fatalf("Failed to parse template: %v", err)}-chatTemplate, err := template.Parse("{{ .prompt }}")if err != nil {t.Fatalf("Failed to parse template: %v", err)}-toolsTemplate, err := template.Parse("{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}")if err != nil {t.Fatalf("Failed to parse template: %v", err)@@ -176,7 +261,7 @@ func TestModelCheckCapabilities(t *testing.T) {{name: "completion model without tools capability",model: Model{- ModelPath: completionModelPath,+ ModelPath: simpleModelPath,Template: chatTemplate,},checkCaps: []model.Capability{model.CapabilityTools},@@ -185,7 +270,7 @@ func TestModelCheckCapabilities(t *testing.T) {{name: "model with all needed capabilities",model: Model{- ModelPath: completionModelPath,+ ModelPath: simpleModelPath,Template: toolsInsertTemplate,},checkCaps: []model.Capability{model.CapabilityTools, model.CapabilityInsert},@@ -193,7 +278,7 @@ func TestModelCheckCapabilities(t *testing.T) {{name: "model missing insert capability",model: Model{- ModelPath: completionModelPath,+ ModelPath: simpleModelPath,Template: toolsTemplate,},checkCaps: []model.Capability{model.CapabilityInsert},@@ -202,7 +287,7 @@ func TestModelCheckCapabilities(t *testing.T) {{name: "model missing vision capability",model: Model{- ModelPath: completionModelPath,+ ModelPath: simpleModelPath,Template: toolsTemplate,},checkCaps: []model.Capability{model.CapabilityVision},@@ -227,7 +312,7 @@ func TestModelCheckCapabilities(t *testing.T) {{name: "unknown capability",model: Model{- ModelPath: completionModelPath,+ ModelPath: simpleModelPath,Template: chatTemplate,},checkCaps: []model.Capability{"unknown"},diff --git a/server/quantization_test.go b/server/quantization_test.goindex 8b726c836..4f717c2c2 100644--- a/server/quantization_test.go+++ b/server/quantization_test.go@@ -257,8 +257,16 @@ func TestQuantizeModel(t *testing.T) {for _, tt := range cases {t.Run(tt.name, func(t *testing.T) {- p, _ := createBinFile(t, tt.kv, tt.tensors)- fp, err := os.Open(p)+ f, err := os.CreateTemp(t.TempDir(), tt.name)+ if err != nil {+ t.Fatal(err.Error())+ }+ defer f.Close()+ err = fsggml.WriteGGUF(f, tt.kv, tt.tensors)+ if err != nil {+ t.Fatalf("failed to create initial model: %s", err)+ }+ fp, err := os.Open(f.Name())if err != nil {t.Fatal(err.Error())}diff --git a/server/sched_test.go b/server/sched_test.goindex 3892fbbab..01fb9a703 100644--- a/server/sched_test.go+++ b/server/sched_test.go@@ -112,7 +112,11 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, estb.ctx, b.ctxDone = context.WithCancel(ctx)t.Helper()- p, _ := createBinFile(t, ggml.KV{+ f, err := os.CreateTemp(t.TempDir(), modelName)+ require.NoError(t, err)+ defer f.Close()++ require.NoError(t, ggml.WriteGGUF(f, ggml.KV{"general.architecture": "llama","llama.context_length": uint32(32),"llama.embedding_length": uint32(4096),@@ -125,14 +129,14 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, est}, []*ggml.Tensor{{Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},{Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- })+ }))+ require.NoError(t, err)++ fname := f.Name()+ model := &Model{Name: modelName, ModelPath: fname}+ b.f, err = llm.LoadModel(model.ModelPath, 0)+ require.NoError(t, err)- model := &Model{Name: modelName, ModelPath: p}- f, err := llm.LoadModel(model.ModelPath, 0)- if err != nil {- t.Fatal(err)- }- b.f = fif duration == nil {duration = &api.Duration{Duration: 5 * time.Millisecond}}