Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
convert: only extract large files
Michael Yangeafc607abb34
diff --git a/convert/convert.go b/convert/convert.goindex 30c5a53f8..b9461e4fe 100644--- a/convert/convert.go+++ b/convert/convert.go@@ -5,9 +5,8 @@ import ("errors""fmt""io"+ "io/fs""log/slog"- "os"- "path/filepath""github.com/ollama/ollama/llm")@@ -67,8 +66,8 @@ type Converter interface {// and files it finds in the input path.// Supported input model formats include safetensors.// Supported input tokenizers files include tokenizer.json (preferred) and tokenizer.model.-func Convert(path string, ws io.WriteSeeker) error {- bts, err := os.ReadFile(filepath.Join(path, "config.json"))+func Convert(fsys fs.FS, ws io.WriteSeeker) error {+ bts, err := fs.ReadFile(fsys, "config.json")if err != nil {return err}@@ -98,7 +97,7 @@ func Convert(path string, ws io.WriteSeeker) error {return err}- t, err := parseTokenizer(path, conv.specialTokenTypes())+ t, err := parseTokenizer(fsys, conv.specialTokenTypes())if err != nil {return err}@@ -114,7 +113,7 @@ func Convert(path string, ws io.WriteSeeker) error {slog.Debug("vocabulary", "size", len(t.Vocabulary.Tokens))}- ts, err := parseTensors(path)+ ts, err := parseTensors(fsys)if err != nil {return err}diff --git a/convert/convert_test.go b/convert/convert_test.goindex 0fbd436f5..67a2fcfe3 100644--- a/convert/convert_test.go+++ b/convert/convert_test.go@@ -6,6 +6,7 @@ import ("flag""fmt""io"+ "io/fs""log/slog""math""os"@@ -17,7 +18,7 @@ import ("golang.org/x/exp/maps")-func convertFull(t *testing.T, d string) (*os.File, llm.KV, llm.Tensors) {+func convertFull(t *testing.T, fsys fs.FS) (*os.File, llm.KV, llm.Tensors) {t.Helper()f, err := os.CreateTemp(t.TempDir(), "f16")@@ -26,7 +27,7 @@ func convertFull(t *testing.T, d string) (*os.File, llm.KV, llm.Tensors) {}defer f.Close()- if err := Convert(d, f); err != nil {+ if err := Convert(fsys, f); err != nil {t.Fatal(err)}@@ -76,7 +77,7 @@ func TestConvertFull(t *testing.T) {t.Skipf("%s not found", p)}- f, kv, tensors := convertFull(t, p)+ f, kv, tensors := convertFull(t, os.DirFS(p))actual := make(map[string]string)for k, v := range kv {if s, ok := v.(json.Marshaler); !ok {diff --git a/convert/fs.go b/convert/fs.gonew file mode 100644index 000000000..bf6da6c25--- /dev/null+++ b/convert/fs.go@@ -0,0 +1,58 @@+package convert++import (+ "archive/zip"+ "errors"+ "io"+ "io/fs"+ "os"+ "path/filepath"+)++type ZipReader struct {+ r *zip.Reader+ p string++ // limit is the maximum size of a file that can be read directly+ // from the zip archive. Files larger than this size will be extracted+ limit int64+}++func NewZipReader(r *zip.Reader, p string, limit int64) fs.FS {+ return &ZipReader{r, p, limit}+}++func (z *ZipReader) Open(name string) (fs.File, error) {+ r, err := z.r.Open(name)+ if err != nil {+ return nil, err+ }+ defer r.Close()++ if fi, err := r.Stat(); err != nil {+ return nil, err+ } else if fi.Size() < z.limit {+ return r, nil+ }++ if !filepath.IsLocal(name) {+ return nil, zip.ErrInsecurePath+ }++ n := filepath.Join(z.p, name)+ if _, err := os.Stat(n); errors.Is(err, os.ErrNotExist) {+ w, err := os.Create(n)+ if err != nil {+ return nil, err+ }+ defer w.Close()++ if _, err := io.Copy(w, r); err != nil {+ return nil, err+ }+ } else if err != nil {+ return nil, err+ }++ return os.Open(n)+}diff --git a/convert/reader.go b/convert/reader.goindex 11ccaa819..56a8ae895 100644--- a/convert/reader.go+++ b/convert/reader.go@@ -3,7 +3,7 @@ package convertimport ("errors""io"- "path/filepath"+ "io/fs""strings")@@ -55,8 +55,8 @@ func (t *tensorBase) SetRepacker(fn repacker) {type repacker func(string, []float32, []uint64) ([]float32, error)-func parseTensors(d string) ([]Tensor, error) {- patterns := map[string]func(...string) ([]Tensor, error){+func parseTensors(fsys fs.FS) ([]Tensor, error) {+ patterns := map[string]func(fs.FS, ...string) ([]Tensor, error){"model-*-of-*.safetensors": parseSafetensors,"model.safetensors": parseSafetensors,"pytorch_model-*-of-*.bin": parseTorch,@@ -65,13 +65,13 @@ func parseTensors(d string) ([]Tensor, error) {}for pattern, parseFn := range patterns {- matches, err := filepath.Glob(filepath.Join(d, pattern))+ matches, err := fs.Glob(fsys, pattern)if err != nil {return nil, err}if len(matches) > 0 {- return parseFn(matches...)+ return parseFn(fsys, matches...)}}diff --git a/convert/reader_safetensors.go b/convert/reader_safetensors.goindex c5fe663cc..1c1695044 100644--- a/convert/reader_safetensors.go+++ b/convert/reader_safetensors.go@@ -6,7 +6,7 @@ import ("encoding/json""fmt""io"- "os"+ "io/fs""slices""github.com/d4l3k/go-bfloat16"@@ -20,10 +20,10 @@ type safetensorMetadata struct {Offsets []int64 `json:"data_offsets"`}-func parseSafetensors(ps ...string) ([]Tensor, error) {+func parseSafetensors(fsys fs.FS, ps ...string) ([]Tensor, error) {var ts []Tensorfor _, p := range ps {- f, err := os.Open(p)+ f, err := fsys.Open(p)if err != nil {return nil, err}@@ -50,6 +50,7 @@ func parseSafetensors(ps ...string) ([]Tensor, error) {for _, key := range keys {if value := headers[key]; value.Type != "" {ts = append(ts, safetensor{+ fs: fsys,path: p,dtype: value.Type,offset: safetensorsPad(n, value.Offsets[0]),@@ -72,6 +73,7 @@ func safetensorsPad(n, offset int64) int64 {}type safetensor struct {+ fs fs.FSpath stringdtype stringoffset int64@@ -80,14 +82,20 @@ type safetensor struct {}func (st safetensor) WriteTo(w io.Writer) (int64, error) {- f, err := os.Open(st.path)+ f, err := st.fs.Open(st.path)if err != nil {return 0, err}defer f.Close()- if _, err = f.Seek(st.offset, io.SeekStart); err != nil {- return 0, err+ if seeker, ok := f.(io.Seeker); ok {+ if _, err := seeker.Seek(st.offset, io.SeekStart); err != nil {+ return 0, err+ }+ } else {+ if _, err := io.CopyN(io.Discard, f, st.offset); err != nil {+ return 0, err+ }}var f32s []float32diff --git a/convert/reader_torch.go b/convert/reader_torch.goindex 1428706e2..531996bf2 100644--- a/convert/reader_torch.go+++ b/convert/reader_torch.go@@ -2,12 +2,13 @@ package convertimport ("io"+ "io/fs""github.com/nlpodyssey/gopickle/pytorch""github.com/nlpodyssey/gopickle/types")-func parseTorch(ps ...string) ([]Tensor, error) {+func parseTorch(fsys fs.FS, ps ...string) ([]Tensor, error) {var ts []Tensorfor _, p := range ps {pt, err := pytorch.Load(p)diff --git a/convert/tokenizer.go b/convert/tokenizer.goindex 43d8c14ef..cca40eb00 100644--- a/convert/tokenizer.go+++ b/convert/tokenizer.go@@ -7,9 +7,9 @@ import ("encoding/json""errors""fmt"+ "io/fs""log/slog""os"- "path/filepath""slices")@@ -32,8 +32,8 @@ type Tokenizer struct {Template string}-func parseTokenizer(d string, specialTokenTypes []string) (*Tokenizer, error) {- v, err := parseVocabulary(d)+func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error) {+ v, err := parseVocabulary(fsys)if err != nil {return nil, err}@@ -44,7 +44,7 @@ func parseTokenizer(d string, specialTokenTypes []string) (*Tokenizer, error) {}addedTokens := make(map[string]token)- if f, err := os.Open(filepath.Join(d, "tokenizer.json")); errors.Is(err, os.ErrNotExist) {+ if f, err := fsys.Open("tokenizer.json"); errors.Is(err, os.ErrNotExist) {} else if err != nil {return nil, err} else {@@ -87,7 +87,7 @@ func parseTokenizer(d string, specialTokenTypes []string) (*Tokenizer, error) {}}- if f, err := os.Open(filepath.Join(d, "tokenizer_config.json")); errors.Is(err, os.ErrNotExist) {+ if f, err := fsys.Open("tokenizer_config.json"); errors.Is(err, os.ErrNotExist) {} else if err != nil {return nil, err} else {@@ -172,8 +172,8 @@ type Vocabulary struct {Types []int32}-func parseVocabularyFromTokenizer(p string) (*Vocabulary, error) {- f, err := os.Open(filepath.Join(p, "tokenizer.json"))+func parseVocabularyFromTokenizer(fsys fs.FS) (*Vocabulary, error) {+ f, err := fsys.Open("tokenizer.json")if err != nil {return nil, err}@@ -219,20 +219,20 @@ func parseVocabularyFromTokenizer(p string) (*Vocabulary, error) {return &v, nil}-func parseVocabulary(d string) (*Vocabulary, error) {- patterns := map[string]func(string) (*Vocabulary, error){+func parseVocabulary(fsys fs.FS) (*Vocabulary, error) {+ patterns := map[string]func(fs.FS) (*Vocabulary, error){"tokenizer.model": parseSentencePiece,"tokenizer.json": parseVocabularyFromTokenizer,}for pattern, parseFn := range patterns {- if _, err := os.Stat(filepath.Join(d, pattern)); errors.Is(err, os.ErrNotExist) {+ if _, err := fs.Stat(fsys, pattern); errors.Is(err, os.ErrNotExist) {continue} else if err != nil {return nil, err}- return parseFn(d)+ return parseFn(fsys)}return nil, errors.New("unknown tensor format")diff --git a/convert/tokenizer_spm.go b/convert/tokenizer_spm.goindex 75d9fe267..babf702c7 100644--- a/convert/tokenizer_spm.go+++ b/convert/tokenizer_spm.go@@ -5,8 +5,8 @@ import ("encoding/json""errors""fmt"+ "io/fs""os"- "path/filepath""slices""google.golang.org/protobuf/proto"@@ -14,8 +14,8 @@ import ("github.com/ollama/ollama/convert/sentencepiece")-func parseSentencePiece(d string) (*Vocabulary, error) {- bts, err := os.ReadFile(filepath.Join(d, "tokenizer.model"))+func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {+ bts, err := fs.ReadFile(fsys, "tokenizer.model")if err != nil {return nil, err}@@ -41,7 +41,7 @@ func parseSentencePiece(d string) (*Vocabulary, error) {}}- f, err := os.Open(filepath.Join(d, "added_tokens.json"))+ f, err := fsys.Open("added_tokens.json")if errors.Is(err, os.ErrNotExist) {return &v, nil} else if err != nil {diff --git a/server/model.go b/server/model.goindex 81272a34c..f2946a0be 100644--- a/server/model.go+++ b/server/model.go@@ -81,88 +81,43 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRereturn layers, nil}-func extractFromZipFile(p string, file *os.File, fn func(api.ProgressResponse)) error {- stat, err := file.Stat()- if err != nil {- return err- }-- r, err := zip.NewReader(file, stat.Size())+func parseFromZipFile(_ context.Context, f *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {+ fi, err := f.Stat()if err != nil {- return err- }-- fn(api.ProgressResponse{Status: "unpacking model metadata"})- for _, f := range r.File {- if !filepath.IsLocal(f.Name) {- return fmt.Errorf("%w: %s", zip.ErrInsecurePath, f.Name)- }-- n := filepath.Join(p, f.Name)- if err := os.MkdirAll(filepath.Dir(n), 0o750); err != nil {- return err- }-- // TODO(mxyng): this should not write out all files to disk- outfile, err := os.Create(n)- if err != nil {- return err- }- defer outfile.Close()-- infile, err := f.Open()- if err != nil {- return err- }- defer infile.Close()-- if _, err = io.Copy(outfile, infile); err != nil {- return err- }-- if err := outfile.Close(); err != nil {- return err- }-- if err := infile.Close(); err != nil {- return err- }+ return nil, err}- return nil-}--func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {- tempDir, err := os.MkdirTemp(filepath.Dir(file.Name()), "")+ r, err := zip.NewReader(f, fi.Size())if err != nil {return nil, err}- defer os.RemoveAll(tempDir)- if err := extractFromZipFile(tempDir, file, fn); err != nil {+ p, err := os.MkdirTemp(filepath.Dir(f.Name()), "")+ if err != nil {return nil, err}+ defer os.RemoveAll(p)fn(api.ProgressResponse{Status: "converting model"})-// TODO(mxyng): this should write directly into a layer// e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")- temp, err := os.CreateTemp(tempDir, "fp16")+ t, err := os.CreateTemp(p, "fp16")if err != nil {return nil, err}- defer temp.Close()- defer os.Remove(temp.Name())+ defer t.Close()+ defer os.Remove(t.Name())- if err := convert.Convert(tempDir, temp); err != nil {+ fn(api.ProgressResponse{Status: "converting model"})+ if err := convert.Convert(convert.NewZipReader(r, p, 32<<20), t); err != nil {return nil, err}- if _, err := temp.Seek(0, io.SeekStart); err != nil {+ if _, err := t.Seek(0, io.SeekStart); err != nil {return nil, err}- layer, err := NewLayer(temp, "application/vnd.ollama.image.model")+ layer, err := NewLayer(t, "application/vnd.ollama.image.model")if err != nil {return nil, err}diff --git a/server/model_test.go b/server/model_test.goindex 5829adfce..0a2225d5b 100644--- a/server/model_test.go+++ b/server/model_test.go@@ -1,16 +1,11 @@package serverimport (- "archive/zip""bytes""encoding/json"- "errors""fmt"- "io""os""path/filepath"- "slices"- "strings""testing""github.com/google/go-cmp/cmp"@@ -18,103 +13,6 @@ import ("github.com/ollama/ollama/template")-func createZipFile(t *testing.T, name string) *os.File {- t.Helper()-- f, err := os.CreateTemp(t.TempDir(), "")- if err != nil {- t.Fatal(err)- }-- zf := zip.NewWriter(f)- defer zf.Close()-- zh, err := zf.CreateHeader(&zip.FileHeader{Name: name})- if err != nil {- t.Fatal(err)- }-- if _, err := io.Copy(zh, bytes.NewReader([]byte(""))); err != nil {- t.Fatal(err)- }-- return f-}--func TestExtractFromZipFile(t *testing.T) {- cases := []struct {- name string- expect []string- err error- }{- {- name: "good",- expect: []string{"good"},- },- {- name: strings.Join([]string{"path", "..", "to", "good"}, string(os.PathSeparator)),- expect: []string{filepath.Join("to", "good")},- },- {- name: strings.Join([]string{"path", "..", "to", "..", "good"}, string(os.PathSeparator)),- expect: []string{"good"},- },- {- name: strings.Join([]string{"path", "to", "..", "..", "good"}, string(os.PathSeparator)),- expect: []string{"good"},- },- {- name: strings.Join([]string{"..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "..", "bad"}, string(os.PathSeparator)),- err: zip.ErrInsecurePath,- },- {- name: strings.Join([]string{"path", "..", "..", "to", "bad"}, string(os.PathSeparator)),- err: zip.ErrInsecurePath,- },- }-- for _, tt := range cases {- t.Run(tt.name, func(t *testing.T) {- f := createZipFile(t, tt.name)- defer f.Close()-- tempDir := t.TempDir()- if err := extractFromZipFile(tempDir, f, func(api.ProgressResponse) {}); !errors.Is(err, tt.err) {- t.Fatal(err)- }-- var matches []string- if err := filepath.Walk(tempDir, func(p string, fi os.FileInfo, err error) error {- if err != nil {- return err- }-- if !fi.IsDir() {- matches = append(matches, p)- }-- return nil- }); err != nil {- t.Fatal(err)- }-- var actual []string- for _, match := range matches {- rel, err := filepath.Rel(tempDir, match)- if err != nil {- t.Error(err)- }-- actual = append(actual, rel)- }-- if !slices.Equal(actual, tt.expect) {- t.Fatalf("expected %d files, got %d", len(tt.expect), len(matches))- }- })- }-}-func readFile(t *testing.T, base, name string) *bytes.Buffer {t.Helper()