Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
refactor convert
Michael Yang5e9db9fb0bce
diff --git a/convert/convert.go b/convert/convert.goindex 103de457c..4ad64d721 100644--- a/convert/convert.go+++ b/convert/convert.go@@ -1,200 +1,123 @@package convertimport (- "cmp"- "encoding/binary""encoding/json"+ "errors""fmt""io""log/slog""os""path/filepath"- "slices"- "strings"- "google.golang.org/protobuf/proto"-- "github.com/ollama/ollama/convert/sentencepiece""github.com/ollama/ollama/llm")-const (- _ int32 = iota- tokenTypeNormal- tokenTypeUnknown- tokenTypeControl- tokenTypeUserDefined- tokenTypeUnused- tokenTypeByte-)--type Params struct {- Architectures []string `json:"architectures"`- VocabSize int `json:"vocab_size"`- HiddenSize int `json:"hidden_size"` // n_embd- HiddenLayers int `json:"num_hidden_layers"` // n_layer- ContextSize int `json:"max_position_embeddings"`- IntermediateSize int `json:"intermediate_size"`- AttentionHeads int `json:"num_attention_heads"` // n_head- KeyValHeads int `json:"num_key_value_heads"`- NormEPS float64 `json:"rms_norm_eps"`- BoSTokenID int `json:"bos_token_id"`- EoSTokenID int `json:"eos_token_id"`- HeadDimension int `json:"head_dim"`- PaddingTokenID int `json:"pad_token_id"`- RopeFrequencyBase float64 `json:"rope_theta"`-- Experts int `json:"num_local_experts"`- ExpertsUsed int `json:"num_experts_per_tok"`-- PreTokenizer string-- ByteOrder+type Parameters struct {+ Architectures []string `json:"architectures"`+ VocabSize uint32 `json:"vocab_size"`}-type ByteOrder interface {- binary.ByteOrder- binary.AppendByteOrder-}+func (Parameters) KV(t *Tokenizer) llm.KV {+ kv := llm.KV{+ "general.file_type": uint32(1),+ "general.quantization_version": uint32(2),+ "tokenizer.ggml.pre": t.Pre,+ "tokenizer.ggml.model": t.Vocabulary.Model,+ "tokenizer.ggml.tokens": t.Vocabulary.Tokens,+ "tokenizer.ggml.scores": t.Vocabulary.Scores,+ "tokenizer.ggml.token_type": t.Vocabulary.Types,+ }-type ModelArch interface {- GetTensors() error- LoadVocab() error- WriteGGUF(io.WriteSeeker) error-}+ if t.Template != "" {+ kv["tokenizer.chat_template"] = t.Template+ }-type ModelFormat interface {- GetLayerName(string) (string, error)- GetTensors(string, *Params) ([]llm.Tensor, error)- GetParams(string) (*Params, error)- GetModelArch(string, string, *Params) (ModelArch, error)-}+ for _, sv := range t.SpecialVocabulary {+ kv[fmt.Sprintf("tokenizer.ggml.%s_token_id", sv.Key())] = uint32(sv.ID)+ kv[fmt.Sprintf("tokenizer.ggml.add_%s_token", sv.Key())] = sv.AddToken+ }-type ModelData struct {- Path string- Name string- Params *Params- Vocab *Vocab- Tensors []llm.Tensor- Format ModelFormat+ return kv}-func GetModelFormat(dirname string) (ModelFormat, error) {- files, err := filepath.Glob(filepath.Join(dirname, "*"))- if err != nil {- return nil, err- }-- for _, fn := range files {- if strings.HasSuffix(fn, ".safetensors") {- return &SafetensorFormat{}, nil- } else if strings.HasSuffix(fn, ".bin") || strings.HasSuffix(fn, ".pth") {- slog.Debug("model is torch")- return &TorchFormat{}, nil- }+func (Parameters) specialTypes() []string {+ return []string{+ "bos", "eos", "unk", "sep", "pad", "cls", "mask",}+}- return nil, fmt.Errorf("couldn't determine model format")+func (Parameters) writeFile(ws io.WriteSeeker, kv llm.KV, ts []*llm.Tensor) error {+ return llm.WriteGGUF(ws, kv, ts)}-// Details on gguf's tokenizer can be found at:-// https://github.com/ggerganov/ggml/blob/master/docs/gguf.md#tokenizer-type Vocab struct {- Tokens []string- Scores []float32- Types []int32- Merges []string+type Converter interface {+ // KV maps parameters to LLM key-values+ KV(*Tokenizer) llm.KV+ // Tensors maps input tensors to LLM tensors. Model specific modifications can be done here.+ Tensors([]Tensor) []*llm.Tensor++ // tensorName returns the LLM tensor name for a specific input name+ tensorName(string) string+ // specialTypes returns any special token types the model uses+ specialTypes() []string+ writeFile(io.WriteSeeker, llm.KV, []*llm.Tensor) error}-func LoadSentencePieceTokens(dirpath string, params *Params) (*Vocab, error) {- slog.Info(fmt.Sprintf("reading vocab from %s", filepath.Join(dirpath, "tokenizer.model")))- in, err := os.ReadFile(filepath.Join(dirpath, "tokenizer.model"))+func Convert(d string, ws io.WriteSeeker) error {+ f, err := os.Open(filepath.Join(d, "config.json"))if err != nil {- return nil, err+ return err}+ defer f.Close()- // To regenerate sentencepiece from the protobufs use:- // protoc -I=./ --go_out=./ sentencepiece_model.proto- modelProto := &sentencepiece.ModelProto{}- if err := proto.Unmarshal(in, modelProto); err != nil {- return nil, err+ var p Parameters+ if err := json.NewDecoder(f).Decode(&p); err != nil {+ return err}- v := &Vocab{- Tokens: make([]string, 0),- Scores: make([]float32, 0),- Types: make([]int32, 0),+ if len(p.Architectures) < 1 {+ return errors.New("unknown architecture")}- pieces := modelProto.GetPieces()- for _, p := range pieces {- v.Tokens = append(v.Tokens, p.GetPiece())- v.Scores = append(v.Scores, p.GetScore())- t := p.GetType()- switch t {- case sentencepiece.ModelProto_SentencePiece_UNKNOWN:- case sentencepiece.ModelProto_SentencePiece_CONTROL:- case sentencepiece.ModelProto_SentencePiece_UNUSED:- case sentencepiece.ModelProto_SentencePiece_BYTE:- default:- t = sentencepiece.ModelProto_SentencePiece_NORMAL- }- v.Types = append(v.Types, int32(t))+ var c Converter+ switch p.Architectures[0] {+ case "LlamaForCausalLM", "MistralForCausalLM":+ c = &llama{}+ case "MixtralForCausalLM":+ c = &mixtral{}+ case "GemmaForCausalLM":+ c = &gemma{}+ default:+ return errors.New("unsupported architecture")}- slog.Info(fmt.Sprintf("vocab size: %d", len(v.Tokens)))-- // add any additional tokens- addIn, err := os.ReadFile(filepath.Join(dirpath, "added_tokens.json"))- if os.IsNotExist(err) {- return v, nil- } else if err != nil {- return nil, err- }-- slog.Info("reading user defined tokens")-- var extraTokenData map[string]int- if err := json.Unmarshal(addIn, &extraTokenData); err != nil {- return nil, err+ bts, err := os.ReadFile(filepath.Join(d, "config.json"))+ if err != nil {+ return err}- type token struct {- key string- pos int+ if err := json.Unmarshal(bts, c); err != nil {+ return err}- extraTokens := make([]token, 0)- for k, id := range extraTokenData {- extraTokens = append(extraTokens, token{k, id})+ t, err := parseTokenizer(d, c.specialTypes())+ if err != nil {+ return err}- slices.SortFunc(extraTokens, func(a, b token) int {- return cmp.Compare(a.pos, b.pos)- })-- numToks := len(v.Tokens)-- for cnt, t := range extraTokens {- // the token id should match the specific index for the total number of tokens- if t.pos != cnt+numToks {- return nil, fmt.Errorf("token ID '%d' for '%s' doesn't match total token size", t.pos, t.key)+ if vocabSize := int(p.VocabSize); vocabSize > len(t.Vocabulary.Tokens) {+ slog.Warn("vocabulary is smaller than expected, padding with dummy tokens", "expect", p.VocabSize, "actual", len(t.Vocabulary.Tokens))+ for i := range vocabSize - len(t.Vocabulary.Tokens) {+ t.Vocabulary.Tokens = append(t.Vocabulary.Tokens, fmt.Sprintf("[PAD%d]", i))+ t.Vocabulary.Scores = append(t.Vocabulary.Scores, -1)+ t.Vocabulary.Types = append(t.Vocabulary.Types, tokenTypeUserDefined)}- v.Tokens = append(v.Tokens, t.key)- v.Scores = append(v.Scores, -1000.0)- v.Types = append(v.Types, tokenTypeUserDefined)}- slog.Info(fmt.Sprintf("vocab size w/ extra tokens: %d", len(v.Tokens)))-- if params.VocabSize > len(v.Tokens) {- missingTokens := params.VocabSize - len(v.Tokens)- slog.Warn(fmt.Sprintf("vocab is missing %d tokens", missingTokens))- for cnt := range missingTokens {- v.Tokens = append(v.Tokens, fmt.Sprintf("<dummy%05d>", cnt+1))- v.Scores = append(v.Scores, -1)- v.Types = append(v.Types, tokenTypeUserDefined)- }++ ts, err := parseTensors(d)+ if err != nil {+ return err}- return v, nil+ return c.writeFile(ws, c.KV(t), c.Tensors(ts))}diff --git a/convert/convert_gemma.go b/convert/convert_gemma.gonew file mode 100644index 000000000..332fee7f2--- /dev/null+++ b/convert/convert_gemma.go@@ -0,0 +1,103 @@+package convert++import (+ "strings"++ "github.com/pdevine/tensor"+ "github.com/pdevine/tensor/native"++ "github.com/ollama/ollama/llm"+)++type gemma struct {+ Parameters+ MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`+ HiddenSize uint32 `json:"hidden_size"`+ HiddenLayers uint32 `json:"num_hidden_layers"`+ IntermediateSize uint32 `json:"intermediate_size"`+ NumAttentionHeads uint32 `json:"num_attention_heads"`+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`+ RMSNormEPS float32 `json:"rms_norm_eps"`+ HeadDim uint32 `json:"head_dim"`+}++var _ Converter = (*gemma)(nil)++func (p *gemma) KV(t *Tokenizer) llm.KV {+ kv := p.Parameters.KV(t)+ kv["general.architecture"] = "gemma"+ kv["general.name"] = "gemma"+ kv["gemma.context_length"] = p.MaxPositionEmbeddings+ kv["gemma.embedding_length"] = p.HiddenSize+ kv["gemma.block_count"] = p.HiddenLayers+ kv["gemma.feed_forward_length"] = p.IntermediateSize+ kv["gemma.attention.head_count"] = p.NumAttentionHeads+ kv["gemma.attention.head_count_kv"] = p.NumKeyValueHeads+ kv["gemma.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS+ kv["gemma.attention.key_length"] = p.HeadDim+ kv["gemma.attention.value_length"] = p.HeadDim+ kv["tokenizer.ggml.eot_token_id"] = uint32(107)+ kv["tokenizer.ggml.middle_token_id"] = uint32(68)+ kv["tokenizer.ggml.prefix_token_id"] = uint32(67)+ kv["tokenizer.ggml.suffix_token_id"] = uint32(69)+ return kv+}++func (p *gemma) Tensors(ts []Tensor) []*llm.Tensor {+ var out []*llm.Tensor+ for _, t := range ts {+ name := p.tensorName(t.Name())+ if strings.HasSuffix(name, "_norm.weight") {+ t.SetRepacker(p.addOne)+ }++ out = append(out, &llm.Tensor{+ Name: name,+ Kind: t.Kind(),+ Shape: t.Shape(),+ WriterTo: t,+ })+ }++ return out+}++func (p *gemma) tensorName(n string) string {+ return strings.NewReplacer(+ "model.embed_tokens", "token_embd",+ "model.norm", "output_norm",+ "model.layers", "blk",+ "input_layernorm", "attn_norm",+ "self_attn.q_proj", "attn_q",+ "self_attn.k_proj", "attn_k",+ "self_attn.v_proj", "attn_v",+ "self_attn.o_proj", "attn_output",+ "mlp.gate_proj", "ffn_gate",+ "mlp.down_proj", "ffn_down",+ "mlp.up_proj", "ffn_up",+ "post_attention_layernorm", "ffn_norm",+ "block_sparse_moe.gate", "ffn_inp",+ ).Replace(n)+}++func (*gemma) addOne(_ string, data []float32, shape []uint64) ([]float32, error) {+ n := tensor.New(tensor.WithShape(int(shape[0])), tensor.WithBacking(data))+ ones := tensor.Ones(tensor.Float32, int(shape[0]))++ n, err := n.Add(ones)+ if err != nil {+ return nil, err+ }++ ts, err := native.SelectF32(n, 0)+ if err != nil {+ return nil, err+ }++ var f32s []float32+ for _, t := range ts {+ f32s = append(f32s, t...)+ }++ return f32s, nil+}diff --git a/convert/convert_llama.go b/convert/convert_llama.gonew file mode 100644index 000000000..700049d32--- /dev/null+++ b/convert/convert_llama.go@@ -0,0 +1,182 @@+package convert++import (+ "cmp"+ "fmt"+ "strings"++ "github.com/ollama/ollama/llm"+ "github.com/pdevine/tensor"+ "github.com/pdevine/tensor/native"+)++type llama struct {+ Parameters+ NLayers uint32 `json:"n_layers"`+ NumHiddenLayers uint32 `json:"num_hidden_layers"`+ NLayer uint32 `json:"n_layer"`+ MaxPositionEmbeddings uint32 `json:"max_position_embeddings"`+ NCtx uint32 `json:"n_ctx"`+ HiddenSize uint32 `json:"hidden_size"`+ NEmbd uint32 `json:"n_embd"`+ IntermediateSize uint32 `json:"intermediate_size"`+ NInner uint32 `json:"n_inner"`+ NumAttentionHeads uint32 `json:"num_attention_heads"`+ NHead uint32 `json:"n_head"`+ NumKeyValueHeads uint32 `json:"num_key_value_heads"`+ RopeTheta float32 `json:"rope_theta"`+ RopeScaling struct {+ Type string `json:"type"`+ Factor float32 `json:"factor"`+ } `json:"rope_scaling"`+ RMSNormEPS float32 `json:"rms_norm_eps"`+ LayerNormEPS float32 `json:"layer_norm_eps"`+ LayerNormEpsilon float32 `json:"layer_norm_epsilon"`+ NormEpsilon float32 `json:"norm_epsilon"`+ HeadDim uint32 `json:"head_dim"`+}++var _ Converter = (*llama)(nil)++func (p *llama) KV(t *Tokenizer) llm.KV {+ kv := p.Parameters.KV(t)+ kv["general.architecture"] = "llama"+ kv["general.name"] = "llama"+ kv["llama.vocab_size"] = p.VocabSize++ kv["llama.block_count"] = cmp.Or(p.NLayers, p.NumHiddenLayers, p.NLayer)++ if contextLength := cmp.Or(p.MaxPositionEmbeddings, p.NCtx); contextLength > 0 {+ kv["llama.context_length"] = contextLength+ }++ if embeddingLength := cmp.Or(p.HiddenSize, p.NEmbd); embeddingLength > 0 {+ kv["llama.embedding_length"] = cmp.Or(p.HiddenSize, p.NEmbd)+ }++ if feedForwardLength := cmp.Or(p.IntermediateSize, p.NInner); feedForwardLength > 0 {+ kv["llama.feed_forward_length"] = cmp.Or(p.IntermediateSize, p.NInner)+ }++ if headCount := cmp.Or(p.NumAttentionHeads, p.NHead); headCount > 0 {+ kv["llama.attention.head_count"] = cmp.Or(p.NumAttentionHeads, p.NHead)+ kv["llama.rope.dimension_count"] = p.HiddenSize / headCount+ }++ if p.RopeTheta > 0 {+ kv["llama.rope.freq_base"] = p.RopeTheta+ }++ if p.RopeScaling.Type == "linear" {+ kv["llama.rope.scaling.type"] = p.RopeScaling.Type+ kv["llama.rope.scaling.factor"] = p.RopeScaling.Factor+ }++ if p.NumKeyValueHeads > 0 {+ kv["llama.attention.head_count_kv"] = p.NumKeyValueHeads+ }++ if p.RMSNormEPS > 0 {+ kv["llama.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS+ }++ if layerNormEpsilon := cmp.Or(p.LayerNormEPS, p.LayerNormEpsilon, p.NormEpsilon); layerNormEpsilon > 0 {+ kv["llama.attention.layer_norm_epsilon"] = layerNormEpsilon+ }++ if p.HeadDim > 0 {+ kv["llama.attention.key_length"] = p.HeadDim+ kv["llama.attention.value_length"] = p.HeadDim+ }++ if len(t.Merges) > 0 {+ kv["tokenizer.ggml.merges"] = t.Merges+ }++ return kv+}++func (p *llama) Tensors(ts []Tensor) []*llm.Tensor {+ var out []*llm.Tensor+ for _, t := range ts {+ name := p.tensorName(t.Name())+ if strings.HasSuffix(name, "attn_q.weight") ||+ strings.HasSuffix(name, "attn_k.weight") {+ t.SetRepacker(p.repack)+ }++ out = append(out, &llm.Tensor{+ Name: name,+ Kind: t.Kind(),+ Shape: t.Shape(),+ WriterTo: t,+ })+ }++ return out+}++func (p *llama) tensorName(n string) string {+ return strings.NewReplacer(+ "lm_head", "output",+ "model.embed_tokens", "token_embd",+ "model.norm", "output_norm",+ "model.layers", "blk",+ "input_layernorm", "attn_norm",+ "self_attn.q_proj", "attn_q",+ "self_attn.k_proj", "attn_k",+ "self_attn.v_proj", "attn_v",+ "self_attn.o_proj", "attn_output",+ "mlp.gate_proj", "ffn_gate",+ "mlp.down_proj", "ffn_down",+ "mlp.up_proj", "ffn_up",+ "post_attention_layernorm", "ffn_norm",+ // mixtral+ "block_sparse_moe.gate", "ffn_gate_inp",+ ).Replace(n)+}++func (p *llama) repack(name string, data []float32, shape []uint64) ([]float32, error) {+ var dims []int+ for _, dim := range shape {+ dims = append(dims, int(dim))+ }++ var heads uint32+ if strings.HasSuffix(name, "q_proj.weight") {+ heads = p.NumAttentionHeads+ } else if strings.HasSuffix(name, "k_proj.weight") {+ heads = cmp.Or(p.NumKeyValueHeads, p.NumAttentionHeads)+ } else {+ return nil, fmt.Errorf("unknown tensor for repack: %s", name)+ }++ n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))+ if err := n.Reshape(append([]int{int(heads), 2, dims[0] / int(heads) / 2}, dims[1:]...)...); err != nil {+ return nil, err+ }++ if err := n.T(0, 2, 1, 3); err != nil {+ return nil, err+ }++ if err := n.Reshape(dims...); err != nil {+ return nil, err+ }++ if err := n.Transpose(); err != nil {+ return nil, err+ }++ ts, err := native.SelectF32(n, 1)+ if err != nil {+ return nil, err+ }++ var f32s []float32+ for _, t := range ts {+ f32s = append(f32s, t...)+ }++ return f32s, nil+}diff --git a/convert/convert_mixtral.go b/convert/convert_mixtral.gonew file mode 100644index 000000000..c55a27f81--- /dev/null+++ b/convert/convert_mixtral.go@@ -0,0 +1,89 @@+package convert++import (+ "fmt"+ "io"+ "slices"+ "strings"++ "github.com/ollama/ollama/llm"+)++type mixtral struct {+ llama+ NumLocalExperts uint32 `json:"num_local_experts"`+ NumExpertsPerToken uint32 `json:"num_experts_per_tok"`+}++var _ Converter = (*mixtral)(nil)++func (p *mixtral) KV(t *Tokenizer) llm.KV {+ kv := p.llama.KV(t)++ if p.NumLocalExperts > 0 {+ kv["llama.expert_count"] = p.NumLocalExperts+ }++ if p.NumExpertsPerToken > 0 {+ kv["llama.expert_used_count"] = p.NumExpertsPerToken+ }++ return kv+}++func (p *mixtral) Tensors(ts []Tensor) []*llm.Tensor {+ oldnew := []string{+ "model.layers", "blk",+ "w1", "ffn_gate_exps",+ "w2", "ffn_down_exps",+ "w3", "ffn_up_exps",+ }++ for i := range p.NumLocalExperts {+ oldnew = append(oldnew, fmt.Sprintf(".block_sparse_moe.experts.%d.", i), ".")+ }++ // group experts of the same layer (model.layers.%d) and type (w[123]) into a single tensor+ namer := strings.NewReplacer(oldnew...)+ experts := make(map[string]experts)++ // merge experts into a single tensor while removing them from ts+ ts = slices.DeleteFunc(ts, func(t Tensor) bool {+ if !strings.Contains(t.Name(), ".block_sparse_moe.experts.") {+ return false+ }++ name := namer.Replace(t.Name())+ experts[name] = append(experts[name], t)+ return true+ })++ var out []*llm.Tensor+ for n, e := range experts {+ // TODO(mxyng): sanity check experts+ out = append(out, &llm.Tensor{+ Name: n,+ Kind: e[0].Kind(),+ Shape: append([]uint64{uint64(len(e))}, e[0].Shape()...),+ WriterTo: e,+ })+ }++ return append(out, p.llama.Tensors(ts)...)+}++type experts []Tensor++func (e experts) WriteTo(w io.Writer) (int64, error) {+ // TODO(mxyng): experts _should_ be numerically sorted by expert but this should check+ for _, t := range e {+ // the canonical merged experts tensor stacks all experts along a new, 0 axis,+ // e.g. `tensor.Stack(0, e[0], e[1:]...)`, which requires allocating temporary buffers+ // this accomplishes the same thing by writing each expert tensor in sequence+ if _, err := t.WriteTo(w); err != nil {+ return 0, err+ }+ }++ return 0, nil+}diff --git a/convert/convert_test.go b/convert/convert_test.goindex a3727bedc..0fbd436f5 100644--- a/convert/convert_test.go+++ b/convert/convert_test.go@@ -20,36 +20,13 @@ import (func convertFull(t *testing.T, d string) (*os.File, llm.KV, llm.Tensors) {t.Helper()- mf, err := GetModelFormat(d)- if err != nil {- t.Fatal(err)- }-- params, err := mf.GetParams(d)- if err != nil {- t.Fatal(err)- }-- arch, err := mf.GetModelArch("", d, params)- if err != nil {- t.Fatal(err)- }-- if err := arch.LoadVocab(); err != nil {- t.Fatal(err)- }-- if err := arch.GetTensors(); err != nil {- t.Fatal(err)- }-f, err := os.CreateTemp(t.TempDir(), "f16")if err != nil {t.Fatal(err)}defer f.Close()- if err := arch.WriteGGUF(f); err != nil {+ if err := Convert(d, f); err != nil {t.Fatal(err)}diff --git a/convert/gemma.go b/convert/gemma.godeleted file mode 100644index d01ffedf1..000000000--- a/convert/gemma.go+++ /dev/null@@ -1,102 +0,0 @@-package convert--import (- "fmt"- "io"- "log/slog"- "strings"-- "github.com/pdevine/tensor"- "github.com/pdevine/tensor/native"-- "github.com/ollama/ollama/llm"-)--type GemmaModel struct {- ModelData-}--func addOnes(data []float32, vectorSize int) ([]float32, error) {- n := tensor.New(tensor.WithShape(vectorSize), tensor.WithBacking(data))- ones := tensor.Ones(tensor.Float32, vectorSize)-- n, err := n.Add(ones)- if err != nil {- return nil, err- }-- ts, err := native.SelectF32(n, 0)- if err != nil {- return nil, err- }-- var f32s []float32- for _, t := range ts {- f32s = append(f32s, t...)- }-- return f32s, nil-}--func (m *GemmaModel) GetTensors() error {- t, err := m.Format.GetTensors(m.Path, m.Params)- if err != nil {- return err- }-- slog.Debug(fmt.Sprintf("Total tensors: %d", len(t)))- for _, l := range t {- if strings.HasSuffix(l.Name, "norm.weight") {- wt := l.WriterTo.(safetensorWriterTo)- wt.repacker = m.Repack- l.WriterTo = wt- }- m.Tensors = append(m.Tensors, l)- }-- return nil-}--func (m *GemmaModel) LoadVocab() error {- v, err := LoadSentencePieceTokens(m.Path, m.Params)- if err != nil {- return err- }- m.Vocab = v- return nil-}--func (m *GemmaModel) Repack(_ string, data []float32, shape []uint64) ([]float32, error) {- return addOnes(data, int(shape[0]))-}--func (m *GemmaModel) WriteGGUF(ws io.WriteSeeker) error {- kv := llm.KV{- "general.architecture": "gemma",- "general.name": m.Name,- "gemma.context_length": uint32(m.Params.ContextSize),- "gemma.embedding_length": uint32(m.Params.HiddenSize),- "gemma.block_count": uint32(m.Params.HiddenLayers),- "gemma.feed_forward_length": uint32(m.Params.IntermediateSize),- "gemma.attention.head_count": uint32(m.Params.AttentionHeads),- "gemma.attention.head_count_kv": uint32(m.Params.KeyValHeads),- "gemma.attention.layer_norm_rms_epsilon": float32(m.Params.NormEPS),- "gemma.attention.key_length": uint32(m.Params.HeadDimension),- "gemma.attention.value_length": uint32(m.Params.HeadDimension),- "general.file_type": uint32(1),- "tokenizer.ggml.model": "llama",-- "tokenizer.ggml.tokens": m.Vocab.Tokens,- "tokenizer.ggml.scores": m.Vocab.Scores,- "tokenizer.ggml.token_type": m.Vocab.Types,-- "tokenizer.ggml.bos_token_id": uint32(m.Params.BoSTokenID),- "tokenizer.ggml.eos_token_id": uint32(m.Params.EoSTokenID),- "tokenizer.ggml.padding_token_id": uint32(m.Params.PaddingTokenID),- "tokenizer.ggml.unknown_token_id": uint32(3),- "tokenizer.ggml.add_bos_token": true,- "tokenizer.ggml.add_eos_token": false,- }-- return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)-}diff --git a/convert/llama.go b/convert/llama.godeleted file mode 100644index b4211b02d..000000000--- a/convert/llama.go+++ /dev/null@@ -1,159 +0,0 @@-package convert--import (- "cmp"- "errors"- "fmt"- "io"- "os"- "path/filepath"- "regexp"- "strings"-- "github.com/pdevine/tensor"- "github.com/pdevine/tensor/native"-- "github.com/ollama/ollama/llm"-)--type LlamaModel struct {- ModelData-}--func (m *LlamaModel) GetTensors() error {- t, err := m.Format.GetTensors(m.Path, m.Params)- if err != nil {- return err- }-- pattern := `^blk\.[0-9]+\.attn_(?P<layer>q|k)\.weight$`- re, err := regexp.Compile(pattern)- if err != nil {- return err- }-- for _, l := range t {- matches := re.FindAllStringSubmatch(l.Name, -1)- if len(matches) > 0 {- switch m.Format.(type) {- case *TorchFormat:- wt := l.WriterTo.(torchWriterTo)- wt.repacker = m.Repack- l.WriterTo = wt- case *SafetensorFormat:- wt := l.WriterTo.(safetensorWriterTo)- wt.repacker = m.Repack- l.WriterTo = wt- }- }- m.Tensors = append(m.Tensors, l)- }-- return nil-}--func (m *LlamaModel) LoadVocab() (err error) {- pre, ts, merges, err := parseTokens(filepath.Join(m.Path, "tokenizer.json"))- if errors.Is(err, os.ErrNotExist) {- return nil- } else if err != nil {- return err- }-- m.Vocab = &Vocab{}- for _, t := range ts {- m.Vocab.Tokens = append(m.Vocab.Tokens, t.Content)- m.Vocab.Types = append(m.Vocab.Types, t.Type())- }-- m.Vocab.Merges = merges- m.Params.PreTokenizer = pre- return nil-}--func (m *LlamaModel) WriteGGUF(ws io.WriteSeeker) error {- kv := llm.KV{- "general.architecture": "llama",- "general.name": m.Name,- "llama.vocab_size": uint32(len(m.Vocab.Tokens)),- "llama.context_length": uint32(m.Params.ContextSize),- "llama.embedding_length": uint32(m.Params.HiddenSize),- "llama.block_count": uint32(m.Params.HiddenLayers),- "llama.feed_forward_length": uint32(m.Params.IntermediateSize),- "llama.rope.freq_base": float32(m.Params.RopeFrequencyBase),- "llama.rope.dimension_count": uint32(m.Params.HiddenSize / m.Params.AttentionHeads),- "llama.attention.head_count": uint32(m.Params.AttentionHeads),- "llama.attention.head_count_kv": uint32(m.Params.KeyValHeads),- "llama.attention.layer_norm_rms_epsilon": float32(m.Params.NormEPS),- "general.file_type": uint32(1),- "tokenizer.ggml.model": "gpt2",-- "tokenizer.ggml.pre": m.Params.PreTokenizer,- "tokenizer.ggml.tokens": m.Vocab.Tokens,- "tokenizer.ggml.token_type": m.Vocab.Types,-- "tokenizer.ggml.bos_token_id": uint32(m.Params.BoSTokenID),- "tokenizer.ggml.eos_token_id": uint32(m.Params.EoSTokenID),- "tokenizer.ggml.unknown_token_id": uint32(0),- }-- if len(m.Vocab.Merges) > 0 {- kv["tokenizer.ggml.merges"] = m.Vocab.Merges- } else {- kv["tokenizer.ggml.scores"] = m.Vocab.Scores- }-- return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)-}--func (m *LlamaModel) Repack(name string, data []float32, shape []uint64) ([]float32, error) {- return llamaRepack(name, m.Params, data, shape)-}--func llamaRepack(name string, params *Params, data []float32, shape []uint64) ([]float32, error) {- var dims []int- for _, dim := range shape {- if dim != 0 {- dims = append(dims, int(dim))- }- }-- var heads int- switch {- case strings.HasSuffix(name, "attn_q.weight"):- heads = params.AttentionHeads- case strings.HasSuffix(name, "attn_k.weight"):- heads = cmp.Or(params.KeyValHeads, params.AttentionHeads)- default:- return nil, fmt.Errorf("unknown tensor name: %s", name)- }-- n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))- if err := n.Reshape(append([]int{heads, 2, dims[0] / heads / 2}, dims[1:]...)...); err != nil {- return nil, err- }-- if err := n.T(0, 2, 1, 3); err != nil {- return nil, err- }-- if err := n.Reshape(dims...); err != nil {- return nil, err- }-- if err := n.Transpose(); err != nil {- return nil, err- }-- ts, err := native.SelectF32(n, 1)- if err != nil {- return nil, err- }-- var f32s []float32- for _, t := range ts {- f32s = append(f32s, t...)- }-- return f32s, nil-}diff --git a/convert/mistral.go b/convert/mistral.godeleted file mode 100644index 8fe066d6a..000000000--- a/convert/mistral.go+++ /dev/null@@ -1,84 +0,0 @@-package convert--import (- "io"- "regexp"-- "github.com/ollama/ollama/llm"-)--type MistralModel struct {- ModelData-}--func (m *MistralModel) GetTensors() error {- t, err := m.Format.GetTensors(m.Path, m.Params)- if err != nil {- return err- }-- pattern := `^blk\.[0-9]+\.attn_(?P<layer>q|k)\.weight$`- re, err := regexp.Compile(pattern)- if err != nil {- return err- }-- for _, l := range t {- matches := re.FindAllStringSubmatch(l.Name, -1)- if len(matches) > 0 {- wt := l.WriterTo.(safetensorWriterTo)- wt.repacker = m.Repack- l.WriterTo = wt- }- m.Tensors = append(m.Tensors, l)- }-- return nil-}--func (m *MistralModel) LoadVocab() error {- v, err := LoadSentencePieceTokens(m.Path, m.Params)- if err != nil {- return err- }- m.Vocab = v- return nil-}--func (m *MistralModel) WriteGGUF(ws io.WriteSeeker) error {- kv := llm.KV{- "general.architecture": "llama",- "general.name": m.Name,- "llama.context_length": uint32(m.Params.ContextSize),- "llama.embedding_length": uint32(m.Params.HiddenSize),- "llama.block_count": uint32(m.Params.HiddenLayers),- "llama.feed_forward_length": uint32(m.Params.IntermediateSize),- "llama.rope.dimension_count": uint32(m.Params.HiddenSize / m.Params.AttentionHeads),- "llama.attention.head_count": uint32(m.Params.AttentionHeads),- "llama.attention.head_count_kv": uint32(m.Params.KeyValHeads),- "llama.attention.layer_norm_rms_epsilon": float32(m.Params.NormEPS),- "general.file_type": uint32(1),- "tokenizer.ggml.model": "llama",-- "tokenizer.ggml.tokens": m.Vocab.Tokens,- "tokenizer.ggml.scores": m.Vocab.Scores,- "tokenizer.ggml.token_type": m.Vocab.Types,-- "tokenizer.ggml.bos_token_id": uint32(m.Params.BoSTokenID),- "tokenizer.ggml.eos_token_id": uint32(m.Params.EoSTokenID),- "tokenizer.ggml.add_bos_token": true,- "tokenizer.ggml.add_eos_token": false,- "tokenizer.ggml.unknown_token_id": uint32(0),- }-- if m.Params.HeadDimension > 0 {- kv["llama.attention.key_length"] = uint32(m.Params.HeadDimension)- kv["llama.attention.value_length"] = uint32(m.Params.HeadDimension)- }-- return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)-}--func (m *MistralModel) Repack(name string, data []float32, shape []uint64) ([]float32, error) {- return llamaRepack(name, m.Params, data, shape)-}diff --git a/convert/mixtral.go b/convert/mixtral.godeleted file mode 100644index baea68cd3..000000000--- a/convert/mixtral.go+++ /dev/null@@ -1,87 +0,0 @@-package convert--import (- "io"- "regexp"-- "github.com/ollama/ollama/llm"-)--type MixtralModel struct {- ModelData-}--func (m *MixtralModel) GetTensors() error {- t, err := m.Format.GetTensors(m.Path, m.Params)- if err != nil {- return err- }-- pattern := `^blk\.[0-9]+\.attn_(?P<layer>q|k)\.weight$`- re, err := regexp.Compile(pattern)- if err != nil {- return err- }-- for _, l := range t {- matches := re.FindAllStringSubmatch(l.Name, -1)- if len(matches) > 0 {- wt := l.WriterTo.(safetensorWriterTo)- wt.repacker = m.Repack- l.WriterTo = wt- }- m.Tensors = append(m.Tensors, l)- }-- return nil-}--func (m *MixtralModel) LoadVocab() error {- v, err := LoadSentencePieceTokens(m.Path, m.Params)- if err != nil {- return err- }- m.Vocab = v- return nil-}--func (m *MixtralModel) WriteGGUF(ws io.WriteSeeker) error {- kv := llm.KV{- "general.architecture": "llama",- "general.name": m.Name,- "llama.block_count": uint32(m.Params.HiddenLayers),- "llama.context_length": uint32(m.Params.ContextSize),- "llama.embedding_length": uint32(m.Params.HiddenSize),- "llama.feed_forward_length": uint32(m.Params.IntermediateSize),- "llama.attention.head_count": uint32(m.Params.AttentionHeads),- "llama.attention.head_count_kv": uint32(m.Params.KeyValHeads),-- "llama.rope.freq_base": float32(m.Params.RopeFrequencyBase),- "llama.attention.layer_norm_rms_epsilon": float32(m.Params.NormEPS),-- "llama.expert_count": uint32(m.Params.Experts),- "llama.expert_used_count": uint32(m.Params.ExpertsUsed),-- "llama.vocab_size": uint32(len(m.Vocab.Tokens)),- "llama.rope.dimension_count": uint32(m.Params.HiddenSize / m.Params.AttentionHeads),-- "general.file_type": uint32(1),- "tokenizer.ggml.model": "llama",-- "tokenizer.ggml.tokens": m.Vocab.Tokens,- "tokenizer.ggml.scores": m.Vocab.Scores,- "tokenizer.ggml.token_type": m.Vocab.Types,-- "tokenizer.ggml.bos_token_id": uint32(m.Params.BoSTokenID),- "tokenizer.ggml.eos_token_id": uint32(m.Params.EoSTokenID),- "tokenizer.ggml.unknown_token_id": uint32(0),- "tokenizer.ggml.add_bos_token": true,- "tokenizer.ggml.add_eos_token": false,- }-- return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)-}--func (m *MixtralModel) Repack(name string, data []float32, shape []uint64) ([]float32, error) {- return llamaRepack(name, m.Params, data, shape)-}diff --git a/convert/reader.go b/convert/reader.gonew file mode 100644index 000000000..9be8ac2eb--- /dev/null+++ b/convert/reader.go@@ -0,0 +1,74 @@+package convert++import (+ "errors"+ "io"+ "path/filepath"+ "strings"+)++type Tensor interface {+ Name() string+ Shape() []uint64+ Kind() uint32+ SetRepacker(repacker)+ WriteTo(io.Writer) (int64, error)+}++type tensorBase struct {+ name string+ shape []uint64+ repacker+}++func (t tensorBase) Name() string {+ return t.name+}++func (t tensorBase) Shape() []uint64 {+ return t.shape+}++func (t tensorBase) Kind() uint32 {+ if strings.HasSuffix(t.name, ".block_sparse_moe.gate.weight") {+ return 0+ }++ switch len(t.shape) {+ case 0:+ panic("invalid tensor shape")+ case 1:+ return 0+ default:+ return 1+ }+}++func (t *tensorBase) SetRepacker(fn repacker) {+ t.repacker = fn+}++type repacker func(string, []float32, []uint64) ([]float32, error)++func parseTensors(d string) ([]Tensor, error) {+ patterns := map[string]func(...string) ([]Tensor, error){+ "model-*-of-*.safetensors": parseSafetensors,+ "model.safetensors": parseSafetensors,+ "pytorch_model-*-of-*.bin": parseTorch,+ "pytorch_model.bin": parseTorch,+ "consolidated.*.pth": parseTorch,+ }++ for pattern, parseFn := range patterns {+ matches, err := filepath.Glob(filepath.Join(d, pattern))+ if err != nil {+ return nil, err+ }++ if len(matches) > 0 {+ return parseFn(matches...)+ }+ }++ return nil, errors.New("unknown tensor format")+}diff --git a/convert/reader_safetensors.go b/convert/reader_safetensors.gonew file mode 100644index 000000000..440581af8--- /dev/null+++ b/convert/reader_safetensors.go@@ -0,0 +1,140 @@+package convert++import (+ "bytes"+ "encoding/binary"+ "encoding/json"+ "fmt"+ "io"+ "os"+ "slices"++ "github.com/d4l3k/go-bfloat16"+ "github.com/x448/float16"+ "golang.org/x/exp/maps"+)++type safetensorMetadata struct {+ Type string `json:"dtype"`+ Shape []uint64 `json:"shape"`+ Offsets []int64 `json:"data_offsets"`+}++func parseSafetensors(ps ...string) ([]Tensor, error) {+ var ts []Tensor+ for _, p := range ps {+ f, err := os.Open(p)+ if err != nil {+ return nil, err+ }+ defer f.Close()++ var n int64+ if err := binary.Read(f, binary.LittleEndian, &n); err != nil {+ return nil, err+ }++ b := bytes.NewBuffer(make([]byte, 0, n))+ if _, err = io.CopyN(b, f, n); err != nil {+ return nil, err+ }++ var headers map[string]safetensorMetadata+ if err := json.NewDecoder(b).Decode(&headers); err != nil {+ return nil, err+ }++ keys := maps.Keys(headers)+ slices.Sort(keys)++ for _, key := range keys {+ if value := headers[key]; value.Type != "" {+ ts = append(ts, safetensor{+ path: p,+ dtype: value.Type,+ offset: safetensorsPad(n, value.Offsets[0]),+ size: safetensorsPad(n, value.Offsets[1]) - safetensorsPad(n, value.Offsets[0]),+ tensorBase: &tensorBase{+ name: key,+ shape: value.Shape,+ },+ })+ }+ }+ }++ return ts, nil+}++func safetensorsPad(n, s int64) int64 {+ return 8 + n + s+}++type safetensor struct {+ path string+ dtype string+ offset int64+ size int64+ *tensorBase+}++func (st safetensor) WriteTo(w io.Writer) (int64, error) {+ f, err := os.Open(st.path)+ if err != nil {+ return 0, err+ }+ defer f.Close()++ if _, err = f.Seek(st.offset, io.SeekStart); err != nil {+ return 0, err+ }++ var f32s []float32+ switch st.dtype {+ case "F32":+ f32s = make([]float32, st.size/4)+ if err = binary.Read(f, binary.LittleEndian, f32s); err != nil {+ return 0, err+ }+ case "F16":+ u16s := make([]uint16, st.size/2)+ if err = binary.Read(f, binary.LittleEndian, u16s); err != nil {+ return 0, err+ }++ for _, b := range u16s {+ f32s = append(f32s, float16.Frombits(b).Float32())+ }++ case "BF16":+ u8s := make([]uint8, st.size)+ if err = binary.Read(f, binary.LittleEndian, u8s); err != nil {+ return 0, err+ }++ f32s = bfloat16.DecodeFloat32(u8s)+ default:+ return 0, fmt.Errorf("unknown data type: %s", st.dtype)+ }++ if st.repacker != nil {+ f32s, err = st.repacker(st.Name(), f32s, st.Shape())+ if err != nil {+ return 0, err+ }+ }++ switch st.Kind() {+ case 0:+ return 0, binary.Write(w, binary.LittleEndian, f32s)+ case 1:+ f16s := make([]uint16, len(f32s))+ for i := range f32s {+ f16s[i] = float16.Fromfloat32(f32s[i]).Bits()+ }++ return 0, binary.Write(w, binary.LittleEndian, f16s)+ default:+ return 0, fmt.Errorf("unknown storage type: %d", st.Kind())+ }+}diff --git a/convert/reader_torch.go b/convert/reader_torch.gonew file mode 100644index 000000000..1428706e2--- /dev/null+++ b/convert/reader_torch.go@@ -0,0 +1,46 @@+package convert++import (+ "io"++ "github.com/nlpodyssey/gopickle/pytorch"+ "github.com/nlpodyssey/gopickle/types"+)++func parseTorch(ps ...string) ([]Tensor, error) {+ var ts []Tensor+ for _, p := range ps {+ pt, err := pytorch.Load(p)+ if err != nil {+ return nil, err+ }++ for _, k := range pt.(*types.Dict).Keys() {+ t := pt.(*types.Dict).MustGet(k)++ var shape []uint64+ for dim := range t.(*pytorch.Tensor).Size {+ shape = append(shape, uint64(dim))+ }++ ts = append(ts, torch{+ storage: t.(*pytorch.Tensor).Source,+ tensorBase: &tensorBase{+ name: k.(string),+ shape: shape,+ },+ })+ }+ }++ return ts, nil+}++type torch struct {+ storage pytorch.StorageInterface+ *tensorBase+}++func (pt torch) WriteTo(w io.Writer) (int64, error) {+ return 0, nil+}diff --git a/convert/safetensors.go b/convert/safetensors.godeleted file mode 100644index f45687f10..000000000--- a/convert/safetensors.go+++ /dev/null@@ -1,309 +0,0 @@-package convert--import (- "bytes"- "encoding/binary"- "encoding/json"- "fmt"- "io"- "os"- "path/filepath"- "regexp"- "slices"- "strings"-- "github.com/d4l3k/go-bfloat16"- "github.com/x448/float16"-- "github.com/ollama/ollama/llm"-)--type safetensorWriterTo struct {- t *llm.Tensor-- params *Params- bo ByteOrder-- filename string- dtype string-- offset, size int64- repacker func(string, []float32, []uint64) ([]float32, error)-}--type safetensorMetadata struct {- Type string `json:"dtype"`- Shape []uint64 `json:"shape"`- Offsets []int64 `json:"data_offsets"`-}--type SafetensorFormat struct{}--func (m *SafetensorFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor, error) {- var tensors []llm.Tensor- matches, err := filepath.Glob(filepath.Join(dirpath, "*.safetensors"))- if err != nil {- return nil, err- }-- var offset uint64- for _, f := range matches {- var t []llm.Tensor- var err error- t, offset, err = m.readTensors(f, offset, params)- if err != nil {- return nil, err- }-- tensors = append(tensors, t...)- }- return tensors, nil-}--func (m *SafetensorFormat) readTensors(fn string, offset uint64, params *Params) ([]llm.Tensor, uint64, error) {- f, err := os.Open(fn)- if err != nil {- return nil, 0, err- }- defer f.Close()-- var n int64- if err := binary.Read(f, binary.LittleEndian, &n); err != nil {- return nil, 0, err- }-- b := bytes.NewBuffer(make([]byte, 0, n))- if _, err = io.CopyN(b, f, n); err != nil {- return nil, 0, err- }-- var headers map[string]safetensorMetadata- if err := json.NewDecoder(b).Decode(&headers); err != nil {- return nil, 0, err- }-- var keys []string- for key := range headers {- if !strings.HasSuffix(key, "self_attn.rotary_embd.inv_freq") {- keys = append(keys, key)- }- }-- slices.Sort(keys)-- var tensors []llm.Tensor- for _, key := range keys {- value := headers[key]-- var kind uint32- switch len(value.Shape) {- case 0:- // valuedata- continue- case 2:- kind = 1- }-- name, err := m.GetLayerName(key)- if err != nil {- return nil, 0, err- }-- shape := make([]uint64, len(value.Shape))- copy(shape, value.Shape)-- pad := func(s int64) int64 {- return 8 + n + s- }-- t := llm.Tensor{- Name: name,- Kind: kind,- Offset: offset,- Shape: shape,- }-- t.WriterTo = safetensorWriterTo{- t: &t,- params: params,- bo: params.ByteOrder,- filename: fn,- dtype: value.Type,- offset: pad(value.Offsets[0]),- size: pad(value.Offsets[1]) - pad(value.Offsets[0]),- }-- offset += t.Size()- tensors = append(tensors, t)- }-- return tensors, offset, nil-}--func (m *SafetensorFormat) GetParams(dirpath string) (*Params, error) {- f, err := os.Open(filepath.Join(dirpath, "config.json"))- if err != nil {- return nil, err- }- defer f.Close()-- var params Params-- if err := json.NewDecoder(f).Decode(¶ms); err != nil {- return nil, err- }-- params.ByteOrder = binary.LittleEndian- return ¶ms, nil-}--func (m *SafetensorFormat) GetLayerName(n string) (string, error) {- directMap := map[string]string{- "model.embed_tokens.weight": "token_embd.weight",- "lm_head.weight": "output.weight",- "model.norm.weight": "output_norm.weight",- }-- tMap := map[string]string{- "model.layers.(\\d+).input_layernorm.weight": "blk.$1.attn_norm.weight",- "model.layers.(\\d+).mlp.down_proj.weight": "blk.$1.ffn_down.weight",- "model.layers.(\\d+).mlp.gate_proj.weight": "blk.$1.ffn_gate.weight",- "model.layers.(\\d+).mlp.up_proj.weight": "blk.$1.ffn_up.weight",- "model.layers.(\\d+).post_attention_layernorm.weight": "blk.$1.ffn_norm.weight",- "model.layers.(\\d+).self_attn.k_proj.weight": "blk.$1.attn_k.weight",- "model.layers.(\\d+).self_attn.o_proj.weight": "blk.$1.attn_output.weight",- "model.layers.(\\d+).self_attn.q_proj.weight": "blk.$1.attn_q.weight",- "model.layers.(\\d+).self_attn.v_proj.weight": "blk.$1.attn_v.weight",- "model.layers.(\\d+).block_sparse_moe.gate.weight": "blk.$1.ffn_gate_inp.weight",- "model.layers.(\\d+).block_sparse_moe.experts.(\\d+).w1.weight": "blk.$1.ffn_gate.$2.weight",- "model.layers.(\\d+).block_sparse_moe.experts.(\\d+).w2.weight": "blk.$1.ffn_down.$2.weight",- "model.layers.(\\d+).block_sparse_moe.experts.(\\d+).w3.weight": "blk.$1.ffn_up.$2.weight",- }-- v, ok := directMap[n]- if ok {- return v, nil- }-- // quick hack to rename the layers to gguf format- for k, v := range tMap {- re := regexp.MustCompile(k)- newName := re.ReplaceAllString(n, v)- if newName != n {- return newName, nil- }- }-- return "", fmt.Errorf("couldn't find a layer name for '%s'", n)-}--func (r safetensorWriterTo) WriteTo(w io.Writer) (n int64, err error) {- f, err := os.Open(r.filename)- if err != nil {- return 0, err- }- defer f.Close()-- if _, err = f.Seek(r.offset, io.SeekStart); err != nil {- return 0, err- }-- var f32s []float32- switch r.dtype {- case "F32":- f32s = make([]float32, r.size/4)- if err = binary.Read(f, r.bo, f32s); err != nil {- return 0, err- }- case "F16":- u16s := make([]uint16, r.size/2)- if err = binary.Read(f, r.bo, u16s); err != nil {- return 0, err- }-- for _, b := range u16s {- f32s = append(f32s, float16.Frombits(b).Float32())- }-- case "BF16":- u8s := make([]uint8, r.size)- if err = binary.Read(f, r.bo, u8s); err != nil {- return 0, err- }-- f32s = bfloat16.DecodeFloat32(u8s)- default:- return 0, fmt.Errorf("unknown data type: %s", r.dtype)- }-- if r.repacker != nil {- f32s, err = r.repacker(r.t.Name, f32s, r.t.Shape)- if err != nil {- return 0, err- }- }-- switch r.t.Kind {- case 0:- return 0, binary.Write(w, r.bo, f32s)- case 1:- f16s := make([]uint16, len(f32s))- for i := range f32s {- f16s[i] = float16.Fromfloat32(f32s[i]).Bits()- }-- return 0, binary.Write(w, r.bo, f16s)- default:- return 0, fmt.Errorf("unknown storage type: %d", r.t.Kind)- }-}--func (m *SafetensorFormat) GetModelArch(name, dirPath string, params *Params) (ModelArch, error) {- switch len(params.Architectures) {- case 0:- return nil, fmt.Errorf("No architecture specified to convert")- case 1:- switch params.Architectures[0] {- case "LlamaForCausalLM":- return &LlamaModel{- ModelData{- Name: name,- Path: dirPath,- Params: params,- Format: m,- },- }, nil- case "MistralForCausalLM":- return &MistralModel{- ModelData{- Name: name,- Path: dirPath,- Params: params,- Format: m,- },- }, nil- case "MixtralForCausalLM":- return &MixtralModel{- ModelData{- Name: name,- Path: dirPath,- Params: params,- Format: m,- },- }, nil- case "GemmaForCausalLM":- return &GemmaModel{- ModelData{- Name: name,- Path: dirPath,- Params: params,- Format: m,- },- }, nil- default:- return nil, fmt.Errorf("Models based on '%s' are not yet supported", params.Architectures[0])- }- }-- return nil, fmt.Errorf("Unknown error")-}diff --git a/convert/testdata/Mistral-7B-Instruct-v0.2.json b/convert/testdata/Mistral-7B-Instruct-v0.2.jsonindex 1da4d2ad2..88d447b3a 100644--- a/convert/testdata/Mistral-7B-Instruct-v0.2.json+++ b/convert/testdata/Mistral-7B-Instruct-v0.2.json@@ -4,7 +4,7 @@"general.quantization_version": "2","llama.block_count": "32","llama.context_length": "32768",- "llama.embedding_length": "",+ "llama.embedding_length": "4096","llama.feed_forward_length": "14336","llama.attention.head_count": "32","llama.attention.head_count_kv": "8",diff --git a/convert/testdata/Mixtral-8x7B-Instruct-v0.1.json b/convert/testdata/Mixtral-8x7B-Instruct-v0.1.jsonindex 0967ef424..a15965324 100644--- a/convert/testdata/Mixtral-8x7B-Instruct-v0.1.json+++ b/convert/testdata/Mixtral-8x7B-Instruct-v0.1.json@@ -1 +1,348 @@-{}+{+ "general.architecture": "llama",+ "general.file_type": "1",+ "general.quantization_version": "2",+ "llama.block_count": "32",+ "llama.context_length": "32768",+ "llama.embedding_length": "4096",+ "llama.feed_forward_length": "14336",+ "llama.rope.dimension_count": "128",+ "llama.rope.freq_base": "1e+06",+ "llama.attention.head_count": "32",+ "llama.attention.head_count_kv": "8",+ "llama.attention.layer_norm_rms_epsilon": "1e-05",+ "llama.expert_count": "8",+ "llama.expert_used_count": "2",+ "tokenizer.ggml.model": "llama",+ "tokenizer.ggml.add_bos_token": "true",+ "tokenizer.ggml.add_eos_token": "false",+ "tokenizer.ggml.bos_token_id": "1",+ "tokenizer.ggml.eos_token_id": "2",+ "tokenizer.ggml.unknown_token_id": "0",+ "tokenizer.ggml.scores": "e3d3eea80bb41a1213f2d0aa3e8a38581d1f19323be77dbd779c9c7e3b72e676",+ "tokenizer.ggml.token_type": "6040635e6bd38d98af06698feb75c1802bad35180ee6ae0a503e38c0f60fd71e",+ "tokenizer.ggml.tokens": "604ac4bfbd019e430d7b6cdf18c6c0cd5b967900601f0307f714ec7773aa5ca6",+ "token_embd.weight": "1d1d1d39a867d5a4bfb32792a47247d2638c10c95a6259391d02843583505cc4",+ "blk.0.ffn_gate_exps.weight": "2e5cd43ac3f26c44f071926ff6c3f239ecc52a34bc9a5b5906d3d4c1bf2fbbfa",+ "blk.0.ffn_down_exps.weight": "a4dfc7e7c96e7402eb70279601675b956bb7331da8101e63fe5c0a611b6972e5",+ "blk.0.ffn_up_exps.weight": "2d5d87b378b2319c344ed2c642598b6f7cb6beeb582a8ea51abc9ae690d473c3",+ "blk.0.ffn_gate_inp.weight": "a46aaf5aba7401ce6e41f158242b4879d34901661f3ede85496cbd0ce79d6314",+ "blk.0.attn_norm.weight": "3fe37d913bdd2b65076bcdd6efe64a37b0b03cacbb1b80b9f7089068aa35f38c",+ "blk.0.ffn_norm.weight": "5e14308a3c894734eb204c8f558bdc817e94bbd5b4e9cb4094e91ba388c8f7f2",+ "blk.0.attn_k.weight": "73d943dcac0911e87bd771f4aa1c901e1bfe1aed293af06e1a67812159859f67",+ "blk.0.attn_output.weight": "4c5f754c855e262e8d4c94c6fbbb57af06399dc0e170d7d99a1a17fc9aab9227",+ "blk.0.attn_q.weight": "d6fd7403c873d49c05f6f03208f30d99ad34cb3b71c9990c47334d502a8e4c7b",+ "blk.0.attn_v.weight": "cf17cf64b2d683bd9de6cebaf60e5c264df6fdc38fe719dde9d54c80334f6366",+ "blk.1.ffn_gate_inp.weight": "0d524de81cd915816b4e714bf595ad6946a9130b3de731cd89428b2781230809",+ "blk.1.attn_k.weight": "2ea47f412992b374c70674730fe84700e0c8cce177086ce9b6635e42408964bd",+ "blk.1.attn_output.weight": "b4b2520794d54113e86c8ff678eacfc62e35be4395a594a6c8c22b4383ebcc0c",+ "blk.1.attn_q.weight": "5db930c98c4f91f6eab57eb974c72210b158e366d23d6d2890b2759c053bee33",+ "blk.1.attn_v.weight": "079bdde09668394bf7af9f8bc175017b4f48f0ab64e6dd855a4d7561d1693c0f",+ "blk.1.ffn_gate_exps.weight": "146a62de19f9ab093deb101f9640534ffc3dc40d69f508be12fc0475d01b0c7a",+ "blk.1.ffn_down_exps.weight": "949da94a3c0f375160672a979e85f7def284264b10d48d038238aad5f5ece793",+ "blk.1.ffn_up_exps.weight": "7016a3f467d9e3f2f4b4019579ed86b757469cd367f2b225483305376b4bb3c1",+ "blk.1.attn_norm.weight": "1614d1e6ed537737275eb888666c7bac533f4eefbe73dec92b591045ca9e1afd",+ "blk.1.ffn_norm.weight": "405a455fa7d1ec36894652ceb554bbcb09a07fd6405f42741e66dc4a4665c19c",+ "blk.2.ffn_gate_exps.weight": "90d5003fc7421f44220c0842d43128955e91488f6f785fe570b62d81b719e964",+ "blk.2.ffn_down_exps.weight": "ecdc2b5a8b504ef0a7833acff47d69b0c1fa9c22126de1bb120ff5e48c3d6e2c",+ "blk.2.ffn_up_exps.weight": "2cbd9485a32460d315eb50a2f3b00863fd77245bfe885b7565efac1cdb1f191e",+ "blk.2.ffn_gate_inp.weight": "0d0a17a1a2c7a61f2cca49ecbb479154dc93a870873257bc4f225e7607f2e2c2",+ "blk.2.attn_norm.weight": "b2e4c5a977f87a6f880896bd73596234c9b83622fa0d7add5892501e3155913c",+ "blk.2.ffn_norm.weight": "0ab875b4280afa922376cfc7b9aa3f7071c9432ea1254091ce7de3749df0e8e6",+ "blk.2.attn_k.weight": "bb884af51fb51550acfef54ccf1b58ce8284e587806e6a2f88c8265e1ad05a5e",+ "blk.2.attn_output.weight": "0f03099ba1ef342ea61af9cd71d028123bbd8b1dd7d7fd9b509aef77815427d9",+ "blk.2.attn_q.weight": "8fad0d29eb4c9d24e564774ee3316b9eb7a4c4985e4567111d2c836c830f6cf3",+ "blk.2.attn_v.weight": "fe04c847ff677632401a94e7b6b6fdca60391ab21cb23bd791533115de6303a1",+ "blk.3.ffn_gate_inp.weight": "29e3aaa724590c070e614af8288939603d2641b0ef11e8c0f476bebb2776673c",+ "blk.3.attn_k.weight": "231cc5631def10f7f292d8862d6125ff555164cd70480ac76362149fad204497",+ "blk.3.attn_output.weight": "86467a605c62852e05fda1a7ef43150df2cf715fe59785dbcba09f1c27cfa086",+ "blk.3.attn_q.weight": "901822402453922225c2d6ac79616691d48217635d5ff7338daa971d5ddee210",+ "blk.3.attn_v.weight": "27030784f44375720df2f090933645a31a022d3fb3b14573e5ca0b78f44070c1",+ "blk.3.ffn_gate_exps.weight": "231ba59cc0b988d125d77bf627aa3f04636684870af88f081f3944b48a160d86",+ "blk.3.ffn_down_exps.weight": "530c3ab44ae4d66e8afa4d10c153ba5dfcdfb7321989a988e62e9d12e7234625",+ "blk.3.ffn_up_exps.weight": "b85c2d4d9d11332e702b3c0a6610d4f525f9a93e5d12f5c7c55c592c40755e75",+ "blk.3.attn_norm.weight": "05dbb6d88cfa6b199f9d705ccbda97c0ef13f9ec875c595398a1a42d009a4555",+ "blk.3.ffn_norm.weight": "6880b1c27d46969ce36fac049c05dc8b89e4bb47dc89df357e32df7e18fc512e",+ "blk.4.ffn_gate_exps.weight": "a883b4f225b760c5a2f6605dc5e2167ab85bb398c70bf64ceb539fcbd6128dcd",+ "blk.4.ffn_down_exps.weight": "d291bb656aae77947d4b525e2819bf4112afece53ff31de9dab999af1f65f9c4",+ "blk.4.ffn_up_exps.weight": "38592afb8ba3dcfb26970f906174f7d3fa62da44fa4be4fc6912a19030ea9164",+ "blk.4.ffn_gate_inp.weight": "1596cb74e8fd6c3080b937b06468bb397b0dbb661e6d180a6bcbdc43e8bfd0c6",+ "blk.4.attn_norm.weight": "f90c83c5ff4366281d283384efc941620542b9cfdea160d678dc54a75e33f758",+ "blk.4.ffn_norm.weight": "d28d8c49d1746b7cc085562d1074905fd14023844de823dc4fb22202bb280790",+ "blk.4.attn_k.weight": "792bbf412cc357140fdaba543e547a9b2f7582919e307bbd9a80c7d6d8f5f1f9",+ "blk.4.attn_output.weight": "d98e4a062d2631d9c315f1990d5f6ca9a88e7e0e46387f611ccb0353f876aa12",+ "blk.4.attn_q.weight": "1a11a55a91d9f748a72176ff6b1c174844df406e00d1b66b9aa64dc6ee4bcd1d",+ "blk.4.attn_v.weight": "04cb3c02b12a6313c7ac7044513441083d534fb4c5a3f63bbaa58f7edbd2fadb",+ "blk.5.ffn_gate_inp.weight": "cbd5cdf015d33a2da6703eb74c22fcb97581fb9175435173b6dc4f9e8364320d",+ "blk.5.attn_k.weight": "4fdf3405e4d657403f5647b51233521310ee984b4b81bbcd901cb3e6ab76b7ff",+ "blk.5.attn_output.weight": "4a25662c46979a29600ed77e1907cf81fb16ef30e724c155444e54ccb76af481",+ "blk.5.attn_q.weight": "e2acb30e30b97300039bb20ad0878f05159d5657fa811748a51d5b6fb35d631e",+ "blk.5.attn_v.weight": "306504b6a26aa123c63dbbed3f4ced0ed2ee8fb6a30bf0093539b817539f5ece",+ "blk.5.ffn_gate_exps.weight": "7e34df9b9944dbeea5e8565786d3aa6937314a4b87acd4d0874687877c5a39fd",+ "blk.5.ffn_down_exps.weight": "c4b7a57a42b5ac0a8ae27dcd5cb2646d7a7cc7123126d44a56ab128e85f60b13",+ "blk.5.ffn_up_exps.weight": "09d47593b6dd6c664a9155bff02fc2eb7ac4a70219a88162d05c802a01d3c6ba",+ "blk.5.attn_norm.weight": "58804a036d6ac4c1fe357b8b6a97a5c37cae1c2f06ee0086c041d449c1c6ef6a",+ "blk.5.ffn_norm.weight": "d872dee6789f0826211aa46ca9d0869e3e96bcace9e77d6559a7b6f3e524f3ca",+ "blk.6.ffn_gate_inp.weight": "fb1eae732e974d6c1d020a5b4ef98c5f33016f984701bcea656f999a99daad66",+ "blk.6.attn_k.weight": "55e9c59c5051ab5519b3a7962e1b5fa96a3c0251cb6200dc2f177885ad2de470",+ "blk.6.attn_output.weight": "f3c834a8d0027370350e2b6294d95434d31432e57be6313b013c15a56303d61c",+ "blk.6.attn_q.weight": "efaefe5f11c2140dc7cb532b0832c2a0b363a165cbda21f00fadae77efca377b",+ "blk.6.attn_v.weight": "900bd734d75616d846a90a121c97e081c956a3d1ab012f66dd0bc62c43e1ec3c",+ "blk.6.ffn_gate_exps.weight": "312a99661b1468fcaed2474621116f1681432755e973f3ee79d01912974fd424",+ "blk.6.ffn_down_exps.weight": "ac9cd7db67a2ef0d2b5def86873673d05e48d49d147dd944469dbb8e2d4c46f6",+ "blk.6.ffn_up_exps.weight": "57613e7e09579400a1a09fee4445acfbfe83f2f327fdf317877787d96ada6b84",+ "blk.6.attn_norm.weight": "0e8801e09885c633bc01a9a5b85d4e878d30158a4eb41a937dc5b760ebd044cb",+ "blk.6.ffn_norm.weight": "b8c58062ac93072f878446b0e7f958c737aa47fb769fc3a8f593133d12db2dd1",+ "blk.7.ffn_gate_exps.weight": "1ef611732ff13edfa8d30981ed9dac00c15ceba9fc012ed0b199e9280a849948",+ "blk.7.ffn_down_exps.weight": "856c6811945c7b0fa461ca17811cfa43436b4cdf5326bad23cbc30883486d7cc",+ "blk.7.ffn_up_exps.weight": "6725e3e33994302ee13fa5ec163631ce2dcaa08aadde8fc166c2265d4561c5c5",+ "blk.7.ffn_gate_inp.weight": "36b49d7f80c1003dc392b2c1b9960cd49889dd69e77b26b9e4b13d01f3d0a32a",+ "blk.7.attn_norm.weight": "7a0ec49acc5e20ee71c6f80ca02f4f1e564c485e0ae0621309e7c2eb0c616cf0",+ "blk.7.ffn_norm.weight": "eeae035c39ab6e64bc06a4baa1bf6e50d4c8b8797cb0ad8abd48be86974802c0",+ "blk.7.attn_k.weight": "e8f78c1def01a7a38d2d9bf7becb17755e28fefe4927856f7890fbee52840187",+ "blk.7.attn_output.weight": "5367f05ac3bb49ef8745ba5902e1bdd4442415a3ebff2c7e1a3918d7be6fe948",+ "blk.7.attn_q.weight": "37c95fc5acc55a4f6e5f02cab9be60e4fe54c08b65f98f4455741b4aa542ff4e",+ "blk.7.attn_v.weight": "c89f1343486ba55814233511e94090f7365662a8a4214aa4c278cdadc79196c2",+ "blk.8.ffn_gate_inp.weight": "4e239afe8c7afb8de3a005757c887cf14b1622ca2d224227591cb0e5301f4c17",+ "blk.8.attn_k.weight": "2ad0229f30fdcc1e85ce64e00d8f75902238294844a81d5af43e14ba75c02983",+ "blk.8.attn_output.weight": "2e44a4722acb3b521b81d0b910f8ca2f6c286d874a92ddd02150566454061699",+ "blk.8.attn_q.weight": "1cd2b09cb2f43e08de776b5f7eac197a5a6d4ffdfd52b21baa36319450147bd0",+ "blk.8.attn_v.weight": "5a22c57ebfd33ac500cbcfd321d5b5b1783f8728801db6f3f8bed51c7183e4db",+ "blk.8.ffn_gate_exps.weight": "91063fe56cb4f3ff3b41052bb5046fcf8ef61516a603ee90aab893a9d68c15a7",+ "blk.8.ffn_down_exps.weight": "d4c3abc8f1d1b462f67f70bd8f404b3fcf45dceeaa8527fa120527254c383c90",+ "blk.8.ffn_up_exps.weight": "76a1a1f08ec577716a2e7027b45293e9205751126424f1bebe1de89c78f087d5",+ "blk.8.attn_norm.weight": "f980d774da39eb76c52358afac3e38cb4c81cb323deaabbe5c41822e3f17a98e",+ "blk.8.ffn_norm.weight": "1c937658cf90f1a85db9a5f26e077730fdd4b694607dbeeb825c5fb2bc407e0b",+ "blk.9.ffn_gate_exps.weight": "a2532471ecb7896d5c78e5a34e10cfaf4125265e1595166c8d0d0dfbe2a3187f",+ "blk.9.ffn_down_exps.weight": "b47921a28412d48fee450b8b9d97cee42344a2e69f06d407fd9523d7adf13333",+ "blk.9.ffn_up_exps.weight": "7c461bd1b2a73b439cff6a10d94afa01e8b06f7e6f09d9a6f28e3876aef48bce",+ "blk.9.ffn_gate_inp.weight": "1648dfb08b5c06d7953a5a97ecb764995fae9487fb729a1c867023b2538149d0",+ "blk.9.attn_norm.weight": "8635db0f299882a63b7cfcd1d4259c9e53fab22c31d3d054de36b1001380b31b",+ "blk.9.ffn_norm.weight": "f9309aa323062d174c463613afef9b0a33501b510bfaa58a8e0e866d12ffef3c",+ "blk.9.attn_k.weight": "dfe62030441e947a588512d18d9c6e4ed72c2f71c227d622c095e4263b23dadf",+ "blk.9.attn_output.weight": "1977beb75c6349c50ba7dd3865d7c0a9c5c5ddc854413147b0eec98ac4fda351",+ "blk.9.attn_q.weight": "eb132596719605cd6bd1782487f121994629e115190edd69240b12af66e734f5",+ "blk.9.attn_v.weight": "9e708f15d332d7c5187b0693b1a977eb30a2fa10bf7df48ed9d7537c0aa6ed99",+ "blk.10.ffn_gate_inp.weight": "97503a5d166c1925f9b65c0eed980753d411714d66896f3d0fad5286c7aba702",+ "blk.10.attn_k.weight": "1ebdd222336bd25b48df1b138cdbe09021c4a5562ea7cb78cadd1255d2be3a39",+ "blk.10.attn_output.weight": "5e98faa38e9d514b9057e1c8342c509cbe1083defd518e506f6bad89117d1f5a",+ "blk.10.attn_q.weight": "3323a26c87d936d1dd87c577d0b763459fced726679612c874b3de5fc6d969c5",+ "blk.10.attn_v.weight": "d5fa73cb56aca388e205f44455e4b4f676fdc12ed7fac4542fbb3b41ecea59ad",+ "blk.10.ffn_gate_exps.weight": "225021b53782800906cd13b70be3a4161e8b300b97f984a959ccad6a6e8adcbd",+ "blk.10.ffn_down_exps.weight": "f08eb91526bd22f5fd0402fe925d6141cdbb308a1ced0330858d0c85c71f5ef3",+ "blk.10.ffn_up_exps.weight": "a9f688350c3b53eaada5103b5848bd9a3d7d6b327a70fa16c24bf28ece933eac",+ "blk.10.attn_norm.weight": "5ba426c9dfc79805015ccd76cd1068b0ad3bb7a8453e14bb1d35486f122d8f95",+ "blk.10.ffn_norm.weight": "98891d6acbc3986b2581b7a3af9f5946a392d9188972c6a8b15d4e745a4f2482",+ "blk.11.ffn_gate_inp.weight": "b2365a60566e7dace892e1cb0e62eb73ce387352601723e847052b34874feaa6",+ "blk.11.attn_k.weight": "0efbc1d1430505543ff71532a4fcda821aeac616ef6c1dca40e00d4f2ff70bea",+ "blk.11.attn_output.weight": "3d5bd4d9a41236f30d4293edb9ae27beaa113ffb31b4fbfadff3a4c370dfd3e6",+ "blk.11.attn_q.weight": "aa11e9db14dd9c77951511443077c2a1a78070753d7bd3d9811038473f69e325",+ "blk.11.attn_v.weight": "5adc567f377aa11d1763d35f50e53fb2896a8b03b623ac36acc45efa2486d512",+ "blk.11.ffn_gate_exps.weight": "71d07d982aabfab9eed3c733d49c20f023bf475368fc71db5084d91beadc4b47",+ "blk.11.ffn_down_exps.weight": "9a06e61461e48b3925a9f7d9cca634d048c8b62163d7bc5c43e35899f959319e",+ "blk.11.ffn_up_exps.weight": "bc05494d0dcec61021b3ac0c5bc1bf502736cadf48224e213bc139d562699a89",+ "blk.11.attn_norm.weight": "a5758a10bdd0404ae1470e8e9db903985d4d07f60553c5001a5e7b660d4f7ada",+ "blk.11.ffn_norm.weight": "814ae037563aad3771787316bec4806c95bf6f5991dd6474b4b1e5cc13dc18ee",+ "blk.12.ffn_gate_exps.weight": "3a68b831ba1606fb9ef6dffed4732032447ecef23ea563ff4e79317586c7eb49",+ "blk.12.ffn_down_exps.weight": "268b25e13f4b7beab08686e83705a41b21d15251809ee4784526f78a580da829",+ "blk.12.ffn_up_exps.weight": "9105751a5b5b42ca2614d0456f24f779d2e2ac8cdff0f96842aa7ae2b70f341e",+ "blk.12.ffn_gate_inp.weight": "d0de1558cc1d458c5c504f63ddc59785c323df7330474bb0644c346104b40a3a",+ "blk.12.attn_norm.weight": "859a4c8113678e2e202d10299850e0cfb52eb11ea50bcbf4fe3ff39bdd394154",+ "blk.12.ffn_norm.weight": "7fbf4c459c1760218877e9ee3f5ad49e960956a4369bcfe96c143f04ff9ddf97",+ "blk.12.attn_k.weight": "0a7e254fdf3730a57372b6ff421a613eabaea68cdefd64800857941411318374",+ "blk.12.attn_output.weight": "ceb763fc15d88af149d8fb78e82db2b7dab3aeae584af8cf7611a12356a397e5",+ "blk.12.attn_q.weight": "a43402d23c46cb2d3cb3c2a98c81b19d10026b7e6742370fed6b2880b6e049b5",+ "blk.12.attn_v.weight": "3bc24f2c0480ce91ef72993ee8f1cf962f7359e12183424583ffa1246bf3db52",+ "blk.13.ffn_gate_inp.weight": "a6d68c82bfe66d8bab68f980f5f18268a9e2c0cd6b8832ed39010e0de198ae05",+ "blk.13.attn_k.weight": "0166c39546b37dc2e01b2b396ba43e183f797dd04eaa51a6d103d8b58ee4bace",+ "blk.13.attn_output.weight": "2ce5eb198deab9557475a58b69b11e9874b547e05c23f223c6e42fa35ddca069",+ "blk.13.attn_q.weight": "745c1bbdf434284a7fae98f45e821c076dd9c2a2467dba6a9d8cf0041e419dbc",+ "blk.13.attn_v.weight": "9ece68d5ac64d1421ea7aa32e1cff9cc1fecf5175f4c4da858dd31d8633e3337",+ "blk.13.ffn_gate_exps.weight": "ccfdcb4670b131689de12d396a010b5ea737795cf5c15a14a304d720b3c7c899",+ "blk.13.ffn_down_exps.weight": "8b8fb328664764f1aaa5cbdec336d5654e981e965a02ef622bde5f07ea1c164d",+ "blk.13.ffn_up_exps.weight": "d2ace0236c2fb3365fdc85499d676a7f65813c48e5085348b1df1799922766ec",+ "blk.13.attn_norm.weight": "1ed29d7d89ce52d7cb4d57e895ff7115430466e917136c049c385c030ed44e9c",+ "blk.13.ffn_norm.weight": "a194fc542597a4dcfdfaec5e3cba2a2b2b21b21edfc87c39c0d7f7651355bc4d",+ "blk.14.ffn_gate_exps.weight": "a625e3574e5e740e7f8e2f9c40390f2f382c720aab5b10534e298002dd8d1fb9",+ "blk.14.ffn_down_exps.weight": "bc366f015b83c865946afd74c8a884943e0ea2c671314a0b7bb72f21a44d2f78",+ "blk.14.ffn_up_exps.weight": "ee3199bf2086de77b49f57f487676be8ee70e102a2fb5a5ef8ddbbc28a9eff41",+ "blk.14.ffn_gate_inp.weight": "2b437870c850fa2e2044d032bb02908af634356e37466fdae260b933e48ee8b4",+ "blk.14.attn_norm.weight": "cd8344d193a1cbd42bd898e17f4bcb1ca0b2918420fbdafa9249a6f2b7f4ae06",+ "blk.14.ffn_norm.weight": "70eec40374e558fed5b07257283cf36342b6b0129285a00007deb59c32c9f7c8",+ "blk.14.attn_k.weight": "4053bdb507e0543d724b632570bac86b31707696d90a0db44c49b2a082e0d599",+ "blk.14.attn_output.weight": "0182632cb0e06a07241b8293d25d109fbc1862e1e337d435f908e8681e2eb1ab",+ "blk.14.attn_q.weight": "ffc7794a4c1b6f793c842dba969435330a7a80b9212e457b4b2ac33e68b41241",+ "blk.14.attn_v.weight": "6411805292d528e61bbaad8f9aab9dd073529a17946c057fb06864fad9cf3211",+ "blk.15.ffn_gate_inp.weight": "77d0744567c76e6abb67f81ba9c715b2b544841186d5b948309571eff213bafb",+ "blk.15.attn_k.weight": "1f7957954ea4c6521c257b35a360e868ffa02bdb3de91f146d5e06bb4a545c98",+ "blk.15.attn_output.weight": "d7809d36bd8d3342240c46fd87bcc7f9821a222f48d9a95e45ae50460265d3cf",+ "blk.15.attn_q.weight": "25f509313ae4d8401b871904059f472a26f5714e7c791c725de77a1a522c976e",+ "blk.15.attn_v.weight": "96fedf5a591fc0f020e6de10fd72ff12b3ef9cf70cd21dabaa0d3e7b06f54e73",+ "blk.15.ffn_gate_exps.weight": "8f950d976b2fd9a3d213b84123cf114c1377efde9352767fb2ddee89e177c8ef",+ "blk.15.ffn_down_exps.weight": "6fd09d1557bb94b06efbd4f6a1ca4be532a202ba290e9315bc8da3d12a5c4c4a",+ "blk.15.ffn_up_exps.weight": "cbeb59ae7b0266a928dc7e3a6e70a9330b92f9ee1b17ee1ed91022108204a33c",+ "blk.15.attn_norm.weight": "2005330911ac2edc7b6d27aca021c67d30d16eb632e49b1a13f30fdb2717aed0",+ "blk.15.ffn_norm.weight": "0e9198f3b548eb78acc8961f2b3350d238d26cec110933ba753a8cf0035c501c",+ "blk.16.ffn_gate_inp.weight": "a41d1f99d739c8b150c3945b6949763988d0c6a4c5a2b5855592ca1a48ed23d5",+ "blk.16.attn_k.weight": "b624e2ec88c2d3047f60530fb87e72cb4a5e655a9663f6f3e9b09e5ad32cddaa",+ "blk.16.attn_output.weight": "687759ea75e45108526ffc1573d6fdf084728079bfc2dc89b9979e76280f43c4",+ "blk.16.attn_q.weight": "beff3a45c7e9ec82ffc6d3c701126be28654d10aabd747d03441210491fd31b6",+ "blk.16.attn_v.weight": "43a349b13f0b9d040cacecd942bcb168c030fef8c75c987d59a4fce6c14e855b",+ "blk.16.ffn_gate_exps.weight": "793406d6c13d727c82bb7b692ca98d65ca975baee69fc57be5378d77c5a19b62",+ "blk.16.ffn_down_exps.weight": "9bad3dd150d0230404b7f886ac7ff8803225757e813f195cdb26bad245243b4d",+ "blk.16.ffn_up_exps.weight": "7449d663023fea3496475bf0a9c1de7272ad0ce9adcb3265e8e424badaa674dc",+ "blk.16.attn_norm.weight": "a424ce34c195a401df1ce37ac4f2794e8a6720b1ee8acb21428e2b68c65e0125",+ "blk.16.ffn_norm.weight": "405a68bb8e16e1064df2de55ca3cd9ceddda1d9fc0af007a9bd7cad4b2676248",+ "blk.17.ffn_gate_exps.weight": "97c6e5321491ca5dc039ee88da0eb0e78f347372785411809af84b3298cb19dd",+ "blk.17.ffn_down_exps.weight": "1617ac19788a1be19bac69277408761e6bdf5719d63a8c7fea14d41cc27641b5",+ "blk.17.ffn_up_exps.weight": "4ead1c365f112581c10610ea3f63d2a1474311d2503d2060fed4b458ef337f5d",+ "blk.17.ffn_gate_inp.weight": "ed4b3393f2523f2b5e0fc7680a1caa2842e605728a529b5af68a7fa8d7abf940",+ "blk.17.attn_norm.weight": "beac17ef86a7fb2b5840cc72f7a95a5e3d6bd24e7fa698e0b0ebb9bdac45c561",+ "blk.17.ffn_norm.weight": "81cb58ec6d6dc02a0b4ede10adc336dc865fa76f982d4eab0e4a37b40f5b0fac",+ "blk.17.attn_k.weight": "eab569e5ea8c8b05e5a6a209fba031129453c2e28181eee3e736b3b04b36bbec",+ "blk.17.attn_output.weight": "f85b70f01438ce8fe5d10599b113f30bf18dee2bbae0657d3eba295870001db3",+ "blk.17.attn_q.weight": "887ceebfbf6a2b94b43d2df4439ac3a5bbc29311d4b28addc04d525546032047",+ "blk.17.attn_v.weight": "2df9414d65014c06a93da22ba3a668be7b83e2e8008e98d7771f7dfebed98298",+ "blk.18.ffn_gate_inp.weight": "9b07741a0950fc667e5fd25937e33bc22e1f764f80eb4ff3119f005327ae0f6e",+ "blk.18.attn_k.weight": "8649598dbb63938744c39bcda5ce8c31773e29c573be8d4d2c114f5030f8d3e8",+ "blk.18.attn_output.weight": "f8e391adb92622298ca834d5d1eda48b69c3b1c51c5a584ef6c54a725c298d75",+ "blk.18.attn_q.weight": "84bf8708a2eed618f48f69c178ed7dd11fa4c468102376e72e910ebd037d131f",+ "blk.18.attn_v.weight": "31db3cd773f09548c2c1b1eac2718e46364a7810970fe9c433fad9d8de5397eb",+ "blk.18.ffn_gate_exps.weight": "be2a2ba378002f1b61f86c273a69eede9b93786d5ce96b4fee1861f730dca4c4",+ "blk.18.ffn_down_exps.weight": "d35196159e37705db50a5343e3989f7335477f1a4add67ef42ad64a638cd07ae",+ "blk.18.ffn_up_exps.weight": "c6ceedd86e97913a6dcadc838e7abb762d629fb8dd55f15cf02fd9bd66d2ba78",+ "blk.18.attn_norm.weight": "41f0b1ad83d6e3cb9fbe0d27878c2e7ad4a351b9f554a6bc9117c01745cdf6e5",+ "blk.18.ffn_norm.weight": "96646204bd0d82f25dc77faba4dbd86b1332e449313e6684e00122da8be99057",+ "blk.19.ffn_gate_exps.weight": "c6eb7f61e7938bda0492dbc05e51e8f631c99224fe18e99861fc4fc53ba9e9ff",+ "blk.19.ffn_down_exps.weight": "4384803da3a3a3d44120d7dd192fe2c9bbd9a1a0cb492dbec1fdd7565230f1e8",+ "blk.19.ffn_up_exps.weight": "22d73de2fbb8bb0f1bd2caf17fad8a355c47d914143f7f6e6d0128f66f074a60",+ "blk.19.ffn_gate_inp.weight": "9a0cc4a2301a5634022fbce41189021bf0d1a961792d2d9330fd35556d18e5bd",+ "blk.19.attn_norm.weight": "c5cc56ec5df9a1f7d5ad71fbda49f1433132e58895d45cb44c73420bd61ebd6b",+ "blk.19.ffn_norm.weight": "77e17de741742ef2482fc7872fd423c8e3c1454dc4d2be89ee939084b6d78bc0",+ "blk.19.attn_k.weight": "a92ea36ce2e3569656306aeefb835ccd5d1b03b33a86e0d3d030644cc923b813",+ "blk.19.attn_output.weight": "5e2a912b37855f84ea964907a1a86d609cbdd79efa0c93c3e8e2fc07caf7c226",+ "blk.19.attn_q.weight": "4ef3a5913292ac3c1a6fd3e9e53d011021f2b41d0276cf849706d1ca925cf7a7",+ "blk.19.attn_v.weight": "42981b75b68ae852cee638b5433605c147da4392aaa6d7a06e756115b0171f39",+ "blk.20.ffn_gate_inp.weight": "71381b9879a7c80b9f7b475abc0aa31b8cd71ccc00856ebe89764a2acb9df2dc",+ "blk.20.attn_k.weight": "1928b7ebc054eb3967929ed6fb446314d5352f4aaf8b475ce55c6345019f2ea4",+ "blk.20.attn_output.weight": "6071ecd9ca91af0d2ba93fef4a1a56f3b243dd70f862a21a2d164d56f386043b",+ "blk.20.attn_q.weight": "002e95042a40f36ceed5829e3d0c8072e5f5e4ee86a089e2902b2348fed24dd5",+ "blk.20.attn_v.weight": "42f509cdb1c0e298f89f896e349be86952c5168e49b3f83bb17badbcb7596d57",+ "blk.20.ffn_gate_exps.weight": "a684a3ffe4b0a57c819a5fa9cb3521de223f392732927271e97ce925b6e33765",+ "blk.20.ffn_down_exps.weight": "e3081a7bc7ba750d8a4886bc8ca4f231b55db4ca082b54b4106c7531964725cb",+ "blk.20.ffn_up_exps.weight": "fad0fd5eca36ab154788da28be8ec25bb5d6db06c9d133db89e96df358a2f6a2",+ "blk.20.attn_norm.weight": "c3e3f2429715ae95e884ef1246b0b461b23c5cc0ed08beecf70a14cddd184820",+ "blk.20.ffn_norm.weight": "ff31f609dda65ca496b0584fabea6550e42edd05ebf229812aa6b7bb5ede15e6",+ "blk.21.ffn_gate_exps.weight": "366f09ef0ecfb86808eb3296cc9abdb957951d27f6533c03f1422b54061da660",+ "blk.21.ffn_down_exps.weight": "3fc495947d27fcca7fc0893c8a96e5d48ba27b2c8c58f8fcfb8dcfcd5539741c",+ "blk.21.ffn_up_exps.weight": "6713ed51410bcc8283cbb001c4ad784098f25701e8021f4fa4f411e186859c4a",+ "blk.21.ffn_gate_inp.weight": "6d4c92c01ec801647134d907bf1108878156df266a6107abc10526332b328b93",+ "blk.21.attn_norm.weight": "27605719ae2df24f4f2e85a730927cab20367631612cb501631f6bbf38eb1209",+ "blk.21.ffn_norm.weight": "ca80ee8177db185b15a4a378c1cb6f7143c76546a7f1726bda23f329323d4ffa",+ "blk.21.attn_k.weight": "9e49f743d4a5bda9b4bd9c40c2ca37cdae5aec7e54cb193897ac8b4945ada14d",+ "blk.21.attn_output.weight": "ab923540879753feaed152f5950f69cdd83d8f2413ca873f5f038b63ab0aea12",+ "blk.21.attn_q.weight": "62617fc3f1c9d2aa672a4d91a121c7a91b92d145b65e75f0b06b4bb7c825dc36",+ "blk.21.attn_v.weight": "15f8b2e72f8e8e992f2f6b3e93238a9d7be7bd6136f91c9d04b4b4cd0cd60369",+ "blk.22.ffn_gate_inp.weight": "3ddb1773d9257b68add7a2a4e94dad25ed926803e02707863dd742ab9b2dc179",+ "blk.22.attn_k.weight": "680e45a9e8d5feddee5266e119dc053bf80718fa9af1cf6803e6f493b265f1eb",+ "blk.22.attn_output.weight": "0d5fae3402fb2c5aa3a860010e3973fc8e3168d1015f7a76b7b2964681693206",+ "blk.22.attn_q.weight": "eee7e3d426ab533bd18d62c9aa142eedbde394bed07db58313e0fccc82a23237",+ "blk.22.attn_v.weight": "26b5be1fe3c2b6824c5a648a3e4bdf17691904526fca158fbc3ebb627b67e2f4",+ "blk.22.ffn_gate_exps.weight": "32ab7a7735313d60f6a75229b1aeee940b6aee176c9648536bf5921b0dc2929a",+ "blk.22.ffn_down_exps.weight": "67590808f6a67777d3eb7976c31fe616d388b98fecbb12253b72d1241d70753f",+ "blk.22.ffn_up_exps.weight": "fc245c0183e6d90829ff5e71a4ec93e4860b3d4c1a17b9dda2fb64f5f5c9ed32",+ "blk.22.attn_norm.weight": "128e99d206d4d6724758ec97468af767fa0aea592149c324b731659c1e74a1a8",+ "blk.22.ffn_norm.weight": "e45f498033f0cffa15da0eff2c47b4472e43fcf8921729fc4eeb2e3a6b3c78e2",+ "blk.23.ffn_gate_inp.weight": "d63e686f5325fbc89fa242c2c52a3b8ff54f867dca914c9ae6eea13e9d6f46e5",+ "blk.23.attn_k.weight": "f71f5a577f46ea12b1818f3a5ff4b85ddc45f9a2afb0fa2e041d71a3e31c6779",+ "blk.23.attn_output.weight": "92b13563c1e0eac0d748fb67b235dfd7a64c8f16e2dafb316885744582e23b4b",+ "blk.23.attn_q.weight": "2f9b9c35dc4f912f3f51c06e2d68f417b51a0de0a84aac530a64f9d3d7b0a2dd",+ "blk.23.attn_v.weight": "268e40813806e74a5c364b19556d087bf8374e76e7b6fcf55c381eb7da13ccd1",+ "blk.23.ffn_gate_exps.weight": "12f857e7a7ce228afac34d99b602c8d6fe96984f2a21118f459a58cb767ee65e",+ "blk.23.ffn_down_exps.weight": "cdb082c16599c3bb36a28066dcc122d9529b54fa91b6cf0153437ec960a5e16d",+ "blk.23.ffn_up_exps.weight": "f4b99f6f44d7b8b5a305894e88633bf5938fc1f6303a2b2092399da9c8b64d7c",+ "blk.23.attn_norm.weight": "a691392210383915916b4d3886d5e4d56e7855e27e37e414fbd73bf66b3712e6",+ "blk.23.ffn_norm.weight": "0c3dc72f667e5ae19b69bfa9f2bd2a01a57681f89ef9527bad4eb0d8c7b70da8",+ "blk.24.ffn_gate_exps.weight": "86baca2a3157994df7fd8ced5e08436d5c1810dc29c0715637c36de723e0e7d1",+ "blk.24.ffn_down_exps.weight": "ac5d559562b35c34993e34b071f66d15c65be5907797078c2d2a49aba54e3192",+ "blk.24.ffn_up_exps.weight": "fce0a099cf09777f44fbab3606ceb75f7fae6f0b80725f9e871654b8cdf9262a",+ "blk.24.ffn_gate_inp.weight": "e7c6800c0cfc56b565b2d35ad6f1dbfdb70dd0b05b338bc8da2286ffc3678d79",+ "blk.24.attn_norm.weight": "dc6cc18ec52d102d015153c4a1132f9d7a504e29cbdec81c5edbf3b9e65815e1",+ "blk.24.ffn_norm.weight": "480d5a1397af5e0e657f1e67d20ec0cdef5724e71246a326843321b87ffabd33",+ "blk.24.attn_k.weight": "338c0597954a9b95a782545b2fe36469553e73f86ae2d2b5697767b28e1c7daa",+ "blk.24.attn_output.weight": "a77d23b79933c67e52f1eef7f83a3dff4f767ce0bbcc39572f8cec4acd457643",+ "blk.24.attn_q.weight": "45c9478593002be1998e96e70668aafa2dd3972380fbc1df12fb05c24ba959e0",+ "blk.24.attn_v.weight": "515729420885408a6a9614bc27cda393ed907521318d14d21335d39a3eff0b61",+ "blk.25.ffn_gate_inp.weight": "aae4ac40e9ab3925241f9d784b54b38851d9bc999a6c3bc03fc3f17c9b28a67c",+ "blk.25.attn_k.weight": "4ab4808d02396c35b00b426f536015673b71c17ae6cd55bbc2e6bfe7a4c59d0c",+ "blk.25.attn_output.weight": "1990bb982b77e0c947cd1a8ef0b36227ee1259e6dbbc2829e5c136edf88675eb",+ "blk.25.attn_q.weight": "a1490f3048e8c0ec8784f8550c43adf5cc8d0f2f90131c934713fe4b1b015bd7",+ "blk.25.attn_v.weight": "f15e53c6d45b3b6f58808fa968425d65e0b26b7f9b268127a77abb1227c67431",+ "blk.25.ffn_gate_exps.weight": "656662447ff54f56ee80f78a1b9483f7efdc40f7375d0cd8a9c72ccf21f77e7b",+ "blk.25.ffn_down_exps.weight": "db06f101bccbaef19cced0f6c185166e18202465f4a42cddfd535fbe5cbabb4a",+ "blk.25.ffn_up_exps.weight": "584a7b02456f27fe1d8d3c7ccd21d426b6ea887795a3ed77f704596a1e3841d7",+ "blk.25.attn_norm.weight": "8f0f3597982930fd237e9d609776c64f2b909a455b21678f83a7ebd4bbb83e64",+ "blk.25.ffn_norm.weight": "3e7079c32582afba0c55e032f254adc18d2997705eec860185e9a6dd3d82f07e",+ "blk.26.ffn_gate_exps.weight": "e70341691b583b86489812b29b77aa41eb658b1865733d6118da54c66e3bfcc6",+ "blk.26.ffn_down_exps.weight": "5c1b812d11dfb064af816ced5ab6463bf9722eefdfc341b8a93705d5038fd781",+ "blk.26.ffn_up_exps.weight": "e18118362ae54ef7432781c83884f9fb230a9d934e342aabeda8822ea5f71fb6",+ "blk.26.ffn_gate_inp.weight": "cd1c5f6710166b9567c6b74c97b2348b191c60aa860958c6bc264ab095261dff",+ "blk.26.attn_norm.weight": "71d087531af2520bda2e676c489e8529cef5db8aeea1eec0a937a8b4f2fa2e54",+ "blk.26.ffn_norm.weight": "7f704e936fda28eb5c2cc339f0f6a5f78170b5aa43c01265b21668870d819c82",+ "blk.26.attn_k.weight": "1cc62a0ce0ae251275d898c52c4a9fba5995fca10955d2011d10dd1a59e1afb8",+ "blk.26.attn_output.weight": "636e881b1505f9cef656a4be98bec6a4765321d51f9bf1dac8933397cf44b765",+ "blk.26.attn_q.weight": "89a3c4d202d7d6adebb9e0c1bcfd8b775f6456386f1be25e86e43acc949c1e16",+ "blk.26.attn_v.weight": "ff2cc963b597cdf1a21703f3e7022af3bb4c65a34a19e19d9309a7c5e198b5bd",+ "blk.27.ffn_gate_inp.weight": "6150139498fefe380bb99d11e72028da47a15ecb73dfc5b2774f726f4bed8f9e",+ "blk.27.attn_k.weight": "f286eb9e5c56c7b801a497aedc40158c2a27877d7f9fb59b3fc67834798902d2",+ "blk.27.attn_output.weight": "5dc3d3a05f9f7729509147fd09c16fb53f85f520cdab5cb69abf4bae3fd460c7",+ "blk.27.attn_q.weight": "8462e40f86b24251960d6f35a9ea99b8793a01937faf1aec2859f2e5395dbb61",+ "blk.27.attn_v.weight": "bac1a99e38e25953f8315f7212eb9777dc216cadb09b959977885ae62724ceca",+ "blk.27.ffn_gate_exps.weight": "6a15eca7f0f6ecfd93db2e55c63875348ec4a78c4ff643ec46df9e958c0101e4",+ "blk.27.ffn_down_exps.weight": "2e1c91247c4359e2073a8e5f26fd7f6426da7be3ed5bc65dcfff701f0a5022b2",+ "blk.27.ffn_up_exps.weight": "65d6f5c553c9332085eae4aeadf25090b5d7768212ea7b08ed698102c21b29a1",+ "blk.27.attn_norm.weight": "7fab8ae63ec8e91ce625cd130ab96d8427dad3a7413bb21b25ec5f408c5b9f5a",+ "blk.27.ffn_norm.weight": "532720546b0fdcd423a02ca6e3e9d8aacb84b1b3e8269968f88a47fe2a69bab4",+ "blk.28.ffn_gate_inp.weight": "a305ea58d98962d9dcf0c53ad2389b7acc8936fb35a0e3fc9410e7767cd49dea",+ "blk.28.attn_k.weight": "8315e8a2e4f78dfdf36d4fc18fffc74bc95fe42c3ae4f9af2b6c874612c0f71b",+ "blk.28.attn_output.weight": "9b5fdedd32d39ef46a22cca7cd5355d7b93bd07ea305f466a8aad6ca5a4f3778",+ "blk.28.attn_q.weight": "4e8fb96997c30e231c437130f410d7c91d541a816f6c568b5f3bfdb4b8dece74",+ "blk.28.attn_v.weight": "1fec739cf3bd7b4913f72ca358d4cf31391c304de44ac0ae31ecb825beaa7cfd",+ "blk.28.ffn_gate_exps.weight": "9f259789d535e09268266b9a8020f32d6a6779966c909d91d3a10574f06238a2",+ "blk.28.ffn_down_exps.weight": "516d3f8abaedb01b9916a4b67d4672159769138ef2850158bc1b32c41e31f0e8",+ "blk.28.ffn_up_exps.weight": "f2f1d88d2c31ed588806fb5ad981d68f5134d7284c4fc022fd018de2eef437fc",+ "blk.28.attn_norm.weight": "960fd005598deadaebd969996f4367a9dbfad90539a863674fe95730935acc64",+ "blk.28.ffn_norm.weight": "e1993b37ced93d4049e9af2c47b0d9207d8f7e6f2cc3a52f57bef30bc806d805",+ "blk.29.ffn_gate_exps.weight": "58927146338f443513337476b3cd30e6341742f096c2beb5890d400f10121298",+ "blk.29.ffn_down_exps.weight": "03a3386e4f0b75a28c5608e23b2de8f0de25f21954e4aa7fc343431bde9db07e",+ "blk.29.ffn_up_exps.weight": "6916b7490a7ae7b04a5d81cc1e7ac9b20c483434f3b186b12d87fe176bf1567b",+ "blk.29.ffn_gate_inp.weight": "98e710e467a3d567abe4ce29d78b8e8dc033148762290c0c5e1ae4d78efd8c78",+ "blk.29.attn_norm.weight": "4e64cb307d37be20d55f38c94faf7e451d11df5e60df347906cbaf9c5441be71",+ "blk.29.ffn_norm.weight": "696c23a52f742679bd44440d687a4c44b4302d57f1e9dc5610d23374336187e7",+ "blk.29.attn_k.weight": "e85253652fd6120c623634ba66b725bf7cd491318b54ccdad2c7df8851d64c0a",+ "blk.29.attn_output.weight": "4f650a71efb150d1f24cd4d114d4187bf570ac424da3b92ea6455abdf1aea705",+ "blk.29.attn_q.weight": "69fa7da901026ebcbbbc848455b425458b7e3295007d7fc093acf4b38e2166ea",+ "blk.29.attn_v.weight": "17e2e7590b317b21f106de546aafd955579703d1e95d6aea044ee72ec3a514c9",+ "blk.30.ffn_gate_inp.weight": "3a03284b4aa60d59d4a2ec86253469b61fc656372afca427cb77a5332fbcc62c",+ "blk.30.attn_k.weight": "d518cfd0db9708e769eb1399e87ee49357dc54d5afdbac3d4c0ca46c64e789eb",+ "blk.30.attn_output.weight": "9b44378714d784c5ef9ab604359091baca4e0ec222afa139b7f840eaefb371fd",+ "blk.30.attn_q.weight": "cbb95365bbfbcad0c9cd99b4eebb5a5d32de68ce08e4063b5ec3e792b7548044",+ "blk.30.attn_v.weight": "e7985c04fe1740e35a9598f43b67b0922b4fc2d00b68a92a9f917b82c3248de1",+ "blk.30.ffn_gate_exps.weight": "8ac4bbd07935d98f895ba94dc174e5ad5046c3c222b53729d60f987c05e7eb70",+ "blk.30.ffn_down_exps.weight": "dd672cc71e82abf05064a18121b8e55fe1a4f19bc1d7cb9a142f4add54bc336e",+ "blk.30.ffn_up_exps.weight": "12282f664a2a12aa25e2deac58946108715ebb978bafed5274cef24569107646",+ "blk.30.attn_norm.weight": "1a33458fee054c6c9c896a4bb0a4e1fbfa0293b2408c7dd2b81d692e966e7273",+ "blk.30.ffn_norm.weight": "311e33b68051f507f1478ed8f2693fddb846170ddb7285a91be43f795c2ce31e",+ "blk.31.ffn_gate_exps.weight": "8af43d9867a51cd8392fb48b981b0ceee0ae979c491c07d711b3b56b5162c786",+ "blk.31.ffn_down_exps.weight": "5579cb7758c1600b19d1f540deffe081b575962e37437b3b2efb2fb0a2924e40",+ "blk.31.ffn_up_exps.weight": "f2e7c005276b3a001fb40753f027fa10b4d5a346f43cf4b4bbdeec6e74e1cf6a",+ "blk.31.ffn_gate_inp.weight": "89885dc0e30b6b16a90c0331d7fa3174671e941364e8102d934f02132237e61b",+ "blk.31.attn_norm.weight": "99e4e9bf86a9edf8c404153a7e8a82324ba79da462622196e2faba161bd95172",+ "blk.31.ffn_norm.weight": "55335997cf6de781bf332b943de96ff4646966b05d9fee86b76ea897e27b6ca7",+ "blk.31.attn_k.weight": "cee570762b78da6316b637892cc4b080e40f57af5551ffb1866b9a8e80e96628",+ "blk.31.attn_output.weight": "fa321ff55ec7819ead7b819fd45215262f39744569765ba2113c989c03588802",+ "blk.31.attn_q.weight": "9e2c409b878f8a2a1436874abf428fceb1c534b21f9ad4dd6f532b8a469007f0",+ "blk.31.attn_v.weight": "a845d0be68ba537b4a775bfba4d897faf7c82a811a2612b0b7420cc4f3574cb8",+ "output.weight": "16101cbb74b54cda9ebc07ca3c762e3263a56efb3cc011156184b95807d7cf13",+ "output_norm.weight": "d7aa61585baedd60157aafe157930785742c55989c288573566a971b02423564"+}diff --git a/convert/tokenizer.go b/convert/tokenizer.goindex fd6df5f53..baee04aa7 100644--- a/convert/tokenizer.go+++ b/convert/tokenizer.go@@ -3,19 +3,148 @@ package convertimport ("cmp""crypto/sha256"+ "encoding/hex""encoding/json"+ "errors""fmt""log/slog""os"+ "path/filepath""slices"+)- "golang.org/x/exp/maps"+const (+ _ int32 = iota+ tokenTypeNormal+ tokenTypeUnknown+ tokenTypeControl+ tokenTypeUserDefined+ tokenTypeUnused+ tokenTypeByte)type Tokenizer struct {- Version string `json:"version"`- AddedTokens []Token `json:"added_tokens"`- Model TokenizerModel `json:"model"`+ *Vocabulary+ SpecialVocabulary []*SpecialVocabulary+ Merges []string++ Pre string+ Template string+}++func parseTokenizer(d string, specialTypes []string) (*Tokenizer, error) {+ v, err := parseVocabulary(d)+ if err != nil {+ return nil, err+ }++ t := &Tokenizer{+ Vocabulary: v,+ Pre: "default",+ }++ addedTokens := make(map[string]token)+ if f, err := os.Open(filepath.Join(d, "tokenizer.json")); errors.Is(err, os.ErrNotExist) {+ } else if err != nil {+ return nil, err+ } else {+ defer f.Close()++ var tt tokenizer+ if err := json.NewDecoder(f).Decode(&tt); err != nil {+ return nil, err+ }++ for _, t := range tt.AddedTokens {+ addedTokens[t.Content] = t+ }++ t.Merges = tt.Model.Merges++ sha256sum := sha256.New()+ for _, pt := range tt.PreTokenizer.PreTokenizers {+ switch pt.Type {+ case "Split":+ if pt.Pattern.Regex != "" {+ sha256sum.Write([]byte(pt.Pattern.Regex))+ }+ }+ }++ switch digest := hex.EncodeToString(sha256sum.Sum(nil)); digest {+ case "d98f9631be1e9607a9848c26c1f9eac1aa9fc21ac6ba82a2fc0741af9780a48f":+ t.Pre = "llama-bpe"+ case "03df5c5863ad70781dcfdef491ead25140f895fe8010964be0daefe27be32b02":+ t.Pre = "deepseek-llm"+ case "21cde974d587f0d54dc8d56b183cc1e6239600172035c68fbd6d4b9f8da0576e":+ t.Pre = "deepseek-coder"+ case "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855":+ // noop, empty pretokenizer+ default:+ slog.Warn("unknown pretokenizer, using default", "digest", digest)+ }+ }++ if f, err := os.Open(filepath.Join(d, "tokenizer_config.json")); errors.Is(err, os.ErrNotExist) {+ } else if err != nil {+ return nil, err+ } else {+ defer f.Close()++ var p map[string]json.RawMessage+ if err := json.NewDecoder(f).Decode(&p); err != nil {+ return nil, err+ }++ if template, ok := p["chat_template"]; ok {+ if err := json.Unmarshal(template, &t.Template); err != nil {+ return nil, err+ }+ }++ for _, st := range specialTypes {+ sv := SpecialVocabulary{Type: st}+ if bts, ok := p[fmt.Sprintf("add_%s_token", st)]; ok {+ if err := json.Unmarshal(bts, &sv.AddToken); err != nil {+ return nil, err+ }+ }++ if bts, ok := p[fmt.Sprintf("%s_token", st)]; ok {+ var content string+ if err := json.Unmarshal(bts, &content); err != nil {+ var mm map[string]any+ if err := json.Unmarshal(bts, &mm); err != nil {+ continue+ }++ content, ok = mm["content"].(string)+ if !ok {+ continue+ }+ }++ sv.Content = content+ }++ if id, ok := addedTokens[sv.Content]; ok {+ sv.ID = id.ID+ t.SpecialVocabulary = append(t.SpecialVocabulary, &sv)+ }+ }+ }++ return t, nil+}++type tokenizer struct {+ Version string `json:"version"`+ AddedTokens []token `json:"added_tokens"`+ Model struct {+ Type string `json:"type"`+ Vocab map[string]int `json:"vocab"`+ Merges []string `json:"merges"`+ } `json:"model"`PreTokenizer struct {PreTokenizers []struct {@@ -27,80 +156,106 @@ type Tokenizer struct {} `json:"pre_tokenizer"`}-type TokenizerModel struct {- Type string `json:"type"`- Vocab map[string]int `json:"vocab"`- Merges []string `json:"merges"`- Tokens []Token-}--type Token struct {+type token struct {ID int `json:"id"`Content string `json:"content"`Special bool `json:"special"`UserDefined bool}-func (t *Token) Type() int32 {- switch {- case t.Special:- return tokenTypeControl- case t.UserDefined:- return tokenTypeUserDefined- default:- return tokenTypeNormal- }+type Vocabulary struct {+ Model string+ Tokens []string+ Scores []float32+ Types []int32}-func (t *Tokenizer) maxID() int {- return max(- slices.Max(maps.Values(t.Model.Vocab)),- slices.MaxFunc(t.AddedTokens, func(a, b Token) int {- return cmp.Compare(a.ID, b.ID)- }).ID,- )-}--func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, err error) {- f, err := os.Open(dirpath)+func parseVocabularyFromTokenizer(p string) (*Vocabulary, error) {+ f, err := os.Open(filepath.Join(p, "tokenizer.json"))if err != nil {- panic(err)+ return nil, err}defer f.Close()- var t Tokenizer+ var t tokenizerif err := json.NewDecoder(f).Decode(&t); err != nil {- return "", nil, nil, err+ return nil, err}- tokens = make([]Token, t.maxID()+1)+ var tokens []tokenfor k, v := range t.Model.Vocab {- tokens[v] = Token{ID: v, Content: k, Special: false, UserDefined: false}+ tokens = append(tokens, token{+ ID: v,+ Content: k,+ })}- for _, v := range t.AddedTokens {- v.UserDefined = true- tokens[v.ID] = v+ for _, t := range t.AddedTokens {+ t.UserDefined = true+ tokens = append(tokens, t)}- sha256sum := sha256.New()- for _, pt := range t.PreTokenizer.PreTokenizers {- if pt.Type == "Split" && pt.Pattern.Regex != "" {- sha256sum.Write([]byte(pt.Pattern.Regex))+ slices.SortFunc(tokens, func(i, j token) int {+ return cmp.Compare(i.ID, j.ID)+ })++ v := Vocabulary{Model: "gpt2"}+ for _, t := range tokens {+ v.Tokens = append(v.Tokens, t.Content)+ v.Scores = append(v.Scores, float32(t.ID))++ switch {+ case t.Special:+ v.Types = append(v.Types, tokenTypeControl)+ case t.UserDefined:+ v.Types = append(v.Types, tokenTypeUserDefined)+ default:+ v.Types = append(v.Types, tokenTypeNormal)}}- switch digest := fmt.Sprintf("%x", sha256sum.Sum(nil)); digest {- case "d98f9631be1e9607a9848c26c1f9eac1aa9fc21ac6ba82a2fc0741af9780a48f":- pre = "llama-bpe"- case "03df5c5863ad70781dcfdef491ead25140f895fe8010964be0daefe27be32b02":- pre = "deepseek-llm"- case "21cde974d587f0d54dc8d56b183cc1e6239600172035c68fbd6d4b9f8da0576e":- pre = "deepseek-coder"- default:- slog.Warn("unknown pretokenizer, using default", "digest", digest)- pre = "default"+ return &v, nil+}++func parseVocabulary(d string) (*Vocabulary, error) {+ patterns := map[string]func(string) (*Vocabulary, error){+ "tokenizer.model": parseSentencePiece,+ "tokenizer.json": parseVocabularyFromTokenizer,+ }++ for pattern, parseFn := range patterns {+ matches, err := filepath.Glob(filepath.Join(d, pattern))+ if err != nil {+ return nil, err+ }++ if len(matches) > 0 {+ return parseFn(d)+ }+ }++ return nil, errors.New("unknown tensor format")+}++type SpecialVocabulary struct {+ Type string+ ID int+ Content string+ AddToken bool+}++func (sv SpecialVocabulary) Key() string {+ switch t := sv.Type; t {+ case "bos", "eos", "cls", "mask":+ return t+ case "unk":+ return "unknown"+ case "sep":+ //nolint:misspell // this is an upstream typo+ return "seperator"+ case "pad":+ return "padding"}- return pre, tokens, t.Model.Merges, nil+ panic("unknown special vocabulary type")}diff --git a/convert/tokenizer_spm.go b/convert/tokenizer_spm.gonew file mode 100644index 000000000..75d9fe267--- /dev/null+++ b/convert/tokenizer_spm.go@@ -0,0 +1,83 @@+package convert++import (+ "cmp"+ "encoding/json"+ "errors"+ "fmt"+ "os"+ "path/filepath"+ "slices"++ "google.golang.org/protobuf/proto"++ "github.com/ollama/ollama/convert/sentencepiece"+)++func parseSentencePiece(d string) (*Vocabulary, error) {+ bts, err := os.ReadFile(filepath.Join(d, "tokenizer.model"))+ if err != nil {+ return nil, err+ }++ var spm sentencepiece.ModelProto+ if err := proto.Unmarshal(bts, &spm); err != nil {+ return nil, err+ }++ v := Vocabulary{Model: "llama"}+ for _, piece := range spm.GetPieces() {+ v.Tokens = append(v.Tokens, piece.GetPiece())+ v.Scores = append(v.Scores, piece.GetScore())++ switch t := piece.GetType(); t {+ case sentencepiece.ModelProto_SentencePiece_UNKNOWN,+ sentencepiece.ModelProto_SentencePiece_CONTROL,+ sentencepiece.ModelProto_SentencePiece_UNUSED,+ sentencepiece.ModelProto_SentencePiece_BYTE:+ v.Types = append(v.Types, int32(t))+ default:+ v.Types = append(v.Types, int32(sentencepiece.ModelProto_SentencePiece_NORMAL))+ }+ }++ f, err := os.Open(filepath.Join(d, "added_tokens.json"))+ if errors.Is(err, os.ErrNotExist) {+ return &v, nil+ } else if err != nil {+ return nil, err+ }+ defer f.Close()++ var atm map[string]int+ if err := json.NewDecoder(f).Decode(&atm); err != nil {+ return nil, err+ }++ type t struct {+ id int+ content string+ }++ var ts []t+ for content, id := range atm {+ ts = append(ts, t{id, content})+ }++ slices.SortFunc(ts, func(i, j t) int {+ return cmp.Compare(i.id, j.id)+ })++ n := len(v.Tokens)+ for i, t := range ts {+ if t.id != i+n {+ return nil, fmt.Errorf("invalid token id: %d", t.id)+ }++ v.Tokens = append(v.Tokens, t.content)+ v.Scores = append(v.Scores, -1000.0)+ v.Types = append(v.Types, tokenTypeUserDefined)+ }++ return &v, nil+}diff --git a/convert/torch.go b/convert/torch.godeleted file mode 100644index 55414adc6..000000000--- a/convert/torch.go+++ /dev/null@@ -1,287 +0,0 @@-package convert--import (- "encoding/binary"- "encoding/json"- "fmt"- "io"- "log/slog"- "os"- "path/filepath"- "regexp"- "strings"-- "github.com/nlpodyssey/gopickle/pytorch"- "github.com/nlpodyssey/gopickle/types"- "github.com/x448/float16"-- "github.com/ollama/ollama/llm"-)--type torchWriterTo struct {- t *llm.Tensor-- params *Params- bo ByteOrder-- storage pytorch.StorageInterface- repacker func(string, []float32, []uint64) ([]float32, error)-}--type TorchFormat struct{}--func (tf *TorchFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor, error) {- slog.Debug("getting torch tensors")-- var files []string- if pt, _ := filepath.Glob(filepath.Join(dirpath, "consolidated*.pth")); len(pt) > 0 {- files = append(files, pt...)- } else if pt, _ := filepath.Glob(filepath.Join(dirpath, "pytorch_model*.pth")); len(pt) > 0 {- files = append(files, pt...)- }-- var offset uint64- var tensors []llm.Tensor- for _, fn := range files {- m, err := pytorch.Load(fn)- if err != nil {- slog.Error(fmt.Sprintf("error unpickling: %q", err))- return []llm.Tensor{}, err- }-- for _, k := range m.(*types.Dict).Keys() {- if strings.HasSuffix(k.(string), "self_attn.rotary_emb.inv_freq") {- continue- }-- t, _ := m.(*types.Dict).Get(k)- tshape := t.(*pytorch.Tensor).Size-- var size uint64- var kind uint32- switch len(tshape) {- case 0:- continue- case 1:- // convert to float32- kind = 0- size = uint64(tshape[0] * 4)- case 2:- // convert to float16- kind = 1- size = uint64(tshape[0] * tshape[1] * 2)- }-- ggufName, err := tf.GetLayerName(k.(string))- if err != nil {- slog.Error(err.Error())- return nil, err- }- slog.Debug(fmt.Sprintf("'%35s': '%30s' %10d [%#v]", k.(string), ggufName, size, tshape))-- shape := []uint64{0, 0, 0, 0}- for i := range tshape {- shape[i] = uint64(tshape[i])- }-- tensor := llm.Tensor{- Name: ggufName,- Kind: kind,- Offset: offset, // calculate the offset- Shape: shape,- }-- tensor.WriterTo = torchWriterTo{- t: &tensor,- params: params,- bo: params.ByteOrder,- storage: t.(*pytorch.Tensor).Source,- }-- tensors = append(tensors, tensor)- offset += size- }- }-- return tensors, nil-}--func getAltParams(dirpath string) (*Params, error) {- f, err := os.Open(filepath.Join(dirpath, "params.json"))- if err != nil {- slog.Error("no params.json")- return nil, err- }- defer f.Close()-- type TorchParams struct {- HiddenSize int `json:"dim"`- AttentionHeads int `json:"n_heads"`- KeyValHeads int `json:"n_kv_heads"`- HiddenLayers int `json:"n_layers"`- RopeTheta float64 `json:"rope_theta"`- NormEPS float64 `json:"norm_eps"`- }-- var tparams TorchParams-- d := json.NewDecoder(f)- err = d.Decode(&tparams)- if err != nil {- return nil, err- }-- params := &Params{- Architectures: []string{"LlamaForCausalLM"},- HiddenSize: tparams.HiddenSize,- AttentionHeads: tparams.AttentionHeads,- KeyValHeads: tparams.KeyValHeads,- HiddenLayers: tparams.HiddenLayers,- NormEPS: tparams.NormEPS,- }-- switch {- case tparams.RopeTheta == 1000000:- // Codellama- params.ContextSize = 16384- case tparams.NormEPS == 1e-06:- // llama2- slog.Debug("Found llama2 - setting context size to 4096")- params.ContextSize = 4096- default:- params.ContextSize = 2048- }-- params.ByteOrder = binary.LittleEndian- return params, nil-}--func (m *TorchFormat) GetParams(dirpath string) (*Params, error) {- f, err := os.Open(filepath.Join(dirpath, "config.json"))- if err != nil {- if os.IsNotExist(err) {- // try params.json instead- return getAltParams(dirpath)- } else {- return nil, err- }- }-- var params Params- d := json.NewDecoder(f)- err = d.Decode(¶ms)- if err != nil {- return nil, err- }-- params.ByteOrder = binary.LittleEndian- return ¶ms, nil-}--func (m *TorchFormat) GetLayerName(n string) (string, error) {- directMap := map[string]string{- "tok_embeddings.weight": "token_embd.weight",- "output.weight": "output.weight",- "norm.weight": "output_norm.weight",- "rope.freqs": "rope_freqs.weight",- "model.embed_tokens.weight": "token_embd.weight",- "lm_head.weight": "output.weight",- "model.norm.weight": "output_norm.weight",- }-- lMap := map[string]string{- "layers.(\\d+).attention_norm.weight": "blk.$1.attn_norm.weight",- "layers.(\\d+).attention_output_norm.weight": "blk.$1.attn_norm.weight",- "layers.(\\d+).feed_forward.w2.weight": "blk.$1.ffn_down.weight",- "layers.(\\d+).feed_forward.w1.weight": "blk.$1.ffn_gate.weight",- "layers.(\\d+).feed_forward.w3.weight": "blk.$1.ffn_up.weight",- "layers.(\\d+).ffn_norm.weight": "blk.$1.ffn_norm.weight",- "layers.(\\d+).attention.wk.weight": "blk.$1.attn_k.weight",- "layers.(\\d+).attention.wo.weight": "blk.$1.attn_output.weight",- "layers.(\\d+).attention.wq.weight": "blk.$1.attn_q.weight",- "layers.(\\d+).attention.wv.weight": "blk.$1.attn_v.weight",- "model.layers.(\\d+).input_layernorm.weight": "blk.$1.attn_norm.weight",- "model.layers.(\\d+).mlp.down_proj.weight": "blk.$1.ffn_down.weight",- "model.layers.(\\d+).mlp.gate_proj.weight": "blk.$1.ffn_gate.weight",- "model.layers.(\\d+).mlp.up_proj.weight": "blk.$1.ffn_up.weight",- "model.layers.(\\d+).post_attention_layernorm.weight": "blk.$1.ffn_norm.weight",- "model.layers.(\\d+).self_attn.k_proj.weight": "blk.$1.attn_k.weight",- "model.layers.(\\d+).self_attn.o_proj.weight": "blk.$1.attn_output.weight",- "model.layers.(\\d+).self_attn.q_proj.weight": "blk.$1.attn_q.weight",- "model.layers.(\\d+).self_attn.v_proj.weight": "blk.$1.attn_v.weight",- }-- v, ok := directMap[n]- if ok {- return v, nil- }-- // quick hack to rename the layers to gguf format- for k, v := range lMap {- re := regexp.MustCompile(k)- newName := re.ReplaceAllString(n, v)- if newName != n {- return newName, nil- }- }-- return "", fmt.Errorf("couldn't find a layer name for '%s'", n)-}--func (r torchWriterTo) WriteTo(w io.Writer) (n int64, err error) {- var f32s []float32- switch s := r.storage.(type) {- case *pytorch.FloatStorage:- f32s = s.Data- case *pytorch.HalfStorage:- f32s = s.Data- case *pytorch.BFloat16Storage:- f32s = s.Data- default:- return 0, fmt.Errorf("unknown data type: %T", s)- }-- if r.repacker != nil {- f32s, err = r.repacker(r.t.Name, f32s, r.t.Shape)- if err != nil {- return 0, err- }- }-- switch r.t.Kind {- case 0:- return 0, binary.Write(w, r.bo, f32s)- case 1:- f16s := make([]uint16, len(f32s))- for i := range f32s {- f16s[i] = float16.Fromfloat32(f32s[i]).Bits()- }-- return 0, binary.Write(w, r.bo, f16s)- default:- return 0, fmt.Errorf("unknown storage type: %d", r.t.Kind)- }-}--func (m *TorchFormat) GetModelArch(name, dirPath string, params *Params) (ModelArch, error) {- switch len(params.Architectures) {- case 0:- return nil, fmt.Errorf("No architecture specified to convert")- case 1:- switch params.Architectures[0] {- case "LlamaForCausalLM":- return &LlamaModel{- ModelData{- Name: name,- Path: dirPath,- Params: params,- Format: m,- },- }, nil- default:- return nil, fmt.Errorf("Models based on '%s' are not yet supported", params.Architectures[0])- }- }-- return nil, fmt.Errorf("Unknown error")-}diff --git a/llm/gguf.go b/llm/gguf.goindex aadfc4ba8..e61babf2b 100644--- a/llm/gguf.go+++ b/llm/gguf.go@@ -2,11 +2,16 @@ package llmimport ("bytes"+ "cmp""encoding/binary""encoding/json""fmt""io"+ "log/slog"+ "slices""strings"++ "golang.org/x/exp/maps")type containerGGUF struct {@@ -88,7 +93,7 @@ type gguf struct {kv KVtensors []*Tensor- parameters uint64+ parameters uint64tensorOffset uint64scratch [16 << 10]byte@@ -101,10 +106,6 @@ func newGGUF(container *containerGGUF) *gguf {}}-func NewGGUFV3(bo binary.ByteOrder) *gguf {- return newGGUF(&containerGGUF{ByteOrder: bo, Version: 3})-}-func (llm *gguf) KV() KV {return llm.kv}@@ -203,7 +204,7 @@ func (llm *gguf) Decode(rs io.ReadSeeker) error {return fmt.Errorf("failed to read tensor dimensions: %w", err)}- shape := [4]uint64{1, 1, 1, 1}+ shape := make([]uint64, dims)for i := 0; uint32(i) < dims; i++ {shape[i], err = readGGUF[uint64](llm, rs)if err != nil {@@ -245,7 +246,7 @@ func (llm *gguf) Decode(rs io.ReadSeeker) error {return err}- padding := llm.padding(offset, int64(alignment))+ padding := ggufPadding(offset, int64(alignment))llm.tensorOffset = uint64(offset + padding)for _, tensor := range llm.tensors {@@ -254,7 +255,7 @@ func (llm *gguf) Decode(rs io.ReadSeeker) error {return fmt.Errorf("failed to get current offset: %w", err)}- padding := llm.padding(offset, int64(alignment))+ padding := ggufPadding(offset, int64(alignment))if _, err := rs.Seek(padding, io.SeekCurrent); err != nil {return fmt.Errorf("failed to seek to init padding: %w", err)}@@ -273,12 +274,12 @@ func readGGUF[T any](llm *gguf, r io.Reader) (T, error) {return t, err}-func writeGGUF[V any](llm *gguf, w io.Writer, t uint32, v V) error {- if err := binary.Write(w, llm.ByteOrder, t); err != nil {+func writeGGUF[V any](w io.Writer, t uint32, v V) error {+ if err := binary.Write(w, binary.LittleEndian, t); err != nil {return err}- return binary.Write(w, llm.ByteOrder, v)+ return binary.Write(w, binary.LittleEndian, v)}func readGGUFV1String(llm *gguf, r io.Reader) (string, error) {@@ -342,12 +343,12 @@ func readGGUFString(llm *gguf, r io.Reader) (string, error) {return string(buf), nil}-func writeGGUFString(llm *gguf, w io.Writer, s string) error {- if err := binary.Write(w, llm.ByteOrder, ggufTypeString); err != nil {+func writeGGUFString(w io.Writer, s string) error {+ if err := binary.Write(w, binary.LittleEndian, ggufTypeString); err != nil {return err}- if err := binary.Write(w, llm.ByteOrder, uint64(len(s))); err != nil {+ if err := binary.Write(w, binary.LittleEndian, uint64(len(s))); err != nil {return err}@@ -488,21 +489,21 @@ func readGGUFArray(llm *gguf, r io.Reader) (*array, error) {return a, nil}-func writeGGUFArray[S ~[]E, E any](llm *gguf, w io.Writer, t uint32, s S) error {- if err := binary.Write(w, llm.ByteOrder, ggufTypeArray); err != nil {+func writeGGUFArray[S ~[]E, E any](w io.Writer, t uint32, s S) error {+ if err := binary.Write(w, binary.LittleEndian, ggufTypeArray); err != nil {return err}- if err := binary.Write(w, llm.ByteOrder, t); err != nil {+ if err := binary.Write(w, binary.LittleEndian, t); err != nil {return err}- if err := binary.Write(w, llm.ByteOrder, uint64(len(s))); err != nil {+ if err := binary.Write(w, binary.LittleEndian, uint64(len(s))); err != nil {return err}for _, e := range s {- if err := binary.Write(w, llm.ByteOrder, e); err != nil {+ if err := binary.Write(w, binary.LittleEndian, e); err != nil {return err}}@@ -510,201 +511,158 @@ func writeGGUFArray[S ~[]E, E any](llm *gguf, w io.Writer, t uint32, s S) errorreturn nil}-var ggufKVOrder = map[string][]string{- "llama": {- "general.architecture",- "general.name",- "llama.vocab_size",- "llama.context_length",- "llama.embedding_length",- "llama.block_count",- "llama.feed_forward_length",- "llama.attention.head_count",- "llama.attention.head_count_kv",- "llama.attention.layer_norm_rms_epsilon",- "llama.rope.freq_base",- "llama.rope.dimension_count",- "llama.expert_count",- "llama.expert_used_count",- "gemma.context_length",- "gemma.embedding_length",- "gemma.block_count",- "gemma.feed_forward_length",- "gemma.attention.head_count",- "gemma.attention.head_count_kv",- "gemma.attention.layer_norm_rms_epsilon",- "gemma.attention.key_length",- "gemma.attention.value_length",- "general.file_type",- "tokenizer.ggml.pre",- "tokenizer.ggml.model",- "tokenizer.ggml.tokens",- "tokenizer.ggml.scores",- "tokenizer.ggml.merges",- "tokenizer.ggml.token_type",- "tokenizer.ggml.bos_token_id",- "tokenizer.ggml.eos_token_id",- "tokenizer.ggml.unknown_token_id",- "tokenizer.ggml.padding_token_id",- "tokenizer.ggml.add_bos_token",- "tokenizer.ggml.add_eos_token",- "tokenizer.chat_template",- "bert.pooling_type",- },-}--func (llm *gguf) Encode(ws io.WriteSeeker, kv KV, tensors []Tensor) error {- switch llm.Version {- case 3:- llm.V3.NumTensor = uint64(len(tensors))- llm.V3.NumKV = uint64(len(kv))- default:- return fmt.Errorf("not implemented: ggufv%d", llm.Version)- }-- if err := binary.Write(ws, llm.ByteOrder, []byte("GGUF")); err != nil {+func WriteGGUF(ws io.WriteSeeker, kv KV, ts []*Tensor) error {+ if err := binary.Write(ws, binary.LittleEndian, []byte("GGUF")); err != nil {return err}- if err := binary.Write(ws, llm.ByteOrder, llm.Version); err != nil {+ if err := binary.Write(ws, binary.LittleEndian, uint32(3)); err != nil {return err}- if err := binary.Write(ws, llm.ByteOrder, llm.numTensor()); err != nil {+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(ts))); err != nil {return err}- if err := binary.Write(ws, llm.ByteOrder, llm.numKV()); err != nil {+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(kv))); err != nil {return err}- kvCheck := make(map[string]bool)- for k := range kv {- kvCheck[k] = false+ keys := maps.Keys(kv)+ slices.Sort(keys)++ for _, key := range keys {+ if err := ggufWriteKV(ws, key, kv[key]); err != nil {+ return err+ }}- for _, k := range ggufKVOrder["llama"] {- v, ok := kv[k]- if !ok {- continue+ slices.SortFunc(ts, func(a, b *Tensor) int {+ var i, j int+ if n, err := fmt.Sscanf(a.Name, "blk.%d", &i); err != nil || n != 1 {+ return cmp.Compare(a.Name, b.Name)+ } else if n, err := fmt.Sscanf(b.Name, "blk.%d", &j); err != nil || n != 1 {+ return cmp.Compare(a.Name, b.Name)}- kvCheck[k] = true- if err := binary.Write(ws, llm.ByteOrder, uint64(len(k))); err != nil {+ return cmp.Compare(i, j)+ })++ var s uint64+ for _, t := range ts {+ t.Offset = s+ if err := ggufWriteTensorInfo(ws, t); err != nil {return err}+ s += t.Size()+ }- if err := binary.Write(ws, llm.ByteOrder, []byte(k)); err != nil {+ var alignment int64 = 32+ for _, t := range ts {+ if err := ggufWriteTensor(ws, t, alignment); err != nil {return err}+ }- var err error- switch v := v.(type) {- case uint32:- err = writeGGUF(llm, ws, ggufTypeUint32, v)- case float32:- err = writeGGUF(llm, ws, ggufTypeFloat32, v)- case bool:- err = writeGGUF(llm, ws, ggufTypeBool, v)- case string:- err = writeGGUFString(llm, ws, v)- case []int32:- err = writeGGUFArray(llm, ws, ggufTypeInt32, v)- case []uint32:- err = writeGGUFArray(llm, ws, ggufTypeUint32, v)- case []float32:- err = writeGGUFArray(llm, ws, ggufTypeFloat32, v)- case []string:- if err := binary.Write(ws, llm.ByteOrder, ggufTypeArray); err != nil {- return err- }-- if err := binary.Write(ws, llm.ByteOrder, ggufTypeString); err != nil {- return err- }-- if err := binary.Write(ws, llm.ByteOrder, uint64(len(v))); err != nil {- return err- }-- for _, e := range v {- if err := binary.Write(ws, llm.ByteOrder, uint64(len(e))); err != nil {- return err- }+ return nil+}- if err := binary.Write(ws, llm.ByteOrder, []byte(e)); err != nil {- return err- }- }- default:- return fmt.Errorf("improper type for '%s'", k)- }- if err != nil {- return err- }+func ggufWriteKV(ws io.WriteSeeker, k string, v any) error {+ slog.Debug(k, "type", fmt.Sprintf("%T", v))+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(k))); err != nil {+ return err}- for k, v := range kvCheck {- if !v {- return fmt.Errorf("Didn't know how to write kv %s", k)- }+ if err := binary.Write(ws, binary.LittleEndian, []byte(k)); err != nil {+ return err}- for _, tensor := range tensors {- if err := binary.Write(ws, llm.ByteOrder, uint64(len(tensor.Name))); err != nil {+ var err error+ switch v := v.(type) {+ case uint32:+ err = writeGGUF(ws, ggufTypeUint32, v)+ case float32:+ err = writeGGUF(ws, ggufTypeFloat32, v)+ case bool:+ err = writeGGUF(ws, ggufTypeBool, v)+ case string:+ err = writeGGUFString(ws, v)+ case []int32:+ err = writeGGUFArray(ws, ggufTypeInt32, v)+ case []uint32:+ err = writeGGUFArray(ws, ggufTypeUint32, v)+ case []float32:+ err = writeGGUFArray(ws, ggufTypeFloat32, v)+ case []string:+ if err := binary.Write(ws, binary.LittleEndian, ggufTypeArray); err != nil {return err}- if err := binary.Write(ws, llm.ByteOrder, []byte(tensor.Name)); err != nil {+ if err := binary.Write(ws, binary.LittleEndian, ggufTypeString); err != nil {return err}- var dims int- for cnt := range len(tensor.Shape) {- if tensor.Shape[cnt] > 0 {- dims++- }- }-- if err := binary.Write(ws, llm.ByteOrder, uint32(dims)); err != nil {+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(v))); err != nil {return err}- for i := range dims {- if err := binary.Write(ws, llm.ByteOrder, tensor.Shape[dims-1-i]); err != nil {+ for _, e := range v {+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(e))); err != nil {return err}- }- if err := binary.Write(ws, llm.ByteOrder, tensor.Kind); err != nil {- return err+ if err := binary.Write(ws, binary.LittleEndian, []byte(e)); err != nil {+ return err+ }}+ default:+ return fmt.Errorf("improper type for '%s'", k)+ }- if err := binary.Write(ws, llm.ByteOrder, tensor.Offset); err != nil {- return err- }+ return err+}++func ggufWriteTensorInfo(ws io.WriteSeeker, t *Tensor) error {+ slog.Debug(t.Name, "kind", t.Kind, "shape", t.Shape, "offset", t.Offset)+ if err := binary.Write(ws, binary.LittleEndian, uint64(len(t.Name))); err != nil {+ return err}- var alignment int64 = 32- for _, tensor := range tensors {- offset, err := ws.Seek(0, io.SeekCurrent)- if err != nil {- return err- }+ if err := binary.Write(ws, binary.LittleEndian, []byte(t.Name)); err != nil {+ return err+ }- padding := llm.padding(offset, alignment)- if err := binary.Write(ws, llm.ByteOrder, bytes.Repeat([]byte{0}, int(padding))); err != nil {- return err- }+ if err := binary.Write(ws, binary.LittleEndian, uint32(len(t.Shape))); err != nil {+ return err+ }- if _, err := tensor.WriteTo(ws); err != nil {+ for i := range len(t.Shape) {+ if err := binary.Write(ws, binary.LittleEndian, t.Shape[len(t.Shape)-i-1]); err != nil {return err}}- return nil+ if err := binary.Write(ws, binary.LittleEndian, t.Kind); err != nil {+ return err+ }++ return binary.Write(ws, binary.LittleEndian, t.Offset)+}++func ggufWriteTensor(ws io.WriteSeeker, t *Tensor, alignment int64) error {+ offset, err := ws.Seek(0, io.SeekCurrent)+ if err != nil {+ return err+ }++ if err := binary.Write(ws, binary.LittleEndian, bytes.Repeat([]byte{0}, int(ggufPadding(offset, alignment)))); err != nil {+ return err+ }++ _, err = t.WriteTo(ws)+ return err}-func (gguf) padding(offset, align int64) int64 {+func ggufPadding(offset, align int64) int64 {return (align - offset%align) % align}diff --git a/llm/memory_test.go b/llm/memory_test.goindex 06ae74387..18c797ee2 100644--- a/llm/memory_test.go+++ b/llm/memory_test.go@@ -2,7 +2,6 @@ package llmimport ("bytes"- "encoding/binary""fmt""os""testing"@@ -20,10 +19,9 @@ func TestEstimateGPULayers(t *testing.T) {f, err := os.CreateTemp(t.TempDir(), modelName)require.NoError(t, err)defer f.Close()- gguf := NewGGUFV3(binary.LittleEndian)inputLayerCount := 5- tensors := []Tensor{+ tensors := []*Tensor{{Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},{Name: "blk.1.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},{Name: "blk.2.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},@@ -32,7 +30,7 @@ func TestEstimateGPULayers(t *testing.T) {{Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},}assert.Len(t, tensors, inputLayerCount+1)- err = gguf.Encode(f, KV{+ err = WriteGGUF(f, KV{"general.architecture": "llama","general.name": "name","llama.context_length": uint32(32),diff --git a/server/model.go b/server/model.goindex c6d3078f1..81272a34c 100644--- a/server/model.go+++ b/server/model.go@@ -143,30 +143,6 @@ func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(areturn nil, err}- mf, err := convert.GetModelFormat(tempDir)- if err != nil {- return nil, err- }-- params, err := mf.GetParams(tempDir)- if err != nil {- return nil, err- }-- mArch, err := mf.GetModelArch("", tempDir, params)- if err != nil {- return nil, err- }-- fn(api.ProgressResponse{Status: "processing tensors"})- if err := mArch.GetTensors(); err != nil {- return nil, err- }-- if err := mArch.LoadVocab(); err != nil {- return nil, err- }-fn(api.ProgressResponse{Status: "converting model"})// TODO(mxyng): this should write directly into a layer@@ -178,7 +154,7 @@ func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(adefer temp.Close()defer os.Remove(temp.Name())- if err = mArch.WriteGGUF(temp); err != nil {+ if err := convert.Convert(tempDir, temp); err != nil {return nil, err}diff --git a/server/routes_create_test.go b/server/routes_create_test.goindex 8c7142094..4d616d8de 100644--- a/server/routes_create_test.go+++ b/server/routes_create_test.go@@ -2,7 +2,6 @@ package serverimport ("bytes"- "encoding/binary""encoding/json""fmt""io"@@ -20,7 +19,7 @@ import (var stream bool = false-func createBinFile(t *testing.T, kv map[string]any, ti []llm.Tensor) string {+func createBinFile(t *testing.T, kv map[string]any, ti []*llm.Tensor) string {t.Helper()f, err := os.CreateTemp(t.TempDir(), "")@@ -29,7 +28,7 @@ func createBinFile(t *testing.T, kv map[string]any, ti []llm.Tensor) string {}defer f.Close()- if err := llm.NewGGUFV3(binary.LittleEndian).Encode(f, kv, ti); err != nil {+ if err := llm.WriteGGUF(f, kv, ti); err != nil {t.Fatal(err)}diff --git a/server/routes_generate_test.go b/server/routes_generate_test.goindex 5c0caff1c..02f95dd28 100644--- a/server/routes_generate_test.go+++ b/server/routes_generate_test.go@@ -101,7 +101,7 @@ func TestGenerateChat(t *testing.T) {"tokenizer.ggml.tokens": []string{""},"tokenizer.ggml.scores": []float32{0},"tokenizer.ggml.token_type": []int32{0},- }, []llm.Tensor{+ }, []*llm.Tensor{{Name: "token_embd.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},{Name: "blk.0.attn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},{Name: "blk.0.ffn_down.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},@@ -149,7 +149,7 @@ func TestGenerateChat(t *testing.T) {Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{"general.architecture": "bert","bert.pooling_type": uint32(0),- }, []llm.Tensor{})),+ }, []*llm.Tensor{})),Stream: &stream,})@@ -399,7 +399,7 @@ func TestGenerate(t *testing.T) {"tokenizer.ggml.tokens": []string{""},"tokenizer.ggml.scores": []float32{0},"tokenizer.ggml.token_type": []int32{0},- }, []llm.Tensor{+ }, []*llm.Tensor{{Name: "token_embd.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},{Name: "blk.0.attn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},{Name: "blk.0.ffn_down.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))},@@ -447,7 +447,7 @@ func TestGenerate(t *testing.T) {Modelfile: fmt.Sprintf("FROM %s", createBinFile(t, llm.KV{"general.architecture": "bert","bert.pooling_type": uint32(0),- }, []llm.Tensor{})),+ }, []*llm.Tensor{})),Stream: &stream,})diff --git a/server/sched_test.go b/server/sched_test.goindex 6959dace3..f3c555147 100644--- a/server/sched_test.go+++ b/server/sched_test.go@@ -3,7 +3,6 @@ package serverimport ("bytes""context"- "encoding/binary""fmt""log/slog""os"@@ -114,8 +113,7 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, estrequire.NoError(t, err)defer f.Close()- gguf := llm.NewGGUFV3(binary.LittleEndian)- err = gguf.Encode(f, llm.KV{+ require.NoError(t, llm.WriteGGUF(f, llm.KV{"general.architecture": "llama","general.name": "name","llama.context_length": uint32(32),@@ -126,10 +124,10 @@ func newScenarioRequest(t *testing.T, ctx context.Context, modelName string, est"tokenizer.ggml.tokens": []string{" "},"tokenizer.ggml.scores": []float32{0},"tokenizer.ggml.token_type": []int32{0},- }, []llm.Tensor{+ }, []*llm.Tensor{{Name: "blk.0.attn.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},{Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))},- })+ }))require.NoError(t, err)fname := f.Name()