Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
model: implement bert in ollama engine (#9080)
Michael Yang3f6642f6fcf9
diff --git a/convert/convert_bert.go b/convert/convert_bert.goindex a9f4b8a77..6b0d0030a 100644--- a/convert/convert_bert.go+++ b/convert/convert_bert.go@@ -28,6 +28,7 @@ type bertModel struct {LayerNormEPS float32 `json:"layer_norm_eps"`LayerNormEpsilon float32 `json:"layer_norm_epsilon"`NormEpsilon float32 `json:"norm_epsilon"`+ normalizeEmbeddings boolPoolingType uint32}@@ -54,9 +55,11 @@ func (p *bertModel) parseMore(fsys fs.FS) error {var pooling stringfor _, m := range modules {- if m.Type == "sentence_transformers.models.Pooling" {+ switch m.Type {+ case "sentence_transformers.models.Pooling":pooling = m.Path- break+ case "sentence_transformers.models.Normalize":+ p.normalizeEmbeddings = true}}@@ -90,6 +93,7 @@ func (p *bertModel) KV(t *Tokenizer) ggml.KV {kv["general.architecture"] = "bert"kv["bert.attention.causal"] = falsekv["bert.pooling_type"] = p.PoolingType+ kv["bert.normalize_embeddings"] = p.normalizeEmbeddingskv["bert.block_count"] = cmp.Or(p.NLayers, p.NumHiddenLayers, p.NLayer)diff --git a/ml/backend.go b/ml/backend.goindex 154a0f1b5..ef7564782 100644--- a/ml/backend.go+++ b/ml/backend.go@@ -416,6 +416,7 @@ type Tensor interface {AddID(ctx Context, t2, ids Tensor) TensorSoftmax(ctx Context) Tensor+ L2Norm(ctx Context, eps float32) TensorLayerNorm(ctx Context, weight, bias Tensor, eps float32) TensorRMSNorm(ctx Context, weight Tensor, eps float32) TensorScale(ctx Context, s float64) Tensordiff --git a/ml/backend/ggml/ggml.go b/ml/backend/ggml/ggml.goindex 931386d56..d5e2e9c9c 100644--- a/ml/backend/ggml/ggml.go+++ b/ml/backend/ggml/ggml.go@@ -1205,6 +1205,13 @@ func (t *Tensor) AddID(ctx ml.Context, t2, ids ml.Tensor) ml.Tensor {}}+func (t *Tensor) L2Norm(ctx ml.Context, eps float32) ml.Tensor {+ return &Tensor{+ b: t.b,+ t: C.ggml_l2_norm(ctx.(*Context).ctx, t.t, C.float(eps)),+ }+}+func (t *Tensor) LayerNorm(ctx ml.Context, w, b ml.Tensor, eps float32) ml.Tensor {tt := C.ggml_norm(ctx.(*Context).ctx, t.t, C.float(eps))if w != nil {diff --git a/ml/nn/pooling/pooling.go b/ml/nn/pooling/pooling.gonew file mode 100644index 000000000..f84690c41--- /dev/null+++ b/ml/nn/pooling/pooling.go@@ -0,0 +1,36 @@+package pooling++import (+ "github.com/ollama/ollama/ml"+)++type Type uint32++const (+ TypeNone Type = iota+ TypeMean+ TypeCLS+ TypeLast+ TypeRank++ TypeUnknown = 0xFFFFFFFE+ TypeUnspecified = 0xFFFFFFFF+)++func Pooling(ctx ml.Context, hiddenStates ml.Tensor, poolingType Type) ml.Tensor {+ switch poolingType {+ case TypeNone:+ return hiddenStates+ case TypeMean:+ hiddenStates = hiddenStates.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx).Mean(ctx)+ return hiddenStates.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)+ case TypeCLS:+ return hiddenStates.View(ctx, 0, hiddenStates.Dim(0))+ case TypeLast:+ panic("not implemented")+ case TypeRank:+ panic("not implemented")+ default:+ panic("not implemented")+ }+}diff --git a/model/model.go b/model/model.goindex 3a72f09aa..efef71d8b 100644--- a/model/model.go+++ b/model/model.go@@ -24,7 +24,11 @@ import ("github.com/ollama/ollama/model/input")-var ErrNoVisionModel = errors.New("this model is missing data required for image input")+var (+ ErrNoVisionModel = errors.New("this model is missing data required for image input")+ ErrUnsupportedModel = errors.New("model not supported")+ ErrUnsupportedTokenizer = errors.New("tokenizer not supported")+)// Model implements a specific model architecture, defining the forward pass and any model-specific configurationtype Model interface {@@ -242,7 +246,7 @@ func setPointer(base Base, v reflect.Value, tags []Tag) {vv = vv.Elem()}- vv = vv.Elem()+ vv = reflect.Indirect(vv)if v.IsNil() {vv = reflect.New(v.Type().Elem()).Elem()}diff --git a/model/models/bert/model.go b/model/models/bert/model.gonew file mode 100644index 000000000..fd1dbd773--- /dev/null+++ b/model/models/bert/model.go@@ -0,0 +1,181 @@+package bert++import (+ "cmp"+ "math"++ "github.com/ollama/ollama/fs"+ "github.com/ollama/ollama/ml"+ "github.com/ollama/ollama/ml/nn"+ "github.com/ollama/ollama/ml/nn/pooling"+ "github.com/ollama/ollama/model"+ "github.com/ollama/ollama/model/input"+)++type Model struct {+ model.Base+ model.TextProcessor++ TokenEmbedding *nn.Embedding `gguf:"token_embd"`+ TypeEmbedding *nn.Embedding `gguf:"token_types"`+ PositionEmbedding *nn.Embedding `gguf:"position_embd"`+ TokenEmbeddingNorm *nn.LayerNorm `gguf:"token_embd_norm"`++ Layers []EncoderLayer `gguf:"blk"`++ Options+}++// Forward implements model.Model.+func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {+ hiddenStates := m.TokenEmbedding.Forward(ctx, batch.Inputs)+ hiddenStates = hiddenStates.Add(ctx, m.TypeEmbedding.Weight.View(ctx, 0, m.hiddenSize))+ hiddenStates = hiddenStates.Add(ctx, m.PositionEmbedding.Forward(ctx, ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))))+ hiddenStates = m.TokenEmbeddingNorm.Forward(ctx, hiddenStates, m.eps)++ for _, layer := range m.Layers {+ hiddenStates = layer.Forward(ctx, hiddenStates, &m.Options)+ }++ hiddenStates = pooling.Pooling(ctx, hiddenStates, m.poolingType)+ if m.normalize {+ hiddenStates = hiddenStates.L2Norm(ctx, 1e-12)+ }++ return hiddenStates, nil+}++type EncoderLayer struct {+ *Attention+ AttentionNorm *nn.LayerNorm `gguf:"attn_output_norm"`++ *MLP+ MLPNorm *nn.LayerNorm `gguf:"layer_output_norm"`+}++func (e *EncoderLayer) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *Options) ml.Tensor {+ // Attention+ residual := hiddenStates+ hiddenStates = e.Attention.Forward(ctx, hiddenStates, opts)+ hiddenStates = hiddenStates.Add(ctx, residual)+ hiddenStates = e.AttentionNorm.Forward(ctx, hiddenStates, opts.eps)++ // MLP+ residual = hiddenStates+ hiddenStates = e.MLP.Forward(ctx, hiddenStates, opts)+ hiddenStates = hiddenStates.Add(ctx, residual)+ hiddenStates = e.MLPNorm.Forward(ctx, hiddenStates, opts.eps)++ return hiddenStates+}++type Attention struct {+ Query *nn.Linear `gguf:"attn_q"`+ QueryNorm *nn.LayerNorm `gguf:"attn_q_norm"`++ Key *nn.Linear `gguf:"attn_k"`+ KeyNorm *nn.LayerNorm `gguf:"attn_k_norm"`++ Value *nn.Linear `gguf:"attn_v"`++ Output *nn.Linear `gguf:"attn_output"`+}++func (a *Attention) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *Options) ml.Tensor {+ batchSize := hiddenStates.Dim(1)++ query := a.Query.Forward(ctx, hiddenStates)+ if a.QueryNorm != nil {+ query = a.QueryNorm.Forward(ctx, query, opts.eps)+ }+ query = query.Reshape(ctx, opts.headDim(), opts.numHeads, batchSize)++ key := a.Key.Forward(ctx, hiddenStates)+ if a.KeyNorm != nil {+ key = a.KeyNorm.Forward(ctx, key, opts.eps)+ }+ key = key.Reshape(ctx, opts.headDim(), cmp.Or(opts.numKVHeads, opts.numHeads), batchSize)++ value := a.Value.Forward(ctx, hiddenStates)+ value = value.Reshape(ctx, opts.headDim(), cmp.Or(opts.numKVHeads, opts.numHeads), batchSize)++ attention := nn.Attention(ctx, query, key, value, 1/math.Sqrt(float64(opts.headDim())), nil)+ attention = attention.Reshape(ctx, opts.hiddenSize, batchSize)+ return a.Output.Forward(ctx, attention)+}++type MLP struct {+ Up *nn.Linear `gguf:"ffn_up"`+ Down *nn.Linear `gguf:"ffn_down"`+}++func (m *MLP) Forward(ctx ml.Context, hiddenStates ml.Tensor, opts *Options) ml.Tensor {+ return m.Down.Forward(ctx, m.Up.Forward(ctx, hiddenStates).GELU(ctx))+}++type Options struct {+ hiddenSize,+ numHeads,+ numKVHeads,+ keyLength,+ valueLength int+ poolingType pooling.Type+ eps float32+ normalize bool+}++func (o Options) headDim() int {+ return cmp.Or(o.keyLength, o.valueLength, o.hiddenSize/o.numHeads)+}++func New(c fs.Config) (model.Model, error) {+ var processor model.TextProcessor+ switch c.String("tokenizer.ggml.model", "bert") {+ case "bert":+ processor = model.NewWordPiece(+ &model.Vocabulary{+ Values: c.Strings("tokenizer.ggml.tokens"),+ Scores: c.Floats("tokenizer.ggml.scores"),+ Types: c.Ints("tokenizer.ggml.token_type"),+ AddBOS: c.Bool("tokenizer.ggml.add_bos_token", true),+ BOS: []int32{+ int32(cmp.Or(+ c.Uint("tokenizer.ggml.cls_token_id"),+ c.Uint("tokenizer.ggml.bos_token_id"),+ )),+ },+ AddEOS: c.Bool("tokenizer.ggml.add_eos_token", true),+ EOS: []int32{+ int32(cmp.Or(+ c.Uint("tokenizer.ggml.separator_token_id"),+ //nolint:misspell+ // NOTE: "seperator_token_id" is a typo in model metadata but we need to+ // support it for compatibility.+ c.Uint("tokenizer.ggml.seperator_token_id"),+ c.Uint("tokenizer.ggml.eos_token_id"),+ )),+ },+ },+ )+ default:+ return nil, model.ErrUnsupportedTokenizer+ }++ return &Model{+ TextProcessor: processor,+ Layers: make([]EncoderLayer, c.Uint("block_count")),+ Options: Options{+ hiddenSize: int(c.Uint("embedding_length")),+ numHeads: int(c.Uint("attention.head_count")),+ numKVHeads: int(c.Uint("attention.head_count_kv")),+ eps: c.Float("attention.layer_norm_epsilon"),+ poolingType: pooling.Type(c.Uint("pooling_type")),+ normalize: c.Bool("normalize_embeddings", true),+ },+ }, nil+}++func init() {+ model.Register("bert", New)+ model.Register("bert_embed", New)+}diff --git a/model/models/gemma2/model.go b/model/models/gemma2/model.goindex 84c89e1fe..8ccb9f92a 100644--- a/model/models/gemma2/model.go+++ b/model/models/gemma2/model.go@@ -24,7 +24,7 @@ type Options struct {type Model struct {model.Base- model.SentencePieceModel+ model.SentencePieceTokenEmbedding *nn.Embedding `gguf:"token_embd"`Layers []Layer `gguf:"blk"`@@ -40,7 +40,7 @@ const (func New(c fs.Config) (model.Model, error) {m := Model{- SentencePieceModel: model.NewSentencePieceModel(+ SentencePiece: model.NewSentencePiece(&model.Vocabulary{Values: c.Strings("tokenizer.ggml.tokens"),Scores: c.Floats("tokenizer.ggml.scores"),diff --git a/model/models/gemma3/embed.go b/model/models/gemma3/embed.goindex 7d1e269ff..395a0344d 100644--- a/model/models/gemma3/embed.go+++ b/model/models/gemma3/embed.go@@ -1,48 +1,38 @@package gemma3import (- "errors"-"github.com/ollama/ollama/fs""github.com/ollama/ollama/kvcache""github.com/ollama/ollama/ml""github.com/ollama/ollama/ml/nn"+ "github.com/ollama/ollama/ml/nn/pooling""github.com/ollama/ollama/model""github.com/ollama/ollama/model/input")type embedModel struct {model.Base- model.SentencePieceModel+ model.SentencePiece*TextModel- PoolingType uint32+ poolingType pooling.TypeDense [2]*nn.Linear `gguf:"dense"`}func (m *embedModel) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {hiddenStates := m.TextModel.Forward(ctx, batch, m.Cache)-- switch m.PoolingType {- case 0: // None- case 1: // Mean- hiddenStates = hiddenStates.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx).Mean(ctx)- hiddenStates = hiddenStates.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)- default:- return nil, errors.New("unsupported pooling type")- }-+ hiddenStates = pooling.Pooling(ctx, hiddenStates, m.poolingType)for _, dense := range m.Dense {hiddenStates = dense.Forward(ctx, hiddenStates)}-+ hiddenStates = hiddenStates.L2Norm(ctx, 1e-12)return hiddenStates, nil}func newEmbedModel(c fs.Config) (model.Model, error) {m := &embedModel{- SentencePieceModel: model.NewSentencePieceModel(+ SentencePiece: model.NewSentencePiece(&model.Vocabulary{Values: c.Strings("tokenizer.ggml.tokens"),Scores: c.Floats("tokenizer.ggml.scores"),@@ -60,7 +50,7 @@ func newEmbedModel(c fs.Config) (model.Model, error) {},),TextModel: newTextModel(c),- PoolingType: c.Uint("pooling_type", 0),+ poolingType: pooling.Type(c.Uint("pooling_type", 0)),}m.Cache = kvcache.NewWrapperCache(diff --git a/model/models/gemma3/model.go b/model/models/gemma3/model.goindex 5c92b6bf9..27da889e4 100644--- a/model/models/gemma3/model.go+++ b/model/models/gemma3/model.go@@ -16,7 +16,7 @@ import (type Model struct {model.Base- model.SentencePieceModel+ model.SentencePiece*VisionModel `gguf:"v"`*TextModel@@ -55,7 +55,7 @@ func (p *MultiModalProjector) Forward(ctx ml.Context, visionOutputs ml.Tensor, ifunc New(c fs.Config) (model.Model, error) {m := Model{- SentencePieceModel: model.NewSentencePieceModel(+ SentencePiece: model.NewSentencePiece(&model.Vocabulary{Values: c.Strings("tokenizer.ggml.tokens"),Scores: c.Floats("tokenizer.ggml.scores"),diff --git a/model/models/gemma3n/model.go b/model/models/gemma3n/model.goindex 6e83a9724..e59e3193f 100644--- a/model/models/gemma3n/model.go+++ b/model/models/gemma3n/model.go@@ -10,7 +10,7 @@ import (type Model struct {model.Base- model.SentencePieceModel+ model.SentencePiece*TextModel}@@ -23,7 +23,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {func New(c fs.Config) (model.Model, error) {m := Model{TextModel: newTextModel(c),- SentencePieceModel: model.NewSentencePieceModel(+ SentencePiece: model.NewSentencePiece(&model.Vocabulary{Values: c.Strings("tokenizer.ggml.tokens"),Scores: c.Floats("tokenizer.ggml.scores"),diff --git a/model/models/models.go b/model/models/models.goindex c880a4720..cc9980789 100644--- a/model/models/models.go+++ b/model/models/models.go@@ -1,6 +1,7 @@package modelsimport (+ _ "github.com/ollama/ollama/model/models/bert"_ "github.com/ollama/ollama/model/models/gemma2"_ "github.com/ollama/ollama/model/models/gemma3"_ "github.com/ollama/ollama/model/models/gemma3n"diff --git a/model/sentencepiece.go b/model/sentencepiece.goindex 827ce00d9..db07beee9 100644--- a/model/sentencepiece.go+++ b/model/sentencepiece.go@@ -12,18 +12,18 @@ import (const spmWhitespaceSep = "▁"-type SentencePieceModel struct {+type SentencePiece struct {maxTokenLen intvocab *Vocabulary}-var _ TextProcessor = (*SentencePieceModel)(nil)+var _ TextProcessor = (*SentencePiece)(nil)-func (spm SentencePieceModel) Vocabulary() *Vocabulary {+func (spm SentencePiece) Vocabulary() *Vocabulary {return spm.vocab}-func NewSentencePieceModel(vocab *Vocabulary) SentencePieceModel {+func NewSentencePiece(vocab *Vocabulary) SentencePiece {logutil.Trace("Tokens", "num tokens", len(vocab.Values), "vals", vocab.Values[:5], "scores", vocab.Scores[:5], "types", vocab.Types[:5])counter := map[int]int{}@@ -42,17 +42,17 @@ func NewSentencePieceModel(vocab *Vocabulary) SentencePieceModel {"user defined", counter[TOKEN_TYPE_USER_DEFINED], "unused", counter[TOKEN_TYPE_UNUSED], "byte", counter[TOKEN_TYPE_BYTE],"max token len", maxTokenLen)- return SentencePieceModel{+ return SentencePiece{maxTokenLen: maxTokenLen,vocab: vocab,}}-func (spm SentencePieceModel) Is(id int32, special Special) bool {+func (spm SentencePiece) Is(id int32, special Special) bool {return spm.vocab.Is(id, special)}-func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error) {+func (spm SentencePiece) Encode(s string, addSpecial bool) ([]int32, error) {fragments := []fragment{{value: s}}for _, special := range spm.vocab.SpecialVocabulary() {id := spm.vocab.Encode(special)@@ -218,7 +218,7 @@ func (q *queue) Pop() interface{} {return item}-func (spm SentencePieceModel) Decode(ids []int32) (string, error) {+func (spm SentencePiece) Decode(ids []int32) (string, error) {var sb strings.Builderfor _, id := range ids {data := spm.vocab.Decode(id)diff --git a/model/sentencepiece_test.go b/model/sentencepiece_test.goindex 50ac26787..8f4570c17 100644--- a/model/sentencepiece_test.go+++ b/model/sentencepiece_test.go@@ -12,7 +12,7 @@ import ("github.com/ollama/ollama/convert/sentencepiece")-func loadSentencePieceVocab(t *testing.T) SentencePieceModel {+func loadSentencePieceVocab(t *testing.T) SentencePiece {t.Helper()bts, err := os.ReadFile(filepath.Join("testdata", "gemma2", "tokenizer.model"))@@ -45,7 +45,7 @@ func loadSentencePieceVocab(t *testing.T) SentencePieceModel {}}- return NewSentencePieceModel(&v)+ return NewSentencePiece(&v)}func TestSentencePieceEncode(t *testing.T) {@@ -115,7 +115,7 @@ func TestSentencePieceEncode(t *testing.T) {})}-func TestSentencePieceModelDecodeByteTokens(t *testing.T) {+func TestSentencePieceDecodeByteTokens(t *testing.T) {vocab := &Vocabulary{Values: []string{"normal",@@ -134,7 +134,7 @@ func TestSentencePieceModelDecodeByteTokens(t *testing.T) {Scores: []float32{0, 0, 0, 0, 0},}- spm := NewSentencePieceModel(vocab)+ spm := NewSentencePiece(vocab)tests := []struct {name stringdiff --git a/model/wordpiece.go b/model/wordpiece.gonew file mode 100644index 000000000..e8d5e848a--- /dev/null+++ b/model/wordpiece.go@@ -0,0 +1,167 @@+package model++import (+ "fmt"+ "iter"+ "strings"+ "unicode"++ "github.com/ollama/ollama/logutil"+)++type WordPiece struct {+ vocab *Vocabulary+}++// ggmlPrefix is the prefix used by GGML vocabularies to indicate word boundaries.+// this differs from original word piece which uses "##" to indicate subwords.+const ggmlPrefix = "▁"++var wordPieceReplacer = strings.NewReplacer(+ " .", ".",+ " ?", "?",+ " !", "!",+ " ,", ",",+ " ' ", "'",+ " n't", "n't",+ " 'm", "'m",+ " do not", " don't",+ " 's", "'s",+ " 've", "'ve",+ " 're", "'re",+)++// Decode implements TextProcessor.+func (wpm WordPiece) Decode(ids []int32) (string, error) {+ var sb strings.Builder+ for i, id := range ids {+ if id < 0 || int(id) >= len(wpm.vocab.Values) {+ return "", fmt.Errorf("invalid token id: %d", id)+ }++ var separator string+ piece := wpm.vocab.Values[id]+ if i > 0 &&+ (strings.HasPrefix(piece, ggmlPrefix) ||+ (strings.HasPrefix(piece, "[") && strings.HasSuffix(piece, "]"))) {+ separator = " "+ }++ sb.WriteString(wordPieceReplacer.Replace(separator + strings.TrimPrefix(piece, ggmlPrefix)))+ }++ return sb.String(), nil+}++// words splits a string into words, treating CJK characters as separate words.+// TODO: this is specifically for BERT and may need to be adjusted or refactored for other models.+func (wpm WordPiece) words(s string) iter.Seq[string] {+ return func(yield func(string) bool) {+ runes := make([]rune, 0, len(s)*3)+ for _, r := range s {+ switch {+ case r >= 0x4E00 && r <= 0x9FFF,+ r >= 0x3400 && r <= 0x4DBF,+ r >= 0x20000 && r <= 0x2A6DF,+ r >= 0x2A700 && r <= 0x2B73F,+ r >= 0x2B740 && r <= 0x2B81F,+ r >= 0x2B820 && r <= 0x2CEAF,+ r >= 0xF900 && r <= 0xFAFF,+ r >= 0x2F800 && r <= 0x2FA1F:+ runes = append(runes, ' ', r, ' ')+ default:+ runes = append(runes, r)+ }+ }++ for w := range strings.FieldsFuncSeq(string(runes), unicode.IsSpace) {+ // split on but keep punctuation+ var start int+ for start < len(w) {+ end := strings.IndexFunc(w[start:], unicode.IsPunct)+ if end < 0 {+ end = len(w) - start+ } else if end == 0 {+ end = 1+ }++ if !yield(w[start : start+end]) {+ return+ }++ start += end+ }+ }+ }+}++// Encode implements TextProcessor.+func (wpm WordPiece) Encode(s string, addSpecial bool) ([]int32, error) {+ var ids []int32++ // TODO: use [UNK] from config+ unk := wpm.vocab.Encode("[UNK]")+ for word := range wpm.words(s) {+ var start int+ var pieces []int32+ for start < len(word) {+ end := len(word)++ var piece int32+ for start < end {+ subword := word[start:end]+ if start == 0 {+ subword = ggmlPrefix + subword+ }++ // TODO: some models might not want [ToLower]+ piece = wpm.vocab.Encode(strings.ToLower(subword))+ if piece >= 0 {+ break+ }++ end--+ }++ if piece < 0 {+ // Unknown token+ pieces = pieces[:0]+ break+ }++ pieces = append(pieces, piece)+ start = end+ }++ if len(pieces) > 0 {+ ids = append(ids, pieces...)+ } else {+ ids = append(ids, unk)+ }+ }++ if addSpecial && len(ids) > 0 {+ ids = wpm.vocab.addSpecials(ids)+ }++ logutil.Trace("encoded", "string", s, "ids", ids)+ return ids, nil+}++// Is implements TextProcessor.+func (wpm WordPiece) Is(id int32, special Special) bool {+ return wpm.vocab.Is(id, special)+}++// Vocabulary implements TextProcessor.+func (wpm WordPiece) Vocabulary() *Vocabulary {+ return wpm.vocab+}++var _ TextProcessor = (*WordPiece)(nil)++func NewWordPiece(vocab *Vocabulary) WordPiece {+ return WordPiece{+ vocab: vocab,+ }+}diff --git a/model/wordpiece_test.go b/model/wordpiece_test.gonew file mode 100644index 000000000..258fbffcb--- /dev/null+++ b/model/wordpiece_test.go@@ -0,0 +1,51 @@+package model++import (+ "slices"+ "testing"++ "github.com/google/go-cmp/cmp"+)++func TestWordPiece(t *testing.T) {+ wpm := NewWordPiece(+ &Vocabulary{+ Values: []string{"[UNK]", "[CLS]", "[SEP]", "▁hello", "▁world", "s", "▁!", "▁@", "▁#"},+ AddBOS: true,+ AddEOS: true,+ BOS: []int32{1},+ EOS: []int32{2},+ })++ ids, err := wpm.Encode("Hello world!", true)+ if err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff([]int32{1, 3, 4, 6, 2}, ids); diff != "" {+ t.Errorf("unexpected ids (-want +got):\n%s", diff)+ }++ words, err := wpm.Decode(ids)+ if err != nil {+ t.Fatal(err)+ }++ if diff := cmp.Diff("[CLS] hello world! [SEP]", words); diff != "" {+ t.Errorf("unexpected words (-want +got):\n%s", diff)+ }+}++func TestWordPieceWords(t *testing.T) {+ var wpm WordPiece++ basic := slices.Collect(wpm.words("Hey friend! How are you?!?"))+ if diff := cmp.Diff([]string{"Hey", "friend", "!", "How", "are", "you", "?", "!", "?"}, basic); diff != "" {+ t.Errorf("unexpected words (-want +got):\n%s", diff)+ }++ chinese := slices.Collect(wpm.words("野口里佳 Noguchi Rika"))+ if diff := cmp.Diff([]string{"野", "口", "里", "佳", "Noguchi", "Rika"}, chinese); diff != "" {+ t.Errorf("unexpected words (-want +got):\n%s", diff)+ }+}diff --git a/server/routes.go b/server/routes.goindex 5114cb74f..739ce69da 100644--- a/server/routes.go+++ b/server/routes.go@@ -488,7 +488,6 @@ func (s *Server) EmbedHandler(c *gin.Context) {}truncate := true-if req.Truncate != nil && !*req.Truncate {truncate = false}@@ -555,7 +554,16 @@ func (s *Server) EmbedHandler(c *gin.Context) {return}+ if bos := kvData.Uint("tokenizer.ggml.bos_token_id"); tokens[0] != int(bos) && kvData.Bool("add_bos_token", true) {+ ctxLen--+ }++ if eos := kvData.Uint("tokenizer.ggml.eos_token_id"); tokens[len(tokens)-1] != int(eos) && kvData.Bool("add_eos_token", true) {+ ctxLen--+ }+tokens = tokens[:ctxLen]+s, err = r.Detokenize(c.Request.Context(), tokens)if err != nil {c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})