Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
ollamarunner: Improve multimodal input handling
Jesse Grossa7e63b82be6d
diff --git a/model/model.go b/model/model.goindex f8ed87410..75b7f6397 100644--- a/model/model.go+++ b/model/model.go@@ -3,7 +3,6 @@ package modelimport ("errors""fmt"- "image"_ "image/jpeg"_ "image/png""log/slog"@@ -22,14 +21,40 @@ import (_ "github.com/ollama/ollama/ml/backend")+// Input represents one token in the input stream+type Input struct {+ // Token is a single element of text.+ Token int32++ // Multimodal is opaque data representing a non-text+ // element such as an image (or part of one if the image+ // can be processed in pieces). It may be either together+ // with Token or on its own.+ Multimodal any++ // MultimodalHash is a unique representation of the data+ // stored in Multimodal, used for caching and comparing+ // equality.+ MultimodalHash uint64+}++// MultimodalIndex is a multimodal element (such as an image)+// together with an index into the slice of Inputs with the+// corresponding token. Note that the index is not the same+// as the position - to find that use the index with the+// Positions slice.+type MultimodalIndex struct {+ Index int+ Multimodal any+}+// Options contains the inputs for a model forward passtype Options struct {- Inputs []int32- Positions []int32- Sequences []int- Outputs []int32-- Images []image.Image+ Inputs []int32+ Multimodal []MultimodalIndex+ Positions []int32+ Sequences []int+ Outputs []int32}type config struct {@@ -59,6 +84,37 @@ type Model interface {Config() config}+// MultimodalProcessor must be implemented by multimodal models.+type MultimodalProcessor interface {+ // EncodeMultimodal processes a single input (such as an image) and+ // generates an output (typically an embedding) that can be used by the model.+ //+ // The return value is most typically an ml.Tensor, however, different+ // type are possible, such as an object containing a tensor plus+ // additional metadata, a slice of tensors or even just the original input.+ //+ // The result may be cached by the runner.+ EncodeMultimodal(ml.Context, []byte) (any, error)++ // PostTokenize is called after tokenization to allow the model to edit the+ // input stream to correctly arrange multimodal elements.+ //+ // The input is a slice of tokens with the results of EncodeMultimodal interleaved+ // in the order that the user provided them. Each element of the slice will be+ // either a single token or single multimodal object.+ //+ // The model must ensure that inputs are stored according to how they will be+ // processed and stored in the cache. For example, Llava-style models should insert+ // placeholder tokens equal to the feature size of the corresponding image with+ // the image itself attached to and split across these tokens. When Forward is called+ // a partial subset of these tokens may be submitted according to the batch size.+ //+ // This function is also responsible for updating MultimodalHash for any Multimodal+ // that is modified to ensure that there is a unique hash value that accurately+ // represents the contents.+ PostTokenize(ml.Context, []Input) ([]Input, error)+}+var models = make(map[string]func(ml.Config) (Model, error))// Register registers a model constructor for the given architecturediff --git a/model/models/mllama/model.go b/model/models/mllama/model.goindex 8fee0cdb0..945c72955 100644--- a/model/models/mllama/model.go+++ b/model/models/mllama/model.go@@ -1,7 +1,12 @@package mllamaimport (+ "bytes"+ "encoding/binary""fmt"+ "hash/fnv"+ "image"+ "slices""github.com/ollama/ollama/kvcache""github.com/ollama/ollama/ml"@@ -56,41 +61,79 @@ func New(c ml.Config) (model.Model, error) {return &m, nil}-func (m *Model) Forward(ctx ml.Context, opts model.Options) (ml.Tensor, error) {- var crossAttentionStates ml.Tensor- if opts.Images != nil {- f32s, aspectRatioID, err := m.ImageProcessor.ProcessImage(opts.Images[0])- if err != nil {- return nil, err- }+func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {+ image, _, err := image.Decode(bytes.NewReader(multimodalData))+ if err != nil {+ return nil, err+ }- pixelValues, err := ctx.FromFloatSlice(f32s,- m.ImageProcessor.imageSize,- m.ImageProcessor.imageSize,- m.ImageProcessor.numChannels,- m.ImageProcessor.maxNumTiles,- )- if err != nil {- return nil, err- }+ f32s, aspectRatioID, err := m.ImageProcessor.ProcessImage(image)+ if err != nil {+ return nil, err+ }- aspectRatio, err := ctx.FromIntSlice([]int32{int32(aspectRatioID)}, 1)- if err != nil {- return nil, err- }+ pixelValues, err := ctx.FromFloatSlice(f32s,+ m.ImageProcessor.imageSize,+ m.ImageProcessor.imageSize,+ m.ImageProcessor.numChannels,+ m.ImageProcessor.maxNumTiles,+ )+ if err != nil {+ return nil, err+ }- positions := make([]int32, 1601)- for i := range positions {- positions[i] = int32(i)- }+ aspectRatio, err := ctx.FromIntSlice([]int32{int32(aspectRatioID)}, 1)+ if err != nil {+ return nil, err+ }++ positions := make([]int32, 1601)+ for i := range positions {+ positions[i] = int32(i)+ }++ positionIDs, err := ctx.FromIntSlice(positions, len(positions))+ if err != nil {+ return nil, err+ }- positionIDs, err := ctx.FromIntSlice(positions, len(positions))- if err != nil {- return nil, err+ crossAttentionStates := m.VisionModel.Forward(ctx, pixelValues, positionIDs, aspectRatio)+ return m.Projector.Forward(ctx, crossAttentionStates), nil+}++func (m *Model) PostTokenize(ctx ml.Context, inputs []model.Input) ([]model.Input, error) {+ var images []model.Input+ fnvHash := fnv.New64a()++ for i := range inputs {+ if inputs[i].Multimodal == nil {+ if len(images) > 0 {+ inputs[i].Multimodal = images[0].Multimodal+ inputs[i].MultimodalHash = images[0].MultimodalHash+ for j := 1; j < len(images); j++ {+ inputs[i].Multimodal = inputs[i].Multimodal.(ml.Tensor).Concat(ctx, images[j].Multimodal.(ml.Tensor), 3)+ fnvHash.Reset()+ binary.Write(fnvHash, binary.NativeEndian, inputs[i].MultimodalHash)+ binary.Write(fnvHash, binary.NativeEndian, inputs[j].MultimodalHash)+ inputs[i].MultimodalHash = fnvHash.Sum64()+ }+ images = nil+ }+ } else {+ images = append(images, inputs[i])+ inputs[i].Token = -1}+ }++ inputs = slices.DeleteFunc(inputs, func(input model.Input) bool { return input.Token == -1 })- crossAttentionStates = m.VisionModel.Forward(ctx, pixelValues, positionIDs, aspectRatio)- crossAttentionStates = m.Projector.Forward(ctx, crossAttentionStates)+ return inputs, nil+}++func (m *Model) Forward(ctx ml.Context, opts model.Options) (ml.Tensor, error) {+ var crossAttentionStates ml.Tensor+ if opts.Multimodal != nil {+ crossAttentionStates = opts.Multimodal[0].Multimodal.(ml.Tensor)}inputs, err := ctx.FromIntSlice(opts.Inputs, len(opts.Inputs))diff --git a/runner/ollamarunner/cache.go b/runner/ollamarunner/cache.goindex e1fa98b1a..2fd060a1a 100644--- a/runner/ollamarunner/cache.go+++ b/runner/ollamarunner/cache.go@@ -5,7 +5,6 @@ import ("fmt""log/slog""math"- "reflect""time""github.com/ollama/ollama/kvcache"@@ -39,10 +38,7 @@ func NewInputCache(model model.Model, kvCacheType string, kvSize int32, numSlotsslots := make([]InputCacheSlot, numSlots)for i := range slots {- slots[i] = InputCacheSlot{- Id: i,- Inputs: make([]input, 0),- }+ slots[i] = InputCacheSlot{Id: i}}cache := model.Config().Cache@@ -83,7 +79,7 @@ type InputCacheSlot struct {Id int// Inputs that are stored in the KV cache- Inputs []input+ Inputs []model.Input// is this cache actively being processed as part of a sequence?InUse bool@@ -92,7 +88,7 @@ type InputCacheSlot struct {lastUsed time.Time}-func (c *InputCache) LoadCacheSlot(prompt []input, cachePrompt bool) (*InputCacheSlot, []input, error) {+func (c *InputCache) LoadCacheSlot(prompt []model.Input, cachePrompt bool) (*InputCacheSlot, []model.Input, error) {var slot *InputCacheSlotvar numPast int32var err error@@ -143,7 +139,7 @@ func (c *InputCache) LoadCacheSlot(prompt []input, cachePrompt bool) (*InputCachreturn slot, prompt, nil}-func (c *InputCache) findLongestCacheSlot(prompt []input) (*InputCacheSlot, int32, error) {+func (c *InputCache) findLongestCacheSlot(prompt []model.Input) (*InputCacheSlot, int32, error) {longest := int32(-1)var longestSlot *InputCacheSlot@@ -166,7 +162,7 @@ func (c *InputCache) findLongestCacheSlot(prompt []input) (*InputCacheSlot, int3return longestSlot, longest, nil}-func (c *InputCache) findBestCacheSlot(prompt []input) (*InputCacheSlot, int32, error) {+func (c *InputCache) findBestCacheSlot(prompt []model.Input) (*InputCacheSlot, int32, error) {oldest := time.Now()var oldestSlot *InputCacheSlot@@ -202,7 +198,7 @@ func (c *InputCache) findBestCacheSlot(prompt []input) (*InputCacheSlot, int32,if longest > 0 && longestSlot != oldestSlot {slog.Debug("forking cache slot", "src", longestSlot.Id, "dst", oldestSlot.Id, "inputs", longest, "total",len(longestSlot.Inputs))- oldestSlot.Inputs = make([]input, longest)+ oldestSlot.Inputs = make([]model.Input, longest)copy(oldestSlot.Inputs, longestSlot.Inputs[:longest])if c.cache != nil {c.cache.CopyPrefix(longestSlot.Id, oldestSlot.Id, longest)@@ -212,7 +208,7 @@ func (c *InputCache) findBestCacheSlot(prompt []input) (*InputCacheSlot, int32,return oldestSlot, longest, nil}-func countCommonPrefix(a []input, b []input) int32 {+func countCommonPrefix(a []model.Input, b []model.Input) int32 {var count int32for i := range a {@@ -220,7 +216,7 @@ func countCommonPrefix(a []input, b []input) int32 {break}- if !reflect.DeepEqual(a[i], b[i]) {+ if a[i].Token != b[i].Token || a[i].MultimodalHash != b[i].MultimodalHash {break}diff --git a/runner/ollamarunner/cache_test.go b/runner/ollamarunner/cache_test.goindex 99e67b4fa..9ce03b73f 100644--- a/runner/ollamarunner/cache_test.go+++ b/runner/ollamarunner/cache_test.go@@ -4,6 +4,8 @@ import ("image""testing""time"++ "github.com/ollama/ollama/model")func TestCountCommon(t *testing.T) {@@ -13,44 +15,50 @@ func TestCountCommon(t *testing.T) {tests := []struct {name string- t1 []input- t2 []input+ t1 []model.Input+ t2 []model.Inputexpected int32}{{name: "Equal",- t1: []input{{token: 1}, {token: 2}, {token: 3}},- t2: []input{{token: 1}, {token: 2}, {token: 3}},+ t1: []model.Input{{Token: 1}, {Token: 2}, {Token: 3}},+ t2: []model.Input{{Token: 1}, {Token: 2}, {Token: 3}},expected: 3,},{name: "Prefix",- t1: []input{{token: 1}},- t2: []input{{token: 1}, {token: 2}, {token: 3}},+ t1: []model.Input{{Token: 1}},+ t2: []model.Input{{Token: 1}, {Token: 2}, {Token: 3}},expected: 1,},{name: "Image Prefix",- t1: []input{{image: imgA}},- t2: []input{{image: imgA}, {image: imgB}, {image: imgC}},+ t1: []model.Input{{Multimodal: imgA, MultimodalHash: 1}},+ t2: []model.Input{{Multimodal: imgA, MultimodalHash: 1}, {Multimodal: imgB, MultimodalHash: 2}, {Multimodal: imgC, MultimodalHash: 3}},expected: 1,},{name: "Mixed",- t1: []input{{token: 1}, {image: imgA}},- t2: []input{{token: 1}, {image: imgA}, {token: 5}},+ t1: []model.Input{{Token: 1}, {Multimodal: imgA, MultimodalHash: 1}},+ t2: []model.Input{{Token: 1}, {Multimodal: imgA, MultimodalHash: 1}, {Token: 5}},expected: 2,},+ {+ name: "Mixed, Same Length",+ t1: []model.Input{{Token: 1}, {Multimodal: imgA, MultimodalHash: 1}},+ t2: []model.Input{{Token: 1}, {Multimodal: imgB, MultimodalHash: 2}},+ expected: 1,+ },{name: "Empty",- t1: []input{},- t2: []input{{token: 1}, {token: 2}, {token: 3}},+ t1: []model.Input{},+ t2: []model.Input{{Token: 1}, {Token: 2}, {Token: 3}},expected: 0,},{name: "Both Empty",- t1: []input{},- t2: []input{},+ t1: []model.Input{},+ t2: []model.Input{},expected: 0,},}@@ -74,7 +82,7 @@ func TestFindCacheSlot(t *testing.T) {tests := []struct {name stringcache InputCache- prompt []input+ prompt []model.Inputlongest expectedbest expected}{@@ -83,18 +91,18 @@ func TestFindCacheSlot(t *testing.T) {cache: InputCache{slots: []InputCacheSlot{{Id: 0,- Inputs: []input{},+ Inputs: []model.Input{},InUse: false,lastUsed: time.Time{},},{Id: 1,- Inputs: []input{},+ Inputs: []model.Input{},InUse: false,lastUsed: time.Time{},},}},- prompt: []input{{token: 1}},+ prompt: []model.Input{{Token: 1}},longest: expected{result: 0, len: 0},best: expected{result: 0, len: 0},},@@ -103,18 +111,18 @@ func TestFindCacheSlot(t *testing.T) {cache: InputCache{slots: []InputCacheSlot{{Id: 0,- Inputs: []input{{token: 1}},+ Inputs: []model.Input{{Token: 1}},InUse: false,lastUsed: time.Now().Add(-time.Second),},{Id: 1,- Inputs: []input{{token: 1}, {token: 2}},+ Inputs: []model.Input{{Token: 1}, {Token: 2}},InUse: false,lastUsed: time.Now().Add(-2 * time.Second),},}},- prompt: []input{{token: 1}, {token: 2}},+ prompt: []model.Input{{Token: 1}, {Token: 2}},longest: expected{result: 1, len: 2},best: expected{result: 1, len: 2},},@@ -123,18 +131,18 @@ func TestFindCacheSlot(t *testing.T) {cache: InputCache{slots: []InputCacheSlot{{Id: 0,- Inputs: []input{{token: 1}, {token: 2}},+ Inputs: []model.Input{{Token: 1}, {Token: 2}},InUse: false,lastUsed: time.Now().Add(-time.Second),},{Id: 1,- Inputs: []input{},+ Inputs: []model.Input{},InUse: false,lastUsed: time.Time{},},}},- prompt: []input{{token: 2}},+ prompt: []model.Input{{Token: 2}},longest: expected{result: 0, len: 0},best: expected{result: 1, len: 0},},@@ -144,19 +152,19 @@ func TestFindCacheSlot(t *testing.T) {slots: []InputCacheSlot{{Id: 0,- Inputs: []input{{token: 1}, {token: 2}},+ Inputs: []model.Input{{Token: 1}, {Token: 2}},InUse: false,lastUsed: time.Now().Add(-time.Second),},{Id: 1,- Inputs: []input{},+ Inputs: []model.Input{},InUse: false,lastUsed: time.Time{},},},},- prompt: []input{{token: 1}},+ prompt: []model.Input{{Token: 1}},longest: expected{result: 0, len: 1},best: expected{result: 1, len: 1},},@@ -165,18 +173,18 @@ func TestFindCacheSlot(t *testing.T) {cache: InputCache{slots: []InputCacheSlot{{Id: 0,- Inputs: []input{{token: 1}},+ Inputs: []model.Input{{Token: 1}},InUse: false,lastUsed: time.Now().Add(-time.Second),},{Id: 1,- Inputs: []input{{token: 1}, {token: 2}},+ Inputs: []model.Input{{Token: 1}, {Token: 2}},InUse: false,lastUsed: time.Now().Add(-2 * time.Second),},}},- prompt: []input{{token: 2}, {token: 3}},+ prompt: []model.Input{{Token: 2}, {Token: 3}},longest: expected{result: 0, len: 0},best: expected{result: 1, len: 0},},@@ -185,18 +193,18 @@ func TestFindCacheSlot(t *testing.T) {cache: InputCache{slots: []InputCacheSlot{{Id: 0,- Inputs: []input{{token: 1}, {token: 2}},+ Inputs: []model.Input{{Token: 1}, {Token: 2}},InUse: true,lastUsed: time.Now().Add(-time.Second),},{Id: 1,- Inputs: []input{{token: 1}},+ Inputs: []model.Input{{Token: 1}},InUse: false,lastUsed: time.Now().Add(-2 * time.Second),},}},- prompt: []input{{token: 1}, {token: 2}},+ prompt: []model.Input{{Token: 1}, {Token: 2}},longest: expected{result: 1, len: 1},best: expected{result: 1, len: 2},},diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.goindex 9ba6563f0..e5189fa56 100644--- a/runner/ollamarunner/runner.go+++ b/runner/ollamarunner/runner.go@@ -1,13 +1,12 @@package ollamarunnerimport (- "bytes""context""encoding/json""errors""flag""fmt"- "image"+ "hash/maphash""log""log/slog""net"@@ -33,22 +32,19 @@ import (_ "github.com/ollama/ollama/model/models")-// input is an element of the prompt to process, either a token or an image-type input struct {- token int32-- image image.Image-}-type Sequence struct {+ // ctx for allocating tensors that last the lifetime of the sequence, such as+ // multimodal embeddings+ ctx ml.Context+// batch indexiBatch int// prompt inputs left to evaluate- inputs []input+ inputs []model.Input// inputs that have been added to a batch but not yet submitted to Forward- pendingInputs []input+ pendingInputs []model.Input// tokens that have been generated but not returned yet (e.g. for stop sequences)pendingResponses []string@@ -101,8 +97,9 @@ func (s *Server) NewSequence(prompt string, images []ImageData, params NewSequens.ready.Wait()startTime := time.Now()+ ctx := s.model.Backend().NewContext()- inputs, err := s.inputs(prompt, images)+ inputs, err := s.inputs(ctx, prompt, images)if err != nil {return nil, fmt.Errorf("failed to process inputs: %w", err)} else if len(inputs) == 0 {@@ -128,6 +125,7 @@ func (s *Server) NewSequence(prompt string, images []ImageData, params NewSequen// TODO(jessegross): Ingest cached history for grammarreturn &Sequence{+ ctx: ctx,inputs: inputs,numPromptInputs: len(inputs),startProcessingTime: startTime,@@ -146,19 +144,22 @@ func (s *Server) NewSequence(prompt string, images []ImageData, params NewSequen// inputs processes the prompt and images into a list of inputs// by splitting the prompt on [img-<n>] tags, tokenizing text and// decoding images-func (s *Server) inputs(prompt string, images []ImageData) ([]input, error) {- var inputs []input+func (s *Server) inputs(ctx ml.Context, prompt string, images []ImageData) ([]model.Input, error) {+ var inputs []model.Inputvar parts []stringvar matches [][]string- // TODO(jessegross): This can sometimes trigger for matching text in the- // user's prompt. We previously tried to avoid it by only looking for images- // on image models. We don't have a clear indication now but it would be better- // to properly escape it in any case.- re := regexp.MustCompile(`\[img-(\d+)\]`)- parts = re.Split(prompt, -1)- matches = re.FindAllStringSubmatch(prompt, -1)+ multimodalProcessor, visionModel := s.model.(model.MultimodalProcessor)+ if visionModel {+ re := regexp.MustCompile(`\[img-(\d+)\]`)+ parts = re.Split(prompt, -1)+ matches = re.FindAllStringSubmatch(prompt, -1)+ } else {+ parts = []string{prompt}+ }++ postTokenize := falsefor i, part := range parts {// text - tokenizetokens, err := s.model.(model.TextProcessor).Encode(part, i == 0)@@ -167,7 +168,7 @@ func (s *Server) inputs(prompt string, images []ImageData) ([]input, error) {}for _, t := range tokens {- inputs = append(inputs, input{token: t})+ inputs = append(inputs, model.Input{Token: t})}// image - decode and store@@ -186,12 +187,25 @@ func (s *Server) inputs(prompt string, images []ImageData) ([]input, error) {return nil, fmt.Errorf("invalid image index: %d", n)}- image, _, err := image.Decode(bytes.NewReader(images[imageIndex].Data))+ imageEmbeddings, err := multimodalProcessor.EncodeMultimodal(ctx, images[imageIndex].Data)if err != nil {return nil, err}- inputs = append(inputs, input{image: image})+ s.multimodalHash.Reset()+ _, _ = s.multimodalHash.Write(images[imageIndex].Data)+ imageHash := s.multimodalHash.Sum64()++ inputs = append(inputs, model.Input{Multimodal: imageEmbeddings, MultimodalHash: imageHash})+ postTokenize = true+ }+ }++ if visionModel && postTokenize {+ var err error+ inputs, err = multimodalProcessor.PostTokenize(ctx, inputs)+ if err != nil {+ return nil, err}}@@ -238,6 +252,10 @@ type Server struct {// next sequence for prompt processing to avoid starvationnextSeq int++ // multimodalHash generates hashes for comparing equality+ // of non-text data+ multimodalHash maphash.Hash}func (s *Server) allNil() bool {@@ -283,6 +301,7 @@ func (s *Server) removeSequence(seqIndex int, reason string) {close(seq.responses)close(seq.embedding)seq.cache.InUse = false+ seq.ctx.Close()s.seqs[seqIndex] = nils.seqsSem.Release(1)}@@ -311,7 +330,6 @@ func (s *Server) processBatch() error {defer s.mu.Unlock()var options model.Options- imgSeq := -1seqIdx := s.nextSeq - 1for range s.seqs {@@ -330,7 +348,7 @@ func (s *Server) processBatch() error {if !s.cache.enabled {seq.inputs = append(seq.cache.Inputs, seq.inputs...)- seq.cache.Inputs = []input{}+ seq.cache.Inputs = []model.Input{}}for i, input := range seq.inputs {@@ -349,25 +367,21 @@ func (s *Server) processBatch() error {break}- // TODO(jessegross): Image inputs need to be rethought - it's- // it doesn't work well for different types of models or multiple sequences- if input.image != nil {- if len(seq.pendingInputs) != len(options.Images) {- break- }-- if imgSeq != seqIdx && imgSeq != -1 {- s.nextSeq = seqIdx- break- }+ // TODO(jessegross): This is a workaround for generating an attention mask and also providing a hint+ // to the encoder cache.+ //+ // Break the batch when switching from text to images so that images are always at the beginning.+ if input.Multimodal != nil && !(len(seq.pendingInputs) == 0 ||+ (len(options.Multimodal) > 0 && options.Multimodal[len(options.Multimodal)-1].Index == len(options.Inputs)-1)) {+ s.nextSeq = seqIdx+ break+ }- imgSeq = seqIdx- options.Images = append(options.Images, input.image)- seq.pendingInputs = append(seq.pendingInputs, input)- continue+ options.Inputs = append(options.Inputs, input.Token)+ if input.Multimodal != nil {+ options.Multimodal = append(options.Multimodal, model.MultimodalIndex{Index: len(options.Inputs) - 1, Multimodal: input.Multimodal})}- options.Inputs = append(options.Inputs, input.token)options.Positions = append(options.Positions, int32(len(seq.cache.Inputs)+len(seq.pendingInputs)))options.Sequences = append(options.Sequences, seq.cache.Id)@@ -403,7 +417,7 @@ func (s *Server) processBatch() error {// After calling Forward, pending inputs are now in the cacheif len(seq.pendingInputs) > 0 {seq.cache.Inputs = append(seq.cache.Inputs, seq.pendingInputs...)- seq.pendingInputs = []input{}+ seq.pendingInputs = []model.Input{}}// don't sample prompt processing@@ -449,7 +463,7 @@ func (s *Server) processBatch() error {return err}- seq.inputs = []input{{token: token}}+ seq.inputs = []model.Input{{Token: token}}seq.pendingResponses = append(seq.pendingResponses, piece)sequence := strings.Join(seq.pendingResponses, "")