Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
ollamarunner: Multi-modal worst case graph
Jesse Grossfe623c2cf44e
diff --git a/runner/ollamarunner/multimodal.go b/runner/ollamarunner/multimodal.goindex 16d359219..d78612fee 100644--- a/runner/ollamarunner/multimodal.go+++ b/runner/ollamarunner/multimodal.go@@ -48,12 +48,12 @@ func (m multimodalStore) addMultimodal(embedding []input.Multimodal) {// getMultimodal takes a source set of tensors (which may contain a whole or// parts of one or more images) and returns the equivalent that can be used in// the current context-func (m multimodalStore) getMultimodal(backend ml.Backend, ctx ml.Context, in []input.Multimodal) ([]input.Multimodal, error) {+func (m multimodalStore) getMultimodal(backend ml.Backend, ctx ml.Context, in []input.Multimodal, reserve bool) ([]input.Multimodal, error) {out := make([]input.Multimodal, len(in))for i := range out {if in[i].Tensor != nil {var err error- out[i].Tensor, err = m.getTensor(backend, ctx, in[i].Tensor)+ out[i].Tensor, err = m.getTensor(backend, ctx, in[i].Tensor, reserve)if err != nil {return nil, err}@@ -65,7 +65,7 @@ func (m multimodalStore) getMultimodal(backend ml.Backend, ctx ml.Context, in []return out, nil}-func (m multimodalStore) getTensor(backend ml.Backend, ctx ml.Context, in ml.Tensor) (ml.Tensor, error) {+func (m multimodalStore) getTensor(backend ml.Backend, ctx ml.Context, in ml.Tensor, reserve bool) (ml.Tensor, error) {entry := m[in]if entry.data == nil {@@ -83,19 +83,32 @@ func (m multimodalStore) getTensor(backend ml.Backend, ctx ml.Context, in ml.Tenreturn nil, nil}- computeCtx.Forward(tensors...).Compute(tensors...)-+ computeCtx.Forward(tensors...)entry.data = make([][]float32, len(entry.mm))- for i, t := range entry.mm {- if t.Tensor != nil {- entry.data[i] = t.Tensor.Floats()++ if !reserve {+ computeCtx.Compute(tensors...)++ for i, t := range entry.mm {+ if t.Tensor != nil {+ entry.data[i] = t.Tensor.Floats()+ }+ }+ } else {+ err := computeCtx.Reserve()+ if err != nil {+ return nil, err}}}for i, t := range entry.mm {if in == t.Tensor {- return ctx.Input().FromFloatSlice(entry.data[i], t.Tensor.Shape()...)+ if !reserve {+ return ctx.Input().FromFloatSlice(entry.data[i], t.Tensor.Shape()...)+ } else {+ return ctx.Input().Empty(t.Tensor.DType(), t.Tensor.Shape()...), nil+ }}}diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.goindex 4e203b7be..cd42d4341 100644--- a/runner/ollamarunner/runner.go+++ b/runner/ollamarunner/runner.go@@ -1,12 +1,14 @@package ollamarunnerimport (+ "bytes""context""encoding/json""errors""flag""fmt""hash/maphash"+ "image""log""log/slog""net"@@ -20,6 +22,7 @@ import ("time""unicode/utf8"+ "golang.org/x/image/bmp""golang.org/x/sync/semaphore""github.com/ollama/ollama/api"@@ -444,7 +447,7 @@ func (s *Server) processBatch() error {batchInputs = append(batchInputs, inp.Token)if inp.Multimodal != nil {- mm, err := seq.mmStore.getMultimodal(s.model.Backend(), ctx, inp.Multimodal)+ mm, err := seq.mmStore.getMultimodal(s.model.Backend(), ctx, inp.Multimodal, false)if err != nil {return err}@@ -732,12 +735,71 @@ func (s *Server) reserveWorstCaseGraph() error {ctx := s.model.Backend().NewContext()defer ctx.Close()+ var err error+ inputs := make([]input.Input, s.batchSize)+ mmStore := newMultimodalStore()++ // Multimodal strategy:+ // - Encode a 2048x2048 image. This assumes that a single image of this+ // size is sufficient to trigger the worst case. This is currently true+ // because for existing models, only a single image fits in a batch.+ // - Add the embedding to a full batch of tokens - this is necessary because+ // the model may be looking for non-image data, such as <image> tags.+ // - Run PostTokenize to execute any transformations between generated+ // embeddings and what the forward pass expects.+ // - The result may now be larger than a batch (images may not fit in a+ // single batch), so trim based on what will fit and must be grouped together.+ // - Fill out the rest of the space with text tokens.+ if multimodalProcessor, ok := s.model.(model.MultimodalProcessor); ok {+ mmCtx := s.model.Backend().NewContext()+ defer mmCtx.Close()++ img := image.NewGray(image.Rect(0, 0, 2048, 2048))+ var buf bytes.Buffer+ bmp.Encode(&buf, img)++ if inputs[0].Multimodal, err = multimodalProcessor.EncodeMultimodal(mmCtx, buf.Bytes()); err == nil {+ mmStore.addMultimodal(inputs[0].Multimodal)++ inputs, err = multimodalProcessor.PostTokenize(inputs)+ if err != nil {+ return err+ }++ for i, inp := range inputs {+ minBatch := 1 + inp.SameBatch+ if minBatch > s.batchSize {+ inputs = inputs[i:min(i+minBatch, len(inputs))]+ break+ } else if i+minBatch > s.batchSize {+ inputs = inputs[:i]+ break+ }+ }++ if len(inputs) < s.batchSize {+ newInputs := make([]input.Input, s.batchSize)+ copy(newInputs, inputs)+ inputs = newInputs+ }+ }+ }+var batch input.Batch- inputs := make([]int32, s.batchSize)+ batchInputs := make([]int32, len(inputs))batch.Positions = make([]int32, len(inputs))batch.Sequences = make([]int, len(inputs))- for i := range inputs {+ for i, inp := range inputs {+ batchInputs[i] = inp.Token+ if inp.Multimodal != nil {+ mm, err := mmStore.getMultimodal(s.model.Backend(), ctx, inp.Multimodal, true)+ if err != nil {+ return err+ }+ batch.Multimodal = append(batch.Multimodal, input.MultimodalIndex{Index: i, Multimodal: mm})+ }+batch.Positions[i] = int32(i)}@@ -746,8 +808,7 @@ func (s *Server) reserveWorstCaseGraph() error {batch.Outputs[i] = int32(i)}- var err error- batch.Inputs, err = ctx.Input().FromIntSlice(inputs, len(inputs))+ batch.Inputs, err = ctx.Input().FromIntSlice(batchInputs, len(batchInputs))if err != nil {return err}