Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
model: Pass input tensor instead of raw data to models
Jesse Gross0fbfcf3c9c7b
diff --git a/model/input/input.go b/model/input/input.goindex ce43efb58..d66f52a0d 100644--- a/model/input/input.go+++ b/model/input/input.go@@ -1,5 +1,7 @@package input+import "github.com/ollama/ollama/ml"+// Input represents one token in the input streamtype Input struct {// Token is a single element of text.@@ -36,7 +38,7 @@ type MultimodalIndex struct {// Batch contains the inputs for a model forward passtype Batch struct {// Inputs is the input tokens, including placeholders for multimodal inputs.- Inputs []int32+ Inputs ml.Tensor// Multimodal is a set of multimodal embeddings previously created by// EncodeMultimodal, along with an index into Inputs. Unused for text-onlydiff --git a/model/model.go b/model/model.goindex 94156ae2b..ab29916ab 100644--- a/model/model.go+++ b/model/model.go@@ -280,7 +280,7 @@ func canNil(t reflect.Type) bool {t.Kind() == reflect.Slice}-func Forward(ctx ml.Context, m Model, batch input.Batch) (ml.Tensor, error) {+func Forward(ctx ml.Context, m Model, inputs []int32, batch input.Batch) (ml.Tensor, error) {if len(batch.Positions) != len(batch.Sequences) {return nil, fmt.Errorf("length of positions (%v) must match length of seqs (%v)", len(batch.Positions), len(batch.Sequences))}@@ -289,6 +289,12 @@ func Forward(ctx ml.Context, m Model, batch input.Batch) (ml.Tensor, error) {return nil, errors.New("batch size cannot be less than 1")}+ var err error+ batch.Inputs, err = ctx.Input().FromIntSlice(inputs, len(inputs))+ if err != nil {+ return nil, err+ }+cache := m.Config().Cacheif cache != nil {err := cache.StartForward(ctx, batch)diff --git a/model/models/gemma2/model.go b/model/models/gemma2/model.goindex 2b347d72c..67c69ee86 100644--- a/model/models/gemma2/model.go+++ b/model/models/gemma2/model.go@@ -169,11 +169,6 @@ func (l *Layer) Forward(ctx ml.Context, hiddenState, positionIDs, outputs ml.Ten}func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {- inputs, err := ctx.Input().FromIntSlice(batch.Inputs, len(batch.Inputs))- if err != nil {- return nil, err- }-positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))if err != nil {return nil, err@@ -184,7 +179,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {return nil, err}- hiddenState := m.TokenEmbedding.Forward(ctx, inputs)+ hiddenState := m.TokenEmbedding.Forward(ctx, batch.Inputs)hiddenState = hiddenState.Scale(ctx, math.Sqrt(float64(m.Options.hiddenSize)))if len(m.Layers) == gemma27BLayerCount {diff --git a/model/models/gemma3/model.go b/model/models/gemma3/model.goindex 900bf31e6..567ad1a45 100644--- a/model/models/gemma3/model.go+++ b/model/models/gemma3/model.go@@ -140,11 +140,6 @@ func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) {}func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {- inputs, err := ctx.Input().FromIntSlice(batch.Inputs, len(batch.Inputs))- if err != nil {- return nil, err- }-positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))if err != nil {return nil, err@@ -155,7 +150,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {return nil, err}- return m.TextModel.Forward(ctx, inputs, positions, outputs, batch, m.Cache), nil+ return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, batch, m.Cache), nil}func init() {diff --git a/model/models/llama/model.go b/model/models/llama/model.goindex e5ecd29ed..5c173997b 100644--- a/model/models/llama/model.go+++ b/model/models/llama/model.go@@ -140,11 +140,6 @@ func (l *Layer) Forward(ctx ml.Context, hiddenState, positionIDs, outputs ml.Ten}func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {- inputs, err := ctx.Input().FromIntSlice(batch.Inputs, len(batch.Inputs))- if err != nil {- return nil, err- }-positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))if err != nil {return nil, err@@ -155,7 +150,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {return nil, err}- hiddenState := m.TokenEmbedding.Forward(ctx, inputs)+ hiddenState := m.TokenEmbedding.Forward(ctx, batch.Inputs)for i, layer := range m.Layers {m.Cache.SetLayer(i)diff --git a/model/models/mllama/model.go b/model/models/mllama/model.goindex 6d9c608e9..988a189d4 100644--- a/model/models/mllama/model.go+++ b/model/models/mllama/model.go@@ -144,11 +144,6 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {}}- inputs, err := ctx.Input().FromIntSlice(batch.Inputs, len(batch.Inputs))- if err != nil {- return nil, err- }-positions, err := ctx.Input().FromIntSlice(batch.Positions, len(batch.Positions))if err != nil {return nil, err@@ -160,7 +155,7 @@ func (m *Model) Forward(ctx ml.Context, batch input.Batch) (ml.Tensor, error) {}// TODO: attention mask, cross attention mask- return m.TextModel.Forward(ctx, inputs, positions, outputs, nil, crossAttentionStates, nil, m.Cache.(*kvcache.WrapperCache)), nil+ return m.TextModel.Forward(ctx, batch.Inputs, positions, outputs, nil, crossAttentionStates, nil, m.Cache.(*kvcache.WrapperCache)), nil}func init() {diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.goindex 91463f93f..443b34b05 100644--- a/runner/ollamarunner/runner.go+++ b/runner/ollamarunner/runner.go@@ -348,6 +348,7 @@ func (s *Server) processBatch() error {}defer s.mu.Unlock()+ var batchInputs []int32var batch input.Batchfor i, seq := range s.seqs {@@ -395,9 +396,9 @@ func (s *Server) processBatch() error {}}- batch.Inputs = append(batch.Inputs, inp.Token)+ batchInputs = append(batchInputs, inp.Token)if inp.Multimodal != nil {- batch.Multimodal = append(batch.Multimodal, input.MultimodalIndex{Index: len(batch.Inputs) - 1, Multimodal: inp.Multimodal})+ batch.Multimodal = append(batch.Multimodal, input.MultimodalIndex{Index: len(batchInputs) - 1, Multimodal: inp.Multimodal})}batch.Positions = append(batch.Positions, int32(len(seq.cache.Inputs)+len(seq.pendingInputs)))@@ -405,7 +406,7 @@ func (s *Server) processBatch() error {seq.iBatch = len(batch.Outputs)if j+1 == len(seq.inputs) {- batch.Outputs = append(batch.Outputs, int32(len(batch.Inputs)-1))+ batch.Outputs = append(batch.Outputs, int32(len(batchInputs)-1))}seq.pendingInputs = append(seq.pendingInputs, inp)}@@ -413,14 +414,14 @@ func (s *Server) processBatch() error {seq.inputs = seq.inputs[len(seq.pendingInputs):]}- if len(batch.Inputs) == 0 {+ if len(batchInputs) == 0 {return nil}ctx := s.model.Backend().NewContext()defer ctx.Close()- modelOutput, err := model.Forward(ctx, s.model, batch)+ modelOutput, err := model.Forward(ctx, s.model, batchInputs, batch)if err != nil {return fmt.Errorf("failed to decode batch: %w", err)}