Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Allow models to force a new batch
Jesse Gross06007c0a187f
diff --git a/model/input/input.go b/model/input/input.goindex 0cb3f3f41..a1247bca7 100644--- a/model/input/input.go+++ b/model/input/input.go@@ -15,6 +15,12 @@ type Input struct {// stored in Multimodal, used for caching and comparing// equality.MultimodalHash uint64++ // BatchBreak forces a new batch to be started with this+ // input. For example, this can be used to align images+ // with batches. Note that batches may be divided in additional+ // locations as well.+ BatchBreak bool}// MultimodalIndex is a multimodal element (such as an image)diff --git a/model/models/gemma3/model.go b/model/models/gemma3/model.goindex 7418bb12f..2fe04348d 100644--- a/model/models/gemma3/model.go+++ b/model/models/gemma3/model.go@@ -112,8 +112,8 @@ func (m *Model) PostTokenize(ctx ml.Context, inputs []input.Input) ([]input.Inpuresult = append(result, inp)} else {imageInputs := []input.Input{- {Token: 108}, // "\n\n"- {Token: 255999}, // "<start_of_image>""+ {Token: 108}, // "\n\n"+ {Token: 255999, BatchBreak: true}, // "<start_of_image>""}result = append(result, imageInputs...)diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.goindex c1475cbb2..9b997bd37 100644--- a/runner/ollamarunner/runner.go+++ b/runner/ollamarunner/runner.go@@ -363,7 +363,7 @@ func (s *Server) processBatch() error {}}- if j >= s.batchSize {+ if j >= s.batchSize || (inp.BatchBreak && len(seq.pendingInputs) != 0) {break}diff --git a/server/prompt.go b/server/prompt.goindex d053f2a8d..5b5b958f1 100644--- a/server/prompt.go+++ b/server/prompt.go@@ -26,7 +26,6 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.var system []api.MessageisMllama := checkMllamaModelFamily(m)- isGemma3 := checkGemma3ModelFamily(m)var imageNumTokens int// TODO: Ideally we would compute this from the projector metadata but some pieces are implementation dependent@@ -41,7 +40,7 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.n := len(msgs) - 1// in reverse, find all messages that fit into context windowfor i := n; i >= 0; i-- {- if (isMllama || isGemma3) && len(msgs[i].Images) > 1 {+ if isMllama && len(msgs[i].Images) > 1 {return "", nil, errTooManyImages}@@ -158,12 +157,3 @@ func checkMllamaModelFamily(m *Model) bool {}return false}--func checkGemma3ModelFamily(m *Model) bool {- for _, arch := range m.Config.ModelFamilies {- if arch == "gemma3" {- return true- }- }- return false-}