Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
ollamarunner: Don't truncate a SameBatch
Jesse Gross493385eb3e81
diff --git a/runner/ollamarunner/cache.go b/runner/ollamarunner/cache.goindex af48ff22e..30292f641 100644--- a/runner/ollamarunner/cache.go+++ b/runner/ollamarunner/cache.go@@ -225,6 +225,8 @@ func countCommonPrefix(a []input.Input, b []input.Input) int32 {return count}+// TODO(jessegross): If we need to reprocess the inputs we should ensure that+// we don't split up a SameBatchfunc (c *InputCache) ShiftDiscard(inputLen int32, numKeep int32) int32 {targetFree := (c.numCtx - numKeep) / 2targetFree = max(targetFree, 1)diff --git a/runner/ollamarunner/runner.go b/runner/ollamarunner/runner.goindex 458387184..f3286abae 100644--- a/runner/ollamarunner/runner.go+++ b/runner/ollamarunner/runner.go@@ -115,16 +115,41 @@ func (s *Server) NewSequence(prompt string, images []llm.ImageData, params NewSeparams.numKeep = int32(len(inputs))}- // TODO(jessegross): We should ensure that we always leave minBatch of context space to shift,- // otherwise we might truncate or split the batch against the model's wishes-// Ensure that at least 1 input can be discarded during shiftparams.numKeep = min(params.numKeep, s.cache.numCtx-1)if int32(len(inputs)) > s.cache.numCtx {discard := int32(len(inputs)) - s.cache.numCtx+ promptStart := params.numKeep + discard++ // If we need to truncate in the middle of a unbreakable batch, remove the entire batch+ sameBatch := 0+ for i, inp := range inputs {+ if sameBatch > 0 {+ sameBatch--++ if promptStart == int32(i) {+ promptStart+++ }+ } else if promptStart == int32(i) {+ break+ }++ if inp.SameBatch != 0 {+ if int32(i) < params.numKeep {+ return nil, fmt.Errorf("SameBatch may not be specified within numKeep (index: %v numKeep: %v SameBatch: %v)", i, params.numKeep, inp.SameBatch)+ }++ sameBatch = inp.SameBatch+ }+ }++ if promptStart >= int32(len(inputs)) {+ return nil, errors.New("entire prompt removed by truncation")+ }+newInputs := inputs[:params.numKeep]- newInputs = append(newInputs, inputs[params.numKeep+discard:]...)+ newInputs = append(newInputs, inputs[promptStart:]...)slog.Warn("truncating input prompt", "limit", s.cache.numCtx, "prompt", len(inputs), "keep", params.numKeep, "new", len(newInputs))inputs = newInputs