Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

ollamarunner: Base cached tokens on current prompt

Jesse Gross499ae7311fd2
diff --git a/runner/llamarunner/cache.go b/runner/llamarunner/cache.go
index 2e55b09dc..2e273e69c 100644
--- a/runner/llamarunner/cache.go
+++ b/runner/llamarunner/cache.go
@@ -104,8 +104,8 @@ func (c *InputCache) LoadCacheSlot(prompt []input, cachePrompt bool) (*InputCach
slog.Debug("loading cache slot", "id", slot.Id, "cache", len(slot.Inputs), "prompt", len(prompt),
"used", numPast, "remaining", len(prompt)-numPast)
+ slot.Inputs = prompt[:numPast]
prompt = prompt[numPast:]
- slot.Inputs = slot.Inputs[:numPast]
return slot, prompt, nil
}
diff --git a/runner/ollamarunner/cache.go b/runner/ollamarunner/cache.go
index 2138d7988..43880a41b 100644
--- a/runner/ollamarunner/cache.go
+++ b/runner/ollamarunner/cache.go
@@ -136,8 +136,8 @@ func (c *InputCache) LoadCacheSlot(prompt []input.Input) (*InputCacheSlot, []inp
slog.Debug("loading cache slot", "id", slot.Id, "cache", len(slot.Inputs), "prompt", len(prompt),
"used", numPast, "remaining", int32(len(prompt))-numPast)
+ slot.Inputs = prompt[:numPast]
prompt = prompt[numPast:]
- slot.Inputs = slot.Inputs[:numPast]
return slot, prompt, nil
}