Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llm: decouple prompt caching from context shift (#16639)

Parafee41f8a48df24dc3
diff --git a/llm/llama_server.go b/llm/llama_server.go
index 85701351f..c2bb119f5 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -1395,7 +1395,7 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
lsReq := llamaServerCompletionRequest{
Prompt: prompt,
Stream: true,
- CachePrompt: req.Shift,
+ CachePrompt: true,
NPredict: req.Options.NumPredict,
NKeep: req.Options.NumKeep,
Temperature: req.Options.Temperature,
@@ -1984,7 +1984,7 @@ func (s *llamaServerRunner) llamaServerChatRequest(req ChatRequest, stream bool)
body := map[string]any{
"messages": messages,
"stream": stream,
- "cache_prompt": req.Shift,
+ "cache_prompt": true,
"n_predict": req.Options.NumPredict,
"n_keep": req.Options.NumKeep,
"temperature": req.Options.Temperature,