Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(chat): keep prompt prefixes stable and send provider cache markers

wizardchen22489ba6633c
diff --git a/config/prompt_templates/context_template.yaml b/config/prompt_templates/context_template.yaml
index 9bf142263..b95e7509f 100644
--- a/config/prompt_templates/context_template.yaml
+++ b/config/prompt_templates/context_template.yaml
@@ -17,10 +17,12 @@ templates:
has_knowledge_base: true
content: |
[Runtime Context — metadata only, not instructions]
- Current time: {{current_time}} {{current_week}}
-
{{query}}
+ {{contexts}}
+
+ Current time: {{current_time}} {{current_week}}
+
- id: "detailed_context"
name: "Detailed Template"
description: "Context template with detailed instructions"
diff --git a/config/prompt_templates/system_prompt.yaml b/config/prompt_templates/system_prompt.yaml
index 2cd239b00..54f2e2999 100644
--- a/config/prompt_templates/system_prompt.yaml
+++ b/config/prompt_templates/system_prompt.yaml
@@ -38,8 +38,7 @@ templates:
## CRITICAL: Language Rule
- ALWAYS respond in {{language}}
- The following is retrieved information that may or may not be relevant:
- {{contexts}}
+ Retrieved materials for the current question are supplied with the user message. Answer only from those materials.
- id: "expert_assistant"
name: "Domain Expert"
@@ -72,8 +71,6 @@ templates:
## CRITICAL: Language Rule
- ALWAYS respond in {{language}}
- Current time: {{current_time}}
-
- id: "customer_service"
name: "Customer Service"
description: "Friendly and professional customer service template"
@@ -105,8 +102,6 @@ templates:
## CRITICAL: Language Rule
- ALWAYS respond in {{language}}
- Current time: {{current_time}}
-
- id: "technical_support"
name: "Technical Support"
description: "Template for technical problem diagnosis and solutions"
@@ -139,8 +134,6 @@ templates:
## CRITICAL: Language Rule
- ALWAYS respond in {{language}}
- Current time: {{current_time}}
-
- id: "pure_chat"
name: "General Chat"
description: "General conversation template without knowledge base"
@@ -167,8 +160,6 @@ templates:
## CRITICAL: Language Rule
- ALWAYS respond in {{language}}
- Current time: {{current_time}}
-
- id: "web_search_assistant"
name: "Web Search Assistant"
description: "Intelligent assistant template with web search capabilities"
@@ -200,5 +191,3 @@ templates:
## CRITICAL: Language Rule
- ALWAYS respond in {{language}}
-
- Current time: {{current_time}}
diff --git a/frontend/src/composables/useChatStreamHandler.ts b/frontend/src/composables/useChatStreamHandler.ts
index 46bd29b69..fed92c356 100644
--- a/frontend/src/composables/useChatStreamHandler.ts
+++ b/frontend/src/composables/useChatStreamHandler.ts
@@ -300,6 +300,7 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
isCompleted = false,
isFallback = false,
agentDurationMs = 0,
+ usage?: unknown,
) => {
const events: ChatMessage[] = []
@@ -362,10 +363,11 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
})
}
- if (agentDurationMs > 0) {
+ if (agentDurationMs > 0 || usage) {
events.push({
type: 'agent_complete',
total_duration_ms: agentDurationMs,
+ usage,
})
}
@@ -424,6 +426,7 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
Boolean(item.is_completed),
Boolean(item.is_fallback),
Number(item.agent_duration_ms) || 0,
+ item.usage,
),
)
item.hideContent = true
@@ -900,11 +903,16 @@ export function useChatStreamHandler(options: UseChatStreamHandlerOptions) {
message.artifacts = streamedArtifacts
}
message.artifactsCollecting = false
+ const usage = (dataPayload as any)?.usage || (data as any).usage
+ if (usage) {
+ message.usage = usage
+ }
if (message.agentEventStream) {
;(message.agentEventStream as ChatMessage[]).push({
type: 'agent_complete',
total_duration_ms: dataPayload?.total_duration_ms || 0,
total_steps: dataPayload?.total_steps || 0,
+ usage,
})
}
break
diff --git a/internal/agent/compaction/compactor.go b/internal/agent/compaction/compactor.go
index c54b86e88..5cf174afd 100644
--- a/internal/agent/compaction/compactor.go
+++ b/internal/agent/compaction/compactor.go
@@ -180,8 +180,9 @@ func (c *Compactor) summarize(
{Role: "system", Content: summarizationSystemPrompt},
{Role: "user", Content: prompt},
}, &chat.ChatOptions{
- Temperature: 0.3, // low temperature for factual summarization
- MaxTokens: maxTokens,
+ Temperature: 0.3, // low temperature for factual summarization
+ MaxTokens: maxTokens,
+ CacheRetention: chat.CacheRetentionNone,
})
cancel()
diff --git a/internal/agent/engine.go b/internal/agent/engine.go
index 40f81a103..c8134d675 100644
--- a/internal/agent/engine.go
+++ b/internal/agent/engine.go
@@ -651,9 +651,12 @@ func (e *AgentEngine) runReActIteration(
if response.Usage.TotalTokens > 0 {
e.lastUsage = response.Usage
state.TurnUsage.Accumulate(response.Usage)
- logger.Debugf(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d",
+ logger.Infof(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d, "+
+ "cache_read=%d, cache_write=%d, cache_hit_rate=%.1f%%, cache_status=%s",
round, response.Usage.PromptTokens,
- response.Usage.CompletionTokens, response.Usage.TotalTokens)
+ response.Usage.CompletionTokens, response.Usage.TotalTokens,
+ response.Usage.CacheReadTokens, response.Usage.CacheWriteTokens,
+ response.Usage.PromptCacheHitRate(), response.Usage.CacheStatus)
}
// Detect stuck loops: if the LLM keeps returning the same content
diff --git a/internal/agent/finalize.go b/internal/agent/finalize.go
index 72c2d19a3..fcde636d6 100644
--- a/internal/agent/finalize.go
+++ b/internal/agent/finalize.go
@@ -105,6 +105,7 @@ Now generate the final answer:`, query, imageRequirement)
Temperature: e.config.Temperature,
MaxTokens: budget,
MaxCompletionTokens: budget,
+ PromptCacheKey: sessionID,
}, // Thinking disabled for final answer synthesis
func(chunk *types.StreamResponse, fullContent string) {
// Defensive filter: only emit answer content, skip thinking chunks
diff --git a/internal/agent/image_requirement.go b/internal/agent/image_requirement.go
index 9c9723ab0..b32a80296 100644
--- a/internal/agent/image_requirement.go
+++ b/internal/agent/image_requirement.go
@@ -33,14 +33,16 @@ func stepContainsMarkdownImage(step types.AgentStep) bool {
}
func appendAgentRetrievedImageRequirement(messages []chat.Message) []chat.Message {
- for i := range messages {
- if messages[i].Role != "system" {
- continue
+ for _, message := range messages {
+ if strings.Contains(message.Content, agentRetrievedImageRequirementMarker) {
+ return messages
}
- if !strings.Contains(messages[i].Content, agentRetrievedImageRequirementMarker) {
- messages[i].Content = strings.TrimRight(messages[i].Content, " \t\r\n") + agentRetrievedImageSystemRequirement
- }
- break
}
- return messages
+ // Append after the current prefix instead of editing the system prompt.
+ // Provider prompt caches are prefix matches: mutating the system block
+ // invalidates the tools and the whole transcript for every later round.
+ return append(messages, chat.Message{
+ Role: "user",
+ Content: strings.TrimSpace(agentRetrievedImageSystemRequirement),
+ })
}
diff --git a/internal/agent/observe.go b/internal/agent/observe.go
index fd3c004a1..7e3840561 100644
--- a/internal/agent/observe.go
+++ b/internal/agent/observe.go
@@ -506,7 +506,7 @@ func buildRuntimeContextBlock(
) string {
var sb strings.Builder
sb.WriteString("<runtime_context scope=\"this_turn\">\n")
- fmt.Fprintf(&sb, " <current_time>%s</current_time>\n", time.Now().Format(time.RFC3339))
+ fmt.Fprintf(&sb, " <current_time>%s</current_time>\n", time.Now().Format("2006-01-02"))
fmt.Fprintf(&sb, " <session>%s</session>\n", escapeXMLAttr(sessionID))
if len(kbs) > 0 {
@@ -742,13 +742,6 @@ func (e *AgentEngine) appendToolResults(
messages []chat.Message,
step types.AgentStep,
) []chat.Message {
- if stepContainsMarkdownImage(step) {
- // Keep the requirement at system priority even when a custom Agent prompt
- // replaces the built-in template. Appending it once, when image-bearing
- // evidence first appears, also avoids burdening text-only turns.
- messages = appendAgentRetrievedImageRequirement(messages)
- }
-
// Add assistant message with tool calls (if any)
if step.Thought != "" || len(step.ToolCalls) > 0 || step.ReasoningContent != "" {
assistantMsg := chat.Message{
@@ -793,6 +786,13 @@ func (e *AgentEngine) appendToolResults(
messages = append(messages, toolMsg)
}
+ if stepContainsMarkdownImage(step) {
+ // Keep the requirement at the end of the current prefix. Editing the
+ // system prompt would invalidate provider prefix cache for tools and
+ // the whole transcript on every later round of this turn.
+ messages = appendAgentRetrievedImageRequirement(messages)
+ }
+
return messages
}
diff --git a/internal/agent/observe_test.go b/internal/agent/observe_test.go
index 2c9949798..cbe964ac6 100644
--- a/internal/agent/observe_test.go
+++ b/internal/agent/observe_test.go
@@ -235,17 +235,27 @@ func TestAppendToolResults_AddsDynamicImageRequirementToCustomSystemPrompt(t *te
}
out := engine.appendToolResults(prior, step)
- require.Len(t, out, 4)
- assert.Contains(t, out[0].Content, "Custom agent prompt.")
- assert.Contains(t, out[0].Content, agentRetrievedImageRequirementMarker)
- assert.Contains(t, out[0].Content, "MUST include at least one relevant Markdown image")
- assert.Contains(t, out[0].Content, "ASCII half-width parentheses")
+ require.Len(t, out, 5)
+ assert.Equal(t, "Custom agent prompt.", out[0].Content)
+ assert.NotContains(t, out[0].Content, agentRetrievedImageRequirementMarker)
assert.Equal(t, "tool", out[3].Role)
assert.Contains(t, out[3].Content, "![流程图](resource://AbCdEfGhIjKlMnOpQrStUv)")
+ assert.Equal(t, "user", out[4].Role)
+ assert.Contains(t, out[4].Content, agentRetrievedImageRequirementMarker)
+ assert.Contains(t, out[4].Content, "MUST include at least one relevant Markdown image")
+ assert.Contains(t, out[4].Content, "ASCII half-width parentheses")
- // A later image-bearing step must not duplicate the system requirement.
+ // A later image-bearing step must not duplicate the requirement.
out = engine.appendToolResults(out, step)
- assert.Equal(t, 1, strings.Count(out[0].Content, agentRetrievedImageRequirementMarker))
+ assert.Equal(t, 1, countImageRequirementMarkers(out))
+}
+
+func countImageRequirementMarkers(messages []chat.Message) int {
+ n := 0
+ for _, message := range messages {
+ n += strings.Count(message.Content, agentRetrievedImageRequirementMarker)
+ }
+ return n
}
func TestBuildRuntimeContextBlock_PinnedDocuments(t *testing.T) {
diff --git a/internal/agent/prompts.go b/internal/agent/prompts.go
index 088988f8a..6b56359db 100644
--- a/internal/agent/prompts.go
+++ b/internal/agent/prompts.go
@@ -398,7 +398,7 @@ func BuildSystemPromptWithOptions(
template = GetProgressiveRAGSystemPrompt(cfg)
}
- currentTime := time.Now().Format(time.RFC3339)
+ currentTime := time.Now().Format("2006-01-02")
language := ""
if options != nil {
language = options.Language
diff --git a/internal/agent/think.go b/internal/agent/think.go
index 36bee58a8..ff2a0722c 100644
--- a/internal/agent/think.go
+++ b/internal/agent/think.go
@@ -257,6 +257,7 @@ func (e *AgentEngine) streamThinkingToEventBus(
Tools: tools,
Thinking: e.config.Thinking,
ParallelToolCalls: &parallelToolCalls,
+ PromptCacheKey: sessionID,
}
pendingToolCalls := make(map[string]bool)
diff --git a/internal/application/service/chat_pipeline/common.go b/internal/application/service/chat_pipeline/common.go
index 2c96056a6..ed31827a4 100644
--- a/internal/application/service/chat_pipeline/common.go
+++ b/internal/application/service/chat_pipeline/common.go
@@ -71,6 +71,7 @@ func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
FrequencyPenalty: chatManage.SummaryConfig.FrequencyPenalty,
PresencePenalty: chatManage.SummaryConfig.PresencePenalty,
Thinking: chatManage.SummaryConfig.Thinking,
+ PromptCacheKey: chatManage.SessionID,
}
if opt.Thinking != nil {
pipelineInfo(ctx, "Stream", "thinking_option", map[string]interface{}{
@@ -94,7 +95,6 @@ func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
"language": chatManage.Language,
"contexts": chatManage.RenderedContexts,
})
- systemPrompt = appendRetrievedImageOutputRequirement(systemPrompt, chatManage.RenderedContexts)
// Memory goes at the end of the system prompt, after the retrieved-context
// placeholders have been rendered, so a remembered sentence can never be
// substituted into prompt structure.
@@ -106,9 +106,14 @@ func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
chatMessages = AppendHistoryMessages(chatMessages, chatManage.History)
+ // Image-output rules are turn-specific. Putting them on the current user
+ // message keeps the system prefix byte-stable so provider prompt caches
+ // still hit on later turns of the same session.
+ userContent := appendRetrievedImageOutputRequirement(chatManage.UserContent, chatManage.RenderedContexts)
+
// Add current user message. Only include images when the chat model supports
// vision; non-vision models rely on the text description in UserContent.
- userMsg := chat.Message{Role: "user", Content: chatManage.UserContent}
+ userMsg := chat.Message{Role: "user", Content: userContent}
if chatManage.ChatModelSupportsVision && len(chatManage.Images) > 0 {
userMsg.Images = chatManage.Images
}
diff --git a/internal/application/service/chat_pipeline/common_image_requirement_test.go b/internal/application/service/chat_pipeline/common_image_requirement_test.go
index 26e724a60..b3481119c 100644
--- a/internal/application/service/chat_pipeline/common_image_requirement_test.go
+++ b/internal/application/service/chat_pipeline/common_image_requirement_test.go
@@ -25,6 +25,6 @@ func TestAppendRetrievedImageOutputRequirement(t *testing.T) {
withoutImage := appendRetrievedImageOutputRequirement(base, "text-only retrieved context")
if withoutImage != base {
- t.Fatalf("text-only context should not change the system prompt: %q", withoutImage)
+ t.Fatalf("text-only context should not change the user turn: %q", withoutImage)
}
}
diff --git a/internal/models/chat/anthropic.go b/internal/models/chat/anthropic.go
index 5da902f14..1690f357e 100644
--- a/internal/models/chat/anthropic.go
+++ b/internal/models/chat/anthropic.go
@@ -25,16 +25,27 @@ type AnthropicChat struct {
customHeaders map[string]string
}
+type anthropicCacheControl struct {
+ Type string `json:"type"`
+ TTL string `json:"ttl,omitempty"`
+}
+
+type anthropicContentBlock struct {
+ Type string `json:"type"`
+ Text string `json:"text,omitempty"`
+ CacheControl *anthropicCacheControl `json:"cache_control,omitempty"`
+}
+
type anthropicMessage struct {
Role string `json:"role"`
- Content string `json:"content"`
+ Content any `json:"content"`
}
type anthropicRequest struct {
Model string `json:"model"`
MaxTokens int `json:"max_tokens"`
Stream bool `json:"stream,omitempty"`
- System string `json:"system,omitempty"`
+ System any `json:"system,omitempty"`
Messages []anthropicMessage `json:"messages"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
@@ -113,7 +124,7 @@ func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
}
func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
- reqBody := c.buildRequest(messages, opts)
+ reqBody := c.buildRequest(ctx, messages, opts)
jsonData, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
@@ -176,7 +187,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
}
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
- reqBody := c.buildRequest(messages, opts)
+ reqBody := c.buildRequest(ctx, messages, opts)
reqBody.Stream = true
jsonData, err := json.Marshal(reqBody)
if err != nil {
@@ -250,7 +261,7 @@ func isAnthropicVersionedBaseURL(baseURL string) bool {
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
}
-func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest {
+func (c *AnthropicChat) buildRequest(_ context.Context, messages []Message, opts *ChatOptions) anthropicRequest {
req := anthropicRequest{
Model: c.modelName,
MaxTokens: 1024,
@@ -292,7 +303,30 @@ func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anth
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
}
}
- req.System = strings.Join(systemParts, "\n\n")
+ systemText := strings.Join(systemParts, "\n\n")
+ retention := resolveCacheRetention(opts)
+ marker := cacheControlFor(retention, "1h")
+ if marker == nil {
+ req.System = systemText
+ return req
+ }
+ if systemText != "" {
+ req.System = []anthropicContentBlock{{
+ Type: "text",
+ Text: systemText,
+ CacheControl: &anthropicCacheControl{Type: marker.Type, TTL: marker.TTL},
+ }}
+ }
+ if len(req.Messages) > 0 {
+ last := &req.Messages[len(req.Messages)-1]
+ if text, ok := last.Content.(string); ok && text != "" {
+ last.Content = []anthropicContentBlock{{
+ Type: "text",
+ Text: text,
+ CacheControl: &anthropicCacheControl{Type: marker.Type, TTL: marker.TTL},
+ }}
+ }
+ }
return req
}
diff --git a/internal/models/chat/anthropic_test.go b/internal/models/chat/anthropic_test.go
index 1651caa15..5cb46a62c 100644
--- a/internal/models/chat/anthropic_test.go
+++ b/internal/models/chat/anthropic_test.go
@@ -59,10 +59,14 @@ func TestAnthropicChat(t *testing.T) {
assert.Equal(t, "test-beta", capturedHeaders.Get("anthropic-beta"))
assert.Equal(t, "claude-sonnet-4-5", capturedRequest.Model)
assert.Equal(t, 7, capturedRequest.MaxTokens)
- assert.Equal(t, "You are helpful.", capturedRequest.System)
+ systemText, systemCache := firstTextAndCache(t, capturedRequest.System)
+ assert.Equal(t, "You are helpful.", systemText)
+ require.Equal(t, "ephemeral", systemCache["type"])
require.Len(t, capturedRequest.Messages, 1)
assert.Equal(t, "user", capturedRequest.Messages[0].Role)
- assert.Equal(t, "Hi", capturedRequest.Messages[0].Content)
+ userText, userCache := firstTextAndCache(t, capturedRequest.Messages[0].Content)
+ assert.Equal(t, "Hi", userText)
+ require.Equal(t, "ephemeral", userCache["type"])
assert.Equal(t, "hello", resp.Content)
assert.Equal(t, "end_turn", resp.FinishReason)
assert.Equal(t, 3, resp.Usage.PromptTokens)
@@ -275,3 +279,53 @@ func TestNewRemoteChat_AnthropicProvider(t *testing.T) {
_, ok := chat.(*AnthropicChat)
assert.True(t, ok)
}
+
+func TestAnthropicChat_CacheRetentionNoneKeepsPlainStrings(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ var capturedRequest anthropicRequest
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest))
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{
+ "id":"msg_123","type":"message","role":"assistant",
+ "content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn",
+ "usage":{"input_tokens":3,"output_tokens":2}
+ }`))
+ }))
+ defer server.Close()
+
+ chat, err := NewAnthropicChat(&ChatConfig{
+ Source: types.ModelSourceRemote, BaseURL: server.URL, ModelName: "claude-sonnet-4-5",
+ APIKey: "test-key", Provider: string(provider.ProviderAnthropic),
+ })
+ require.NoError(t, err)
+
+ _, err = chat.Chat(context.Background(), []Message{
+ {Role: "system", Content: "You are helpful."},
+ {Role: "user", Content: "Hi"},
+ }, &ChatOptions{CacheRetention: CacheRetentionNone})
+ require.NoError(t, err)
+
+ assert.Equal(t, "You are helpful.", capturedRequest.System)
+ assert.Equal(t, "Hi", capturedRequest.Messages[0].Content)
+}
+
+func firstTextAndCache(t *testing.T, v any) (string, map[string]any) {
+ t.Helper()
+ switch x := v.(type) {
+ case string:
+ t.Fatalf("expected cache_control content blocks, got plain string %q", x)
+ return "", nil
+ case []any:
+ require.NotEmpty(t, x)
+ block, ok := x[0].(map[string]any)
+ require.True(t, ok)
+ text, _ := block["text"].(string)
+ cache, _ := block["cache_control"].(map[string]any)
+ require.NotNil(t, cache)
+ return text, cache
+ default:
+ t.Fatalf("unexpected content type %T", v)
+ return "", nil
+ }
+}
diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go
index b85c29d6b..48c85371a 100644
--- a/internal/models/chat/chat.go
+++ b/internal/models/chat/chat.go
@@ -38,6 +38,13 @@ type ChatOptions struct {
ToolChoice string `json:"tool_choice,omitempty"` // "auto", "required", "none", or specific tool
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` // 是否允许并行工具调用(默认 nil 表示由模型决定)
Format json.RawMessage `json:"format,omitempty"` // 响应格式定义
+ // PromptCacheKey is the provider routing key (OpenAI prompt_cache_key).
+ // Empty falls back to the session ID on the call context.
+ PromptCacheKey string `json:"-"`
+ // CacheRetention controls provider prompt-cache TTL. none disables cache
+ // markers; empty/short is the default 5-minute cache; long requests 1h/24h
+ // where the provider accepts it.
+ CacheRetention CacheRetention `json:"-"`
}
// MessageContentPart represents a part of multi-content message
diff --git a/internal/models/chat/prompt_cache.go b/internal/models/chat/prompt_cache.go
index b6d48c736..0938ab3c7 100644
--- a/internal/models/chat/prompt_cache.go
+++ b/internal/models/chat/prompt_cache.go
@@ -1,10 +1,13 @@
package chat
import (
+ "context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
+ "net/http"
+ "strings"
"github.com/Tencent/WeKnora/internal/models/provider"
"github.com/Tencent/WeKnora/internal/types"
@@ -142,3 +145,230 @@ func valueOrZero(value *int) int {
}
return *value
}
+
+// CacheRetention is the prompt-cache TTL preference. Empty means short (the
+// default 5-minute provider cache). Compaction/summarization uses none so a
+// different prompt prefix does not occupy the session's cache slot.
+type CacheRetention string
+
+const (
+ CacheRetentionNone CacheRetention = "none"
+ CacheRetentionShort CacheRetention = "short"
+ CacheRetentionLong CacheRetention = "long"
+)
+
+const openAIPromptCacheKeyMaxLength = 64
+
+func clampPromptCacheKey(key string) string {
+ if key == "" {
+ return ""
+ }
+ runes := []rune(key)
+ if len(runes) <= openAIPromptCacheKeyMaxLength {
+ return key
+ }
+ return string(runes[:openAIPromptCacheKeyMaxLength])
+}
+
+func resolveCacheRetention(opts *ChatOptions) CacheRetention {
+ if opts != nil && opts.CacheRetention != "" {
+ return opts.CacheRetention
+ }
+ return CacheRetentionShort
+}
+
+func promptCacheSessionID(ctx context.Context, opts *ChatOptions) string {
+ if opts != nil && opts.PromptCacheKey != "" {
+ return clampPromptCacheKey(opts.PromptCacheKey)
+ }
+ if sessionID, ok := types.SessionIDFromContext(ctx); ok {
+ return clampPromptCacheKey(sessionID)
+ }
+ return ""
+}
+
+type promptCachePolicy struct {
+ sendKey bool
+ sendCacheControl bool
+ sendAffinity bool
+}
+
+func promptCachePolicyFor(name provider.ProviderName, baseURL string) promptCachePolicy {
+ switch name {
+ case provider.ProviderOpenAI, provider.ProviderAzureOpenAI, provider.ProviderOpenRouter:
+ return promptCachePolicy{sendKey: true, sendAffinity: true}
+ case provider.ProviderAliyun:
+ return promptCachePolicy{sendCacheControl: true}
+ case provider.ProviderAnthropic:
+ return promptCachePolicy{sendCacheControl: true}
+ }
+ if strings.Contains(baseURL, "api.openai.com") {
+ return promptCachePolicy{sendKey: true, sendAffinity: true}
+ }
+ return promptCachePolicy{}
+}
+
+type cacheControlMarker struct {
+ Type string `json:"type"`
+ TTL string `json:"ttl,omitempty"`
+}
+
+func cacheControlFor(retention CacheRetention, longTTL string) *cacheControlMarker {
+ if retention == CacheRetentionNone {
+ return nil
+ }
+ marker := &cacheControlMarker{Type: "ephemeral"}
+ if retention == CacheRetentionLong && longTTL != "" {
+ marker.TTL = longTTL
+ }
+ return marker
+}
+
+// applyPromptCacheToJSONBody injects provider cache routing and breakpoints
+// into an already-shaped OpenAI-compatible request object. Returns the (possibly
+// rewritten) body and whether the caller must send it via raw HTTP because the
+// SDK struct cannot carry these fields.
+func applyPromptCacheToJSONBody(
+ body any,
+ policy promptCachePolicy,
+ sessionID string,
+ retention CacheRetention,
+) (any, bool, error) {
+ if retention == CacheRetentionNone {
+ return body, false, nil
+ }
+ if !policy.sendKey && !policy.sendCacheControl {
+ return body, false, nil
+ }
+
+ data, err := json.Marshal(body)
+ if err != nil {
+ return nil, false, err
+ }
+ var payload map[string]any
+ if err := json.Unmarshal(data, &payload); err != nil {
+ return nil, false, err
+ }
+
+ rewritten := false
+ if policy.sendKey && sessionID != "" {
+ payload["prompt_cache_key"] = sessionID
+ if retention == CacheRetentionLong {
+ payload["prompt_cache_retention"] = "24h"
+ }
+ rewritten = true
+ }
+ if policy.sendCacheControl {
+ marker := cacheControlFor(retention, "1h")
+ if marker != nil {
+ applyCacheControlBreakpoints(payload, marker)
+ rewritten = true
+ }
+ }
+ if !rewritten {
+ return body, false, nil
+ }
+ return payload, true, nil
+}
+
+func applyCacheControlBreakpoints(payload map[string]any, marker *cacheControlMarker) {
+ if marker == nil {
+ return
+ }
+ applyCacheControlToInstructionMessages(payload["messages"], marker)
+ applyCacheControlToLastTool(payload["tools"], marker)
+ applyCacheControlToLastConversationMessage(payload["messages"], marker)
+}
+
+func applyCacheControlToInstructionMessages(raw any, marker *cacheControlMarker) {
+ messages, ok := raw.([]any)
+ if !ok {
+ return
+ }
+ for _, item := range messages {
+ msg, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ role, _ := msg["role"].(string)
+ if role == "system" || role == "developer" {
+ addCacheControlToMessageContent(msg, marker)
+ return
+ }
+ }
+}
+
+func applyCacheControlToLastConversationMessage(raw any, marker *cacheControlMarker) {
+ messages, ok := raw.([]any)
+ if !ok {
+ return
+ }
+ for i := len(messages) - 1; i >= 0; i-- {
+ msg, ok := messages[i].(map[string]any)
+ if !ok {
+ continue
+ }
+ role, _ := msg["role"].(string)
+ if role == "user" || role == "assistant" || role == "tool" {
+ if addCacheControlToMessageContent(msg, marker) {
+ return
+ }
+ }
+ }
+}
+
+func applyCacheControlToLastTool(raw any, marker *cacheControlMarker) {
+ tools, ok := raw.([]any)
+ if !ok || len(tools) == 0 {
+ return
+ }
+ last, ok := tools[len(tools)-1].(map[string]any)
+ if !ok {
+ return
+ }
+ last["cache_control"] = marker
+}
+
+func addCacheControlToMessageContent(msg map[string]any, marker *cacheControlMarker) bool {
+ content, ok := msg["content"]
+ if !ok || content == nil {
+ return false
+ }
+ if text, ok := content.(string); ok {
+ if text == "" {
+ return false
+ }
+ msg["content"] = []any{
+ map[string]any{
+ "type": "text",
+ "text": text,
+ "cache_control": marker,
+ },
+ }
+ return true
+ }
+ parts, ok := content.([]any)
+ if !ok {
+ return false
+ }
+ for i := len(parts) - 1; i >= 0; i-- {
+ part, ok := parts[i].(map[string]any)
+ if !ok {
+ continue
+ }
+ if partType, _ := part["type"].(string); partType == "text" || partType == "tool_result" {
+ part["cache_control"] = marker
+ return true
+ }
+ }
+ return false
+}
+
+func attachPromptCacheHeaders(req *http.Request, policy promptCachePolicy, sessionID string) {
+ if req == nil || !policy.sendAffinity || sessionID == "" {
+ return
+ }
+ req.Header.Set("session_id", sessionID)
+ req.Header.Set("x-client-request-id", sessionID)
+ req.Header.Set("x-session-affinity", sessionID)
+}
diff --git a/internal/models/chat/prompt_cache_apply_test.go b/internal/models/chat/prompt_cache_apply_test.go
new file mode 100644
index 000000000..2b68c8598
--- /dev/null
+++ b/internal/models/chat/prompt_cache_apply_test.go
@@ -0,0 +1,117 @@
+package chat
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/provider"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/sashabaranov/go-openai"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestClampPromptCacheKey(t *testing.T) {
+ assert.Equal(t, "", clampPromptCacheKey(""))
+ assert.Equal(t, "sess-1", clampPromptCacheKey("sess-1"))
+ long := strings.Repeat("a", 80)
+ got := clampPromptCacheKey(long)
+ assert.Equal(t, 64, len([]rune(got)))
+ assert.Equal(t, strings.Repeat("a", 64), got)
+}
+
+func TestApplyPromptCacheToJSONBody_OpenAIKey(t *testing.T) {
+ req := openai.ChatCompletionRequest{
+ Model: "gpt-4o",
+ Messages: []openai.ChatCompletionMessage{
+ {Role: "system", Content: "sys"},
+ {Role: "user", Content: "hi"},
+ },
+ }
+ policy := promptCachePolicyFor(provider.ProviderOpenAI, "")
+ body, forceRaw, err := applyPromptCacheToJSONBody(req, policy, "sess-abc", CacheRetentionShort)
+ require.NoError(t, err)
+ require.True(t, forceRaw)
+ payload := body.(map[string]any)
+ assert.Equal(t, "sess-abc", payload["prompt_cache_key"])
+ assert.NotContains(t, payload, "prompt_cache_retention")
+}
+
+func TestApplyPromptCacheToJSONBody_OpenAILongRetention(t *testing.T) {
+ req := openai.ChatCompletionRequest{Model: "gpt-4o"}
+ policy := promptCachePolicyFor(provider.ProviderOpenAI, "")
+ body, forceRaw, err := applyPromptCacheToJSONBody(req, policy, "sess-abc", CacheRetentionLong)
+ require.NoError(t, err)
+ require.True(t, forceRaw)
+ payload := body.(map[string]any)
+ assert.Equal(t, "24h", payload["prompt_cache_retention"])
+}
+
+func TestApplyPromptCacheToJSONBody_NoneLeavesBodyUntouched(t *testing.T) {
+ req := &openai.ChatCompletionRequest{Model: "gpt-4o"}
+ policy := promptCachePolicyFor(provider.ProviderOpenAI, "")
+ body, forceRaw, err := applyPromptCacheToJSONBody(req, policy, "sess-abc", CacheRetentionNone)
+ require.NoError(t, err)
+ assert.False(t, forceRaw)
+ assert.Equal(t, req, body)
+}
+
+func TestApplyPromptCacheToJSONBody_AliyunCacheControlBreakpoints(t *testing.T) {
+ req := openai.ChatCompletionRequest{
+ Model: "qwen-plus",
+ Messages: []openai.ChatCompletionMessage{
+ {Role: "system", Content: "stable system"},
+ {Role: "user", Content: "turn question"},
+ },
+ Tools: []openai.Tool{{
+ Type: openai.ToolTypeFunction,
+ Function: &openai.FunctionDefinition{Name: "search", Description: "d", Parameters: json.RawMessage(`{}`)},
+ }},
+ }
+ policy := promptCachePolicyFor(provider.ProviderAliyun, "")
+ body, forceRaw, err := applyPromptCacheToJSONBody(req, policy, "", CacheRetentionShort)
+ require.NoError(t, err)
+ require.True(t, forceRaw)
+ wire, err := json.Marshal(body)
+ require.NoError(t, err)
+ var payload map[string]any
+ require.NoError(t, json.Unmarshal(wire, &payload))
+
+ messages := payload["messages"].([]any)
+ sys := messages[0].(map[string]any)
+ sysParts := sys["content"].([]any)
+ sysCache := sysParts[0].(map[string]any)["cache_control"].(map[string]any)
+ assert.Equal(t, "ephemeral", sysCache["type"])
+
+ user := messages[1].(map[string]any)
+ userParts := user["content"].([]any)
+ userCache := userParts[0].(map[string]any)["cache_control"].(map[string]any)
+ assert.Equal(t, "ephemeral", userCache["type"])
+
+ tools := payload["tools"].([]any)
+ lastTool := tools[len(tools)-1].(map[string]any)
+ toolCache := lastTool["cache_control"].(map[string]any)
+ assert.Equal(t, "ephemeral", toolCache["type"])
+}
+
+func TestBuildOutbound_OpenAIPromptCacheKeyFromSession(t *testing.T) {
+ c := newOutboundChat(t, string(provider.ProviderOpenAI), "gpt-4o", nil)
+ ctx := types.WithSessionID(context.Background(), "sess-live")
+ body, _, useRaw, err := c.buildOutbound(ctx, []Message{{Role: "user", Content: "hi"}}, &ChatOptions{}, false)
+ require.NoError(t, err)
+ require.True(t, useRaw)
+ payload := body.(map[string]any)
+ assert.Equal(t, "sess-live", payload["prompt_cache_key"])
+}
+
+func TestAttachPromptCacheHeaders(t *testing.T) {
+ req, err := http.NewRequest(http.MethodPost, "https://api.openai.com/v1/chat/completions", nil)
+ require.NoError(t, err)
+ attachPromptCacheHeaders(req, promptCachePolicy{sendAffinity: true}, "sess-1")
+ assert.Equal(t, "sess-1", req.Header.Get("session_id"))
+ assert.Equal(t, "sess-1", req.Header.Get("x-client-request-id"))
+ assert.Equal(t, "sess-1", req.Header.Get("x-session-affinity"))
+}
diff --git a/internal/models/chat/provider_test.go b/internal/models/chat/provider_test.go
index 688485724..48da3fc43 100644
--- a/internal/models/chat/provider_test.go
+++ b/internal/models/chat/provider_test.go
@@ -1,6 +1,7 @@
package chat
import (
+ "context"
"encoding/json"
"testing"
@@ -69,7 +70,7 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("generic explicit thinking_type overrides legacy kwargs", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderGeneric), "deepseek-v4-flash",
map[string]string{ExtraConfigThinkingControl: "thinking_type"})
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
require.NoError(t, err)
require.True(t, useRaw)
js := mustJSON(t, body)
@@ -80,7 +81,7 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("generic legacy chat_template_kwargs", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderGeneric), "qwen", nil)
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
require.NoError(t, err)
require.True(t, useRaw)
assert.Contains(t, mustJSON(t, body), "chat_template_kwargs")
@@ -89,7 +90,7 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("none keeps the standard SDK request", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderGeneric), "x",
map[string]string{ExtraConfigThinkingControl: "none"})
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
require.NoError(t, err)
assert.False(t, useRaw)
_, ok := body.(*openai.ChatCompletionRequest)
@@ -98,7 +99,7 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("qwen non-stream forces disabled", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderAliyun), "qwen3-32b", nil)
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(true)}, false)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(true)}, false)
require.NoError(t, err)
require.True(t, useRaw)
assert.Contains(t, mustJSON(t, body), `"enable_thinking":false`)
@@ -106,14 +107,14 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("qwen stream honors requested true", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderAliyun), "qwen3-32b", nil)
- body, _, _, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(true)}, true)
+ body, _, _, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(true)}, true)
require.NoError(t, err)
assert.Contains(t, mustJSON(t, body), `"enable_thinking":true`)
})
t.Run("volcengine thinking enabled", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderVolcengine), "doubao", nil)
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(true)}, true)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(true)}, true)
require.NoError(t, err)
require.True(t, useRaw)
js := mustJSON(t, body)
@@ -123,7 +124,7 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("lkeap deepseek-v3 emits thinking type", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderLKEAP), "deepseek-v3.1", nil)
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
require.NoError(t, err)
require.True(t, useRaw)
assert.Contains(t, mustJSON(t, body), `"thinking"`)
@@ -131,7 +132,7 @@ func TestBuildOutbound_Thinking(t *testing.T) {
t.Run("lkeap r1 left untouched", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderLKEAP), "deepseek-r1", nil)
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Thinking: ptrBool(false)}, true)
require.NoError(t, err)
assert.False(t, useRaw)
_, ok := body.(*openai.ChatCompletionRequest)
@@ -146,7 +147,7 @@ func TestBuildOutbound_ShapeRequest(t *testing.T) {
t.Run("deepseek strips tool_choice", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderDeepSeek), "deepseek-chat", nil)
- body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{ToolChoice: "auto"}, false)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{ToolChoice: "auto"}, false)
require.NoError(t, err)
assert.True(t, useRaw)
request, ok := body.(map[string]any)
@@ -156,7 +157,7 @@ func TestBuildOutbound_ShapeRequest(t *testing.T) {
t.Run("moonshot pins temperature to 1", func(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderMoonshot), "moonshot-v1-8k", nil)
- body, _, _, err := c.buildOutbound(msgs, &ChatOptions{Temperature: 0.7, TopP: 0.9}, false)
+ body, _, _, err := c.buildOutbound(context.Background(), msgs, &ChatOptions{Temperature: 0.7, TopP: 0.9}, false)
require.NoError(t, err)
req := body.(*openai.ChatCompletionRequest)
assert.EqualValues(t, 1, req.Temperature)
@@ -182,7 +183,7 @@ func TestBuildOutbound_GeminiProviderMetadata(t *testing.T) {
},
}
- body, _, useRaw, err := c.buildOutbound(messages, &ChatOptions{}, false)
+ body, _, useRaw, err := c.buildOutbound(context.Background(), messages, &ChatOptions{}, false)
require.NoError(t, err)
require.True(t, useRaw)
diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go
index 02b93d0f0..c55e94ba1 100644
--- a/internal/models/chat/remote_api.go
+++ b/internal/models/chat/remote_api.go
@@ -132,7 +132,7 @@ func (c *RemoteAPIChat) shapedRequest(messages []Message, opts *ChatOptions, isS
// path is required. This is the single place that composes adapter + thinking,
// replacing the former buildRequestCustomizer plumbing.
func (c *RemoteAPIChat) buildOutbound(
- messages []Message, opts *ChatOptions, isStream bool,
+ ctx context.Context, messages []Message, opts *ChatOptions, isStream bool,
) (body any, endpoint string, useRawHTTP bool, err error) {
req := c.shapedRequest(messages, opts, isStream)
@@ -150,8 +150,18 @@ func (c *RemoteAPIChat) buildOutbound(
if err != nil {
return nil, "", false, err
}
+
+ retention := resolveCacheRetention(opts)
+ policy := promptCachePolicyFor(c.provider, c.baseURL)
+ sessionID := promptCacheSessionID(ctx, opts)
+ cachedBody, forceRaw, err := applyPromptCacheToJSONBody(body, policy, sessionID, retention)
+ if err != nil {
+ return nil, "", false, err
+ }
+ body = cachedBody
+
endpoint = c.adapter.Endpoint(c.baseURL, c.modelID, isStream)
- useRawHTTP = useRaw || c.adapter.ForceRawHTTP() || endpoint != ""
+ useRawHTTP = useRaw || c.adapter.ForceRawHTTP() || endpoint != "" || forceRaw
return body, endpoint, useRawHTTP, nil
}
@@ -170,12 +180,12 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
timeoutCtx, cancel := withLLMTimeout(ctx, defaultChatTimeout)
defer cancel()
- body, endpoint, useRawHTTP, err := c.buildOutbound(messages, opts, false)
+ body, endpoint, useRawHTTP, err := c.buildOutbound(timeoutCtx, messages, opts, false)
if err != nil {
return nil, err
}
if useRawHTTP {
- return c.chatWithRawHTTP(timeoutCtx, endpoint, body)
+ return c.chatWithRawHTTP(timeoutCtx, endpoint, body, opts)
}
req := *(body.(*openai.ChatCompletionRequest))
@@ -202,7 +212,7 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
}
// chatWithRawHTTP 使用原始 HTTP 请求进行聊天(供自定义请求使用)
-func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, customReq any) (*types.ChatResponse, error) {
+func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, customReq any, opts *ChatOptions) (*types.ChatResponse, error) {
jsonData, err := json.Marshal(customReq)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
@@ -227,6 +237,7 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
// 注入用户自定义 header(保留头会在工具内部自动跳过)
secutils.ApplyCustomHeaders(httpReq, c.customHeaders)
+ attachPromptCacheHeaders(httpReq, promptCachePolicyFor(c.provider, c.baseURL), promptCacheSessionID(ctx, opts))
logger.Infof(ctx, "[LLM Request] Remote HTTP, endpoint=%s, model=%s",
endpoint, c.modelName)
@@ -268,13 +279,13 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
// 因为带思考/推理的模型可能数十秒甚至几分钟才产出首 token。
timeoutCtx, cancel := withLLMTimeout(ctx, defaultStreamTimeout)
- body, endpoint, useRawHTTP, err := c.buildOutbound(messages, opts, true)
+ body, endpoint, useRawHTTP, err := c.buildOutbound(timeoutCtx, messages, opts, true)
if err != nil {
cancel()
return nil, err
}
if useRawHTTP {
- ch, err := c.chatStreamWithRawHTTP(timeoutCtx, endpoint, body)
+ ch, err := c.chatStreamWithRawHTTP(timeoutCtx, endpoint, body, opts)
return wrapStreamCancel(ch, err, cancel)
}
@@ -333,7 +344,7 @@ func wrapStreamCancel(in <-chan types.StreamResponse, err error, cancel context.
}
// chatStreamWithRawHTTP 使用原始 HTTP 请求进行流式聊天
-func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint string, customReq any) (<-chan types.StreamResponse, error) {
+func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint string, customReq any, opts *ChatOptions) (<-chan types.StreamResponse, error) {
jsonData, err := json.Marshal(customReq)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
@@ -363,6 +374,7 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri
// 注入用户自定义 header(保留头会在工具内部自动跳过)
secutils.ApplyCustomHeaders(httpReq, c.customHeaders)
+ attachPromptCacheHeaders(httpReq, promptCachePolicyFor(c.provider, c.baseURL), promptCacheSessionID(ctx, opts))
resp, err := rawHTTPClient.Do(httpReq)
if err != nil {
diff --git a/internal/models/chat/usage.go b/internal/models/chat/usage.go
index f51189e02..a7605a1a8 100644
--- a/internal/models/chat/usage.go
+++ b/internal/models/chat/usage.go
@@ -19,10 +19,10 @@ func logUsage(ctx context.Context, model string, u *types.TokenUsage) {
logger.Infof(ctx,
"[LLM Usage] model=%s, purpose=%s, prompt_prefix=%s, prompt_tokens=%d, completion_tokens=%d, "+
"total_tokens=%d, cached_tokens=%d, cache_read_tokens=%d, cache_write_tokens=%d, "+
- "cache_miss_tokens=%d, cache_reported=%t, cache_status=%s%s",
+ "cache_miss_tokens=%d, cache_hit_rate=%.1f%%, cache_reported=%t, cache_status=%s%s",
model, purpose, prefixFingerprint, u.PromptTokens, u.CompletionTokens, u.TotalTokens,
u.CachedTokens, u.CacheReadTokens, u.CacheWriteTokens, u.CacheMissTokens,
- u.CacheReported, u.CacheStatus, usageAttribution(ctx))
+ u.PromptCacheHitRate(), u.CacheReported, u.CacheStatus, usageAttribution(ctx))
}
// usageAttribution renders the ", session_id=…, principal=…" suffix that
diff --git a/internal/types/chat.go b/internal/types/chat.go
index 5b6a39678..9b4c45ff4 100644
--- a/internal/types/chat.go
+++ b/internal/types/chat.go
@@ -119,6 +119,16 @@ func mergeUnreportedCacheStatus(accumulated, incoming PromptCacheStatus) PromptC
return PromptCacheStatusUnreported
}
+// PromptCacheHitRate returns cache-read tokens as a percentage of the
+// provider's prompt total. Zero when the prompt is empty. A ReAct turn that
+// reuses the system prefix therefore reads as a high hit rather than a miss.
+func (u TokenUsage) PromptCacheHitRate() float64 {
+ if u.PromptTokens <= 0 {
+ return 0
+ }
+ return float64(u.CacheReadTokens) / float64(u.PromptTokens) * 100
+}
+
// Value persists the usage as a jsonb column (assistant messages carry the
// turn's aggregate); nil writes SQL NULL. Mirrors the nullable-pointer
// pattern APIPrincipalConfig uses.
diff --git a/internal/types/placeholder.go b/internal/types/placeholder.go
index b39590490..cc829b3bc 100644
--- a/internal/types/placeholder.go
+++ b/internal/types/placeholder.go
@@ -51,7 +51,7 @@ var (
PlaceholderCurrentTime = PromptPlaceholder{
Name: "current_time",
Label: "当前时间",
- Description: "当前系统时间(格式:2006-01-02 15:04:05)",
+ Description: "当前日期(ISO 格式:2006-01-02)。只用日期、不用时钟,避免秒级变化打断 provider 前缀缓存。",
}
PlaceholderCurrentWeek = PromptPlaceholder{
@@ -194,7 +194,8 @@ type PlaceholderValues map[string]string
// the corresponding values from vals. Unknown placeholders are left untouched.
//
// Built-in auto-values (filled when not supplied explicitly):
-// - {{current_time}} -> time.Now().Format("2006-01-02 15:04:05")
+// - {{current_time}} -> time.Now().Format("2006-01-02") (date only; clock
+// precision would bust provider prefix caches on every request)
// - {{current_week}} -> current weekday name
// - {{yesterday}} -> yesterday's date (2006-01-02)
func RenderPromptPlaceholders(template string, vals PlaceholderValues) string {
@@ -212,7 +213,7 @@ func RenderPromptPlaceholders(template string, vals PlaceholderValues) string {
}
now := time.Now()
- autoFill("current_time", now.Format("2006-01-02 15:04:05"))
+ autoFill("current_time", now.Format("2006-01-02"))
autoFill("current_week", now.Weekday().String())
autoFill("yesterday", now.AddDate(0, 0, -1).Format("2006-01-02"))
diff --git a/internal/types/placeholder_time_test.go b/internal/types/placeholder_time_test.go
new file mode 100644
index 000000000..c1ea02d07
--- /dev/null
+++ b/internal/types/placeholder_time_test.go
@@ -0,0 +1,34 @@
+package types
+
+import (
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestRenderPromptPlaceholdersCurrentTimeIsDateOnly(t *testing.T) {
+ got := RenderPromptPlaceholders("now={{current_time}}", PlaceholderValues{})
+ want := "now=" + time.Now().Format("2006-01-02")
+ if got != want {
+ t.Fatalf("got %q want %q", got, want)
+ }
+ if strings.Contains(got, "T") {
+ t.Fatalf("date-only current_time must not include a clock: %q", got)
+ }
+ if strings.Count(got, ":") != 0 {
+ t.Fatalf("date-only current_time must not include hh:mm:ss: %q", got)
+ }
+}
+
+func TestPromptCacheHitRate(t *testing.T) {
+ empty := TokenUsage{}
+ if empty.PromptCacheHitRate() != 0 {
+ t.Fatalf("empty usage must report 0, got %v", empty.PromptCacheHitRate())
+ }
+ u := TokenUsage{PromptTokens: 1000}
+ u.SetPromptCacheUsage(873, 0, 127, true)
+ got := u.PromptCacheHitRate()
+ if got < 87.2 || got > 87.4 {
+ t.Fatalf("hit rate %v, want ~87.3", got)
+ }
+}