Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

Report cached prompt tokens (#17943)

Daniel Hiltgen855f4bf989da
diff --git a/anthropic/anthropic.go b/anthropic/anthropic.go
index 60cd78dd6..7ed5cb767 100755
--- a/anthropic/anthropic.go
+++ b/anthropic/anthropic.go
@@ -217,8 +217,31 @@ type MessagesResponse struct {
// Usage contains token usage information
type Usage struct {
- InputTokens int `json:"input_tokens"`
- OutputTokens int `json:"output_tokens"`
+ InputTokens int `json:"input_tokens"`
+ CacheReadInputTokens *int `json:"cache_read_input_tokens,omitempty"`
+ OutputTokens int `json:"output_tokens"`
+}
+
+// UsageFromMetrics separates total prompt tokens into uncached and cache-read counts.
+func UsageFromMetrics(metrics api.Metrics) Usage {
+ total := max(0, metrics.PromptEvalCount)
+ var cached *int
+ if metrics.PromptEvalCachedCount != nil {
+ count := min(max(0, *metrics.PromptEvalCachedCount), total)
+ cached = &count
+ }
+ return Usage{
+ InputTokens: total - intValue(cached),
+ CacheReadInputTokens: cached,
+ OutputTokens: metrics.EvalCount,
+ }
+}
+
+func intValue(v *int) int {
+ if v == nil {
+ return 0
+ }
+ return *v
}
// Streaming event types
@@ -273,8 +296,9 @@ type MessageDelta struct {
// DeltaUsage contains cumulative token usage
type DeltaUsage struct {
- InputTokens int `json:"input_tokens"`
- OutputTokens int `json:"output_tokens"`
+ InputTokens int `json:"input_tokens"`
+ CacheReadInputTokens *int `json:"cache_read_input_tokens,omitempty"`
+ OutputTokens int `json:"output_tokens"`
}
// MessageStopEvent signals the end of the message
@@ -688,10 +712,7 @@ func ToMessagesResponse(id string, r api.ChatResponse) MessagesResponse {
Model: r.Model,
Content: content,
StopReason: stopReason,
- Usage: Usage{
- InputTokens: r.Metrics.PromptEvalCount,
- OutputTokens: r.Metrics.EvalCount,
- },
+ Usage: UsageFromMetrics(r.Metrics),
}
}
@@ -721,6 +742,7 @@ type StreamConverter struct {
firstWrite bool
contentIndex int
inputTokens int
+ cacheReadTokens *int
outputTokens int
estimatedInputTokens int // Estimated tokens from request (used when actual metrics are 0)
thinkingStarted bool
@@ -752,8 +774,10 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {
if c.firstWrite {
c.firstWrite = false
// Use actual metrics if available, otherwise use estimate
- c.inputTokens = r.Metrics.PromptEvalCount
- if c.inputTokens == 0 && c.estimatedInputTokens > 0 {
+ usage := UsageFromMetrics(r.Metrics)
+ c.inputTokens = usage.InputTokens
+ c.cacheReadTokens = usage.CacheReadInputTokens
+ if c.inputTokens == 0 && intValue(c.cacheReadTokens) == 0 && c.estimatedInputTokens > 0 {
c.inputTokens = c.estimatedInputTokens
}
@@ -768,8 +792,9 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {
Model: c.Model,
Content: []ContentBlock{},
Usage: Usage{
- InputTokens: c.inputTokens,
- OutputTokens: 0,
+ InputTokens: c.inputTokens,
+ CacheReadInputTokens: c.cacheReadTokens,
+ OutputTokens: 0,
},
},
},
@@ -950,8 +975,10 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {
})
}
- c.inputTokens = r.Metrics.PromptEvalCount
- c.outputTokens = r.Metrics.EvalCount
+ usage := UsageFromMetrics(r.Metrics)
+ c.inputTokens = usage.InputTokens
+ c.cacheReadTokens = usage.CacheReadInputTokens
+ c.outputTokens = usage.OutputTokens
stopReason := mapStopReason(r.DoneReason, len(c.toolCallsSent) > 0)
events = append(events, StreamEvent{
@@ -962,8 +989,9 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {
StopReason: stopReason,
},
Usage: DeltaUsage{
- InputTokens: c.inputTokens,
- OutputTokens: c.outputTokens,
+ InputTokens: c.inputTokens,
+ CacheReadInputTokens: c.cacheReadTokens,
+ OutputTokens: c.outputTokens,
},
},
})
diff --git a/anthropic/anthropic_test.go b/anthropic/anthropic_test.go
index d29127319..6ceb7171c 100755
--- a/anthropic/anthropic_test.go
+++ b/anthropic/anthropic_test.go
@@ -16,6 +16,10 @@ const (
testImage = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=`
)
+func testIntPtr(v int) *int {
+ return &v
+}
+
// textContent is a convenience for constructing []ContentBlock with a single text block in tests.
func textContent(s string) []ContentBlock {
return []ContentBlock{{Type: "text", Text: &s}}
@@ -30,6 +34,61 @@ func makeArgs(kvs ...any) api.ToolCallFunctionArguments {
return args
}
+func TestUsageFromMetricsBoundsCacheReads(t *testing.T) {
+ tests := []struct {
+ name string
+ metrics api.Metrics
+ want Usage
+ }{
+ {
+ name: "negative counts",
+ metrics: api.Metrics{PromptEvalCount: -1, PromptEvalCachedCount: testIntPtr(-2), EvalCount: 3},
+ want: Usage{CacheReadInputTokens: testIntPtr(0), OutputTokens: 3},
+ },
+ {
+ name: "cache reads exceed prompt",
+ metrics: api.Metrics{PromptEvalCount: 3, PromptEvalCachedCount: testIntPtr(5), EvalCount: 2},
+ want: Usage{CacheReadInputTokens: testIntPtr(3), OutputTokens: 2},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if diff := cmp.Diff(tt.want, UsageFromMetrics(tt.metrics)); diff != "" {
+ t.Errorf("usage mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}
+
+func TestUsageCacheReadJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ count *int
+ want string
+ }{
+ {name: "unreported", want: `{"input_tokens":10,"output_tokens":2}`},
+ {name: "zero", count: testIntPtr(0), want: `{"input_tokens":10,"cache_read_input_tokens":0,"output_tokens":2}`},
+ {name: "positive", count: testIntPtr(4), want: `{"input_tokens":6,"cache_read_input_tokens":4,"output_tokens":2}`},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ data, err := json.Marshal(UsageFromMetrics(api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: tt.count,
+ EvalCount: 2,
+ }))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := string(data); got != tt.want {
+ t.Errorf("json = %s, want %s", got, tt.want)
+ }
+ })
+ }
+}
+
func TestFromMessagesRequest_Basic(t *testing.T) {
req := MessagesRequest{
Model: "test-model",
@@ -861,8 +920,9 @@ func TestToMessagesResponse_Basic(t *testing.T) {
Done: true,
DoneReason: "stop",
Metrics: api.Metrics{
- PromptEvalCount: 10,
- EvalCount: 5,
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ EvalCount: 5,
},
}
@@ -886,9 +946,17 @@ func TestToMessagesResponse_Basic(t *testing.T) {
if result.StopReason != "end_turn" {
t.Errorf("expected stop_reason 'end_turn', got %q", result.StopReason)
}
- if result.Usage.InputTokens != 10 || result.Usage.OutputTokens != 5 {
+ if result.Usage.InputTokens != 6 || intValue(result.Usage.CacheReadInputTokens) != 4 || result.Usage.OutputTokens != 5 {
t.Errorf("unexpected usage: %+v", result.Usage)
}
+
+ data, err := json.Marshal(result.Usage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(data), `"cache_read_input_tokens":4`) {
+ t.Errorf("unexpected usage json: %s", data)
+ }
}
func TestToMessagesResponse_PreservesClaudeAutoClassifierOutput(t *testing.T) {
@@ -1072,7 +1140,7 @@ func TestStreamConverter_Basic(t *testing.T) {
Role: "assistant",
Content: "Hello",
},
- Metrics: api.Metrics{PromptEvalCount: 10},
+ Metrics: api.Metrics{PromptEvalCount: 10, PromptEvalCachedCount: testIntPtr(4)},
}
events1 := conv.Process(resp1)
@@ -1100,7 +1168,7 @@ func TestStreamConverter_Basic(t *testing.T) {
},
Done: true,
DoneReason: "stop",
- Metrics: api.Metrics{PromptEvalCount: 10, EvalCount: 5},
+ Metrics: api.Metrics{PromptEvalCount: 10, PromptEvalCachedCount: testIntPtr(4), EvalCount: 5},
}
events2 := conv.Process(resp2)
@@ -1118,7 +1186,7 @@ func TestStreamConverter_Basic(t *testing.T) {
t.Errorf("unexpected stop reason: %+v", data.Delta.StopReason)
}
- if data.Usage.InputTokens != 10 || data.Usage.OutputTokens != 5 {
+ if data.Usage.InputTokens != 6 || intValue(data.Usage.CacheReadInputTokens) != 4 || data.Usage.OutputTokens != 5 {
t.Errorf("unexpected usage: %+v", data.Usage)
}
} else {
diff --git a/api/metrics_test.go b/api/metrics_test.go
new file mode 100644
index 000000000..b0c5fa5c9
--- /dev/null
+++ b/api/metrics_test.go
@@ -0,0 +1,74 @@
+package api
+
+import (
+ "encoding/json"
+ "io"
+ "os"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestMetricsCachedPromptJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ count *int
+ want string
+ }{
+ {name: "unreported", want: `{}`},
+ {name: "zero", count: testIntPtr(0), want: `{"prompt_eval_cached_count":0}`},
+ {name: "positive", count: testIntPtr(4), want: `{"prompt_eval_cached_count":4}`},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ data, err := json.Marshal(Metrics{PromptEvalCachedCount: tt.count})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := string(data); got != tt.want {
+ t.Errorf("json = %s, want %s", got, tt.want)
+ }
+
+ var metrics Metrics
+ if err := json.Unmarshal(data, &metrics); err != nil {
+ t.Fatal(err)
+ }
+ if tt.count == nil {
+ if metrics.PromptEvalCachedCount != nil {
+ t.Errorf("cached count = %v, want nil", metrics.PromptEvalCachedCount)
+ }
+ } else if metrics.PromptEvalCachedCount == nil || *metrics.PromptEvalCachedCount != *tt.count {
+ t.Errorf("cached count = %v, want %d", metrics.PromptEvalCachedCount, *tt.count)
+ }
+ })
+ }
+}
+
+func TestMetricsSummaryCachedPromptTokens(t *testing.T) {
+ read, write, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ original := os.Stderr
+ os.Stderr = write
+ t.Cleanup(func() { os.Stderr = original })
+
+ (&Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ PromptEvalDuration: time.Second,
+ }).Summary()
+ write.Close()
+ os.Stderr = original
+
+ output, err := io.ReadAll(read)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, want := range []string{"prompt eval count: 10 token(s)", "prompt eval cached: 4 token(s)", "prompt eval rate: 6.00 tokens/s"} {
+ if !strings.Contains(string(output), want) {
+ t.Errorf("summary missing %q:\n%s", want, output)
+ }
+ }
+}
diff --git a/api/types.go b/api/types.go
index 65f5123b4..82dd9a449 100644
--- a/api/types.go
+++ b/api/types.go
@@ -555,12 +555,13 @@ type DebugInfo struct {
}
type Metrics struct {
- TotalDuration time.Duration `json:"total_duration,omitempty"`
- LoadDuration time.Duration `json:"load_duration,omitempty"`
- PromptEvalCount int `json:"prompt_eval_count,omitempty"`
- PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
- EvalCount int `json:"eval_count,omitempty"`
- EvalDuration time.Duration `json:"eval_duration,omitempty"`
+ TotalDuration time.Duration `json:"total_duration,omitempty"`
+ LoadDuration time.Duration `json:"load_duration,omitempty"`
+ PromptEvalCount int `json:"prompt_eval_count,omitempty"`
+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
+ PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
+ EvalCount int `json:"eval_count,omitempty"`
+ EvalDuration time.Duration `json:"eval_duration,omitempty"`
}
// Options specified in [GenerateRequest]. If you add a new option here, also
@@ -983,9 +984,18 @@ func (m *Metrics) Summary() {
fmt.Fprintf(os.Stderr, "prompt eval count: %d token(s)\n", m.PromptEvalCount)
}
+ cached := 0
+ if m.PromptEvalCachedCount != nil {
+ cached = *m.PromptEvalCachedCount
+ }
+ if cached > 0 {
+ fmt.Fprintf(os.Stderr, "prompt eval cached: %d token(s)\n", cached)
+ }
+
if m.PromptEvalDuration > 0 {
fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", m.PromptEvalDuration)
- fmt.Fprintf(os.Stderr, "prompt eval rate: %.2f tokens/s\n", float64(m.PromptEvalCount)/m.PromptEvalDuration.Seconds())
+ uncached := max(0, m.PromptEvalCount-cached)
+ fmt.Fprintf(os.Stderr, "prompt eval rate: %.2f tokens/s\n", float64(uncached)/m.PromptEvalDuration.Seconds())
}
if m.EvalCount > 0 {
diff --git a/cmd/bench/bench.go b/cmd/bench/bench.go
index 1ecd42cfc..b0433f10c 100644
--- a/cmd/bench/bench.go
+++ b/cmd/bench/bench.go
@@ -36,10 +36,11 @@ type flagOptions struct {
}
type Metrics struct {
- Model string
- Step string
- Count int
- Duration time.Duration
+ Model string
+ Step string
+ Count int
+ CachedPromptCount *int
+ Duration time.Duration
}
type ModelInfo struct {
@@ -194,7 +195,7 @@ func outputFormatHeader(w io.Writer, format string, verbose bool) {
fmt.Fprintf(w, "goarch: %s\n", runtime.GOARCH)
}
case "csv":
- headings := []string{"NAME", "STEP", "COUNT", "NS_PER_COUNT", "TOKEN_PER_SEC"}
+ headings := []string{"NAME", "STEP", "COUNT", "NS_PER_COUNT", "TOKEN_PER_SEC", "CACHED_PROMPT_COUNT"}
fmt.Fprintln(w, strings.Join(headings, ","))
}
}
@@ -221,14 +222,21 @@ func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool)
case "benchstat":
for _, m := range metrics {
if m.Step == "generate" || m.Step == "prefill" {
+ var promptCounts string
+ if m.Step == "prefill" {
+ promptCounts = fmt.Sprintf(" %d processed-prompt-token", m.Count)
+ if m.CachedPromptCount != nil {
+ promptCounts += fmt.Sprintf(" %d cached-prompt-token", *m.CachedPromptCount)
+ }
+ }
if m.Count > 0 {
nsPerToken := float64(m.Duration.Nanoseconds()) / float64(m.Count)
tokensPerSec := float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9
- fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 %.2f ns/token %.2f token/sec\n",
- m.Model, m.Step, nsPerToken, tokensPerSec)
+ fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 %.2f ns/token %.2f token/sec%s\n",
+ m.Model, m.Step, nsPerToken, tokensPerSec, promptCounts)
} else {
- fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 0 ns/token 0 token/sec\n",
- m.Model, m.Step)
+ fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 0 ns/token 0 token/sec%s\n",
+ m.Model, m.Step, promptCounts)
}
} else if m.Step == "ttft" {
fmt.Fprintf(w, "BenchmarkModel/name=%s/step=ttft 1 %d ns/op\n",
@@ -240,6 +248,10 @@ func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool)
}
case "csv":
for _, m := range metrics {
+ cachedPromptCount := ""
+ if m.CachedPromptCount != nil {
+ cachedPromptCount = fmt.Sprint(*m.CachedPromptCount)
+ }
if m.Step == "generate" || m.Step == "prefill" {
var nsPerToken float64
var tokensPerSec float64
@@ -247,9 +259,9 @@ func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool)
nsPerToken = float64(m.Duration.Nanoseconds()) / float64(m.Count)
tokensPerSec = float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9
}
- fmt.Fprintf(w, "%s,%s,%d,%.2f,%.2f\n", m.Model, m.Step, m.Count, nsPerToken, tokensPerSec)
+ fmt.Fprintf(w, "%s,%s,%d,%.2f,%.2f,%s\n", m.Model, m.Step, m.Count, nsPerToken, tokensPerSec, cachedPromptCount)
} else {
- fmt.Fprintf(w, "%s,%s,1,%d,0\n", m.Model, m.Step, m.Duration.Nanoseconds())
+ fmt.Fprintf(w, "%s,%s,1,%d,0,%s\n", m.Model, m.Step, m.Duration.Nanoseconds(), cachedPromptCount)
}
}
default:
@@ -428,12 +440,17 @@ func BenchmarkModel(fOpt flagOptions) error {
}
}
+ cachedPromptCount := 0
+ if responseMetrics.PromptEvalCachedCount != nil {
+ cachedPromptCount = *responseMetrics.PromptEvalCachedCount
+ }
metrics := []Metrics{
{
- Model: model,
- Step: "prefill",
- Count: responseMetrics.PromptEvalCount,
- Duration: responseMetrics.PromptEvalDuration,
+ Model: model,
+ Step: "prefill",
+ Count: max(0, responseMetrics.PromptEvalCount-cachedPromptCount),
+ CachedPromptCount: responseMetrics.PromptEvalCachedCount,
+ Duration: responseMetrics.PromptEvalDuration,
},
{
Model: model,
@@ -464,8 +481,13 @@ func BenchmarkModel(fOpt flagOptions) error {
OutputMetrics(out, *fOpt.format, metrics, *fOpt.verbose)
if *fOpt.debug && *fOpt.promptTokens > 0 {
- fmt.Fprintf(os.Stderr, "Generated prompt targeting ~%d tokens (actual: %d)\n",
- *fOpt.promptTokens, responseMetrics.PromptEvalCount)
+ if responseMetrics.PromptEvalCachedCount == nil {
+ fmt.Fprintf(os.Stderr, "Generated prompt targeting ~%d tokens (actual: %d, cached: unavailable)\n",
+ *fOpt.promptTokens, responseMetrics.PromptEvalCount)
+ } else {
+ fmt.Fprintf(os.Stderr, "Generated prompt targeting ~%d tokens (actual: %d, cached: %d)\n",
+ *fOpt.promptTokens, responseMetrics.PromptEvalCount, cachedPromptCount)
+ }
}
if *fOpt.keepAlive > 0 {
diff --git a/cmd/bench/bench_test.go b/cmd/bench/bench_test.go
index 3c4c67f13..ca5769fc1 100644
--- a/cmd/bench/bench_test.go
+++ b/cmd/bench/bench_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/rand"
"encoding/json"
+ "fmt"
"io"
"net/http"
"net/http/httptest"
@@ -15,6 +16,10 @@ import (
"github.com/ollama/ollama/api"
)
+func testIntPtr(v int) *int {
+ return &v
+}
+
func createTestFlagOptions() flagOptions {
models := "test-model"
format := "benchstat"
@@ -1083,7 +1088,7 @@ func TestBenchmarkModel_CSVFormat(t *testing.T) {
}
})
- if !strings.Contains(output, "NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC") {
+ if !strings.Contains(output, "NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC,CACHED_PROMPT_COUNT") {
t.Errorf("Expected CSV header, got: %s", output)
}
if !strings.Contains(output, "test-model,prefill,") {
@@ -1094,6 +1099,35 @@ func TestBenchmarkModel_CSVFormat(t *testing.T) {
}
}
+func TestBenchmarkModel_PrefillExcludesCachedTokens(t *testing.T) {
+ fOpt := createTestFlagOptions()
+ format := "csv"
+ fOpt.format = &format
+ responses := defaultGenerateResponses()
+ responses[len(responses)-1].PromptEvalCachedCount = testIntPtr(4)
+
+ server := createMockOllamaServer(t, mockServerOptions{generateResponses: responses})
+ defer server.Close()
+ t.Setenv("OLLAMA_HOST", server.URL)
+
+ output := captureOutput(func() {
+ if err := BenchmarkModel(fOpt); err != nil {
+ t.Errorf("BenchmarkModel: %v", err)
+ }
+ })
+ wantPrefix := fmt.Sprintf("test-model,prefill,%d,", responses[len(responses)-1].PromptEvalCount-*responses[len(responses)-1].PromptEvalCachedCount)
+ var prefillRow string
+ for row := range strings.SplitSeq(output, "\n") {
+ if strings.HasPrefix(row, wantPrefix) {
+ prefillRow = row
+ break
+ }
+ }
+ if prefillRow == "" || !strings.HasSuffix(prefillRow, ",4") {
+ t.Errorf("prefill row did not exclude cached prompt tokens:\n%s", output)
+ }
+}
+
// --- Unit tests for helper functions ---
func TestGeneratePromptForTokenCount(t *testing.T) {
@@ -1188,7 +1222,7 @@ func TestBuildGenerateRequest_VariesByEpoch(t *testing.T) {
func TestOutputMetrics_Benchstat(t *testing.T) {
var buf bytes.Buffer
metrics := []Metrics{
- {Model: "m1", Step: "prefill", Count: 10, Duration: 100 * time.Millisecond},
+ {Model: "m1", Step: "prefill", Count: 10, CachedPromptCount: testIntPtr(4), Duration: 100 * time.Millisecond},
{Model: "m1", Step: "generate", Count: 50, Duration: 500 * time.Millisecond},
{Model: "m1", Step: "ttft", Count: 1, Duration: 50 * time.Millisecond},
{Model: "m1", Step: "load", Count: 1, Duration: 50 * time.Millisecond},
@@ -1214,6 +1248,9 @@ func TestOutputMetrics_Benchstat(t *testing.T) {
if !strings.Contains(output, "token/sec") {
t.Errorf("Expected token/sec metric for throughput lines, got: %s", output)
}
+ if !strings.Contains(output, "10 processed-prompt-token 4 cached-prompt-token") {
+ t.Errorf("Expected prompt cache counts on prefill line, got: %s", output)
+ }
for _, line := range strings.Split(strings.TrimSpace(output), "\n") {
if !strings.HasPrefix(line, "Benchmark") {
continue
@@ -1242,6 +1279,37 @@ func TestOutputMetrics_BenchstatFormat(t *testing.T) {
if !strings.Contains(output, "ns/token") {
t.Errorf("Expected ns/token unit for prefill, got: %s", output)
}
+ if strings.Contains(output, "cached-prompt-token") {
+ t.Errorf("unexpected cached prompt count when unavailable: %s", output)
+ }
+}
+
+func TestOutputMetrics_CachedPromptAvailability(t *testing.T) {
+ tests := []struct {
+ name string
+ count *int
+ want string
+ }{
+ {name: "unavailable", want: "m1,prefill,10,10000000.00,100.00,"},
+ {name: "zero", count: testIntPtr(0), want: "m1,prefill,10,10000000.00,100.00,0"},
+ {name: "positive", count: testIntPtr(4), want: "m1,prefill,10,10000000.00,100.00,4"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var buf bytes.Buffer
+ OutputMetrics(&buf, "csv", []Metrics{{
+ Model: "m1",
+ Step: "prefill",
+ Count: 10,
+ CachedPromptCount: tt.count,
+ Duration: 100 * time.Millisecond,
+ }}, false)
+ if got := strings.TrimSpace(buf.String()); got != tt.want {
+ t.Errorf("output = %q, want %q", got, tt.want)
+ }
+ })
+ }
}
func TestOutputModelInfo(t *testing.T) {
@@ -1403,7 +1471,7 @@ func TestOutputFormatHeader(t *testing.T) {
var buf bytes.Buffer
outputFormatHeader(&buf, "csv", false)
output := buf.String()
- if !strings.Contains(output, "NAME,STEP,COUNT") {
+ if output != "NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC,CACHED_PROMPT_COUNT\n" {
t.Errorf("Expected CSV header, got: %s", output)
}
})
diff --git a/cmd/tui/chat/render.go b/cmd/tui/chat/render.go
index 4ea56ecfd..fb85688a6 100644
--- a/cmd/tui/chat/render.go
+++ b/cmd/tui/chat/render.go
@@ -582,9 +582,17 @@ func metricsSummaryLines(metrics *api.Metrics) []string {
if metrics.PromptEvalCount > 0 {
lines = append(lines, fmt.Sprintf("prompt eval count: %d token(s)", metrics.PromptEvalCount))
}
+ cached := 0
+ if metrics.PromptEvalCachedCount != nil {
+ cached = *metrics.PromptEvalCachedCount
+ }
+ if cached > 0 {
+ lines = append(lines, fmt.Sprintf("prompt eval cached: %d token(s)", cached))
+ }
if metrics.PromptEvalDuration > 0 {
lines = append(lines, fmt.Sprintf("prompt eval duration: %s", metrics.PromptEvalDuration))
- lines = append(lines, fmt.Sprintf("prompt eval rate: %.2f tokens/s", float64(metrics.PromptEvalCount)/metrics.PromptEvalDuration.Seconds()))
+ uncached := max(0, metrics.PromptEvalCount-cached)
+ lines = append(lines, fmt.Sprintf("prompt eval rate: %.2f tokens/s", float64(uncached)/metrics.PromptEvalDuration.Seconds()))
}
if metrics.EvalCount > 0 {
lines = append(lines, fmt.Sprintf("eval count: %d token(s)", metrics.EvalCount))
@@ -597,9 +605,14 @@ func metricsSummaryLines(metrics *api.Metrics) []string {
}
func metricsEmpty(metrics api.Metrics) bool {
+ cached := 0
+ if metrics.PromptEvalCachedCount != nil {
+ cached = *metrics.PromptEvalCachedCount
+ }
return metrics.TotalDuration <= 0 &&
metrics.LoadDuration <= 0 &&
metrics.PromptEvalCount <= 0 &&
+ cached <= 0 &&
metrics.PromptEvalDuration <= 0 &&
metrics.EvalCount <= 0 &&
metrics.EvalDuration <= 0
diff --git a/cmd/tui/chat/render_test.go b/cmd/tui/chat/render_test.go
index c2d911412..0a31c7bbb 100644
--- a/cmd/tui/chat/render_test.go
+++ b/cmd/tui/chat/render_test.go
@@ -17,6 +17,24 @@ import (
"github.com/ollama/ollama/api"
)
+func testIntPtr(v int) *int {
+ return &v
+}
+
+func TestMetricsSummaryLinesCachedPromptTokens(t *testing.T) {
+ lines := metricsSummaryLines(&api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ PromptEvalDuration: time.Second,
+ })
+ got := strings.Join(lines, "\n")
+ for _, want := range []string{"prompt eval count: 10 token(s)", "prompt eval cached: 4 token(s)", "prompt eval rate: 6.00 tokens/s"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("summary missing %q:\n%s", want, got)
+ }
+ }
+}
+
func TestChatAssistantEntryHasNoLabel(t *testing.T) {
m := chatModel{}
diff --git a/docs/api.md b/docs/api.md
index 95cac42ca..87ac17d0a 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -100,7 +100,8 @@ The final response in the stream also includes additional data about the generat
- `total_duration`: time spent generating the response
- `load_duration`: time spent in nanoseconds loading the model
- `prompt_eval_count`: number of tokens in the prompt
-- `prompt_eval_duration`: time spent in nanoseconds evaluating the prompt
+- `prompt_eval_cached_count`: number of prompt tokens read from the cache
+- `prompt_eval_duration`: time spent in nanoseconds evaluating uncached prompt tokens
- `eval_count`: number of tokens in the response
- `eval_duration`: time in nanoseconds spent generating the response
- `context`: an encoding of the conversation used in this response, this can be sent in the next request to keep a conversational memory
diff --git a/docs/api/usage.mdx b/docs/api/usage.mdx
index 107bb0c4c..e9606a425 100644
--- a/docs/api/usage.mdx
+++ b/docs/api/usage.mdx
@@ -6,8 +6,9 @@ Ollama's API responses include metrics that can be used for measuring performanc
* `total_duration`: How long the response took to generate
* `load_duration`: How long the model took to load
-* `prompt_eval_count`: How many input tokens were processed
-* `prompt_eval_duration`: How long it took to evaluate the prompt
+* `prompt_eval_count`: How many input tokens were in the prompt
+* `prompt_eval_cached_count`: How many prompt tokens were read from the cache
+* `prompt_eval_duration`: How long it took to evaluate the uncached prompt tokens
* `eval_count`: How many output tokens were processes
* `eval_duration`: How long it took to generate the output tokens
@@ -27,6 +28,7 @@ For endpoints that return usage metrics, the response body will include the usag
"total_duration": 174560334,
"load_duration": 101397084,
"prompt_eval_count": 11,
+ "prompt_eval_cached_count": 8,
"prompt_eval_duration": 13074791,
"eval_count": 18,
"eval_duration": 52479709
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index 34aa9f40e..162a906c8 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -147,9 +147,12 @@ components:
prompt_eval_count:
type: integer
description: Number of input tokens in the prompt
+ prompt_eval_cached_count:
+ type: integer
+ description: Number of prompt tokens read from the cache
prompt_eval_duration:
type: integer
- description: Time spent evaluating the prompt in nanoseconds
+ description: Time spent evaluating uncached prompt tokens in nanoseconds
eval_count:
type: integer
description: Number of output tokens generated in the response
@@ -191,9 +194,12 @@ components:
prompt_eval_count:
type: integer
description: Number of input tokens in the prompt
+ prompt_eval_cached_count:
+ type: integer
+ description: Number of prompt tokens read from the cache
prompt_eval_duration:
type: integer
- description: Time spent evaluating the prompt in nanoseconds
+ description: Time spent evaluating uncached prompt tokens in nanoseconds
eval_count:
type: integer
description: Number of output tokens generated in the response
@@ -352,9 +358,12 @@ components:
prompt_eval_count:
type: integer
description: Number of tokens in the prompt
+ prompt_eval_cached_count:
+ type: integer
+ description: Number of prompt tokens read from the cache
prompt_eval_duration:
type: integer
- description: Time spent evaluating the prompt in nanoseconds
+ description: Time spent evaluating uncached prompt tokens in nanoseconds
eval_count:
type: integer
description: Number of tokens generated in the response
diff --git a/llm/llama_server.go b/llm/llama_server.go
index 6f27b7547..fc3edfbbe 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -1412,6 +1412,7 @@ type llamaServerCompletionRequest struct {
JsonSchema json.RawMessage `json:"json_schema,omitempty"`
NProbs int `json:"n_probs,omitempty"`
PreservedTokens []string `json:"preserved_tokens,omitempty"`
+ TimingsPerToken bool `json:"timings_per_token,omitempty"`
}
func llamaServerPreservedTokens(parserTokens []string, toolCallTag string) []string {
@@ -1501,7 +1502,7 @@ type llamaServerChatResponse struct {
}
type llamaServerTimings struct {
- CacheN int `json:"cache_n"`
+ CacheN *int `json:"cache_n"`
PromptN int `json:"prompt_n"`
PromptMS float64 `json:"prompt_ms"`
PredictN int `json:"predicted_n"`
@@ -1509,7 +1510,10 @@ type llamaServerTimings struct {
}
func (t llamaServerTimings) promptEvalCount() int {
- return t.CacheN + t.PromptN
+ if t.CacheN == nil {
+ return t.PromptN
+ }
+ return *t.CacheN + t.PromptN
}
type llamaServerApplyTemplateResponse struct {
@@ -1574,6 +1578,7 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
TypicalP: req.Options.TypicalP,
Seed: req.Options.Seed,
PreservedTokens: llamaServerPreservedTokens(req.PreservedTokens, req.ToolCallTag),
+ TimingsPerToken: req.IncludeIntermediateMetrics,
}
if req.Logprobs {
@@ -1701,8 +1706,13 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
}
if lsResp.Content != "" && !lsResp.Stop {
- resp := CompletionResponse{
- Content: lsResp.Content,
+ resp := CompletionResponse{Content: lsResp.Content}
+ if req.IncludeIntermediateMetrics {
+ resp.PromptEvalCount = lsResp.Timings.promptEvalCount()
+ resp.PromptEvalCachedCount = lsResp.Timings.CacheN
+ resp.PromptEvalDuration = time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond))
+ resp.EvalCount = lsResp.Timings.PredictN
+ resp.EvalDuration = time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond))
}
resp.Logprobs = convertLogprobs(lsResp.CompletionProbabilities, req.TopLogprobs > 0)
fn(resp)
@@ -1715,13 +1725,14 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
}
finalResp = CompletionResponse{
- Content: lsResp.Content,
- Done: true,
- DoneReason: doneReason,
- PromptEvalCount: lsResp.Timings.promptEvalCount(),
- PromptEvalDuration: time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond)),
- EvalCount: lsResp.Timings.PredictN,
- EvalDuration: time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond)),
+ Content: lsResp.Content,
+ Done: true,
+ DoneReason: doneReason,
+ PromptEvalCount: lsResp.Timings.promptEvalCount(),
+ PromptEvalCachedCount: lsResp.Timings.CacheN,
+ PromptEvalDuration: time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond)),
+ EvalCount: lsResp.Timings.PredictN,
+ EvalDuration: time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond)),
}
hasFinalResp = true
}
@@ -2018,6 +2029,7 @@ func (s *llamaServerRunner) Chat(ctx context.Context, req ChatRequest, fn func(C
resp.Done = true
resp.DoneReason = doneReason
resp.PromptEvalCount = lsResp.Timings.promptEvalCount()
+ resp.PromptEvalCachedCount = lsResp.Timings.CacheN
resp.PromptEvalDuration = time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond))
resp.EvalCount = lsResp.Timings.PredictN
resp.EvalDuration = time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond))
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index a9c9bd826..4d4300b5e 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -154,13 +154,13 @@ func TestContextShiftPromptLimit(t *testing.T) {
func TestLlamaServerCompletionSSEParsing(t *testing.T) {
// Simulate llama-server SSE streaming response
sseLines := []string{
- `data: {"content":"Hello","stop":false}`,
+ `data: {"content":"Hello","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10.5,"predicted_n":1,"predicted_ms":9.1}}`,
``,
`:`,
- `data: {"content":" world","stop":false}`,
+ `data: {"content":" world","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10.5,"predicted_n":2,"predicted_ms":20.3}}`,
``,
`:`,
- `data: {"content":"","stop":true,"stop_type":"eos","timings":{"prompt_n":5,"prompt_ms":10.5,"predicted_n":2,"predicted_ms":20.3}}`,
+ `data: {"content":"","stop":true,"stop_type":"eos","timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10.5,"predicted_n":2,"predicted_ms":20.3}}`,
``,
}
@@ -186,6 +186,9 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {
if !reqBody.Stream {
t.Error("stream should be true")
}
+ if !reqBody.TimingsPerToken {
+ t.Error("timings_per_token should be true")
+ }
w.Header().Set("Content-Type", "text/event-stream")
for _, line := range sseLines {
@@ -208,8 +211,9 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {
var responses []CompletionResponse
opts := api.DefaultOptions()
err := runner.Completion(t.Context(), CompletionRequest{
- Prompt: "test prompt",
- Options: &opts,
+ Prompt: "test prompt",
+ Options: &opts,
+ IncludeIntermediateMetrics: true,
}, func(cr CompletionResponse) {
responses = append(responses, cr)
})
@@ -228,11 +232,29 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {
if responses[0].Done {
t.Error("response[0] should not be done")
}
+ if responses[0].PromptEvalCount != 5 || responses[0].EvalCount != 1 {
+ t.Errorf("response[0] counts = (%d, %d), want (5, 1)", responses[0].PromptEvalCount, responses[0].EvalCount)
+ }
+ if got := responses[0].PromptEvalCachedCount; got == nil || *got != 2 {
+ t.Errorf("response[0] cached prompt count = %v, want 2", got)
+ }
+ if responses[0].PromptEvalDuration != 10500*time.Microsecond || responses[0].EvalDuration != 9100*time.Microsecond {
+ t.Errorf("response[0] durations = (%s, %s), want (10.5ms, 9.1ms)", responses[0].PromptEvalDuration, responses[0].EvalDuration)
+ }
// Second token
if responses[1].Content != " world" {
t.Errorf("response[1].Content = %q, want %q", responses[1].Content, " world")
}
+ if responses[1].PromptEvalCount != 5 || responses[1].EvalCount != 2 {
+ t.Errorf("response[1] counts = (%d, %d), want (5, 2)", responses[1].PromptEvalCount, responses[1].EvalCount)
+ }
+ if got := responses[1].PromptEvalCachedCount; got == nil || *got != 2 {
+ t.Errorf("response[1] cached prompt count = %v, want 2", got)
+ }
+ if responses[1].PromptEvalDuration != 10500*time.Microsecond || responses[1].EvalDuration != 20300*time.Microsecond {
+ t.Errorf("response[1] durations = (%s, %s), want (10.5ms, 20.3ms)", responses[1].PromptEvalDuration, responses[1].EvalDuration)
+ }
// Final response
if !responses[2].Done {
@@ -244,6 +266,9 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {
if responses[2].PromptEvalCount != 5 {
t.Errorf("PromptEvalCount = %d, want 5", responses[2].PromptEvalCount)
}
+ if got := responses[2].PromptEvalCachedCount; got == nil || *got != 2 {
+ t.Errorf("PromptEvalCachedCount = %v, want 2", got)
+ }
if responses[2].EvalCount != 2 {
t.Errorf("EvalCount = %d, want 2", responses[2].EvalCount)
}
@@ -291,6 +316,9 @@ func TestLlamaServerCompletionPromptEvalCountIncludesCache(t *testing.T) {
if responses[0].PromptEvalCount != 17 {
t.Errorf("PromptEvalCount = %d, want 17", responses[0].PromptEvalCount)
}
+ if got := responses[0].PromptEvalCachedCount; got == nil || *got != 12 {
+ t.Errorf("PromptEvalCachedCount = %v, want 12", got)
+ }
if responses[0].PromptEvalDuration != 10*time.Millisecond {
t.Errorf("PromptEvalDuration = %s, want 10ms", responses[0].PromptEvalDuration)
}
@@ -341,6 +369,9 @@ func TestLlamaServerChatPromptEvalCountIncludesCache(t *testing.T) {
if responses[1].PromptEvalCount != 17 {
t.Errorf("PromptEvalCount = %d, want 17", responses[1].PromptEvalCount)
}
+ if got := responses[1].PromptEvalCachedCount; got == nil || *got != 12 {
+ t.Errorf("PromptEvalCachedCount = %v, want 12", got)
+ }
if responses[1].PromptEvalDuration != 10*time.Millisecond {
t.Errorf("PromptEvalDuration = %s, want 10ms", responses[1].PromptEvalDuration)
}
diff --git a/llm/server.go b/llm/server.go
index c1f494468..a8a55409a 100644
--- a/llm/server.go
+++ b/llm/server.go
@@ -214,6 +214,8 @@ type CompletionRequest struct {
PreservedTokens []string // parser tokens to render as text; ignored by non-llama-server runners
ToolCallTag string // raw generic tool parser tag, if any
LeadingBOS string // textual BOS emitted by Go rendering, if any
+ // IncludeIntermediateMetrics adds cumulative metrics to non-final responses; final responses always include metrics.
+ IncludeIntermediateMetrics bool
// Logprobs specifies whether to include log probabilities in the response
Logprobs bool
@@ -235,14 +237,15 @@ type ChatRequest struct {
}
type ChatResponse struct {
- Message api.Message `json:"message"`
- DoneReason DoneReason `json:"done_reason"`
- Done bool `json:"done"`
- PromptEvalCount int `json:"prompt_eval_count"`
- PromptEvalDuration time.Duration `json:"prompt_eval_duration"`
- EvalCount int `json:"eval_count"`
- EvalDuration time.Duration `json:"eval_duration"`
- Logprobs []Logprob `json:"logprobs,omitempty"`
+ Message api.Message `json:"message"`
+ DoneReason DoneReason `json:"done_reason"`
+ Done bool `json:"done"`
+ PromptEvalCount int `json:"prompt_eval_count"`
+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
+ PromptEvalDuration time.Duration `json:"prompt_eval_duration"`
+ EvalCount int `json:"eval_count"`
+ EvalDuration time.Duration `json:"eval_duration"`
+ Logprobs []Logprob `json:"logprobs,omitempty"`
}
// DoneReason represents the reason why a completion response is done
@@ -278,13 +281,14 @@ type Logprob struct {
}
type CompletionResponse struct {
- Content string `json:"content"`
- DoneReason DoneReason `json:"done_reason"`
- Done bool `json:"done"`
- PromptEvalCount int `json:"prompt_eval_count"`
- PromptEvalDuration time.Duration `json:"prompt_eval_duration"`
- EvalCount int `json:"eval_count"`
- EvalDuration time.Duration `json:"eval_duration"`
+ Content string `json:"content"`
+ DoneReason DoneReason `json:"done_reason"`
+ Done bool `json:"done"`
+ PromptEvalCount int `json:"prompt_eval_count"`
+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`
+ PromptEvalDuration time.Duration `json:"prompt_eval_duration"`
+ EvalCount int `json:"eval_count"`
+ EvalDuration time.Duration `json:"eval_duration"`
// Logprobs contains log probability information if requested
Logprobs []Logprob `json:"logprobs,omitempty"`
diff --git a/middleware/anthropic.go b/middleware/anthropic.go
index 33c2f02f4..762698e8c 100644
--- a/middleware/anthropic.go
+++ b/middleware/anthropic.go
@@ -100,13 +100,15 @@ type WebSearchAnthropicWriter struct {
terminalSent bool
- observedPromptEvalCount int
- observedEvalCount int
+ observedPromptEvalCount int
+ observedPromptEvalCachedCount *int
+ observedEvalCount int
- loopInFlight bool
- loopBaseInputTok int
- loopBaseOutputTok int
- loopResultCh chan webSearchLoopResult
+ loopInFlight bool
+ loopBaseInputTok int
+ loopBaseCacheReadTok *int
+ loopBaseOutputTok int
+ loopResultCh chan webSearchLoopResult
streamMessageStarted bool
streamHasOpenBlock bool
@@ -198,10 +200,7 @@ func (w *WebSearchAnthropicWriter) Write(data []byte) (int, error) {
loopCtx, cancel := w.startLoopContext()
defer cancel()
- initialUsage := anthropic.Usage{
- InputTokens: max(w.observedPromptEvalCount, chatResponse.Metrics.PromptEvalCount),
- OutputTokens: max(w.observedEvalCount, chatResponse.Metrics.EvalCount),
- }
+ initialUsage := w.usageWithObservedMetrics(chatResponse.Metrics)
logutil.Trace("anthropic middleware: starting sync web_search loop",
"tool_call", anthropic.TraceToolCall(webSearchCall),
"resp", anthropic.TraceChatResponse(chatResponse),
@@ -316,8 +315,10 @@ func (w *WebSearchAnthropicWriter) runWebSearchLoop(ctx context.Context, initial
"resp", anthropic.TraceChatResponse(followUpResponse),
)
- usage.InputTokens += followUpResponse.Metrics.PromptEvalCount
- usage.OutputTokens += followUpResponse.Metrics.EvalCount
+ followUpUsage := anthropic.UsageFromMetrics(followUpResponse.Metrics)
+ usage.InputTokens += followUpUsage.InputTokens
+ usage.CacheReadInputTokens = addOptionalInts(usage.CacheReadInputTokens, followUpUsage.CacheReadInputTokens)
+ usage.OutputTokens += followUpUsage.OutputTokens
nextToolCall, hasWebSearch, hasOtherTools := findWebSearchToolCall(followUpResponse.Message.ToolCalls)
if hasWebSearch && hasOtherTools {
@@ -377,11 +378,9 @@ func (w *WebSearchAnthropicWriter) startLoopWorker(initialResponse api.ChatRespo
return
}
- initialUsage := anthropic.Usage{
- InputTokens: max(w.observedPromptEvalCount, initialResponse.Metrics.PromptEvalCount),
- OutputTokens: max(w.observedEvalCount, initialResponse.Metrics.EvalCount),
- }
+ initialUsage := w.usageWithObservedMetrics(initialResponse.Metrics)
w.loopBaseInputTok = initialUsage.InputTokens
+ w.loopBaseCacheReadTok = initialUsage.CacheReadInputTokens
w.loopBaseOutputTok = initialUsage.OutputTokens
w.loopResultCh = make(chan webSearchLoopResult, 1)
w.loopInFlight = true
@@ -435,14 +434,20 @@ func (w *WebSearchAnthropicWriter) recordObservedUsage(metrics api.Metrics) {
if metrics.PromptEvalCount > w.observedPromptEvalCount {
w.observedPromptEvalCount = metrics.PromptEvalCount
}
+ w.observedPromptEvalCachedCount = maxOptionalInts(w.observedPromptEvalCachedCount, metrics.PromptEvalCachedCount)
if metrics.EvalCount > w.observedEvalCount {
w.observedEvalCount = metrics.EvalCount
}
}
func (w *WebSearchAnthropicWriter) applyObservedUsageDeltaToUsage(usage *anthropic.Usage) {
- if deltaIn := w.observedPromptEvalCount - w.loopBaseInputTok; deltaIn > 0 {
- usage.InputTokens += deltaIn
+ observed := w.currentObservedUsage()
+ if delta := observed.InputTokens - w.loopBaseInputTok; delta > 0 {
+ usage.InputTokens += delta
+ }
+ if observed.CacheReadInputTokens != nil {
+ delta := max(0, optionalIntValue(observed.CacheReadInputTokens)-optionalIntValue(w.loopBaseCacheReadTok))
+ usage.CacheReadInputTokens = addOptionalInts(usage.CacheReadInputTokens, &delta)
}
if deltaOut := w.observedEvalCount - w.loopBaseOutputTok; deltaOut > 0 {
usage.OutputTokens += deltaOut
@@ -450,10 +455,18 @@ func (w *WebSearchAnthropicWriter) applyObservedUsageDeltaToUsage(usage *anthrop
}
func (w *WebSearchAnthropicWriter) currentObservedUsage() anthropic.Usage {
- return anthropic.Usage{
- InputTokens: w.observedPromptEvalCount,
- OutputTokens: w.observedEvalCount,
- }
+ return anthropic.UsageFromMetrics(api.Metrics{
+ PromptEvalCount: w.observedPromptEvalCount,
+ PromptEvalCachedCount: w.observedPromptEvalCachedCount,
+ EvalCount: w.observedEvalCount,
+ })
+}
+
+func (w *WebSearchAnthropicWriter) usageWithObservedMetrics(metrics api.Metrics) anthropic.Usage {
+ metrics.PromptEvalCount = max(metrics.PromptEvalCount, w.observedPromptEvalCount)
+ metrics.PromptEvalCachedCount = maxOptionalInts(metrics.PromptEvalCachedCount, w.observedPromptEvalCachedCount)
+ metrics.EvalCount = max(metrics.EvalCount, w.observedEvalCount)
+ return anthropic.UsageFromMetrics(metrics)
}
func (w *WebSearchAnthropicWriter) startLoopContext() (context.Context, context.CancelFunc) {
@@ -543,7 +556,7 @@ func (w *WebSearchAnthropicWriter) ensureStreamMessageStart(usage anthropic.Usag
}
inputTokens := usage.InputTokens
- if inputTokens == 0 {
+ if inputTokens == 0 && optionalIntValue(usage.CacheReadInputTokens) == 0 {
inputTokens = w.estimatedInputTokens
}
@@ -556,7 +569,8 @@ func (w *WebSearchAnthropicWriter) ensureStreamMessageStart(usage anthropic.Usag
Model: w.req.Model,
Content: []anthropic.ContentBlock{},
Usage: anthropic.Usage{
- InputTokens: inputTokens,
+ InputTokens: inputTokens,
+ CacheReadInputTokens: usage.CacheReadInputTokens,
},
},
}); err != nil {
@@ -669,8 +683,9 @@ func (w *WebSearchAnthropicWriter) writeTerminalResponse(response anthropic.Mess
StopReason: response.StopReason,
},
Usage: anthropic.DeltaUsage{
- InputTokens: response.Usage.InputTokens,
- OutputTokens: response.Usage.OutputTokens,
+ InputTokens: response.Usage.InputTokens,
+ CacheReadInputTokens: response.Usage.CacheReadInputTokens,
+ OutputTokens: response.Usage.OutputTokens,
},
}); err != nil {
return err
diff --git a/middleware/anthropic_test.go b/middleware/anthropic_test.go
index d43dcb990..b17214fbf 100644
--- a/middleware/anthropic_test.go
+++ b/middleware/anthropic_test.go
@@ -2117,7 +2117,7 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {
Message: api.Message{Role: "assistant", Content: "After search."},
Done: true,
DoneReason: "stop",
- Metrics: api.Metrics{PromptEvalCount: 20, EvalCount: 7},
+ Metrics: api.Metrics{PromptEvalCount: 20, PromptEvalCachedCount: testIntPtr(5), EvalCount: 7},
}
_ = json.NewEncoder(w).Encode(resp)
}))
@@ -2145,7 +2145,7 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {
Model: "test-model",
Message: api.Message{Role: "assistant", Content: "Preface "},
Done: false,
- Metrics: api.Metrics{PromptEvalCount: 12, EvalCount: 4},
+ Metrics: api.Metrics{PromptEvalCount: 12, PromptEvalCachedCount: testIntPtr(4), EvalCount: 4},
},
{
Model: "test-model",
@@ -2169,7 +2169,7 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {
Message: api.Message{Role: "assistant"},
Done: true,
DoneReason: "stop",
- Metrics: api.Metrics{PromptEvalCount: 12, EvalCount: 4},
+ Metrics: api.Metrics{PromptEvalCount: 12, PromptEvalCachedCount: testIntPtr(4), EvalCount: 4},
},
}
c.Writer.WriteHeader(http.StatusOK)
@@ -2212,8 +2212,11 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {
if !found {
t.Fatal("expected message_delta event")
}
- if messageDelta.Usage.InputTokens != 32 {
- t.Fatalf("expected aggregated input tokens 32 (12 passthrough + 20 followup), got %d", messageDelta.Usage.InputTokens)
+ if messageDelta.Usage.InputTokens != 23 {
+ t.Fatalf("expected 23 uncached input tokens, got %d", messageDelta.Usage.InputTokens)
+ }
+ if got := messageDelta.Usage.CacheReadInputTokens; got == nil || *got != 9 {
+ t.Fatalf("expected 9 cached input tokens, got %v", got)
}
if messageDelta.Usage.OutputTokens != 11 {
t.Fatalf("expected aggregated output tokens 11 (4 passthrough + 7 followup), got %d", messageDelta.Usage.OutputTokens)
diff --git a/middleware/metrics.go b/middleware/metrics.go
new file mode 100644
index 000000000..a9a814580
--- /dev/null
+++ b/middleware/metrics.go
@@ -0,0 +1,24 @@
+package middleware
+
+func optionalIntValue(v *int) int {
+ if v == nil {
+ return 0
+ }
+ return *v
+}
+
+func addOptionalInts(a, b *int) *int {
+ if a == nil && b == nil {
+ return nil
+ }
+ value := optionalIntValue(a) + optionalIntValue(b)
+ return &value
+}
+
+func maxOptionalInts(a, b *int) *int {
+ if a == nil && b == nil {
+ return nil
+ }
+ value := max(optionalIntValue(a), optionalIntValue(b))
+ return &value
+}
diff --git a/middleware/openai.go b/middleware/openai.go
index 98e961513..4a8216794 100644
--- a/middleware/openai.go
+++ b/middleware/openai.go
@@ -705,6 +705,7 @@ func (w *WebSearchResponsesWriter) finishStream() error {
var toolCalls []api.ToolCall
for _, response := range w.buffered {
observed.PromptEvalCount = max(observed.PromptEvalCount, response.Metrics.PromptEvalCount)
+ observed.PromptEvalCachedCount = maxOptionalInts(observed.PromptEvalCachedCount, response.Metrics.PromptEvalCachedCount)
observed.EvalCount = max(observed.EvalCount, response.Metrics.EvalCount)
if response.Message.Content != "" {
contentBuilder.WriteString(response.Message.Content)
@@ -877,6 +878,7 @@ func (w *WebSearchResponsesWriter) runLoop(ctx context.Context, initial api.Chat
return api.ChatResponse{}, calls, usage, err
}
usage.PromptEvalCount += followUp.Metrics.PromptEvalCount
+ usage.PromptEvalCachedCount = addOptionalInts(usage.PromptEvalCachedCount, followUp.Metrics.PromptEvalCachedCount)
usage.EvalCount += followUp.Metrics.EvalCount
next, hasWebSearch, mixed := findWebSearchToolCall(followUp.Message.ToolCalls)
@@ -1023,6 +1025,7 @@ func (w *WebSearchResponsesWriter) writeWebSearchResponse(final api.ChatResponse
response.Usage.InputTokens = usage.PromptEvalCount
response.Usage.OutputTokens = usage.EvalCount
response.Usage.TotalTokens = usage.PromptEvalCount + usage.EvalCount
+ response.Usage.InputTokensDetails.CachedTokens = optionalIntValue(usage.PromptEvalCachedCount)
}
w.ResponseWriter.Header().Set("Content-Type", "application/json")
w.done = true
@@ -1121,7 +1124,12 @@ func (w *WebSearchResponsesWriter) writeWebSearchError(err error, usage api.Metr
response := map[string]any{
"id": w.inner.responseID, "object": "response", "status": "failed", "model": w.req.Model,
"output": []any{}, "error": map[string]any{"code": errorCode, "message": message},
- "usage": map[string]any{"input_tokens": usage.PromptEvalCount, "output_tokens": usage.EvalCount, "total_tokens": usage.PromptEvalCount + usage.EvalCount},
+ "usage": map[string]any{
+ "input_tokens": usage.PromptEvalCount,
+ "output_tokens": usage.EvalCount,
+ "total_tokens": usage.PromptEvalCount + usage.EvalCount,
+ "input_tokens_details": map[string]any{"cached_tokens": optionalIntValue(usage.PromptEvalCachedCount)},
+ },
}
initialEvents := w.inner.converter.Process(api.ChatResponse{})
for _, event := range initialEvents {
diff --git a/middleware/openai_test.go b/middleware/openai_test.go
index 2d50e2464..92b2b4110 100644
--- a/middleware/openai_test.go
+++ b/middleware/openai_test.go
@@ -20,6 +20,10 @@ import (
"github.com/ollama/ollama/openai"
)
+func testIntPtr(v int) *int {
+ return &v
+}
+
// testPropsMap creates a ToolPropertiesMap from a map (convenience function for tests)
func testPropsMap(m map[string]api.ToolProperty) *api.ToolPropertiesMap {
props := api.NewToolPropertiesMap()
@@ -683,8 +687,9 @@ func TestChatWriter_StreamMetricsTrailerSkipsEmptyContentChunk(t *testing.T) {
Done: true,
DoneReason: "stop",
Metrics: api.Metrics{
- PromptEvalCount: 3,
- EvalCount: 1,
+ PromptEvalCount: 3,
+ PromptEvalCachedCount: testIntPtr(1),
+ EvalCount: 1,
},
}
data, err = json.Marshal(trailer)
@@ -710,6 +715,9 @@ func TestChatWriter_StreamMetricsTrailerSkipsEmptyContentChunk(t *testing.T) {
if !strings.Contains(frames[2], `"choices":[]`) {
t.Fatalf("expected usage frame with empty choices, got %s", frames[2])
}
+ if !strings.Contains(frames[2], `"prompt_tokens_details":{"cached_tokens":1}`) {
+ t.Fatalf("expected usage frame with cached prompt tokens, got %s", frames[2])
+ }
if frames[3] != "[DONE]" {
t.Fatalf("expected final frame [DONE], got %q", frames[3])
}
diff --git a/middleware/responses_web_search_test.go b/middleware/responses_web_search_test.go
index daba465a7..a19c5763f 100644
--- a/middleware/responses_web_search_test.go
+++ b/middleware/responses_web_search_test.go
@@ -49,11 +49,11 @@ func TestWebSearchResponsesWriterNonStreaming(t *testing.T) {
if strings.Contains(messages[1].Content, "Cite") || !strings.Contains(messages[1].Content, "URL: https://ollama.com/news") {
t.Fatalf("unexpected search result content: %q", messages[1].Content)
}
- return api.ChatResponse{Done: true, Message: api.Message{Role: "assistant", Content: "Read [Ollama](https://ollama.com/news)."}, Metrics: api.Metrics{PromptEvalCount: 7, EvalCount: 3}}, nil
+ return api.ChatResponse{Done: true, Message: api.Message{Role: "assistant", Content: "Read [Ollama](https://ollama.com/news)."}, Metrics: api.Metrics{PromptEvalCount: 7, PromptEvalCachedCount: testIntPtr(3), EvalCount: 3}}, nil
},
}
- initial := api.ChatResponse{Done: true, Message: api.Message{ToolCalls: []api.ToolCall{{ID: "call_1", Function: api.ToolCallFunction{Name: "web_search", Arguments: testArgs(map[string]any{"query": "ollama news"})}}}}, Metrics: api.Metrics{PromptEvalCount: 5, EvalCount: 2}}
+ initial := api.ChatResponse{Done: true, Message: api.Message{ToolCalls: []api.ToolCall{{ID: "call_1", Function: api.ToolCallFunction{Name: "web_search", Arguments: testArgs(map[string]any{"query": "ollama news"})}}}}, Metrics: api.Metrics{PromptEvalCount: 5, PromptEvalCachedCount: testIntPtr(2), EvalCount: 2}}
data, err := json.Marshal(initial)
if err != nil {
t.Fatal(err)
@@ -78,6 +78,9 @@ func TestWebSearchResponsesWriterNonStreaming(t *testing.T) {
if response.Usage == nil || response.Usage.InputTokens != 12 || response.Usage.OutputTokens != 5 {
t.Fatalf("usage = %#v", response.Usage)
}
+ if response.Usage.InputTokensDetails.CachedTokens != 5 {
+ t.Fatalf("cached input tokens = %d, want 5", response.Usage.InputTokensDetails.CachedTokens)
+ }
if len(response.Output[1].Content[0].Annotations) != 0 {
t.Fatalf("annotations = %#v, want none", response.Output[1].Content[0].Annotations)
}
diff --git a/openai/openai.go b/openai/openai.go
index 2d38607db..66b8e69ad 100644
--- a/openai/openai.go
+++ b/openai/openai.go
@@ -72,10 +72,15 @@ type CompleteChunkChoice struct {
Logprobs *ChoiceLogprobs `json:"logprobs,omitempty"`
}
+type PromptTokensDetails struct {
+ CachedTokens int `json:"cached_tokens"`
+}
+
type Usage struct {
- PromptTokens int `json:"prompt_tokens"`
- CompletionTokens int `json:"completion_tokens"`
- TotalTokens int `json:"total_tokens"`
+ PromptTokens int `json:"prompt_tokens"`
+ PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"`
+ CompletionTokens int `json:"completion_tokens"`
+ TotalTokens int `json:"total_tokens"`
}
type ResponseFormat struct {
@@ -238,11 +243,15 @@ func NewError(code int, message string) ErrorResponse {
// ToUsage converts an api.ChatResponse to Usage
func ToUsage(r api.ChatResponse) Usage {
- return Usage{
+ usage := Usage{
PromptTokens: r.Metrics.PromptEvalCount,
CompletionTokens: r.Metrics.EvalCount,
TotalTokens: r.Metrics.PromptEvalCount + r.Metrics.EvalCount,
}
+ if r.Metrics.PromptEvalCachedCount != nil {
+ usage.PromptTokensDetails = &PromptTokensDetails{CachedTokens: *r.Metrics.PromptEvalCachedCount}
+ }
+ return usage
}
// ToToolCalls converts api.ToolCall to OpenAI ToolCall format
@@ -396,11 +405,15 @@ func FinishChunk(id string, r api.ChatResponse, toolCallSent bool) ChatCompletio
// ToUsageGenerate converts an api.GenerateResponse to Usage
func ToUsageGenerate(r api.GenerateResponse) Usage {
- return Usage{
+ usage := Usage{
PromptTokens: r.Metrics.PromptEvalCount,
CompletionTokens: r.Metrics.EvalCount,
TotalTokens: r.Metrics.PromptEvalCount + r.Metrics.EvalCount,
}
+ if r.Metrics.PromptEvalCachedCount != nil {
+ usage.PromptTokensDetails = &PromptTokensDetails{CachedTokens: *r.Metrics.PromptEvalCachedCount}
+ }
+ return usage
}
// ToCompletion converts an api.GenerateResponse to Completion
diff --git a/openai/openai_test.go b/openai/openai_test.go
index 1f3b258b0..a8c53cdae 100644
--- a/openai/openai_test.go
+++ b/openai/openai_test.go
@@ -3,6 +3,7 @@ package openai
import (
"encoding/base64"
"encoding/json"
+ "strings"
"testing"
"time"
@@ -30,6 +31,10 @@ const (
image = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=`
)
+func testIntPtr(v int) *int {
+ return &v
+}
+
func TestFromChatRequest_Basic(t *testing.T) {
req := ChatCompletionRequest{
Model: "test-model",
@@ -180,8 +185,9 @@ func TestFromCompleteRequest_Basic(t *testing.T) {
func TestToUsage(t *testing.T) {
resp := api.ChatResponse{
Metrics: api.Metrics{
- PromptEvalCount: 10,
- EvalCount: 20,
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ EvalCount: 20,
},
}
@@ -190,6 +196,9 @@ func TestToUsage(t *testing.T) {
if usage.PromptTokens != 10 {
t.Errorf("expected PromptTokens 10, got %d", usage.PromptTokens)
}
+ if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 4 {
+ t.Errorf("expected CachedTokens 4, got %#v", usage.PromptTokensDetails)
+ }
if usage.CompletionTokens != 20 {
t.Errorf("expected CompletionTokens 20, got %d", usage.CompletionTokens)
@@ -198,6 +207,68 @@ func TestToUsage(t *testing.T) {
if usage.TotalTokens != 30 {
t.Errorf("expected TotalTokens 30, got %d", usage.TotalTokens)
}
+
+ data, err := json.Marshal(usage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(data), `"prompt_tokens_details":{"cached_tokens":4}`) {
+ t.Errorf("unexpected usage json: %s", data)
+ }
+}
+
+func TestToUsageOmitsUnreportedCacheDetails(t *testing.T) {
+ usage := ToUsage(api.ChatResponse{Metrics: api.Metrics{PromptEvalCount: 10, EvalCount: 2}})
+ if usage.PromptTokensDetails != nil {
+ t.Fatalf("expected no cache details, got %#v", usage.PromptTokensDetails)
+ }
+
+ data, err := json.Marshal(usage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var payload map[string]any
+ if err := json.Unmarshal(data, &payload); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := payload["prompt_tokens_details"]; ok {
+ t.Fatalf("unexpected cache details in %s", data)
+ }
+}
+
+func TestToUsageIncludesZeroCacheDetails(t *testing.T) {
+ usage := ToUsage(api.ChatResponse{Metrics: api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(0),
+ EvalCount: 2,
+ }})
+ if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 0 {
+ t.Fatalf("expected zero cache details, got %#v", usage.PromptTokensDetails)
+ }
+
+ data, err := json.Marshal(usage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(data), `"prompt_tokens_details":{"cached_tokens":0}`) {
+ t.Errorf("unexpected usage json: %s", data)
+ }
+}
+
+func TestToCompletionUsageIncludesCachedTokens(t *testing.T) {
+ completion := ToCompletion("completion-id", api.GenerateResponse{
+ Metrics: api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ EvalCount: 2,
+ },
+ })
+ if completion.Usage.PromptTokens != 10 || completion.Usage.TotalTokens != 12 {
+ t.Fatalf("unexpected usage: %#v", completion.Usage)
+ }
+ if details := completion.Usage.PromptTokensDetails; details == nil || details.CachedTokens != 4 {
+ t.Fatalf("expected 4 cached tokens, got %#v", details)
+ }
}
func TestNewError(t *testing.T) {
diff --git a/openai/responses.go b/openai/responses.go
index 4a399ad6e..e32c8ac66 100644
--- a/openai/responses.go
+++ b/openai/responses.go
@@ -885,6 +885,13 @@ type ResponsesUsage struct {
OutputTokensDetails ResponsesOutputTokensDetails `json:"output_tokens_details"`
}
+func intValue(v *int) int {
+ if v == nil {
+ return 0
+ }
+ return *v
+}
+
// derefFloat64 returns the value of a float64 pointer, or a default if nil.
func derefFloat64(p *float64, def float64) float64 {
if p != nil {
@@ -1001,11 +1008,10 @@ func ToResponse(model, responseID, itemID string, chatResponse api.ChatResponse,
Temperature: derefFloat64(request.Temperature, 1.0),
Reasoning: reasoning,
Usage: &ResponsesUsage{
- InputTokens: chatResponse.PromptEvalCount,
- OutputTokens: chatResponse.EvalCount,
- TotalTokens: chatResponse.PromptEvalCount + chatResponse.EvalCount,
- // TODO(drifkin): wire through the actual values
- InputTokensDetails: ResponsesInputTokensDetails{CachedTokens: 0},
+ InputTokens: chatResponse.PromptEvalCount,
+ OutputTokens: chatResponse.EvalCount,
+ TotalTokens: chatResponse.PromptEvalCount + chatResponse.EvalCount,
+ InputTokensDetails: ResponsesInputTokensDetails{CachedTokens: intValue(chatResponse.PromptEvalCachedCount)},
// TODO(drifkin): wire through the actual values
OutputTokensDetails: ResponsesOutputTokensDetails{ReasoningTokens: 0},
},
@@ -1608,7 +1614,7 @@ func (c *ResponsesStreamConverter) processCompletion(r api.ChatResponse) []Respo
"output_tokens": r.EvalCount,
"total_tokens": r.PromptEvalCount + r.EvalCount,
"input_tokens_details": map[string]any{
- "cached_tokens": 0,
+ "cached_tokens": intValue(r.PromptEvalCachedCount),
},
"output_tokens_details": map[string]any{
"reasoning_tokens": 0,
diff --git a/openai/responses_test.go b/openai/responses_test.go
index 98c20c581..b43bed29f 100644
--- a/openai/responses_test.go
+++ b/openai/responses_test.go
@@ -2,6 +2,7 @@ package openai
import (
"encoding/json"
+ "strings"
"testing"
"time"
@@ -1743,6 +1744,34 @@ func TestToResponse_WithReasoning(t *testing.T) {
}
}
+func TestToResponse_UsageIncludesCachedTokens(t *testing.T) {
+ response := ToResponse("gpt-oss:20b", "resp_123", "msg_456", api.ChatResponse{
+ CreatedAt: time.Now(),
+ Message: api.Message{Content: "The answer is 42"},
+ Done: true,
+ Metrics: api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ EvalCount: 3,
+ },
+ }, ResponsesRequest{})
+
+ if response.Usage == nil {
+ t.Fatal("expected usage")
+ }
+ if response.Usage.InputTokens != 10 || response.Usage.InputTokensDetails.CachedTokens != 4 || response.Usage.TotalTokens != 13 {
+ t.Errorf("unexpected usage: %+v", response.Usage)
+ }
+
+ data, err := json.Marshal(response.Usage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(data), `"input_tokens_details":{"cached_tokens":4}`) {
+ t.Errorf("unexpected usage json: %s", data)
+ }
+}
+
func TestFromResponsesRequest_Instructions(t *testing.T) {
reqJSON := `{
"model": "gpt-oss:20b",
@@ -2143,6 +2172,11 @@ func TestResponsesStreamConverter_ResponseCompletedIncludesOutput(t *testing.T)
events := converter.Process(api.ChatResponse{
Message: api.Message{},
Done: true,
+ Metrics: api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ EvalCount: 3,
+ },
})
// Find the response.completed event
@@ -2172,6 +2206,11 @@ func TestResponsesStreamConverter_ResponseCompletedIncludesOutput(t *testing.T)
if item["type"] != "message" {
t.Errorf("output[0].type = %q, want %q", item["type"], "message")
}
+ usage := response["usage"].(map[string]any)
+ inputDetails := usage["input_tokens_details"].(map[string]any)
+ if inputDetails["cached_tokens"] != 4 {
+ t.Errorf("cached_tokens = %v, want 4", inputDetails["cached_tokens"])
+ }
}
func TestResponsesStreamConverter_ResponseCreatedIncludesOutput(t *testing.T) {
diff --git a/server/routes.go b/server/routes.go
index 53e392978..263c7e27a 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -677,10 +677,11 @@ func (s *Server) GenerateHandler(c *gin.Context) {
Response: cr.Content,
Done: cr.Done,
Metrics: api.Metrics{
- PromptEvalCount: cr.PromptEvalCount,
- PromptEvalDuration: cr.PromptEvalDuration,
- EvalCount: cr.EvalCount,
- EvalDuration: cr.EvalDuration,
+ PromptEvalCount: cr.PromptEvalCount,
+ PromptEvalCachedCount: cr.PromptEvalCachedCount,
+ PromptEvalDuration: cr.PromptEvalDuration,
+ EvalCount: cr.EvalCount,
+ EvalDuration: cr.EvalDuration,
},
Logprobs: toAPILogprobs(cr.Logprobs),
}
@@ -2750,6 +2751,7 @@ func (s *Server) ChatHandler(c *gin.Context) {
defer close(ch)
structuredOutputsState := structuredOutputsState_None
+ var firstPassMetrics api.Metrics
for {
var tb strings.Builder
@@ -2768,6 +2770,7 @@ func (s *Server) ChatHandler(c *gin.Context) {
if req.Format != nil && structuredOutputsState == structuredOutputsState_None && !forceImmediate && ((builtinParser != nil || thinkingState != nil) && slices.Contains(m.Capabilities(), model.CapabilityThinking)) {
currentFormat = nil
}
+ includeIntermediateMetrics := req.Format != nil && currentFormat == nil
// sets up new context given parent context per request
ctx, cancel := context.WithCancel(c.Request.Context())
@@ -2775,30 +2778,47 @@ func (s *Server) ChatHandler(c *gin.Context) {
var parserErr error
err := r.Completion(ctx, llm.CompletionRequest{
- Prompt: prompt,
- Media: media,
- Format: currentFormat,
- Options: opts,
- Shift: req.Shift == nil || *req.Shift,
- Truncate: truncate,
- Logprobs: req.Logprobs,
- TopLogprobs: req.TopLogprobs,
- PreservedTokens: preservedTokensForCompletion(builtinParser),
- ToolCallTag: toolCallTagForCompletion(toolParser),
- LeadingBOS: leadingBOSForModel(m),
+ Prompt: prompt,
+ Media: media,
+ Format: currentFormat,
+ Options: opts,
+ Shift: req.Shift == nil || *req.Shift,
+ Truncate: truncate,
+ Logprobs: req.Logprobs,
+ TopLogprobs: req.TopLogprobs,
+ PreservedTokens: preservedTokensForCompletion(builtinParser),
+ ToolCallTag: toolCallTagForCompletion(toolParser),
+ LeadingBOS: leadingBOSForModel(m),
+ IncludeIntermediateMetrics: includeIntermediateMetrics,
}, func(r llm.CompletionResponse) {
+ metrics := api.Metrics{
+ PromptEvalCount: r.PromptEvalCount,
+ PromptEvalCachedCount: r.PromptEvalCachedCount,
+ PromptEvalDuration: r.PromptEvalDuration,
+ EvalCount: r.EvalCount,
+ EvalDuration: r.EvalDuration,
+ }
+ if includeIntermediateMetrics {
+ firstPassMetrics = metrics
+ if !r.Done {
+ metrics = api.Metrics{}
+ }
+ } else if structuredOutputsState == structuredOutputsState_Applying && r.Done {
+ // Treat the restart as generation work: retain the original prompt metrics and fold in the second prefill.
+ metrics.PromptEvalCount = firstPassMetrics.PromptEvalCount
+ metrics.PromptEvalCachedCount = firstPassMetrics.PromptEvalCachedCount
+ metrics.PromptEvalDuration = firstPassMetrics.PromptEvalDuration
+ metrics.EvalCount += firstPassMetrics.EvalCount
+ metrics.EvalDuration += firstPassMetrics.EvalDuration + r.PromptEvalDuration
+ }
+
res := api.ChatResponse{
Model: req.Model,
CreatedAt: time.Now().UTC(),
Message: api.Message{Role: "assistant", Content: r.Content},
Done: r.Done,
- Metrics: api.Metrics{
- PromptEvalCount: r.PromptEvalCount,
- PromptEvalDuration: r.PromptEvalDuration,
- EvalCount: r.EvalCount,
- EvalDuration: r.EvalDuration,
- },
- Logprobs: toAPILogprobs(r.Logprobs),
+ Metrics: metrics,
+ Logprobs: toAPILogprobs(r.Logprobs),
}
if r.Done {
@@ -3006,10 +3026,11 @@ func (s *Server) handleNativeChat(c *gin.Context, req api.ChatRequest, m *Model,
Message: r.Message,
Done: r.Done,
Metrics: api.Metrics{
- PromptEvalCount: r.PromptEvalCount,
- PromptEvalDuration: r.PromptEvalDuration,
- EvalCount: r.EvalCount,
- EvalDuration: r.EvalDuration,
+ PromptEvalCount: r.PromptEvalCount,
+ PromptEvalCachedCount: r.PromptEvalCachedCount,
+ PromptEvalDuration: r.PromptEvalDuration,
+ EvalCount: r.EvalCount,
+ EvalDuration: r.EvalDuration,
},
Logprobs: toAPILogprobs(r.Logprobs),
}
diff --git a/server/routes_cloud_test.go b/server/routes_cloud_test.go
index 6868c5806..6ca8929cb 100644
--- a/server/routes_cloud_test.go
+++ b/server/routes_cloud_test.go
@@ -215,7 +215,8 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {
})
t.Run("api chat", func(t *testing.T) {
- upstream, capture := newUpstream(t, `{"message":{"role":"assistant","content":"ok"},"done":true}`)
+ upstreamResponse := `{"message":{"role":"assistant","content":"ok"},"done":true,"prompt_eval_count":12,"prompt_eval_cached_count":9}`
+ upstream, capture := newUpstream(t, upstreamResponse)
defer upstream.Close()
original := cloudProxyBaseURL
@@ -247,6 +248,9 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected status 200, got %d (%s)", resp.StatusCode, string(body))
}
+ if got := string(body); got != upstreamResponse {
+ t.Fatalf("cloud response changed: got %q, want %q", got, upstreamResponse)
+ }
if capture.path != "/api/chat" {
t.Fatalf("expected upstream path /api/chat, got %q", capture.path)
@@ -387,7 +391,8 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {
})
t.Run("v1 chat completions bypasses conversion", func(t *testing.T) {
- upstream, capture := newUpstream(t, `{"id":"chatcmpl_test","object":"chat.completion"}`)
+ upstreamResponse := `{"id":"chatcmpl_test","object":"chat.completion","usage":{"prompt_tokens":12,"prompt_tokens_details":{"cached_tokens":9},"completion_tokens":3,"total_tokens":15}}`
+ upstream, capture := newUpstream(t, upstreamResponse)
defer upstream.Close()
original := cloudProxyBaseURL
@@ -420,6 +425,9 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected status 200, got %d (%s)", resp.StatusCode, string(body))
}
+ if got := string(body); got != upstreamResponse {
+ t.Fatalf("cloud response changed: got %q, want %q", got, upstreamResponse)
+ }
if capture.path != "/v1/chat/completions" {
t.Fatalf("expected upstream path /v1/chat/completions, got %q", capture.path)
@@ -811,10 +819,10 @@ func TestCloudResponsesWebSearchUsesLocalOrchestration(t *testing.T) {
w.Header().Set("Content-Type", "application/x-ndjson")
if chatCalls == 1 {
_, _ = io.WriteString(w, `{"message":{"role":"assistant","tool_calls":[{"id":"call_1","function":{"name":"web_search","arguments":{"query":"latest Ollama release"}}}]},"done":false}`+"\n")
- _, _ = io.WriteString(w, `{"message":{"role":"assistant"},"done":true,"prompt_eval_count":12,"eval_count":4}`+"\n")
+ _, _ = io.WriteString(w, `{"message":{"role":"assistant"},"done":true,"prompt_eval_count":12,"prompt_eval_cached_count":5,"eval_count":4}`+"\n")
return
}
- _, _ = io.WriteString(w, `{"message":{"role":"assistant","content":"Ollama [release](https://ollama.com/release)."},"done":true,"prompt_eval_count":20,"eval_count":6}`)
+ _, _ = io.WriteString(w, `{"message":{"role":"assistant","content":"Ollama [release](https://ollama.com/release)."},"done":true,"prompt_eval_count":20,"prompt_eval_cached_count":17,"eval_count":6}`)
case "/api/web_search":
searchCalls++
w.Header().Set("Content-Type", "application/json")
@@ -873,6 +881,9 @@ func TestCloudResponsesWebSearchUsesLocalOrchestration(t *testing.T) {
if bytes.Contains(body, []byte("response.function_call_arguments")) || bytes.Contains(body, []byte(`"type":"function_call"`)) {
t.Fatalf("private web_search function leaked: %s", body)
}
+ if !bytes.Contains(body, []byte(`"input_tokens_details":{"cached_tokens":22}`)) {
+ t.Fatalf("missing aggregated cached token usage: %s", body)
+ }
}
func TestCloudResponsesUnsupportedWebSearchPassthrough(t *testing.T) {
diff --git a/server/routes_generate_test.go b/server/routes_generate_test.go
index 5de22f78b..f3decb881 100644
--- a/server/routes_generate_test.go
+++ b/server/routes_generate_test.go
@@ -279,13 +279,14 @@ func TestChatHandlerChatTemplateRoute(t *testing.T) {
mock := mockRunner{
ChatFn: func(_ context.Context, req llm.ChatRequest, fn func(llm.ChatResponse)) error {
fn(llm.ChatResponse{
- Message: api.Message{Role: "assistant", Content: "chat template response"},
- Done: true,
- DoneReason: llm.DoneReasonStop,
- PromptEvalCount: 1,
- PromptEvalDuration: time.Millisecond,
- EvalCount: 2,
- EvalDuration: 2 * time.Millisecond,
+ Message: api.Message{Role: "assistant", Content: "chat template response"},
+ Done: true,
+ DoneReason: llm.DoneReasonStop,
+ PromptEvalCount: 2,
+ PromptEvalCachedCount: testIntPtr(1),
+ PromptEvalDuration: time.Millisecond,
+ EvalCount: 2,
+ EvalDuration: 2 * time.Millisecond,
})
return nil
},
@@ -314,6 +315,9 @@ func TestChatHandlerChatTemplateRoute(t *testing.T) {
if actual.Message.Content != "chat template response" {
t.Fatalf("expected chat template response, got %q", actual.Message.Content)
}
+ if actual.PromptEvalCount != 2 || actual.PromptEvalCachedCount == nil || *actual.PromptEvalCachedCount != 1 {
+ t.Errorf("prompt counts = (%d, %v), want (2, 1)", actual.PromptEvalCount, actual.PromptEvalCachedCount)
+ }
if len(mock.ChatRequest.Messages) != 1 || mock.ChatRequest.Messages[0].Content != "hello" {
t.Fatalf("chat_template request messages = %#v", mock.ChatRequest.Messages)
}
@@ -753,12 +757,13 @@ func TestGenerateChat(t *testing.T) {
mock := mockRunner{
CompletionResponse: llm.CompletionResponse{
- Done: true,
- DoneReason: llm.DoneReasonStop,
- PromptEvalCount: 1,
- PromptEvalDuration: 1,
- EvalCount: 1,
- EvalDuration: 1,
+ Done: true,
+ DoneReason: llm.DoneReasonStop,
+ PromptEvalCount: 2,
+ PromptEvalCachedCount: testIntPtr(1),
+ PromptEvalDuration: 1,
+ EvalCount: 1,
+ EvalDuration: 1,
},
}
@@ -970,6 +975,9 @@ func TestGenerateChat(t *testing.T) {
if actual.PromptEvalCount == 0 {
t.Errorf("expected prompt eval count > 0, got 0")
}
+ if actual.PromptEvalCachedCount == nil || *actual.PromptEvalCachedCount != 1 {
+ t.Errorf("expected cached prompt eval count 1, got %v", actual.PromptEvalCachedCount)
+ }
if actual.PromptEvalDuration == 0 {
t.Errorf("expected prompt eval duration > 0, got 0")
@@ -2574,6 +2582,35 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
}
})
+ earlyFirstPassMetrics := api.Metrics{
+ PromptEvalCount: 4,
+ PromptEvalCachedCount: testIntPtr(1),
+ PromptEvalDuration: 5 * time.Millisecond,
+ EvalCount: 6,
+ EvalDuration: 7 * time.Millisecond,
+ }
+ firstPassMetrics := api.Metrics{
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ PromptEvalDuration: 11 * time.Millisecond,
+ EvalCount: 12,
+ EvalDuration: 13 * time.Millisecond,
+ }
+ secondPassMetrics := api.Metrics{
+ PromptEvalCount: 20_000,
+ PromptEvalCachedCount: testIntPtr(19_000),
+ PromptEvalDuration: 21 * time.Millisecond,
+ EvalCount: 22,
+ EvalDuration: 23 * time.Millisecond,
+ }
+ wantMetrics := api.Metrics{
+ PromptEvalCount: firstPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: firstPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: firstPassMetrics.PromptEvalDuration,
+ EvalCount: firstPassMetrics.EvalCount + secondPassMetrics.EvalCount,
+ EvalDuration: firstPassMetrics.EvalDuration + secondPassMetrics.PromptEvalDuration + secondPassMetrics.EvalDuration,
+ }
+
t.Run("structured outputs restart non-stream", func(t *testing.T) {
var (
requestsMu sync.Mutex
@@ -2596,10 +2633,22 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
switch callNum {
case 1:
fn(llm.CompletionResponse{
- Content: " I am thinking through this problem. </think> {\"answer\":\"42\"}",
- Done: false,
- PromptEvalCount: 1,
- PromptEvalDuration: 1,
+ Content: " I am thinking through this problem.",
+ Done: false,
+ PromptEvalCount: earlyFirstPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: earlyFirstPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: earlyFirstPassMetrics.PromptEvalDuration,
+ EvalCount: earlyFirstPassMetrics.EvalCount,
+ EvalDuration: earlyFirstPassMetrics.EvalDuration,
+ })
+ fn(llm.CompletionResponse{
+ Content: " </think> {\"answer\":\"42\"}",
+ Done: false,
+ PromptEvalCount: firstPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: firstPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: firstPassMetrics.PromptEvalDuration,
+ EvalCount: firstPassMetrics.EvalCount,
+ EvalDuration: firstPassMetrics.EvalDuration,
})
select {
@@ -2611,13 +2660,14 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
}
case 2:
fn(llm.CompletionResponse{
- Content: `{"answer":"42"}`,
- Done: true,
- DoneReason: llm.DoneReasonStop,
- PromptEvalCount: 1,
- PromptEvalDuration: 1,
- EvalCount: 1,
- EvalDuration: 1,
+ Content: `{"answer":"42"}`,
+ Done: true,
+ DoneReason: llm.DoneReasonStop,
+ PromptEvalCount: secondPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: secondPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: secondPassMetrics.PromptEvalDuration,
+ EvalCount: secondPassMetrics.EvalCount,
+ EvalDuration: secondPassMetrics.EvalDuration,
})
return nil
default:
@@ -2650,10 +2700,16 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
if requests[0].Format != nil {
t.Errorf("expected first completion format to be nil, got %q", requests[0].Format)
}
+ if !requests[0].IncludeIntermediateMetrics {
+ t.Error("expected first completion to request per-token metrics")
+ }
if !bytes.Equal([]byte(format), []byte(requests[1].Format)) {
t.Errorf("expected second completion format to match original format")
}
+ if requests[1].IncludeIntermediateMetrics {
+ t.Error("expected second completion to use terminal metrics")
+ }
var resp api.ChatResponse
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
@@ -2675,6 +2731,15 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
if resp.DoneReason != "stop" {
t.Errorf("expected done reason stop, got %s", resp.DoneReason)
}
+ if resp.PromptEvalCount != wantMetrics.PromptEvalCount || resp.EvalCount != wantMetrics.EvalCount {
+ t.Errorf("response counts = (%d, %d), want (%d, %d)", resp.PromptEvalCount, resp.EvalCount, wantMetrics.PromptEvalCount, wantMetrics.EvalCount)
+ }
+ if diff := cmp.Diff(wantMetrics.PromptEvalCachedCount, resp.PromptEvalCachedCount); diff != "" {
+ t.Errorf("response cached prompt count mismatch (-want +got):\n%s", diff)
+ }
+ if resp.PromptEvalDuration != wantMetrics.PromptEvalDuration || resp.EvalDuration != wantMetrics.EvalDuration {
+ t.Errorf("response durations = (%s, %s), want (%s, %s)", resp.PromptEvalDuration, resp.EvalDuration, wantMetrics.PromptEvalDuration, wantMetrics.EvalDuration)
+ }
})
t.Run("structured outputs restart streaming", func(t *testing.T) {
@@ -2699,10 +2764,22 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
switch callNum {
case 1:
fn(llm.CompletionResponse{
- Content: " I am thinking through this problem. </think> {\"answer\":\"42\"}",
- Done: false,
- PromptEvalCount: 1,
- PromptEvalDuration: 1,
+ Content: " I am thinking through this problem.",
+ Done: false,
+ PromptEvalCount: earlyFirstPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: earlyFirstPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: earlyFirstPassMetrics.PromptEvalDuration,
+ EvalCount: earlyFirstPassMetrics.EvalCount,
+ EvalDuration: earlyFirstPassMetrics.EvalDuration,
+ })
+ fn(llm.CompletionResponse{
+ Content: " </think> {\"answer\":\"42\"}",
+ Done: false,
+ PromptEvalCount: firstPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: firstPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: firstPassMetrics.PromptEvalDuration,
+ EvalCount: firstPassMetrics.EvalCount,
+ EvalDuration: firstPassMetrics.EvalDuration,
})
select {
@@ -2714,13 +2791,14 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
}
case 2:
fn(llm.CompletionResponse{
- Content: `{"answer":"42"}`,
- Done: true,
- DoneReason: llm.DoneReasonStop,
- PromptEvalCount: 1,
- PromptEvalDuration: 1,
- EvalCount: 1,
- EvalDuration: 1,
+ Content: `{"answer":"42"}`,
+ Done: true,
+ DoneReason: llm.DoneReasonStop,
+ PromptEvalCount: secondPassMetrics.PromptEvalCount,
+ PromptEvalCachedCount: secondPassMetrics.PromptEvalCachedCount,
+ PromptEvalDuration: secondPassMetrics.PromptEvalDuration,
+ EvalCount: secondPassMetrics.EvalCount,
+ EvalDuration: secondPassMetrics.EvalDuration,
})
return nil
default:
@@ -2753,10 +2831,16 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
if requests[0].Format != nil {
t.Errorf("expected first completion format to be nil, got %q", requests[0].Format)
}
+ if !requests[0].IncludeIntermediateMetrics {
+ t.Error("expected first completion to request per-token metrics")
+ }
if !bytes.Equal([]byte(format), []byte(requests[1].Format)) {
t.Errorf("expected second completion format to match original format")
}
+ if requests[1].IncludeIntermediateMetrics {
+ t.Error("expected second completion to use terminal metrics")
+ }
decoder := json.NewDecoder(w.Body)
var events []api.ChatResponse
@@ -2778,8 +2862,15 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
}
first := events[0]
- if first.Message.Thinking != "I am thinking through this problem. " {
- t.Errorf("expected first event thinking %q, got %q", "I am thinking through this problem. ", first.Message.Thinking)
+ var thinking strings.Builder
+ for _, event := range events {
+ thinking.WriteString(event.Message.Thinking)
+ if !event.Done && (event.PromptEvalCount != 0 || event.PromptEvalCachedCount != nil || event.PromptEvalDuration != 0 || event.EvalCount != 0 || event.EvalDuration != 0) {
+ t.Errorf("non-terminal event unexpectedly exposed metrics: %+v", event.Metrics)
+ }
+ }
+ if got := thinking.String(); got != "I am thinking through this problem. " {
+ t.Errorf("thinking = %q, want %q", got, "I am thinking through this problem. ")
}
if first.Message.Content != "" {
@@ -2789,7 +2880,6 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
if first.Done {
t.Error("expected first event to be non-terminal")
}
-
last := events[len(events)-1]
if last.Message.Thinking != "" {
t.Errorf("expected final event thinking to be empty, got %q", last.Message.Thinking)
@@ -2806,6 +2896,15 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {
if last.DoneReason != "stop" {
t.Errorf("expected final done reason stop, got %s", last.DoneReason)
}
+ if last.PromptEvalCount != wantMetrics.PromptEvalCount || last.EvalCount != wantMetrics.EvalCount {
+ t.Errorf("final counts = (%d, %d), want (%d, %d)", last.PromptEvalCount, last.EvalCount, wantMetrics.PromptEvalCount, wantMetrics.EvalCount)
+ }
+ if diff := cmp.Diff(wantMetrics.PromptEvalCachedCount, last.PromptEvalCachedCount); diff != "" {
+ t.Errorf("final cached prompt count mismatch (-want +got):\n%s", diff)
+ }
+ if last.PromptEvalDuration != wantMetrics.PromptEvalDuration || last.EvalDuration != wantMetrics.EvalDuration {
+ t.Errorf("final durations = (%s, %s), want (%s, %s)", last.PromptEvalDuration, last.EvalDuration, wantMetrics.PromptEvalDuration, wantMetrics.EvalDuration)
+ }
})
}
diff --git a/x/mlxrunner/client.go b/x/mlxrunner/client.go
index 71debf9c2..5c6046e3f 100644
--- a/x/mlxrunner/client.go
+++ b/x/mlxrunner/client.go
@@ -111,12 +111,13 @@ func (c *Client) WaitUntilRunning(ctx context.Context) error {
}
type CompletionRequest struct {
- Prompt string
- Media []llm.MediaData
- Format json.RawMessage
- Options api.Options
- Logprobs bool
- TopLogprobs int
+ Prompt string
+ Media []llm.MediaData
+ Format json.RawMessage
+ Options api.Options
+ Logprobs bool
+ TopLogprobs int
+ IncludeIntermediateMetrics bool
}
type CompletionResponse struct {
@@ -124,10 +125,11 @@ type CompletionResponse struct {
Done bool
DoneReason int
- PromptEvalCount int
- PromptEvalDuration time.Duration
- EvalCount int
- EvalDuration time.Duration
+ PromptEvalCount int
+ PromptEvalCachedCount *int
+ PromptEvalDuration time.Duration
+ EvalCount int
+ EvalDuration time.Duration
Logprobs []llm.Logprob
@@ -156,11 +158,12 @@ func (c *Client) Close() error {
// Completion implements llm.LlamaServer.
func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error {
creq := CompletionRequest{
- Prompt: req.Prompt,
- Media: req.Media,
- Format: req.Format,
- Logprobs: req.Logprobs,
- TopLogprobs: req.TopLogprobs,
+ Prompt: req.Prompt,
+ Media: req.Media,
+ Format: req.Format,
+ Logprobs: req.Logprobs,
+ TopLogprobs: req.TopLogprobs,
+ IncludeIntermediateMetrics: req.IncludeIntermediateMetrics,
}
if req.Options != nil {
creq.Options = *req.Options
@@ -205,14 +208,15 @@ func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn f
}
cresp := llm.CompletionResponse{
- Content: raw.Content,
- Done: raw.Done,
- DoneReason: llm.DoneReason(raw.DoneReason),
- PromptEvalCount: raw.PromptEvalCount,
- PromptEvalDuration: raw.PromptEvalDuration,
- EvalCount: raw.EvalCount,
- EvalDuration: raw.EvalDuration,
- Logprobs: raw.Logprobs,
+ Content: raw.Content,
+ Done: raw.Done,
+ DoneReason: llm.DoneReason(raw.DoneReason),
+ PromptEvalCount: raw.PromptEvalCount,
+ PromptEvalCachedCount: raw.PromptEvalCachedCount,
+ PromptEvalDuration: raw.PromptEvalDuration,
+ EvalCount: raw.EvalCount,
+ EvalDuration: raw.EvalDuration,
+ Logprobs: raw.Logprobs,
}
fn(cresp)
diff --git a/x/mlxrunner/client_test.go b/x/mlxrunner/client_test.go
new file mode 100644
index 000000000..fb56db618
--- /dev/null
+++ b/x/mlxrunner/client_test.go
@@ -0,0 +1,60 @@
+package mlxrunner
+
+import (
+ "encoding/json"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "testing"
+
+ "github.com/ollama/ollama/api"
+ "github.com/ollama/ollama/llm"
+)
+
+func testIntPtr(v int) *int {
+ return &v
+}
+
+func TestClientCompletionRequestsIntermediateMetrics(t *testing.T) {
+ var request CompletionRequest
+ want := CompletionResponse{
+ Done: true,
+ PromptEvalCount: 10,
+ PromptEvalCachedCount: testIntPtr(4),
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Errorf("decode request: %v", err)
+ return
+ }
+ if err := json.NewEncoder(w).Encode(want); err != nil {
+ t.Errorf("encode response: %v", err)
+ }
+ }))
+ t.Cleanup(srv.Close)
+
+ _, portString, err := net.SplitHostPort(srv.Listener.Addr().String())
+ if err != nil {
+ t.Fatalf("parse server port: %v", err)
+ }
+ port, err := strconv.Atoi(portString)
+ if err != nil {
+ t.Fatalf("parse server port: %v", err)
+ }
+ client := &Client{port: port, client: srv.Client()}
+ opts := api.DefaultOptions()
+ var got llm.CompletionResponse
+ if err := client.Completion(t.Context(), llm.CompletionRequest{
+ Options: &opts,
+ IncludeIntermediateMetrics: true,
+ }, func(response llm.CompletionResponse) { got = response }); err != nil {
+ t.Fatalf("Completion: %v", err)
+ }
+ if !request.IncludeIntermediateMetrics {
+ t.Fatal("metrics per token was not forwarded to the MLX runner")
+ }
+ if got.PromptEvalCount != want.PromptEvalCount || got.PromptEvalCachedCount == nil || *got.PromptEvalCachedCount != *want.PromptEvalCachedCount {
+ t.Errorf("prompt counts = (%d, %v), want (%d, %d)", got.PromptEvalCount, got.PromptEvalCachedCount, want.PromptEvalCount, *want.PromptEvalCachedCount)
+ }
+}
diff --git a/x/mlxrunner/mtp_test.go b/x/mlxrunner/mtp_test.go
index d3106e275..68eebc1dc 100644
--- a/x/mlxrunner/mtp_test.go
+++ b/x/mlxrunner/mtp_test.go
@@ -9,6 +9,7 @@ import (
"slices"
"strings"
"testing"
+ "time"
"github.com/ollama/ollama/api"
"github.com/ollama/ollama/x/internal/mlxtest"
@@ -438,6 +439,55 @@ func TestRunMTPDecodeGreedy(t *testing.T) {
})
}
+func TestDecodeIntermediateMetrics(t *testing.T) {
+ skipIfNoMLX(t)
+ const eos int32 = 7
+ predict := map[int32]int32{1: 2, 2: 3, 3: eos, eos: 0}
+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})
+ caches, _ := newMTPTestCaches(1)
+ session, ch := newMTPTestSession(caches)
+ session.inputs = []int32{0, 1}
+ session.remaining = []int32{1}
+ req := Request{
+ Responses: ch,
+ Tokens: session.inputs,
+ CompletionRequest: CompletionRequest{
+ Options: api.Options{NumPredict: 20},
+ IncludeIntermediateMetrics: true,
+ },
+ }
+ d := testDecoder(t, r, req, caches, []int32{1}, 1)
+ promptEval := 5 * time.Millisecond
+ if err := r.decode(context.Background(), req, session, d, promptEval); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ d.close()
+
+ var responses []CompletionResponse
+ for len(ch) > 0 {
+ resp := <-ch
+ responses = append(responses, resp)
+ }
+ if len(responses) != 3 {
+ t.Fatalf("got %d responses, want 3", len(responses))
+ }
+ for i, resp := range responses[:2] {
+ if resp.PromptEvalCount != 2 || resp.PromptEvalCachedCount == nil || *resp.PromptEvalCachedCount != 1 || resp.PromptEvalDuration != promptEval {
+ t.Errorf("response[%d] prompt metrics = (%d, %v, %s), want (2, 1, %s)", i, resp.PromptEvalCount, resp.PromptEvalCachedCount, resp.PromptEvalDuration, promptEval)
+ }
+ if resp.EvalCount != i+1 || resp.EvalDuration <= 0 {
+ t.Errorf("response[%d] eval metrics = (%d, %s), want count %d and positive duration", i, resp.EvalCount, resp.EvalDuration, i+1)
+ }
+ }
+ final := responses[2]
+ if final.PromptEvalCount != 2 || final.PromptEvalCachedCount == nil || *final.PromptEvalCachedCount != 1 || final.PromptEvalDuration != promptEval {
+ t.Errorf("final prompt metrics = (%d, %v, %s), want (2, 1, %s)", final.PromptEvalCount, final.PromptEvalCachedCount, final.PromptEvalDuration, promptEval)
+ }
+ if final.EvalCount != 2 || final.EvalDuration <= 0 {
+ t.Errorf("final eval metrics = (%d, %s), want count 2 and positive duration", final.EvalCount, final.EvalDuration)
+ }
+}
+
func TestRunMTPDecodeSampled(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
// The same chain at temperature 1: because oneHotLogits uses a large gap,
diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.go
index ec2844aff..bd260cb44 100644
--- a/x/mlxrunner/pipeline.go
+++ b/x/mlxrunner/pipeline.go
@@ -269,7 +269,13 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
wantTopLogprobs: request.SamplerOpts.TopLogprobs,
}
- final := CompletionResponse{Done: true, PromptEvalCount: len(request.Tokens), DoneReason: 1}
+ cachedPromptCount := len(session.inputs) - len(session.remaining)
+ final := CompletionResponse{
+ Done: true,
+ DoneReason: 1,
+ PromptEvalCount: len(request.Tokens),
+ PromptEvalCachedCount: &cachedPromptCount,
+ }
final.PromptEvalDuration = promptEval
now := time.Now()
@@ -317,6 +323,14 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
if !ok {
continue
}
+ // Two-pass structured output cancels the first pass before its final response.
+ if request.IncludeIntermediateMetrics {
+ resp.PromptEvalCount = len(request.Tokens)
+ resp.PromptEvalCachedCount = final.PromptEvalCachedCount
+ resp.PromptEvalDuration = promptEval
+ resp.EvalCount = generated
+ resp.EvalDuration = time.Since(now)
+ }
select {
case <-ctx.Done():
return ctx.Err()