Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Report cached prompt tokens (#17943)
Daniel Hiltgen855f4bf989da
diff --git a/anthropic/anthropic.go b/anthropic/anthropic.goindex 60cd78dd6..7ed5cb767 100755--- a/anthropic/anthropic.go+++ b/anthropic/anthropic.go@@ -217,8 +217,31 @@ type MessagesResponse struct {// Usage contains token usage informationtype Usage struct {- InputTokens int `json:"input_tokens"`- OutputTokens int `json:"output_tokens"`+ InputTokens int `json:"input_tokens"`+ CacheReadInputTokens *int `json:"cache_read_input_tokens,omitempty"`+ OutputTokens int `json:"output_tokens"`+}++// UsageFromMetrics separates total prompt tokens into uncached and cache-read counts.+func UsageFromMetrics(metrics api.Metrics) Usage {+ total := max(0, metrics.PromptEvalCount)+ var cached *int+ if metrics.PromptEvalCachedCount != nil {+ count := min(max(0, *metrics.PromptEvalCachedCount), total)+ cached = &count+ }+ return Usage{+ InputTokens: total - intValue(cached),+ CacheReadInputTokens: cached,+ OutputTokens: metrics.EvalCount,+ }+}++func intValue(v *int) int {+ if v == nil {+ return 0+ }+ return *v}// Streaming event types@@ -273,8 +296,9 @@ type MessageDelta struct {// DeltaUsage contains cumulative token usagetype DeltaUsage struct {- InputTokens int `json:"input_tokens"`- OutputTokens int `json:"output_tokens"`+ InputTokens int `json:"input_tokens"`+ CacheReadInputTokens *int `json:"cache_read_input_tokens,omitempty"`+ OutputTokens int `json:"output_tokens"`}// MessageStopEvent signals the end of the message@@ -688,10 +712,7 @@ func ToMessagesResponse(id string, r api.ChatResponse) MessagesResponse {Model: r.Model,Content: content,StopReason: stopReason,- Usage: Usage{- InputTokens: r.Metrics.PromptEvalCount,- OutputTokens: r.Metrics.EvalCount,- },+ Usage: UsageFromMetrics(r.Metrics),}}@@ -721,6 +742,7 @@ type StreamConverter struct {firstWrite boolcontentIndex intinputTokens int+ cacheReadTokens *intoutputTokens intestimatedInputTokens int // Estimated tokens from request (used when actual metrics are 0)thinkingStarted bool@@ -752,8 +774,10 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {if c.firstWrite {c.firstWrite = false// Use actual metrics if available, otherwise use estimate- c.inputTokens = r.Metrics.PromptEvalCount- if c.inputTokens == 0 && c.estimatedInputTokens > 0 {+ usage := UsageFromMetrics(r.Metrics)+ c.inputTokens = usage.InputTokens+ c.cacheReadTokens = usage.CacheReadInputTokens+ if c.inputTokens == 0 && intValue(c.cacheReadTokens) == 0 && c.estimatedInputTokens > 0 {c.inputTokens = c.estimatedInputTokens}@@ -768,8 +792,9 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {Model: c.Model,Content: []ContentBlock{},Usage: Usage{- InputTokens: c.inputTokens,- OutputTokens: 0,+ InputTokens: c.inputTokens,+ CacheReadInputTokens: c.cacheReadTokens,+ OutputTokens: 0,},},},@@ -950,8 +975,10 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {})}- c.inputTokens = r.Metrics.PromptEvalCount- c.outputTokens = r.Metrics.EvalCount+ usage := UsageFromMetrics(r.Metrics)+ c.inputTokens = usage.InputTokens+ c.cacheReadTokens = usage.CacheReadInputTokens+ c.outputTokens = usage.OutputTokensstopReason := mapStopReason(r.DoneReason, len(c.toolCallsSent) > 0)events = append(events, StreamEvent{@@ -962,8 +989,9 @@ func (c *StreamConverter) Process(r api.ChatResponse) []StreamEvent {StopReason: stopReason,},Usage: DeltaUsage{- InputTokens: c.inputTokens,- OutputTokens: c.outputTokens,+ InputTokens: c.inputTokens,+ CacheReadInputTokens: c.cacheReadTokens,+ OutputTokens: c.outputTokens,},},})diff --git a/anthropic/anthropic_test.go b/anthropic/anthropic_test.goindex d29127319..6ceb7171c 100755--- a/anthropic/anthropic_test.go+++ b/anthropic/anthropic_test.go@@ -16,6 +16,10 @@ const (testImage = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=`)+func testIntPtr(v int) *int {+ return &v+}+// textContent is a convenience for constructing []ContentBlock with a single text block in tests.func textContent(s string) []ContentBlock {return []ContentBlock{{Type: "text", Text: &s}}@@ -30,6 +34,61 @@ func makeArgs(kvs ...any) api.ToolCallFunctionArguments {return args}+func TestUsageFromMetricsBoundsCacheReads(t *testing.T) {+ tests := []struct {+ name string+ metrics api.Metrics+ want Usage+ }{+ {+ name: "negative counts",+ metrics: api.Metrics{PromptEvalCount: -1, PromptEvalCachedCount: testIntPtr(-2), EvalCount: 3},+ want: Usage{CacheReadInputTokens: testIntPtr(0), OutputTokens: 3},+ },+ {+ name: "cache reads exceed prompt",+ metrics: api.Metrics{PromptEvalCount: 3, PromptEvalCachedCount: testIntPtr(5), EvalCount: 2},+ want: Usage{CacheReadInputTokens: testIntPtr(3), OutputTokens: 2},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if diff := cmp.Diff(tt.want, UsageFromMetrics(tt.metrics)); diff != "" {+ t.Errorf("usage mismatch (-want +got):\n%s", diff)+ }+ })+ }+}++func TestUsageCacheReadJSON(t *testing.T) {+ tests := []struct {+ name string+ count *int+ want string+ }{+ {name: "unreported", want: `{"input_tokens":10,"output_tokens":2}`},+ {name: "zero", count: testIntPtr(0), want: `{"input_tokens":10,"cache_read_input_tokens":0,"output_tokens":2}`},+ {name: "positive", count: testIntPtr(4), want: `{"input_tokens":6,"cache_read_input_tokens":4,"output_tokens":2}`},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ data, err := json.Marshal(UsageFromMetrics(api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: tt.count,+ EvalCount: 2,+ }))+ if err != nil {+ t.Fatal(err)+ }+ if got := string(data); got != tt.want {+ t.Errorf("json = %s, want %s", got, tt.want)+ }+ })+ }+}+func TestFromMessagesRequest_Basic(t *testing.T) {req := MessagesRequest{Model: "test-model",@@ -861,8 +920,9 @@ func TestToMessagesResponse_Basic(t *testing.T) {Done: true,DoneReason: "stop",Metrics: api.Metrics{- PromptEvalCount: 10,- EvalCount: 5,+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ EvalCount: 5,},}@@ -886,9 +946,17 @@ func TestToMessagesResponse_Basic(t *testing.T) {if result.StopReason != "end_turn" {t.Errorf("expected stop_reason 'end_turn', got %q", result.StopReason)}- if result.Usage.InputTokens != 10 || result.Usage.OutputTokens != 5 {+ if result.Usage.InputTokens != 6 || intValue(result.Usage.CacheReadInputTokens) != 4 || result.Usage.OutputTokens != 5 {t.Errorf("unexpected usage: %+v", result.Usage)}++ data, err := json.Marshal(result.Usage)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), `"cache_read_input_tokens":4`) {+ t.Errorf("unexpected usage json: %s", data)+ }}func TestToMessagesResponse_PreservesClaudeAutoClassifierOutput(t *testing.T) {@@ -1072,7 +1140,7 @@ func TestStreamConverter_Basic(t *testing.T) {Role: "assistant",Content: "Hello",},- Metrics: api.Metrics{PromptEvalCount: 10},+ Metrics: api.Metrics{PromptEvalCount: 10, PromptEvalCachedCount: testIntPtr(4)},}events1 := conv.Process(resp1)@@ -1100,7 +1168,7 @@ func TestStreamConverter_Basic(t *testing.T) {},Done: true,DoneReason: "stop",- Metrics: api.Metrics{PromptEvalCount: 10, EvalCount: 5},+ Metrics: api.Metrics{PromptEvalCount: 10, PromptEvalCachedCount: testIntPtr(4), EvalCount: 5},}events2 := conv.Process(resp2)@@ -1118,7 +1186,7 @@ func TestStreamConverter_Basic(t *testing.T) {t.Errorf("unexpected stop reason: %+v", data.Delta.StopReason)}- if data.Usage.InputTokens != 10 || data.Usage.OutputTokens != 5 {+ if data.Usage.InputTokens != 6 || intValue(data.Usage.CacheReadInputTokens) != 4 || data.Usage.OutputTokens != 5 {t.Errorf("unexpected usage: %+v", data.Usage)}} else {diff --git a/api/metrics_test.go b/api/metrics_test.gonew file mode 100644index 000000000..b0c5fa5c9--- /dev/null+++ b/api/metrics_test.go@@ -0,0 +1,74 @@+package api++import (+ "encoding/json"+ "io"+ "os"+ "strings"+ "testing"+ "time"+)++func TestMetricsCachedPromptJSON(t *testing.T) {+ tests := []struct {+ name string+ count *int+ want string+ }{+ {name: "unreported", want: `{}`},+ {name: "zero", count: testIntPtr(0), want: `{"prompt_eval_cached_count":0}`},+ {name: "positive", count: testIntPtr(4), want: `{"prompt_eval_cached_count":4}`},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ data, err := json.Marshal(Metrics{PromptEvalCachedCount: tt.count})+ if err != nil {+ t.Fatal(err)+ }+ if got := string(data); got != tt.want {+ t.Errorf("json = %s, want %s", got, tt.want)+ }++ var metrics Metrics+ if err := json.Unmarshal(data, &metrics); err != nil {+ t.Fatal(err)+ }+ if tt.count == nil {+ if metrics.PromptEvalCachedCount != nil {+ t.Errorf("cached count = %v, want nil", metrics.PromptEvalCachedCount)+ }+ } else if metrics.PromptEvalCachedCount == nil || *metrics.PromptEvalCachedCount != *tt.count {+ t.Errorf("cached count = %v, want %d", metrics.PromptEvalCachedCount, *tt.count)+ }+ })+ }+}++func TestMetricsSummaryCachedPromptTokens(t *testing.T) {+ read, write, err := os.Pipe()+ if err != nil {+ t.Fatal(err)+ }+ original := os.Stderr+ os.Stderr = write+ t.Cleanup(func() { os.Stderr = original })++ (&Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ PromptEvalDuration: time.Second,+ }).Summary()+ write.Close()+ os.Stderr = original++ output, err := io.ReadAll(read)+ if err != nil {+ t.Fatal(err)+ }+ for _, want := range []string{"prompt eval count: 10 token(s)", "prompt eval cached: 4 token(s)", "prompt eval rate: 6.00 tokens/s"} {+ if !strings.Contains(string(output), want) {+ t.Errorf("summary missing %q:\n%s", want, output)+ }+ }+}diff --git a/api/types.go b/api/types.goindex 65f5123b4..82dd9a449 100644--- a/api/types.go+++ b/api/types.go@@ -555,12 +555,13 @@ type DebugInfo struct {}type Metrics struct {- TotalDuration time.Duration `json:"total_duration,omitempty"`- LoadDuration time.Duration `json:"load_duration,omitempty"`- PromptEvalCount int `json:"prompt_eval_count,omitempty"`- PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`- EvalCount int `json:"eval_count,omitempty"`- EvalDuration time.Duration `json:"eval_duration,omitempty"`+ TotalDuration time.Duration `json:"total_duration,omitempty"`+ LoadDuration time.Duration `json:"load_duration,omitempty"`+ PromptEvalCount int `json:"prompt_eval_count,omitempty"`+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`+ PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`+ EvalCount int `json:"eval_count,omitempty"`+ EvalDuration time.Duration `json:"eval_duration,omitempty"`}// Options specified in [GenerateRequest]. If you add a new option here, also@@ -983,9 +984,18 @@ func (m *Metrics) Summary() {fmt.Fprintf(os.Stderr, "prompt eval count: %d token(s)\n", m.PromptEvalCount)}+ cached := 0+ if m.PromptEvalCachedCount != nil {+ cached = *m.PromptEvalCachedCount+ }+ if cached > 0 {+ fmt.Fprintf(os.Stderr, "prompt eval cached: %d token(s)\n", cached)+ }+if m.PromptEvalDuration > 0 {fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", m.PromptEvalDuration)- fmt.Fprintf(os.Stderr, "prompt eval rate: %.2f tokens/s\n", float64(m.PromptEvalCount)/m.PromptEvalDuration.Seconds())+ uncached := max(0, m.PromptEvalCount-cached)+ fmt.Fprintf(os.Stderr, "prompt eval rate: %.2f tokens/s\n", float64(uncached)/m.PromptEvalDuration.Seconds())}if m.EvalCount > 0 {diff --git a/cmd/bench/bench.go b/cmd/bench/bench.goindex 1ecd42cfc..b0433f10c 100644--- a/cmd/bench/bench.go+++ b/cmd/bench/bench.go@@ -36,10 +36,11 @@ type flagOptions struct {}type Metrics struct {- Model string- Step string- Count int- Duration time.Duration+ Model string+ Step string+ Count int+ CachedPromptCount *int+ Duration time.Duration}type ModelInfo struct {@@ -194,7 +195,7 @@ func outputFormatHeader(w io.Writer, format string, verbose bool) {fmt.Fprintf(w, "goarch: %s\n", runtime.GOARCH)}case "csv":- headings := []string{"NAME", "STEP", "COUNT", "NS_PER_COUNT", "TOKEN_PER_SEC"}+ headings := []string{"NAME", "STEP", "COUNT", "NS_PER_COUNT", "TOKEN_PER_SEC", "CACHED_PROMPT_COUNT"}fmt.Fprintln(w, strings.Join(headings, ","))}}@@ -221,14 +222,21 @@ func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool)case "benchstat":for _, m := range metrics {if m.Step == "generate" || m.Step == "prefill" {+ var promptCounts string+ if m.Step == "prefill" {+ promptCounts = fmt.Sprintf(" %d processed-prompt-token", m.Count)+ if m.CachedPromptCount != nil {+ promptCounts += fmt.Sprintf(" %d cached-prompt-token", *m.CachedPromptCount)+ }+ }if m.Count > 0 {nsPerToken := float64(m.Duration.Nanoseconds()) / float64(m.Count)tokensPerSec := float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9- fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 %.2f ns/token %.2f token/sec\n",- m.Model, m.Step, nsPerToken, tokensPerSec)+ fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 %.2f ns/token %.2f token/sec%s\n",+ m.Model, m.Step, nsPerToken, tokensPerSec, promptCounts)} else {- fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 0 ns/token 0 token/sec\n",- m.Model, m.Step)+ fmt.Fprintf(w, "BenchmarkModel/name=%s/step=%s 1 0 ns/token 0 token/sec%s\n",+ m.Model, m.Step, promptCounts)}} else if m.Step == "ttft" {fmt.Fprintf(w, "BenchmarkModel/name=%s/step=ttft 1 %d ns/op\n",@@ -240,6 +248,10 @@ func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool)}case "csv":for _, m := range metrics {+ cachedPromptCount := ""+ if m.CachedPromptCount != nil {+ cachedPromptCount = fmt.Sprint(*m.CachedPromptCount)+ }if m.Step == "generate" || m.Step == "prefill" {var nsPerToken float64var tokensPerSec float64@@ -247,9 +259,9 @@ func OutputMetrics(w io.Writer, format string, metrics []Metrics, verbose bool)nsPerToken = float64(m.Duration.Nanoseconds()) / float64(m.Count)tokensPerSec = float64(m.Count) / (float64(m.Duration.Nanoseconds()) + 1e-12) * 1e9}- fmt.Fprintf(w, "%s,%s,%d,%.2f,%.2f\n", m.Model, m.Step, m.Count, nsPerToken, tokensPerSec)+ fmt.Fprintf(w, "%s,%s,%d,%.2f,%.2f,%s\n", m.Model, m.Step, m.Count, nsPerToken, tokensPerSec, cachedPromptCount)} else {- fmt.Fprintf(w, "%s,%s,1,%d,0\n", m.Model, m.Step, m.Duration.Nanoseconds())+ fmt.Fprintf(w, "%s,%s,1,%d,0,%s\n", m.Model, m.Step, m.Duration.Nanoseconds(), cachedPromptCount)}}default:@@ -428,12 +440,17 @@ func BenchmarkModel(fOpt flagOptions) error {}}+ cachedPromptCount := 0+ if responseMetrics.PromptEvalCachedCount != nil {+ cachedPromptCount = *responseMetrics.PromptEvalCachedCount+ }metrics := []Metrics{{- Model: model,- Step: "prefill",- Count: responseMetrics.PromptEvalCount,- Duration: responseMetrics.PromptEvalDuration,+ Model: model,+ Step: "prefill",+ Count: max(0, responseMetrics.PromptEvalCount-cachedPromptCount),+ CachedPromptCount: responseMetrics.PromptEvalCachedCount,+ Duration: responseMetrics.PromptEvalDuration,},{Model: model,@@ -464,8 +481,13 @@ func BenchmarkModel(fOpt flagOptions) error {OutputMetrics(out, *fOpt.format, metrics, *fOpt.verbose)if *fOpt.debug && *fOpt.promptTokens > 0 {- fmt.Fprintf(os.Stderr, "Generated prompt targeting ~%d tokens (actual: %d)\n",- *fOpt.promptTokens, responseMetrics.PromptEvalCount)+ if responseMetrics.PromptEvalCachedCount == nil {+ fmt.Fprintf(os.Stderr, "Generated prompt targeting ~%d tokens (actual: %d, cached: unavailable)\n",+ *fOpt.promptTokens, responseMetrics.PromptEvalCount)+ } else {+ fmt.Fprintf(os.Stderr, "Generated prompt targeting ~%d tokens (actual: %d, cached: %d)\n",+ *fOpt.promptTokens, responseMetrics.PromptEvalCount, cachedPromptCount)+ }}if *fOpt.keepAlive > 0 {diff --git a/cmd/bench/bench_test.go b/cmd/bench/bench_test.goindex 3c4c67f13..ca5769fc1 100644--- a/cmd/bench/bench_test.go+++ b/cmd/bench/bench_test.go@@ -4,6 +4,7 @@ import ("bytes""crypto/rand""encoding/json"+ "fmt""io""net/http""net/http/httptest"@@ -15,6 +16,10 @@ import ("github.com/ollama/ollama/api")+func testIntPtr(v int) *int {+ return &v+}+func createTestFlagOptions() flagOptions {models := "test-model"format := "benchstat"@@ -1083,7 +1088,7 @@ func TestBenchmarkModel_CSVFormat(t *testing.T) {}})- if !strings.Contains(output, "NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC") {+ if !strings.Contains(output, "NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC,CACHED_PROMPT_COUNT") {t.Errorf("Expected CSV header, got: %s", output)}if !strings.Contains(output, "test-model,prefill,") {@@ -1094,6 +1099,35 @@ func TestBenchmarkModel_CSVFormat(t *testing.T) {}}+func TestBenchmarkModel_PrefillExcludesCachedTokens(t *testing.T) {+ fOpt := createTestFlagOptions()+ format := "csv"+ fOpt.format = &format+ responses := defaultGenerateResponses()+ responses[len(responses)-1].PromptEvalCachedCount = testIntPtr(4)++ server := createMockOllamaServer(t, mockServerOptions{generateResponses: responses})+ defer server.Close()+ t.Setenv("OLLAMA_HOST", server.URL)++ output := captureOutput(func() {+ if err := BenchmarkModel(fOpt); err != nil {+ t.Errorf("BenchmarkModel: %v", err)+ }+ })+ wantPrefix := fmt.Sprintf("test-model,prefill,%d,", responses[len(responses)-1].PromptEvalCount-*responses[len(responses)-1].PromptEvalCachedCount)+ var prefillRow string+ for row := range strings.SplitSeq(output, "\n") {+ if strings.HasPrefix(row, wantPrefix) {+ prefillRow = row+ break+ }+ }+ if prefillRow == "" || !strings.HasSuffix(prefillRow, ",4") {+ t.Errorf("prefill row did not exclude cached prompt tokens:\n%s", output)+ }+}+// --- Unit tests for helper functions ---func TestGeneratePromptForTokenCount(t *testing.T) {@@ -1188,7 +1222,7 @@ func TestBuildGenerateRequest_VariesByEpoch(t *testing.T) {func TestOutputMetrics_Benchstat(t *testing.T) {var buf bytes.Buffermetrics := []Metrics{- {Model: "m1", Step: "prefill", Count: 10, Duration: 100 * time.Millisecond},+ {Model: "m1", Step: "prefill", Count: 10, CachedPromptCount: testIntPtr(4), Duration: 100 * time.Millisecond},{Model: "m1", Step: "generate", Count: 50, Duration: 500 * time.Millisecond},{Model: "m1", Step: "ttft", Count: 1, Duration: 50 * time.Millisecond},{Model: "m1", Step: "load", Count: 1, Duration: 50 * time.Millisecond},@@ -1214,6 +1248,9 @@ func TestOutputMetrics_Benchstat(t *testing.T) {if !strings.Contains(output, "token/sec") {t.Errorf("Expected token/sec metric for throughput lines, got: %s", output)}+ if !strings.Contains(output, "10 processed-prompt-token 4 cached-prompt-token") {+ t.Errorf("Expected prompt cache counts on prefill line, got: %s", output)+ }for _, line := range strings.Split(strings.TrimSpace(output), "\n") {if !strings.HasPrefix(line, "Benchmark") {continue@@ -1242,6 +1279,37 @@ func TestOutputMetrics_BenchstatFormat(t *testing.T) {if !strings.Contains(output, "ns/token") {t.Errorf("Expected ns/token unit for prefill, got: %s", output)}+ if strings.Contains(output, "cached-prompt-token") {+ t.Errorf("unexpected cached prompt count when unavailable: %s", output)+ }+}++func TestOutputMetrics_CachedPromptAvailability(t *testing.T) {+ tests := []struct {+ name string+ count *int+ want string+ }{+ {name: "unavailable", want: "m1,prefill,10,10000000.00,100.00,"},+ {name: "zero", count: testIntPtr(0), want: "m1,prefill,10,10000000.00,100.00,0"},+ {name: "positive", count: testIntPtr(4), want: "m1,prefill,10,10000000.00,100.00,4"},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ var buf bytes.Buffer+ OutputMetrics(&buf, "csv", []Metrics{{+ Model: "m1",+ Step: "prefill",+ Count: 10,+ CachedPromptCount: tt.count,+ Duration: 100 * time.Millisecond,+ }}, false)+ if got := strings.TrimSpace(buf.String()); got != tt.want {+ t.Errorf("output = %q, want %q", got, tt.want)+ }+ })+ }}func TestOutputModelInfo(t *testing.T) {@@ -1403,7 +1471,7 @@ func TestOutputFormatHeader(t *testing.T) {var buf bytes.BufferoutputFormatHeader(&buf, "csv", false)output := buf.String()- if !strings.Contains(output, "NAME,STEP,COUNT") {+ if output != "NAME,STEP,COUNT,NS_PER_COUNT,TOKEN_PER_SEC,CACHED_PROMPT_COUNT\n" {t.Errorf("Expected CSV header, got: %s", output)}})diff --git a/cmd/tui/chat/render.go b/cmd/tui/chat/render.goindex 4ea56ecfd..fb85688a6 100644--- a/cmd/tui/chat/render.go+++ b/cmd/tui/chat/render.go@@ -582,9 +582,17 @@ func metricsSummaryLines(metrics *api.Metrics) []string {if metrics.PromptEvalCount > 0 {lines = append(lines, fmt.Sprintf("prompt eval count: %d token(s)", metrics.PromptEvalCount))}+ cached := 0+ if metrics.PromptEvalCachedCount != nil {+ cached = *metrics.PromptEvalCachedCount+ }+ if cached > 0 {+ lines = append(lines, fmt.Sprintf("prompt eval cached: %d token(s)", cached))+ }if metrics.PromptEvalDuration > 0 {lines = append(lines, fmt.Sprintf("prompt eval duration: %s", metrics.PromptEvalDuration))- lines = append(lines, fmt.Sprintf("prompt eval rate: %.2f tokens/s", float64(metrics.PromptEvalCount)/metrics.PromptEvalDuration.Seconds()))+ uncached := max(0, metrics.PromptEvalCount-cached)+ lines = append(lines, fmt.Sprintf("prompt eval rate: %.2f tokens/s", float64(uncached)/metrics.PromptEvalDuration.Seconds()))}if metrics.EvalCount > 0 {lines = append(lines, fmt.Sprintf("eval count: %d token(s)", metrics.EvalCount))@@ -597,9 +605,14 @@ func metricsSummaryLines(metrics *api.Metrics) []string {}func metricsEmpty(metrics api.Metrics) bool {+ cached := 0+ if metrics.PromptEvalCachedCount != nil {+ cached = *metrics.PromptEvalCachedCount+ }return metrics.TotalDuration <= 0 &&metrics.LoadDuration <= 0 &&metrics.PromptEvalCount <= 0 &&+ cached <= 0 &&metrics.PromptEvalDuration <= 0 &&metrics.EvalCount <= 0 &&metrics.EvalDuration <= 0diff --git a/cmd/tui/chat/render_test.go b/cmd/tui/chat/render_test.goindex c2d911412..0a31c7bbb 100644--- a/cmd/tui/chat/render_test.go+++ b/cmd/tui/chat/render_test.go@@ -17,6 +17,24 @@ import ("github.com/ollama/ollama/api")+func testIntPtr(v int) *int {+ return &v+}++func TestMetricsSummaryLinesCachedPromptTokens(t *testing.T) {+ lines := metricsSummaryLines(&api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ PromptEvalDuration: time.Second,+ })+ got := strings.Join(lines, "\n")+ for _, want := range []string{"prompt eval count: 10 token(s)", "prompt eval cached: 4 token(s)", "prompt eval rate: 6.00 tokens/s"} {+ if !strings.Contains(got, want) {+ t.Errorf("summary missing %q:\n%s", want, got)+ }+ }+}+func TestChatAssistantEntryHasNoLabel(t *testing.T) {m := chatModel{}diff --git a/docs/api.md b/docs/api.mdindex 95cac42ca..87ac17d0a 100644--- a/docs/api.md+++ b/docs/api.md@@ -100,7 +100,8 @@ The final response in the stream also includes additional data about the generat- `total_duration`: time spent generating the response- `load_duration`: time spent in nanoseconds loading the model- `prompt_eval_count`: number of tokens in the prompt-- `prompt_eval_duration`: time spent in nanoseconds evaluating the prompt+- `prompt_eval_cached_count`: number of prompt tokens read from the cache+- `prompt_eval_duration`: time spent in nanoseconds evaluating uncached prompt tokens- `eval_count`: number of tokens in the response- `eval_duration`: time in nanoseconds spent generating the response- `context`: an encoding of the conversation used in this response, this can be sent in the next request to keep a conversational memorydiff --git a/docs/api/usage.mdx b/docs/api/usage.mdxindex 107bb0c4c..e9606a425 100644--- a/docs/api/usage.mdx+++ b/docs/api/usage.mdx@@ -6,8 +6,9 @@ Ollama's API responses include metrics that can be used for measuring performanc* `total_duration`: How long the response took to generate* `load_duration`: How long the model took to load-* `prompt_eval_count`: How many input tokens were processed-* `prompt_eval_duration`: How long it took to evaluate the prompt+* `prompt_eval_count`: How many input tokens were in the prompt+* `prompt_eval_cached_count`: How many prompt tokens were read from the cache+* `prompt_eval_duration`: How long it took to evaluate the uncached prompt tokens* `eval_count`: How many output tokens were processes* `eval_duration`: How long it took to generate the output tokens@@ -27,6 +28,7 @@ For endpoints that return usage metrics, the response body will include the usag"total_duration": 174560334,"load_duration": 101397084,"prompt_eval_count": 11,+ "prompt_eval_cached_count": 8,"prompt_eval_duration": 13074791,"eval_count": 18,"eval_duration": 52479709diff --git a/docs/openapi.yaml b/docs/openapi.yamlindex 34aa9f40e..162a906c8 100644--- a/docs/openapi.yaml+++ b/docs/openapi.yaml@@ -147,9 +147,12 @@ components:prompt_eval_count:type: integerdescription: Number of input tokens in the prompt+ prompt_eval_cached_count:+ type: integer+ description: Number of prompt tokens read from the cacheprompt_eval_duration:type: integer- description: Time spent evaluating the prompt in nanoseconds+ description: Time spent evaluating uncached prompt tokens in nanosecondseval_count:type: integerdescription: Number of output tokens generated in the response@@ -191,9 +194,12 @@ components:prompt_eval_count:type: integerdescription: Number of input tokens in the prompt+ prompt_eval_cached_count:+ type: integer+ description: Number of prompt tokens read from the cacheprompt_eval_duration:type: integer- description: Time spent evaluating the prompt in nanoseconds+ description: Time spent evaluating uncached prompt tokens in nanosecondseval_count:type: integerdescription: Number of output tokens generated in the response@@ -352,9 +358,12 @@ components:prompt_eval_count:type: integerdescription: Number of tokens in the prompt+ prompt_eval_cached_count:+ type: integer+ description: Number of prompt tokens read from the cacheprompt_eval_duration:type: integer- description: Time spent evaluating the prompt in nanoseconds+ description: Time spent evaluating uncached prompt tokens in nanosecondseval_count:type: integerdescription: Number of tokens generated in the responsediff --git a/llm/llama_server.go b/llm/llama_server.goindex 6f27b7547..fc3edfbbe 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -1412,6 +1412,7 @@ type llamaServerCompletionRequest struct {JsonSchema json.RawMessage `json:"json_schema,omitempty"`NProbs int `json:"n_probs,omitempty"`PreservedTokens []string `json:"preserved_tokens,omitempty"`+ TimingsPerToken bool `json:"timings_per_token,omitempty"`}func llamaServerPreservedTokens(parserTokens []string, toolCallTag string) []string {@@ -1501,7 +1502,7 @@ type llamaServerChatResponse struct {}type llamaServerTimings struct {- CacheN int `json:"cache_n"`+ CacheN *int `json:"cache_n"`PromptN int `json:"prompt_n"`PromptMS float64 `json:"prompt_ms"`PredictN int `json:"predicted_n"`@@ -1509,7 +1510,10 @@ type llamaServerTimings struct {}func (t llamaServerTimings) promptEvalCount() int {- return t.CacheN + t.PromptN+ if t.CacheN == nil {+ return t.PromptN+ }+ return *t.CacheN + t.PromptN}type llamaServerApplyTemplateResponse struct {@@ -1574,6 +1578,7 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionRequesTypicalP: req.Options.TypicalP,Seed: req.Options.Seed,PreservedTokens: llamaServerPreservedTokens(req.PreservedTokens, req.ToolCallTag),+ TimingsPerToken: req.IncludeIntermediateMetrics,}if req.Logprobs {@@ -1701,8 +1706,13 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques}if lsResp.Content != "" && !lsResp.Stop {- resp := CompletionResponse{- Content: lsResp.Content,+ resp := CompletionResponse{Content: lsResp.Content}+ if req.IncludeIntermediateMetrics {+ resp.PromptEvalCount = lsResp.Timings.promptEvalCount()+ resp.PromptEvalCachedCount = lsResp.Timings.CacheN+ resp.PromptEvalDuration = time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond))+ resp.EvalCount = lsResp.Timings.PredictN+ resp.EvalDuration = time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond))}resp.Logprobs = convertLogprobs(lsResp.CompletionProbabilities, req.TopLogprobs > 0)fn(resp)@@ -1715,13 +1725,14 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques}finalResp = CompletionResponse{- Content: lsResp.Content,- Done: true,- DoneReason: doneReason,- PromptEvalCount: lsResp.Timings.promptEvalCount(),- PromptEvalDuration: time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond)),- EvalCount: lsResp.Timings.PredictN,- EvalDuration: time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond)),+ Content: lsResp.Content,+ Done: true,+ DoneReason: doneReason,+ PromptEvalCount: lsResp.Timings.promptEvalCount(),+ PromptEvalCachedCount: lsResp.Timings.CacheN,+ PromptEvalDuration: time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond)),+ EvalCount: lsResp.Timings.PredictN,+ EvalDuration: time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond)),}hasFinalResp = true}@@ -2018,6 +2029,7 @@ func (s *llamaServerRunner) Chat(ctx context.Context, req ChatRequest, fn func(Cresp.Done = trueresp.DoneReason = doneReasonresp.PromptEvalCount = lsResp.Timings.promptEvalCount()+ resp.PromptEvalCachedCount = lsResp.Timings.CacheNresp.PromptEvalDuration = time.Duration(lsResp.Timings.PromptMS * float64(time.Millisecond))resp.EvalCount = lsResp.Timings.PredictNresp.EvalDuration = time.Duration(lsResp.Timings.PredictMS * float64(time.Millisecond))diff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex a9c9bd826..4d4300b5e 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -154,13 +154,13 @@ func TestContextShiftPromptLimit(t *testing.T) {func TestLlamaServerCompletionSSEParsing(t *testing.T) {// Simulate llama-server SSE streaming responsesseLines := []string{- `data: {"content":"Hello","stop":false}`,+ `data: {"content":"Hello","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10.5,"predicted_n":1,"predicted_ms":9.1}}`,``,`:`,- `data: {"content":" world","stop":false}`,+ `data: {"content":" world","stop":false,"timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10.5,"predicted_n":2,"predicted_ms":20.3}}`,``,`:`,- `data: {"content":"","stop":true,"stop_type":"eos","timings":{"prompt_n":5,"prompt_ms":10.5,"predicted_n":2,"predicted_ms":20.3}}`,+ `data: {"content":"","stop":true,"stop_type":"eos","timings":{"cache_n":2,"prompt_n":3,"prompt_ms":10.5,"predicted_n":2,"predicted_ms":20.3}}`,``,}@@ -186,6 +186,9 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {if !reqBody.Stream {t.Error("stream should be true")}+ if !reqBody.TimingsPerToken {+ t.Error("timings_per_token should be true")+ }w.Header().Set("Content-Type", "text/event-stream")for _, line := range sseLines {@@ -208,8 +211,9 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {var responses []CompletionResponseopts := api.DefaultOptions()err := runner.Completion(t.Context(), CompletionRequest{- Prompt: "test prompt",- Options: &opts,+ Prompt: "test prompt",+ Options: &opts,+ IncludeIntermediateMetrics: true,}, func(cr CompletionResponse) {responses = append(responses, cr)})@@ -228,11 +232,29 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {if responses[0].Done {t.Error("response[0] should not be done")}+ if responses[0].PromptEvalCount != 5 || responses[0].EvalCount != 1 {+ t.Errorf("response[0] counts = (%d, %d), want (5, 1)", responses[0].PromptEvalCount, responses[0].EvalCount)+ }+ if got := responses[0].PromptEvalCachedCount; got == nil || *got != 2 {+ t.Errorf("response[0] cached prompt count = %v, want 2", got)+ }+ if responses[0].PromptEvalDuration != 10500*time.Microsecond || responses[0].EvalDuration != 9100*time.Microsecond {+ t.Errorf("response[0] durations = (%s, %s), want (10.5ms, 9.1ms)", responses[0].PromptEvalDuration, responses[0].EvalDuration)+ }// Second tokenif responses[1].Content != " world" {t.Errorf("response[1].Content = %q, want %q", responses[1].Content, " world")}+ if responses[1].PromptEvalCount != 5 || responses[1].EvalCount != 2 {+ t.Errorf("response[1] counts = (%d, %d), want (5, 2)", responses[1].PromptEvalCount, responses[1].EvalCount)+ }+ if got := responses[1].PromptEvalCachedCount; got == nil || *got != 2 {+ t.Errorf("response[1] cached prompt count = %v, want 2", got)+ }+ if responses[1].PromptEvalDuration != 10500*time.Microsecond || responses[1].EvalDuration != 20300*time.Microsecond {+ t.Errorf("response[1] durations = (%s, %s), want (10.5ms, 20.3ms)", responses[1].PromptEvalDuration, responses[1].EvalDuration)+ }// Final responseif !responses[2].Done {@@ -244,6 +266,9 @@ func TestLlamaServerCompletionSSEParsing(t *testing.T) {if responses[2].PromptEvalCount != 5 {t.Errorf("PromptEvalCount = %d, want 5", responses[2].PromptEvalCount)}+ if got := responses[2].PromptEvalCachedCount; got == nil || *got != 2 {+ t.Errorf("PromptEvalCachedCount = %v, want 2", got)+ }if responses[2].EvalCount != 2 {t.Errorf("EvalCount = %d, want 2", responses[2].EvalCount)}@@ -291,6 +316,9 @@ func TestLlamaServerCompletionPromptEvalCountIncludesCache(t *testing.T) {if responses[0].PromptEvalCount != 17 {t.Errorf("PromptEvalCount = %d, want 17", responses[0].PromptEvalCount)}+ if got := responses[0].PromptEvalCachedCount; got == nil || *got != 12 {+ t.Errorf("PromptEvalCachedCount = %v, want 12", got)+ }if responses[0].PromptEvalDuration != 10*time.Millisecond {t.Errorf("PromptEvalDuration = %s, want 10ms", responses[0].PromptEvalDuration)}@@ -341,6 +369,9 @@ func TestLlamaServerChatPromptEvalCountIncludesCache(t *testing.T) {if responses[1].PromptEvalCount != 17 {t.Errorf("PromptEvalCount = %d, want 17", responses[1].PromptEvalCount)}+ if got := responses[1].PromptEvalCachedCount; got == nil || *got != 12 {+ t.Errorf("PromptEvalCachedCount = %v, want 12", got)+ }if responses[1].PromptEvalDuration != 10*time.Millisecond {t.Errorf("PromptEvalDuration = %s, want 10ms", responses[1].PromptEvalDuration)}diff --git a/llm/server.go b/llm/server.goindex c1f494468..a8a55409a 100644--- a/llm/server.go+++ b/llm/server.go@@ -214,6 +214,8 @@ type CompletionRequest struct {PreservedTokens []string // parser tokens to render as text; ignored by non-llama-server runnersToolCallTag string // raw generic tool parser tag, if anyLeadingBOS string // textual BOS emitted by Go rendering, if any+ // IncludeIntermediateMetrics adds cumulative metrics to non-final responses; final responses always include metrics.+ IncludeIntermediateMetrics bool// Logprobs specifies whether to include log probabilities in the responseLogprobs bool@@ -235,14 +237,15 @@ type ChatRequest struct {}type ChatResponse struct {- Message api.Message `json:"message"`- DoneReason DoneReason `json:"done_reason"`- Done bool `json:"done"`- PromptEvalCount int `json:"prompt_eval_count"`- PromptEvalDuration time.Duration `json:"prompt_eval_duration"`- EvalCount int `json:"eval_count"`- EvalDuration time.Duration `json:"eval_duration"`- Logprobs []Logprob `json:"logprobs,omitempty"`+ Message api.Message `json:"message"`+ DoneReason DoneReason `json:"done_reason"`+ Done bool `json:"done"`+ PromptEvalCount int `json:"prompt_eval_count"`+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`+ PromptEvalDuration time.Duration `json:"prompt_eval_duration"`+ EvalCount int `json:"eval_count"`+ EvalDuration time.Duration `json:"eval_duration"`+ Logprobs []Logprob `json:"logprobs,omitempty"`}// DoneReason represents the reason why a completion response is done@@ -278,13 +281,14 @@ type Logprob struct {}type CompletionResponse struct {- Content string `json:"content"`- DoneReason DoneReason `json:"done_reason"`- Done bool `json:"done"`- PromptEvalCount int `json:"prompt_eval_count"`- PromptEvalDuration time.Duration `json:"prompt_eval_duration"`- EvalCount int `json:"eval_count"`- EvalDuration time.Duration `json:"eval_duration"`+ Content string `json:"content"`+ DoneReason DoneReason `json:"done_reason"`+ Done bool `json:"done"`+ PromptEvalCount int `json:"prompt_eval_count"`+ PromptEvalCachedCount *int `json:"prompt_eval_cached_count,omitempty"`+ PromptEvalDuration time.Duration `json:"prompt_eval_duration"`+ EvalCount int `json:"eval_count"`+ EvalDuration time.Duration `json:"eval_duration"`// Logprobs contains log probability information if requestedLogprobs []Logprob `json:"logprobs,omitempty"`diff --git a/middleware/anthropic.go b/middleware/anthropic.goindex 33c2f02f4..762698e8c 100644--- a/middleware/anthropic.go+++ b/middleware/anthropic.go@@ -100,13 +100,15 @@ type WebSearchAnthropicWriter struct {terminalSent bool- observedPromptEvalCount int- observedEvalCount int+ observedPromptEvalCount int+ observedPromptEvalCachedCount *int+ observedEvalCount int- loopInFlight bool- loopBaseInputTok int- loopBaseOutputTok int- loopResultCh chan webSearchLoopResult+ loopInFlight bool+ loopBaseInputTok int+ loopBaseCacheReadTok *int+ loopBaseOutputTok int+ loopResultCh chan webSearchLoopResultstreamMessageStarted boolstreamHasOpenBlock bool@@ -198,10 +200,7 @@ func (w *WebSearchAnthropicWriter) Write(data []byte) (int, error) {loopCtx, cancel := w.startLoopContext()defer cancel()- initialUsage := anthropic.Usage{- InputTokens: max(w.observedPromptEvalCount, chatResponse.Metrics.PromptEvalCount),- OutputTokens: max(w.observedEvalCount, chatResponse.Metrics.EvalCount),- }+ initialUsage := w.usageWithObservedMetrics(chatResponse.Metrics)logutil.Trace("anthropic middleware: starting sync web_search loop","tool_call", anthropic.TraceToolCall(webSearchCall),"resp", anthropic.TraceChatResponse(chatResponse),@@ -316,8 +315,10 @@ func (w *WebSearchAnthropicWriter) runWebSearchLoop(ctx context.Context, initial"resp", anthropic.TraceChatResponse(followUpResponse),)- usage.InputTokens += followUpResponse.Metrics.PromptEvalCount- usage.OutputTokens += followUpResponse.Metrics.EvalCount+ followUpUsage := anthropic.UsageFromMetrics(followUpResponse.Metrics)+ usage.InputTokens += followUpUsage.InputTokens+ usage.CacheReadInputTokens = addOptionalInts(usage.CacheReadInputTokens, followUpUsage.CacheReadInputTokens)+ usage.OutputTokens += followUpUsage.OutputTokensnextToolCall, hasWebSearch, hasOtherTools := findWebSearchToolCall(followUpResponse.Message.ToolCalls)if hasWebSearch && hasOtherTools {@@ -377,11 +378,9 @@ func (w *WebSearchAnthropicWriter) startLoopWorker(initialResponse api.ChatResporeturn}- initialUsage := anthropic.Usage{- InputTokens: max(w.observedPromptEvalCount, initialResponse.Metrics.PromptEvalCount),- OutputTokens: max(w.observedEvalCount, initialResponse.Metrics.EvalCount),- }+ initialUsage := w.usageWithObservedMetrics(initialResponse.Metrics)w.loopBaseInputTok = initialUsage.InputTokens+ w.loopBaseCacheReadTok = initialUsage.CacheReadInputTokensw.loopBaseOutputTok = initialUsage.OutputTokensw.loopResultCh = make(chan webSearchLoopResult, 1)w.loopInFlight = true@@ -435,14 +434,20 @@ func (w *WebSearchAnthropicWriter) recordObservedUsage(metrics api.Metrics) {if metrics.PromptEvalCount > w.observedPromptEvalCount {w.observedPromptEvalCount = metrics.PromptEvalCount}+ w.observedPromptEvalCachedCount = maxOptionalInts(w.observedPromptEvalCachedCount, metrics.PromptEvalCachedCount)if metrics.EvalCount > w.observedEvalCount {w.observedEvalCount = metrics.EvalCount}}func (w *WebSearchAnthropicWriter) applyObservedUsageDeltaToUsage(usage *anthropic.Usage) {- if deltaIn := w.observedPromptEvalCount - w.loopBaseInputTok; deltaIn > 0 {- usage.InputTokens += deltaIn+ observed := w.currentObservedUsage()+ if delta := observed.InputTokens - w.loopBaseInputTok; delta > 0 {+ usage.InputTokens += delta+ }+ if observed.CacheReadInputTokens != nil {+ delta := max(0, optionalIntValue(observed.CacheReadInputTokens)-optionalIntValue(w.loopBaseCacheReadTok))+ usage.CacheReadInputTokens = addOptionalInts(usage.CacheReadInputTokens, &delta)}if deltaOut := w.observedEvalCount - w.loopBaseOutputTok; deltaOut > 0 {usage.OutputTokens += deltaOut@@ -450,10 +455,18 @@ func (w *WebSearchAnthropicWriter) applyObservedUsageDeltaToUsage(usage *anthrop}func (w *WebSearchAnthropicWriter) currentObservedUsage() anthropic.Usage {- return anthropic.Usage{- InputTokens: w.observedPromptEvalCount,- OutputTokens: w.observedEvalCount,- }+ return anthropic.UsageFromMetrics(api.Metrics{+ PromptEvalCount: w.observedPromptEvalCount,+ PromptEvalCachedCount: w.observedPromptEvalCachedCount,+ EvalCount: w.observedEvalCount,+ })+}++func (w *WebSearchAnthropicWriter) usageWithObservedMetrics(metrics api.Metrics) anthropic.Usage {+ metrics.PromptEvalCount = max(metrics.PromptEvalCount, w.observedPromptEvalCount)+ metrics.PromptEvalCachedCount = maxOptionalInts(metrics.PromptEvalCachedCount, w.observedPromptEvalCachedCount)+ metrics.EvalCount = max(metrics.EvalCount, w.observedEvalCount)+ return anthropic.UsageFromMetrics(metrics)}func (w *WebSearchAnthropicWriter) startLoopContext() (context.Context, context.CancelFunc) {@@ -543,7 +556,7 @@ func (w *WebSearchAnthropicWriter) ensureStreamMessageStart(usage anthropic.Usag}inputTokens := usage.InputTokens- if inputTokens == 0 {+ if inputTokens == 0 && optionalIntValue(usage.CacheReadInputTokens) == 0 {inputTokens = w.estimatedInputTokens}@@ -556,7 +569,8 @@ func (w *WebSearchAnthropicWriter) ensureStreamMessageStart(usage anthropic.UsagModel: w.req.Model,Content: []anthropic.ContentBlock{},Usage: anthropic.Usage{- InputTokens: inputTokens,+ InputTokens: inputTokens,+ CacheReadInputTokens: usage.CacheReadInputTokens,},},}); err != nil {@@ -669,8 +683,9 @@ func (w *WebSearchAnthropicWriter) writeTerminalResponse(response anthropic.MessStopReason: response.StopReason,},Usage: anthropic.DeltaUsage{- InputTokens: response.Usage.InputTokens,- OutputTokens: response.Usage.OutputTokens,+ InputTokens: response.Usage.InputTokens,+ CacheReadInputTokens: response.Usage.CacheReadInputTokens,+ OutputTokens: response.Usage.OutputTokens,},}); err != nil {return errdiff --git a/middleware/anthropic_test.go b/middleware/anthropic_test.goindex d43dcb990..b17214fbf 100644--- a/middleware/anthropic_test.go+++ b/middleware/anthropic_test.go@@ -2117,7 +2117,7 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {Message: api.Message{Role: "assistant", Content: "After search."},Done: true,DoneReason: "stop",- Metrics: api.Metrics{PromptEvalCount: 20, EvalCount: 7},+ Metrics: api.Metrics{PromptEvalCount: 20, PromptEvalCachedCount: testIntPtr(5), EvalCount: 7},}_ = json.NewEncoder(w).Encode(resp)}))@@ -2145,7 +2145,7 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {Model: "test-model",Message: api.Message{Role: "assistant", Content: "Preface "},Done: false,- Metrics: api.Metrics{PromptEvalCount: 12, EvalCount: 4},+ Metrics: api.Metrics{PromptEvalCount: 12, PromptEvalCachedCount: testIntPtr(4), EvalCount: 4},},{Model: "test-model",@@ -2169,7 +2169,7 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {Message: api.Message{Role: "assistant"},Done: true,DoneReason: "stop",- Metrics: api.Metrics{PromptEvalCount: 12, EvalCount: 4},+ Metrics: api.Metrics{PromptEvalCount: 12, PromptEvalCachedCount: testIntPtr(4), EvalCount: 4},},}c.Writer.WriteHeader(http.StatusOK)@@ -2212,8 +2212,11 @@ func TestWebSearchStreamingUsageUsesObservedChunkMetrics(t *testing.T) {if !found {t.Fatal("expected message_delta event")}- if messageDelta.Usage.InputTokens != 32 {- t.Fatalf("expected aggregated input tokens 32 (12 passthrough + 20 followup), got %d", messageDelta.Usage.InputTokens)+ if messageDelta.Usage.InputTokens != 23 {+ t.Fatalf("expected 23 uncached input tokens, got %d", messageDelta.Usage.InputTokens)+ }+ if got := messageDelta.Usage.CacheReadInputTokens; got == nil || *got != 9 {+ t.Fatalf("expected 9 cached input tokens, got %v", got)}if messageDelta.Usage.OutputTokens != 11 {t.Fatalf("expected aggregated output tokens 11 (4 passthrough + 7 followup), got %d", messageDelta.Usage.OutputTokens)diff --git a/middleware/metrics.go b/middleware/metrics.gonew file mode 100644index 000000000..a9a814580--- /dev/null+++ b/middleware/metrics.go@@ -0,0 +1,24 @@+package middleware++func optionalIntValue(v *int) int {+ if v == nil {+ return 0+ }+ return *v+}++func addOptionalInts(a, b *int) *int {+ if a == nil && b == nil {+ return nil+ }+ value := optionalIntValue(a) + optionalIntValue(b)+ return &value+}++func maxOptionalInts(a, b *int) *int {+ if a == nil && b == nil {+ return nil+ }+ value := max(optionalIntValue(a), optionalIntValue(b))+ return &value+}diff --git a/middleware/openai.go b/middleware/openai.goindex 98e961513..4a8216794 100644--- a/middleware/openai.go+++ b/middleware/openai.go@@ -705,6 +705,7 @@ func (w *WebSearchResponsesWriter) finishStream() error {var toolCalls []api.ToolCallfor _, response := range w.buffered {observed.PromptEvalCount = max(observed.PromptEvalCount, response.Metrics.PromptEvalCount)+ observed.PromptEvalCachedCount = maxOptionalInts(observed.PromptEvalCachedCount, response.Metrics.PromptEvalCachedCount)observed.EvalCount = max(observed.EvalCount, response.Metrics.EvalCount)if response.Message.Content != "" {contentBuilder.WriteString(response.Message.Content)@@ -877,6 +878,7 @@ func (w *WebSearchResponsesWriter) runLoop(ctx context.Context, initial api.Chatreturn api.ChatResponse{}, calls, usage, err}usage.PromptEvalCount += followUp.Metrics.PromptEvalCount+ usage.PromptEvalCachedCount = addOptionalInts(usage.PromptEvalCachedCount, followUp.Metrics.PromptEvalCachedCount)usage.EvalCount += followUp.Metrics.EvalCountnext, hasWebSearch, mixed := findWebSearchToolCall(followUp.Message.ToolCalls)@@ -1023,6 +1025,7 @@ func (w *WebSearchResponsesWriter) writeWebSearchResponse(final api.ChatResponseresponse.Usage.InputTokens = usage.PromptEvalCountresponse.Usage.OutputTokens = usage.EvalCountresponse.Usage.TotalTokens = usage.PromptEvalCount + usage.EvalCount+ response.Usage.InputTokensDetails.CachedTokens = optionalIntValue(usage.PromptEvalCachedCount)}w.ResponseWriter.Header().Set("Content-Type", "application/json")w.done = true@@ -1121,7 +1124,12 @@ func (w *WebSearchResponsesWriter) writeWebSearchError(err error, usage api.Metrresponse := map[string]any{"id": w.inner.responseID, "object": "response", "status": "failed", "model": w.req.Model,"output": []any{}, "error": map[string]any{"code": errorCode, "message": message},- "usage": map[string]any{"input_tokens": usage.PromptEvalCount, "output_tokens": usage.EvalCount, "total_tokens": usage.PromptEvalCount + usage.EvalCount},+ "usage": map[string]any{+ "input_tokens": usage.PromptEvalCount,+ "output_tokens": usage.EvalCount,+ "total_tokens": usage.PromptEvalCount + usage.EvalCount,+ "input_tokens_details": map[string]any{"cached_tokens": optionalIntValue(usage.PromptEvalCachedCount)},+ },}initialEvents := w.inner.converter.Process(api.ChatResponse{})for _, event := range initialEvents {diff --git a/middleware/openai_test.go b/middleware/openai_test.goindex 2d50e2464..92b2b4110 100644--- a/middleware/openai_test.go+++ b/middleware/openai_test.go@@ -20,6 +20,10 @@ import ("github.com/ollama/ollama/openai")+func testIntPtr(v int) *int {+ return &v+}+// testPropsMap creates a ToolPropertiesMap from a map (convenience function for tests)func testPropsMap(m map[string]api.ToolProperty) *api.ToolPropertiesMap {props := api.NewToolPropertiesMap()@@ -683,8 +687,9 @@ func TestChatWriter_StreamMetricsTrailerSkipsEmptyContentChunk(t *testing.T) {Done: true,DoneReason: "stop",Metrics: api.Metrics{- PromptEvalCount: 3,- EvalCount: 1,+ PromptEvalCount: 3,+ PromptEvalCachedCount: testIntPtr(1),+ EvalCount: 1,},}data, err = json.Marshal(trailer)@@ -710,6 +715,9 @@ func TestChatWriter_StreamMetricsTrailerSkipsEmptyContentChunk(t *testing.T) {if !strings.Contains(frames[2], `"choices":[]`) {t.Fatalf("expected usage frame with empty choices, got %s", frames[2])}+ if !strings.Contains(frames[2], `"prompt_tokens_details":{"cached_tokens":1}`) {+ t.Fatalf("expected usage frame with cached prompt tokens, got %s", frames[2])+ }if frames[3] != "[DONE]" {t.Fatalf("expected final frame [DONE], got %q", frames[3])}diff --git a/middleware/responses_web_search_test.go b/middleware/responses_web_search_test.goindex daba465a7..a19c5763f 100644--- a/middleware/responses_web_search_test.go+++ b/middleware/responses_web_search_test.go@@ -49,11 +49,11 @@ func TestWebSearchResponsesWriterNonStreaming(t *testing.T) {if strings.Contains(messages[1].Content, "Cite") || !strings.Contains(messages[1].Content, "URL: https://ollama.com/news") {t.Fatalf("unexpected search result content: %q", messages[1].Content)}- return api.ChatResponse{Done: true, Message: api.Message{Role: "assistant", Content: "Read [Ollama](https://ollama.com/news)."}, Metrics: api.Metrics{PromptEvalCount: 7, EvalCount: 3}}, nil+ return api.ChatResponse{Done: true, Message: api.Message{Role: "assistant", Content: "Read [Ollama](https://ollama.com/news)."}, Metrics: api.Metrics{PromptEvalCount: 7, PromptEvalCachedCount: testIntPtr(3), EvalCount: 3}}, nil},}- initial := api.ChatResponse{Done: true, Message: api.Message{ToolCalls: []api.ToolCall{{ID: "call_1", Function: api.ToolCallFunction{Name: "web_search", Arguments: testArgs(map[string]any{"query": "ollama news"})}}}}, Metrics: api.Metrics{PromptEvalCount: 5, EvalCount: 2}}+ initial := api.ChatResponse{Done: true, Message: api.Message{ToolCalls: []api.ToolCall{{ID: "call_1", Function: api.ToolCallFunction{Name: "web_search", Arguments: testArgs(map[string]any{"query": "ollama news"})}}}}, Metrics: api.Metrics{PromptEvalCount: 5, PromptEvalCachedCount: testIntPtr(2), EvalCount: 2}}data, err := json.Marshal(initial)if err != nil {t.Fatal(err)@@ -78,6 +78,9 @@ func TestWebSearchResponsesWriterNonStreaming(t *testing.T) {if response.Usage == nil || response.Usage.InputTokens != 12 || response.Usage.OutputTokens != 5 {t.Fatalf("usage = %#v", response.Usage)}+ if response.Usage.InputTokensDetails.CachedTokens != 5 {+ t.Fatalf("cached input tokens = %d, want 5", response.Usage.InputTokensDetails.CachedTokens)+ }if len(response.Output[1].Content[0].Annotations) != 0 {t.Fatalf("annotations = %#v, want none", response.Output[1].Content[0].Annotations)}diff --git a/openai/openai.go b/openai/openai.goindex 2d38607db..66b8e69ad 100644--- a/openai/openai.go+++ b/openai/openai.go@@ -72,10 +72,15 @@ type CompleteChunkChoice struct {Logprobs *ChoiceLogprobs `json:"logprobs,omitempty"`}+type PromptTokensDetails struct {+ CachedTokens int `json:"cached_tokens"`+}+type Usage struct {- PromptTokens int `json:"prompt_tokens"`- CompletionTokens int `json:"completion_tokens"`- TotalTokens int `json:"total_tokens"`+ PromptTokens int `json:"prompt_tokens"`+ PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"`+ CompletionTokens int `json:"completion_tokens"`+ TotalTokens int `json:"total_tokens"`}type ResponseFormat struct {@@ -238,11 +243,15 @@ func NewError(code int, message string) ErrorResponse {// ToUsage converts an api.ChatResponse to Usagefunc ToUsage(r api.ChatResponse) Usage {- return Usage{+ usage := Usage{PromptTokens: r.Metrics.PromptEvalCount,CompletionTokens: r.Metrics.EvalCount,TotalTokens: r.Metrics.PromptEvalCount + r.Metrics.EvalCount,}+ if r.Metrics.PromptEvalCachedCount != nil {+ usage.PromptTokensDetails = &PromptTokensDetails{CachedTokens: *r.Metrics.PromptEvalCachedCount}+ }+ return usage}// ToToolCalls converts api.ToolCall to OpenAI ToolCall format@@ -396,11 +405,15 @@ func FinishChunk(id string, r api.ChatResponse, toolCallSent bool) ChatCompletio// ToUsageGenerate converts an api.GenerateResponse to Usagefunc ToUsageGenerate(r api.GenerateResponse) Usage {- return Usage{+ usage := Usage{PromptTokens: r.Metrics.PromptEvalCount,CompletionTokens: r.Metrics.EvalCount,TotalTokens: r.Metrics.PromptEvalCount + r.Metrics.EvalCount,}+ if r.Metrics.PromptEvalCachedCount != nil {+ usage.PromptTokensDetails = &PromptTokensDetails{CachedTokens: *r.Metrics.PromptEvalCachedCount}+ }+ return usage}// ToCompletion converts an api.GenerateResponse to Completiondiff --git a/openai/openai_test.go b/openai/openai_test.goindex 1f3b258b0..a8c53cdae 100644--- a/openai/openai_test.go+++ b/openai/openai_test.go@@ -3,6 +3,7 @@ package openaiimport ("encoding/base64""encoding/json"+ "strings""testing""time"@@ -30,6 +31,10 @@ const (image = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=`)+func testIntPtr(v int) *int {+ return &v+}+func TestFromChatRequest_Basic(t *testing.T) {req := ChatCompletionRequest{Model: "test-model",@@ -180,8 +185,9 @@ func TestFromCompleteRequest_Basic(t *testing.T) {func TestToUsage(t *testing.T) {resp := api.ChatResponse{Metrics: api.Metrics{- PromptEvalCount: 10,- EvalCount: 20,+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ EvalCount: 20,},}@@ -190,6 +196,9 @@ func TestToUsage(t *testing.T) {if usage.PromptTokens != 10 {t.Errorf("expected PromptTokens 10, got %d", usage.PromptTokens)}+ if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 4 {+ t.Errorf("expected CachedTokens 4, got %#v", usage.PromptTokensDetails)+ }if usage.CompletionTokens != 20 {t.Errorf("expected CompletionTokens 20, got %d", usage.CompletionTokens)@@ -198,6 +207,68 @@ func TestToUsage(t *testing.T) {if usage.TotalTokens != 30 {t.Errorf("expected TotalTokens 30, got %d", usage.TotalTokens)}++ data, err := json.Marshal(usage)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), `"prompt_tokens_details":{"cached_tokens":4}`) {+ t.Errorf("unexpected usage json: %s", data)+ }+}++func TestToUsageOmitsUnreportedCacheDetails(t *testing.T) {+ usage := ToUsage(api.ChatResponse{Metrics: api.Metrics{PromptEvalCount: 10, EvalCount: 2}})+ if usage.PromptTokensDetails != nil {+ t.Fatalf("expected no cache details, got %#v", usage.PromptTokensDetails)+ }++ data, err := json.Marshal(usage)+ if err != nil {+ t.Fatal(err)+ }+ var payload map[string]any+ if err := json.Unmarshal(data, &payload); err != nil {+ t.Fatal(err)+ }+ if _, ok := payload["prompt_tokens_details"]; ok {+ t.Fatalf("unexpected cache details in %s", data)+ }+}++func TestToUsageIncludesZeroCacheDetails(t *testing.T) {+ usage := ToUsage(api.ChatResponse{Metrics: api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(0),+ EvalCount: 2,+ }})+ if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 0 {+ t.Fatalf("expected zero cache details, got %#v", usage.PromptTokensDetails)+ }++ data, err := json.Marshal(usage)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), `"prompt_tokens_details":{"cached_tokens":0}`) {+ t.Errorf("unexpected usage json: %s", data)+ }+}++func TestToCompletionUsageIncludesCachedTokens(t *testing.T) {+ completion := ToCompletion("completion-id", api.GenerateResponse{+ Metrics: api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ EvalCount: 2,+ },+ })+ if completion.Usage.PromptTokens != 10 || completion.Usage.TotalTokens != 12 {+ t.Fatalf("unexpected usage: %#v", completion.Usage)+ }+ if details := completion.Usage.PromptTokensDetails; details == nil || details.CachedTokens != 4 {+ t.Fatalf("expected 4 cached tokens, got %#v", details)+ }}func TestNewError(t *testing.T) {diff --git a/openai/responses.go b/openai/responses.goindex 4a399ad6e..e32c8ac66 100644--- a/openai/responses.go+++ b/openai/responses.go@@ -885,6 +885,13 @@ type ResponsesUsage struct {OutputTokensDetails ResponsesOutputTokensDetails `json:"output_tokens_details"`}+func intValue(v *int) int {+ if v == nil {+ return 0+ }+ return *v+}+// derefFloat64 returns the value of a float64 pointer, or a default if nil.func derefFloat64(p *float64, def float64) float64 {if p != nil {@@ -1001,11 +1008,10 @@ func ToResponse(model, responseID, itemID string, chatResponse api.ChatResponse,Temperature: derefFloat64(request.Temperature, 1.0),Reasoning: reasoning,Usage: &ResponsesUsage{- InputTokens: chatResponse.PromptEvalCount,- OutputTokens: chatResponse.EvalCount,- TotalTokens: chatResponse.PromptEvalCount + chatResponse.EvalCount,- // TODO(drifkin): wire through the actual values- InputTokensDetails: ResponsesInputTokensDetails{CachedTokens: 0},+ InputTokens: chatResponse.PromptEvalCount,+ OutputTokens: chatResponse.EvalCount,+ TotalTokens: chatResponse.PromptEvalCount + chatResponse.EvalCount,+ InputTokensDetails: ResponsesInputTokensDetails{CachedTokens: intValue(chatResponse.PromptEvalCachedCount)},// TODO(drifkin): wire through the actual valuesOutputTokensDetails: ResponsesOutputTokensDetails{ReasoningTokens: 0},},@@ -1608,7 +1614,7 @@ func (c *ResponsesStreamConverter) processCompletion(r api.ChatResponse) []Respo"output_tokens": r.EvalCount,"total_tokens": r.PromptEvalCount + r.EvalCount,"input_tokens_details": map[string]any{- "cached_tokens": 0,+ "cached_tokens": intValue(r.PromptEvalCachedCount),},"output_tokens_details": map[string]any{"reasoning_tokens": 0,diff --git a/openai/responses_test.go b/openai/responses_test.goindex 98c20c581..b43bed29f 100644--- a/openai/responses_test.go+++ b/openai/responses_test.go@@ -2,6 +2,7 @@ package openaiimport ("encoding/json"+ "strings""testing""time"@@ -1743,6 +1744,34 @@ func TestToResponse_WithReasoning(t *testing.T) {}}+func TestToResponse_UsageIncludesCachedTokens(t *testing.T) {+ response := ToResponse("gpt-oss:20b", "resp_123", "msg_456", api.ChatResponse{+ CreatedAt: time.Now(),+ Message: api.Message{Content: "The answer is 42"},+ Done: true,+ Metrics: api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ EvalCount: 3,+ },+ }, ResponsesRequest{})++ if response.Usage == nil {+ t.Fatal("expected usage")+ }+ if response.Usage.InputTokens != 10 || response.Usage.InputTokensDetails.CachedTokens != 4 || response.Usage.TotalTokens != 13 {+ t.Errorf("unexpected usage: %+v", response.Usage)+ }++ data, err := json.Marshal(response.Usage)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), `"input_tokens_details":{"cached_tokens":4}`) {+ t.Errorf("unexpected usage json: %s", data)+ }+}+func TestFromResponsesRequest_Instructions(t *testing.T) {reqJSON := `{"model": "gpt-oss:20b",@@ -2143,6 +2172,11 @@ func TestResponsesStreamConverter_ResponseCompletedIncludesOutput(t *testing.T)events := converter.Process(api.ChatResponse{Message: api.Message{},Done: true,+ Metrics: api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ EvalCount: 3,+ },})// Find the response.completed event@@ -2172,6 +2206,11 @@ func TestResponsesStreamConverter_ResponseCompletedIncludesOutput(t *testing.T)if item["type"] != "message" {t.Errorf("output[0].type = %q, want %q", item["type"], "message")}+ usage := response["usage"].(map[string]any)+ inputDetails := usage["input_tokens_details"].(map[string]any)+ if inputDetails["cached_tokens"] != 4 {+ t.Errorf("cached_tokens = %v, want 4", inputDetails["cached_tokens"])+ }}func TestResponsesStreamConverter_ResponseCreatedIncludesOutput(t *testing.T) {diff --git a/server/routes.go b/server/routes.goindex 53e392978..263c7e27a 100644--- a/server/routes.go+++ b/server/routes.go@@ -677,10 +677,11 @@ func (s *Server) GenerateHandler(c *gin.Context) {Response: cr.Content,Done: cr.Done,Metrics: api.Metrics{- PromptEvalCount: cr.PromptEvalCount,- PromptEvalDuration: cr.PromptEvalDuration,- EvalCount: cr.EvalCount,- EvalDuration: cr.EvalDuration,+ PromptEvalCount: cr.PromptEvalCount,+ PromptEvalCachedCount: cr.PromptEvalCachedCount,+ PromptEvalDuration: cr.PromptEvalDuration,+ EvalCount: cr.EvalCount,+ EvalDuration: cr.EvalDuration,},Logprobs: toAPILogprobs(cr.Logprobs),}@@ -2750,6 +2751,7 @@ func (s *Server) ChatHandler(c *gin.Context) {defer close(ch)structuredOutputsState := structuredOutputsState_None+ var firstPassMetrics api.Metricsfor {var tb strings.Builder@@ -2768,6 +2770,7 @@ func (s *Server) ChatHandler(c *gin.Context) {if req.Format != nil && structuredOutputsState == structuredOutputsState_None && !forceImmediate && ((builtinParser != nil || thinkingState != nil) && slices.Contains(m.Capabilities(), model.CapabilityThinking)) {currentFormat = nil}+ includeIntermediateMetrics := req.Format != nil && currentFormat == nil// sets up new context given parent context per requestctx, cancel := context.WithCancel(c.Request.Context())@@ -2775,30 +2778,47 @@ func (s *Server) ChatHandler(c *gin.Context) {var parserErr errorerr := r.Completion(ctx, llm.CompletionRequest{- Prompt: prompt,- Media: media,- Format: currentFormat,- Options: opts,- Shift: req.Shift == nil || *req.Shift,- Truncate: truncate,- Logprobs: req.Logprobs,- TopLogprobs: req.TopLogprobs,- PreservedTokens: preservedTokensForCompletion(builtinParser),- ToolCallTag: toolCallTagForCompletion(toolParser),- LeadingBOS: leadingBOSForModel(m),+ Prompt: prompt,+ Media: media,+ Format: currentFormat,+ Options: opts,+ Shift: req.Shift == nil || *req.Shift,+ Truncate: truncate,+ Logprobs: req.Logprobs,+ TopLogprobs: req.TopLogprobs,+ PreservedTokens: preservedTokensForCompletion(builtinParser),+ ToolCallTag: toolCallTagForCompletion(toolParser),+ LeadingBOS: leadingBOSForModel(m),+ IncludeIntermediateMetrics: includeIntermediateMetrics,}, func(r llm.CompletionResponse) {+ metrics := api.Metrics{+ PromptEvalCount: r.PromptEvalCount,+ PromptEvalCachedCount: r.PromptEvalCachedCount,+ PromptEvalDuration: r.PromptEvalDuration,+ EvalCount: r.EvalCount,+ EvalDuration: r.EvalDuration,+ }+ if includeIntermediateMetrics {+ firstPassMetrics = metrics+ if !r.Done {+ metrics = api.Metrics{}+ }+ } else if structuredOutputsState == structuredOutputsState_Applying && r.Done {+ // Treat the restart as generation work: retain the original prompt metrics and fold in the second prefill.+ metrics.PromptEvalCount = firstPassMetrics.PromptEvalCount+ metrics.PromptEvalCachedCount = firstPassMetrics.PromptEvalCachedCount+ metrics.PromptEvalDuration = firstPassMetrics.PromptEvalDuration+ metrics.EvalCount += firstPassMetrics.EvalCount+ metrics.EvalDuration += firstPassMetrics.EvalDuration + r.PromptEvalDuration+ }+res := api.ChatResponse{Model: req.Model,CreatedAt: time.Now().UTC(),Message: api.Message{Role: "assistant", Content: r.Content},Done: r.Done,- Metrics: api.Metrics{- PromptEvalCount: r.PromptEvalCount,- PromptEvalDuration: r.PromptEvalDuration,- EvalCount: r.EvalCount,- EvalDuration: r.EvalDuration,- },- Logprobs: toAPILogprobs(r.Logprobs),+ Metrics: metrics,+ Logprobs: toAPILogprobs(r.Logprobs),}if r.Done {@@ -3006,10 +3026,11 @@ func (s *Server) handleNativeChat(c *gin.Context, req api.ChatRequest, m *Model,Message: r.Message,Done: r.Done,Metrics: api.Metrics{- PromptEvalCount: r.PromptEvalCount,- PromptEvalDuration: r.PromptEvalDuration,- EvalCount: r.EvalCount,- EvalDuration: r.EvalDuration,+ PromptEvalCount: r.PromptEvalCount,+ PromptEvalCachedCount: r.PromptEvalCachedCount,+ PromptEvalDuration: r.PromptEvalDuration,+ EvalCount: r.EvalCount,+ EvalDuration: r.EvalDuration,},Logprobs: toAPILogprobs(r.Logprobs),}diff --git a/server/routes_cloud_test.go b/server/routes_cloud_test.goindex 6868c5806..6ca8929cb 100644--- a/server/routes_cloud_test.go+++ b/server/routes_cloud_test.go@@ -215,7 +215,8 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {})t.Run("api chat", func(t *testing.T) {- upstream, capture := newUpstream(t, `{"message":{"role":"assistant","content":"ok"},"done":true}`)+ upstreamResponse := `{"message":{"role":"assistant","content":"ok"},"done":true,"prompt_eval_count":12,"prompt_eval_cached_count":9}`+ upstream, capture := newUpstream(t, upstreamResponse)defer upstream.Close()original := cloudProxyBaseURL@@ -247,6 +248,9 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {if resp.StatusCode != http.StatusOK {t.Fatalf("expected status 200, got %d (%s)", resp.StatusCode, string(body))}+ if got := string(body); got != upstreamResponse {+ t.Fatalf("cloud response changed: got %q, want %q", got, upstreamResponse)+ }if capture.path != "/api/chat" {t.Fatalf("expected upstream path /api/chat, got %q", capture.path)@@ -387,7 +391,8 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {})t.Run("v1 chat completions bypasses conversion", func(t *testing.T) {- upstream, capture := newUpstream(t, `{"id":"chatcmpl_test","object":"chat.completion"}`)+ upstreamResponse := `{"id":"chatcmpl_test","object":"chat.completion","usage":{"prompt_tokens":12,"prompt_tokens_details":{"cached_tokens":9},"completion_tokens":3,"total_tokens":15}}`+ upstream, capture := newUpstream(t, upstreamResponse)defer upstream.Close()original := cloudProxyBaseURL@@ -420,6 +425,9 @@ func TestExplicitCloudPassthroughAPIAndV1(t *testing.T) {if resp.StatusCode != http.StatusOK {t.Fatalf("expected status 200, got %d (%s)", resp.StatusCode, string(body))}+ if got := string(body); got != upstreamResponse {+ t.Fatalf("cloud response changed: got %q, want %q", got, upstreamResponse)+ }if capture.path != "/v1/chat/completions" {t.Fatalf("expected upstream path /v1/chat/completions, got %q", capture.path)@@ -811,10 +819,10 @@ func TestCloudResponsesWebSearchUsesLocalOrchestration(t *testing.T) {w.Header().Set("Content-Type", "application/x-ndjson")if chatCalls == 1 {_, _ = io.WriteString(w, `{"message":{"role":"assistant","tool_calls":[{"id":"call_1","function":{"name":"web_search","arguments":{"query":"latest Ollama release"}}}]},"done":false}`+"\n")- _, _ = io.WriteString(w, `{"message":{"role":"assistant"},"done":true,"prompt_eval_count":12,"eval_count":4}`+"\n")+ _, _ = io.WriteString(w, `{"message":{"role":"assistant"},"done":true,"prompt_eval_count":12,"prompt_eval_cached_count":5,"eval_count":4}`+"\n")return}- _, _ = io.WriteString(w, `{"message":{"role":"assistant","content":"Ollama [release](https://ollama.com/release)."},"done":true,"prompt_eval_count":20,"eval_count":6}`)+ _, _ = io.WriteString(w, `{"message":{"role":"assistant","content":"Ollama [release](https://ollama.com/release)."},"done":true,"prompt_eval_count":20,"prompt_eval_cached_count":17,"eval_count":6}`)case "/api/web_search":searchCalls++w.Header().Set("Content-Type", "application/json")@@ -873,6 +881,9 @@ func TestCloudResponsesWebSearchUsesLocalOrchestration(t *testing.T) {if bytes.Contains(body, []byte("response.function_call_arguments")) || bytes.Contains(body, []byte(`"type":"function_call"`)) {t.Fatalf("private web_search function leaked: %s", body)}+ if !bytes.Contains(body, []byte(`"input_tokens_details":{"cached_tokens":22}`)) {+ t.Fatalf("missing aggregated cached token usage: %s", body)+ }}func TestCloudResponsesUnsupportedWebSearchPassthrough(t *testing.T) {diff --git a/server/routes_generate_test.go b/server/routes_generate_test.goindex 5de22f78b..f3decb881 100644--- a/server/routes_generate_test.go+++ b/server/routes_generate_test.go@@ -279,13 +279,14 @@ func TestChatHandlerChatTemplateRoute(t *testing.T) {mock := mockRunner{ChatFn: func(_ context.Context, req llm.ChatRequest, fn func(llm.ChatResponse)) error {fn(llm.ChatResponse{- Message: api.Message{Role: "assistant", Content: "chat template response"},- Done: true,- DoneReason: llm.DoneReasonStop,- PromptEvalCount: 1,- PromptEvalDuration: time.Millisecond,- EvalCount: 2,- EvalDuration: 2 * time.Millisecond,+ Message: api.Message{Role: "assistant", Content: "chat template response"},+ Done: true,+ DoneReason: llm.DoneReasonStop,+ PromptEvalCount: 2,+ PromptEvalCachedCount: testIntPtr(1),+ PromptEvalDuration: time.Millisecond,+ EvalCount: 2,+ EvalDuration: 2 * time.Millisecond,})return nil},@@ -314,6 +315,9 @@ func TestChatHandlerChatTemplateRoute(t *testing.T) {if actual.Message.Content != "chat template response" {t.Fatalf("expected chat template response, got %q", actual.Message.Content)}+ if actual.PromptEvalCount != 2 || actual.PromptEvalCachedCount == nil || *actual.PromptEvalCachedCount != 1 {+ t.Errorf("prompt counts = (%d, %v), want (2, 1)", actual.PromptEvalCount, actual.PromptEvalCachedCount)+ }if len(mock.ChatRequest.Messages) != 1 || mock.ChatRequest.Messages[0].Content != "hello" {t.Fatalf("chat_template request messages = %#v", mock.ChatRequest.Messages)}@@ -753,12 +757,13 @@ func TestGenerateChat(t *testing.T) {mock := mockRunner{CompletionResponse: llm.CompletionResponse{- Done: true,- DoneReason: llm.DoneReasonStop,- PromptEvalCount: 1,- PromptEvalDuration: 1,- EvalCount: 1,- EvalDuration: 1,+ Done: true,+ DoneReason: llm.DoneReasonStop,+ PromptEvalCount: 2,+ PromptEvalCachedCount: testIntPtr(1),+ PromptEvalDuration: 1,+ EvalCount: 1,+ EvalDuration: 1,},}@@ -970,6 +975,9 @@ func TestGenerateChat(t *testing.T) {if actual.PromptEvalCount == 0 {t.Errorf("expected prompt eval count > 0, got 0")}+ if actual.PromptEvalCachedCount == nil || *actual.PromptEvalCachedCount != 1 {+ t.Errorf("expected cached prompt eval count 1, got %v", actual.PromptEvalCachedCount)+ }if actual.PromptEvalDuration == 0 {t.Errorf("expected prompt eval duration > 0, got 0")@@ -2574,6 +2582,35 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {}})+ earlyFirstPassMetrics := api.Metrics{+ PromptEvalCount: 4,+ PromptEvalCachedCount: testIntPtr(1),+ PromptEvalDuration: 5 * time.Millisecond,+ EvalCount: 6,+ EvalDuration: 7 * time.Millisecond,+ }+ firstPassMetrics := api.Metrics{+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ PromptEvalDuration: 11 * time.Millisecond,+ EvalCount: 12,+ EvalDuration: 13 * time.Millisecond,+ }+ secondPassMetrics := api.Metrics{+ PromptEvalCount: 20_000,+ PromptEvalCachedCount: testIntPtr(19_000),+ PromptEvalDuration: 21 * time.Millisecond,+ EvalCount: 22,+ EvalDuration: 23 * time.Millisecond,+ }+ wantMetrics := api.Metrics{+ PromptEvalCount: firstPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: firstPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: firstPassMetrics.PromptEvalDuration,+ EvalCount: firstPassMetrics.EvalCount + secondPassMetrics.EvalCount,+ EvalDuration: firstPassMetrics.EvalDuration + secondPassMetrics.PromptEvalDuration + secondPassMetrics.EvalDuration,+ }+t.Run("structured outputs restart non-stream", func(t *testing.T) {var (requestsMu sync.Mutex@@ -2596,10 +2633,22 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {switch callNum {case 1:fn(llm.CompletionResponse{- Content: " I am thinking through this problem. </think> {\"answer\":\"42\"}",- Done: false,- PromptEvalCount: 1,- PromptEvalDuration: 1,+ Content: " I am thinking through this problem.",+ Done: false,+ PromptEvalCount: earlyFirstPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: earlyFirstPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: earlyFirstPassMetrics.PromptEvalDuration,+ EvalCount: earlyFirstPassMetrics.EvalCount,+ EvalDuration: earlyFirstPassMetrics.EvalDuration,+ })+ fn(llm.CompletionResponse{+ Content: " </think> {\"answer\":\"42\"}",+ Done: false,+ PromptEvalCount: firstPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: firstPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: firstPassMetrics.PromptEvalDuration,+ EvalCount: firstPassMetrics.EvalCount,+ EvalDuration: firstPassMetrics.EvalDuration,})select {@@ -2611,13 +2660,14 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {}case 2:fn(llm.CompletionResponse{- Content: `{"answer":"42"}`,- Done: true,- DoneReason: llm.DoneReasonStop,- PromptEvalCount: 1,- PromptEvalDuration: 1,- EvalCount: 1,- EvalDuration: 1,+ Content: `{"answer":"42"}`,+ Done: true,+ DoneReason: llm.DoneReasonStop,+ PromptEvalCount: secondPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: secondPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: secondPassMetrics.PromptEvalDuration,+ EvalCount: secondPassMetrics.EvalCount,+ EvalDuration: secondPassMetrics.EvalDuration,})return nildefault:@@ -2650,10 +2700,16 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {if requests[0].Format != nil {t.Errorf("expected first completion format to be nil, got %q", requests[0].Format)}+ if !requests[0].IncludeIntermediateMetrics {+ t.Error("expected first completion to request per-token metrics")+ }if !bytes.Equal([]byte(format), []byte(requests[1].Format)) {t.Errorf("expected second completion format to match original format")}+ if requests[1].IncludeIntermediateMetrics {+ t.Error("expected second completion to use terminal metrics")+ }var resp api.ChatResponseif err := json.NewDecoder(w.Body).Decode(&resp); err != nil {@@ -2675,6 +2731,15 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {if resp.DoneReason != "stop" {t.Errorf("expected done reason stop, got %s", resp.DoneReason)}+ if resp.PromptEvalCount != wantMetrics.PromptEvalCount || resp.EvalCount != wantMetrics.EvalCount {+ t.Errorf("response counts = (%d, %d), want (%d, %d)", resp.PromptEvalCount, resp.EvalCount, wantMetrics.PromptEvalCount, wantMetrics.EvalCount)+ }+ if diff := cmp.Diff(wantMetrics.PromptEvalCachedCount, resp.PromptEvalCachedCount); diff != "" {+ t.Errorf("response cached prompt count mismatch (-want +got):\n%s", diff)+ }+ if resp.PromptEvalDuration != wantMetrics.PromptEvalDuration || resp.EvalDuration != wantMetrics.EvalDuration {+ t.Errorf("response durations = (%s, %s), want (%s, %s)", resp.PromptEvalDuration, resp.EvalDuration, wantMetrics.PromptEvalDuration, wantMetrics.EvalDuration)+ }})t.Run("structured outputs restart streaming", func(t *testing.T) {@@ -2699,10 +2764,22 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {switch callNum {case 1:fn(llm.CompletionResponse{- Content: " I am thinking through this problem. </think> {\"answer\":\"42\"}",- Done: false,- PromptEvalCount: 1,- PromptEvalDuration: 1,+ Content: " I am thinking through this problem.",+ Done: false,+ PromptEvalCount: earlyFirstPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: earlyFirstPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: earlyFirstPassMetrics.PromptEvalDuration,+ EvalCount: earlyFirstPassMetrics.EvalCount,+ EvalDuration: earlyFirstPassMetrics.EvalDuration,+ })+ fn(llm.CompletionResponse{+ Content: " </think> {\"answer\":\"42\"}",+ Done: false,+ PromptEvalCount: firstPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: firstPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: firstPassMetrics.PromptEvalDuration,+ EvalCount: firstPassMetrics.EvalCount,+ EvalDuration: firstPassMetrics.EvalDuration,})select {@@ -2714,13 +2791,14 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {}case 2:fn(llm.CompletionResponse{- Content: `{"answer":"42"}`,- Done: true,- DoneReason: llm.DoneReasonStop,- PromptEvalCount: 1,- PromptEvalDuration: 1,- EvalCount: 1,- EvalDuration: 1,+ Content: `{"answer":"42"}`,+ Done: true,+ DoneReason: llm.DoneReasonStop,+ PromptEvalCount: secondPassMetrics.PromptEvalCount,+ PromptEvalCachedCount: secondPassMetrics.PromptEvalCachedCount,+ PromptEvalDuration: secondPassMetrics.PromptEvalDuration,+ EvalCount: secondPassMetrics.EvalCount,+ EvalDuration: secondPassMetrics.EvalDuration,})return nildefault:@@ -2753,10 +2831,16 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {if requests[0].Format != nil {t.Errorf("expected first completion format to be nil, got %q", requests[0].Format)}+ if !requests[0].IncludeIntermediateMetrics {+ t.Error("expected first completion to request per-token metrics")+ }if !bytes.Equal([]byte(format), []byte(requests[1].Format)) {t.Errorf("expected second completion format to match original format")}+ if requests[1].IncludeIntermediateMetrics {+ t.Error("expected second completion to use terminal metrics")+ }decoder := json.NewDecoder(w.Body)var events []api.ChatResponse@@ -2778,8 +2862,15 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {}first := events[0]- if first.Message.Thinking != "I am thinking through this problem. " {- t.Errorf("expected first event thinking %q, got %q", "I am thinking through this problem. ", first.Message.Thinking)+ var thinking strings.Builder+ for _, event := range events {+ thinking.WriteString(event.Message.Thinking)+ if !event.Done && (event.PromptEvalCount != 0 || event.PromptEvalCachedCount != nil || event.PromptEvalDuration != 0 || event.EvalCount != 0 || event.EvalDuration != 0) {+ t.Errorf("non-terminal event unexpectedly exposed metrics: %+v", event.Metrics)+ }+ }+ if got := thinking.String(); got != "I am thinking through this problem. " {+ t.Errorf("thinking = %q, want %q", got, "I am thinking through this problem. ")}if first.Message.Content != "" {@@ -2789,7 +2880,6 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {if first.Done {t.Error("expected first event to be non-terminal")}-last := events[len(events)-1]if last.Message.Thinking != "" {t.Errorf("expected final event thinking to be empty, got %q", last.Message.Thinking)@@ -2806,6 +2896,15 @@ func TestChatWithPromptEndingInThinkTag(t *testing.T) {if last.DoneReason != "stop" {t.Errorf("expected final done reason stop, got %s", last.DoneReason)}+ if last.PromptEvalCount != wantMetrics.PromptEvalCount || last.EvalCount != wantMetrics.EvalCount {+ t.Errorf("final counts = (%d, %d), want (%d, %d)", last.PromptEvalCount, last.EvalCount, wantMetrics.PromptEvalCount, wantMetrics.EvalCount)+ }+ if diff := cmp.Diff(wantMetrics.PromptEvalCachedCount, last.PromptEvalCachedCount); diff != "" {+ t.Errorf("final cached prompt count mismatch (-want +got):\n%s", diff)+ }+ if last.PromptEvalDuration != wantMetrics.PromptEvalDuration || last.EvalDuration != wantMetrics.EvalDuration {+ t.Errorf("final durations = (%s, %s), want (%s, %s)", last.PromptEvalDuration, last.EvalDuration, wantMetrics.PromptEvalDuration, wantMetrics.EvalDuration)+ }})}diff --git a/x/mlxrunner/client.go b/x/mlxrunner/client.goindex 71debf9c2..5c6046e3f 100644--- a/x/mlxrunner/client.go+++ b/x/mlxrunner/client.go@@ -111,12 +111,13 @@ func (c *Client) WaitUntilRunning(ctx context.Context) error {}type CompletionRequest struct {- Prompt string- Media []llm.MediaData- Format json.RawMessage- Options api.Options- Logprobs bool- TopLogprobs int+ Prompt string+ Media []llm.MediaData+ Format json.RawMessage+ Options api.Options+ Logprobs bool+ TopLogprobs int+ IncludeIntermediateMetrics bool}type CompletionResponse struct {@@ -124,10 +125,11 @@ type CompletionResponse struct {Done boolDoneReason int- PromptEvalCount int- PromptEvalDuration time.Duration- EvalCount int- EvalDuration time.Duration+ PromptEvalCount int+ PromptEvalCachedCount *int+ PromptEvalDuration time.Duration+ EvalCount int+ EvalDuration time.DurationLogprobs []llm.Logprob@@ -156,11 +158,12 @@ func (c *Client) Close() error {// Completion implements llm.LlamaServer.func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error {creq := CompletionRequest{- Prompt: req.Prompt,- Media: req.Media,- Format: req.Format,- Logprobs: req.Logprobs,- TopLogprobs: req.TopLogprobs,+ Prompt: req.Prompt,+ Media: req.Media,+ Format: req.Format,+ Logprobs: req.Logprobs,+ TopLogprobs: req.TopLogprobs,+ IncludeIntermediateMetrics: req.IncludeIntermediateMetrics,}if req.Options != nil {creq.Options = *req.Options@@ -205,14 +208,15 @@ func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn f}cresp := llm.CompletionResponse{- Content: raw.Content,- Done: raw.Done,- DoneReason: llm.DoneReason(raw.DoneReason),- PromptEvalCount: raw.PromptEvalCount,- PromptEvalDuration: raw.PromptEvalDuration,- EvalCount: raw.EvalCount,- EvalDuration: raw.EvalDuration,- Logprobs: raw.Logprobs,+ Content: raw.Content,+ Done: raw.Done,+ DoneReason: llm.DoneReason(raw.DoneReason),+ PromptEvalCount: raw.PromptEvalCount,+ PromptEvalCachedCount: raw.PromptEvalCachedCount,+ PromptEvalDuration: raw.PromptEvalDuration,+ EvalCount: raw.EvalCount,+ EvalDuration: raw.EvalDuration,+ Logprobs: raw.Logprobs,}fn(cresp)diff --git a/x/mlxrunner/client_test.go b/x/mlxrunner/client_test.gonew file mode 100644index 000000000..fb56db618--- /dev/null+++ b/x/mlxrunner/client_test.go@@ -0,0 +1,60 @@+package mlxrunner++import (+ "encoding/json"+ "net"+ "net/http"+ "net/http/httptest"+ "strconv"+ "testing"++ "github.com/ollama/ollama/api"+ "github.com/ollama/ollama/llm"+)++func testIntPtr(v int) *int {+ return &v+}++func TestClientCompletionRequestsIntermediateMetrics(t *testing.T) {+ var request CompletionRequest+ want := CompletionResponse{+ Done: true,+ PromptEvalCount: 10,+ PromptEvalCachedCount: testIntPtr(4),+ }+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Errorf("decode request: %v", err)+ return+ }+ if err := json.NewEncoder(w).Encode(want); err != nil {+ t.Errorf("encode response: %v", err)+ }+ }))+ t.Cleanup(srv.Close)++ _, portString, err := net.SplitHostPort(srv.Listener.Addr().String())+ if err != nil {+ t.Fatalf("parse server port: %v", err)+ }+ port, err := strconv.Atoi(portString)+ if err != nil {+ t.Fatalf("parse server port: %v", err)+ }+ client := &Client{port: port, client: srv.Client()}+ opts := api.DefaultOptions()+ var got llm.CompletionResponse+ if err := client.Completion(t.Context(), llm.CompletionRequest{+ Options: &opts,+ IncludeIntermediateMetrics: true,+ }, func(response llm.CompletionResponse) { got = response }); err != nil {+ t.Fatalf("Completion: %v", err)+ }+ if !request.IncludeIntermediateMetrics {+ t.Fatal("metrics per token was not forwarded to the MLX runner")+ }+ if got.PromptEvalCount != want.PromptEvalCount || got.PromptEvalCachedCount == nil || *got.PromptEvalCachedCount != *want.PromptEvalCachedCount {+ t.Errorf("prompt counts = (%d, %v), want (%d, %d)", got.PromptEvalCount, got.PromptEvalCachedCount, want.PromptEvalCount, *want.PromptEvalCachedCount)+ }+}diff --git a/x/mlxrunner/mtp_test.go b/x/mlxrunner/mtp_test.goindex d3106e275..68eebc1dc 100644--- a/x/mlxrunner/mtp_test.go+++ b/x/mlxrunner/mtp_test.go@@ -9,6 +9,7 @@ import ("slices""strings""testing"+ "time""github.com/ollama/ollama/api""github.com/ollama/ollama/x/internal/mlxtest"@@ -438,6 +439,55 @@ func TestRunMTPDecodeGreedy(t *testing.T) {})}+func TestDecodeIntermediateMetrics(t *testing.T) {+ skipIfNoMLX(t)+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ caches, _ := newMTPTestCaches(1)+ session, ch := newMTPTestSession(caches)+ session.inputs = []int32{0, 1}+ session.remaining = []int32{1}+ req := Request{+ Responses: ch,+ Tokens: session.inputs,+ CompletionRequest: CompletionRequest{+ Options: api.Options{NumPredict: 20},+ IncludeIntermediateMetrics: true,+ },+ }+ d := testDecoder(t, r, req, caches, []int32{1}, 1)+ promptEval := 5 * time.Millisecond+ if err := r.decode(context.Background(), req, session, d, promptEval); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()++ var responses []CompletionResponse+ for len(ch) > 0 {+ resp := <-ch+ responses = append(responses, resp)+ }+ if len(responses) != 3 {+ t.Fatalf("got %d responses, want 3", len(responses))+ }+ for i, resp := range responses[:2] {+ if resp.PromptEvalCount != 2 || resp.PromptEvalCachedCount == nil || *resp.PromptEvalCachedCount != 1 || resp.PromptEvalDuration != promptEval {+ t.Errorf("response[%d] prompt metrics = (%d, %v, %s), want (2, 1, %s)", i, resp.PromptEvalCount, resp.PromptEvalCachedCount, resp.PromptEvalDuration, promptEval)+ }+ if resp.EvalCount != i+1 || resp.EvalDuration <= 0 {+ t.Errorf("response[%d] eval metrics = (%d, %s), want count %d and positive duration", i, resp.EvalCount, resp.EvalDuration, i+1)+ }+ }+ final := responses[2]+ if final.PromptEvalCount != 2 || final.PromptEvalCachedCount == nil || *final.PromptEvalCachedCount != 1 || final.PromptEvalDuration != promptEval {+ t.Errorf("final prompt metrics = (%d, %v, %s), want (2, 1, %s)", final.PromptEvalCount, final.PromptEvalCachedCount, final.PromptEvalDuration, promptEval)+ }+ if final.EvalCount != 2 || final.EvalDuration <= 0 {+ t.Errorf("final eval metrics = (%d, %s), want count 2 and positive duration", final.EvalCount, final.EvalDuration)+ }+}+func TestRunMTPDecodeSampled(t *testing.T) {mlxtest.Run(t, func(t *mlxtest.T) {// The same chain at temperature 1: because oneHotLogits uses a large gap,diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex ec2844aff..bd260cb44 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -269,7 +269,13 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSesswantTopLogprobs: request.SamplerOpts.TopLogprobs,}- final := CompletionResponse{Done: true, PromptEvalCount: len(request.Tokens), DoneReason: 1}+ cachedPromptCount := len(session.inputs) - len(session.remaining)+ final := CompletionResponse{+ Done: true,+ DoneReason: 1,+ PromptEvalCount: len(request.Tokens),+ PromptEvalCachedCount: &cachedPromptCount,+ }final.PromptEvalDuration = promptEvalnow := time.Now()@@ -317,6 +323,14 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSessif !ok {continue}+ // Two-pass structured output cancels the first pass before its final response.+ if request.IncludeIntermediateMetrics {+ resp.PromptEvalCount = len(request.Tokens)+ resp.PromptEvalCachedCount = final.PromptEvalCachedCount+ resp.PromptEvalDuration = promptEval+ resp.EvalCount = generated+ resp.EvalDuration = time.Since(now)+ }select {case <-ctx.Done():return ctx.Err()