Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Fix #2289: Anthropic usage reports cache_creation_input_tokens, the prompt tokens read now and kept for the next request (from PR #2290)

xrip7adced3db50a
diff --git a/serve/server.py b/serve/server.py
index 300238b71..807ea898b 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -5611,14 +5611,14 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, can
stop = "stop_sequence" if x.get("stop_sequence") is not None else \
"tool_use" if used_tool and streamed <= finished and x["finish"] == "stop" else \
{"stop": "end_turn", "length": "max_tokens", "cancel": "end_turn"}[x["finish"]]
- # the final counts, Anthropic's way: input_tokens leaves out what the conversation cache already held,
- # which is cache_read_input_tokens (message_start could only say the whole prompt). At least the last
- # prompt token is always read, and Claude Code takes an input_tokens of 0 as "not given" and keeps
- # message_start's whole prompt beside cache_read_input_tokens: the prompt counted twice (#1013)
+ # the final counts (message_start could only say the whole prompt): what was read now stays for the next
+ # request, a cache write; input_tokens keeps the last token, as 0 makes Claude Code count the prompt twice (#1013)
reused = max(0, min(x.get("reused") or 0, len(ids) - 1))
+ read_now = len(ids) - reused
yield "message_delta", {"type": "message_delta", "delta": {"stop_reason": stop,
"stop_sequence": x.get("stop_sequence")},
- "usage": {"input_tokens": len(ids) - reused, "cache_read_input_tokens": reused,
+ "usage": {"input_tokens": 1, "cache_creation_input_tokens": read_now - 1,
+ "cache_read_input_tokens": reused,
"output_tokens": x["completion_tokens"]}}
yield "message_stop", {"type": "message_stop"}
diff --git a/serve/test_parallel_yield.py b/serve/test_parallel_yield.py
index b807ee7cb..eeabe383a 100644
--- a/serve/test_parallel_yield.py
+++ b/serve/test_parallel_yield.py
@@ -240,7 +240,7 @@ class ParallelServiceYield(tp.FakeEngineCase):
prompt = self.svc.history[[h["output_tokens"] for h in self.svc.history].index(
max(h["output_tokens"] for h in self.svc.history))]["prompt_total"]
self.assertGreaterEqual(u["input_tokens"], 1, u)
- self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], prompt, u)
+ self.assertEqual(u["input_tokens"] + u["cache_creation_input_tokens"] + u["cache_read_input_tokens"], prompt, u)
self.assertEqual(u["cache_read_input_tokens"], 3, u) # the first read's reuse, not the continuation's
def test_a_slot_request_ended_by_its_stop_token_reports_the_slot_leg(self):
diff --git a/serve/test_request_accounting.py b/serve/test_request_accounting.py
index 145fdd560..55ff05408 100644
--- a/serve/test_request_accounting.py
+++ b/serve/test_request_accounting.py
@@ -148,14 +148,14 @@ class RequestAccounting(unittest.TestCase):
usage = self.chat("/v1/messages")["usage"]
self.assertEqual(self.engine.attempts, 2)
self.assertEqual(usage["cache_read_input_tokens"], 5)
- self.assertEqual(usage["input_tokens"], len(self.engine.prompts[0]) - 5)
+ self.assertEqual(usage["input_tokens"] + usage["cache_creation_input_tokens"], len(self.engine.prompts[0]) - 5)
def test_anthropic_stream_classifies_only_original_input_as_cached(self):
events = self.chat("/v1/messages", stream=True)
usage = next(event["usage"] for event in events if event["type"] == "message_delta")
self.assertEqual(self.engine.attempts, 2)
self.assertEqual(usage["cache_read_input_tokens"], 5)
- self.assertEqual(usage["input_tokens"], len(self.engine.prompts[0]) - 5)
+ self.assertEqual(usage["input_tokens"] + usage["cache_creation_input_tokens"], len(self.engine.prompts[0]) - 5)
def test_single_segment_keeps_its_own_usage_and_timings(self):
response = self.chat(budget=10000)
diff --git a/serve/test_server.py b/serve/test_server.py
index 2d4bd85a0..8bbe6e65b 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -74,7 +74,8 @@ class MaxTokens(unittest.TestCase):
return s, b, u.get("prompt_tokens"), u.get("completion_tokens")
s, b = self.post("/v1/messages", {"model": "m", "messages": msgs, **budget})
u = b.get("usage", {})
- return s, b, u.get("input_tokens"), u.get("output_tokens")
+ prompt = sum(u.get(k) or 0 for k in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))
+ return s, b, prompt if u else None, u.get("output_tokens")
def test_anthropic_thinks_only_when_asked(self):
# #278: Anthropic's thinking is opt-in; "thinking", an effort or a reasoning_budget_tokens (#123) asks for it
@@ -3712,7 +3713,9 @@ class UsageAndStatus(unittest.TestCase):
def test_anthropic(self):
u = self.chat("/v1/messages")["usage"]
self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)
- self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], len(self.engine.last_prompt))
+ self.assertEqual(u["input_tokens"], 1) # what was read now is kept: written to the cache
+ self.assertEqual(u["input_tokens"] + u["cache_creation_input_tokens"] + u["cache_read_input_tokens"],
+ len(self.engine.last_prompt))
self.assertGreater(u["output_tokens"], 0)
def test_v1_status(self):