Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Fix #2289: Anthropic usage reports cache_creation_input_tokens, the prompt tokens read now and kept for the next request (from PR #2290)
xrip7adced3db50a
diff --git a/serve/server.py b/serve/server.pyindex 300238b71..807ea898b 100644--- a/serve/server.py+++ b/serve/server.py@@ -5611,14 +5611,14 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, canstop = "stop_sequence" if x.get("stop_sequence") is not None else \"tool_use" if used_tool and streamed <= finished and x["finish"] == "stop" else \{"stop": "end_turn", "length": "max_tokens", "cancel": "end_turn"}[x["finish"]]- # the final counts, Anthropic's way: input_tokens leaves out what the conversation cache already held,- # which is cache_read_input_tokens (message_start could only say the whole prompt). At least the last- # prompt token is always read, and Claude Code takes an input_tokens of 0 as "not given" and keeps- # message_start's whole prompt beside cache_read_input_tokens: the prompt counted twice (#1013)+ # the final counts (message_start could only say the whole prompt): what was read now stays for the next+ # request, a cache write; input_tokens keeps the last token, as 0 makes Claude Code count the prompt twice (#1013)reused = max(0, min(x.get("reused") or 0, len(ids) - 1))+ read_now = len(ids) - reusedyield "message_delta", {"type": "message_delta", "delta": {"stop_reason": stop,"stop_sequence": x.get("stop_sequence")},- "usage": {"input_tokens": len(ids) - reused, "cache_read_input_tokens": reused,+ "usage": {"input_tokens": 1, "cache_creation_input_tokens": read_now - 1,+ "cache_read_input_tokens": reused,"output_tokens": x["completion_tokens"]}}yield "message_stop", {"type": "message_stop"}diff --git a/serve/test_parallel_yield.py b/serve/test_parallel_yield.pyindex b807ee7cb..eeabe383a 100644--- a/serve/test_parallel_yield.py+++ b/serve/test_parallel_yield.py@@ -240,7 +240,7 @@ class ParallelServiceYield(tp.FakeEngineCase):prompt = self.svc.history[[h["output_tokens"] for h in self.svc.history].index(max(h["output_tokens"] for h in self.svc.history))]["prompt_total"]self.assertGreaterEqual(u["input_tokens"], 1, u)- self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], prompt, u)+ self.assertEqual(u["input_tokens"] + u["cache_creation_input_tokens"] + u["cache_read_input_tokens"], prompt, u)self.assertEqual(u["cache_read_input_tokens"], 3, u) # the first read's reuse, not the continuation'sdef test_a_slot_request_ended_by_its_stop_token_reports_the_slot_leg(self):diff --git a/serve/test_request_accounting.py b/serve/test_request_accounting.pyindex 145fdd560..55ff05408 100644--- a/serve/test_request_accounting.py+++ b/serve/test_request_accounting.py@@ -148,14 +148,14 @@ class RequestAccounting(unittest.TestCase):usage = self.chat("/v1/messages")["usage"]self.assertEqual(self.engine.attempts, 2)self.assertEqual(usage["cache_read_input_tokens"], 5)- self.assertEqual(usage["input_tokens"], len(self.engine.prompts[0]) - 5)+ self.assertEqual(usage["input_tokens"] + usage["cache_creation_input_tokens"], len(self.engine.prompts[0]) - 5)def test_anthropic_stream_classifies_only_original_input_as_cached(self):events = self.chat("/v1/messages", stream=True)usage = next(event["usage"] for event in events if event["type"] == "message_delta")self.assertEqual(self.engine.attempts, 2)self.assertEqual(usage["cache_read_input_tokens"], 5)- self.assertEqual(usage["input_tokens"], len(self.engine.prompts[0]) - 5)+ self.assertEqual(usage["input_tokens"] + usage["cache_creation_input_tokens"], len(self.engine.prompts[0]) - 5)def test_single_segment_keeps_its_own_usage_and_timings(self):response = self.chat(budget=10000)diff --git a/serve/test_server.py b/serve/test_server.pyindex 2d4bd85a0..8bbe6e65b 100644--- a/serve/test_server.py+++ b/serve/test_server.py@@ -74,7 +74,8 @@ class MaxTokens(unittest.TestCase):return s, b, u.get("prompt_tokens"), u.get("completion_tokens")s, b = self.post("/v1/messages", {"model": "m", "messages": msgs, **budget})u = b.get("usage", {})- return s, b, u.get("input_tokens"), u.get("output_tokens")+ prompt = sum(u.get(k) or 0 for k in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))+ return s, b, prompt if u else None, u.get("output_tokens")def test_anthropic_thinks_only_when_asked(self):# #278: Anthropic's thinking is opt-in; "thinking", an effort or a reasoning_budget_tokens (#123) asks for it@@ -3712,7 +3713,9 @@ class UsageAndStatus(unittest.TestCase):def test_anthropic(self):u = self.chat("/v1/messages")["usage"]self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)- self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], len(self.engine.last_prompt))+ self.assertEqual(u["input_tokens"], 1) # what was read now is kept: written to the cache+ self.assertEqual(u["input_tokens"] + u["cache_creation_input_tokens"] + u["cache_read_input_tokens"],+ len(self.engine.last_prompt))self.assertGreater(u["output_tokens"], 0)def test_v1_status(self):