FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(server): report the backend kv page size in stats (#599)
云微581fdca310d4
diff --git a/python/freetoken/server/stats.py b/python/freetoken/server/stats.pyindex 1857d3d7..d01828e2 100644--- a/python/freetoken/server/stats.py+++ b/python/freetoken/server/stats.py@@ -138,9 +138,10 @@ def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:config = state.configready_at = getattr(state, "ready_at", None)uptime_s = max(0, int(time.monotonic() - ready_at)) if ready_at is not None else 0+ pools = getattr(state, "cache_pools", None) or {}kv = ({"used_pages": tr.kv_used_pages, "total_pages": tr.kv_total_pages,- "page_size": getattr(config, "page_size", 1)}+ "page_size": pools.get("page_size") or getattr(config, "page_size", 1)}if tr.kv_total_pages > 0 else None)mamba = (diff --git a/tests/server/test_generation_accounting.py b/tests/server/test_generation_accounting.pyindex 8276163b..a6d2bdb9 100644--- a/tests/server/test_generation_accounting.py+++ b/tests/server/test_generation_accounting.py@@ -91,6 +91,19 @@ def _last_row() -> dict:return rows[-1][email protected]("pools, expected", [(None, 1), ({"page_size": 64}, 64)])+def test_stats_reports_backend_page_size(pools, expected):+ from freetoken.server.stats import StatsTracker, build_stats++ tracker = StatsTracker()+ tracker.kv_total_pages = 130+ config = SimpleNamespace(served_model_name="unit-model", max_seq_len=262144,+ page_size=1, served_modalities=(), model_config=SimpleNamespace())+ state = SimpleNamespace(stats=tracker, config=config, cache_pools=pools)+ doc = build_stats(state, 0, 0)+ assert doc["kv"] == {"used_pages": 0, "total_pages": 130, "page_size": expected}++def test_non_stream_records_the_request_with_real_token_totals():request_ring.reset()st = FakeState([_ack(prompt=5, completion=1, out="a"), _ack(completion=2, out="bc", finished=True)])