Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(server): report the backend kv page size in stats (#599)

云微581fdca310d4
diff --git a/python/freetoken/server/stats.py b/python/freetoken/server/stats.py
index 1857d3d7..d01828e2 100644
--- a/python/freetoken/server/stats.py
+++ b/python/freetoken/server/stats.py
@@ -138,9 +138,10 @@ def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:
config = state.config
ready_at = getattr(state, "ready_at", None)
uptime_s = max(0, int(time.monotonic() - ready_at)) if ready_at is not None else 0
+ pools = getattr(state, "cache_pools", None) or {}
kv = (
{"used_pages": tr.kv_used_pages, "total_pages": tr.kv_total_pages,
- "page_size": getattr(config, "page_size", 1)}
+ "page_size": pools.get("page_size") or getattr(config, "page_size", 1)}
if tr.kv_total_pages > 0 else None
)
mamba = (
diff --git a/tests/server/test_generation_accounting.py b/tests/server/test_generation_accounting.py
index 8276163b..a6d2bdb9 100644
--- a/tests/server/test_generation_accounting.py
+++ b/tests/server/test_generation_accounting.py
@@ -91,6 +91,19 @@ def _last_row() -> dict:
return rows[-1]
[email protected]("pools, expected", [(None, 1), ({"page_size": 64}, 64)])
+def test_stats_reports_backend_page_size(pools, expected):
+ from freetoken.server.stats import StatsTracker, build_stats
+
+ tracker = StatsTracker()
+ tracker.kv_total_pages = 130
+ config = SimpleNamespace(served_model_name="unit-model", max_seq_len=262144,
+ page_size=1, served_modalities=(), model_config=SimpleNamespace())
+ state = SimpleNamespace(stats=tracker, config=config, cache_pools=pools)
+ doc = build_stats(state, 0, 0)
+ assert doc["kv"] == {"used_pages": 0, "total_pages": 130, "page_size": expected}
+
+
def test_non_stream_records_the_request_with_real_token_totals():
request_ring.reset()
st = FakeState([_ack(prompt=5, completion=1, out="a"), _ack(completion=2, out="bc", finished=True)])