FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(stats): record per-request TTFT and report its mean
jason-fxz12ed6953867d
diff --git a/python/freetoken/control_cli.py b/python/freetoken/control_cli.pyindex d92dd42c..50a32aae 100644--- a/python/freetoken/control_cli.py+++ b/python/freetoken/control_cli.py@@ -212,7 +212,8 @@ def _format_stats(doc: dict[str, Any]) -> str:(f"requests active={requests.get('active', 0)} "f"completed={requests.get('completed', 0)} "- f"p95_ms={requests.get('p95_ms', 0)}"+ f"p95_ms={requests.get('p95_ms', 0)} "+ f"ttft_mean_ms={requests.get('ttft_mean_ms', 0)}"),f"vram_bytes={doc.get('vram_bytes', 0)}",]diff --git a/python/freetoken/server/api_server.py b/python/freetoken/server/api_server.pyindex 85c94037..80dbf6ab 100644--- a/python/freetoken/server/api_server.py+++ b/python/freetoken/server/api_server.py@@ -465,6 +465,7 @@ async def _record_request_middleware(request: Request, call_next):status=response.status_code,model=_served_model_name(),duration_ms=duration_ms,+ ttft_ms=None,prompt_tokens=None,completion_tokens=None,stream=ctype.startswith("text/event-stream"),diff --git a/python/freetoken/server/control_api.py b/python/freetoken/server/control_api.pyindex 4424f49f..7158e4e5 100644--- a/python/freetoken/server/control_api.py+++ b/python/freetoken/server/control_api.py@@ -70,7 +70,9 @@ def register_control_routes(@app.get("/v1/stats")async def stats():- doc = build_stats(get_state(), request_ring.requests_p95_ms())+ doc = build_stats(+ get_state(), request_ring.requests_p95_ms(), request_ring.requests_ttft_mean_ms()+ )# Surface the model's recommended sampling (from its generation_config.json / GGUF# metadata) so clients can seed their sampling controls per-model instead of guessing.if get_model_sampling is not None:diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex d6689680..be05d908 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -463,6 +463,7 @@ def _record_generation(prompt_tokens: int,completion_tokens: int,error: str | None,+ first_token_at: float | None = None,) -> None:"""Log one generation request into the request ring. Every protocol adapter converges here,so token totals are captured whatever endpoint served the request — unlike the HTTP@@ -480,6 +481,7 @@ def _record_generation(status=500 if error else 200,model=_served_model_name(),duration_ms=int((time.monotonic() - start) * 1000),+ ttft_ms=int((first_token_at - start) * 1000) if first_token_at is not None else None,prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,stream=stream,@@ -497,12 +499,15 @@ async def generate_events(start = time.monotonic()prompt_tokens = 0completion_tokens = 0+ first_token_at: float | None = Noneerror: str | None = Nonetry:async for ev in _generate_events_impl(uid, spec, state):if isinstance(ev, GenDone):prompt_tokens = ev.prompt_tokenscompletion_tokens = ev.completion_tokens+ elif first_token_at is None:+ first_token_at = time.monotonic()yield evexcept GenerationError as exc:error = str(exc)@@ -511,6 +516,7 @@ async def generate_events(_record_generation(source=source, stream=True, start=start,prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, error=error,+ first_token_at=first_token_at,)diff --git a/python/freetoken/server/request_ring.py b/python/freetoken/server/request_ring.pyindex b591d558..bf210c46 100644--- a/python/freetoken/server/request_ring.py+++ b/python/freetoken/server/request_ring.py@@ -19,6 +19,7 @@ class RequestRecord:status: intmodel: str | Noneduration_ms: int+ ttft_ms: int | Noneprompt_tokens: int | Nonecompletion_tokens: int | Nonestream: bool | None@@ -59,6 +60,13 @@ class RequestRing:k = max(0, math.ceil(0.95 * len(durs)) - 1)return int(durs[k])+ def ttft_mean_ms(self) -> int:+ """Mean TTFT over the records that have one."""+ vals = [rec.ttft_ms for _idx, rec in self._buf if rec.ttft_ms is not None]+ if not vals:+ return 0+ return int(round(sum(vals) / len(vals)))+def count(self) -> int:return self._next@@ -79,6 +87,10 @@ def requests_p95_ms() -> int:return _RING.p95_ms()+def requests_ttft_mean_ms() -> int:+ return _RING.ttft_mean_ms()++def requests_count() -> int:return _RING.count()diff --git a/python/freetoken/server/stats.py b/python/freetoken/server/stats.pyindex 28082674..e89d23a4 100644--- a/python/freetoken/server/stats.py+++ b/python/freetoken/server/stats.py@@ -127,7 +127,7 @@ def _swa_page_size(config: Any) -> int:return 1-def build_stats(state: Any, p95_ms: int) -> dict:+def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:"""Full /v1/stats doc. throughput is 0 when idle; kv/mamba/swa are nullwhen their total is 0 (owned-KV / non-hybrid / non-SWA). kv and swa share one shape:pages + the pool's own page_size (tokens = pages x page_size)."""@@ -166,6 +166,7 @@ def build_stats(state: Any, p95_ms: int) -> dict:"active": tr.active,"completed": tr.completed,"p95_ms": p95_ms,+ "ttft_mean_ms": ttft_mean_ms,"prompt_tokens_total": tr.prompt_tokens_total,"completion_tokens_total": tr.completion_tokens_total,},diff --git a/tests/server/test_generation_accounting.py b/tests/server/test_generation_accounting.pyindex 331ec5e9..204c1f8c 100644--- a/tests/server/test_generation_accounting.py+++ b/tests/server/test_generation_accounting.py@@ -4,13 +4,27 @@ stream (it fires before the totals are known)."""from __future__ import annotationsimport asyncio+import os+import sysfrom types import SimpleNamespaceimport pytest-from freetoken.message import UserReply-from freetoken.server import api_server, request_ring-from freetoken.server.generation import GenDone, GenSpec, generate_events, generate_full+# Same shim as the sibling server tests: the venv may hold a non-editable install, and without+# this the file only tests the source tree when a test that does insert it collects first.+_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+_PY = os.path.join(_ROOT, "python")+if _PY not in sys.path:+ sys.path.insert(0, _PY)++from freetoken.message import UserReply # noqa: E402+from freetoken.server import api_server, request_ring # noqa: E402+from freetoken.server.generation import ( # noqa: E402+ GenDone,+ GenSpec,+ generate_events,+ generate_full,+)@pytest.fixture(autouse=True)@@ -63,6 +77,14 @@ def _spec() -> GenSpec:return GenSpec(messages=[{"role": "user", "content": "hi"}], sampling_params=SimpleNamespace())+def _row(*, ttft_ms: int | None) -> request_ring.RequestRecord:+ return request_ring.RequestRecord(+ ts="2026-01-01T00:00:00Z", method="POST", path="/v1/messages", status=200,+ model="unit-model", duration_ms=1000, ttft_ms=ttft_ms, prompt_tokens=1,+ completion_tokens=1, stream=True, error=None,+ )++def _last_row() -> dict:rows, _ = request_ring.requests_since(0, 1000)return rows[-1]@@ -133,3 +155,40 @@ def test_no_source_opts_out_of_recording():asyncio.run(generate_full(42, _spec(), st)) # no sourcerows, _ = request_ring.requests_since(0, 1000)assert rows == []+++def test_stream_records_a_ttft_within_the_request_duration():+ request_ring.reset()+ st = FakeState([_ack(prompt=3, completion=1, out="a"), _ack(completion=1, out="b", finished=True)])++ async def drain():+ async for _ev in generate_events(42, _spec(), st, source="/v1/messages"):+ pass++ asyncio.run(drain())+ row = _last_row()+ assert row["ttft_ms"] is not None and 0 <= row["ttft_ms"] <= row["duration_ms"]+++def test_non_stream_records_no_ttft():+ """generate_full hands the client one response: there is no first-token instant to observe."""+ request_ring.reset()+ st = FakeState([_ack(prompt=3, completion=2, out="ab", finished=True)])+ asyncio.run(generate_full(42, _spec(), st, source="/v1/chat/completions"))+ assert _last_row()["ttft_ms"] is None+++def test_ttft_mean_covers_only_the_rows_that_have_one():+ """Non-streaming generations and middleware-logged rows carry no TTFT; averaging them in+ as zeros would drag the mean toward 0 as soon as anything hits /health."""+ request_ring.reset()+ for ms in (100, 200, 300):+ request_ring.record_request(_row(ttft_ms=ms))+ request_ring.record_request(_row(ttft_ms=None))+ assert request_ring.requests_ttft_mean_ms() == 200+++def test_ttft_mean_is_zero_without_samples():+ request_ring.reset()+ request_ring.record_request(_row(ttft_ms=None))+ assert request_ring.requests_ttft_mean_ms() == 0