Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(stats): record per-request TTFT and report its mean

jason-fxz12ed6953867d
diff --git a/python/freetoken/control_cli.py b/python/freetoken/control_cli.py
index d92dd42c..50a32aae 100644
--- a/python/freetoken/control_cli.py
+++ b/python/freetoken/control_cli.py
@@ -212,7 +212,8 @@ def _format_stats(doc: dict[str, Any]) -> str:
(
f"requests active={requests.get('active', 0)} "
f"completed={requests.get('completed', 0)} "
- f"p95_ms={requests.get('p95_ms', 0)}"
+ f"p95_ms={requests.get('p95_ms', 0)} "
+ f"ttft_mean_ms={requests.get('ttft_mean_ms', 0)}"
),
f"vram_bytes={doc.get('vram_bytes', 0)}",
]
diff --git a/python/freetoken/server/api_server.py b/python/freetoken/server/api_server.py
index 85c94037..80dbf6ab 100644
--- a/python/freetoken/server/api_server.py
+++ b/python/freetoken/server/api_server.py
@@ -465,6 +465,7 @@ async def _record_request_middleware(request: Request, call_next):
status=response.status_code,
model=_served_model_name(),
duration_ms=duration_ms,
+ ttft_ms=None,
prompt_tokens=None,
completion_tokens=None,
stream=ctype.startswith("text/event-stream"),
diff --git a/python/freetoken/server/control_api.py b/python/freetoken/server/control_api.py
index 4424f49f..7158e4e5 100644
--- a/python/freetoken/server/control_api.py
+++ b/python/freetoken/server/control_api.py
@@ -70,7 +70,9 @@ def register_control_routes(
@app.get("/v1/stats")
async def stats():
- doc = build_stats(get_state(), request_ring.requests_p95_ms())
+ doc = build_stats(
+ get_state(), request_ring.requests_p95_ms(), request_ring.requests_ttft_mean_ms()
+ )
# Surface the model's recommended sampling (from its generation_config.json / GGUF
# metadata) so clients can seed their sampling controls per-model instead of guessing.
if get_model_sampling is not None:
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index d6689680..be05d908 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -463,6 +463,7 @@ def _record_generation(
prompt_tokens: int,
completion_tokens: int,
error: str | None,
+ first_token_at: float | None = None,
) -> None:
"""Log one generation request into the request ring. Every protocol adapter converges here,
so token totals are captured whatever endpoint served the request — unlike the HTTP
@@ -480,6 +481,7 @@ def _record_generation(
status=500 if error else 200,
model=_served_model_name(),
duration_ms=int((time.monotonic() - start) * 1000),
+ ttft_ms=int((first_token_at - start) * 1000) if first_token_at is not None else None,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
stream=stream,
@@ -497,12 +499,15 @@ async def generate_events(
start = time.monotonic()
prompt_tokens = 0
completion_tokens = 0
+ first_token_at: float | None = None
error: str | None = None
try:
async for ev in _generate_events_impl(uid, spec, state):
if isinstance(ev, GenDone):
prompt_tokens = ev.prompt_tokens
completion_tokens = ev.completion_tokens
+ elif first_token_at is None:
+ first_token_at = time.monotonic()
yield ev
except GenerationError as exc:
error = str(exc)
@@ -511,6 +516,7 @@ async def generate_events(
_record_generation(
source=source, stream=True, start=start,
prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, error=error,
+ first_token_at=first_token_at,
)
diff --git a/python/freetoken/server/request_ring.py b/python/freetoken/server/request_ring.py
index b591d558..bf210c46 100644
--- a/python/freetoken/server/request_ring.py
+++ b/python/freetoken/server/request_ring.py
@@ -19,6 +19,7 @@ class RequestRecord:
status: int
model: str | None
duration_ms: int
+ ttft_ms: int | None
prompt_tokens: int | None
completion_tokens: int | None
stream: bool | None
@@ -59,6 +60,13 @@ class RequestRing:
k = max(0, math.ceil(0.95 * len(durs)) - 1)
return int(durs[k])
+ def ttft_mean_ms(self) -> int:
+ """Mean TTFT over the records that have one."""
+ vals = [rec.ttft_ms for _idx, rec in self._buf if rec.ttft_ms is not None]
+ if not vals:
+ return 0
+ return int(round(sum(vals) / len(vals)))
+
def count(self) -> int:
return self._next
@@ -79,6 +87,10 @@ def requests_p95_ms() -> int:
return _RING.p95_ms()
+def requests_ttft_mean_ms() -> int:
+ return _RING.ttft_mean_ms()
+
+
def requests_count() -> int:
return _RING.count()
diff --git a/python/freetoken/server/stats.py b/python/freetoken/server/stats.py
index 28082674..e89d23a4 100644
--- a/python/freetoken/server/stats.py
+++ b/python/freetoken/server/stats.py
@@ -127,7 +127,7 @@ def _swa_page_size(config: Any) -> int:
return 1
-def build_stats(state: Any, p95_ms: int) -> dict:
+def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:
"""Full /v1/stats doc. throughput is 0 when idle; kv/mamba/swa are null
when their total is 0 (owned-KV / non-hybrid / non-SWA). kv and swa share one shape:
pages + the pool's own page_size (tokens = pages x page_size)."""
@@ -166,6 +166,7 @@ def build_stats(state: Any, p95_ms: int) -> dict:
"active": tr.active,
"completed": tr.completed,
"p95_ms": p95_ms,
+ "ttft_mean_ms": ttft_mean_ms,
"prompt_tokens_total": tr.prompt_tokens_total,
"completion_tokens_total": tr.completion_tokens_total,
},
diff --git a/tests/server/test_generation_accounting.py b/tests/server/test_generation_accounting.py
index 331ec5e9..204c1f8c 100644
--- a/tests/server/test_generation_accounting.py
+++ b/tests/server/test_generation_accounting.py
@@ -4,13 +4,27 @@ stream (it fires before the totals are known)."""
from __future__ import annotations
import asyncio
+import os
+import sys
from types import SimpleNamespace
import pytest
-from freetoken.message import UserReply
-from freetoken.server import api_server, request_ring
-from freetoken.server.generation import GenDone, GenSpec, generate_events, generate_full
+# Same shim as the sibling server tests: the venv may hold a non-editable install, and without
+# this the file only tests the source tree when a test that does insert it collects first.
+_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+_PY = os.path.join(_ROOT, "python")
+if _PY not in sys.path:
+ sys.path.insert(0, _PY)
+
+from freetoken.message import UserReply # noqa: E402
+from freetoken.server import api_server, request_ring # noqa: E402
+from freetoken.server.generation import ( # noqa: E402
+ GenDone,
+ GenSpec,
+ generate_events,
+ generate_full,
+)
@pytest.fixture(autouse=True)
@@ -63,6 +77,14 @@ def _spec() -> GenSpec:
return GenSpec(messages=[{"role": "user", "content": "hi"}], sampling_params=SimpleNamespace())
+def _row(*, ttft_ms: int | None) -> request_ring.RequestRecord:
+ return request_ring.RequestRecord(
+ ts="2026-01-01T00:00:00Z", method="POST", path="/v1/messages", status=200,
+ model="unit-model", duration_ms=1000, ttft_ms=ttft_ms, prompt_tokens=1,
+ completion_tokens=1, stream=True, error=None,
+ )
+
+
def _last_row() -> dict:
rows, _ = request_ring.requests_since(0, 1000)
return rows[-1]
@@ -133,3 +155,40 @@ def test_no_source_opts_out_of_recording():
asyncio.run(generate_full(42, _spec(), st)) # no source
rows, _ = request_ring.requests_since(0, 1000)
assert rows == []
+
+
+def test_stream_records_a_ttft_within_the_request_duration():
+ request_ring.reset()
+ st = FakeState([_ack(prompt=3, completion=1, out="a"), _ack(completion=1, out="b", finished=True)])
+
+ async def drain():
+ async for _ev in generate_events(42, _spec(), st, source="/v1/messages"):
+ pass
+
+ asyncio.run(drain())
+ row = _last_row()
+ assert row["ttft_ms"] is not None and 0 <= row["ttft_ms"] <= row["duration_ms"]
+
+
+def test_non_stream_records_no_ttft():
+ """generate_full hands the client one response: there is no first-token instant to observe."""
+ request_ring.reset()
+ st = FakeState([_ack(prompt=3, completion=2, out="ab", finished=True)])
+ asyncio.run(generate_full(42, _spec(), st, source="/v1/chat/completions"))
+ assert _last_row()["ttft_ms"] is None
+
+
+def test_ttft_mean_covers_only_the_rows_that_have_one():
+ """Non-streaming generations and middleware-logged rows carry no TTFT; averaging them in
+ as zeros would drag the mean toward 0 as soon as anything hits /health."""
+ request_ring.reset()
+ for ms in (100, 200, 300):
+ request_ring.record_request(_row(ttft_ms=ms))
+ request_ring.record_request(_row(ttft_ms=None))
+ assert request_ring.requests_ttft_mean_ms() == 200
+
+
+def test_ttft_mean_is_zero_without_samples():
+ request_ring.reset()
+ request_ring.record_request(_row(ttft_ms=None))
+ assert request_ring.requests_ttft_mean_ms() == 0