Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Monitor/serve: a one-token reply has no decode span, so no tok/s is shown for it (a 1-token probe read 1,250 tok/s); test

Niko12215d5c26d5bc08
diff --git a/serve/server.py b/serve/server.py
index 3ed46d37b..76b5a2c79 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -5516,7 +5516,7 @@ class Service:
"engine_generated": last.get("generated"),
"prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
"decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
- if n and last.get("generated") and last.get("decode_ms") else None,
+ if n and (last.get("generated") or 0) >= 2 and last.get("decode_ms") else None,
"hit_rate": hit_rate, "pcie_share": pcie_share, "ram_blobs": last.get("ram_blobs"),
"file_blobs": last.get("file_blobs"), "file_mb": last.get("file_mb"),
# #457: the speculative drafts from the DONE line (None: the engine did not say)
@@ -5650,7 +5650,7 @@ def answer_speed(last: dict, generated: int, wall_s: float | None = None) -> flo
Only an engine that sent no DONE falls back to the server's own clock from the first token, and a clock younger than
RATE_MIN_SPAN_S says nothing (it printed 1000000 tok/s for a one-token reply). Prompt reading is never in it."""
produced, decode_ms = last.get("generated"), last.get("decode_ms")
- if produced and decode_ms and decode_ms > 0:
+ if produced and produced >= 2 and decode_ms and decode_ms > 0: # one token has no first-to-last span (a 1-token probe printed 1,250 tok/s)
return round(produced / (decode_ms / 1000), 1)
if generated and wall_s is not None and wall_s >= RATE_MIN_SPAN_S:
return round(generated / wall_s, 1)
diff --git a/serve/test_answer_speed.py b/serve/test_answer_speed.py
index 96548681b..325472593 100644
--- a/serve/test_answer_speed.py
+++ b/serve/test_answer_speed.py
@@ -28,6 +28,7 @@ class AnswerSpeed(unittest.TestCase):
self.assertEqual(answer_speed({**self.REAL, "drafts_offered": 500}, 73), 65.9)
def test_a_one_token_reply_has_no_absurd_rate(self):
+ self.assertIsNone(answer_speed({"generated": 1, "decode_ms": 0.8}, 1, None)) # a 1-token probe: 1 / 0.8 ms was 1,250 tok/s
self.assertIsNone(answer_speed({}, 1, 1e-6))
self.assertIsNone(answer_speed({}, 1, None))
diff --git a/strata_cli/overview.py b/strata_cli/overview.py
index d9dabfa35..76125aeca 100644
--- a/strata_cli/overview.py
+++ b/strata_cli/overview.py
@@ -58,7 +58,7 @@ def request_speed(r: dict) -> float | None:
out, ms = _num(r.get("output_tokens")), _num(r.get("decode_ms"))
if r.get("decode_tok_s") is not None and _num(r.get("decode_tok_s")) is not None:
return _num(r.get("decode_tok_s"))
- return round(out / (ms / 1000.0), 1) if out and ms and ms > 0 else None
+ return round(out / (ms / 1000.0), 1) if out and out >= 2 and ms and ms > 0 else None # one token has no decode span (a probe showed 1,250 tok/s)
def _cards(hw: dict, static: dict) -> list[dict]: