Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
1.8 KB
Raw Blame
"""The one definition of answer speed in the server: produced tokens over the engine's decode time, never over a time that
holds the prompt being read (no GPU or model pack).

    python -m unittest serve.test_answer_speed -v
"""
import unittest

from serve.server import answer_speed, request_timings


class AnswerSpeed(unittest.TestCase):
    # A real request from the owner PC (Gyro-S, MTP): 308 prompt tokens read in 1212 ms, 73 generated in 1108 ms.
    REAL = {"generated": 73, "decode_ms": 1108.0, "prompt_ms": 1212.0, "drafts_accepted": 45, "drafts_offered": 68}

    def test_the_engines_decode_time_is_the_denominator(self):
        self.assertEqual(answer_speed(self.REAL, 73, 2.3), 65.9)

    def test_it_matches_the_timings_and_metrics_figure(self):
        self.assertEqual(answer_speed(self.REAL, 73), request_timings(308, 73, self.REAL)["predicted_per_second"])

    def test_prompt_reading_time_is_not_in_it(self):
        wall = (1212.0 + 1108.0) / 1000
        self.assertLess(round(73 / wall, 1), 40)          # the number a wall-clock division would show
        self.assertEqual(answer_speed(self.REAL, 73, wall), 65.9)

    def test_drafts_are_not_counted(self):
        # 68 drafted, 45 accepted: the engine's count (73) is what the model kept; the speed never uses the draft figures.
        self.assertEqual(answer_speed({**self.REAL, "drafts_offered": 500}, 73), 65.9)

    def test_a_one_token_reply_has_no_absurd_rate(self):
        self.assertIsNone(answer_speed({"generated": 1, "decode_ms": 0.8}, 1, None))     # a 1-token probe: 1 / 0.8 ms was 1,250 tok/s
        self.assertIsNone(answer_speed({}, 1, 1e-6))
        self.assertIsNone(answer_speed({}, 1, None))

    def test_without_a_done_line_the_server_clock_from_the_first_token_is_used(self):
        self.assertEqual(answer_speed({}, 50, 2.0), 25.0)


if __name__ == "__main__":
    unittest.main()