Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
"""Engine measurement tests.
The parsing is tested against the engine's real line formats, taken from
`src/program/generate.cpp`, and the failure handling is tested with a stub engine that speaks the
same protocol. That combination is deliberate: real hardware needs a model that does not fit this
machine, but the *contract* — what counts as a measurement and what must be rejected — is fully
testable here, and it is the part that decides whether a tuning result can be trusted.
The risk these tests exist for is specific: a tuning harness that scores a failed or incomplete run
as a slow one will happily recommend a configuration that does not work.
"""
from __future__ import annotations
import os
import stat
import sys
import tempfile
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from strata_control import tune_engine as te # noqa: E402
class TestDoneParsing(unittest.TestCase):
"""`DONE` is what the engine reports; the field order is the engine's, not ours."""
def test_a_normal_done_line(self):
# DONE produced prompt_tokens prompt_ms decode_ms finish drafts_acc drafts_off ...
d = te.parse_done("DONE 128 512 210.5 1830.0 stop 40 55 0 0 0 12 3 3.4 12 0")
self.assertEqual(d["produced"], 128)
self.assertEqual(d["prompt_tokens"], 512)
self.assertAlmostEqual(d["prompt_ms"], 210.5)
self.assertAlmostEqual(d["decode_ms"], 1830.0)
self.assertEqual(d["finish"], "stop")
self.assertEqual(d["drafts_accepted"], 40)
def test_the_variable_tail_is_kept_not_required(self):
"""The engine appends lookup-chain counters only when that option is on."""
short = te.parse_done("DONE 8 16 10.0 20.0 stop 1 2")
self.assertEqual(short["tail"], [])
long = te.parse_done("DONE 8 16 10.0 20.0 stop 1 2 3 4 5 6 7 8.0 9 10 11 12 13")
self.assertEqual(long["tail"], ["3", "4", "5", "6", "7", "8.0", "9", "10", "11", "12", "13"])
def test_a_short_or_malformed_line_is_an_error_not_a_zero(self):
for bad in ("DONE 1 2", "DONE a b c d stop", "not a done line", "DONE 1 2 three 4.0 stop"):
with self.assertRaises(te.MeasurementError, msg=bad):
te.parse_done(bad)
def test_unreadable_optional_fields_do_not_fail_the_parse(self):
d = te.parse_done("DONE 8 16 10.0 20.0 stop x y")
self.assertIsNone(d["drafts_accepted"])
self.assertIsNone(d["drafts_offered"])
class TestReadyAndPP(unittest.TestCase):
def test_ready_carries_the_context(self):
self.assertEqual(te.parse_ready("READY 32768 stop"), 32768)
def test_a_ready_without_a_context_is_an_error(self):
with self.assertRaises(te.MeasurementError):
te.parse_ready("READY")
def test_pp_with_and_without_the_rate(self):
a = te.parse_pp("PP 1024 4096 800.0 1280.0")
self.assertEqual(a["processed"], 1024)
self.assertEqual(a["tok_s"], 1280.0)
b = te.parse_pp("PP 1024 4096 800.0")
self.assertNotIn("tok_s", b)
class TestMeasurements(unittest.TestCase):
"""The numbers, and the cases that must never produce one."""
def done(self, **kw):
base = {"produced": 100, "prompt_tokens": 512, "prompt_ms": 200.0, "decode_ms": 1000.0,
"finish": "stop"}
base.update(kw)
return base
def test_rates_come_from_the_engines_own_timings(self):
m = te.measurement_from_protocol(self.done(), tokens_seen=100)
self.assertAlmostEqual(m.decode_tokens_per_s, 100.0) # 100 tokens / 1.0 s
self.assertAlmostEqual(m.prompt_tokens_per_s, 2560.0) # 512 / 0.2 s
self.assertAlmostEqual(m.tokens_per_s, 100 * 1000 / 1200) # whole request
self.assertEqual(m.finish, "stop")
def test_ttft_is_the_prompt_plus_one_decode_step(self):
m = te.measurement_from_protocol(self.done(), tokens_seen=100)
# 200 ms prompt + (1000 ms / 100 tokens) = 210 ms
self.assertAlmostEqual(m.ttft_ms, 210.0)
def test_client_ttft_is_recorded_separately_and_labelled(self):
m = te.measurement_from_protocol(self.done(), tokens_seen=100, client_ttft_ms=375.0)
self.assertAlmostEqual(m.ttft_ms, 210.0)
self.assertAlmostEqual(m.client_ttft_ms, 375.0)
def test_a_cancelled_run_is_not_a_measurement(self):
with self.assertRaises(te.MeasurementError):
te.measurement_from_protocol(self.done(finish="cancel"), tokens_seen=100)
def test_a_nonfinite_run_is_not_a_measurement(self):
with self.assertRaises(te.MeasurementError):
te.measurement_from_protocol(self.done(finish="nonfinite"), tokens_seen=100)
def test_zero_tokens_is_not_a_measurement_even_with_a_stop_finish(self):
with self.assertRaises(te.MeasurementError) as ctx:
te.measurement_from_protocol(self.done(produced=0), tokens_seen=0)
self.assertIn("no tokens", str(ctx.exception))
def test_a_token_count_that_disagrees_with_the_stream_is_refused(self):
"""If the stream and the summary disagree, neither can be trusted for a rate."""
with self.assertRaises(te.MeasurementError) as ctx:
te.measurement_from_protocol(self.done(produced=100), tokens_seen=42)
self.assertIn("refusing to score", str(ctx.exception))
def test_a_length_finish_is_a_legitimate_measurement(self):
"""Hitting the output cap is a real measurement of the cap, not a failure."""
m = te.measurement_from_protocol(self.done(finish="length"), tokens_seen=100)
self.assertEqual(m.finish, "length")
def test_missing_metrics_stay_missing_rather_than_becoming_zero(self):
m = te.measurement_from_protocol(
{"produced": 10, "prompt_tokens": 0, "prompt_ms": 0.0, "decode_ms": 500.0,
"finish": "stop"}, tokens_seen=10)
metrics = m.to_metrics()
self.assertIn("tokens_per_s", metrics)
self.assertNotIn("prompt_tokens_per_s", metrics,
"an unmeasurable prompt rate must be absent, not zero")
STUB_PRELUDE = (
"import sys\n"
"def say(text):\n print(text, flush=True)\n"
"def wait():\n sys.stdin.readline()\n")
class TestEngineHarnessWithAStub(unittest.TestCase):
"""The harness against a stub that speaks the real protocol.
This cannot prove the engine measures what we think. It proves the harness's own contract: a
boot failure, an early exit and an error line all become errors, and a good response becomes a
measurement with the engine's numbers.
"""
def setUp(self):
self.tmp = Path(tempfile.mkdtemp(prefix="strata-tune-engine-"))
def tearDown(self):
import shutil
shutil.rmtree(self.tmp, ignore_errors=True)
def stub(self, body: str) -> Path:
"""Write a stub engine that speaks the serve protocol.
The stub is a Python script (the harness starts it as ``[sys.executable, stub.py]``, so it
runs the same on every OS; a ``.sh`` stub is WinError 193 on Windows). ``body`` is Python
using ``say(text)`` (print a line and flush) and ``wait()`` (block until the harness sends
a line). Each body is explicit about how it blocks: ``wait()`` keeps the process, and so the
stdout pipe, alive without leaving a child behind.
"""
p = self.tmp / "stub-engine.py"
p.write_text(STUB_PRELUDE + body)
return p
def test_a_good_response_becomes_a_measurement(self):
stub = self.stub(
'say("READY 4096 stop")\n'
'wait()\n'
'say("PP 512 512 200.0 2560.0")\n'
'say("T 1"); say("T 2"); say("T 3")\n'
'say("DONE 3 512 200.0 30.0 stop 0 0")\n'
'wait()\n')
h = te.EngineHarness(stub, {}, workload={"prompt_tokens": 8, "max_new": 3})
try:
h.start()
self.assertEqual(h._context, 4096)
m = h.measure()
self.assertEqual(m.produced, 3)
self.assertAlmostEqual(m.decode_tokens_per_s, 100.0) # 3 / 0.030 s
self.assertEqual(m.finish, "stop")
finally:
h.close()
def test_an_engine_that_never_reports_ready_times_out(self):
# The stub must block *in its own process* with stdout open. A shell `sleep` child would
# hold the pipe open after the shell exits, so the harness would keep waiting on a process
# that is already gone -- which made this test hang rather than fail.
stub = self.stub('wait()\n')
h = te.EngineHarness(stub, {}, boot_timeout_s=1.0)
try:
with self.assertRaises(te.MeasurementError) as ctx:
h.start()
self.assertIn("READY", str(ctx.exception))
finally:
h.close()
def test_an_engine_that_exits_without_ready_is_reported_as_an_early_exit(self):
# It prints something and *then* dies without READY, which is the realistic shape: the
# engine got far enough to log, then failed.
stub = self.stub('say("loading weights")\nsys.exit(0)\n')
h = te.EngineHarness(stub, {}, boot_timeout_s=10.0)
try:
with self.assertRaises(te.MeasurementError) as ctx:
h.start()
self.assertIn("before it was ready", str(ctx.exception))
finally:
h.close()
def test_an_engine_that_exits_immediately_reports_the_code(self):
stub = self.stub('say("starting up")\nsys.exit(7)\n')
h = te.EngineHarness(stub, {})
try:
with self.assertRaises(te.MeasurementError) as ctx:
h.start()
self.assertIn("7", str(ctx.exception))
finally:
h.close()
def test_an_error_line_from_the_engine_is_raised_verbatim(self):
stub = self.stub('say("READY 4096 stop")\nwait()\nsay("ERR out of memory")\nwait()\n')
h = te.EngineHarness(stub, {}, workload={"prompt_tokens": 4, "max_new": 1},
request_timeout_s=3.0)
try:
h.start()
with self.assertRaises(te.MeasurementError) as ctx:
h.measure()
self.assertIn("out of memory", str(ctx.exception))
finally:
h.close()
def test_the_shared_stub_engine_speaks_the_protocol_and_reacts_to_kv(self):
"""tests/tuning/stub_engine.py is the engine every sweep test uses, on every OS."""
stub_path = Path(__file__).with_name("stub_engine.py")
speeds = {}
for kv in ("int8", "fp16"):
h = te.EngineHarness(stub_path, {"--kv": kv}, workload={"prompt_tokens": 8, "max_new": 4})
try:
h.start()
m = h.measure()
self.assertEqual(m.produced, 4)
speeds[kv] = m.decode_tokens_per_s
finally:
h.close()
self.assertAlmostEqual(speeds["int8"], 80.0) # 4 tokens in 50 ms
self.assertAlmostEqual(speeds["fp16"], 40.0) # 4 tokens in 100 ms
def test_a_py_engine_runs_through_the_current_interpreter(self):
from strata_control import tuning
self.assertEqual(tuning.engine_command(Path("x") / "stub.py"), [sys.executable, str(Path("x") / "stub.py")])
self.assertEqual(tuning.engine_command(Path("x") / "strata"), [str(Path("x") / "strata")])
def test_a_missing_binary_is_an_error_before_any_process_starts(self):
h = te.EngineHarness(self.tmp / "nope", {})
with self.assertRaises(te.MeasurementError) as ctx:
h.start()
self.assertIn("not at", str(ctx.exception))
def test_the_declared_flags_are_passed_to_the_engine(self):
stub = self.stub('say("READY 4096 stop")\nwait()\nsay("DONE 1 1 1.0 1.0 stop 0 0")\nwait()\n')
h = te.EngineHarness(stub, {"--kv": "int8", "--kv-grow": "1"})
argv = h._argv()
self.assertIn("--kv", argv)
self.assertIn("int8", argv)
self.assertIn("--kv-grow", argv)
try:
h.start()
h.measure()
finally:
h.close()
def test_make_measure_returns_the_metric_dict_the_job_expects(self):
stub = self.stub('say("READY 4096 stop")\nwait()\nsay("T 1")\nsay("DONE 1 16 8.0 4.0 stop 0 0")\nwait()\n')
measure = te.make_measure(stub, workload={"prompt_tokens": 4, "max_new": 1})
class C:
args: dict = {}
metrics = measure(C())
self.assertIn("tokens_per_s", metrics)
self.assertIsInstance(metrics["tokens_per_s"], float)
self.assertGreater(metrics["tokens_per_s"], 0)
class TestWorkloadIsFixed(unittest.TestCase):
"""A sweep is only comparable if every candidate did the same work."""
def test_the_default_workload_is_deterministic(self):
w = te.DEFAULT_WORKLOAD
self.assertEqual(w["seed"], 1)
self.assertEqual(w["temperature"], 0.0)
self.assertGreater(w["prompt_tokens"], 0)
self.assertGreater(w["max_new"], 0)
def test_the_same_prompt_ids_are_produced_every_time(self):
h = te.EngineHarness(Path("/nonexistent"), {}, workload={"prompt_tokens": 16})
self.assertEqual(list(h._prompt_tokens()), list(h._prompt_tokens()))
self.assertEqual(len(list(h._prompt_tokens())), 16)
if __name__ == "__main__":
unittest.main()