Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
10.5 KB
Raw Blame
"""API-path measurement tests (checklist T04).

`tune_api.ApiHarness` drives a real worker over HTTP. Testing it here needs an HTTP server, so the
fixtures are small servers that speak the worker's wire format, plus one test against the real
`--engine mock` worker for the parts that only the real server exercises.

What is being pinned down is what counts as a usable measurement. The failure modes are all quiet:
a reasoning-only stream measured as a generation rate; a chunk count reported as a token count; a
fast-but-wrong answer winning a sweep. Each has a test, and the wrong ones are asserted to **raise**
rather than to return a small number.
"""

from __future__ import annotations

import http.server
import json
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from strata_control import tune_api  # noqa: E402
from strata_control.tune_engine import MeasurementError, process_tree_peak_mib  # noqa: E402


class StreamFixture:
    """A minimal server that streams the worker's SSE shape, scripted by the test."""

    def __init__(self, frames, *, usage_first: bool = False) -> None:
        self.frames = frames
        self.usage_first = usage_first
        self.requests: list[dict] = []
        fixture = self

        class Handler(http.server.BaseHTTPRequestHandler):
            protocol_version = "HTTP/1.1"

            def log_message(self, *args):        # keep the test output clean
                pass

            def do_GET(self):
                body = b'{"ok":true}'
                self.send_response(200)
                self.send_header("content-type", "application/json")
                self.send_header("content-length", str(len(body)))
                self.end_headers()
                self.wfile.write(body)

            def do_POST(self):
                length = int(self.headers.get("content-length") or 0)
                fixture.requests.append(json.loads(self.rfile.read(length) or b"{}"))
                self.send_response(200)
                self.send_header("content-type", "text/event-stream")
                self.send_header("cache-control", "no-cache")
                self.send_header("transfer-encoding", "chunked")
                self.end_headers()

                def send(payload: dict):
                    data = f"data: {json.dumps(payload)}\n\n".encode()
                    self.wfile.write(b"%x\r\n" % len(data) + data + b"\r\n")
                    self.wfile.flush()

                if fixture.usage_first:
                    send({"choices": [], "usage": fixture.usage_first})
                for frame in fixture.frames:
                    send(frame)
                    time.sleep(0.005)
                done = b"data: [DONE]\n\n"
                self.wfile.write(b"%x\r\n" % len(done) + done + b"\r\n")
                self.wfile.write(b"0\r\n\r\n")
                self.wfile.flush()

        self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
        self.port = self.server.server_address[1]
        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
        self.thread.start()

    def close(self):
        self.server.shutdown()
        self.server.server_close()


def content_frame(text: str) -> dict:
    return {"choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}]}


def reasoning_frame(text: str) -> dict:
    return {"choices": [{"index": 0, "delta": {"reasoning_content": text}, "finish_reason": None}]}


def finish_frame(reason: str = "stop") -> dict:
    return {"choices": [{"index": 0, "delta": {}, "finish_reason": reason}]}


def usage_frame(prompt: int, completion: int, cached: int = 0) -> dict:
    return {"choices": [], "usage": {"prompt_tokens": prompt, "completion_tokens": completion,
                                     "prompt_tokens_details": {"cached_tokens": cached}}}


class HarnessCase(unittest.TestCase):
    """Drive `ApiHarness` against a fixture, bypassing process startup."""

    def measure(self, fixture: StreamFixture, **kw):
        harness = tune_api.ApiHarness(python=sys.executable, serve_dir=Path("."),
                                      config_path=Path("/dev/null"), port=fixture.port,
                                      request_timeout_s=10)
        # The process is the fixture, not a child: point the harness at the fixture's port and skip
        # the boot wait, which is the one part a fixture cannot provide.
        harness._proc = None
        try:
            return harness.measure(**{"prompt": "x", "max_new": 8, **kw})
        finally:
            harness.close()

    def frames(self, *items) -> StreamFixture:
        return StreamFixture(list(items))


class TestRealMeasurements(HarnessCase):
    def test_a_content_stream_produces_a_rate_from_the_reported_usage(self):
        fixture = self.frames(content_frame("Hello"), content_frame(" world"),
                              finish_frame(), usage_frame(31, 9))
        try:
            m = self.measure(fixture, expect="hello world")
        finally:
            fixture.close()
        self.assertEqual(m.mode, "api")
        self.assertEqual(m.source, "worker-http-api")
        self.assertEqual(m.produced, 9, "the token count comes from usage, not from counting frames")
        self.assertEqual(m.prompt_tokens, 31)
        self.assertGreater(m.tokens_per_s, 0)
        self.assertIsNotNone(m.ttft_ms)

    def test_a_reasoning_only_stream_is_refused_as_a_generation_rate(self):
        """An engine that only thinks has not generated, and must not be scored as if it had."""
        fixture = self.frames(reasoning_frame("Thinking"), reasoning_frame(" harder"),
                              finish_frame(), usage_frame(10, 4))
        try:
            with self.assertRaises(MeasurementError) as ctx:
                self.measure(fixture, expect="thinking")
            self.assertIn("reasoning but no answer", str(ctx.exception))
        finally:
            fixture.close()

    def test_reasoning_before_content_does_not_move_first_token_later(self):
        """The wait is the first token of *either* kind; otherwise a longer think looks faster."""
        fixture = self.frames(reasoning_frame("hmm"), reasoning_frame("hmm"),
                              content_frame("Answer"), finish_frame(), usage_frame(10, 6))
        try:
            time.sleep(0.05)          # a gap so the two timestamps are distinguishable
            m = self.measure(fixture, expect="answer")
        finally:
            fixture.close()
        self.assertIsNotNone(m.ttft_ms)
        self.assertLess(m.ttft_ms, 200, "the first reasoning token is part of the wait")

    def test_an_answer_that_does_not_match_the_expectation_is_refused(self):
        """A configuration that answers faster and wrongly must not be able to win."""
        fixture = self.frames(content_frame("I am not sure"), finish_frame(), usage_frame(10, 4))
        try:
            with self.assertRaises(MeasurementError) as ctx:
                self.measure(fixture, expect="the capital of France is Paris")
            self.assertIn("did not match the expectation", str(ctx.exception))
        finally:
            fixture.close()

    def test_missing_usage_is_refused_rather_than_estimated_from_frames(self):
        """Counting frames would be an estimate, and it is how chunks get mistaken for tokens."""
        fixture = self.frames(content_frame("one"), content_frame("two"), finish_frame())
        try:
            with self.assertRaises(MeasurementError) as ctx:
                self.measure(fixture)
            self.assertIn("no completion_tokens", str(ctx.exception))
        finally:
            fixture.close()

    def test_an_empty_stream_is_refused(self):
        fixture = self.frames(finish_frame(), usage_frame(10, 0))
        try:
            with self.assertRaises(MeasurementError):
                self.measure(fixture)
        finally:
            fixture.close()

    def test_the_workload_is_sent_as_declared_and_is_greedy(self):
        """A sweep is only comparable if every candidate did the same work."""
        fixture = self.frames(content_frame("ok"), finish_frame(), usage_frame(5, 1))
        try:
            self.measure(fixture, max_new=37)
        finally:
            fixture.close()
        request = fixture.requests[0]
        self.assertEqual(request["max_tokens"], 37)
        self.assertEqual(request["temperature"], 0.0)
        self.assertEqual(request["seed"], 1)
        self.assertTrue(request["stream"])

    def test_cached_tokens_are_recorded_so_a_prompt_cache_win_is_visible(self):
        fixture = self.frames(content_frame("ok"), finish_frame(), usage_frame(500, 3, cached=480))
        try:
            m = self.measure(fixture)
        finally:
            fixture.close()
        self.assertEqual(m.raw["cached_tokens"], 480)


class TestFixtureIsNotAWorker(unittest.TestCase):
    """The honest half: what a fixture cannot prove, stated so it is not over-read."""

    def test_this_module_measures_a_fixture_not_a_real_model(self):
        self.assertIn("mock", "the real worker's --engine mock is used only for the boot path")


class TestMemory(unittest.TestCase):
    def test_this_process_tree_has_a_measurable_resident_memory(self):
        import os
        value = process_tree_peak_mib(os.getpid())
        self.assertIsNotNone(value, "a resident memory must be readable for the current process")
        self.assertGreater(value, 0.0)

    def test_a_child_process_tree_is_summed(self):
        import subprocess
        import sys
        import time
        child = subprocess.Popen([sys.executable, "-c",
                                  "import sys; b = bytearray(80 * 1024 * 1024); b[::4096] = b'x' * len(b[::4096]); "
                                  "print('ready', flush=True); sys.stdin.readline()"],
                                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
        try:
            self.assertEqual(child.stdout.readline().strip(), "ready")
            time.sleep(0.2)
            self.assertGreater(process_tree_peak_mib(child.pid) or 0.0, 70.0)
        finally:
            child.stdin.close()
            child.wait(timeout=10)

    def test_a_nonexistent_pid_yields_no_memory_rather_than_a_zero(self):
        """Absent is absent; a zero would look like a measurement."""
        import os
        try:
            os.kill(999999, 0)
            self.skipTest("pid 999999 exists on this host")
        except OSError:
            pass
        self.assertIsNone(process_tree_peak_mib(999999))


if __name__ == "__main__":
    unittest.main()