Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
6.1 KB
Raw Blame
"""The real session's request side, against a tiny HTTP server that answers like the worker does (a `timings` block on the
engine's own clock)."""

from __future__ import annotations

import json
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path

from strata_control import tune_session
from strata_control.tune_session import RequestSet, WorkerEngine, WorkerSession, parse_facts, failure_kind, make_prompt


class Handler(BaseHTTPRequestHandler):
    seen: list[dict] = []

    def log_message(self, *a):  # noqa: ANN002
        pass

    def do_GET(self):  # noqa: N802
        body = json.dumps({"data": [{"id": "stub-model"}]}).encode()
        self.send_response(200)
        self.send_header("content-length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)

    def do_POST(self):  # noqa: N802
        req = json.loads(self.rfile.read(int(self.headers["content-length"])))
        Handler.seen.append({"auth": self.headers.get("authorization"), **req})
        n = req["max_tokens"]
        prompt_tokens = len(req["messages"][0]["content"]) // 4
        tune = req.get("strata_tune") or {}
        rate = 80.0 * (1.1 if tune.get("spec_min_p") == 0.7 else 1.0)
        body = json.dumps({"choices": [{"message": {"content": "x"}}], "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": n},
                           "timings": {"cache_n": 0, "prompt_n": prompt_tokens, "prompt_ms": prompt_tokens / 2.0,
                                       "prompt_per_second": 2000.0, "predicted_n": n, "predicted_ms": n / rate * 1000,
                                       "predicted_per_second": rate}}).encode()
        self.send_response(200)
        self.send_header("content-length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)


class FakeProc:
    pid = 0

    def poll(self):
        return None


class SessionTest(unittest.TestCase):
    def setUp(self):
        Handler.seen = []
        self.srv = HTTPServer(("127.0.0.1", 0), Handler)
        threading.Thread(target=self.srv.serve_forever, daemon=True).start()
        self.addCleanup(self.srv.shutdown)
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        s = WorkerSession(base_config={"args": []}, serve_dir=".", workdir=self.tmp.name)
        log = Path(self.tmp.name) / "e.log"
        log.write_text("strata generate: expert cache 3160 slots, 4.26 GiB of VRAM\nstrata generate: 5 pool workers on logical processors 2,4\n")
        self.eng = WorkerEngine(s, FakeProc(), self.srv.server_address[1], "k", [log], 12.0, None, lambda x: None)

    def test_measure_reads_the_engines_own_clock(self):
        r = self.eng.measure(RequestSet(warmups=1, decodes=3, decode_tokens=200, pp_tokens=4096))
        self.assertEqual(r.status, "PASS")
        self.assertAlmostEqual(r.decode_tok_s, 80.0, places=1)
        self.assertEqual(r.pp_tok_s, 2000.0)
        self.assertEqual(r.facts["slots"], 3160)
        self.assertEqual(r.facts["workers"], 5)
        self.assertTrue(all(h["auth"] == "Bearer k" for h in Handler.seen))
        self.assertTrue(all(h["model"] == "stub-model" for h in Handler.seen))

    def test_prompts_are_unique_and_unchecked_by_cache(self):
        self.eng.prompt_once(4096)
        self.eng.prompt_once(4096)
        a, b = [h["messages"][0]["content"] for h in Handler.seen[-2:]]
        self.assertNotEqual(a[:40], b[:40])
        self.assertTrue(all(h.get("strata_checkpoint") is False for h in Handler.seen))

    def test_per_request_settings_are_sent_and_change_the_rate(self):
        base = self.eng.decode_once(None, 200)
        fast = self.eng.decode_once({"spec_min_p": 0.7}, 200)
        self.assertAlmostEqual(fast / base, 1.1, places=3)
        self.assertEqual(Handler.seen[-1]["strata_tune"], {"spec_min_p": 0.7})

    def test_prompt_length_calibrates_once(self):
        s = self.eng.s
        before = s.chars_per_token
        self.eng.prompt_once(4096)
        self.assertTrue(s.calibrated)
        self.assertNotEqual(s.chars_per_token, 0)
        text = make_prompt(1000, "abc", s.chars_per_token)
        self.assertIn("Reference abc.", text)


class ParsingTest(unittest.TestCase):
    def test_failure_kinds(self):
        self.assertEqual(failure_kind("error: unknown argument: --adapt-min-gain\nusage...", 2)[0], "UNSUPPORTED")
        self.assertEqual(failure_kind("native dense upload ... out of memory", 1)[0], "OOM")
        self.assertEqual(failure_kind("", -9)[0], "OOM")
        self.assertEqual(failure_kind("something else broke", 1)[0], "FAIL_BOOT")

    def test_cuda_init_failure_with_the_memory_guard_line_is_not_oom(self):
        # #3502: the informational memory guard line was present in every failed run
        text = ("ggml_cuda_init: failed to initialize CUDA: no CUDA-capable device is detected\n"
                "strata generate: memory guard: limit 63.8 GiB (source: meminfo), current 12.6 GiB, reclaimable cache "
                "credited 0.0 GiB; available 51.2 GiB (MemAvailable 51.2 GiB), floor 2560 MiB\n"
                "strata generate: expert arena: cudaHostRegister of the whole arena FAILED (out of memory)\n")
        status, why = failure_kind(text, 1)
        self.assertEqual(status, "FAIL_BOOT")
        self.assertIn("Unload the model", why)
        self.assertNotIn("memory guard", why)
        info = "strata generate: memory guard: limit 63.8 GiB, available 51.2 GiB\nsomething else broke"
        self.assertEqual(failure_kind(info, 1)[0], "FAIL_BOOT")

    def test_gpu_cache_wording_is_not_the_low_ram_mode(self):
        self.assertFalse(parse_facts("R4 hit path ON - resident experts are computed on the GPU")["resident_ram"])
        self.assertTrue(parse_facts("WARNING: the whole resident RAM mode does not fit (FileExpertSource: resident complement 34.47 GiB")["resident_ram"])

    def test_facts(self):
        f = parse_facts("strata generate: 6 CPU threads\nhybrid experts: 37% on the GPU\n")
        self.assertEqual(f["workers"], 6)
        self.assertTrue(f["hybrid_experts"])
        self.assertTrue(parse_facts("resident RAM mode on")["resident_ram"])


if __name__ == "__main__":
    unittest.main()