Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
16.3 KB
Raw Blame
"""POST /v1/images/generations (serve/images.py): the request shape, the sd-cli command, the GPU turn and its policies,
over HTTP with a stand-in sd-cli (serve/fixtures/fake_sd_cli.py) and the mock engine.

    python -m serve.test_images
"""
import base64
import json
import os
import sys
import tempfile
import threading
import time
import unittest
import urllib.error
import urllib.request
from pathlib import Path
from unittest import mock

from serve import images as images_api
from serve.frontend import ChatTemplate
from serve.server import ByteTokenizer, MockEngine, Service, serve

ROOT = Path(__file__).resolve().parents[1]
FAKE = ROOT / "serve" / "fixtures" / "fake_sd_cli.py"
REAL_ARGS = images_api.sd_cli_args


def fake_args(cfg, job, out):
    """sd-cli's command, run through Python (the stand-in is a script)."""
    return [sys.executable, str(FAKE), *REAL_ARGS(cfg, job, out)[1:]]


def image_cfg(d, **kw):
    files = {}
    for k in ("diffusion_model", "vae", "llm"):
        p = Path(d) / f"{k}.bin"
        p.write_bytes(b"x" * 1024)
        files[k] = str(p)
    return {"exe": str(FAKE), "backend": "vulkan", "quant": "Q8_0", "args": ["--diffusion-fa"],
            "out_dir": str(Path(d) / "out"), **files, **kw}


class Request(unittest.TestCase):
    CFG = {"max_n": 4, "max_side": 2048}

    def test_defaults(self):
        j = images_api.parse_request({"prompt": "a cat"}, self.CFG)
        self.assertEqual((j["width"], j["height"], j["n"], j["steps"], j["response_format"]),
                         (1024, 1024, 1, 8, "b64_json"))
        self.assertTrue(0 <= j["seed"] < 2**31)

    def test_fields(self):
        j = images_api.parse_request({"prompt": "a cat", "size": "1536x1024", "n": 2, "steps": 4, "seed": 7,
                                      "response_format": "url", "quality": "hd", "model": "dall-e-3"}, self.CFG)
        self.assertEqual((j["width"], j["height"], j["n"], j["steps"], j["seed"], j["response_format"]),
                         (1536, 1024, 2, 4, 7, "url"))
        self.assertEqual(images_api.parse_request({"prompt": "x", "size": "auto"}, self.CFG)["width"], 1024)

    def test_refused(self):
        bad = [{}, {"prompt": ""}, {"prompt": 5}, {"prompt": "x", "n": 0}, {"prompt": "x", "n": 5},
               {"prompt": "x", "n": True}, {"prompt": "x", "size": "1000x1000"}, {"prompt": "x", "size": "128x128"},
               {"prompt": "x", "size": "4096x1024"}, {"prompt": "x", "size": "big"}, {"prompt": "x", "steps": 0},
               {"prompt": "x", "seed": -1}, {"prompt": "x", "seed": "1"}, {"prompt": "x", "response_format": "png"},
               {"prompt": "x", "output_format": "jpeg"}, {"prompt": "a\0b"}]
        for req in bad:
            with self.subTest(req=req), self.assertRaises(images_api.ImageRequestError):
                images_api.parse_request(req, self.CFG)


class Command(unittest.TestCase):
    def test_the_model_cards_command(self):
        """Unsloth's stable-diffusion.cpp command for Turbo: 8 steps, CFG 1, euler, the Turbo sigmas, --diffusion-fa."""
        cfg = {"exe": "sd-cli", "diffusion_model": "dit.gguf", "vae": "vae.st", "llm": "te.gguf",
               "args": ["--diffusion-fa", "--vae-tiling"]}
        job = {"prompt": "a sloth", "width": 1024, "height": 768, "n": 1, "steps": 8, "seed": 42}
        self.assertEqual(images_api.sd_cli_args(cfg, job, "o/img_%d.png"), [
            "sd-cli", "--diffusion-model", "dit.gguf", "--vae", "vae.st", "--llm", "te.gguf", "-p", "a sloth",
            "--steps", "8", "--cfg-scale", "1.0", "--sampling-method", "euler", "-W", "1024", "-H", "768",
            "--sigmas", "1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0",
            "-s", "42", "-o", "o/img_%d.png", "--diffusion-fa", "--vae-tiling"])

    def test_other_steps_and_batch(self):
        cfg = {"exe": "sd-cli", "diffusion_model": "d", "vae": "v", "llm": "l"}
        cmd = images_api.sd_cli_args(cfg, {"prompt": "p", "width": 512, "height": 512, "n": 3, "steps": 4,
                                           "seed": 1}, "x")
        self.assertNotIn("--sigmas", cmd)          # Turbo's sigmas are an 8-step schedule
        self.assertEqual(cmd[cmd.index("-b") + 1], "3")

    def test_native_switch(self):
        cfg = {"exe": "sd-cli", "diffusion_model": "d", "vae": "v", "llm": "l", "native": True}
        job = {"prompt": "p", "width": 512, "height": 512, "n": 1, "steps": 8, "seed": 1}
        self.assertEqual(images_api.sd_cli_args(cfg, job, "x")[-2:], ["--model-args", "qwen_image_2_1_native=1"])
        cfg["native"] = False
        self.assertNotIn("--model-args", images_api.sd_cli_args(cfg, job, "x"))

    def test_env(self):
        env = images_api.image_env({"backend": "cuda", "gpu": 1, "env": {"X": 2}}, {"PATH": "p"})
        self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["CUDA_DEVICE_ORDER"], env["X"]), ("1", "PCI_BUS_ID", "2"))
        self.assertEqual(images_api.image_env({"backend": "cuda", "gpu": [1, 0]}, {})["CUDA_VISIBLE_DEVICES"], "1,0")     # a list is a device list
        with tempfile.TemporaryDirectory() as d:
            env = images_api.image_env({"backend": "hip", "gpu": 0, "lib_dirs": [d, d + "-missing"]}, {})
            self.assertEqual(env["HIP_VISIBLE_DEVICES"], "0")
            self.assertEqual(env["PATH" if os.name == "nt" else "LD_LIBRARY_PATH"], d)
        self.assertEqual(images_api.image_env({"backend": "vulkan", "gpu": 2}, {})["GGML_VK_VISIBLE_DEVICES"], "2")

    def test_config_checked(self):
        with self.assertRaises(ValueError):
            images_api.ImageGen({"exe": "x", "diffusion_model": "d", "vae": "v"})
        with tempfile.TemporaryDirectory() as d:
            with self.assertRaises(ValueError):
                images_api.ImageGen(image_cfg(d, gpu_policy="sometimes"))


class FakeEngine:
    """An engine with the parts the GPU turn uses: alive/unload/restart, vram (elastic) and session files."""

    def __init__(self, elastic=False):
        self.up, self.calls, self.info = True, [], {"vram_elastic": elastic}
        self.spawn = (["strata"], ["--vram-elastic"] if elastic else [])
        self.unloaded = False

    def alive(self):
        return self.up

    def exit_code(self):
        return 0

    def unload(self):
        self.calls.append("unload")
        self.up, self.unloaded = False, True

    def restart(self):
        self.calls.append("restart")
        self.up, self.unloaded = True, False

    def vram(self, reserve):
        self.calls.append(("vram", reserve))
        return {"vram_free_mib": 9000}

    def session_file(self, action, path):
        self.calls.append((action, os.path.basename(path)))
        if action == "save":
            Path(path).write_bytes(b"kv")
        else:
            assert Path(path).read_bytes() == b"kv"
        return {"tokens": 123, "bytes": 2, "ms": 1.0}

    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
        return iter(())


class Policy(unittest.TestCase):
    def svc(self, engine):
        tok = ByteTokenizer()
        return Service(engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))

    def test_resolve(self):
        with tempfile.TemporaryDirectory() as d:
            gen = images_api.ImageGen(image_cfg(d))
            eng = FakeEngine()
            svc = self.svc(eng)
            with mock.patch.object(svc, "free_vram_mib", return_value=20000):
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "keep")
            with mock.patch.object(svc, "free_vram_mib", return_value=None):
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "keep")
            with mock.patch.object(svc, "free_vram_mib", return_value=4000):
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "unload")
                svc.engine = FakeEngine(elastic=True)
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "shrink")
                gen.cfg["uma"] = True
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "keep")
                gen.cfg["uma"] = False
                gen.policy = "unload"
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "unload")
                svc.engine.up = False                   # nothing loaded: nothing to give back
                self.assertEqual(images_api.resolve_policy(svc, gen, 15000), "keep")
            self.assertEqual(images_api.resolve_policy(self.svc(MockEngine(ByteTokenizer(), "x")), gen, 1), "keep")

    def test_unload_saves_and_the_next_start_restores(self):
        with tempfile.TemporaryDirectory() as d:
            gen = images_api.ImageGen(image_cfg(d, gpu_policy="unload"))
            eng = FakeEngine()
            svc = self.svc(eng)
            job = {"width": 1024, "height": 1024}
            with images_api.gpu_turn(svc, gen, job) as policy:
                self.assertEqual(policy, "unload")
                self.assertTrue(svc.fifo.locked())          # no chat turn meanwhile
                self.assertFalse(eng.up)
                self.assertTrue(svc.status["busy"])
            self.assertFalse(svc.fifo.locked())
            self.assertFalse(svc.status["busy"])
            self.assertEqual([c if isinstance(c, str) else c[0] for c in eng.calls], ["save", "unload"])
            self.assertTrue(svc.restore_after_load)
            with svc.fifo:
                svc.ensure_loaded()                          # the next request: start, then the conversation back
            self.assertEqual([c if isinstance(c, str) else c[0] for c in eng.calls],
                             ["save", "unload", "restart", "restore"])
            self.assertIsNone(svc.restore_after_load)
            self.assertEqual(list(Path(gen.dir).glob("llm-session-*")), [])

    def test_eager_reload(self):
        with tempfile.TemporaryDirectory() as d:
            gen = images_api.ImageGen(image_cfg(d, gpu_policy="unload", reload="eager"))
            eng = FakeEngine()
            svc = self.svc(eng)
            with images_api.gpu_turn(svc, gen, {"width": 512, "height": 512}):
                pass
            self.assertEqual([c if isinstance(c, str) else c[0] for c in eng.calls],
                             ["save", "unload", "restart", "restore"])
            self.assertTrue(eng.up)

    def test_shrink_gives_the_cache_back(self):
        with tempfile.TemporaryDirectory() as d:
            gen = images_api.ImageGen(image_cfg(d, gpu_policy="shrink"))
            eng = FakeEngine(elastic=True)
            svc = self.svc(eng)
            svc.vram_reserve = 512
            with images_api.gpu_turn(svc, gen, {"width": 1024, "height": 1024}):
                self.assertEqual(eng.calls[0][0], "vram")
                self.assertEqual(eng.calls[0][1], 6 * 1024)     # 2 GiB + 4 GiB per megapixel (+ 3 KiB of files)
            self.assertEqual(eng.calls[-1], ("vram", 512))
            self.assertTrue(eng.up)

    def test_keep_touches_nothing(self):
        with tempfile.TemporaryDirectory() as d:
            gen = images_api.ImageGen(image_cfg(d, gpu_policy="keep"))
            eng = FakeEngine()
            svc = self.svc(eng)
            with images_api.gpu_turn(svc, gen, {"width": 1024, "height": 1024}):
                pass
            self.assertEqual(eng.calls, [])


@mock.patch.object(images_api, "sd_cli_args", fake_args)
class OverHttp(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        tok = ByteTokenizer()
        self.engine = MockEngine(tok, "</think>\n\nok")
        self.svc = Service(self.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
        self.svc.images = images_api.ImageGen(image_cfg(self.tmp.name, keep_files=3))
        self.httpd = serve(self.svc, port=0)
        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"

    def tearDown(self):
        self.httpd.shutdown()
        self.httpd.server_close()
        self.tmp.cleanup()

    def post(self, body, path="/v1/images/generations"):
        req = urllib.request.Request(self.base + path, data=json.dumps(body).encode(),
                                     headers={"Content-Type": "application/json"})
        try:
            with urllib.request.urlopen(req, timeout=60) as r:
                return r.status, json.loads(r.read())
        except urllib.error.HTTPError as e:
            with e:
                return e.code, json.loads(e.read())

    def get(self, path):
        try:
            with urllib.request.urlopen(self.base + path, timeout=30) as r:
                return r.status, r.read(), r.headers.get("Content-Type")
        except urllib.error.HTTPError as e:
            with e:
                return e.code, e.read(), None

    def test_b64(self):
        code, body = self.post({"prompt": "a red fox", "seed": 11, "n": 2, "size": "512x768"})
        self.assertEqual(code, 200, body)
        self.assertEqual(len(body["data"]), 2)
        pngs = [base64.b64decode(x["b64_json"]) for x in body["data"]]
        sys.path.insert(0, str(FAKE.parent))
        import fake_sd_cli
        self.assertEqual(pngs, [fake_sd_cli.png(512, 768, 11, "a red fox"), fake_sd_cli.png(512, 768, 12, "a red fox")])
        self.assertEqual([x["seed"] for x in body["data"]], [11, 12])
        self.assertEqual(body["size"], "512x768")
        self.assertEqual(body["strata"]["steps"], 8)
        self.assertEqual(body["strata"]["sampling_s"], 0.08)
        self.assertEqual(body["strata"]["gpu_policy"], "keep")
        self.assertEqual(self.svc.images.totals["images"], 2)
        self.assertEqual([p.name for p in Path(self.svc.images.dir).iterdir()], [])   # nothing left behind

    def test_url(self):
        names = []
        for i in range(4):
            code, body = self.post({"prompt": "a lighthouse", "seed": i, "response_format": "url"})
            self.assertEqual(code, 200, body)
            url = body["data"][0]["url"]
            self.assertTrue(url.startswith(self.base + "/v1/images/files/"), url)
            names.append(url.rsplit("/", 1)[1])
            code, data, ctype = self.get(url[len(self.base):])
            self.assertEqual((code, ctype), (200, "image/png"))
            self.assertTrue(data.startswith(b"\x89PNG"))
        self.assertEqual(self.get("/v1/images/files/" + names[0])[0], 404)   # keep_files 3: the oldest went
        self.assertEqual(self.get("/v1/images/files/../../setup.py")[0], 404)
        self.assertEqual(self.get("/v1/images/files/" + "0" * 32 + ".png")[0], 404)

    def test_bad_request_and_failure(self):
        self.assertEqual(self.post({"prompt": "x", "size": "999x999"})[0], 400)
        with mock.patch.dict(os.environ, {"FAKE_SD_FAIL": "1"}):
            code, body = self.post({"prompt": "x"})
        self.assertEqual(code, 500)
        self.assertIn("ErrorDeviceLost", body["error"]["message"])
        self.assertEqual(self.svc.images.totals["failed"], 1)
        self.assertFalse(self.svc.status["busy"])
        self.assertFalse(self.svc.fifo.locked())

    def test_listed(self):
        code, data, _ = self.get("/v1/models")
        ids = [m["id"] for m in json.loads(data)["data"]]
        self.assertIn("qwen-image-2.1-turbo", ids)
        st = json.loads(self.get("/v1/status")[1])
        self.assertTrue(st["image_generation"]["enabled"])
        self.assertTrue(json.loads(self.get("/health")[1])["image_generation"])

    def test_not_set_up(self):
        self.svc.images = None
        code, body = self.post({"prompt": "x"})
        self.assertEqual((code, body["error"]["code"]), (404, "images_not_set_up"))

    def test_chat_waits_for_the_image(self):
        """One job at a time on the GPU: a chat request sent during an image job is answered after it."""
        done = {}
        with mock.patch.dict(os.environ, {"FAKE_SD_SLEEP": "1.5"}):
            t = threading.Thread(target=lambda: done.setdefault("img", (self.post({"prompt": "x"}), time.time())))
            t.start()
            time.sleep(0.5)
            req = urllib.request.Request(self.base + "/v1/chat/completions", data=json.dumps(
                {"model": "m", "max_tokens": 8, "messages": [{"role": "user", "content": "hi"}]}).encode(),
                headers={"Content-Type": "application/json"})
            with urllib.request.urlopen(req, timeout=60) as r:
                self.assertEqual(r.status, 200)
            chat_at = time.time()
            t.join()
        (code, _), img_at = done["img"]
        self.assertEqual(code, 200)
        self.assertGreaterEqual(chat_at, img_at - 0.05)


if __name__ == "__main__":
    unittest.main()