Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
46.1 KB
Raw Blame
"""Decision engines behind POST /v1/systemone: the DECIDE command, end to end.

A decision engine (Laya, GLiNER2.5-Decide) does not generate. It announces
`CAPS decide=1 chat=0`, and /v1/systemone hands it the request as one DECIDE
record instead of scoring options through the logprob channel. Three layers are
pinned here:

  DecideProtocol      the wire, against a fake engine process: the DECIDE frame
                      byte for byte, the DECISION + DONE reply, a refusal
                      (DECIDE_INVALID) as the caller's 422, the CAPS flags.
  DecisionGateway     the HTTP route against a fake decision engine: the reply
                      has exactly the shape the LLM path returns, the record
                      carries the state and the options as the reference reads
                      them, the generating endpoints answer 400 with a pointer,
                      validation stays the route's own.
  LayaTinyEndToEnd    the real laya engine on the tiny fixture behind the real
                      gateway: the probabilities that come back over HTTP are
                      the `laya` package's (laya_tiny/ref.json), and the
                      latency is split into gateway and engine time.
  GlinerDecideTinyEndToEnd
                      the same for the gliner_decide engine on its tiny
                      fixture, against the `gliner2` package's answers
                      (gliner_decide_tiny/ref.json).
  ClefTinyEndToEnd    qwen36 with Clef's decision head (clef_tiny_c) behind the
                      real gateway: CAPS decide=1 decide_record=raw and no
                      chat=0, /v1/systemone gives Clef's own probabilities
                      (clef_tiny/ref.json), and the same engine still chats,
                      with a decision before and after the chat unchanged.

Each end-to-end class is skipped without its fixture and binary (`make
laya-tiny-check` / `make gliner-decide-tiny-check` / `make clef-tiny-check`
build them and turn the skip into a failure).
"""
import json
import math
import os
import sys
import threading
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from urllib.error import HTTPError
from urllib.request import Request, urlopen

HERE = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE / "tests"))
import openai_server  # noqa: E402
from family_registry import family_by_id  # noqa: E402
from openai_server import APIError, APIServer, Engine, READY  # noqa: E402
from test_openai_server import BlockingStream, FakeProcess  # noqa: E402

FIXTURE = HERE / "laya_tiny"
BINARY = HERE / ("laya.exe" if os.name == "nt" else "laya")
REQUIRED = os.environ.get("LAYA_TINY_REQUIRED") == "1"
GLINER_FIXTURE = HERE / "gliner_decide_tiny"
GLINER_BINARY = HERE / ("gliner_decide.exe" if os.name == "nt" else "gliner_decide")
GLINER_REQUIRED = os.environ.get("GLINER_DECIDE_TINY_REQUIRED") == "1"
CLEF_REF = HERE / "clef_tiny" / "ref.json"
CLEF_CONTAINER = HERE / "clef_tiny_c"
QWEN36 = HERE / ("qwen36.exe" if os.name == "nt" else "qwen36")
CLEF_REQUIRED = os.environ.get("CLEF_TINY_REQUIRED") == "1"

TICKET = {"from": "[email protected]", "subject": "Duplicate charge on invoice #4411",
          "body": "Hi, we were billed twice for March. Please refund the duplicate today or we "
                  "will cancel our plan."}
QUESTIONS = {
    "department": {"type": "choice", "instructions": "Which department should handle this request?",
                   "criteria": {"billing": "invoices, payments, refunds",
                                "technical": "bugs, outages, system errors",
                                "sales": "pricing, new contracts", "other": "everything else"}},
    "urgency": {"type": "score", "instructions": "How urgent is this request?",
                "criteria": ["not urgent", "soon", "critical deadline or blocking issue"]},
    "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"},
    "refund_requested": {"type": "noul", "instructions": "Does the user explicitly request a refund?"},
}


def decision_process(on_decide):
    """A FakeProcess whose handshake announces a decision engine."""
    process = FakeProcess(on_decide)
    process.stdout = BlockingStream(READY + b"CAPS decide=1 chat=0 max_len=512\n"
                                    b"STAT 0 0.0 0.0 1.00 0 0\n")
    return process


class DecideProtocol(unittest.TestCase):
    def start(self, respond):
        process = decision_process(respond)
        with patch("openai_server.subprocess.Popen", return_value=process):
            engine = Engine("laya", "model", family=family_by_id("laya"))
        self.addCleanup(engine.close)
        return engine, process

    def test_caps_say_decide_and_no_chat(self):
        engine, _ = self.start(lambda process, frame: None)
        self.assertTrue(engine.decides)
        self.assertFalse(engine.chats)
        self.assertTrue(openai_server.engine_decides(engine))
        self.assertFalse(openai_server.engine_chats(engine))

    def test_round_trip_is_byte_exact(self):
        record = {"state": "sé", "state_type": "string", "questions": [
            {"id": "q", "type": "noul", "instructions": "ok?",
             "options": [{"label": "false", "text": None}, {"label": "true", "text": None}]}]}
        payload = json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode()
        expected = f"DECIDE 1 0 {len(payload)}\n".encode() + payload + b"\n"
        reply = json.dumps({"answers": [{"id": "q", "logits": [0.0, 1.0], "probs": [0.25, 0.75],
                                         "temperature": 2.0, "tokens": 9}],
                            "input_tokens": 9, "engine_ms": 4.5}).encode()

        def respond(process, frame):
            self.assertEqual(frame, expected)
            process.stdout.feed(f"DECISION 1 {len(reply)}\n".encode() + reply +
                                b"\nDONE 1 STAT 0 2000.000 0.0 1.00 9 0\n")

        engine, process = self.start(respond)
        decision, stats = engine.decide(record)
        self.assertEqual(process.writes, [expected])
        self.assertEqual(decision["answers"][0]["probs"], [0.25, 0.75])
        self.assertEqual(stats["prompt_tokens"], 9)
        self.assertEqual(stats["completion_tokens"], 0)

    def test_a_refused_record_is_the_callers_422(self):
        def respond(process, frame):
            process.stdout.feed(b"ERROR 1 DECIDE_INVALID questions.bucket: only 38 of its 40 "
                                b"option markers fit in max_len=160\n")

        engine, _ = self.start(respond)
        with self.assertRaises(APIError) as caught:
            engine.decide({"state": "x", "state_type": "string", "questions": []})
        self.assertEqual(caught.exception.status, 422)
        self.assertEqual(caught.exception.param, "questions.bucket")
        self.assertIn("only 38 of its 40", caught.exception.message)

    def test_an_engine_failure_is_not_the_callers(self):
        def respond(process, frame):
            process.stdout.feed(b"ERROR 1 DECIDE_FAILED out of memory\n")

        engine, _ = self.start(respond)
        with self.assertRaises(RuntimeError):
            engine.decide({"state": "x", "state_type": "string", "questions": []})

    def test_a_nul_never_reaches_the_engine(self):
        engine, process = self.start(lambda process, frame: self.fail("written"))
        with self.assertRaises(APIError) as caught:
            engine.decide({"state": "a\0b", "state_type": "string", "questions": []})
        self.assertEqual(caught.exception.status, 422)
        self.assertEqual(process.writes, [])


class FakeDecisionEngine:
    """A decision engine that answers from a table: the probabilities of each
    question by id, in the record's option order."""

    decides = True
    chats = False
    kv_slots = 1

    def __init__(self, table, refuse=None):
        self.table = table
        self.refuse = refuse
        self.records = []

    def decide(self, record, cache_slot=0, cancelled=None):
        self.records.append(record)
        if self.refuse:
            raise APIError(422, self.refuse, "questions.q", "invalid_question")
        answers = [{"id": q["id"], "logits": [math.log(p) for p in self.table[q["id"]]],
                    "probs": self.table[q["id"]], "temperature": 1.0, "tokens": 10}
                   for q in record["questions"]]
        return {"answers": answers, "input_tokens": 10 * len(answers), "engine_ms": 12.5}, {}

    def generate(self, *args, **kwargs):
        raise AssertionError("a decision engine is never asked to generate")

    def close(self):
        pass


class DecisionGateway(unittest.TestCase):
    def serve(self, engine):
        self.engine = engine
        self.server = APIServer(("127.0.0.1", 0), engine, "laya")
        self.addCleanup(self.server.server_close)
        self.addCleanup(self.server.shutdown)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()
        self.base = f"http://127.0.0.1:{self.server.server_port}"

    def post(self, path, body):
        request = Request(self.base + path, data=json.dumps(body).encode(),
                          headers={"Content-Type": "application/json"})
        with urlopen(request, timeout=10) as response:
            return json.loads(response.read()), dict(response.headers)

    def post_error(self, path, body):
        try:
            self.post(path, body)
        except HTTPError as error:
            return error.code, json.loads(error.read())
        self.fail("expected an error")

    def table(self):
        return {"department": [0.7, 0.1, 0.1, 0.1], "urgency": [0.2, 0.3, 0.5],
                "churn_risk": [0.25, 0.75], "refund_requested": [0.9, 0.1]}

    def test_reply_has_the_shape_the_llm_path_returns(self):
        self.serve(FakeDecisionEngine(self.table()))
        out, headers = self.post("/v1/systemone", {"model": "jev-latest", "state": TICKET,
                                                   "questions": QUESTIONS})
        self.assertEqual(out["model"], "laya")
        self.assertEqual(list(out["answers"]), list(QUESTIONS))
        self.assertEqual(out["answers"]["churn_risk"], {"type": "noul", "noul": 0.75})
        self.assertEqual(out["answers"]["refund_requested"], {"type": "noul", "noul": 0.1})
        department = out["answers"]["department"]
        self.assertEqual(set(department), {"type", "choice", "probabilities", "confidence"})
        self.assertEqual(department["choice"], "billing")
        self.assertEqual(list(department["probabilities"]), ["billing", "technical", "sales", "other"])
        self.assertAlmostEqual(department["confidence"], (4 * 0.7 - 1) / 3, places=6)
        urgency = out["answers"]["urgency"]
        self.assertEqual(set(urgency), {"type", "score", "legend", "probabilities", "confidence"})
        self.assertEqual(urgency["legend"], {"0": "not urgent", "1": "soon",
                                             "2": "critical deadline or blocking issue"})
        self.assertEqual(list(urgency["probabilities"]), ["0", "1", "2"])
        self.assertAlmostEqual(urgency["score"], 0 * 0.2 + 1 * 0.3 + 2 * 0.5, places=6)
        self.assertEqual(out["usage"], {"input_tokens": 40, "output_tokens": 0, "cost": 0})
        self.assertEqual(out["provider"], "colibri")
        self.assertEqual(headers["x-colibri-engine-ms"], "12.5")

    def test_the_record_carries_what_the_reference_reads(self):
        self.serve(FakeDecisionEngine({"q": [0.5, 0.5], "c": [0.6, 0.4], "s": [0.5, 0.5],
                                       "d": [0.5, 0.5]}))
        self.post("/v1/systemone", {"model": "jev-latest",
                                    "state": {"ticket": 4711, "text": "café", "ok": True},
                                    "questions": {
            "q": {"type": "noul", "instructions": {"ask": "money?"},
                  "criteria": {"true": "about a payment"}},
            "c": {"type": "choice", "instructions": "  Which?  ",
                  "criteria": {"a": None, "b": {"desc": "structured"}}},
            "s": {"type": "score", "criteria": ["low", {"n": 2}]},
            "d": {"type": "noul"}}})
        record = self.engine.records[0]
        # a JSON state as json.dumps(ensure_ascii=False) writes it, and its type
        self.assertEqual(record["state"], '{"ticket": 4711, "text": "café", "ok": true}')
        self.assertEqual(record["state_type"], "object")
        q, c, s, d = record["questions"]
        self.assertEqual(q["instructions"], '{"ask": "money?"}')
        self.assertEqual(q["options"], [{"label": "false", "text": None},
                                        {"label": "true", "text": "about a payment"}])
        self.assertEqual(c["instructions"], "  Which?  ")                  # exactly as sent
        self.assertEqual(c["options"], [{"label": "a", "text": None},
                                        {"label": "b", "text": '{"desc": "structured"}'}])
        self.assertEqual(s["options"], [{"label": "0", "text": "low"},
                                        {"label": "1", "text": '{"n": 2}'}])
        self.assertEqual(s["instructions"], "Rate this on the scale below.")
        self.assertEqual(d["instructions"], "Is this true?")

    def test_a_raw_form_engine_gets_the_callers_values(self):
        """An engine that said decide_record=raw (Clef) renders the request the way
        its reference does: it gets the caller's values, not the gateway's
        defaults, and JSON with sorted keys."""
        engine = FakeDecisionEngine({"q": [0.5, 0.5], "c": [0.2, 0.3, 0.5], "s": [0.5, 0.5],
                                     "d": [0.5, 0.5]})
        engine.decide_record = "raw"
        engine.chats = True
        self.serve(engine)
        self.post("/v1/systemone", {"state": {"ticket": 4711, "text": "café", "ok": True},
                                    "questions": {
            "q": {"type": "noul", "instructions": {"ask": "money?", "a": 1},
                  "criteria": {"true": "about a payment", "false": None}},
            "c": {"type": "choice", "instructions": "",
                  "criteria": {"b": None, "a": {"z": 1, "y": [2.5]}, "c": ""}},
            "s": {"type": "score", "criteria": ["low", {"n": 2}]},
            "d": {"type": "noul"}}})
        record = self.engine.records[0]
        self.assertEqual(record["record"], "raw")
        self.assertEqual(record["state"], '{"ok":true,"text":"café","ticket":4711}')
        q, c, s, d = record["questions"]
        self.assertEqual(q["instructions"], '{"a":1,"ask":"money?"}')
        self.assertEqual(q["options"], [{"label": "false", "text": None},
                                        {"label": "true", "text": "about a payment"}])
        self.assertEqual(c["instructions"], "")                         # not replaced
        self.assertEqual(c["options"], [{"label": "b", "text": None},
                                        {"label": "a", "text": '{"y":[2.5],"z":1}', "json": True},
                                        {"label": "c", "text": ""}])
        self.assertEqual(s["instructions"], None)
        self.assertEqual(s["options"][1], {"label": "1", "text": '{"n":2}', "json": True})
        self.assertEqual(d["options"], [{"label": "false"}, {"label": "true"}])   # undescribed
        with urlopen(self.base + "/v1/models", timeout=10) as response:
            self.assertEqual(json.loads(response.read())["data"][0]["capabilities"], ["chat", "systemone"])

    def test_a_list_state_is_a_conversation(self):
        self.serve(FakeDecisionEngine({"q": [0.5, 0.5]}))
        turns = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
        self.post("/v1/systemone", {"state": turns, "questions": {"q": {"type": "noul",
                                                                        "instructions": "?"}}})
        self.assertEqual(self.engine.records[0]["state_type"], "array")
        self.assertEqual(json.loads(self.engine.records[0]["state"]), turns)

    def test_generating_endpoints_point_to_systemone(self):
        self.serve(FakeDecisionEngine({}))
        for path, body in (("/v1/chat/completions", {"messages": [{"role": "user", "content": "hi"}]}),
                           ("/v1/completions", {"prompt": "hi"}),
                           ("/v1/messages", {"max_tokens": 5, "messages": [{"role": "user", "content": "hi"}]})):
            with self.subTest(path=path):
                status, error = self.post_error(path, dict(body, model="laya"))
                self.assertEqual(status, 400)
                if path != "/v1/messages":              # Anthropic's envelope has no code
                    self.assertEqual(error["error"]["code"], "unsupported_endpoint")
                self.assertIn("POST /v1/systemone", error["error"]["message"])

    def test_v1_brio_is_gone_here_too(self):
        self.serve(FakeDecisionEngine({}))
        status, error = self.post_error("/v1/brio", {"model": "laya", "state": "x", "options": ["a", "b"]})
        self.assertEqual((status, error["error"]["code"]), (404, "not_found"))

    def test_models_card_and_health_say_decision(self):
        self.serve(FakeDecisionEngine({}))
        with urlopen(self.base + "/v1/models", timeout=10) as response:
            card = json.loads(response.read())["data"][0]
        self.assertEqual(card["capabilities"], ["systemone", "decision"])
        with urlopen(self.base + "/health", timeout=10) as response:
            self.assertEqual(json.loads(response.read())["capabilities"], ["systemone", "decision"])

    def test_colibris_options_on_a_decision_engine(self):
        """normalize and pin_state act on a language model's logprobs and photos;
        a decision engine has neither and ignores them. A prefix is refused: it
        would be text the model never reads. cache_slot reaches the engine."""
        engine = FakeDecisionEngine({"q": [0.5, 0.5]})
        engine.kv_slots = 2
        engine.slots = []
        decide = engine.decide
        engine.decide = lambda record, cache_slot=0, cancelled=None: (
            engine.slots.append(cache_slot), decide(record, cache_slot, cancelled))[1]
        self.engine = engine
        self.server = APIServer(("127.0.0.1", 0), engine, "laya", kv_slots=2)
        self.addCleanup(self.server.server_close)
        self.addCleanup(self.server.shutdown)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()
        self.base = f"http://127.0.0.1:{self.server.server_port}"
        question = {"q": {"type": "noul", "instructions": "?"}}
        out, _ = self.post("/v1/systemone", {"state": "x", "questions": question,
                                             "normalize": "mean", "pin_state": True, "cache_slot": 1})
        self.assertEqual(out["answers"]["q"]["noul"], 0.5)
        self.assertEqual(engine.slots, [1])
        status, error = self.post_error("/v1/systemone", {"state": "x", "questions": question,
                                                          "prefix": "rules"})
        self.assertEqual((status, error["error"]["param"]), (422, "prefix"))

    def test_validation_is_the_routes_own(self):
        self.serve(FakeDecisionEngine({}))
        for body, param in (({"questions": {"q": {"type": "noul"}}}, "state"),
                            ({"state": "x", "questions": {}}, "questions"),
                            ({"state": "x", "questions": {"q": {"type": "maybe"}}}, "questions.q.type"),
                            ({"state": "x", "questions": {"q": {"type": "score", "criteria": []}}},
                             "questions.q.criteria")):
            with self.subTest(param=param):
                status, error = self.post_error("/v1/systemone", body)
                self.assertEqual(status, 422)
                self.assertEqual(error["error"]["param"], param)
        self.assertEqual(self.engine.records, [])

    def test_an_engine_refusal_reaches_the_client_as_422(self):
        self.serve(FakeDecisionEngine({}, refuse="questions.q: only 3 of its 30 option markers fit"))
        status, error = self.post_error("/v1/systemone", {"state": "x", "questions": {
            "q": {"type": "choice", "criteria": {"a": "x", "b": "y"}}}})
        self.assertEqual(status, 422)
        self.assertIn("only 3 of its 30", error["error"]["message"])


class DecisionRegistryAndLauncher(unittest.TestCase):
    """A Laya checkpoint has no root config.json: the registry knows it by its
    rl_agent_config.json and its encoder's config. A GLiNER2 checkpoint's
    config.json says "extractor": the registry reads its architecture and its
    encoder_config. coli describes both and refuses to chat with them."""

    def checkpoint(self, encoder):
        import tempfile
        directory = tempfile.TemporaryDirectory()
        self.addCleanup(directory.cleanup)
        root = Path(directory.name)
        (root / "encoder").mkdir()
        (root / "rl_agent_config.json").write_text(json.dumps({"head_layers": 2}), encoding="utf-8")
        (root / "encoder" / "config.json").write_text(json.dumps(encoder), encoding="utf-8")
        return root

    def test_the_registry_knows_a_laya_checkpoint(self):
        from family_registry import UnknownFamilyError, default_model_id, display_for, resolve_model
        english = resolve_model(self.checkpoint({"model_type": "modernbert", "hidden_size": 1024,
                                                 "num_hidden_layers": 28}))
        self.assertEqual((english.descriptor.id, english.descriptor.modality), ("laya", "decision"))
        self.assertTrue(english.descriptor.capabilities.decision)
        self.assertEqual(display_for(english), ("Laya", "421M"))
        multilingual = resolve_model(self.checkpoint({"model_type": "modernbert", "hidden_size": 768,
                                                      "num_hidden_layers": 22}))
        self.assertEqual(display_for(multilingual), ("Laya multilingual", "322M"))
        self.assertEqual(default_model_id(multilingual), "laya-multilingual")
        with self.assertRaises(UnknownFamilyError):
            resolve_model(self.checkpoint({"model_type": "deberta-v2"}))

    def gliner_checkpoint(self, config, encoder):
        import tempfile
        directory = tempfile.TemporaryDirectory()
        self.addCleanup(directory.cleanup)
        root = Path(directory.name)
        (root / "encoder_config").mkdir()
        (root / "config.json").write_text(json.dumps(config), encoding="utf-8")
        (root / "encoder_config" / "config.json").write_text(json.dumps(encoder), encoding="utf-8")
        return root

    def test_the_registry_knows_a_gliner2_checkpoint(self):
        from family_registry import UnknownFamilyError, default_model_id, display_for, resolve_model
        large = {"model_type": "deberta-v2", "hidden_size": 1024, "num_hidden_layers": 24}
        decide = resolve_model(self.gliner_checkpoint(
            {"model_type": "extractor", "architecture": "span"}, large))
        self.assertEqual((decide.descriptor.id, decide.descriptor.modality),
                         ("gliner_decide", "decision"))
        self.assertEqual(decide.model_type, "gliner2_span_deberta-v2")
        self.assertTrue(decide.descriptor.capabilities.decision)
        self.assertEqual(display_for(decide), ("GLiNER2.5-Decide", "340M"))
        self.assertEqual(default_model_id(decide), "gliner2.5-decide")
        # a legacy config without an architecture is "span", as AutoExtractor reads it
        legacy = resolve_model(self.gliner_checkpoint({"model_type": "extractor"}, large))
        self.assertEqual(legacy.descriptor.id, "gliner_decide")
        # another geometry is named by its own model_type, not by the 340M's
        base = resolve_model(self.gliner_checkpoint(
            {"model_type": "extractor"}, {"model_type": "deberta-v2", "hidden_size": 768,
                                          "num_hidden_layers": 12}))
        self.assertEqual(display_for(base), ("gliner2_span_deberta-v2", ""))
        for config, encoder in (({"model_type": "extractor", "architecture": "boundary"}, large),
                                ({"model_type": "extractor"}, {"model_type": "modernbert"})):
            with self.subTest(config=config, encoder=encoder), self.assertRaises(UnknownFamilyError):
                resolve_model(self.gliner_checkpoint(config, encoder))

    def test_coli_describes_it_and_refuses_to_chat(self):
        import subprocess
        # each fixture with its weights generated and its engine built: coli plan reads the
        # weights' header, and coli chat says the engine is not built before anything else
        fixtures = [path for path, binary in ((FIXTURE, BINARY), (GLINER_FIXTURE, GLINER_BINARY))
                    if (path / "model.safetensors").exists() and binary.exists()]
        if not fixtures:
            self.skipTest("no tiny decision fixture with its engine built "
                          "(make laya-tiny-check / gliner-decide-tiny-check)")
        def coli(*args):
            return subprocess.run([sys.executable, str(HERE / "coli"), *args], capture_output=True,
                                  text=True, timeout=120, cwd=str(HERE))
        for fixture in fixtures:
            with self.subTest(fixture=fixture.name):
                info = coli("info", "--model", str(fixture))
                self.assertIn("decision model", info.stdout)
                plan = coli("plan", "--model", str(fixture), "--json")
                self.assertEqual(json.loads(plan.stdout)["modality"], "decision")
                doctor = json.loads(coli("doctor", "--model", str(fixture), "--json").stdout)
                checks = [{"id": c["id"], "status": c["status"]} for c in doctor["checks"]]
                self.assertIn({"id": "model.family", "status": "pass"}, checks)
                self.assertNotIn({"id": "model.files", "status": "fail"}, checks)
                for command in (("chat", "--model", str(fixture), "--no-attach"),
                                ("run", "--model", str(fixture), "hello")):
                    refused = coli(*command)
                    self.assertNotEqual(refused.returncode, 0)
                    self.assertIn("POST /v1/systemone", refused.stdout + refused.stderr)


    def test_the_registry_knows_a_clef_checkpoint(self):
        import tempfile
        from family_registry import (FamilyConfigError, checkpoint_decides, default_model_id,
                                     display_for, resolve_model)
        directory = tempfile.TemporaryDirectory()
        self.addCleanup(directory.cleanup)
        root = Path(directory.name)
        text = {"model_type": "qwen3_5_text", "num_hidden_layers": 64, "hidden_size": 5120,
                "intermediate_size": 17408}
        (root / "config.json").write_text(json.dumps({"model_type": "qwen3_5", "text_config": text}),
                                          encoding="utf-8")
        stock = resolve_model(root)
        self.assertEqual((stock.decision_head, display_for(stock)), ("", ("Qwen3.8-27B", "27B")))
        self.assertFalse(checkpoint_decides(stock))
        (root / "joint_head_config.json").write_text(json.dumps({"hidden_size": 5120}), encoding="utf-8")
        with self.assertRaises(FamilyConfigError):          # half a head
            resolve_model(root)
        (root / "joint_head.safetensors").write_bytes(b"")
        clef = resolve_model(root)
        self.assertEqual((clef.descriptor.id, clef.descriptor.modality), ("qwen36", "text"))
        self.assertEqual(clef.decision_head, "clef")
        self.assertTrue(checkpoint_decides(clef))
        self.assertEqual(display_for(clef), ("Clef", "27B"))
        self.assertEqual(default_model_id(clef), "clef")


def _laya_ready():
    return (FIXTURE / "model.safetensors").exists() and BINARY.exists()


class LayaTinyEndToEnd(unittest.TestCase):
    """The real engine behind the real gateway, on the tiny fixture."""

    @classmethod
    def setUpClass(cls):
        if not _laya_ready():
            message = "laya or laya_tiny missing: run make laya-tiny-check"
            if REQUIRED:
                raise AssertionError(message)
            raise unittest.SkipTest(message)
        cls.ref = {case["name"]: case for case in
                   json.loads((FIXTURE / "ref.json").read_text(encoding="utf-8"))["cases"]}

    def setUp(self):
        engine = Engine(BINARY, FIXTURE, env=dict(os.environ, OMP_NUM_THREADS="2"),
                        family=family_by_id("laya"))
        self.addCleanup(engine.process.stdout.close)
        self.addCleanup(engine.close)
        self.server = APIServer(("127.0.0.1", 0), engine, "laya-tiny")
        self.addCleanup(self.server.server_close)
        self.addCleanup(self.server.shutdown)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()
        self.base = f"http://127.0.0.1:{self.server.server_port}"

    def post(self, path, body):
        request = Request(self.base + path, data=json.dumps(body).encode(),
                          headers={"Content-Type": "application/json"})
        with urlopen(request, timeout=120) as response:
            return json.loads(response.read()), dict(response.headers)

    def test_systemone_answers_with_the_reference_probabilities(self):
        # the cases the route accepts as Jev requests (it refuses a bare number as a
        # score level, which the engine-level oracle still covers)
        for name in ("readme_ticket", "plain_two_options", "conversation_truncated_left",
                     "json_instructions_and_state", "special_text", "unicode"):
            case = self.ref[name]
            with self.subTest(case=name):
                out, headers = self.post("/v1/systemone", {"model": "jev-latest",
                                                           "state": case["state"],
                                                           "questions": case["questions"]})
                for want in case["answers"]:
                    got = out["answers"][want["id"]]
                    probs = want["probs"]
                    if got["type"] == "noul":
                        self.assertAlmostEqual(got["noul"], probs[1], delta=2e-5)
                        continue
                    values = list(got["probabilities"].values())
                    self.assertEqual(len(values), len(probs))
                    for a, b in zip(values, probs):
                        self.assertAlmostEqual(a, b, delta=2e-5)
                    if got["type"] == "choice":
                        labels = list(case["questions"][want["id"]]["criteria"])
                        self.assertEqual(got["choice"], labels[probs.index(max(probs))])
                self.assertEqual(out["usage"]["input_tokens"],
                                 sum(len(a["ids"]) for a in case["answers"]))
                self.assertEqual(out["usage"]["output_tokens"], 0)
                self.assertIn("x-colibri-engine-ms", headers)

    def test_a_question_whose_options_do_not_fit_is_a_422(self):
        case = self.ref["too_many_options"]
        request = Request(self.base + "/v1/systemone",
                          data=json.dumps({"state": case["state"], "questions": case["questions"]}).encode(),
                          headers={"Content-Type": "application/json"})
        with self.assertRaises(HTTPError) as caught:
            urlopen(request, timeout=60)
        self.assertEqual(caught.exception.code, 422)
        error = json.loads(caught.exception.read())["error"]
        self.assertEqual(error["param"], "questions.bucket")
        self.assertIn("only 38 of its 40 option markers fit", error["message"])

    def test_chat_is_refused(self):
        request = Request(self.base + "/v1/chat/completions",
                          data=json.dumps({"model": "laya-tiny",
                                           "messages": [{"role": "user", "content": "hi"}]}).encode(),
                          headers={"Content-Type": "application/json"})
        with self.assertRaises(HTTPError) as caught:
            urlopen(request, timeout=10)
        self.assertEqual(caught.exception.code, 400)

    def test_latency_gateway_and_engine(self):
        """Not a gate: the numbers the report quotes, measured the same way every
        time. Wall time at the client, the engine's own compute time, and the rest
        (HTTP, JSON, the pipe) as the gateway's share."""
        case = self.ref["readme_ticket"]
        body = {"model": "jev-latest", "state": case["state"], "questions": case["questions"]}
        self.post("/v1/systemone", body)                       # warm
        walls, engines = [], []
        for _ in range(10):
            started = time.perf_counter()
            _, headers = self.post("/v1/systemone", body)
            walls.append((time.perf_counter() - started) * 1e3)
            engines.append(float(headers["x-colibri-engine-ms"]))
        walls.sort(), engines.sort()
        wall, engine = walls[len(walls) // 2], engines[len(engines) // 2]
        sys.stderr.write(f"\n[laya tiny e2e] /v1/systemone, 4 questions: median {wall:.1f} ms at the "
                         f"client, {engine:.1f} ms in the engine, {wall - engine:.1f} ms gateway\n")
        self.assertGreater(wall, engine)


def _gliner_ready():
    return (GLINER_FIXTURE / "model.safetensors").exists() and GLINER_BINARY.exists()


class GlinerDecideTinyEndToEnd(unittest.TestCase):
    """The gliner_decide engine behind the real gateway, on its tiny fixture."""

    @classmethod
    def setUpClass(cls):
        if not _gliner_ready():
            message = "gliner_decide or gliner_decide_tiny missing: run make gliner-decide-tiny-check"
            if GLINER_REQUIRED:
                raise AssertionError(message)
            raise unittest.SkipTest(message)
        ref = json.loads((GLINER_FIXTURE / "ref.json").read_text(encoding="utf-8"))
        cls.max_len = ref["max_len"]
        cls.ref = {case["name"]: case for case in ref["cases"]}

    def setUp(self):
        env = dict(os.environ, OMP_NUM_THREADS="2", COLI_GLINER_MAX_LEN=str(self.max_len))
        engine = Engine(GLINER_BINARY, GLINER_FIXTURE, env=env, family=family_by_id("gliner_decide"))
        self.addCleanup(engine.process.stdout.close)
        self.addCleanup(engine.close)
        self.server = APIServer(("127.0.0.1", 0), engine, "gliner-decide-tiny")
        self.addCleanup(self.server.server_close)
        self.addCleanup(self.server.shutdown)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()
        self.base = f"http://127.0.0.1:{self.server.server_port}"

    def post(self, path, body):
        request = Request(self.base + path, data=json.dumps(body).encode(),
                          headers={"Content-Type": "application/json"})
        with urlopen(request, timeout=120) as response:
            return json.loads(response.read()), dict(response.headers)

    def test_systemone_answers_with_the_reference_probabilities(self):
        # the cases the route accepts as Jev requests (it refuses a bare number as a
        # criterion, which the engine-level oracle still covers)
        for name in ("ticket_three_types", "two_options_and_noul_criteria",
                     "twelve_labels_no_instructions", "long_state_cut", "conversation",
                     "json_state_and_instructions", "special_tokens_and_whitespace",
                     "urls_mail_handles", "unicode", "many_options", "empty_state"):
            case = self.ref[name]
            with self.subTest(case=name):
                out, headers = self.post("/v1/systemone", {"model": "jev-latest",
                                                           "state": case["state"],
                                                           "questions": case["questions"]})
                self.assertEqual(out["model"], "gliner-decide-tiny")
                for want in case["answers"]:
                    got = out["answers"][want["id"]]
                    probs = want["probs"]
                    if got["type"] == "noul":
                        self.assertAlmostEqual(got["noul"], probs[1], delta=2e-5)
                        continue
                    values = list(got["probabilities"].values())
                    self.assertEqual(len(values), len(probs))
                    for a, b in zip(values, probs):
                        self.assertAlmostEqual(a, b, delta=2e-5)
                    if got["type"] == "choice":
                        labels = list(case["questions"][want["id"]]["criteria"])
                        self.assertEqual(got["choice"], labels[probs.index(max(probs))])
                    else:
                        expected = sum(i * p for i, p in enumerate(probs))
                        self.assertAlmostEqual(got["score"], expected, delta=1e-4)
                # one sequence for the whole request
                self.assertEqual(out["usage"]["input_tokens"], len(case["ids"]))
                self.assertEqual(out["usage"]["output_tokens"], 0)
                self.assertIn("x-colibri-engine-ms", headers)

    def test_questions_that_do_not_fit_are_a_422(self):
        case = self.ref["questions_over_max_len"]
        request = Request(self.base + "/v1/systemone",
                          data=json.dumps({"state": case["state"], "questions": case["questions"]}).encode(),
                          headers={"Content-Type": "application/json"})
        with self.assertRaises(HTTPError) as caught:
            urlopen(request, timeout=60)
        self.assertEqual(caught.exception.code, 422)
        error = json.loads(caught.exception.read())["error"]
        self.assertEqual(error["param"], "questions")
        self.assertIn(case["refused"].split(": ", 1)[1], error["message"])

    def test_chat_is_refused(self):
        request = Request(self.base + "/v1/chat/completions",
                          data=json.dumps({"model": "gliner-decide-tiny",
                                           "messages": [{"role": "user", "content": "hi"}]}).encode(),
                          headers={"Content-Type": "application/json"})
        with self.assertRaises(HTTPError) as caught:
            urlopen(request, timeout=10)
        self.assertEqual(caught.exception.code, 400)

    def test_latency_gateway_and_engine(self):
        """Not a gate: wall time at the client, the engine's own compute time, and
        the rest (HTTP, JSON, the pipe) as the gateway's share."""
        case = self.ref["ticket_three_types"]
        body = {"model": "jev-latest", "state": case["state"], "questions": case["questions"]}
        self.post("/v1/systemone", body)                       # warm
        walls, engines = [], []
        for _ in range(10):
            started = time.perf_counter()
            _, headers = self.post("/v1/systemone", body)
            walls.append((time.perf_counter() - started) * 1e3)
            engines.append(float(headers["x-colibri-engine-ms"]))
        walls.sort(), engines.sort()
        wall, engine = walls[len(walls) // 2], engines[len(engines) // 2]
        sys.stderr.write(f"\n[gliner_decide tiny e2e] /v1/systemone, 4 questions: median {wall:.1f} ms "
                         f"at the client, {engine:.1f} ms in the engine, {wall - engine:.1f} ms gateway\n")
        self.assertGreater(wall, engine)


def _clef_ready():
    return CLEF_REF.exists() and (CLEF_CONTAINER / "joint_head.safetensors").exists() and QWEN36.exists()


class ClefTinyEndToEnd(unittest.TestCase):
    """qwen36 with Clef's head behind the real gateway, on the tiny fixture."""

    # the cases the route accepts as Jev requests (it refuses a bare number as a
    # criterion or an instruction, which the engine-level oracle still covers)
    CASES = ("model_card_invoice", "model_card_systemone", "ticket_four_questions", "noul_variants",
             "json_state", "conversation", "special_text", "many_questions", "single_option",
             "empty_state")

    @classmethod
    def setUpClass(cls):
        if not _clef_ready():
            message = "qwen36, clef_tiny or clef_tiny_c missing: run make clef-tiny-check"
            if CLEF_REQUIRED:
                raise AssertionError(message)
            raise unittest.SkipTest(message)
        cls.ref = {case["name"]: case for case in
                   json.loads(CLEF_REF.read_text(encoding="utf-8"))["cases"]}

    def setUp(self):
        arch = patch.object(openai_server, "ARCH", "qwen36")
        arch.start()
        self.addCleanup(arch.stop)
        engine = Engine(QWEN36, CLEF_CONTAINER, family=family_by_id("qwen36"),
                        env=dict(os.environ, COLI_DENSE_I8="0", OMP_NUM_THREADS="2",
                                 COLI_NO_OMP_TUNE="1"))
        self.engine = engine
        self.addCleanup(engine.process.stdout.close)
        self.addCleanup(engine.close)
        self.server = APIServer(("127.0.0.1", 0), engine, "clef-tiny")
        self.addCleanup(self.server.server_close)
        self.addCleanup(self.server.shutdown)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()
        self.base = f"http://127.0.0.1:{self.server.server_port}"

    def post(self, path, body):
        request = Request(self.base + path, data=json.dumps(body).encode(),
                          headers={"Content-Type": "application/json"})
        with urlopen(request, timeout=300) as response:
            return json.loads(response.read()), dict(response.headers)

    def check(self, name, out):
        case = self.ref[name]
        for want in case["answers"]:
            got = out["answers"][want["id"]]
            p = dict(zip(want["option_ids"], want["probs"]))
            if got["type"] == "noul":
                self.assertAlmostEqual(got["noul"], p["true"], delta=2e-5, msg=name)
                continue
            self.assertEqual(set(got["probabilities"]), set(p), name)
            for label, value in got["probabilities"].items():
                self.assertAlmostEqual(value, p[label], delta=2e-5, msg=f"{name}.{want['id']}.{label}")
            if got["type"] == "choice":
                self.assertEqual(got["choice"], max(p, key=p.get), name)
        self.assertEqual(out["usage"], {"input_tokens": len(case["input_ids"]), "output_tokens": 0,
                                        "cost": 0})

    def test_caps_say_decide_raw_and_chat(self):
        self.assertTrue(self.engine.decides)
        self.assertTrue(self.engine.chats)
        self.assertEqual(self.engine.decide_record, "raw")
        with urlopen(self.base + "/v1/models", timeout=10) as response:
            card = json.loads(response.read())
        self.assertEqual(card["data"][0]["capabilities"], ["chat", "systemone"])

    def test_systemone_answers_with_clefs_probabilities(self):
        for name in self.CASES:
            case = self.ref[name]
            with self.subTest(case=name):
                out, headers = self.post("/v1/systemone", {"model": "jev-latest", "state": case["state"],
                                                           "questions": case["questions"]})
                self.check(name, out)
                self.assertIn("x-colibri-engine-ms", headers)

    def test_the_same_engine_chats_between_two_decisions(self):
        case = self.ref["model_card_systemone"]
        body = {"state": case["state"], "questions": case["questions"]}
        first, _ = self.post("/v1/systemone", body)
        chat, _ = self.post("/v1/chat/completions", {"model": "clef-tiny", "max_tokens": 6,
                                                      "temperature": 0,
                                                      "messages": [{"role": "user", "content": "hi"}]})
        self.assertEqual(chat["object"], "chat.completion")
        self.assertGreater(chat["usage"]["completion_tokens"], 0)
        second, _ = self.post("/v1/systemone", body)
        self.assertEqual(second["answers"], first["answers"])
        self.check("model_card_systemone", second)

    def test_the_gateway_keeps_clefs_trunk_in_f16_when_the_budget_holds_it(self):
        """int8 moves Clef's probabilities by up to 0.22 on the real checkpoint, f16 by
        0.012 (docs/clef.md): the gateway picks f16 when the planner's RAM budget
        holds it, int8 otherwise, and an operator's own COLI_DENSE_BITS wins."""
        env = {}
        openai_server.decision_head_env(env, str(CLEF_CONTAINER))
        self.assertEqual(env.get("COLI_DENSE_BITS"), "16")
        env = {"RAM_GB": "0.001"}
        self.assertIn("int8", openai_server.decision_head_env(env, str(CLEF_CONTAINER)))
        self.assertNotIn("COLI_DENSE_BITS", env)
        env = {"COLI_DENSE_BITS": "4"}
        self.assertIsNone(openai_server.decision_head_env(env, str(CLEF_CONTAINER)))
        self.assertEqual(env, {"COLI_DENSE_BITS": "4"})
        self.assertIsNone(openai_server.decision_head_env({}, str(HERE / "laya_tiny")))

    def test_a_schema_past_the_budget_is_a_422(self):
        """COLI_CLEF_MAX_LEN is the reference's max_length: past it the schema alone
        does not fit and the request is the caller's 422, not a 500."""
        self.engine.close()
        engine = Engine(QWEN36, CLEF_CONTAINER, family=family_by_id("qwen36"),
                        env=dict(os.environ, COLI_DENSE_I8="0", OMP_NUM_THREADS="2",
                                 COLI_NO_OMP_TUNE="1", COLI_CLEF_MAX_LEN="600"))
        self.addCleanup(engine.process.stdout.close)
        self.addCleanup(engine.close)
        self.server.engine = engine
        case = self.ref["schema_too_long"]
        request = Request(self.base + "/v1/systemone",
                          data=json.dumps({"state": case["state"], "questions": case["questions"]}).encode(),
                          headers={"Content-Type": "application/json"})
        with self.assertRaises(HTTPError) as caught:
            urlopen(request, timeout=120)
        self.assertEqual(caught.exception.code, 422)
        self.assertIn("1389", json.loads(caught.exception.read())["error"]["message"])

    def test_latency_gateway_and_engine(self):
        """Not a gate: wall time at the client against the engine's own time."""
        case = self.ref["ticket_four_questions"]
        body = {"model": "jev-latest", "state": case["state"], "questions": case["questions"]}
        self.post("/v1/systemone", body)
        walls, engines = [], []
        for _ in range(5):
            started = time.perf_counter()
            _, headers = self.post("/v1/systemone", body)
            walls.append((time.perf_counter() - started) * 1e3)
            engines.append(float(headers["x-colibri-engine-ms"]))
        walls.sort(), engines.sort()
        wall, engine = walls[len(walls) // 2], engines[len(engines) // 2]
        sys.stderr.write(f"\n[clef tiny e2e] /v1/systemone, 4 questions: median {wall:.1f} ms at the "
                         f"client, {engine:.1f} ms in the engine, {wall - engine:.1f} ms gateway\n")
        self.assertGreater(wall, engine)


if __name__ == "__main__":
    unittest.main()