Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
registry: a Clef checkpoint is qwen36 with a decision head; serve and SDK tests
JustVugg2edb1184dd6b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex d4cddf77..edf8e2e4 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -1169,11 +1169,12 @@ jobs:- uses: actions/setup-node@v4with:node-version: '20'- - name: Install the pinned SDKs+ - name: Install the pinned SDKs, and torch for the Clef fixture's converterrun: |pip install -r c/tools/requirements-jev-sdk.txt+ pip install numpy -r c/tools/oracle-requirements.txtcd c/tools/jev-sdk && npm ci- - name: Both SDKs against a language model (tiny MiMo) and a decision engine (tiny Laya)+ - name: Both SDKs against a language model (tiny MiMo), a decision engine (tiny Laya) and Clefrun: make -C c jev-sdk-checkqwenimage-tiny-check:diff --git a/c/Makefile b/c/Makefileindex 96ee466f..1a0ca727 100644--- a/c/Makefile+++ b/c/Makefile@@ -1318,11 +1318,12 @@ clef-tiny-check: clef-tiny-generate qwen36$(EXE)CLEF_TINY_REQUIRED=1 $(PYTHON) -m unittest -v tests.test_clef_tiny tests.test_decision_serve# TypeSafe's own SDKs (Python and TypeScript), unmodified, against `coli serve`-# on a language model (tiny MiMo) and a decision engine (tiny Laya): switching-# from Jev is a base-URL change. Needs the pinned SDKs first:+# on a language model (tiny MiMo), a decision engine (tiny Laya) and a chat+# model with a decision head (tiny Clef on qwen36): switching from Jev is a+# base-URL change. Needs the pinned SDKs first, and torch for the Clef converter:# pip install -r tools/requirements-jev-sdk.txt && (cd tools/jev-sdk && npm ci).PHONY: jev-sdk-check-jev-sdk-check: mimo$(EXE) laya$(EXE) mimo-tiny-generate laya-tiny-generate+jev-sdk-check: mimo$(EXE) laya$(EXE) qwen36$(EXE) mimo-tiny-generate laya-tiny-generate clef-tiny-generateJEV_SDK_REQUIRED=1 $(PYTHON) -m unittest -v tests.test_jev_sdk# In make check: every format and ragged edges against the naive product, anddiff --git a/c/coli b/c/coliindex 28a0ee85..c20f51a7 100755--- a/c/coli+++ b/c/coli@@ -273,6 +273,21 @@ def is_decision_model(model):except (FamilyConfigError, UnknownFamilyError, OSError, TypeError, ValueError):return False+def decision_head_line(model, resolved):+ """`coli info`'s line for a decision head over a chat model (Clef): what it+ adds, and that the chat stays."""+ from family_registry import decision_head_of+ head = decision_head_of(resolved)+ try:+ with open(os.path.join(model, head.files[0]), encoding="utf-8") as f:+ cfg = json.load(f)+ shape = (f"{cfg.get('routing_layers')} routing + {cfg.get('layers')} decoder layers, "+ f"width {cfg.get('width')}")+ except (OSError, ValueError, AttributeError):+ shape = "head config unreadable"+ return (f"{head.display_name} joint schema head ({shape}): POST /v1/systemone answers "+ f"natively; chat, completions and messages work as on any {resolved.descriptor.id} model")+def decision_refusal(family):"""What every chatting command says about a decision model, and where to go."""return (f"{C.yel}{family.display_name} is a decision model:{C.r} it answers typed questions "@@ -1290,6 +1305,8 @@ def cmd_info(a):sts=[x for x in os.listdir(a.model) if x.endswith('.safetensors')]sz=sum(os.path.getsize(os.path.join(a.model,x)) for x in sts)row("shards", f"{len(sts)} files · {sz/1e9:.0f} GB on disk")+ if resolved is not None and resolved.decision_head:+ row("decision", decision_head_line(a.model, resolved))else:# No config.json means no family, and no family means no engine: say so# instead of letting the GLM default below stand in. A directory of shardsdiff --git a/c/doctor.py b/c/doctor.pyindex ce78fa0d..4490dffa 100644--- a/c/doctor.py+++ b/c/doctor.py@@ -550,6 +550,25 @@ def missing_shared_libraries(engine_path):for line in result.stdout.splitlines() if "not found" in line})+def _decision_head_check(model, resolved):+ """A decision head beside a chat model's shards (Clef): its config must read+ and fit the backbone, or the engine refuses to start."""+ from family_registry import decision_head_of+ head = decision_head_of(resolved)+ try:+ config = json.loads((Path(model) / head.files[0]).read_text(encoding="utf-8"))+ hidden = config.get("hidden_size")+ except (OSError, ValueError, AttributeError) as error:+ return _check("model.decision_head", "fail", f"{head.files[0]} is unreadable: {error}")+ want = resolved.family_config.get("hidden_size")+ if hidden != want:+ return _check("model.decision_head", "fail",+ f"{head.display_name} head reads hidden size {hidden}, the backbone has {want}")+ return _check("model.decision_head", "pass",+ f"{head.display_name} decision head: POST /v1/systemone answers natively",+ head=head.id, hidden_size=hidden)++def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,engine_path, available_memory=None, available_disk=None, gpus=None,linkage=None, deep=False, mirror_dir=None, kv_slots=1,@@ -582,6 +601,8 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,family_id=resolved.descriptor.id,model_type=resolved.model_type,descriptor=public_metadata(resolved.descriptor)))+ if resolved.decision_head:+ checks.append(_decision_head_check(model, resolved))except (FamilyConfigError, UnknownFamilyError) as error:checks.append(_check("model.family", "fail", str(error)))else:diff --git a/c/family_registry.py b/c/family_registry.pyindex 0b93cae5..cd575597 100644--- a/c/family_registry.py+++ b/c/family_registry.py@@ -179,6 +179,45 @@ class ResolvedFamily:config: dictfamily_config: dictmodel_dir: str+ # A decision head over a text family's backbone (Cloudflare's Clef: a qwen36+ # container with joint_head_config.json and joint_head.safetensors). The+ # engine then answers POST /v1/systemone natively AND chats; the family, its+ # planner and its limits stay the backbone's. "" for every other checkpoint.+ decision_head: str = ""+++@dataclass(frozen=True, slots=True)+class DecisionHead:+ """A decision head a text engine loads beside its backbone (docs/clef.md)."""+ id: str+ family: str # the engine that runs the backbone and the head+ files: tuple # all of them next to the shards+ display_name: str+ model_id: str+ # (geometry, display_scale): the backbone sizes the head ships on+ scales: tuple = ()+++DECISION_HEADS = (+ DecisionHead("clef", "qwen36", ("joint_head_config.json", "joint_head.safetensors"),+ "Clef", "clef",+ scales=(((("num_hidden_layers", 64), ("hidden_size", 5120),+ ("intermediate_size", 17408)), "27B"),)),+)+++def decision_head_of(resolved):+ """The DecisionHead a resolved checkpoint carries, or None."""+ for head in DECISION_HEADS:+ if head.id == resolved.decision_head:+ return head+ return None+++def checkpoint_decides(resolved):+ """True when the checkpoint answers POST /v1/systemone natively: a decision+ family (Laya), or a text family with a decision head (Clef)."""+ return resolved.descriptor.modality == "decision" or bool(resolved.decision_head)def _required_int(config, key, family, minimum=1):@@ -1874,8 +1913,18 @@ def resolve_model(model_dir):family_config = config.get("text_config", config)if not isinstance(family_config, dict):raise FamilyConfigError(f"{family.id}: text_config is not an object")+ head = ""+ for candidate in DECISION_HEADS:+ present = [name for name in candidate.files if (model / name).is_file()]+ if candidate.family == family.id and present:+ if len(present) != len(candidate.files):+ raise FamilyConfigError(f"{model}: {present[0]} without "+ f"{', '.join(n for n in candidate.files if n not in present)}: "+ f"a {candidate.display_name} decision head needs all of "+ f"{', '.join(candidate.files)}")+ head = candidate.idreturn ResolvedFamily(family, _normalize_model_type(config.get("model_type")),- config, family_config, str(model))+ config, family_config, str(model), head)def default_model_id(resolved):@@ -1883,6 +1932,9 @@ def default_model_id(resolved):one of its own, else the family's."""family = resolved.descriptorconfig = resolved.family_config+ head = decision_head_of(resolved)+ if head:+ return head.model_idfor variant in family.display_variants:if variant.model_id and all(config.get(key) == value for key, value in variant.geometry):return variant.model_id@@ -1899,9 +1951,15 @@ def display_for(resolved):count."""family = resolved.descriptor+ config = resolved.family_config+ head = decision_head_of(resolved)+ if head:+ for geometry, scale in head.scales:+ if all(config.get(key) == value for key, value in geometry):+ return head.display_name, scale+ return head.display_name, ""if not family.display_variants:return family.display_name, family.display_scale- config = resolved.family_configfor variant in family.display_variants:if all(config.get(key) == value for key, value in variant.geometry):return variant.display_name, variant.display_scalediff --git a/c/openai_server.py b/c/openai_server.pyindex a4c6c4d0..2494c857 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -24,7 +24,7 @@ import v4_dsml # vendored DeepSeek V4 DSML reference primitimport v41_dsml # ...and V4.1's, whose tag names differ by a spaceimport image_engine # the qwenimage serve protocol, PNG and request rulesfrom family_registry import (FamilyConfigError, UnknownFamilyError, family_by_id,- family_ids, resolve_model)+ display_for, family_ids, resolve_model)from family_registry import default_model_id as registry_default_model_idfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Path@@ -5942,7 +5942,7 @@ class APIServer(ThreadingHTTPServer):"""The served model as Jev's GET /v1/models lists it. colibri does not know amodel's release date; it gives the day this server started."""try:- name = family_by_id(ARCH).display_name+ name = getattr(self, "display_name", None) or family_by_id(ARCH).display_nameexcept Exception:name = self.model_idreturn {"name": self.model_id,@@ -7975,6 +7975,10 @@ def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,pending_model_id = family.default_model_idmodel_id = pending_model_idserver.model_id = model_id+ try: # what the checkpoint on disk is called (a Clef, a 27B), for the Jev card+ server.display_name = display_for(resolve_model(model))[0]+ except Exception:+ server.display_name = Noneif kv_slots > family.limits.max_kv_slots:raise ValueError(f"{family.id} engine supports at most "f"{family.limits.max_kv_slots} KV slot(s)")diff --git a/c/tests/test_decision_serve.py b/c/tests/test_decision_serve.pyindex bd32f2fd..f2f519b0 100644--- a/c/tests/test_decision_serve.py+++ b/c/tests/test_decision_serve.py@@ -16,9 +16,15 @@ of scoring options through the logprob channel. Three layers are pinned here:gateway: the probabilities that come back over HTTP arethe `laya` package's (laya_tiny/ref.json), and thelatency is split into gateway and engine time.--LayaTinyEndToEnd is skipped without the fixture and the binary-(`make laya-tiny-check` builds both and turns the skip into a failure).+ ClefTinyEndToEnd qwen36 with Clef's decision head (clef_tiny_c) behind the+ real gateway: CAPS decide=1 decide_record=raw and no+ chat=0, /v1/systemone gives Clef's own probabilities+ (clef_tiny/ref.json), and the same engine still chats,+ with a decision before and after the chat unchanged.++LayaTinyEndToEnd and ClefTinyEndToEnd are skipped without their fixture and+binary (`make laya-tiny-check` and `make clef-tiny-check` build them and turn+the skip into a failure)."""import jsonimport math@@ -43,6 +49,10 @@ from test_openai_server import BlockingStream, FakeProcess # noqa: E402FIXTURE = HERE / "laya_tiny"BINARY = HERE / ("laya.exe" if os.name == "nt" else "laya")REQUIRED = os.environ.get("LAYA_TINY_REQUIRED") == "1"+CLEF_REF = HERE / "clef_tiny" / "ref.json"+CLEF_CONTAINER = HERE / "clef_tiny_c"+QWEN36 = HERE / ("qwen36.exe" if os.name == "nt" else "qwen36")+CLEF_REQUIRED = os.environ.get("CLEF_TINY_REQUIRED") == "1"TICKET = {"from": "[email protected]", "subject": "Duplicate charge on invoice #4411","body": "Hi, we were billed twice for March. Please refund the duplicate today or we "@@ -238,6 +248,40 @@ class DecisionGateway(unittest.TestCase):self.assertEqual(s["instructions"], "Rate this on the scale below.")self.assertEqual(d["instructions"], "Is this true?")+ def test_a_raw_form_engine_gets_the_callers_values(self):+ """An engine that said decide_record=raw (Clef) renders the request the way+ its reference does: it gets the caller's values, not the gateway's+ defaults, and JSON with sorted keys."""+ engine = FakeDecisionEngine({"q": [0.5, 0.5], "c": [0.2, 0.3, 0.5], "s": [0.5, 0.5],+ "d": [0.5, 0.5]})+ engine.decide_record = "raw"+ engine.chats = True+ self.serve(engine)+ self.post("/v1/systemone", {"state": {"ticket": 4711, "text": "café", "ok": True},+ "questions": {+ "q": {"type": "noul", "instructions": {"ask": "money?", "a": 1},+ "criteria": {"true": "about a payment", "false": None}},+ "c": {"type": "choice", "instructions": "",+ "criteria": {"b": None, "a": {"z": 1, "y": [2.5]}, "c": ""}},+ "s": {"type": "score", "criteria": ["low", {"n": 2}]},+ "d": {"type": "noul"}}})+ record = self.engine.records[0]+ self.assertEqual(record["record"], "raw")+ self.assertEqual(record["state"], '{"ok":true,"text":"café","ticket":4711}')+ q, c, s, d = record["questions"]+ self.assertEqual(q["instructions"], '{"a":1,"ask":"money?"}')+ self.assertEqual(q["options"], [{"label": "false", "text": None},+ {"label": "true", "text": "about a payment"}])+ self.assertEqual(c["instructions"], "") # not replaced+ self.assertEqual(c["options"], [{"label": "b", "text": None},+ {"label": "a", "text": '{"y":[2.5],"z":1}', "json": True},+ {"label": "c", "text": ""}])+ self.assertEqual(s["instructions"], None)+ self.assertEqual(s["options"][1], {"label": "1", "text": '{"n":2}', "json": True})+ self.assertEqual(d["options"], [{"label": "false"}, {"label": "true"}]) # undescribed+ with urlopen(self.base + "/v1/models", timeout=10) as response:+ self.assertEqual(json.loads(response.read())["data"][0]["capabilities"], ["chat", "systemone"])+def test_a_list_state_is_a_conversation(self):self.serve(FakeDecisionEngine({"q": [0.5, 0.5]}))turns = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]@@ -369,6 +413,32 @@ class DecisionRegistryAndLauncher(unittest.TestCase):self.assertIn("POST /v1/systemone", refused.stdout + refused.stderr)+ def test_the_registry_knows_a_clef_checkpoint(self):+ import tempfile+ from family_registry import (FamilyConfigError, checkpoint_decides, default_model_id,+ display_for, resolve_model)+ directory = tempfile.TemporaryDirectory()+ self.addCleanup(directory.cleanup)+ root = Path(directory.name)+ text = {"model_type": "qwen3_5_text", "num_hidden_layers": 64, "hidden_size": 5120,+ "intermediate_size": 17408}+ (root / "config.json").write_text(json.dumps({"model_type": "qwen3_5", "text_config": text}),+ encoding="utf-8")+ stock = resolve_model(root)+ self.assertEqual((stock.decision_head, display_for(stock)), ("", ("Qwen3.8-27B", "27B")))+ self.assertFalse(checkpoint_decides(stock))+ (root / "joint_head_config.json").write_text(json.dumps({"hidden_size": 5120}), encoding="utf-8")+ with self.assertRaises(FamilyConfigError): # half a head+ resolve_model(root)+ (root / "joint_head.safetensors").write_bytes(b"")+ clef = resolve_model(root)+ self.assertEqual((clef.descriptor.id, clef.descriptor.modality), ("qwen36", "text"))+ self.assertEqual(clef.decision_head, "clef")+ self.assertTrue(checkpoint_decides(clef))+ self.assertEqual(display_for(clef), ("Clef", "27B"))+ self.assertEqual(default_model_id(clef), "clef")++def _laya_ready():return (FIXTURE / "model.safetensors").exists() and BINARY.exists()@@ -472,5 +542,133 @@ class LayaTinyEndToEnd(unittest.TestCase):self.assertGreater(wall, engine)+def _clef_ready():+ return CLEF_REF.exists() and (CLEF_CONTAINER / "joint_head.safetensors").exists() and QWEN36.exists()+++class ClefTinyEndToEnd(unittest.TestCase):+ """qwen36 with Clef's head behind the real gateway, on the tiny fixture."""++ # the cases the route accepts as Jev requests (it refuses a bare number as a+ # criterion or an instruction, which the engine-level oracle still covers)+ CASES = ("model_card_invoice", "model_card_systemone", "ticket_four_questions", "noul_variants",+ "json_state", "conversation", "special_text", "many_questions", "single_option",+ "empty_state")++ @classmethod+ def setUpClass(cls):+ if not _clef_ready():+ message = "qwen36, clef_tiny or clef_tiny_c missing: run make clef-tiny-check"+ if CLEF_REQUIRED:+ raise AssertionError(message)+ raise unittest.SkipTest(message)+ cls.ref = {case["name"]: case for case in+ json.loads(CLEF_REF.read_text(encoding="utf-8"))["cases"]}++ def setUp(self):+ arch = patch.object(openai_server, "ARCH", "qwen36")+ arch.start()+ self.addCleanup(arch.stop)+ engine = Engine(QWEN36, CLEF_CONTAINER, family=family_by_id("qwen36"),+ env=dict(os.environ, COLI_DENSE_I8="0", OMP_NUM_THREADS="2",+ COLI_NO_OMP_TUNE="1"))+ self.engine = engine+ self.addCleanup(engine.process.stdout.close)+ self.addCleanup(engine.close)+ self.server = APIServer(("127.0.0.1", 0), engine, "clef-tiny")+ self.addCleanup(self.server.server_close)+ self.addCleanup(self.server.shutdown)+ threading.Thread(target=self.server.serve_forever, daemon=True).start()+ self.base = f"http://127.0.0.1:{self.server.server_port}"++ def post(self, path, body):+ request = Request(self.base + path, data=json.dumps(body).encode(),+ headers={"Content-Type": "application/json"})+ with urlopen(request, timeout=300) as response:+ return json.loads(response.read()), dict(response.headers)++ def check(self, name, out):+ case = self.ref[name]+ for want in case["answers"]:+ got = out["answers"][want["id"]]+ p = dict(zip(want["option_ids"], want["probs"]))+ if got["type"] == "noul":+ self.assertAlmostEqual(got["noul"], p["true"], delta=2e-5, msg=name)+ continue+ self.assertEqual(set(got["probabilities"]), set(p), name)+ for label, value in got["probabilities"].items():+ self.assertAlmostEqual(value, p[label], delta=2e-5, msg=f"{name}.{want['id']}.{label}")+ if got["type"] == "choice":+ self.assertEqual(got["choice"], max(p, key=p.get), name)+ self.assertEqual(out["usage"], {"input_tokens": len(case["input_ids"]), "output_tokens": 0,+ "cost": 0})++ def test_caps_say_decide_raw_and_chat(self):+ self.assertTrue(self.engine.decides)+ self.assertTrue(self.engine.chats)+ self.assertEqual(self.engine.decide_record, "raw")+ with urlopen(self.base + "/v1/models", timeout=10) as response:+ card = json.loads(response.read())+ self.assertEqual(card["data"][0]["capabilities"], ["chat", "systemone"])++ def test_systemone_answers_with_clefs_probabilities(self):+ for name in self.CASES:+ case = self.ref[name]+ with self.subTest(case=name):+ out, headers = self.post("/v1/systemone", {"model": "jev-latest", "state": case["state"],+ "questions": case["questions"]})+ self.check(name, out)+ self.assertIn("x-colibri-engine-ms", headers)++ def test_the_same_engine_chats_between_two_decisions(self):+ case = self.ref["model_card_systemone"]+ body = {"state": case["state"], "questions": case["questions"]}+ first, _ = self.post("/v1/systemone", body)+ chat, _ = self.post("/v1/chat/completions", {"model": "clef-tiny", "max_tokens": 6,+ "temperature": 0,+ "messages": [{"role": "user", "content": "hi"}]})+ self.assertEqual(chat["object"], "chat.completion")+ self.assertGreater(chat["usage"]["completion_tokens"], 0)+ second, _ = self.post("/v1/systemone", body)+ self.assertEqual(second["answers"], first["answers"])+ self.check("model_card_systemone", second)++ def test_a_schema_past_the_budget_is_a_422(self):+ """COLI_CLEF_MAX_LEN is the reference's max_length: past it the schema alone+ does not fit and the request is the caller's 422, not a 500."""+ self.engine.close()+ engine = Engine(QWEN36, CLEF_CONTAINER, family=family_by_id("qwen36"),+ env=dict(os.environ, COLI_DENSE_I8="0", OMP_NUM_THREADS="2",+ COLI_NO_OMP_TUNE="1", COLI_CLEF_MAX_LEN="600"))+ self.addCleanup(engine.process.stdout.close)+ self.addCleanup(engine.close)+ self.server.engine = engine+ case = self.ref["schema_too_long"]+ request = Request(self.base + "/v1/systemone",+ data=json.dumps({"state": case["state"], "questions": case["questions"]}).encode(),+ headers={"Content-Type": "application/json"})+ with self.assertRaises(HTTPError) as caught:+ urlopen(request, timeout=120)+ self.assertEqual(caught.exception.code, 422)+ self.assertIn("1389", json.loads(caught.exception.read())["error"]["message"])++ def test_latency_gateway_and_engine(self):+ """Not a gate: wall time at the client against the engine's own time."""+ case = self.ref["ticket_four_questions"]+ body = {"model": "jev-latest", "state": case["state"], "questions": case["questions"]}+ self.post("/v1/systemone", body)+ walls, engines = [], []+ for _ in range(5):+ started = time.perf_counter()+ _, headers = self.post("/v1/systemone", body)+ walls.append((time.perf_counter() - started) * 1e3)+ engines.append(float(headers["x-colibri-engine-ms"]))+ walls.sort(), engines.sort()+ wall, engine = walls[len(walls) // 2], engines[len(engines) // 2]+ sys.stderr.write(f"\n[clef tiny e2e] /v1/systemone, 4 questions: median {wall:.1f} ms at the "+ f"client, {engine:.1f} ms in the engine, {wall - engine:.1f} ms gateway\n")+ self.assertGreater(wall, engine)++if __name__ == "__main__":unittest.main()diff --git a/c/tests/test_jev_sdk.py b/c/tests/test_jev_sdk.pyindex 2023449c..2a6b9de9 100644--- a/c/tests/test_jev_sdk.py+++ b/c/tests/test_jev_sdk.py@@ -11,9 +11,10 @@ The official clients, unmodified, pointed at colibri with nothing but@typesafe-ai/sdk TypeScript, new TypeSafeClient({apiKey, baseURL}).systemOne(...) and .models.list(), run under node-Each one is driven against two servers, each a real `coli serve` process:+Each one is driven against three servers, each a real `coli serve` process:a language model (the tiny MiMo fixture, which scores options through the-logprob channel) and a decision engine (the tiny Laya fixture, DECIDE). The+logprob channel), a decision engine (the tiny Laya fixture, DECIDE) and a+chat model with a decision head (the tiny Clef on qwen36, DECIDE too). Thesame requests go to both: every question type, several questions at once,text, object and array states, objects and arrays as instructions andcriteria, and a request the server refuses (422), which each SDK must raise@@ -44,6 +45,8 @@ import mimo_serve_fixture # noqa: E402REQUIRED = os.environ.get("JEV_SDK_REQUIRED") == "1"LAYA_FIXTURE = HERE / "laya_tiny"LAYA_BINARY = HERE / ("laya.exe" if os.name == "nt" else "laya")+CLEF_CONTAINER = HERE / "clef_tiny_c"+QWEN36_BINARY = HERE / ("qwen36.exe" if os.name == "nt" else "qwen36")TS_SDK_DIR = Path(os.environ.get("JEV_TS_SDK_DIR", HERE / "tools" / "jev-sdk"))TS_SCRIPT = HERE / "tests" / "jev_sdk_client.mjs"API_KEY = "sk-colibri-test"@@ -148,6 +151,10 @@ def servers():found.append(("laya-tiny (decision engine)", LAYA_FIXTURE, {}))elif REQUIRED:raise AssertionError("laya or laya_tiny missing (make laya laya-tiny-generate)")+ if QWEN36_BINARY.exists() and (CLEF_CONTAINER / "joint_head.safetensors").exists():+ found.append(("clef-tiny (chat model with a decision head)", CLEF_CONTAINER, {}))+ elif REQUIRED:+ raise AssertionError("qwen36 or clef_tiny_c missing (make qwen36 clef-tiny-generate)")if not found:raise unittest.SkipTest("no tiny fixture to serve (make mimo laya laya-tiny-generate)")return founddiff --git a/c/tools/compare_clef.py b/c/tools/compare_clef.pynew file mode 100644index 00000000..6718adc9--- /dev/null+++ b/c/tools/compare_clef.py@@ -0,0 +1,311 @@+#!/usr/bin/env python3+"""qwen36's Clef path against Clef's own code on the real checkpoint (maintainer tool).++Runs the same requests through both and reports, per request and overall, the+largest probability difference, whether every decision (argmax) agrees, and the+engine's time per request. The requests are the model card's examples+(Cloudflare/clef README: the invoice record and the systemone() call), the+Jev / System One examples of docs/systemone.md and docs/laya.md, and cases of+our own: a contract read in full, a conversation, unicode, JSON states and+instructions, a 20-intent choice, nine questions in one record.++Two halves, usually on the same box at different times (a 27B in bf16 and the+engine do not fit in RAM together):++ reference Clef's joint_schema_model.py (the copy in the checkpoint) on the+ Hugging Face checkpoint, in bf16 as the release runs, through+ transformers. The backbone's later layers can be offloaded to disk+ (--cpu-layers) so it fits beside nothing else; the answers are the+ same, only slower. Written to --reference and read back from it.+ engine ./qwen36 on the converted container (tools/convert_qwen36.py),+ CLEF_RECORDS mode: the records the gateway sends in the raw form,+ one process, every request in turn.++ python tools/compare_clef.py --hf ~/clef --reference clef_ref.json --cpu-layers 36+ python tools/compare_clef.py --container ~/clef_c --reference clef_ref.json \\+ [--env COLI_DENSE_BITS=4] [--json out.json]+"""+import argparse+import json+import os+import statistics+import subprocess+import sys+import tempfile+import time+from pathlib import Path+from types import SimpleNamespace++HERE = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(HERE))+from openai_server import systemone_decision_record # noqa: E402++CONTRACT = (+ "MASTER SERVICES AGREEMENT. This agreement is made between the Provider and the Customer. "+ "1. Services. The Provider will deliver the hosted platform described in Schedule A, including "+ "updates, security patches and support during business hours. 2. Fees. The Customer pays the "+ "fees in Schedule B within thirty days of each invoice; late amounts accrue interest at one "+ "percent per month. 3. Term. The initial term is twelve months and renews automatically for "+ "successive twelve-month periods unless either party gives sixty days written notice. "+ "4. Termination. Either party may terminate for material breach not cured within thirty days "+ "of notice. 5. Confidentiality. Each party keeps the other's confidential information secret "+ "and uses it only to perform this agreement. 6. Data protection. The Provider processes "+ "personal data only on documented instructions and notifies the Customer of any breach within "+ "seventy-two hours. 7. Liability. Neither party is liable for indirect damages; total liability "+ "is capped at the fees paid in the twelve months before the claim. 8. Governing law. This "+ "agreement is governed by the laws of the State of New York. ")+BANKING = ["activate_my_card", "apple_pay", "card_arrival", "card_not_working", "change_pin",+ "compromised_card", "declined_card_payment", "exchange_rate", "failed_transfer",+ "lost_or_stolen_card", "pending_transfer", "refund_not_showing_up", "top_up_failed",+ "transfer_timing", "verify_my_identity", "wrong_amount_of_cash_received",+ "cash_withdrawal_charge", "direct_debit_payment_not_recognised", "terminate_account",+ "age_limit"]+++def records():+ return [+ ("card invoice (encode_record example)",+ {"invoice": {"vendor": "Acme", "total": 1250.0, "currency": "USD", "status": "overdue"}},+ {"status": {"type": "choice", "instructions": "What is the invoice status?",+ "criteria": {"paid": "Invoice is paid.", "overdue": "Invoice is past due.",+ "draft": "Not sent."}},+ "large": {"type": "noul", "instructions": "Is the total above 1000 USD?"}}),+ ("card systemone example", "Our checkout started returning errors and orders are blocked.",+ {"department": {"type": "choice", "instructions": "Which team should handle the message?",+ "criteria": {"billing": "Payments or invoices", "technical": "Bugs or outages"}},+ "urgency": {"type": "score", "criteria": ["Can wait", "This week", "Today"]},+ "outage": {"type": "noul", "instructions": "Is a service down?"}}),+ ("systemone.md curl",+ "Hi, I have been trying to connect my Stripe account for 3 days and the integration keeps "+ "failing. I am losing sales. Please help ASAP.",+ {"urgency": {"type": "noul", "instructions": "Does this message express urgency?"},+ "department": {"type": "choice", "instructions": "Which team should handle this?",+ "criteria": {"billing": "payments, invoices, Stripe payouts",+ "technical": "bugs, outages, integration errors",+ "sales": "pricing, plans, upgrades"}},+ "severity": {"type": "score", "instructions": "How severe is the customer impact?",+ "criteria": ["no impact", "minor inconvenience", "blocked on one task",+ "losing money", "business down"]}}),+ ("laya.md readme",+ "Hi, we were billed twice for March. Please refund the duplicate today or we will cancel "+ "our plan.",+ {"department": {"type": "choice", "instructions": "Which department should handle this?",+ "criteria": {"billing": "invoices, payments, refunds",+ "technical": "bugs, outages, system errors",+ "other": "everything else"}},+ "urgency": {"type": "score", "instructions": "How urgent is this?",+ "criteria": ["not urgent", "soon", "blocking"]},+ "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"}}),+ ("ticket json + refund criteria",+ {"from": "[email protected]", "subject": "Duplicate charge on invoice #4411",+ "body": "Hi, we were billed twice for March. Please refund the duplicate today or we will "+ "cancel our plan."},+ {"refund": {"type": "noul", "instructions": "Is a refund requested?",+ "criteria": {"true": "the customer asks for money back",+ "false": "no refund is mentioned"}},+ "sentiment": {"type": "score", "instructions": "What is the sender's tone?",+ "criteria": ["angry", "negative", "neutral", "positive", "delighted"]}}),+ ("contract in full", CONTRACT,+ {"renewal": {"type": "noul", "instructions": "Does the agreement renew automatically?"},+ "notice": {"type": "choice", "instructions": "How much notice stops the renewal?",+ "criteria": {"thirty days": None, "sixty days": None, "ninety days": None}},+ "law": {"type": "choice", "instructions": "Which law governs the agreement?",+ "criteria": {"New York": None, "California": None, "Delaware": None,+ "England and Wales": None}},+ "breach_hours": {"type": "score", "instructions": "How fast must a data breach be reported?",+ "criteria": ["within 24 hours", "within 72 hours", "within 30 days"]}}),+ ("code review", "The PR touches the engine and carries no tests. CI is red on two jobs.",+ {"verdict": {"type": "choice", "instructions": "What should we do with the pull request?",+ "criteria": {"merge": None, "request changes": None, "close": None}}}),+ ("conversation churn",+ [{"role": "user", "content": "Hi, I cannot log in to my account since yesterday."},+ {"role": "assistant", "content": "Sorry about that. Did you try to reset the password?"},+ {"role": "user", "content": "Yes, twice, and the reset email never arrives."},+ {"role": "user", "content": "If it is not fixed today I will cancel the subscription."}],+ {"churn": {"type": "noul", "instructions": "Does the user threaten to cancel?"},+ "topic": {"type": "choice", "instructions": "What is the conversation about?",+ "criteria": {"login": "access to the account", "billing": "payments",+ "other": "anything else"}}}),+ ("unicode italian", "Buongiorno, la fattura di marzo è sbagliata: ci avete addebitato due "+ "volte. Grazie.",+ {"language": {"type": "choice", "instructions": "Which language is the message in?",+ "criteria": {"italian": "italiano", "german": "deutsch", "english": "english"}},+ "billing": {"type": "noul", "instructions": "Is it about billing?"}}),+ ("unicode mixed", "我们的账户被锁了,请尽快处理。 Grüße aus München \U0001F642",+ {"locked": {"type": "noul", "instructions": "Is an account locked?"},+ "polite": {"type": "noul", "instructions": "Is the message polite?"}}),+ ("json instructions", {"ticket": {"id": 4411, "amount": 129.5, "currency": "EUR", "paid": True}},+ {"action": {"type": "choice",+ "instructions": {"task": "pick the next action", "context": "support desk"},+ "criteria": {"refund": "give the money back", "escalate": "a human decides",+ "close": "nothing to do"}},+ "paid": {"type": "noul", "instructions": ["Was", "the", "ticket", "paid?"]}}),+ ("banking 20 intents", "I tried to pay with my card at the supermarket and it was refused "+ "twice, although there is money on the account.",+ {"intent": {"type": "choice", "instructions": "Which intent does the customer express?",+ "criteria": {name: name.replace("_", " ") for name in BANKING}}}),+ ("capital (mmlu-like)", "Question: What is the capital of Australia?",+ {"answer": {"type": "choice", "instructions": "Pick the correct answer.",+ "criteria": {"A": "Sydney", "B": "Canberra", "C": "Melbourne", "D": "Perth"}}}),+ ("arithmetic", "A train travels 120 km in 2 hours.",+ {"speed": {"type": "choice", "instructions": "What is its average speed?",+ "criteria": {"40 km/h": None, "60 km/h": None, "80 km/h": None, "240 km/h": None}},+ "faster_than_100": {"type": "noul", "instructions": "Is it faster than 100 km/h?"}}),+ ("nine questions", "Invoice INV-2207 from Globex: 3,400 EUR for consulting in August, due "+ "September 30. Approved by Maria. Paid on October 2 by bank transfer.",+ {"vendor": {"type": "choice", "criteria": {"Acme": None, "Globex": None, "Initech": None}},+ "currency": {"type": "choice", "criteria": {"USD": None, "EUR": None, "GBP": None}},+ "paid": {"type": "noul", "instructions": "Has it been paid?"},+ "late": {"type": "noul", "instructions": "Was it paid after the due date?"},+ "approved": {"type": "noul", "instructions": "Was it approved?"},+ "method": {"type": "choice", "instructions": "How was it paid?",+ "criteria": {"card": None, "bank transfer": None, "cash": None}},+ "amount": {"type": "score", "instructions": "How large is the amount?",+ "criteria": ["below 1,000", "1,000 to 5,000", "above 5,000"]},+ "service": {"type": "choice", "instructions": "What was bought?",+ "criteria": {"consulting": None, "hardware": None, "software": None}},+ "month": {"type": "choice", "instructions": "Which month of service?",+ "criteria": {"July": None, "August": None, "September": None}}}),+ ]+++def reference_answers(args, cases):+ """Clef's own code, unrounded, in bf16 as the release runs."""+ import importlib.util+ import torch+ from safetensors.torch import load_file+ from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration+ if args.threads:+ torch.set_num_threads(args.threads)+ spec = importlib.util.spec_from_file_location("joint_schema_model", Path(args.hf) / "joint_schema_model.py")+ jsm = importlib.util.module_from_spec(spec)+ sys.modules["joint_schema_model"] = jsm+ spec.loader.exec_module(jsm)+ dtype = getattr(torch, args.dtype)+ kwargs = {}+ if args.cpu_layers:+ # load_release_model with device_map={"": "cpu"}, except that the layers past+ # --cpu-layers (and the vision tower, which a text record never runs) stay on+ # disk and are read in at each forward: the same weights, less RAM+ config = json.loads((Path(args.hf) / "config.json").read_text())+ layers = config["text_config"]["num_hidden_layers"]+ device_map = {"model.visual": "disk", "lm_head": "cpu",+ "model.language_model.embed_tokens": "cpu",+ "model.language_model.norm": "cpu", "model.language_model.rotary_emb": "cpu"}+ for i in range(layers):+ device_map[f"model.language_model.layers.{i}"] = "cpu" if i < args.cpu_layers else "disk"+ offload = Path(tempfile.mkdtemp(prefix="clef-offload-"))+ kwargs = {"device_map": device_map, "offload_folder": str(offload)}+ backbone = Qwen3_5ForConditionalGeneration.from_pretrained(args.hf, dtype=dtype, **kwargs)+ backbone.config.use_cache = False+ head = jsm.JointSchemaHead(**json.loads((Path(args.hf) / "joint_head_config.json").read_text()))+ head.load_state_dict(load_file(str(Path(args.hf) / "joint_head.safetensors")), strict=True)+ head = head.to(device="cpu", dtype=dtype)+ model = jsm.ClefModel(backbone, head).eval()+ # AutoProcessor builds the video processor, which needs torchvision; a text-only+ # record reaches nothing of the processor but its tokenizer+ processor = SimpleNamespace(tokenizer=AutoTokenizer.from_pretrained(args.hf))+ out = []+ for name, state, questions in cases:+ request = {"model": "clef", "state": state, "questions": questions}+ started = time.perf_counter()+ encoded = jsm.encode_record(processor.tokenizer, request, processor=processor)+ batch = jsm.collate_records([encoded], processor.tokenizer.pad_token_id, torch.device("cpu"))+ with torch.inference_mode():+ logits = model(batch)[0]+ seconds = time.perf_counter() - started+ answers = [{"id": q.question_id, "option_ids": list(q.option_ids),+ "logits": [float(x) for x in l.float()],+ "probs": [float(x) for x in l.float().softmax(-1)]}+ for q, l in zip(encoded.questions, logits)]+ out.append({"name": name, "input_ids": list(encoded.input_ids), "answers": answers,+ "seconds": seconds})+ print(f"reference {name}: {len(encoded.input_ids)} tokens, {seconds:.1f} s", flush=True)+ return {"dtype": args.dtype, "cases": out}+++def engine_answers(args, cases):+ payloads = [{"payload": json.dumps(systemone_decision_record({"state": s, "questions": q}, "raw"),+ ensure_ascii=False)} for _, s, q in cases]+ env = dict(os.environ, SNAP=args.container, CLEF_IDS="1")+ for item in args.env:+ key, _, value = item.partition("=")+ env[key] = value+ if args.threads:+ env["OMP_NUM_THREADS"] = str(args.threads)+ with tempfile.TemporaryDirectory() as tmp:+ path = Path(tmp) / "records.json"+ path.write_text(json.dumps(payloads, ensure_ascii=False), encoding="utf-8")+ env["CLEF_RECORDS"] = str(path)+ run = subprocess.run([args.engine, "8", "8"], capture_output=True, text=True,+ encoding="utf-8", env=env)+ if run.returncode:+ sys.exit(run.stderr[-4000:])+ return [json.loads(line) for line in run.stdout.splitlines()]+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--hf", help="the Hugging Face checkpoint (for the reference)")+ parser.add_argument("--container", help="the converted container (for the engine)")+ parser.add_argument("--engine", default=str(HERE / "qwen36"))+ parser.add_argument("--reference", required=True, help="the reference's answers: read from "+ "this file when it exists, else computed (--hf) and written to it")+ parser.add_argument("--dtype", default="bfloat16", help="the reference's dtype (the release: bfloat16)")+ parser.add_argument("--cpu-layers", type=int, default=0,+ help="reference: keep this many backbone layers in RAM, the rest on disk")+ parser.add_argument("--env", action="append", default=[], help="KEY=VALUE for the engine")+ parser.add_argument("--threads", type=int, default=0)+ parser.add_argument("--json", help="write the per-request comparison here")+ args = parser.parse_args()+ cases = records()+ path = Path(args.reference)+ if path.exists():+ reference = json.loads(path.read_text(encoding="utf-8"))+ else:+ if not args.hf:+ sys.exit("no reference file yet: pass --hf to compute it")+ reference = reference_answers(args, cases)+ path.write_text(json.dumps(reference, ensure_ascii=False) + "\n", encoding="utf-8")+ if not args.container:+ return 0+ started = time.perf_counter()+ got = engine_answers(args, cases)+ wall = time.perf_counter() - started+ results, worst, agree, total, ids_same = [], 0.0, 0, 0, 0+ for (name, state, questions), mine, ref in zip(cases, got, reference["cases"]):+ if "error" in mine:+ sys.exit(f"{name}: the engine refused: {mine['error']}")+ same_ids = mine["input_ids"] == ref["input_ids"]+ ids_same += same_ids+ record = systemone_decision_record({"state": state, "questions": questions}, "raw")+ diffs, same = [], True+ for want, answer, question in zip(ref["answers"], mine["answers"], record["questions"]):+ labels = [option["label"] for option in question["options"]]+ order = [labels.index(option) for option in want["option_ids"]]+ diffs.append(max(abs(answer["probs"][i] - want["probs"][k]) for k, i in enumerate(order)))+ pick = order[max(range(len(order)), key=lambda k: want["probs"][k])]+ same_q = pick == max(range(len(answer["probs"])), key=lambda i: answer["probs"][i])+ same = same and same_q+ agree += same_q+ total += 1+ worst = max(worst, max(diffs))+ results.append({"name": name, "questions": len(questions), "tokens": mine["input_tokens"],+ "input_ids_identical": same_ids, "max_abs_prob_diff": max(diffs),+ "decisions_agree": same, "engine_ms": mine["engine_ms"],+ "reference_s": ref.get("seconds")})+ print(f"{name:38s} q={len(questions):2d} tok={mine['input_tokens']:5d} ids={'same' if same_ids else 'DIFF'} "+ f"max|dp|={max(diffs):.1e} agree={'yes' if same else 'NO '} engine {mine['engine_ms'] / 1e3:6.1f} s",+ flush=True)+ print(f"\n{len(results)} requests, {total} questions: input ids identical on {ids_same}/{len(results)}, "+ f"decisions agree on {agree}/{total}; max |dp| {worst:.2e} (reference {reference['dtype']})")+ engine_s = [r["engine_ms"] / 1e3 for r in results]+ print(f"engine per request: median {statistics.median(engine_s):.1f} s, total {sum(engine_s):.0f} s "+ f"({wall:.0f} s with the load)")+ if args.json:+ Path(args.json).write_text(json.dumps(results, indent=1) + "\n", encoding="utf-8")+ return 0 if agree == total and ids_same == len(results) else 1+++if __name__ == "__main__":+ sys.exit(main())