Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

registry: a Clef checkpoint is qwen36 with a decision head; serve and SDK tests

JustVugg2edb1184dd6b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d4cddf77..edf8e2e4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1169,11 +1169,12 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: '20'
- - name: Install the pinned SDKs
+ - name: Install the pinned SDKs, and torch for the Clef fixture's converter
run: |
pip install -r c/tools/requirements-jev-sdk.txt
+ pip install numpy -r c/tools/oracle-requirements.txt
cd c/tools/jev-sdk && npm ci
- - name: Both SDKs against a language model (tiny MiMo) and a decision engine (tiny Laya)
+ - name: Both SDKs against a language model (tiny MiMo), a decision engine (tiny Laya) and Clef
run: make -C c jev-sdk-check
qwenimage-tiny-check:
diff --git a/c/Makefile b/c/Makefile
index 96ee466f..1a0ca727 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1318,11 +1318,12 @@ clef-tiny-check: clef-tiny-generate qwen36$(EXE)
CLEF_TINY_REQUIRED=1 $(PYTHON) -m unittest -v tests.test_clef_tiny tests.test_decision_serve
# TypeSafe's own SDKs (Python and TypeScript), unmodified, against `coli serve`
-# on a language model (tiny MiMo) and a decision engine (tiny Laya): switching
-# from Jev is a base-URL change. Needs the pinned SDKs first:
+# on a language model (tiny MiMo), a decision engine (tiny Laya) and a chat
+# model with a decision head (tiny Clef on qwen36): switching from Jev is a
+# base-URL change. Needs the pinned SDKs first, and torch for the Clef converter:
# pip install -r tools/requirements-jev-sdk.txt && (cd tools/jev-sdk && npm ci)
.PHONY: jev-sdk-check
-jev-sdk-check: mimo$(EXE) laya$(EXE) mimo-tiny-generate laya-tiny-generate
+jev-sdk-check: mimo$(EXE) laya$(EXE) qwen36$(EXE) mimo-tiny-generate laya-tiny-generate clef-tiny-generate
JEV_SDK_REQUIRED=1 $(PYTHON) -m unittest -v tests.test_jev_sdk
# In make check: every format and ragged edges against the naive product, and
diff --git a/c/coli b/c/coli
index 28a0ee85..c20f51a7 100755
--- a/c/coli
+++ b/c/coli
@@ -273,6 +273,21 @@ def is_decision_model(model):
except (FamilyConfigError, UnknownFamilyError, OSError, TypeError, ValueError):
return False
+def decision_head_line(model, resolved):
+ """`coli info`'s line for a decision head over a chat model (Clef): what it
+ adds, and that the chat stays."""
+ from family_registry import decision_head_of
+ head = decision_head_of(resolved)
+ try:
+ with open(os.path.join(model, head.files[0]), encoding="utf-8") as f:
+ cfg = json.load(f)
+ shape = (f"{cfg.get('routing_layers')} routing + {cfg.get('layers')} decoder layers, "
+ f"width {cfg.get('width')}")
+ except (OSError, ValueError, AttributeError):
+ shape = "head config unreadable"
+ return (f"{head.display_name} joint schema head ({shape}): POST /v1/systemone answers "
+ f"natively; chat, completions and messages work as on any {resolved.descriptor.id} model")
+
def decision_refusal(family):
"""What every chatting command says about a decision model, and where to go."""
return (f"{C.yel}{family.display_name} is a decision model:{C.r} it answers typed questions "
@@ -1290,6 +1305,8 @@ def cmd_info(a):
sts=[x for x in os.listdir(a.model) if x.endswith('.safetensors')]
sz=sum(os.path.getsize(os.path.join(a.model,x)) for x in sts)
row("shards", f"{len(sts)} files · {sz/1e9:.0f} GB on disk")
+ if resolved is not None and resolved.decision_head:
+ row("decision", decision_head_line(a.model, resolved))
else:
# No config.json means no family, and no family means no engine: say so
# instead of letting the GLM default below stand in. A directory of shards
diff --git a/c/doctor.py b/c/doctor.py
index ce78fa0d..4490dffa 100644
--- a/c/doctor.py
+++ b/c/doctor.py
@@ -550,6 +550,25 @@ def missing_shared_libraries(engine_path):
for line in result.stdout.splitlines() if "not found" in line})
+def _decision_head_check(model, resolved):
+ """A decision head beside a chat model's shards (Clef): its config must read
+ and fit the backbone, or the engine refuses to start."""
+ from family_registry import decision_head_of
+ head = decision_head_of(resolved)
+ try:
+ config = json.loads((Path(model) / head.files[0]).read_text(encoding="utf-8"))
+ hidden = config.get("hidden_size")
+ except (OSError, ValueError, AttributeError) as error:
+ return _check("model.decision_head", "fail", f"{head.files[0]} is unreadable: {error}")
+ want = resolved.family_config.get("hidden_size")
+ if hidden != want:
+ return _check("model.decision_head", "fail",
+ f"{head.display_name} head reads hidden size {hidden}, the backbone has {want}")
+ return _check("model.decision_head", "pass",
+ f"{head.display_name} decision head: POST /v1/systemone answers natively",
+ head=head.id, hidden_size=hidden)
+
+
def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
engine_path, available_memory=None, available_disk=None, gpus=None,
linkage=None, deep=False, mirror_dir=None, kv_slots=1,
@@ -582,6 +601,8 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
family_id=resolved.descriptor.id,
model_type=resolved.model_type,
descriptor=public_metadata(resolved.descriptor)))
+ if resolved.decision_head:
+ checks.append(_decision_head_check(model, resolved))
except (FamilyConfigError, UnknownFamilyError) as error:
checks.append(_check("model.family", "fail", str(error)))
else:
diff --git a/c/family_registry.py b/c/family_registry.py
index 0b93cae5..cd575597 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -179,6 +179,45 @@ class ResolvedFamily:
config: dict
family_config: dict
model_dir: str
+ # A decision head over a text family's backbone (Cloudflare's Clef: a qwen36
+ # container with joint_head_config.json and joint_head.safetensors). The
+ # engine then answers POST /v1/systemone natively AND chats; the family, its
+ # planner and its limits stay the backbone's. "" for every other checkpoint.
+ decision_head: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class DecisionHead:
+ """A decision head a text engine loads beside its backbone (docs/clef.md)."""
+ id: str
+ family: str # the engine that runs the backbone and the head
+ files: tuple # all of them next to the shards
+ display_name: str
+ model_id: str
+ # (geometry, display_scale): the backbone sizes the head ships on
+ scales: tuple = ()
+
+
+DECISION_HEADS = (
+ DecisionHead("clef", "qwen36", ("joint_head_config.json", "joint_head.safetensors"),
+ "Clef", "clef",
+ scales=(((("num_hidden_layers", 64), ("hidden_size", 5120),
+ ("intermediate_size", 17408)), "27B"),)),
+)
+
+
+def decision_head_of(resolved):
+ """The DecisionHead a resolved checkpoint carries, or None."""
+ for head in DECISION_HEADS:
+ if head.id == resolved.decision_head:
+ return head
+ return None
+
+
+def checkpoint_decides(resolved):
+ """True when the checkpoint answers POST /v1/systemone natively: a decision
+ family (Laya), or a text family with a decision head (Clef)."""
+ return resolved.descriptor.modality == "decision" or bool(resolved.decision_head)
def _required_int(config, key, family, minimum=1):
@@ -1874,8 +1913,18 @@ def resolve_model(model_dir):
family_config = config.get("text_config", config)
if not isinstance(family_config, dict):
raise FamilyConfigError(f"{family.id}: text_config is not an object")
+ head = ""
+ for candidate in DECISION_HEADS:
+ present = [name for name in candidate.files if (model / name).is_file()]
+ if candidate.family == family.id and present:
+ if len(present) != len(candidate.files):
+ raise FamilyConfigError(f"{model}: {present[0]} without "
+ f"{', '.join(n for n in candidate.files if n not in present)}: "
+ f"a {candidate.display_name} decision head needs all of "
+ f"{', '.join(candidate.files)}")
+ head = candidate.id
return ResolvedFamily(family, _normalize_model_type(config.get("model_type")),
- config, family_config, str(model))
+ config, family_config, str(model), head)
def default_model_id(resolved):
@@ -1883,6 +1932,9 @@ def default_model_id(resolved):
one of its own, else the family's."""
family = resolved.descriptor
config = resolved.family_config
+ head = decision_head_of(resolved)
+ if head:
+ return head.model_id
for variant in family.display_variants:
if variant.model_id and all(config.get(key) == value for key, value in variant.geometry):
return variant.model_id
@@ -1899,9 +1951,15 @@ def display_for(resolved):
count.
"""
family = resolved.descriptor
+ config = resolved.family_config
+ head = decision_head_of(resolved)
+ if head:
+ for geometry, scale in head.scales:
+ if all(config.get(key) == value for key, value in geometry):
+ return head.display_name, scale
+ return head.display_name, ""
if not family.display_variants:
return family.display_name, family.display_scale
- config = resolved.family_config
for variant in family.display_variants:
if all(config.get(key) == value for key, value in variant.geometry):
return variant.display_name, variant.display_scale
diff --git a/c/openai_server.py b/c/openai_server.py
index a4c6c4d0..2494c857 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -24,7 +24,7 @@ import v4_dsml # vendored DeepSeek V4 DSML reference primit
import v41_dsml # ...and V4.1's, whose tag names differ by a space
import image_engine # the qwenimage serve protocol, PNG and request rules
from family_registry import (FamilyConfigError, UnknownFamilyError, family_by_id,
- family_ids, resolve_model)
+ display_for, family_ids, resolve_model)
from family_registry import default_model_id as registry_default_model_id
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
@@ -5942,7 +5942,7 @@ class APIServer(ThreadingHTTPServer):
"""The served model as Jev's GET /v1/models lists it. colibri does not know a
model's release date; it gives the day this server started."""
try:
- name = family_by_id(ARCH).display_name
+ name = getattr(self, "display_name", None) or family_by_id(ARCH).display_name
except Exception:
name = self.model_id
return {"name": self.model_id,
@@ -7975,6 +7975,10 @@ def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,
pending_model_id = family.default_model_id
model_id = pending_model_id
server.model_id = model_id
+ try: # what the checkpoint on disk is called (a Clef, a 27B), for the Jev card
+ server.display_name = display_for(resolve_model(model))[0]
+ except Exception:
+ server.display_name = None
if kv_slots > family.limits.max_kv_slots:
raise ValueError(f"{family.id} engine supports at most "
f"{family.limits.max_kv_slots} KV slot(s)")
diff --git a/c/tests/test_decision_serve.py b/c/tests/test_decision_serve.py
index bd32f2fd..f2f519b0 100644
--- a/c/tests/test_decision_serve.py
+++ b/c/tests/test_decision_serve.py
@@ -16,9 +16,15 @@ of scoring options through the logprob channel. Three layers are pinned here:
gateway: the probabilities that come back over HTTP are
the `laya` package's (laya_tiny/ref.json), and the
latency is split into gateway and engine time.
-
-LayaTinyEndToEnd is skipped without the fixture and the binary
-(`make laya-tiny-check` builds both and turns the skip into a failure).
+ ClefTinyEndToEnd qwen36 with Clef's decision head (clef_tiny_c) behind the
+ real gateway: CAPS decide=1 decide_record=raw and no
+ chat=0, /v1/systemone gives Clef's own probabilities
+ (clef_tiny/ref.json), and the same engine still chats,
+ with a decision before and after the chat unchanged.
+
+LayaTinyEndToEnd and ClefTinyEndToEnd are skipped without their fixture and
+binary (`make laya-tiny-check` and `make clef-tiny-check` build them and turn
+the skip into a failure).
"""
import json
import math
@@ -43,6 +49,10 @@ from test_openai_server import BlockingStream, FakeProcess # noqa: E402
FIXTURE = HERE / "laya_tiny"
BINARY = HERE / ("laya.exe" if os.name == "nt" else "laya")
REQUIRED = os.environ.get("LAYA_TINY_REQUIRED") == "1"
+CLEF_REF = HERE / "clef_tiny" / "ref.json"
+CLEF_CONTAINER = HERE / "clef_tiny_c"
+QWEN36 = HERE / ("qwen36.exe" if os.name == "nt" else "qwen36")
+CLEF_REQUIRED = os.environ.get("CLEF_TINY_REQUIRED") == "1"
TICKET = {"from": "[email protected]", "subject": "Duplicate charge on invoice #4411",
"body": "Hi, we were billed twice for March. Please refund the duplicate today or we "
@@ -238,6 +248,40 @@ class DecisionGateway(unittest.TestCase):
self.assertEqual(s["instructions"], "Rate this on the scale below.")
self.assertEqual(d["instructions"], "Is this true?")
+ def test_a_raw_form_engine_gets_the_callers_values(self):
+ """An engine that said decide_record=raw (Clef) renders the request the way
+ its reference does: it gets the caller's values, not the gateway's
+ defaults, and JSON with sorted keys."""
+ engine = FakeDecisionEngine({"q": [0.5, 0.5], "c": [0.2, 0.3, 0.5], "s": [0.5, 0.5],
+ "d": [0.5, 0.5]})
+ engine.decide_record = "raw"
+ engine.chats = True
+ self.serve(engine)
+ self.post("/v1/systemone", {"state": {"ticket": 4711, "text": "café", "ok": True},
+ "questions": {
+ "q": {"type": "noul", "instructions": {"ask": "money?", "a": 1},
+ "criteria": {"true": "about a payment", "false": None}},
+ "c": {"type": "choice", "instructions": "",
+ "criteria": {"b": None, "a": {"z": 1, "y": [2.5]}, "c": ""}},
+ "s": {"type": "score", "criteria": ["low", {"n": 2}]},
+ "d": {"type": "noul"}}})
+ record = self.engine.records[0]
+ self.assertEqual(record["record"], "raw")
+ self.assertEqual(record["state"], '{"ok":true,"text":"café","ticket":4711}')
+ q, c, s, d = record["questions"]
+ self.assertEqual(q["instructions"], '{"a":1,"ask":"money?"}')
+ self.assertEqual(q["options"], [{"label": "false", "text": None},
+ {"label": "true", "text": "about a payment"}])
+ self.assertEqual(c["instructions"], "") # not replaced
+ self.assertEqual(c["options"], [{"label": "b", "text": None},
+ {"label": "a", "text": '{"y":[2.5],"z":1}', "json": True},
+ {"label": "c", "text": ""}])
+ self.assertEqual(s["instructions"], None)
+ self.assertEqual(s["options"][1], {"label": "1", "text": '{"n":2}', "json": True})
+ self.assertEqual(d["options"], [{"label": "false"}, {"label": "true"}]) # undescribed
+ with urlopen(self.base + "/v1/models", timeout=10) as response:
+ self.assertEqual(json.loads(response.read())["data"][0]["capabilities"], ["chat", "systemone"])
+
def test_a_list_state_is_a_conversation(self):
self.serve(FakeDecisionEngine({"q": [0.5, 0.5]}))
turns = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
@@ -369,6 +413,32 @@ class DecisionRegistryAndLauncher(unittest.TestCase):
self.assertIn("POST /v1/systemone", refused.stdout + refused.stderr)
+ def test_the_registry_knows_a_clef_checkpoint(self):
+ import tempfile
+ from family_registry import (FamilyConfigError, checkpoint_decides, default_model_id,
+ display_for, resolve_model)
+ directory = tempfile.TemporaryDirectory()
+ self.addCleanup(directory.cleanup)
+ root = Path(directory.name)
+ text = {"model_type": "qwen3_5_text", "num_hidden_layers": 64, "hidden_size": 5120,
+ "intermediate_size": 17408}
+ (root / "config.json").write_text(json.dumps({"model_type": "qwen3_5", "text_config": text}),
+ encoding="utf-8")
+ stock = resolve_model(root)
+ self.assertEqual((stock.decision_head, display_for(stock)), ("", ("Qwen3.8-27B", "27B")))
+ self.assertFalse(checkpoint_decides(stock))
+ (root / "joint_head_config.json").write_text(json.dumps({"hidden_size": 5120}), encoding="utf-8")
+ with self.assertRaises(FamilyConfigError): # half a head
+ resolve_model(root)
+ (root / "joint_head.safetensors").write_bytes(b"")
+ clef = resolve_model(root)
+ self.assertEqual((clef.descriptor.id, clef.descriptor.modality), ("qwen36", "text"))
+ self.assertEqual(clef.decision_head, "clef")
+ self.assertTrue(checkpoint_decides(clef))
+ self.assertEqual(display_for(clef), ("Clef", "27B"))
+ self.assertEqual(default_model_id(clef), "clef")
+
+
def _laya_ready():
return (FIXTURE / "model.safetensors").exists() and BINARY.exists()
@@ -472,5 +542,133 @@ class LayaTinyEndToEnd(unittest.TestCase):
self.assertGreater(wall, engine)
+def _clef_ready():
+ return CLEF_REF.exists() and (CLEF_CONTAINER / "joint_head.safetensors").exists() and QWEN36.exists()
+
+
+class ClefTinyEndToEnd(unittest.TestCase):
+ """qwen36 with Clef's head behind the real gateway, on the tiny fixture."""
+
+ # the cases the route accepts as Jev requests (it refuses a bare number as a
+ # criterion or an instruction, which the engine-level oracle still covers)
+ CASES = ("model_card_invoice", "model_card_systemone", "ticket_four_questions", "noul_variants",
+ "json_state", "conversation", "special_text", "many_questions", "single_option",
+ "empty_state")
+
+ @classmethod
+ def setUpClass(cls):
+ if not _clef_ready():
+ message = "qwen36, clef_tiny or clef_tiny_c missing: run make clef-tiny-check"
+ if CLEF_REQUIRED:
+ raise AssertionError(message)
+ raise unittest.SkipTest(message)
+ cls.ref = {case["name"]: case for case in
+ json.loads(CLEF_REF.read_text(encoding="utf-8"))["cases"]}
+
+ def setUp(self):
+ arch = patch.object(openai_server, "ARCH", "qwen36")
+ arch.start()
+ self.addCleanup(arch.stop)
+ engine = Engine(QWEN36, CLEF_CONTAINER, family=family_by_id("qwen36"),
+ env=dict(os.environ, COLI_DENSE_I8="0", OMP_NUM_THREADS="2",
+ COLI_NO_OMP_TUNE="1"))
+ self.engine = engine
+ self.addCleanup(engine.process.stdout.close)
+ self.addCleanup(engine.close)
+ self.server = APIServer(("127.0.0.1", 0), engine, "clef-tiny")
+ self.addCleanup(self.server.server_close)
+ self.addCleanup(self.server.shutdown)
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
+ self.base = f"http://127.0.0.1:{self.server.server_port}"
+
+ def post(self, path, body):
+ request = Request(self.base + path, data=json.dumps(body).encode(),
+ headers={"Content-Type": "application/json"})
+ with urlopen(request, timeout=300) as response:
+ return json.loads(response.read()), dict(response.headers)
+
+ def check(self, name, out):
+ case = self.ref[name]
+ for want in case["answers"]:
+ got = out["answers"][want["id"]]
+ p = dict(zip(want["option_ids"], want["probs"]))
+ if got["type"] == "noul":
+ self.assertAlmostEqual(got["noul"], p["true"], delta=2e-5, msg=name)
+ continue
+ self.assertEqual(set(got["probabilities"]), set(p), name)
+ for label, value in got["probabilities"].items():
+ self.assertAlmostEqual(value, p[label], delta=2e-5, msg=f"{name}.{want['id']}.{label}")
+ if got["type"] == "choice":
+ self.assertEqual(got["choice"], max(p, key=p.get), name)
+ self.assertEqual(out["usage"], {"input_tokens": len(case["input_ids"]), "output_tokens": 0,
+ "cost": 0})
+
+ def test_caps_say_decide_raw_and_chat(self):
+ self.assertTrue(self.engine.decides)
+ self.assertTrue(self.engine.chats)
+ self.assertEqual(self.engine.decide_record, "raw")
+ with urlopen(self.base + "/v1/models", timeout=10) as response:
+ card = json.loads(response.read())
+ self.assertEqual(card["data"][0]["capabilities"], ["chat", "systemone"])
+
+ def test_systemone_answers_with_clefs_probabilities(self):
+ for name in self.CASES:
+ case = self.ref[name]
+ with self.subTest(case=name):
+ out, headers = self.post("/v1/systemone", {"model": "jev-latest", "state": case["state"],
+ "questions": case["questions"]})
+ self.check(name, out)
+ self.assertIn("x-colibri-engine-ms", headers)
+
+ def test_the_same_engine_chats_between_two_decisions(self):
+ case = self.ref["model_card_systemone"]
+ body = {"state": case["state"], "questions": case["questions"]}
+ first, _ = self.post("/v1/systemone", body)
+ chat, _ = self.post("/v1/chat/completions", {"model": "clef-tiny", "max_tokens": 6,
+ "temperature": 0,
+ "messages": [{"role": "user", "content": "hi"}]})
+ self.assertEqual(chat["object"], "chat.completion")
+ self.assertGreater(chat["usage"]["completion_tokens"], 0)
+ second, _ = self.post("/v1/systemone", body)
+ self.assertEqual(second["answers"], first["answers"])
+ self.check("model_card_systemone", second)
+
+ def test_a_schema_past_the_budget_is_a_422(self):
+ """COLI_CLEF_MAX_LEN is the reference's max_length: past it the schema alone
+ does not fit and the request is the caller's 422, not a 500."""
+ self.engine.close()
+ engine = Engine(QWEN36, CLEF_CONTAINER, family=family_by_id("qwen36"),
+ env=dict(os.environ, COLI_DENSE_I8="0", OMP_NUM_THREADS="2",
+ COLI_NO_OMP_TUNE="1", COLI_CLEF_MAX_LEN="600"))
+ self.addCleanup(engine.process.stdout.close)
+ self.addCleanup(engine.close)
+ self.server.engine = engine
+ case = self.ref["schema_too_long"]
+ request = Request(self.base + "/v1/systemone",
+ data=json.dumps({"state": case["state"], "questions": case["questions"]}).encode(),
+ headers={"Content-Type": "application/json"})
+ with self.assertRaises(HTTPError) as caught:
+ urlopen(request, timeout=120)
+ self.assertEqual(caught.exception.code, 422)
+ self.assertIn("1389", json.loads(caught.exception.read())["error"]["message"])
+
+ def test_latency_gateway_and_engine(self):
+ """Not a gate: wall time at the client against the engine's own time."""
+ case = self.ref["ticket_four_questions"]
+ body = {"model": "jev-latest", "state": case["state"], "questions": case["questions"]}
+ self.post("/v1/systemone", body)
+ walls, engines = [], []
+ for _ in range(5):
+ started = time.perf_counter()
+ _, headers = self.post("/v1/systemone", body)
+ walls.append((time.perf_counter() - started) * 1e3)
+ engines.append(float(headers["x-colibri-engine-ms"]))
+ walls.sort(), engines.sort()
+ wall, engine = walls[len(walls) // 2], engines[len(engines) // 2]
+ sys.stderr.write(f"\n[clef tiny e2e] /v1/systemone, 4 questions: median {wall:.1f} ms at the "
+ f"client, {engine:.1f} ms in the engine, {wall - engine:.1f} ms gateway\n")
+ self.assertGreater(wall, engine)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tests/test_jev_sdk.py b/c/tests/test_jev_sdk.py
index 2023449c..2a6b9de9 100644
--- a/c/tests/test_jev_sdk.py
+++ b/c/tests/test_jev_sdk.py
@@ -11,9 +11,10 @@ The official clients, unmodified, pointed at colibri with nothing but
@typesafe-ai/sdk TypeScript, new TypeSafeClient({apiKey, baseURL})
.systemOne(...) and .models.list(), run under node
-Each one is driven against two servers, each a real `coli serve` process:
+Each one is driven against three servers, each a real `coli serve` process:
a language model (the tiny MiMo fixture, which scores options through the
-logprob channel) and a decision engine (the tiny Laya fixture, DECIDE). The
+logprob channel), a decision engine (the tiny Laya fixture, DECIDE) and a
+chat model with a decision head (the tiny Clef on qwen36, DECIDE too). The
same requests go to both: every question type, several questions at once,
text, object and array states, objects and arrays as instructions and
criteria, and a request the server refuses (422), which each SDK must raise
@@ -44,6 +45,8 @@ import mimo_serve_fixture # noqa: E402
REQUIRED = os.environ.get("JEV_SDK_REQUIRED") == "1"
LAYA_FIXTURE = HERE / "laya_tiny"
LAYA_BINARY = HERE / ("laya.exe" if os.name == "nt" else "laya")
+CLEF_CONTAINER = HERE / "clef_tiny_c"
+QWEN36_BINARY = HERE / ("qwen36.exe" if os.name == "nt" else "qwen36")
TS_SDK_DIR = Path(os.environ.get("JEV_TS_SDK_DIR", HERE / "tools" / "jev-sdk"))
TS_SCRIPT = HERE / "tests" / "jev_sdk_client.mjs"
API_KEY = "sk-colibri-test"
@@ -148,6 +151,10 @@ def servers():
found.append(("laya-tiny (decision engine)", LAYA_FIXTURE, {}))
elif REQUIRED:
raise AssertionError("laya or laya_tiny missing (make laya laya-tiny-generate)")
+ if QWEN36_BINARY.exists() and (CLEF_CONTAINER / "joint_head.safetensors").exists():
+ found.append(("clef-tiny (chat model with a decision head)", CLEF_CONTAINER, {}))
+ elif REQUIRED:
+ raise AssertionError("qwen36 or clef_tiny_c missing (make qwen36 clef-tiny-generate)")
if not found:
raise unittest.SkipTest("no tiny fixture to serve (make mimo laya laya-tiny-generate)")
return found
diff --git a/c/tools/compare_clef.py b/c/tools/compare_clef.py
new file mode 100644
index 00000000..6718adc9
--- /dev/null
+++ b/c/tools/compare_clef.py
@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""qwen36's Clef path against Clef's own code on the real checkpoint (maintainer tool).
+
+Runs the same requests through both and reports, per request and overall, the
+largest probability difference, whether every decision (argmax) agrees, and the
+engine's time per request. The requests are the model card's examples
+(Cloudflare/clef README: the invoice record and the systemone() call), the
+Jev / System One examples of docs/systemone.md and docs/laya.md, and cases of
+our own: a contract read in full, a conversation, unicode, JSON states and
+instructions, a 20-intent choice, nine questions in one record.
+
+Two halves, usually on the same box at different times (a 27B in bf16 and the
+engine do not fit in RAM together):
+
+ reference Clef's joint_schema_model.py (the copy in the checkpoint) on the
+ Hugging Face checkpoint, in bf16 as the release runs, through
+ transformers. The backbone's later layers can be offloaded to disk
+ (--cpu-layers) so it fits beside nothing else; the answers are the
+ same, only slower. Written to --reference and read back from it.
+ engine ./qwen36 on the converted container (tools/convert_qwen36.py),
+ CLEF_RECORDS mode: the records the gateway sends in the raw form,
+ one process, every request in turn.
+
+ python tools/compare_clef.py --hf ~/clef --reference clef_ref.json --cpu-layers 36
+ python tools/compare_clef.py --container ~/clef_c --reference clef_ref.json \\
+ [--env COLI_DENSE_BITS=4] [--json out.json]
+"""
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+HERE = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(HERE))
+from openai_server import systemone_decision_record # noqa: E402
+
+CONTRACT = (
+ "MASTER SERVICES AGREEMENT. This agreement is made between the Provider and the Customer. "
+ "1. Services. The Provider will deliver the hosted platform described in Schedule A, including "
+ "updates, security patches and support during business hours. 2. Fees. The Customer pays the "
+ "fees in Schedule B within thirty days of each invoice; late amounts accrue interest at one "
+ "percent per month. 3. Term. The initial term is twelve months and renews automatically for "
+ "successive twelve-month periods unless either party gives sixty days written notice. "
+ "4. Termination. Either party may terminate for material breach not cured within thirty days "
+ "of notice. 5. Confidentiality. Each party keeps the other's confidential information secret "
+ "and uses it only to perform this agreement. 6. Data protection. The Provider processes "
+ "personal data only on documented instructions and notifies the Customer of any breach within "
+ "seventy-two hours. 7. Liability. Neither party is liable for indirect damages; total liability "
+ "is capped at the fees paid in the twelve months before the claim. 8. Governing law. This "
+ "agreement is governed by the laws of the State of New York. ")
+BANKING = ["activate_my_card", "apple_pay", "card_arrival", "card_not_working", "change_pin",
+ "compromised_card", "declined_card_payment", "exchange_rate", "failed_transfer",
+ "lost_or_stolen_card", "pending_transfer", "refund_not_showing_up", "top_up_failed",
+ "transfer_timing", "verify_my_identity", "wrong_amount_of_cash_received",
+ "cash_withdrawal_charge", "direct_debit_payment_not_recognised", "terminate_account",
+ "age_limit"]
+
+
+def records():
+ return [
+ ("card invoice (encode_record example)",
+ {"invoice": {"vendor": "Acme", "total": 1250.0, "currency": "USD", "status": "overdue"}},
+ {"status": {"type": "choice", "instructions": "What is the invoice status?",
+ "criteria": {"paid": "Invoice is paid.", "overdue": "Invoice is past due.",
+ "draft": "Not sent."}},
+ "large": {"type": "noul", "instructions": "Is the total above 1000 USD?"}}),
+ ("card systemone example", "Our checkout started returning errors and orders are blocked.",
+ {"department": {"type": "choice", "instructions": "Which team should handle the message?",
+ "criteria": {"billing": "Payments or invoices", "technical": "Bugs or outages"}},
+ "urgency": {"type": "score", "criteria": ["Can wait", "This week", "Today"]},
+ "outage": {"type": "noul", "instructions": "Is a service down?"}}),
+ ("systemone.md curl",
+ "Hi, I have been trying to connect my Stripe account for 3 days and the integration keeps "
+ "failing. I am losing sales. Please help ASAP.",
+ {"urgency": {"type": "noul", "instructions": "Does this message express urgency?"},
+ "department": {"type": "choice", "instructions": "Which team should handle this?",
+ "criteria": {"billing": "payments, invoices, Stripe payouts",
+ "technical": "bugs, outages, integration errors",
+ "sales": "pricing, plans, upgrades"}},
+ "severity": {"type": "score", "instructions": "How severe is the customer impact?",
+ "criteria": ["no impact", "minor inconvenience", "blocked on one task",
+ "losing money", "business down"]}}),
+ ("laya.md readme",
+ "Hi, we were billed twice for March. Please refund the duplicate today or we will cancel "
+ "our plan.",
+ {"department": {"type": "choice", "instructions": "Which department should handle this?",
+ "criteria": {"billing": "invoices, payments, refunds",
+ "technical": "bugs, outages, system errors",
+ "other": "everything else"}},
+ "urgency": {"type": "score", "instructions": "How urgent is this?",
+ "criteria": ["not urgent", "soon", "blocking"]},
+ "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"}}),
+ ("ticket json + refund criteria",
+ {"from": "[email protected]", "subject": "Duplicate charge on invoice #4411",
+ "body": "Hi, we were billed twice for March. Please refund the duplicate today or we will "
+ "cancel our plan."},
+ {"refund": {"type": "noul", "instructions": "Is a refund requested?",
+ "criteria": {"true": "the customer asks for money back",
+ "false": "no refund is mentioned"}},
+ "sentiment": {"type": "score", "instructions": "What is the sender's tone?",
+ "criteria": ["angry", "negative", "neutral", "positive", "delighted"]}}),
+ ("contract in full", CONTRACT,
+ {"renewal": {"type": "noul", "instructions": "Does the agreement renew automatically?"},
+ "notice": {"type": "choice", "instructions": "How much notice stops the renewal?",
+ "criteria": {"thirty days": None, "sixty days": None, "ninety days": None}},
+ "law": {"type": "choice", "instructions": "Which law governs the agreement?",
+ "criteria": {"New York": None, "California": None, "Delaware": None,
+ "England and Wales": None}},
+ "breach_hours": {"type": "score", "instructions": "How fast must a data breach be reported?",
+ "criteria": ["within 24 hours", "within 72 hours", "within 30 days"]}}),
+ ("code review", "The PR touches the engine and carries no tests. CI is red on two jobs.",
+ {"verdict": {"type": "choice", "instructions": "What should we do with the pull request?",
+ "criteria": {"merge": None, "request changes": None, "close": None}}}),
+ ("conversation churn",
+ [{"role": "user", "content": "Hi, I cannot log in to my account since yesterday."},
+ {"role": "assistant", "content": "Sorry about that. Did you try to reset the password?"},
+ {"role": "user", "content": "Yes, twice, and the reset email never arrives."},
+ {"role": "user", "content": "If it is not fixed today I will cancel the subscription."}],
+ {"churn": {"type": "noul", "instructions": "Does the user threaten to cancel?"},
+ "topic": {"type": "choice", "instructions": "What is the conversation about?",
+ "criteria": {"login": "access to the account", "billing": "payments",
+ "other": "anything else"}}}),
+ ("unicode italian", "Buongiorno, la fattura di marzo è sbagliata: ci avete addebitato due "
+ "volte. Grazie.",
+ {"language": {"type": "choice", "instructions": "Which language is the message in?",
+ "criteria": {"italian": "italiano", "german": "deutsch", "english": "english"}},
+ "billing": {"type": "noul", "instructions": "Is it about billing?"}}),
+ ("unicode mixed", "我们的账户被锁了,请尽快处理。 Grüße aus München \U0001F642",
+ {"locked": {"type": "noul", "instructions": "Is an account locked?"},
+ "polite": {"type": "noul", "instructions": "Is the message polite?"}}),
+ ("json instructions", {"ticket": {"id": 4411, "amount": 129.5, "currency": "EUR", "paid": True}},
+ {"action": {"type": "choice",
+ "instructions": {"task": "pick the next action", "context": "support desk"},
+ "criteria": {"refund": "give the money back", "escalate": "a human decides",
+ "close": "nothing to do"}},
+ "paid": {"type": "noul", "instructions": ["Was", "the", "ticket", "paid?"]}}),
+ ("banking 20 intents", "I tried to pay with my card at the supermarket and it was refused "
+ "twice, although there is money on the account.",
+ {"intent": {"type": "choice", "instructions": "Which intent does the customer express?",
+ "criteria": {name: name.replace("_", " ") for name in BANKING}}}),
+ ("capital (mmlu-like)", "Question: What is the capital of Australia?",
+ {"answer": {"type": "choice", "instructions": "Pick the correct answer.",
+ "criteria": {"A": "Sydney", "B": "Canberra", "C": "Melbourne", "D": "Perth"}}}),
+ ("arithmetic", "A train travels 120 km in 2 hours.",
+ {"speed": {"type": "choice", "instructions": "What is its average speed?",
+ "criteria": {"40 km/h": None, "60 km/h": None, "80 km/h": None, "240 km/h": None}},
+ "faster_than_100": {"type": "noul", "instructions": "Is it faster than 100 km/h?"}}),
+ ("nine questions", "Invoice INV-2207 from Globex: 3,400 EUR for consulting in August, due "
+ "September 30. Approved by Maria. Paid on October 2 by bank transfer.",
+ {"vendor": {"type": "choice", "criteria": {"Acme": None, "Globex": None, "Initech": None}},
+ "currency": {"type": "choice", "criteria": {"USD": None, "EUR": None, "GBP": None}},
+ "paid": {"type": "noul", "instructions": "Has it been paid?"},
+ "late": {"type": "noul", "instructions": "Was it paid after the due date?"},
+ "approved": {"type": "noul", "instructions": "Was it approved?"},
+ "method": {"type": "choice", "instructions": "How was it paid?",
+ "criteria": {"card": None, "bank transfer": None, "cash": None}},
+ "amount": {"type": "score", "instructions": "How large is the amount?",
+ "criteria": ["below 1,000", "1,000 to 5,000", "above 5,000"]},
+ "service": {"type": "choice", "instructions": "What was bought?",
+ "criteria": {"consulting": None, "hardware": None, "software": None}},
+ "month": {"type": "choice", "instructions": "Which month of service?",
+ "criteria": {"July": None, "August": None, "September": None}}}),
+ ]
+
+
+def reference_answers(args, cases):
+ """Clef's own code, unrounded, in bf16 as the release runs."""
+ import importlib.util
+ import torch
+ from safetensors.torch import load_file
+ from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration
+ if args.threads:
+ torch.set_num_threads(args.threads)
+ spec = importlib.util.spec_from_file_location("joint_schema_model", Path(args.hf) / "joint_schema_model.py")
+ jsm = importlib.util.module_from_spec(spec)
+ sys.modules["joint_schema_model"] = jsm
+ spec.loader.exec_module(jsm)
+ dtype = getattr(torch, args.dtype)
+ kwargs = {}
+ if args.cpu_layers:
+ # load_release_model with device_map={"": "cpu"}, except that the layers past
+ # --cpu-layers (and the vision tower, which a text record never runs) stay on
+ # disk and are read in at each forward: the same weights, less RAM
+ config = json.loads((Path(args.hf) / "config.json").read_text())
+ layers = config["text_config"]["num_hidden_layers"]
+ device_map = {"model.visual": "disk", "lm_head": "cpu",
+ "model.language_model.embed_tokens": "cpu",
+ "model.language_model.norm": "cpu", "model.language_model.rotary_emb": "cpu"}
+ for i in range(layers):
+ device_map[f"model.language_model.layers.{i}"] = "cpu" if i < args.cpu_layers else "disk"
+ offload = Path(tempfile.mkdtemp(prefix="clef-offload-"))
+ kwargs = {"device_map": device_map, "offload_folder": str(offload)}
+ backbone = Qwen3_5ForConditionalGeneration.from_pretrained(args.hf, dtype=dtype, **kwargs)
+ backbone.config.use_cache = False
+ head = jsm.JointSchemaHead(**json.loads((Path(args.hf) / "joint_head_config.json").read_text()))
+ head.load_state_dict(load_file(str(Path(args.hf) / "joint_head.safetensors")), strict=True)
+ head = head.to(device="cpu", dtype=dtype)
+ model = jsm.ClefModel(backbone, head).eval()
+ # AutoProcessor builds the video processor, which needs torchvision; a text-only
+ # record reaches nothing of the processor but its tokenizer
+ processor = SimpleNamespace(tokenizer=AutoTokenizer.from_pretrained(args.hf))
+ out = []
+ for name, state, questions in cases:
+ request = {"model": "clef", "state": state, "questions": questions}
+ started = time.perf_counter()
+ encoded = jsm.encode_record(processor.tokenizer, request, processor=processor)
+ batch = jsm.collate_records([encoded], processor.tokenizer.pad_token_id, torch.device("cpu"))
+ with torch.inference_mode():
+ logits = model(batch)[0]
+ seconds = time.perf_counter() - started
+ answers = [{"id": q.question_id, "option_ids": list(q.option_ids),
+ "logits": [float(x) for x in l.float()],
+ "probs": [float(x) for x in l.float().softmax(-1)]}
+ for q, l in zip(encoded.questions, logits)]
+ out.append({"name": name, "input_ids": list(encoded.input_ids), "answers": answers,
+ "seconds": seconds})
+ print(f"reference {name}: {len(encoded.input_ids)} tokens, {seconds:.1f} s", flush=True)
+ return {"dtype": args.dtype, "cases": out}
+
+
+def engine_answers(args, cases):
+ payloads = [{"payload": json.dumps(systemone_decision_record({"state": s, "questions": q}, "raw"),
+ ensure_ascii=False)} for _, s, q in cases]
+ env = dict(os.environ, SNAP=args.container, CLEF_IDS="1")
+ for item in args.env:
+ key, _, value = item.partition("=")
+ env[key] = value
+ if args.threads:
+ env["OMP_NUM_THREADS"] = str(args.threads)
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "records.json"
+ path.write_text(json.dumps(payloads, ensure_ascii=False), encoding="utf-8")
+ env["CLEF_RECORDS"] = str(path)
+ run = subprocess.run([args.engine, "8", "8"], capture_output=True, text=True,
+ encoding="utf-8", env=env)
+ if run.returncode:
+ sys.exit(run.stderr[-4000:])
+ return [json.loads(line) for line in run.stdout.splitlines()]
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--hf", help="the Hugging Face checkpoint (for the reference)")
+ parser.add_argument("--container", help="the converted container (for the engine)")
+ parser.add_argument("--engine", default=str(HERE / "qwen36"))
+ parser.add_argument("--reference", required=True, help="the reference's answers: read from "
+ "this file when it exists, else computed (--hf) and written to it")
+ parser.add_argument("--dtype", default="bfloat16", help="the reference's dtype (the release: bfloat16)")
+ parser.add_argument("--cpu-layers", type=int, default=0,
+ help="reference: keep this many backbone layers in RAM, the rest on disk")
+ parser.add_argument("--env", action="append", default=[], help="KEY=VALUE for the engine")
+ parser.add_argument("--threads", type=int, default=0)
+ parser.add_argument("--json", help="write the per-request comparison here")
+ args = parser.parse_args()
+ cases = records()
+ path = Path(args.reference)
+ if path.exists():
+ reference = json.loads(path.read_text(encoding="utf-8"))
+ else:
+ if not args.hf:
+ sys.exit("no reference file yet: pass --hf to compute it")
+ reference = reference_answers(args, cases)
+ path.write_text(json.dumps(reference, ensure_ascii=False) + "\n", encoding="utf-8")
+ if not args.container:
+ return 0
+ started = time.perf_counter()
+ got = engine_answers(args, cases)
+ wall = time.perf_counter() - started
+ results, worst, agree, total, ids_same = [], 0.0, 0, 0, 0
+ for (name, state, questions), mine, ref in zip(cases, got, reference["cases"]):
+ if "error" in mine:
+ sys.exit(f"{name}: the engine refused: {mine['error']}")
+ same_ids = mine["input_ids"] == ref["input_ids"]
+ ids_same += same_ids
+ record = systemone_decision_record({"state": state, "questions": questions}, "raw")
+ diffs, same = [], True
+ for want, answer, question in zip(ref["answers"], mine["answers"], record["questions"]):
+ labels = [option["label"] for option in question["options"]]
+ order = [labels.index(option) for option in want["option_ids"]]
+ diffs.append(max(abs(answer["probs"][i] - want["probs"][k]) for k, i in enumerate(order)))
+ pick = order[max(range(len(order)), key=lambda k: want["probs"][k])]
+ same_q = pick == max(range(len(answer["probs"])), key=lambda i: answer["probs"][i])
+ same = same and same_q
+ agree += same_q
+ total += 1
+ worst = max(worst, max(diffs))
+ results.append({"name": name, "questions": len(questions), "tokens": mine["input_tokens"],
+ "input_ids_identical": same_ids, "max_abs_prob_diff": max(diffs),
+ "decisions_agree": same, "engine_ms": mine["engine_ms"],
+ "reference_s": ref.get("seconds")})
+ print(f"{name:38s} q={len(questions):2d} tok={mine['input_tokens']:5d} ids={'same' if same_ids else 'DIFF'} "
+ f"max|dp|={max(diffs):.1e} agree={'yes' if same else 'NO '} engine {mine['engine_ms'] / 1e3:6.1f} s",
+ flush=True)
+ print(f"\n{len(results)} requests, {total} questions: input ids identical on {ids_same}/{len(results)}, "
+ f"decisions agree on {agree}/{total}; max |dp| {worst:.2e} (reference {reference['dtype']})")
+ engine_s = [r["engine_ms"] / 1e3 for r in results]
+ print(f"engine per request: median {statistics.median(engine_s):.1f} s, total {sum(engine_s):.0f} s "
+ f"({wall:.0f} s with the load)")
+ if args.json:
+ Path(args.json).write_text(json.dumps(results, indent=1) + "\n", encoding="utf-8")
+ return 0 if agree == total and ids_same == len(results) else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())