Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

systemone: TypeSafe's own SDKs work against colibri with the base URL changed

JustVugg0b0e6df379cd
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 82dac24f..3c5604be 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1101,6 +1101,24 @@ jobs:
- name: Generate the fixture, build laya, compare with the committed reference
run: make -C c laya-tiny-check
+ jev-sdk-check:
+ name: TypeSafe's SDKs (Python, TypeScript) against coli serve, base URL changed only
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ - uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ - name: Install the pinned SDKs
+ run: |
+ pip install -r c/tools/requirements-jev-sdk.txt
+ cd c/tools/jev-sdk && npm ci
+ - name: Both SDKs against a language model (tiny MiMo) and a decision engine (tiny Laya)
+ run: make -C c jev-sdk-check
+
qwenimage-tiny-check:
name: Qwen-Image tiny oracle (text encoder, DiT, scheduler, VAE vs diffusers)
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index 3d7ee1d0..58a352c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,7 @@ c/qwen38
c/qwen38.exe
c/laya
c/laya.exe
+c/tools/jev-sdk/node_modules/
# deepseek_v4 and the per-unit objects its amalgamated Makefile leaves behind.
# The same omission as #858, one file over: every engine is listed here except
# the one added last, so `make deepseek-v4` left 26 untracked .o files in
diff --git a/c/Makefile b/c/Makefile
index 74e3756d..23d039ff 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1285,6 +1285,14 @@ laya-tiny-generate:
laya-tiny-check: laya-tiny-generate laya$(EXE)
LAYA_TINY_REQUIRED=1 $(PYTHON) -m unittest -v tests.test_laya_tiny tests.test_decision_serve
+# TypeSafe's own SDKs (Python and TypeScript), unmodified, against `coli serve`
+# on a language model (tiny MiMo) and a decision engine (tiny Laya): switching
+# from Jev is a base-URL change. Needs the pinned SDKs first:
+# pip install -r tools/requirements-jev-sdk.txt && (cd tools/jev-sdk && npm ci)
+.PHONY: jev-sdk-check
+jev-sdk-check: mimo$(EXE) laya$(EXE) mimo-tiny-generate laya-tiny-generate
+ JEV_SDK_REQUIRED=1 $(PYTHON) -m unittest -v tests.test_jev_sdk
+
# In make check: every format and ragged edges against the naive product, and
# the int8 x int8 VNNI path against the exact integer sum.
tests/test_qi_gemm$(EXE): tests/test_qi_gemm.c
diff --git a/c/openai_server.py b/c/openai_server.py
index 27916d5b..d135f391 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -4962,7 +4962,7 @@ def systemone_decision_record(body):
"text": None if text is None or text == "" else decision_text(text)}
for label, text in criteria.items()]
elif kind == "score":
- options = [{"label": str(i + 1), "text": decision_text(text)}
+ options = [{"label": str(i), "text": decision_text(text)}
for i, text in enumerate(criteria)]
else:
given = {str(key).lower(): text for key, text in (criteria or {}).items()}
@@ -5845,6 +5845,17 @@ class APIServer(ThreadingHTTPServer):
entry["capabilities"] = ["decision"]
return entry
+ def jev_model_card(self):
+ """The served model as Jev's GET /v1/models lists it. colibri does not know a
+ model's release date; it gives the day this server started."""
+ try:
+ name = family_by_id(ARCH).display_name
+ except Exception:
+ name = self.model_id
+ return {"name": self.model_id,
+ "description": f"{name}, served by colibri; POST /v1/systemone answers typed questions.",
+ "release_date": time.strftime("%Y-%m-%d", time.gmtime(self.created))}
+
def input_modalities(self):
"""What a request to this server may carry: text, plus image when BOTH the
family has a placeholder expansion and the engine said it loaded its tower.
@@ -6072,6 +6083,8 @@ class APIHandler(BaseHTTPRequestHandler):
self.send_header("Content-Length", str(len(data)))
if request_id:
self.send_header("x-request-id", request_id)
+ # what the TypeSafe SDKs read as `request_id` (an error carries it too)
+ self.send_header("x-typesafe-request-id", request_id)
for name, value in (headers or {}).items():
self.send_header(name, value)
self.send_cors_headers()
@@ -6084,9 +6097,10 @@ class APIHandler(BaseHTTPRequestHandler):
return
self.send_header("Access-Control-Allow-Origin", "*" if "*" in self.server.cors_origins else origin)
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
- self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type, x-api-key, anthropic-version")
+ self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type, x-api-key, anthropic-version, "
+ "X-TypeSafe-SDK, X-TypeSafe-Runtime, X-TypeSafe-Retry-Count")
self.send_header("Access-Control-Expose-Headers",
- "x-request-id, x-colibri-queue-wait-ms, Retry-After")
+ "x-request-id, x-typesafe-request-id, x-colibri-queue-wait-ms, Retry-After")
self.send_header("Access-Control-Max-Age", "600")
if "*" not in self.server.cors_origins:
self.send_header("Vary", "Origin")
@@ -6283,7 +6297,10 @@ class APIHandler(BaseHTTPRequestHandler):
return
self.require_auth()
if path == "/v1/models":
- self.send_json(200, {"object": "list", "data": [self.server.model_entry()]},
+ # `data` is OpenAI's list; `models` is Jev's (name, description,
+ # release_date), which the TypeSafe SDKs' models.list() reads.
+ self.send_json(200, {"object": "list", "data": [self.server.model_entry()],
+ "models": [self.server.jev_model_card()]},
request_id)
elif path.startswith("/v1/models/") and unquote(path[11:]) == self.server.model_id:
self.send_json(200, self.server.model_entry(), request_id)
@@ -6479,7 +6496,7 @@ class APIHandler(BaseHTTPRequestHandler):
state = "\n".join(parts)
if not state and not question and form == "options":
raise APIError(400, "Provide `state`, `messages` or `question`.", "state")
- if not state and form != "options":
+ if not state and form != "options" and not body.get("_empty_state_ok"):
raise APIError(400, f"`{form}` needs a `state` (or `messages`) to decide on.", "state")
# "sum" (the joint log-probability of the option as a continuation)
# is the default: "mean" compares per-token averages, which silently
@@ -6688,16 +6705,24 @@ class APIHandler(BaseHTTPRequestHandler):
@staticmethod
def _systemone_confidence(probabilities):
"""(n * peak - 1) / (n - 1): 1 when all the mass is on one label, 0 when flat."""
- values = list(probabilities)
+ values = [float(v) for v in probabilities]
n = len(values)
if n < 2:
return 1.0
return round(max(0.0, (n * max(values) - 1.0) / (n - 1)), 6)
+ @staticmethod
+ def _systemone_legend(value, index):
+ """A score level as the reply's `legend` gives it back: the criterion the
+ caller sent (text, object or array), or `level <n>` for one sent as null."""
+ return value if isinstance(value, (str, dict, list)) else f"level {index}"
+
def systemone(self, body, request_id):
- state = self._systemone_text(body.get("state"), "state")
- if state is None:
+ if body.get("state") is None:
raise APIError(422, "`state` is required: the content the questions are about.", "state")
+ # Text, an object or an array; an empty text is a state with nothing in it,
+ # which the Jev schema allows, and the questions are asked without context.
+ state = self._systemone_text(body.get("state"), "state")
raw = body.get("questions")
if not isinstance(raw, dict) or not raw:
raise APIError(422, "`questions` must be a non-empty object of id: question.", "questions")
@@ -6740,47 +6765,70 @@ class APIHandler(BaseHTTPRequestHandler):
labels.append(label)
text = self._systemone_text(description, f"{where}.criteria.{label}")
lines.append(f"- {label}: {text}" if text else f"- {label}")
- if len(labels) < 2:
- raise APIError(422, f"`{where}.criteria` needs at least two labels.", f"{where}.criteria")
text = (instructions or "Which of the following applies?") + "\nOptions:\n" + "\n".join(lines)
plan.append((qid, "choice", text + "\nAnswer with one of the options.", labels, None))
elif kind == "score":
- if not isinstance(criteria, list) or not 2 <= len(criteria) <= 10:
- raise APIError(422, f"`{where}.criteria` must be an array of 2 to 10 level descriptions.",
+ # Jev numbers the levels from zero, in the order given (its OpenAPI
+ # schema: "Each description's position determines its score,
+ # starting at zero"). The model reads them as 1..n, the numbering a
+ # rubric is usually written in, and the reply maps them back.
+ if not isinstance(criteria, list) or not 1 <= len(criteria) <= 255:
+ raise APIError(422, f"`{where}.criteria` must be an array of 1 to 255 level descriptions.",
f"{where}.criteria")
- levels = [self._systemone_text(c, f"{where}.criteria[{i}]") or f"level {i + 1}"
+ levels = [self._systemone_text(c, f"{where}.criteria[{i}]") or f"level {i}"
for i, c in enumerate(criteria)]
text = (instructions or "Rate this on the scale below.") + "\nScale:\n" + \
"\n".join(f"{i + 1}: {d}" for i, d in enumerate(levels))
plan.append((qid, "score", text + "\nAnswer with the number.",
- [str(i + 1) for i in range(len(levels))], levels))
+ [str(i + 1) for i in range(len(levels))],
+ [self._systemone_legend(c, i) for i, c in enumerate(criteria)]))
else:
raise APIError(422, f"`{where}.type` must be \"noul\", \"choice\" or \"score\".", f"{where}.type")
if engine_decides(self.server.engine):
self._systemone_decide(body, plan, request_id)
return
- inner = {"state": state, "_max_options": 255,
- "questions": [{"question": text, "options": options} for _, _, text, options, _ in plan]}
- result = self.brio(inner, request_id, send=False)
+ # A question with one option has its answer already: nothing to score.
+ asked = [entry for entry in plan if len(entry[3]) > 1]
+ result = {"answers": [], "usage": {"prompt_tokens": 0, "read_tokens": 0}}
+ if asked:
+ inner = {"state": state or "", "_max_options": 255, "_empty_state_ok": True,
+ "questions": [{"question": text, "options": options}
+ for _, _, text, options, _ in asked]}
+ result = self.brio(inner, request_id, send=False)
+ scored = iter(result["answers"])
answers = {}
- for (qid, kind, _, options, levels), got in zip(plan, result["answers"]):
- p = {c["option"]: c["p"] for c in got["choices"]}
+ for qid, kind, _, options, levels in plan:
+ if len(options) > 1:
+ got = next(scored)
+ p = {c["option"]: c["p"] for c in got["choices"]}
+ else:
+ p = {options[0]: 1.0}
if kind == "noul":
- answers[qid] = {"type": "noul", "noul": round(p.get("yes", 0.0), 6)}
+ answers[qid] = {"type": "noul", "noul": round(float(p.get("yes", 0.0)), 6)}
elif kind == "choice":
- answers[qid] = {"type": "choice", "choice": got["answer"],
- "probabilities": {o: round(p[o], 6) for o in options},
+ answers[qid] = {"type": "choice", "choice": max(options, key=lambda o: p[o]),
+ "probabilities": {o: round(float(p[o]), 6) for o in options},
"confidence": self._systemone_confidence(p.values())}
else:
+ values = [float(p[o]) for o in options]
answers[qid] = {"type": "score",
- "score": round(sum(int(k) * v for k, v in p.items()), 6),
- "legend": {str(i + 1): d for i, d in enumerate(levels)},
- "probabilities": {o: round(p[o], 6) for o in options},
- "confidence": self._systemone_confidence(p.values())}
- reply = {"model": self.server.model_id, "answers": answers,
- "usage": {"input_tokens": result["usage"]["prompt_tokens"],
- "output_tokens": result["usage"]["read_tokens"]}}
- self.send_json(200, reply, request_id, result.get("_headers"))
+ "score": round(sum(i * v for i, v in enumerate(values)), 6),
+ "legend": {str(i): legend for i, legend in enumerate(levels)},
+ "probabilities": {str(i): round(v, 6) for i, v in enumerate(values)},
+ "confidence": self._systemone_confidence(values)}
+ self.send_json(200, self._systemone_reply(answers, request_id,
+ result["usage"]["prompt_tokens"],
+ result["usage"]["read_tokens"]),
+ request_id, result.get("_headers"))
+
+ def _systemone_reply(self, answers, request_id, input_tokens, output_tokens):
+ """The reply both paths send, in the Jev shape: `model` is the served
+ model, `provider` says who answered, and `usage.cost` is what this server
+ charges, nothing."""
+ return {"id": request_id, "model": self.server.model_id, "provider": "colibri",
+ "answers": answers,
+ "usage": {"input_tokens": int(input_tokens), "output_tokens": int(output_tokens),
+ "cost": 0}}
def _systemone_decide(self, body, plan, request_id):
"""The native path: the request as one DECIDE record, the engine's
@@ -6812,21 +6860,19 @@ class APIHandler(BaseHTTPRequestHandler):
None, "engine_error", "server_error")
best = max(range(n), key=lambda i: (probs[i], -i))
if kind == "noul":
- answers[qid] = {"type": "noul", "noul": round(probs[1], 6)}
+ answers[qid] = {"type": "noul", "noul": round(float(probs[1]), 6)}
elif kind == "choice":
labels = [option["label"] for option in question["options"]]
answers[qid] = {"type": "choice", "choice": labels[best],
- "probabilities": {label: round(p, 6) for label, p in zip(labels, probs)},
+ "probabilities": {label: round(float(p), 6) for label, p in zip(labels, probs)},
"confidence": self._systemone_confidence(probs)}
else:
answers[qid] = {"type": "score",
- "score": round(sum((i + 1) * p for i, p in enumerate(probs)), 6),
- "legend": {str(i + 1): d for i, d in enumerate(levels)},
- "probabilities": {str(i + 1): round(p, 6) for i, p in enumerate(probs)},
+ "score": round(sum(i * float(p) for i, p in enumerate(probs)), 6),
+ "legend": {str(i): legend for i, legend in enumerate(levels)},
+ "probabilities": {str(i): round(float(p), 6) for i, p in enumerate(probs)},
"confidence": self._systemone_confidence(probs)}
- reply = {"model": self.server.model_id, "answers": answers,
- "usage": {"input_tokens": int(decision.get("input_tokens") or 0),
- "output_tokens": 0}}
+ reply = self._systemone_reply(answers, request_id, decision.get("input_tokens") or 0, 0)
headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000)),
"x-colibri-elapsed-ms": str(round((time.time() - started) * 1000))}
engine_ms = decision.get("engine_ms")
@@ -7027,8 +7073,21 @@ class APIHandler(BaseHTTPRequestHandler):
self.send_json(error.status, self.error_body(error), request_id, error.headers)
def error_body(self, error):
- """Anthropic clients parse a different error envelope; the OpenAI one is unchanged."""
- if urlsplit(self.path).path != "/v1/messages":
+ """Anthropic clients parse a different error envelope; the OpenAI one is unchanged.
+ A /v1/systemone validation error also carries Jev's `detail` list (FastAPI's
+ form: loc, msg, type), which a client written for Jev may read."""
+ path = urlsplit(self.path).path
+ if path == "/v1/systemone" and error.status == 422:
+ body = error_object(error)
+ loc = ["body"]
+ for part in re.split(r"\.(?![^\[]*\])", error.param or ""):
+ match = re.fullmatch(r"(.*?)((?:\[\d+\])*)", part)
+ if match.group(1):
+ loc.append(match.group(1))
+ loc.extend(int(i) for i in re.findall(r"\[(\d+)\]", match.group(2)))
+ body["detail"] = [{"loc": loc, "msg": error.message, "type": "value_error"}]
+ return body
+ if path != "/v1/messages":
return error_object(error)
return {"type": "error", "error": {"type": error.error_type, "message": error.message}}
diff --git a/c/tests/jev_sdk_client.mjs b/c/tests/jev_sdk_client.mjs
new file mode 100644
index 00000000..8e2b680d
--- /dev/null
+++ b/c/tests/jev_sdk_client.mjs
@@ -0,0 +1,32 @@
+// Drives TypeSafe's TypeScript SDK (@typesafe-ai/sdk), unmodified, for
+// tests/test_jev_sdk.py: TYPESAFE_API_KEY and TYPESAFE_BASE_URL are the only
+// change, read by the SDK itself. The request comes as argv[2]; the parsed
+// reply, its request id and the model list go to stdout as JSON, or the
+// SDK's error (name, status, message) when it raised one.
+import { createRequire } from "node:module";
+import path from "node:path";
+
+const require = createRequire(path.join(process.env.JEV_TS_SDK_DIR || process.cwd(), "package.json"));
+const { TypeSafeClient, choice, noul, score } = require("@typesafe-ai/sdk");
+
+const request = JSON.parse(process.argv[2]);
+// The test's own settings: a CPU box can take longer than the 10 s default,
+// and a failure should show once rather than after two retries.
+const client = new TypeSafeClient({ timeout: 120000, retry: { maxRetries: 0 } });
+const body = request.helpers
+ ? {
+ state: "I was charged twice. Please help.",
+ questions: {
+ tone: choice("What is the tone?", { calm: null, angry: null }),
+ angry: noul("Is the sender angry?"),
+ level: score("How strong?", ["weak", "strong"]),
+ },
+ }
+ : { state: request.state, questions: request.questions };
+try {
+ const { data, requestId } = await client.systemOne(body).withResponse();
+ const models = await client.models.list();
+ process.stdout.write(JSON.stringify({ result: data, requestId, models }));
+} catch (error) {
+ process.stdout.write(JSON.stringify({ error: { name: error.name, status: error.status, message: error.message } }));
+}
diff --git a/c/tests/test_decision_serve.py b/c/tests/test_decision_serve.py
index 86d2fcc4..b6e10515 100644
--- a/c/tests/test_decision_serve.py
+++ b/c/tests/test_decision_serve.py
@@ -202,11 +202,12 @@ class DecisionGateway(unittest.TestCase):
self.assertAlmostEqual(department["confidence"], (4 * 0.7 - 1) / 3, places=6)
urgency = out["answers"]["urgency"]
self.assertEqual(set(urgency), {"type", "score", "legend", "probabilities", "confidence"})
- self.assertEqual(urgency["legend"], {"1": "not urgent", "2": "soon",
- "3": "critical deadline or blocking issue"})
- self.assertEqual(list(urgency["probabilities"]), ["1", "2", "3"])
- self.assertAlmostEqual(urgency["score"], 1 * 0.2 + 2 * 0.3 + 3 * 0.5, places=6)
- self.assertEqual(out["usage"], {"input_tokens": 40, "output_tokens": 0})
+ self.assertEqual(urgency["legend"], {"0": "not urgent", "1": "soon",
+ "2": "critical deadline or blocking issue"})
+ self.assertEqual(list(urgency["probabilities"]), ["0", "1", "2"])
+ self.assertAlmostEqual(urgency["score"], 0 * 0.2 + 1 * 0.3 + 2 * 0.5, places=6)
+ self.assertEqual(out["usage"], {"input_tokens": 40, "output_tokens": 0, "cost": 0})
+ self.assertEqual(out["provider"], "colibri")
self.assertEqual(headers["x-colibri-engine-ms"], "12.5")
def test_the_record_carries_what_the_reference_reads(self):
@@ -232,8 +233,8 @@ class DecisionGateway(unittest.TestCase):
self.assertEqual(c["instructions"], " Which? ") # exactly as sent
self.assertEqual(c["options"], [{"label": "a", "text": None},
{"label": "b", "text": '{"desc": "structured"}'}])
- self.assertEqual(s["options"], [{"label": "1", "text": "low"},
- {"label": "2", "text": '{"n": 2}'}])
+ self.assertEqual(s["options"], [{"label": "0", "text": "low"},
+ {"label": "1", "text": '{"n": 2}'}])
self.assertEqual(s["instructions"], "Rate this on the scale below.")
self.assertEqual(d["instructions"], "Is this true?")
@@ -271,7 +272,7 @@ class DecisionGateway(unittest.TestCase):
for body, param in (({"questions": {"q": {"type": "noul"}}}, "state"),
({"state": "x", "questions": {}}, "questions"),
({"state": "x", "questions": {"q": {"type": "maybe"}}}, "questions.q.type"),
- ({"state": "x", "questions": {"q": {"type": "score", "criteria": ["one"]}}},
+ ({"state": "x", "questions": {"q": {"type": "score", "criteria": []}}},
"questions.q.criteria")):
with self.subTest(param=param):
status, error = self.post_error("/v1/systemone", body)
diff --git a/c/tests/test_jev_sdk.py b/c/tests/test_jev_sdk.py
new file mode 100644
index 00000000..2023449c
--- /dev/null
+++ b/c/tests/test_jev_sdk.py
@@ -0,0 +1,276 @@
+"""TypeSafe's own SDKs against `coli serve`: switching from Jev is a base-URL change.
+
+The official clients, unmodified, pointed at colibri with nothing but
+`base_url` and an API key changed:
+
+ typesafe_sdk Python, TypeSafeClient(api_key=..., base_url=...)
+ .system_one(...) and .models.list(); it validates the
+ reply strictly (pydantic, strict=True), so a missing
+ field, a wrong type or a score level not numbered from
+ zero fails here
+ @typesafe-ai/sdk TypeScript, new TypeSafeClient({apiKey, baseURL})
+ .systemOne(...) and .models.list(), run under node
+
+Each one is driven against two servers, each a real `coli serve` process:
+a language model (the tiny MiMo fixture, which scores options through the
+logprob channel) and a decision engine (the tiny Laya fixture, DECIDE). The
+same requests go to both: every question type, several questions at once,
+text, object and array states, objects and arrays as instructions and
+criteria, and a request the server refuses (422), which each SDK must raise
+as its own UnprocessableEntity error.
+
+Skipped when an SDK, the engines or the fixtures are missing. CI installs the
+pinned SDKs (tools/requirements-jev-sdk.txt, tools/jev-sdk/package.json) and
+sets JEV_SDK_REQUIRED=1, which turns the skips into failures:
+
+ make -C c jev-sdk-check
+"""
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+import urllib.request
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(HERE / "tests"))
+import mimo_serve_fixture # noqa: E402
+
+REQUIRED = os.environ.get("JEV_SDK_REQUIRED") == "1"
+LAYA_FIXTURE = HERE / "laya_tiny"
+LAYA_BINARY = HERE / ("laya.exe" if os.name == "nt" else "laya")
+TS_SDK_DIR = Path(os.environ.get("JEV_TS_SDK_DIR", HERE / "tools" / "jev-sdk"))
+TS_SCRIPT = HERE / "tests" / "jev_sdk_client.mjs"
+API_KEY = "sk-colibri-test"
+
+STATE = {"subject": "Duplicate charge on invoice #4411",
+ "body": "Hi, we were billed twice for March. Please refund the duplicate today or we "
+ "will cancel our plan."}
+# Raw dictionaries: what both SDKs put on the wire, and what a Jev client builds.
+QUESTIONS = {
+ "department": {"type": "choice", "instructions": "Which department should handle this?",
+ "criteria": {"billing": "invoices, payments, refunds",
+ "technical": {"covers": ["bugs", "outages"]},
+ "other": None}},
+ "urgency": {"type": "score", "instructions": "How urgent is this?",
+ "criteria": ["not urgent", "soon", "blocking"]},
+ "churn": {"type": "noul", "instructions": "Does the user threaten to cancel?",
+ "criteria": {"true": "they say they will leave", "false": None}},
+ "refund": {"type": "noul", "instructions": {"ask": "is a refund requested?"}},
+}
+
+
+def skip_or_fail(message):
+ if REQUIRED:
+ raise AssertionError(message)
+ raise unittest.SkipTest(message)
+
+
+def free_port():
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ return sock.getsockname()[1]
+
+
+class ColiServe:
+ """`coli serve` in its own process, on a free port, until stop()."""
+
+ def __init__(self, model, env=None):
+ self.port = free_port()
+ self.base = f"http://127.0.0.1:{self.port}"
+ self.log = tempfile.TemporaryFile()
+ child_env = dict(os.environ, COLI_API_KEY=API_KEY, OMP_NUM_THREADS="2",
+ COLI_NO_OMP_TUNE="1", **(env or {}))
+ self.process = subprocess.Popen(
+ [sys.executable, str(HERE / "coli"), "serve", "--model", str(model),
+ "--host", "127.0.0.1", "--port", str(self.port)],
+ env=child_env, stdout=self.log, stderr=subprocess.STDOUT)
+ deadline = time.monotonic() + 120
+ while True:
+ try:
+ with urllib.request.urlopen(self.base + "/health", timeout=2):
+ break
+ except OSError:
+ if self.process.poll() is not None or time.monotonic() > deadline:
+ self.stop()
+ raise AssertionError(f"coli serve did not start:\n{self.output()}")
+ time.sleep(0.2)
+
+ def output(self):
+ self.log.seek(0)
+ return self.log.read().decode("utf-8", "replace")[-4000:]
+
+ def stop(self):
+ if self.process.poll() is None:
+ self.process.terminate()
+ try:
+ self.process.wait(timeout=30)
+ except subprocess.TimeoutExpired:
+ self.process.kill()
+ self.process.wait(timeout=10)
+ self.log.close()
+
+
+_long_mimo = None
+
+
+def long_mimo_fixture():
+ """The served MiMo fixture with room for a Jev request. Its tokenizer reads one
+ byte per token and the fixture has 256 positions, less than one request with
+ a JSON state; the copy raises max_position_embeddings (RoPE has no limit of
+ its own), and is this module's, so no other test sees it."""
+ global _long_mimo
+ if _long_mimo is None:
+ root = Path(tempfile.mkdtemp(prefix="jev-sdk-mimo-"))
+ import atexit
+ atexit.register(shutil.rmtree, root, True)
+ _long_mimo = root / "fixture"
+ shutil.copytree(mimo_serve_fixture.served_fixture(), _long_mimo)
+ config = json.loads((_long_mimo / "config.json").read_text(encoding="utf-8"))
+ config["max_position_embeddings"] = 4096
+ (_long_mimo / "config.json").write_text(json.dumps(config), encoding="utf-8")
+ return _long_mimo
+
+
+def servers():
+ """(name, model dir, env) of the two servers, or a skip naming what is missing."""
+ found = []
+ if mimo_serve_fixture.available():
+ found.append(("mimo-tiny (language model)", long_mimo_fixture(), {"CTX": "4096"}))
+ elif REQUIRED:
+ raise AssertionError("mimo or mimo_tiny missing (make mimo; tools/make_mimo_tiny.py)")
+ if LAYA_BINARY.exists() and (LAYA_FIXTURE / "model.safetensors").exists():
+ found.append(("laya-tiny (decision engine)", LAYA_FIXTURE, {}))
+ elif REQUIRED:
+ raise AssertionError("laya or laya_tiny missing (make laya laya-tiny-generate)")
+ if not found:
+ raise unittest.SkipTest("no tiny fixture to serve (make mimo laya laya-tiny-generate)")
+ return found
+
+
+class _ServedTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.served = []
+ for name, model, env in servers():
+ cls.served.append((name, ColiServe(model, env)))
+
+ @classmethod
+ def tearDownClass(cls):
+ for _, server in cls.served:
+ server.stop()
+
+ def check_answers(self, where, answers):
+ """The answer contract, read the way a Jev client reads it."""
+ department = answers["department"]
+ self.assertIn(department["choice"], {"billing", "technical", "other"}, where)
+ self.assertEqual(set(department["probabilities"]), {"billing", "technical", "other"}, where)
+ self.assertAlmostEqual(sum(department["probabilities"].values()), 1.0, places=4, msg=where)
+ self.assertEqual(department["choice"],
+ max(department["probabilities"], key=department["probabilities"].get), where)
+ self.assertTrue(0.0 <= department["confidence"] <= 1.0, where)
+ urgency = answers["urgency"]
+ self.assertEqual({int(k) for k in urgency["probabilities"]}, {0, 1, 2}, where)
+ self.assertEqual({int(k) for k in urgency["legend"]}, {0, 1, 2}, where)
+ self.assertAlmostEqual(sum(urgency["probabilities"].values()), 1.0, places=4, msg=where)
+ expected = sum(int(k) * p for k, p in urgency["probabilities"].items())
+ self.assertAlmostEqual(urgency["score"], expected, places=4, msg=where)
+ for key in ("churn", "refund"):
+ self.assertTrue(0.0 <= answers[key]["noul"] <= 1.0, where)
+
+
+class PythonSdk(_ServedTest):
+ @classmethod
+ def setUpClass(cls):
+ try:
+ import typesafe_sdk # noqa: F401
+ except ImportError:
+ skip_or_fail("typesafe_sdk is not installed (pip install -r tools/requirements-jev-sdk.txt)")
+ super().setUpClass()
+
+ def client(self, server):
+ from typesafe_sdk import RetryPolicy, TypeSafeClient
+ # The documented switch: base_url and the key. The longer timeout and no
+ # retries are this test's, so a slow CI box fails once and visibly.
+ return TypeSafeClient(api_key=API_KEY, base_url=server.base, timeout=120.0,
+ retry=RetryPolicy(max_retries=0))
+
+ def test_system_one_with_dictionaries_and_question_objects(self):
+ from typesafe_sdk import Choice, Noul, Score, SystemOneResponse
+ for name, server in self.served:
+ with self.subTest(server=name), self.client(server) as client:
+ result = client.system_one(state=STATE, questions=QUESTIONS)
+ self.assertIsInstance(result, SystemOneResponse)
+ answers = {key: answer.model_dump() for key, answer in result.answers.items()}
+ self.check_answers(name, answers)
+ self.assertIn(result.choices["department"].choice, {"billing", "technical", "other"})
+ self.assertEqual(sorted(result.scores["urgency"].legend), [0, 1, 2])
+ self.assertGreater(result.usage.input_tokens, 0)
+ self.assertTrue(result.request_id.startswith("req_"))
+ self.assertEqual(result.model, client.models.list().models[0].name)
+ # the SDK's own question objects, a text state and an array state
+ typed = {"tone": Choice(instructions="What is the tone?",
+ criteria={"calm": None, "angry": None}),
+ "angry": Noul(instructions="Is the sender angry?"),
+ "level": Score(instructions="How strong?", criteria=["weak", "strong"])}
+ for state in ("I was charged twice. Please help.",
+ [{"role": "user", "content": "charged twice"},
+ {"role": "assistant", "content": "sorry about that"}]):
+ out = client.system_one(state=state, questions=typed)
+ self.assertIn(out.choices["tone"].choice, {"calm", "angry"})
+ self.assertTrue(0.0 <= out.nouls["angry"].noul <= 1.0)
+ self.assertEqual(sorted(out.scores["level"].probabilities), [0, 1])
+
+ def test_a_refusal_is_the_sdks_unprocessable_entity(self):
+ from typesafe_sdk import TypeSafeUnprocessableEntityError
+ for name, server in self.served:
+ with self.subTest(server=name), self.client(server) as client:
+ with self.assertRaises(TypeSafeUnprocessableEntityError) as caught:
+ client.system_one(state="x", questions={"q": {"type": "guess"}})
+ self.assertIn("type", str(caught.exception))
+ self.assertTrue(caught.exception.request_id.startswith("req_"))
+
+
+class TypeScriptSdk(_ServedTest):
+ @classmethod
+ def setUpClass(cls):
+ if shutil.which("node") is None:
+ skip_or_fail("node is not installed")
+ if not (TS_SDK_DIR / "node_modules" / "@typesafe-ai" / "sdk").is_dir():
+ skip_or_fail(f"@typesafe-ai/sdk is not installed (cd {TS_SDK_DIR} && npm ci)")
+ super().setUpClass()
+
+ def run_client(self, server, request):
+ env = dict(os.environ, TYPESAFE_API_KEY=API_KEY, TYPESAFE_BASE_URL=server.base,
+ JEV_TS_SDK_DIR=str(TS_SDK_DIR))
+ result = subprocess.run(["node", str(TS_SCRIPT), json.dumps(request)], env=env,
+ capture_output=True, text=True, timeout=300,
+ cwd=str(TS_SDK_DIR))
+ self.assertEqual(result.returncode, 0, result.stderr[-3000:])
+ return json.loads(result.stdout)
+
+ def test_system_one_and_models(self):
+ for name, server in self.served:
+ with self.subTest(server=name):
+ out = self.run_client(server, {"state": STATE, "questions": QUESTIONS})
+ self.check_answers(name, out["result"]["answers"])
+ self.assertTrue(out["requestId"].startswith("req_"))
+ self.assertEqual(out["models"][0]["name"], out["result"]["model"])
+ helpers = self.run_client(server, {"helpers": True})
+ self.assertIn(helpers["result"]["answers"]["tone"]["choice"], {"calm", "angry"})
+
+ def test_a_refusal_is_the_sdks_unprocessable_entity(self):
+ for name, server in self.served:
+ with self.subTest(server=name):
+ out = self.run_client(server, {"state": "x", "questions": {"q": {"type": "guess"}}})
+ self.assertEqual(out["error"]["name"], "UnprocessableEntityError")
+ self.assertEqual(out["error"]["status"], 422)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_systemone_api.py b/c/tests/test_systemone_api.py
index 0e63e10e..acf6dcb0 100644
--- a/c/tests/test_systemone_api.py
+++ b/c/tests/test_systemone_api.py
@@ -108,21 +108,64 @@ class SystemOneApi(unittest.TestCase):
self.assertIn("- a\n- b", self.pins()[1])
def test_score_expected_value_legend_and_confidence(self):
+ # Jev numbers the levels from zero ("Each description's position determines
+ # its score, starting at zero", its OpenAPI schema); the model reads 1..n.
self.serve({"4": -0.1, "1": -5.0, "2": -5.0, "3": -5.0, "5": -5.0})
levels = ["not urgent", "low", "moderate", "high", "critical"]
out = self.post({"model": "jev-latest", "state": STATE, "questions": {
"urgency": {"type": "score", "instructions": "How urgent is this?", "criteria": levels}}})
answer = out["answers"]["urgency"]
self.assertEqual(answer["type"], "score")
- self.assertEqual(answer["legend"], {str(i + 1): d for i, d in enumerate(levels)})
- self.assertEqual(list(answer["probabilities"]), ["1", "2", "3", "4", "5"])
+ self.assertEqual(answer["legend"], {str(i): d for i, d in enumerate(levels)})
+ self.assertEqual(list(answer["probabilities"]), ["0", "1", "2", "3", "4"])
ps = answer["probabilities"]
self.assertAlmostEqual(sum(ps.values()), 1.0, places=5)
self.assertAlmostEqual(answer["score"], sum(int(k) * v for k, v in ps.items()), places=5)
- self.assertGreater(answer["score"], 3.9) # the mass sits on level 4
+ self.assertGreater(answer["score"], 2.9) # the mass sits on level 3, "high"
self.assertAlmostEqual(answer["confidence"], confidence(list(ps.values())), places=5)
self.assertIn("4: high", self.pins()[1])
+ def test_legend_gives_back_the_criteria_as_sent(self):
+ self.serve({"1": -0.1, "2": -3.0})
+ rubric = [{"level": "calm", "examples": ["thanks"]}, "angry"]
+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {
+ "tone": {"type": "score", "criteria": rubric}}})
+ self.assertEqual(out["answers"]["tone"]["legend"], {"0": rubric[0], "1": "angry"})
+
+ def test_reply_carries_the_fields_the_jev_sdks_read(self):
+ self.serve({"yes": -0.1, "no": -2.0})
+ request = Request(self.base + "/v1/systemone",
+ data=json.dumps({"model": "jev-latest", "state": STATE, "questions": {
+ "q": {"type": "noul", "instructions": "Urgent?"}}}).encode(),
+ headers={"Content-Type": "application/json"})
+ with urlopen(request, timeout=10) as response:
+ out = json.loads(response.read())
+ request_id = response.headers["x-typesafe-request-id"]
+ self.assertEqual(out["id"], request_id)
+ self.assertTrue(request_id.startswith("req_"))
+ self.assertEqual(out["provider"], "colibri")
+ self.assertEqual(out["model"], "test-model")
+ self.assertEqual(out["usage"]["cost"], 0)
+ self.assertIsInstance(out["answers"]["q"]["noul"], float)
+
+ def test_one_label_or_one_level_is_answered_without_the_engine(self):
+ self.serve({"yes": -0.1, "no": -2.0})
+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {
+ "only": {"type": "choice", "criteria": {"billing": "payments"}},
+ "flat": {"type": "score", "criteria": ["the one level"]}}})
+ self.assertEqual(out["answers"]["only"], {"type": "choice", "choice": "billing",
+ "probabilities": {"billing": 1.0}, "confidence": 1.0})
+ self.assertEqual(out["answers"]["flat"]["probabilities"], {"0": 1.0})
+ self.assertEqual(out["answers"]["flat"]["score"], 0.0)
+ self.assertEqual(self.engine.calls, [])
+
+ def test_an_empty_text_state_is_a_state(self):
+ self.serve({"yes": -0.1, "no": -2.0})
+ out = self.post({"model": "jev-latest", "state": "", "questions": {
+ "q": {"type": "noul", "instructions": "Is there anything here?"}}})
+ self.assertIn("noul", out["answers"]["q"])
+ self.assertFalse(any(c["prompt"].startswith("Context:") for c in self.engine.calls))
+
def test_structured_state_and_instructions_are_serialized(self):
self.serve({"yes": -0.1, "no": -2.0})
out = self.post({"model": "jev-latest",
@@ -144,7 +187,7 @@ class SystemOneApi(unittest.TestCase):
pins = self.pins()
self.assertEqual(len(pins), 4, pins) # the state, then each question once
self.assertEqual(pins[0], f"Context:\n{STATE}\n\n")
- self.assertEqual(set(out["usage"]), {"input_tokens", "output_tokens"})
+ self.assertEqual(set(out["usage"]), {"input_tokens", "output_tokens", "cost"})
self.assertGreater(out["usage"]["input_tokens"], 0)
self.assertGreater(out["usage"]["output_tokens"], 0)
@@ -161,12 +204,31 @@ class SystemOneApi(unittest.TestCase):
"questions": {"q": {"type": "choice", "criteria": ["a", "b"]}}})
self.assertEqual(code, 422)
code, body = self.post_error({"model": "jev-latest", "state": STATE,
- "questions": {"q": {"type": "score", "criteria": ["only one"]}}})
+ "questions": {"q": {"type": "score", "criteria": []}}})
self.assertEqual(code, 422)
- self.assertIn("2 to 10", body["error"]["message"])
+ self.assertIn("1 to 255", body["error"]["message"])
code, body = self.post_error({"model": "jev-latest", "state": STATE, "questions": []})
self.assertEqual(code, 422)
+ def test_validation_errors_carry_jevs_detail_list(self):
+ self.serve({})
+ code, body = self.post_error({"model": "jev-latest", "state": STATE, "questions": {
+ "q": {"type": "score", "criteria": ["fine", 3]}}})
+ self.assertEqual(code, 422)
+ self.assertEqual(body["error"]["param"], "questions.q.criteria[1]")
+ self.assertEqual(body["detail"], [{"loc": ["body", "questions", "q", "criteria", 1],
+ "msg": body["error"]["message"], "type": "value_error"}])
+
+ def test_models_lists_the_jev_card_beside_the_openai_one(self):
+ self.serve({})
+ with urlopen(self.base + "/v1/models", timeout=10) as response:
+ listing = json.loads(response.read())
+ self.assertEqual(listing["data"][0]["id"], "test-model")
+ card = listing["models"][0]
+ self.assertEqual(set(card), {"name", "description", "release_date"})
+ self.assertEqual(card["name"], "test-model")
+ self.assertRegex(card["release_date"], r"^\d{4}-\d{2}-\d{2}$")
+
def test_brio_route_still_checks_the_model_name(self):
self.serve({"a": -0.1, "b": -1.0})
request = Request(self.base + "/v1/brio",
diff --git a/c/tools/jev-sdk/package-lock.json b/c/tools/jev-sdk/package-lock.json
new file mode 100644
index 00000000..8a4a599c
--- /dev/null
+++ b/c/tools/jev-sdk/package-lock.json
@@ -0,0 +1,22 @@
+{
+ "name": "colibri-jev-sdk-check",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "colibri-jev-sdk-check",
+ "dependencies": {
+ "@typesafe-ai/sdk": "0.6.0"
+ }
+ },
+ "node_modules/@typesafe-ai/sdk": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz",
+ "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20"
+ }
+ }
+ }
+}
diff --git a/c/tools/jev-sdk/package.json b/c/tools/jev-sdk/package.json
new file mode 100644
index 00000000..9942fd8d
--- /dev/null
+++ b/c/tools/jev-sdk/package.json
@@ -0,0 +1,8 @@
+{
+ "name": "colibri-jev-sdk-check",
+ "private": true,
+ "description": "TypeSafe's TypeScript SDK, pinned, for tests/test_jev_sdk.py (npm ci here, then make jev-sdk-check)",
+ "dependencies": {
+ "@typesafe-ai/sdk": "0.6.0"
+ }
+}
diff --git a/c/tools/requirements-jev-sdk.txt b/c/tools/requirements-jev-sdk.txt
new file mode 100644
index 00000000..6c22d670
--- /dev/null
+++ b/c/tools/requirements-jev-sdk.txt
@@ -0,0 +1,16 @@
+# TypeSafe's Python SDK and what it pulls in, pinned, for tests/test_jev_sdk.py
+# (make jev-sdk-check). numpy builds the tiny fixtures the test serves.
+typesafe_sdk==0.7.2
+httpx2==2.13.1
+httpcore2==2.13.1
+pydantic==2.13.5
+pydantic_core==2.46.5
+tenacity==9.1.4
+typing_extensions==4.16.0
+typing-inspection==0.4.4
+annotated-types==0.8.0
+anyio==4.15.1
+h11==0.16.0
+idna==3.20
+truststore==0.10.4
+numpy