Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
gateway: POST /v1/systemone, the request and reply of Jev, served by the brio channel
JustVugg48c974d199d8
diff --git a/c/openai_server.py b/c/openai_server.pyindex b319b314..d3de8dea 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -3956,14 +3956,19 @@ class APIHandler(BaseHTTPRequestHandler):self._check_host()self.require_auth()body = self.read_json()- self.check_model(body)path = urlsplit(self.path).path+ # A client written for Jev sends "jev-latest": on that route the+ # served model answers whatever name was asked for.+ if path != "/v1/systemone":+ self.check_model(body)if path == "/v1/chat/completions":self.chat_completion(body, request_id)elif path == "/v1/completions":self.completion(body, request_id)elif path == "/v1/brio":self.brio(body, request_id)+ elif path == "/v1/systemone":+ self.systemone(body, request_id)elif path == "/v1/messages":self.anthropic_messages(body, request_id)else:@@ -4009,11 +4014,11 @@ class APIHandler(BaseHTTPRequestHandler):# malformato perche' non lo scrive il modello. Prima queste due forme# esistevano solo come script di misura: chi integrava doveva riscriverle.@staticmethod- def _brio_options(options, where):+ def _brio_options(options, where, limit=64):if not isinstance(options, list) or not options:raise APIError(400, f"`{where}` must be a non-empty array of strings.", where)- if len(options) > 64:- raise APIError(400, f"`{where}` accepts at most 64 entries.", where)+ if len(options) > limit:+ raise APIError(400, f"`{where}` accepts at most {limit} entries.", where)seen = set()for option in options:if not isinstance(option, str) or not option.strip():@@ -4025,7 +4030,11 @@ class APIHandler(BaseHTTPRequestHandler):raise APIError(400, f"`{where}` needs at least two options to choose between.", where)return options- def brio(self, body, request_id):+ def brio(self, body, request_id, send=True):+ # `send=False` returns the result instead of writing it: /v1/systemone+ # builds a `questions` request and re-shapes the answer. `_max_options`+ # is that caller's word too (Jev allows 255 labels); clamped.+ option_limit = min(int(body.get("_max_options", 64) or 64), 255)forms = [k for k in ("options", "questions", "schema") if body.get(k) is not None]if len(forms) != 1:raise APIError(400, "Provide exactly one of `options`, `questions` or `schema`.",@@ -4055,7 +4064,8 @@ class APIHandler(BaseHTTPRequestHandler):if per not in ("mean", "sum"):raise APIError(400, "`normalize` must be \"mean\" or \"sum\".", "normalize")questions.append((text, self._brio_options(entry.get("options"),- f"questions[{i}].options"), per))+ f"questions[{i}].options",+ option_limit), per))else:raw = body["schema"]if not isinstance(raw, dict) or not raw:@@ -4231,9 +4241,143 @@ class APIHandler(BaseHTTPRequestHandler):"read_tokens": read_total,"total_tokens": prompt_max + read_total},})- self.send_json(200, result, request_id,- {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000)),- "x-colibri-elapsed-ms": str(round((time.time() - started) * 1000))})+ headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000)),+ "x-colibri-elapsed-ms": str(round((time.time() - started) * 1000))}+ if not send:+ result["_headers"] = headers+ return result+ self.send_json(200, result, request_id, headers)++ # ------------------------------------------------------------ Jev-compatible+ #+ # POST /v1/systemone speaks the request and the reply of TypeSafe's Jev+ # API (docs.typesafe.ai/api): a client written for it points at colibri+ # and changes the base URL, nothing else. The three primitives map onto+ # the `questions` form of /v1/brio, the same channel: the state is+ # photographed once and every question pays only its own tokens.+ #+ # noul -> one yes/no question. `noul` is the probability of yes. The+ # optional criteria (what true and false mean) go into the+ # question text.+ # choice -> the labels of `criteria` are the options; their descriptions+ # go into the question text, because a label alone ("billing")+ # does not say what it means. `confidence` follows their+ # documented formula, (n * peak - 1) / (n - 1).+ # score -> the levels of `criteria` are the options "1".."n"; `score`+ # is the expected value under the distribution, `legend` the+ # levels by number, `confidence` as for choice.+ #+ # What differs, stated rather than hidden: `model` echoes the served+ # model, not "jev-latest"; `usage.output_tokens` counts the option tokens+ # READ, since this engine generates nothing; validation errors are 422 as+ # theirs are, with this server's error envelope. docs/brio.md has the+ # mapping table.+ _SYSTEMONE_MAX_QUESTIONS = 64++ @staticmethod+ def _systemone_text(value, where):+ """Jev's EntryType: a string, or JSON given as an object or an array."""+ if value is None:+ return None+ if isinstance(value, str):+ return value.strip() or None+ if isinstance(value, (dict, list)):+ return json.dumps(value, ensure_ascii=False, indent=2)+ raise APIError(422, f"`{where}` must be a string, an object or an array.", where)++ @staticmethod+ def _systemone_confidence(probabilities):+ """(n * peak - 1) / (n - 1): 1 when all the mass is on one label, 0 when flat."""+ values = list(probabilities)+ n = len(values)+ if n < 2:+ return 1.0+ return round(max(0.0, (n * max(values) - 1.0) / (n - 1)), 6)++ def systemone(self, body, request_id):+ state = self._systemone_text(body.get("state"), "state")+ if state is None:+ raise APIError(422, "`state` is required: the content the questions are about.", "state")+ raw = body.get("questions")+ if not isinstance(raw, dict) or not raw:+ raise APIError(422, "`questions` must be a non-empty object of id: question.", "questions")+ if len(raw) > self._SYSTEMONE_MAX_QUESTIONS:+ raise APIError(422, f"`questions` accepts at most {self._SYSTEMONE_MAX_QUESTIONS} entries.",+ "questions")+ plan = [] # (id, kind, text, options, levels)+ for qid, question in raw.items():+ where = f"questions.{qid}"+ if not isinstance(qid, str) or not qid.strip():+ raise APIError(422, "Every question id must be a non-empty string.", "questions")+ if not isinstance(question, dict):+ raise APIError(422, f"`{where}` must be an object.", where)+ kind = question.get("type")+ instructions = self._systemone_text(question.get("instructions"), f"{where}.instructions")+ criteria = question.get("criteria")+ if kind == "noul":+ if criteria is not None and not isinstance(criteria, dict):+ raise APIError(422, f"`{where}.criteria` must be an object with `true` and/or `false`.",+ f"{where}.criteria")+ yes = self._systemone_text((criteria or {}).get("true"), f"{where}.criteria.true")+ no = self._systemone_text((criteria or {}).get("false"), f"{where}.criteria.false")+ text = instructions or "Is this true?"+ if yes:+ text += f"\nyes: {yes}"+ if no:+ text += f"\nno: {no}"+ plan.append((qid, "noul", text + "\nAnswer yes or no.", ["yes", "no"], None))+ elif kind == "choice":+ if not isinstance(criteria, dict) or not criteria:+ raise APIError(422, f"`{where}.criteria` must be a non-empty object of label: description.",+ f"{where}.criteria")+ if len(criteria) > 255:+ raise APIError(422, f"`{where}.criteria` accepts at most 255 labels.", f"{where}.criteria")+ labels, lines = [], []+ for label, description in criteria.items():+ if not isinstance(label, str) or not label.strip():+ raise APIError(422, f"Every label of `{where}.criteria` must be a non-empty string.",+ f"{where}.criteria")+ labels.append(label)+ text = self._systemone_text(description, f"{where}.criteria.{label}")+ lines.append(f"- {label}: {text}" if text else f"- {label}")+ if len(labels) < 2:+ raise APIError(422, f"`{where}.criteria` needs at least two labels.", f"{where}.criteria")+ text = (instructions or "Which of the following applies?") + "\nOptions:\n" + "\n".join(lines)+ plan.append((qid, "choice", text + "\nAnswer with one of the options.", labels, None))+ elif kind == "score":+ if not isinstance(criteria, list) or not 2 <= len(criteria) <= 10:+ raise APIError(422, f"`{where}.criteria` must be an array of 2 to 10 level descriptions.",+ f"{where}.criteria")+ levels = [self._systemone_text(c, f"{where}.criteria[{i}]") or f"level {i + 1}"+ for i, c in enumerate(criteria)]+ text = (instructions or "Rate this on the scale below.") + "\nScale:\n" + \+ "\n".join(f"{i + 1}: {d}" for i, d in enumerate(levels))+ plan.append((qid, "score", text + "\nAnswer with the number.",+ [str(i + 1) for i in range(len(levels))], levels))+ else:+ raise APIError(422, f"`{where}.type` must be \"noul\", \"choice\" or \"score\".", f"{where}.type")+ inner = {"state": state, "_max_options": 255,+ "questions": [{"question": text, "options": options} for _, _, text, options, _ in plan]}+ result = self.brio(inner, request_id, send=False)+ answers = {}+ for (qid, kind, _, options, levels), got in zip(plan, result["answers"]):+ p = {c["option"]: c["p"] for c in got["choices"]}+ if kind == "noul":+ answers[qid] = {"type": "noul", "noul": round(p.get("yes", 0.0), 6)}+ elif kind == "choice":+ answers[qid] = {"type": "choice", "choice": got["answer"],+ "probabilities": {o: round(p[o], 6) for o in options},+ "confidence": self._systemone_confidence(p.values())}+ else:+ answers[qid] = {"type": "score",+ "score": round(sum(int(k) * v for k, v in p.items()), 6),+ "legend": {str(i + 1): d for i, d in enumerate(levels)},+ "probabilities": {o: round(p[o], 6) for o in options},+ "confidence": self._systemone_confidence(p.values())}+ reply = {"model": self.server.model_id, "answers": answers,+ "usage": {"input_tokens": result["usage"]["prompt_tokens"],+ "output_tokens": result["usage"]["read_tokens"]}}+ self.send_json(200, reply, request_id, result.get("_headers"))def _fail(self, error, request_id):"""Report an error, unless the response is already on the wire. Once a streaming 200diff --git a/c/tests/test_systemone_api.py b/c/tests/test_systemone_api.pynew file mode 100644index 00000000..beef44a5--- /dev/null+++ b/c/tests/test_systemone_api.py@@ -0,0 +1,179 @@+"""POST /v1/systemone: the request and the reply of TypeSafe's Jev API, served+by the brio channel.++A client written for Jev sends `state`, `model` and a map of questions typed+noul / choice / score, and reads `answers` keyed by its own ids plus+`usage.input_tokens/output_tokens`. This pins the mapping onto the `questions`+form of /v1/brio with the deterministic scoring engine of test_brio_api: what+each primitive puts in the prompt, what comes back, the confidence formula+their docs give, that any model name is accepted on this route, and that the+state is still photographed once for all the questions.+"""+import json+import math+import threading+import unittest+from urllib.error import HTTPError+from urllib.request import Request, urlopen++from openai_server import APIServer+from tests.test_brio_api import ScoringEngine++STATE = ("Hi, I've been trying to connect my Stripe account for 3 days and the "+ "integration keeps failing. I'm losing sales. Please help ASAP.")+++def confidence(ps):+ n = len(ps)+ return (n * max(ps) - 1.0) / (n - 1)+++class SystemOneApi(unittest.TestCase):+ def serve(self, table):+ self.engine = ScoringEngine(table)+ self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model")+ self.addCleanup(self.server.shutdown)+ self.addCleanup(self.server.server_close)+ threading.Thread(target=self.server.serve_forever, daemon=True).start()+ self.base = f"http://127.0.0.1:{self.server.server_port}"++ def post(self, body):+ request = Request(self.base + "/v1/systemone", data=json.dumps(body).encode(),+ headers={"Content-Type": "application/json"})+ with urlopen(request, timeout=10) as response:+ return json.loads(response.read())++ def post_error(self, body):+ try:+ self.post(body)+ except HTTPError as error:+ return error.code, json.loads(error.read())+ self.fail("expected an error")++ def pins(self):+ return [c["prompt"] for c in self.engine.calls if c["pin"]]++ def test_noul_is_the_probability_of_yes(self):+ self.serve({"yes": -0.1, "no": -2.0})+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {+ "urgency": {"type": "noul", "instructions": "Does this message express urgency?"}}})+ answer = out["answers"]["urgency"]+ self.assertEqual(answer["type"], "noul")+ want = math.exp(-0.1) / (math.exp(-0.1) + math.exp(-2.0))+ self.assertAlmostEqual(answer["noul"], want, places=5)+ self.assertNotIn("confidence", answer) # their docs: noul carries none+ # the question is asked as a yes/no question, the state as the context+ asked = [c["prompt"] for c in self.engine.calls if "Does this message" in c["prompt"]]+ self.assertTrue(asked and "Answer yes or no." in asked[0], asked[:1])++ def test_noul_criteria_go_into_the_question(self):+ self.serve({"yes": -0.1, "no": -2.0})+ self.post({"model": "jev-latest", "state": STATE, "questions": {+ "q": {"type": "noul", "instructions": "Is the customer at risk of churning?",+ "criteria": {"true": "they threaten to leave or mention losses",+ "false": "a routine question"}}}})+ pinned = self.pins()[1]+ self.assertIn("yes: they threaten to leave", pinned)+ self.assertIn("no: a routine question", pinned)++ def test_choice_labels_probabilities_and_confidence(self):+ self.serve({"billing": -0.2, "technical": -3.0, "sales": -4.0})+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {+ "department": {"type": "choice", "instructions": "Which team should handle this?",+ "criteria": {"billing": "payments, invoices, Stripe",+ "technical": "bugs and outages",+ "sales": "pricing and plans"}}}})+ answer = out["answers"]["department"]+ self.assertEqual(answer["type"], "choice")+ self.assertEqual(answer["choice"], "billing")+ self.assertEqual(list(answer["probabilities"]), ["billing", "technical", "sales"])+ self.assertAlmostEqual(sum(answer["probabilities"].values()), 1.0, places=5)+ self.assertAlmostEqual(answer["confidence"],+ confidence(list(answer["probabilities"].values())), places=5)+ # the descriptions are in the question, the labels are the options scored+ pinned = self.pins()[1]+ self.assertIn("- billing: payments, invoices, Stripe", pinned)+ self.assertIn("- sales: pricing and plans", pinned)+ scored = [c["prompt"] for c in self.engine.calls if not c["pin"]]+ self.assertTrue(any(p.endswith(" technical") for p in scored), scored[-3:])++ def test_choice_with_null_descriptions_still_lists_the_labels(self):+ self.serve({"a": -0.1, "b": -1.0})+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {+ "q": {"type": "choice", "criteria": {"a": None, "b": None}}}})+ self.assertEqual(out["answers"]["q"]["choice"], "a")+ self.assertIn("- a\n- b", self.pins()[1])++ def test_score_expected_value_legend_and_confidence(self):+ self.serve({"4": -0.1, "1": -5.0, "2": -5.0, "3": -5.0, "5": -5.0})+ levels = ["not urgent", "low", "moderate", "high", "critical"]+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {+ "urgency": {"type": "score", "instructions": "How urgent is this?", "criteria": levels}}})+ answer = out["answers"]["urgency"]+ self.assertEqual(answer["type"], "score")+ self.assertEqual(answer["legend"], {str(i + 1): d for i, d in enumerate(levels)})+ self.assertEqual(list(answer["probabilities"]), ["1", "2", "3", "4", "5"])+ ps = answer["probabilities"]+ self.assertAlmostEqual(sum(ps.values()), 1.0, places=5)+ self.assertAlmostEqual(answer["score"], sum(int(k) * v for k, v in ps.items()), places=5)+ self.assertGreater(answer["score"], 3.9) # the mass sits on level 4+ self.assertAlmostEqual(answer["confidence"], confidence(list(ps.values())), places=5)+ self.assertIn("4: high", self.pins()[1])++ def test_structured_state_and_instructions_are_serialized(self):+ self.serve({"yes": -0.1, "no": -2.0})+ out = self.post({"model": "jev-latest",+ "state": {"ticket": 4711, "text": "refund not received"},+ "questions": {"q": {"type": "noul",+ "instructions": {"ask": "is this about money?"}}}})+ self.assertEqual(out["model"], "test-model") # the served model answers+ state_pin = self.pins()[0]+ self.assertIn('"ticket": 4711', state_pin)+ self.assertIn('"ask": "is this about money?"', self.pins()[1])++ def test_state_is_photographed_once_for_all_questions(self):+ self.serve({"yes": -0.1, "no": -2.0, "a": -0.1, "b": -1.0})+ out = self.post({"model": "jev-latest", "state": STATE, "questions": {+ "one": {"type": "noul", "instructions": "q1?"},+ "two": {"type": "choice", "criteria": {"a": "x", "b": "y"}},+ "three": {"type": "score", "criteria": ["bad", "good"]}}})+ self.assertEqual(list(out["answers"]), ["one", "two", "three"])+ pins = self.pins()+ self.assertEqual(len(pins), 4, pins) # the state, then each question once+ self.assertEqual(pins[0], f"Context:\n{STATE}\n\n")+ self.assertEqual(set(out["usage"]), {"input_tokens", "output_tokens"})+ self.assertGreater(out["usage"]["input_tokens"], 0)+ self.assertGreater(out["usage"]["output_tokens"], 0)++ def test_validation_errors_are_422(self):+ self.serve({})+ code, body = self.post_error({"model": "jev-latest", "questions": {"q": {"type": "noul"}}})+ self.assertEqual(code, 422)+ self.assertIn("state", body["error"]["message"])+ code, body = self.post_error({"model": "jev-latest", "state": STATE,+ "questions": {"q": {"type": "guess"}}})+ self.assertEqual(code, 422)+ self.assertIn("type", body["error"]["message"])+ code, body = self.post_error({"model": "jev-latest", "state": STATE,+ "questions": {"q": {"type": "choice", "criteria": ["a", "b"]}}})+ self.assertEqual(code, 422)+ code, body = self.post_error({"model": "jev-latest", "state": STATE,+ "questions": {"q": {"type": "score", "criteria": ["only one"]}}})+ self.assertEqual(code, 422)+ self.assertIn("2 to 10", body["error"]["message"])+ code, body = self.post_error({"model": "jev-latest", "state": STATE, "questions": []})+ self.assertEqual(code, 422)++ def test_brio_route_still_checks_the_model_name(self):+ self.serve({"a": -0.1, "b": -1.0})+ request = Request(self.base + "/v1/brio",+ data=json.dumps({"model": "jev-latest", "state": STATE,+ "options": ["a", "b"]}).encode(),+ headers={"Content-Type": "application/json"})+ with self.assertRaises(HTTPError) as caught:+ urlopen(request, timeout=10)+ self.assertEqual(caught.exception.code, 404)+++if __name__ == "__main__":+ unittest.main()diff --git a/docs/api.md b/docs/api.mdindex 5d1bdc60..3d014bbc 100644--- a/docs/api.md+++ b/docs/api.md@@ -22,7 +22,9 @@ curl http://127.0.0.1:8000/v1/chat/completions \```Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,-`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and+`POST /v1/chat/completions`, legacy `POST /v1/completions`, `POST /v1/brio`+(closed-set scoring, [brio.md](brio.md)) and `POST /v1/systemone`, the+request and reply of TypeSafe's Jev API served by the same channel. Chat andcompletion requests support JSON responses, SSE streaming, usage counts,`max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to fourcustom `stop` sequences. `max_tokens` is a ceiling, not a target: when thediff --git a/docs/brio.md b/docs/brio.mdindex 0cc134fc..89a0ae73 100644--- a/docs/brio.md+++ b/docs/brio.md@@ -181,6 +181,74 @@ const { json, fields } = await r.json();// json.queue, json.urgent are guaranteed to be values from your lists```+### For code written against Jev: `POST /v1/systemone`++If your application already speaks TypeSafe's Jev API, point it at colibri+and change the base URL; the request and the reply are the same shape. The+route accepts any `model` name (a Jev client sends `jev-latest`) and answers+with the model the server actually runs.++```bash+curl -s http://127.0.0.1:8000/v1/systemone \+ -H "Authorization: Bearer $COLI_API_KEY" -H "Content-Type: application/json" \+ -d '{+ "state": "Hi, I have been trying to connect my Stripe account for 3 days and the integration keeps failing. I am losing sales. Please help ASAP.",+ "model": "jev-latest",+ "questions": {+ "urgency": {"type": "noul", "instructions": "Does this message express urgency?"},+ "department": {"type": "choice", "instructions": "Which team should handle this?",+ "criteria": {"billing": "payments, invoices, Stripe payouts",+ "technical": "bugs, outages, integration errors",+ "sales": "pricing, plans, upgrades"}},+ "severity": {"type": "score", "instructions": "How severe is the customer impact?",+ "criteria": ["no impact", "minor inconvenience", "blocked on one task",+ "losing money", "business down"]}+ }}'+```++The reply, from Qwen3.6-35B-A3B on a CPU box (1m46 with the experts streamed+from disk, the state read once for the three questions):++```json+{"model": "qwen36",+ "answers": {+ "urgency": {"type": "noul", "noul": 0.847606},+ "department": {"type": "choice", "choice": "technical",+ "probabilities": {"billing": 0.059806, "technical": 0.939983, "sales": 0.000211},+ "confidence": 0.909974},+ "severity": {"type": "score", "score": 3.831772,+ "legend": {"1": "no impact", "2": "minor inconvenience", "3": "blocked on one task",+ "4": "losing money", "5": "business down"},+ "probabilities": {"1": 0.044403, "2": 0.02624, "3": 0.034899, "4": 0.842097, "5": 0.052361},+ "confidence": 0.802621}},+ "usage": {"input_tokens": 86, "output_tokens": 15}}+```++How the three primitives map onto the `questions` form above, so you know+what the model is actually asked:++| Jev primitive | what is scored | what goes into the question text | reply |+|---|---|---|---|+| `noul` | `yes` / `no` | `instructions`, then `yes: <criteria.true>` and `no: <criteria.false>` when given, then "Answer yes or no." | `noul` = probability of yes |+| `choice` | the labels of `criteria` (up to 255) | `instructions`, then one line per label with its description | `choice`, `probabilities` by label, `confidence` |+| `score` | the level numbers `"1".."n"` (2 to 10 levels) | `instructions`, then `k: <description>` per level, then "Answer with the number." | `score` = expected value under the distribution, `legend` by number, `probabilities`, `confidence` |++`state` and `instructions` may be a string, an object or an array; JSON is+serialized as text. `confidence` is the formula their documentation gives,+`(n * peak - 1) / (n - 1)`: 1 when all the mass sits on one label, 0 when+flat; `noul` carries none, as in theirs.++What differs, stated rather than hidden:++- `model` in the reply is the served model, not a Jev version.+- `usage.output_tokens` counts the option tokens the engine **read**; nothing+ is generated. `input_tokens` is the longest prompt of the request.+- Validation errors are `422`, as theirs, with this server's error envelope+ (`error.message`, `error.param`).+- The probabilities are this model's, normalised over the options with the+ `mean` rule above. The numbers will not match Jev's on the same input; the+ contract is the same, the model is yours.+### Do not put the options in the promptWrite the state and the question; leave the option list to the `options` field. On a