Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

systemone: immediate replies, honest normalisation, photos that pay

JustVugg751850e4c88f
diff --git a/c/coli b/c/coli
index eb379f77..d018f31c 100755
--- a/c/coli
+++ b/c/coli
@@ -3479,7 +3479,8 @@ def main():
ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000)
ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID"))
ps.add_argument("--api-key",default=os.environ.get("COLI_API_KEY"))
- ps.add_argument("--cors-origin",action="append",default=None)
+ ps.add_argument("--cors-origin",action="append",default=None,
+ help="allowed browser origin, repeatable; plain values replace the default list, +ORIGIN adds to it")
ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8")))
ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300")))
ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1")))
@@ -3506,7 +3507,8 @@ def main():
for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)),
("--model-id",dict(default=os.environ.get("COLI_MODEL_ID"))),
("--api-key",dict(default=os.environ.get("COLI_API_KEY"))),
- ("--cors-origin",dict(action="append",default=None)),
+ ("--cors-origin",dict(action="append",default=None,
+ help="allowed browser origin, repeatable; plain values replace the default list, +ORIGIN adds to it")),
("--max-queue",dict(type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8")))),
("--queue-timeout",dict(type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300")))),
("--kv-slots",dict(type=int,default=int(os.environ.get("COLI_KV_SLOTS","1")))),
diff --git a/c/openai_server.py b/c/openai_server.py
index d135f391..84922e5e 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -5777,6 +5777,19 @@ IMAGE_OPTION_KEYS = ("default_width", "default_height", "default_steps", "min_si
"max_side", "multiple")
+def cors_origin_list(given):
+ """The allowed browser origins from --cors-origin values. None keeps the
+ defaults. A value written `+origin` adds to the list instead of replacing
+ it: only `+` values extend the defaults, and plain values replace them as
+ they always have, with any `+` values added after."""
+ if given is None:
+ return DEFAULT_CORS_ORIGINS
+ plain = [origin for origin in given if not origin.startswith("+")]
+ added = [origin[1:] for origin in given if origin.startswith("+") and origin[1:]]
+ base = list(DEFAULT_CORS_ORIGINS) if not plain else plain
+ return tuple(dict.fromkeys(base + added))
+
+
def _positive_env(name, default):
try:
value = int(os.environ.get(name, "") or default)
@@ -5826,6 +5839,12 @@ class APIServer(ThreadingHTTPServer):
self.allowed_hosts = tuple(
h.strip().lower() for h in allowed_hosts if h and h.strip())
self.created = int(time.time())
+ # Hashes of the states /v1/systemone scored lately: a state seen again is
+ # worth a photo, one seen once is probably a feed (systemone's pin rule).
+ self._states_seen = collections.OrderedDict()
+ self._states_lock = threading.Lock()
+ # slot -> (fixed prefix, its tokens) photographed by /v1/systemone
+ self.pinned_prefixes = {}
self._conn_lock = threading.Lock()
self._conn_live = 0
self._conn_by_ip = {}
@@ -5837,14 +5856,33 @@ class APIServer(ThreadingHTTPServer):
without trial and error."""
entry = model_object(self.model_id, self.created)
entry["input_modalities"] = self.input_modalities()
+ entry["capabilities"] = self.capabilities()
if is_image_engine(self.engine):
info = getattr(self.engine, "info", None) or {}
- entry["capabilities"] = ["image_generation"]
entry["image"] = {key: info.get(key) for key in IMAGE_OPTION_KEYS}
- elif engine_decides(self.engine) and not engine_chats(self.engine):
- entry["capabilities"] = ["decision"]
return entry
+ def state_seen(self, state, capacity=256):
+ """True if `state` was scored before (and remember it either way)."""
+ key = hashlib.sha1(state.encode("utf-8", "replace")).digest()
+ with self._states_lock:
+ seen = key in self._states_seen
+ self._states_seen[key] = True
+ self._states_seen.move_to_end(key)
+ while len(self._states_seen) > capacity:
+ self._states_seen.popitem(last=False)
+ return seen
+
+ def capabilities(self):
+ """What the served model does, for /v1/models and /health: an image model
+ draws (image_generation); every text engine answers POST /v1/systemone
+ (systemone), and a decision engine does only that (decision)."""
+ if is_image_engine(self.engine):
+ return ["image_generation"]
+ if engine_decides(self.engine) and not engine_chats(self.engine):
+ return ["systemone", "decision"]
+ return ["chat", "systemone"]
+
def jev_model_card(self):
"""The served model as Jev's GET /v1/models lists it. colibri does not know a
model's release date; it gives the day this server started."""
@@ -5971,6 +6009,12 @@ class _DeadlineReader:
class APIHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
+ # TCP_NODELAY on every connection. A response is the header block and the
+ # body in two writes; with Nagle on, the body waits for the client's delayed
+ # ACK of the headers on a kept-alive connection, about 40 ms per request
+ # (measured: 44 ms median for a /v1/systemone round trip on loopback, 1 ms
+ # without). That is most of a decision's time on a fast engine.
+ disable_nagle_algorithm = True
timeout = 30 # per socket OPERATION. On its own this does not stop a slowloris:
# it restarts on every byte received, so a drip renews it forever.
# READ_DEADLINE below is the cumulative bound that actually does.
@@ -6100,7 +6144,8 @@ class APIHandler(BaseHTTPRequestHandler):
self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type, x-api-key, anthropic-version, "
"X-TypeSafe-SDK, X-TypeSafe-Runtime, X-TypeSafe-Retry-Count")
self.send_header("Access-Control-Expose-Headers",
- "x-request-id, x-typesafe-request-id, x-colibri-queue-wait-ms, Retry-After")
+ "x-request-id, x-typesafe-request-id, x-colibri-queue-wait-ms, "
+ "x-colibri-elapsed-ms, x-colibri-engine-ms, Retry-After")
self.send_header("Access-Control-Max-Age", "600")
if "*" not in self.server.cors_origins:
self.send_header("Vary", "Origin")
@@ -6262,11 +6307,9 @@ class APIHandler(BaseHTTPRequestHandler):
if tiers: payload["tiers"] = tiers
hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None
if hwinfo: payload["hwinfo"] = hwinfo
+ payload["capabilities"] = self.server.capabilities()
if is_image_engine(self.server.engine):
- payload["capabilities"] = ["image_generation"]
payload["image"] = dict(getattr(self.server.engine, "info", None) or {})
- elif engine_decides(self.server.engine) and not engine_chats(self.server.engine):
- payload["capabilities"] = ["decision"]
self.send_json(200, payload, request_id)
return
if path == "/experts":
@@ -6520,7 +6563,15 @@ class APIHandler(BaseHTTPRequestHandler):
or not 0 <= cache_slot < self.server.kv_slots:
raise APIError(400, "Invalid cache slot.", "cache_slot")
- state_prefix = f"Context:\n{state}\n\n" if state else ""
+ # /v1/systemone's two knobs, private keys like _max_options: `_prefix`
+ # is a fixed text the client sends before a changing state (a game's
+ # rules), photographed so only the state and the question are read;
+ # `_pin_state` False skips the photo of the state, which costs a round
+ # trip and a snapshot and pays only if the state comes back.
+ fixed = body.get("_prefix") or ""
+ fixed_prefix = f"{fixed}\n\n" if fixed else ""
+ pin_state = body.get("_pin_state", True)
+ state_prefix = fixed_prefix + (f"Context:\n{state}\n\n" if state else "")
started = time.time()
read_total = 0
prompt_max = 0
@@ -6552,8 +6603,13 @@ class APIHandler(BaseHTTPRequestHandler):
"""Fotografa `prefix`, poi un giro per opzione: ognuna paga solo
i propri token. Torna (scored, entropia, token del prefisso)."""
nonlocal read_total, prompt_max
- n_prefix, _ = score(prefix, True)
+ n_prefix, fresh = score(prefix, True)
prompt_max = max(prompt_max, n_prefix)
+ known = self.server.pinned_prefixes.get(cache_slot)
+ if known and known[0] == fixed_prefix and len(fresh) > n_prefix - known[1]:
+ # The engine read the fixed prefix again: its photo was
+ # evicted. Forget it, so the next request takes it anew.
+ self.server.pinned_prefixes.pop(cache_slot, None)
scored = []
for option in choices:
# Il cliente se n'e andato: smettere subito invece di
@@ -6608,7 +6664,16 @@ class APIHandler(BaseHTTPRequestHandler):
# modalita. Con essa, ogni domanda successiva paga solo i propri
# token. Il costo e' uno snapshot in piu' su una richiesta one-shot,
# riusato o sfrattato.
- if state_prefix:
+ # The fixed prefix is photographed once per slot, not per request:
+ # sending it again would read it again (an identical prompt is not a
+ # strict prefix of itself, so no photo can resume it), which is the
+ # cost the prefix exists to save. If the photo is evicted, the next
+ # question shows it (it reads the prefix fresh) and it is retaken.
+ if fixed_prefix and self.server.pinned_prefixes.get(cache_slot, ("",))[0] != fixed_prefix:
+ n_fixed, _ = score(fixed_prefix, True)
+ prompt_max = max(prompt_max, n_fixed)
+ self.server.pinned_prefixes[cache_slot] = (fixed_prefix, n_fixed)
+ if state_prefix and state_prefix != fixed_prefix and pin_state:
n_state, _ = score(state_prefix, True)
prompt_max = max(prompt_max, n_state)
@@ -6784,16 +6849,40 @@ class APIHandler(BaseHTTPRequestHandler):
[self._systemone_legend(c, i) for i, c in enumerate(criteria)]))
else:
raise APIError(422, f"`{where}.type` must be \"noul\", \"choice\" or \"score\".", f"{where}.type")
+ options = self._systemone_options(body)
if engine_decides(self.server.engine):
- self._systemone_decide(body, plan, request_id)
+ if options["prefix"]:
+ raise APIError(422, "`prefix` is a language-model option (a fixed text photographed "
+ "before a changing state); a decision engine reads the whole "
+ "request in one pass: put that text in `state` or `instructions`.",
+ "prefix")
+ self._systemone_decide(body, plan, request_id, options["cache_slot"])
return
# A question with one option has its answer already: nothing to score.
asked = [entry for entry in plan if len(entry[3]) > 1]
result = {"answers": [], "usage": {"prompt_tokens": 0, "read_tokens": 0}}
if asked:
+ # Photograph the state when it can pay: within this request (two or
+ # more questions read it), or because it came back from an earlier
+ # one. A state that changes on every call (a game, a feed) is read
+ # straight through instead. `pin_state` says it explicitly.
+ if options["pin_state"] is None:
+ options["pin_state"] = len(asked) > 1 or self.server.state_seen(state or "")
+ else:
+ self.server.state_seen(state or "")
inner = {"state": state or "", "_max_options": 255, "_empty_state_ok": True,
- "questions": [{"question": text, "options": options}
- for _, _, text, options, _ in asked]}
+ "normalize": options["normalize"], "_pin_state": options["pin_state"],
+ "questions": [{"question": text, "options": choices}
+ for _, _, text, choices, _ in asked]}
+ if options["prefix"]:
+ inner["_prefix"] = options["prefix"]
+ if options["cache_slot"] is not None:
+ inner["cache_slot"] = options["cache_slot"]
+ elif options["prefix"]:
+ # The photo of the prefix lives in a slot: route by the part
+ # that does not change, as the state is routed without one.
+ inner["cache_slot"] = conversation_cache_slot(
+ [{"role": "system", "content": options["prefix"]}], self.server.kv_slots)
result = self.brio(inner, request_id, send=False)
scored = iter(result["answers"])
answers = {}
@@ -6821,6 +6910,36 @@ class APIHandler(BaseHTTPRequestHandler):
result["usage"]["read_tokens"]),
request_id, result.get("_headers"))
+ def _systemone_options(self, body):
+ """colibri's optional fields on /v1/systemone, none of which a Jev client
+ sends (docs/brio.md lists them):
+
+ normalize "sum" (default) or "mean": how a language model's option
+ log-probabilities become one score per option
+ pin_state true / false: photograph the state for the next request;
+ omitted, the server decides (see systemone)
+ prefix a fixed text read before the state and always photographed
+ cache_slot the KV slot, as on the chat endpoints
+
+ On a decision engine normalize and pin_state have nothing to act on and
+ are ignored; prefix is refused."""
+ normalize = body.get("normalize", "sum")
+ if normalize not in ("sum", "mean"):
+ raise APIError(422, "`normalize` must be \"sum\" or \"mean\".", "normalize")
+ pin_state = body.get("pin_state")
+ if pin_state is not None and not isinstance(pin_state, bool):
+ raise APIError(422, "`pin_state` must be true or false.", "pin_state")
+ prefix = body.get("prefix")
+ if prefix is not None and (not isinstance(prefix, str) or not prefix.strip()):
+ raise APIError(422, "`prefix` must be a non-empty string.", "prefix")
+ cache_slot = body.get("cache_slot")
+ if cache_slot is not None and (isinstance(cache_slot, bool) or not isinstance(cache_slot, int)
+ or not 0 <= cache_slot < self.server.kv_slots):
+ raise APIError(422, f"`cache_slot` must be an integer from 0 to {self.server.kv_slots - 1}.",
+ "cache_slot")
+ return {"normalize": normalize, "pin_state": pin_state, "prefix": prefix,
+ "cache_slot": cache_slot}
+
def _systemone_reply(self, answers, request_id, input_tokens, output_tokens):
"""The reply both paths send, in the Jev shape: `model` is the served
model, `provider` says who answered, and `usage.cost` is what this server
@@ -6830,13 +6949,14 @@ class APIHandler(BaseHTTPRequestHandler):
"usage": {"input_tokens": int(input_tokens), "output_tokens": int(output_tokens),
"cost": 0}}
- def _systemone_decide(self, body, plan, request_id):
+ def _systemone_decide(self, body, plan, request_id, cache_slot=None):
"""The native path: the request as one DECIDE record, the engine's
probabilities back, shaped exactly like the LLM path's reply. No prompt is
rendered and no option is scored on its own: one round trip, one forward."""
record = systemone_decision_record(body)
- cache_slot = conversation_cache_slot([{"role": "system", "content": record["state"]}],
- self.server.kv_slots)
+ if cache_slot is None:
+ cache_slot = conversation_cache_slot([{"role": "system", "content": record["state"]}],
+ self.server.kv_slots)
started = time.time()
with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:
queue_wait, cache_slot = admission
@@ -7942,7 +8062,7 @@ def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,
if allowed_hosts and "*" in allowed_hosts:
print("WARNING: --allowed-host '*' accepts ANY Host header "
"(DNS-rebinding guard disabled)", file=sys.stderr)
- origins = DEFAULT_CORS_ORIGINS if cors_origins is None else tuple(cors_origins)
+ origins = cors_origin_list(cors_origins)
# Bind before starting the 744B engine. A stale/occupied port must fail in
# milliseconds rather than loading hundreds of GB and leaking a child.
server = APIServer((host, port), None, model_id, api_key, max_tokens, origins,
@@ -8023,7 +8143,8 @@ def main():
parser.add_argument("--model-id", default=os.environ.get("COLI_MODEL_ID"))
parser.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))
parser.add_argument("--cors-origin", action="append", default=None,
- help="allowed browser origin; repeat as needed (use '*' for any origin)")
+ help="allowed browser origin; repeat as needed (use '*' for any origin). "
+ "Plain values replace the default list; +ORIGIN adds to it")
# Absent = not explicitly set: mirrors coli's --cap (see cap_for_arch and issue
# #379 -- glm arch resolves platform-aware, non-glm gets the legacy 8). An
# explicit value, 0 included, reaches the engine verbatim.
diff --git a/c/tests/test_decision_serve.py b/c/tests/test_decision_serve.py
index b6e10515..3820d66e 100644
--- a/c/tests/test_decision_serve.py
+++ b/c/tests/test_decision_serve.py
@@ -263,9 +263,34 @@ class DecisionGateway(unittest.TestCase):
self.serve(FakeDecisionEngine({}))
with urlopen(self.base + "/v1/models", timeout=10) as response:
card = json.loads(response.read())["data"][0]
- self.assertEqual(card["capabilities"], ["decision"])
+ self.assertEqual(card["capabilities"], ["systemone", "decision"])
with urlopen(self.base + "/health", timeout=10) as response:
- self.assertEqual(json.loads(response.read())["capabilities"], ["decision"])
+ self.assertEqual(json.loads(response.read())["capabilities"], ["systemone", "decision"])
+
+ def test_colibris_options_on_a_decision_engine(self):
+ """normalize and pin_state act on a language model's logprobs and photos;
+ a decision engine has neither and ignores them. A prefix is refused: it
+ would be text the model never reads. cache_slot reaches the engine."""
+ engine = FakeDecisionEngine({"q": [0.5, 0.5]})
+ engine.kv_slots = 2
+ engine.slots = []
+ decide = engine.decide
+ engine.decide = lambda record, cache_slot=0, cancelled=None: (
+ engine.slots.append(cache_slot), decide(record, cache_slot, cancelled))[1]
+ self.engine = engine
+ self.server = APIServer(("127.0.0.1", 0), engine, "laya", kv_slots=2)
+ self.addCleanup(self.server.server_close)
+ self.addCleanup(self.server.shutdown)
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
+ self.base = f"http://127.0.0.1:{self.server.server_port}"
+ question = {"q": {"type": "noul", "instructions": "?"}}
+ out, _ = self.post("/v1/systemone", {"state": "x", "questions": question,
+ "normalize": "mean", "pin_state": True, "cache_slot": 1})
+ self.assertEqual(out["answers"]["q"]["noul"], 0.5)
+ self.assertEqual(engine.slots, [1])
+ status, error = self.post_error("/v1/systemone", {"state": "x", "questions": question,
+ "prefix": "rules"})
+ self.assertEqual((status, error["error"]["param"]), (422, "prefix"))
def test_validation_is_the_routes_own(self):
self.serve(FakeDecisionEngine({}))
diff --git a/c/tests/test_systemone_api.py b/c/tests/test_systemone_api.py
index acf6dcb0..9005ce5a 100644
--- a/c/tests/test_systemone_api.py
+++ b/c/tests/test_systemone_api.py
@@ -9,14 +9,17 @@ each primitive puts in the prompt, what comes back, the confidence formula
their docs give, that any model name is accepted on this route, and that the
state is still photographed once for all the questions.
"""
+import http.client
import json
import math
import threading
+import time
import unittest
from urllib.error import HTTPError
from urllib.request import Request, urlopen
-from openai_server import APIServer
+import openai_server
+from openai_server import APIServer, cors_origin_list
if __package__:
from .test_brio_api import ScoringEngine
else:
@@ -31,10 +34,22 @@ def confidence(ps):
return (n * max(ps) - 1.0) / (n - 1)
+class SlotScoringEngine(ScoringEngine):
+ """ScoringEngine that also writes down the KV slot of every call."""
+
+ def __init__(self, table, kv_slots=1):
+ super().__init__(table)
+ self.kv_slots = kv_slots
+
+ def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0, *args, **kwargs):
+ super().generate(prompt, max_tokens, temperature, top_p, on_text, cache_slot, *args, **kwargs)
+ self.calls[-1]["slot"] = cache_slot
+
+
class SystemOneApi(unittest.TestCase):
- def serve(self, table):
- self.engine = ScoringEngine(table)
- self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model")
+ def serve(self, table, kv_slots=1):
+ self.engine = SlotScoringEngine(table, kv_slots)
+ self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model", kv_slots=kv_slots)
self.addCleanup(self.server.shutdown)
self.addCleanup(self.server.server_close)
threading.Thread(target=self.server.serve_forever, daemon=True).start()
@@ -75,7 +90,7 @@ class SystemOneApi(unittest.TestCase):
"q": {"type": "noul", "instructions": "Is the customer at risk of churning?",
"criteria": {"true": "they threaten to leave or mention losses",
"false": "a routine question"}}}})
- pinned = self.pins()[1]
+ pinned = self.pins()[-1]
self.assertIn("yes: they threaten to leave", pinned)
self.assertIn("no: a routine question", pinned)
@@ -94,7 +109,7 @@ class SystemOneApi(unittest.TestCase):
self.assertAlmostEqual(answer["confidence"],
confidence(list(answer["probabilities"].values())), places=5)
# the descriptions are in the question, the labels are the options scored
- pinned = self.pins()[1]
+ pinned = self.pins()[-1]
self.assertIn("- billing: payments, invoices, Stripe", pinned)
self.assertIn("- sales: pricing and plans", pinned)
scored = [c["prompt"] for c in self.engine.calls if not c["pin"]]
@@ -105,7 +120,7 @@ class SystemOneApi(unittest.TestCase):
out = self.post({"model": "jev-latest", "state": STATE, "questions": {
"q": {"type": "choice", "criteria": {"a": None, "b": None}}}})
self.assertEqual(out["answers"]["q"]["choice"], "a")
- self.assertIn("- a\n- b", self.pins()[1])
+ self.assertIn("- a\n- b", self.pins()[-1])
def test_score_expected_value_legend_and_confidence(self):
# Jev numbers the levels from zero ("Each description's position determines
@@ -123,7 +138,7 @@ class SystemOneApi(unittest.TestCase):
self.assertAlmostEqual(answer["score"], sum(int(k) * v for k, v in ps.items()), places=5)
self.assertGreater(answer["score"], 2.9) # the mass sits on level 3, "high"
self.assertAlmostEqual(answer["confidence"], confidence(list(ps.values())), places=5)
- self.assertIn("4: high", self.pins()[1])
+ self.assertIn("4: high", self.pins()[-1])
def test_legend_gives_back_the_criteria_as_sent(self):
self.serve({"1": -0.1, "2": -3.0})
@@ -173,9 +188,9 @@ class SystemOneApi(unittest.TestCase):
"questions": {"q": {"type": "noul",
"instructions": {"ask": "is this about money?"}}}})
self.assertEqual(out["model"], "test-model") # the served model answers
- state_pin = self.pins()[0]
- self.assertIn('"ticket": 4711', state_pin)
- self.assertIn('"ask": "is this about money?"', self.pins()[1])
+ question_pin = self.pins()[-1] # the state, then the question
+ self.assertIn('"ticket": 4711', question_pin)
+ self.assertIn('"ask": "is this about money?"', question_pin)
def test_state_is_photographed_once_for_all_questions(self):
self.serve({"yes": -0.1, "no": -2.0, "a": -0.1, "b": -1.0})
@@ -210,6 +225,100 @@ class SystemOneApi(unittest.TestCase):
code, body = self.post_error({"model": "jev-latest", "state": STATE, "questions": []})
self.assertEqual(code, 422)
+ # ---- colibri's options (none of which a Jev client sends) --------------
+ def test_normalize_defaults_to_sum_and_mean_can_be_asked(self):
+ # "a b" reads two tokens at -1.0 each, "c" one at -1.5: the joint
+ # probability (sum, -2.0 vs -1.5) picks "c", the per-token mean "a b".
+ question = {"q": {"type": "choice", "criteria": {"a b": None, "c": None}}}
+ self.serve({"a b": -1.0, "c": -1.5})
+ self.assertEqual(self.post({"state": STATE, "questions": question})["answers"]["q"]["choice"], "c")
+ out = self.post({"state": STATE, "questions": question, "normalize": "mean"})
+ self.assertEqual(out["answers"]["q"]["choice"], "a b")
+ code, body = self.post_error({"state": STATE, "questions": question, "normalize": "max"})
+ self.assertEqual((code, body["error"]["param"]), (422, "normalize"))
+
+ def test_a_state_seen_once_is_not_photographed(self):
+ self.serve({"yes": -0.1, "no": -2.0})
+ question = {"q": {"type": "noul", "instructions": "Urgent?"}}
+ state_pins = lambda: [p for p in self.pins() if p.endswith("\n\n") and "Question:" not in p]
+ self.post({"state": "frame 1", "questions": question})
+ self.post({"state": "frame 2", "questions": question})
+ self.assertEqual(state_pins(), []) # a feed: never read twice
+ self.post({"state": "frame 2", "questions": question})
+ self.assertEqual(state_pins(), ["Context:\nframe 2\n\n"]) # it came back: worth it
+ self.post({"state": "frame 3", "questions": dict(question, other={"type": "noul"})})
+ self.assertEqual(state_pins()[-1], "Context:\nframe 3\n\n") # two questions share it
+ before = len(state_pins())
+ self.post({"state": "frame 4", "questions": dict(question, other={"type": "noul"}),
+ "pin_state": False})
+ self.assertEqual(len(state_pins()), before)
+ self.post({"state": "frame 5", "questions": question, "pin_state": True})
+ self.assertEqual(state_pins()[-1], "Context:\nframe 5\n\n")
+ code, body = self.post_error({"state": "x", "questions": question, "pin_state": "yes"})
+ self.assertEqual((code, body["error"]["param"]), (422, "pin_state"))
+
+ def test_a_fixed_prefix_is_photographed_before_the_state(self):
+ self.serve({"yes": -0.1, "no": -2.0})
+ rules = "You play Flappy Bird. Flap when the pipe gap is above the bird."
+ for frame in ("bird y=10, gap y=40", "bird y=12, gap y=40"):
+ self.post({"state": frame, "prefix": rules,
+ "questions": {"flap": {"type": "noul", "instructions": "Flap now?"}}})
+ pins = self.pins()
+ self.assertEqual(pins.count(rules + "\n\n"), 2)
+ question_pins = [p for p in pins if "Question:" in p]
+ self.assertTrue(all(p.startswith(rules + "\n\nContext:\nbird y=") for p in question_pins))
+ code, body = self.post_error({"state": "x", "prefix": "", "questions": {"q": {"type": "noul"}}})
+ self.assertEqual((code, body["error"]["param"]), (422, "prefix"))
+
+ def test_cache_slot_is_forwarded_and_the_default_is_unchanged(self):
+ self.serve({"yes": -0.1, "no": -2.0}, kv_slots=2)
+ question = {"q": {"type": "noul", "instructions": "Urgent?"}}
+ self.post({"state": STATE, "questions": question, "cache_slot": 1})
+ self.assertEqual({c["slot"] for c in self.engine.calls}, {1})
+ self.engine.calls.clear()
+ self.post({"state": STATE, "questions": question})
+ want = openai_server.conversation_cache_slot([{"role": "system", "content": STATE}], 2)
+ self.assertEqual({c["slot"] for c in self.engine.calls}, {want})
+ code, body = self.post_error({"state": STATE, "questions": question, "cache_slot": 2})
+ self.assertEqual((code, body["error"]["param"]), (422, "cache_slot"))
+
+ def test_models_says_the_model_answers_systemone(self):
+ self.serve({})
+ with urlopen(self.base + "/v1/models", timeout=10) as response:
+ card = json.loads(response.read())["data"][0]
+ self.assertEqual(card["capabilities"], ["chat", "systemone"])
+
+ def test_a_browser_can_read_the_timing_headers(self):
+ self.serve({"yes": -0.1, "no": -2.0})
+ request = Request(self.base + "/v1/systemone",
+ data=json.dumps({"state": STATE, "questions": {
+ "q": {"type": "noul", "instructions": "Urgent?"}}}).encode(),
+ headers={"Content-Type": "application/json",
+ "Origin": "http://localhost:5173"})
+ with urlopen(request, timeout=10) as response:
+ exposed = {h.strip().lower() for h in
+ response.headers["Access-Control-Expose-Headers"].split(",")}
+ self.assertIn("x-colibri-elapsed-ms", response.headers)
+ self.assertTrue({"x-colibri-elapsed-ms", "x-colibri-queue-wait-ms",
+ "x-colibri-engine-ms", "x-typesafe-request-id"} <= exposed)
+
+ def test_a_kept_alive_round_trip_does_not_wait_for_a_delayed_ack(self):
+ """The header block and the body are two writes. With Nagle on, the body
+ waits for the client's delayed ACK of the headers: about 40 ms per request
+ on loopback (measured 44 ms median before TCP_NODELAY, 1 ms after)."""
+ self.serve({"yes": -0.1, "no": -2.0})
+ connection = http.client.HTTPConnection("127.0.0.1", self.server.server_port, timeout=10)
+ self.addCleanup(connection.close)
+ body = json.dumps({"state": STATE, "questions": {"q": {"type": "noul", "instructions": "?"}}})
+ times = []
+ for _ in range(15):
+ started = time.perf_counter()
+ connection.request("POST", "/v1/systemone", body, {"Content-Type": "application/json"})
+ connection.getresponse().read()
+ times.append((time.perf_counter() - started) * 1e3)
+ median = sorted(times[3:])[len(times[3:]) // 2]
+ self.assertLess(median, 20.0, f"keep-alive round trips: {[round(t, 1) for t in times]} ms")
+
def test_validation_errors_carry_jevs_detail_list(self):
self.serve({})
code, body = self.post_error({"model": "jev-latest", "state": STATE, "questions": {
@@ -240,5 +349,16 @@ class SystemOneApi(unittest.TestCase):
self.assertEqual(caught.exception.code, 404)
+class CorsOrigins(unittest.TestCase):
+ def test_plus_adds_to_the_defaults_and_plain_values_replace_them(self):
+ defaults = openai_server.DEFAULT_CORS_ORIGINS
+ self.assertEqual(cors_origin_list(None), defaults)
+ self.assertEqual(cors_origin_list(["+http://game.local:8080"]),
+ (*defaults, "http://game.local:8080"))
+ self.assertEqual(cors_origin_list(["https://ui.example"]), ("https://ui.example",))
+ self.assertEqual(cors_origin_list(["https://ui.example", "+http://game.local:8080"]),
+ ("https://ui.example", "http://game.local:8080"))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/docs/SETTINGS.md b/docs/SETTINGS.md
index 5d5f0074..919530f3 100644
--- a/docs/SETTINGS.md
+++ b/docs/SETTINGS.md
@@ -67,7 +67,7 @@ Flags may also be given **after** the subcommand. Most flags map onto an engine
| `--port` | `8000` | Port. |
| `--model-id` | `$COLI_MODEL_ID` or `glm-5.2-colibri` | Model id reported by the API. |
| `--api-key` | `$COLI_API_KEY` | Require this bearer token. |
-| `--cors-origin` | none (repeatable) | Allowed CORS origin(s). |
+| `--cors-origin` | none (repeatable) | Allowed CORS origin(s). A plain value replaces the default list; `+ORIGIN` adds to it. |
| `--allowed-host` | `$COLI_ALLOWED_HOSTS` or none (repeatable) | Additional Host header accepted by the DNS-rebinding guard. |
| `--max-queue` | `$COLI_MAX_QUEUE` or `8` | Max queued requests. |
| `--queue-timeout` | `$COLI_QUEUE_TIMEOUT` or `300` | Seconds a request may wait. |
@@ -111,7 +111,7 @@ Run directly (or via `coli serve`). OpenAI-compatible `/v1/chat/completions`.
| `--port` | `8000` | Port. |
| `--model-id` | `$COLI_MODEL_ID` or `glm-5.2-colibri` | Model id in API responses. |
| `--api-key` | `$COLI_API_KEY` | Required bearer token. |
-| `--cors-origin` | none (repeatable) | Allowed CORS origin(s). |
+| `--cors-origin` | none (repeatable) | Allowed CORS origin(s). A plain value replaces the default list; `+ORIGIN` adds to it. |
| `--allowed-host` | `$COLI_ALLOWED_HOSTS` or none (repeatable) | Additional Host header accepted by the DNS-rebinding guard. |
| `--cap` | `0` (auto) | Expert-cache cap; `0` = engine default (`8`, or `1` on Metal + macOS + fast model volume — #379). |
| `--max-tokens` | `1024` | Default max completion tokens. |