Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(api): honor COLI_TEMP as the request default
Blaked7972b78f465
diff --git a/c/coli b/c/coliindex dc608ab7..8614b245 100755--- a/c/coli+++ b/c/coli@@ -1321,6 +1321,7 @@ def cmd_serve(a):"olmoe-colibri" if arch=="olmoe" else"glm-5.2-colibri")try:+ if a.temp is not None: os.environ["COLI_TEMP"] = str(a.temp)openai_server.serve(a.model,a.host,a.port,model_id,a.api_key,a.cap,ngen_for(a,interactive=True),engine,env_for_engine(a,arch),a.cors_origin,a.max_queue,a.queue_timeout,a.kv_slots,allowed_hosts=a.allowed_host)diff --git a/c/openai_server.py b/c/openai_server.pyindex 4e901768..47920643 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1622,7 +1622,16 @@ def generation_options(body, limit):maximum = limittemperature = body.get("temperature")top_p = body.get("top_p")- temperature = 0.7 if temperature is None else temperature+ if temperature is None:+ # The launcher publishes --temp through COLI_TEMP (#509, #968). The+ # gateway must use that value as its request default or the SERVE frame+ # replaces it with 0.7 before any engine can honor the setting.+ try:+ temperature = float(os.environ.get("COLI_TEMP", "0.7"))+ if not math.isfinite(temperature) or not 0 <= temperature <= 2:+ temperature = 0.7+ except ValueError:+ temperature = 0.7top_p = 0.9 if top_p is None else top_pif isinstance(maximum, bool) or not isinstance(maximum, int) or maximum < 1:raise APIError(400, f"`{maximum_param}` must be a positive integer.", maximum_param)diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex d9c02788..f8468595 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -187,6 +187,15 @@ class TemplateTest(unittest.TestCase):opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)self.assertEqual(opts[3], "not a grammar ::=")+ def test_coli_temp_is_the_default_for_requests_that_omit_temperature(self):+ with patch.dict("openai_server.os.environ", {"COLI_TEMP": "0.25"}):+ self.assertEqual(generation_options({}, 8)[1], 0.25)+ self.assertEqual(generation_options({"temperature": 0}, 8)[1], 0.0)+ for invalid in ("malformed", "5", "-1", "nan", "1e999"):+ with self.subTest(invalid=invalid):+ with patch.dict("openai_server.os.environ", {"COLI_TEMP": invalid}):+ self.assertEqual(generation_options({}, 8)[1], 0.7)+def test_validates_stop_sequences(self):self.assertEqual(generation_options({"stop": "END"}, 8)[4], ("END",))self.assertEqual(generation_options({"stop": ["ONE", "TWO"]}, 8)[4],