Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(api): honor COLI_TEMP as the request default

Blaked7972b78f465
diff --git a/c/coli b/c/coli
index dc608ab7..8614b245 100755
--- a/c/coli
+++ b/c/coli
@@ -1321,6 +1321,7 @@ def cmd_serve(a):
"olmoe-colibri" if arch=="olmoe" else
"glm-5.2-colibri")
try:
+ if a.temp is not None: os.environ["COLI_TEMP"] = str(a.temp)
openai_server.serve(a.model,a.host,a.port,model_id,a.api_key,
a.cap,ngen_for(a,interactive=True),engine,env_for_engine(a,arch),a.cors_origin,
a.max_queue,a.queue_timeout,a.kv_slots,allowed_hosts=a.allowed_host)
diff --git a/c/openai_server.py b/c/openai_server.py
index 4e901768..47920643 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1622,7 +1622,16 @@ def generation_options(body, limit):
maximum = limit
temperature = body.get("temperature")
top_p = body.get("top_p")
- temperature = 0.7 if temperature is None else temperature
+ if temperature is None:
+ # The launcher publishes --temp through COLI_TEMP (#509, #968). The
+ # gateway must use that value as its request default or the SERVE frame
+ # replaces it with 0.7 before any engine can honor the setting.
+ try:
+ temperature = float(os.environ.get("COLI_TEMP", "0.7"))
+ if not math.isfinite(temperature) or not 0 <= temperature <= 2:
+ temperature = 0.7
+ except ValueError:
+ temperature = 0.7
top_p = 0.9 if top_p is None else top_p
if isinstance(maximum, bool) or not isinstance(maximum, int) or maximum < 1:
raise APIError(400, f"`{maximum_param}` must be a positive integer.", maximum_param)
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index d9c02788..f8468595 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -187,6 +187,15 @@ class TemplateTest(unittest.TestCase):
opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)
self.assertEqual(opts[3], "not a grammar ::=")
+ def test_coli_temp_is_the_default_for_requests_that_omit_temperature(self):
+ with patch.dict("openai_server.os.environ", {"COLI_TEMP": "0.25"}):
+ self.assertEqual(generation_options({}, 8)[1], 0.25)
+ self.assertEqual(generation_options({"temperature": 0}, 8)[1], 0.0)
+ for invalid in ("malformed", "5", "-1", "nan", "1e999"):
+ with self.subTest(invalid=invalid):
+ with patch.dict("openai_server.os.environ", {"COLI_TEMP": invalid}):
+ self.assertEqual(generation_options({}, 8)[1], 0.7)
+
def test_validates_stop_sequences(self):
self.assertEqual(generation_options({"stop": "END"}, 8)[4], ("END",))
self.assertEqual(generation_options({"stop": ["ONE", "TWO"]}, 8)[4],