Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge pull request #1734 from tarazum/fix/heat-save-stdin-drain

Vincenzo Fornaro50c9e1608648
diff --cc c/openai_server.py
index fcd2fb04,9839680c..488aefce
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@@ -3418,9 -2981,14 +3423,14 @@@ class Engine
child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",
NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))
tune_child_env(child_env, arch)
- resolved_cap = cap_for_arch(arch, cap, child_env)
+ resolved_cap = cap_for_arch(arch, cap, child_env, model=model)
child_env.pop("COLI_PROFILE_CAP", None)
child_env.pop("COLI_PLAN_CAP", None)
+ # Own process group on Windows: a CTRL_BREAK sent to the serve
+ # process group (the graceful stop, handled as SIGBREAK above) must
+ # not reach the engine — the C runtime's default would kill it
+ # before its stdin-EOF teardown (atexit -> HEAT_FILE save) can run.
+ spawn_flags = subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == "win32" else 0
self.process = subprocess.Popen(
[str(executable), str(resolved_cap)], env=child_env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,