Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

coli run: hand every engine its model directory through SNAP

JustVuggea8460332ef1
diff --git a/c/coli b/c/coli
index 375f65bc..d2894aa1 100755
--- a/c/coli
+++ b/c/coli
@@ -465,6 +465,13 @@ def env_for_engine(a, arch, plan=None):
env.setdefault("OMP_PLACES", "cores")
env["NGEN"] = str(ngen_for(a, family=arch))
if a.temp is not None: env["COLI_TEMP"] = str(a.temp)
+ # SNAP is how every engine but GLM learns its model directory. env_for()
+ # sets it for GLM and openai_server.py sets it for the gateway, so `coli
+ # chat` and `coli serve` worked while `coli run` handed the sister engines
+ # an environment without it: OLMoE exited with "started without a model"
+ # (#1501). Set it here, once, for all of them.
+ if getattr(a, "model", None):
+ env.setdefault("SNAP", os.path.abspath(a.model))
if arch == "olmoe":
env["CHAT"] = "1"
env["MAX_NEW"] = str(ngen_for(a, family=arch))
diff --git a/c/compat.h b/c/compat.h
index f5b372b0..0c5db65b 100644
--- a/c/compat.h
+++ b/c/compat.h
@@ -618,9 +618,11 @@ static inline void coli_print_launcher_help(const char *engine)
" %s doctor --model <model directory> check a model is usable\n"
"\n"
"The launcher needs Python 3 and picks the right engine for the model.\n"
+ "(Running the engine by hand: it reads the model directory from the\n"
+ "SNAP environment variable, e.g. SNAP=<model directory> ./%s ...)\n"
"Getting a model, step by step: https://github.com/JustVugg/colibri"
"/blob/main/docs/quickstart.md\n",
- engine, run, run, run, run);
+ engine, run, run, run, run, engine);
coli_hold_console();
}
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index 9497ab6b..e44ea3f6 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -116,6 +116,20 @@ class V4CliTest(unittest.TestCase):
self.assertEqual(env["RAM_GB"], "64")
self.assertEqual(env["CTX"], "4096")
+ def test_sister_engines_get_snap_from_the_model_flag(self):
+ """#1501: `coli run` handed olmoe (and every non-GLM engine) an
+ environment without SNAP, so the engine exited with "started without
+ a model" while chat and serve, which set it elsewhere, worked."""
+ from family_registry import family_ids
+ for arch in [f for f in family_ids() if f != "glm"]:
+ args = argparse.Namespace(ngen=8, temp=None, ram=0, ctx=None, model="models/demo")
+ env = self.cli.env_for_engine(args, arch)
+ self.assertEqual(env.get("SNAP"), os.path.abspath("models/demo"), arch)
+ # an explicit SNAP in the caller's environment still wins
+ with mock.patch.dict(os.environ, {"SNAP": "/elsewhere"}):
+ args = argparse.Namespace(ngen=8, temp=None, ram=0, ctx=None, model="models/demo")
+ self.assertEqual(self.cli.env_for_engine(args, "olmoe")["SNAP"], "/elsewhere")
+
def test_kimi_engine_environment_forwards_ram(self):
"""#855: `--ram` reached the environment for deepseek_v4 only, so on Kimi
K3 it was set and never read -- the flag a user reaches for to bound