Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge remote-tracking branch 'origin/dev' into feat/shipped-expert-profiles

JustVugg34d8a693b330
diff --cc c/setup_flow.py
index a9c0bcac,67943c89..d5abb739
--- a/c/setup_flow.py
+++ b/c/setup_flow.py
@@@ -880,35 -888,59 +888,88 @@@ def resolve_engine(family, entry, decis
return {"launcher_dir": runtime, "engine": engine, "backend": "cpu", "source": f"release {tag}"}
+# Starting expert histories, one per catalog model that has one (profiles/<id>.coli_usage).
+PROFILES_DIR = os.path.join(HERE, "profiles")
+
+
+def seed_expert_profile(entry, model_dir, out=print):
+ """Give a model with no expert history yet the one that ships for it.
+
+ Every engine with an expert tier or pins reads <model>/.coli_usage at start
+ (route_trace.h): the experts it routed to before, which the GPU's expert
+ tier (and PIN=auto's RAM pins) fill first. A fresh install has none, so its
+ first runs start cold. profiles/<id>.coli_usage is that history from a
+ calibration session (chat in several languages, code, reasoning, JSON),
+ copied in when the model has none. It is only a starting point: the engine
+ adds every run's routing to it and saves it back, so a user's own history
+ takes over as they use the model. An existing history is never replaced.
+ Returns True when one was copied."""
+ if entry is None or not model_dir:
+ return False
+ src = os.path.join(PROFILES_DIR, f"{entry.id}.coli_usage")
+ dst = os.path.join(model_dir, ".coli_usage")
+ if not os.path.isfile(src) or os.path.exists(dst) or not os.path.isdir(model_dir):
+ return False
+ try:
+ shutil.copyfile(src, dst)
+ except OSError:
+ return False
+ out(f" expert profile: a starting history for {entry.name} (the experts its GPU tier "
+ "loads first); your own use extends it")
+ return True
+ # What the toolchain needs to build each kind of engine (toolchain()).
+ CAN_BUILD = {"cpu": "can_build", "vulkan": "can_build_vulkan", "cuda": "can_build_cuda"}
+
+
+ def refresh_engine(family, backend, tc, path, out=print, make_args=()):
+ """An engine already here, brought up to date with the sources before it is
+ used: make redoes what changed since it was built, and nothing when nothing
+ did (about 0.02 s). Without this a checkout updated with `git pull` kept its
+ old engine, since one with the right backend was found and used as it was,
+ and the new code never ran (#1852). It is rebuilt as the kind it is (`backend`,
+ from binary_backend), so a Vulkan engine used for the CPU stays a Vulkan one.
+ With no toolchain for that kind, or when the rebuild fails, the engine that
+ was here stays, the failure with a line. Returns "present" or "rebuilt"."""
+ if not tc.get(CAN_BUILD.get(backend, "can_build")):
+ return "present"
+ before = os.stat(path).st_mtime_ns
+ try:
+ build_engine(family, backend, tc, out=out, make_args=make_args, update=True)
+ except BuildError as error:
+ if not os.path.exists(path):
+ raise
+ out(f" could not update it (log: {error.log}); "
+ f"keeping the {family.engine_artifact} engine that was here")
+ return "present"
+ if os.stat(path).st_mtime_ns == before:
+ out(" up to date")
+ return "present"
+ out(" rebuilt: the sources changed since the last build")
+ return "rebuilt"
+
+
+ def refresh_configured(cfg, out=print, tc=None):
+ """The configured engine, brought up to date before a start (refresh_engine).
+ A rerun of the setup, or `coli start`, used to start the engine as it was, so
+ after a `git pull` the old code kept running (#1852). Only an engine built in
+ this checkout: a prebuilt one from a release archive has no sources here. Not
+ while a server runs: it keeps the engine it started with. Returns
+ refresh_engine's answer, or None when there was nothing to check."""
+ try:
+ family = family_by_id(cfg.get("family"))
+ except Exception:
+ return None
+ engine = cfg.get("engine") or ""
+ if (not os.path.isfile(engine)
+ or os.path.abspath(engine) != os.path.abspath(engine_path(HERE, family))):
+ return None
+ if server_status(cfg)["state"] != "stopped":
+ return None
+ tc = tc or toolchain()
+ have = binary_backend(engine)
+ # The CUDA variables the setup built it with: they come from the cards it kept.
+ make_args = cuda_make_args(family, tc, cfg.get("nvidia") or []) if have == "cuda" else ()
+ return refresh_engine(family, have, tc, engine, out=out, make_args=make_args)
def resolve_with_fallback(ui, family, entry, decision, tc, hw, requested, failed=()):
@@@ -1584,11 -1632,7 +1661,12 @@@ def cmd_setup(a, ui=None)
ui.say("The GPU packages are installed now: rebuilding the engine for the GPU.\n")
else:
ui.say(f"Already set up: {describe_existing(cfg)}")
+ try:
+ seeded = setup_catalog.by_id((cfg.get("model") or {}).get("id"))
+ except KeyError:
+ seeded = None # a model given by folder: no catalog entry, no profile
+ seed_expert_profile(seeded, cfg.get("model_dir"), out=ui.say)
+ refresh_configured(cfg, out=ui.say)
return _finish(ui, cfg, a)
# A setup that stopped half way (pending), or a finished one whose engine or
# files went missing (ready, but not runnable): continue with the same choice.
diff --cc c/tests/test_setup_flow.py
index abe26157,eb9bdef4..22a13fcb
--- a/c/tests/test_setup_flow.py
+++ b/c/tests/test_setup_flow.py
@@@ -835,34 -898,27 +898,53 @@@ class WholeSetup(HomeTestCase)
self.assertEqual(code, 0, text)
self.assertIn("Already set up: Tiny test model", text)
self.assertEqual(len(self.hub.requests), before)
- build.assert_not_called()
+ build.assert_called_once() # make, with nothing to redo
+ self.assertTrue(build.call_args.kwargs["update"])
+
+ def test_a_rerun_after_a_pull_rebuilds_the_engine_it_starts(self):
+ code, text = self.run_setup(pick="tiny")
+ self.assertEqual(code, 0, text)
+ engine = Path(self.engines, "qwen36")
+ os.utime(engine, ns=(1_000_000_000, 1_000_000_000))
+
+ def make(family, backend, tc, out=print, make_args=(), update=False):
+ self.assertEqual((backend, update), ("vulkan", True))
+ engine.write_bytes(b"\x7fELF libvulkan.so.1 with the pulled sources")
+ return str(engine)
+
+ with mock.patch.object(setup_flow, "build_engine", side_effect=make):
+ code, text = self.run_setup()
+ self.assertEqual(code, 0, text)
+ self.assertIn("Already set up: Tiny test model", text)
+ self.assertIn("rebuilt: the sources changed since the last build", text)
+ self.assertIn(b"pulled sources", engine.read_bytes())
+ def test_a_fresh_install_gets_the_shipped_expert_profile(self):
+ # profiles/<id>.coli_usage is copied into a model that has no history yet,
+ # never over one it has, and a model without a profile gets nothing.
+ profiles = Path(self.tmp.name, "profiles")
+ profiles.mkdir()
+ Path(profiles, "tiny.coli_usage").write_text("-1 2 4\n-2 1 7\n0 1 3\n")
+ with mock.patch.object(setup_flow, "PROFILES_DIR", str(profiles)):
+ code, text = self.run_setup(pick="tiny")
+ self.assertEqual(code, 0, text)
+ history = Path(self.models, "tiny", ".coli_usage")
+ self.assertEqual(history.read_text(), "-1 2 4\n-2 1 7\n0 1 3\n")
+ self.assertIn("expert profile: a starting history for Tiny test model", text)
+ history.write_text("-1 2 4\n-2 1 7\n1 0 9\n") # the user's own use since
+ code, text = self.run_setup()
+ self.assertEqual(code, 0, text)
+ self.assertEqual(history.read_text(), "-1 2 4\n-2 1 7\n1 0 9\n")
+ self.assertNotIn("expert profile", text)
+ history.unlink() # a rerun seeds a model left without one
+ code, text = self.run_setup()
+ self.assertIn("expert profile", text)
+ self.assertTrue(history.is_file())
+ history.unlink()
+ with mock.patch.object(setup_flow, "PROFILES_DIR", str(Path(self.tmp.name, "none"))):
+ code, text = self.run_setup()
+ self.assertFalse(history.exists())
+
def test_interrupted_download_continues_on_rerun(self):
real = setup_download.download_repo
self.hub.cut_after["model-00000.safetensors"] = 150_000