Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge remote-tracking branch 'origin/dev' into feat/shipped-expert-profiles
JustVugg34d8a693b330
diff --cc c/setup_flow.pyindex a9c0bcac,67943c89..d5abb739--- a/c/setup_flow.py+++ b/c/setup_flow.py@@@ -880,35 -888,59 +888,88 @@@ def resolve_engine(family, entry, decisreturn {"launcher_dir": runtime, "engine": engine, "backend": "cpu", "source": f"release {tag}"}+# Starting expert histories, one per catalog model that has one (profiles/<id>.coli_usage).+PROFILES_DIR = os.path.join(HERE, "profiles")+++def seed_expert_profile(entry, model_dir, out=print):+ """Give a model with no expert history yet the one that ships for it.++ Every engine with an expert tier or pins reads <model>/.coli_usage at start+ (route_trace.h): the experts it routed to before, which the GPU's expert+ tier (and PIN=auto's RAM pins) fill first. A fresh install has none, so its+ first runs start cold. profiles/<id>.coli_usage is that history from a+ calibration session (chat in several languages, code, reasoning, JSON),+ copied in when the model has none. It is only a starting point: the engine+ adds every run's routing to it and saves it back, so a user's own history+ takes over as they use the model. An existing history is never replaced.+ Returns True when one was copied."""+ if entry is None or not model_dir:+ return False+ src = os.path.join(PROFILES_DIR, f"{entry.id}.coli_usage")+ dst = os.path.join(model_dir, ".coli_usage")+ if not os.path.isfile(src) or os.path.exists(dst) or not os.path.isdir(model_dir):+ return False+ try:+ shutil.copyfile(src, dst)+ except OSError:+ return False+ out(f" expert profile: a starting history for {entry.name} (the experts its GPU tier "+ "loads first); your own use extends it")+ return True+ # What the toolchain needs to build each kind of engine (toolchain()).+ CAN_BUILD = {"cpu": "can_build", "vulkan": "can_build_vulkan", "cuda": "can_build_cuda"}+++ def refresh_engine(family, backend, tc, path, out=print, make_args=()):+ """An engine already here, brought up to date with the sources before it is+ used: make redoes what changed since it was built, and nothing when nothing+ did (about 0.02 s). Without this a checkout updated with `git pull` kept its+ old engine, since one with the right backend was found and used as it was,+ and the new code never ran (#1852). It is rebuilt as the kind it is (`backend`,+ from binary_backend), so a Vulkan engine used for the CPU stays a Vulkan one.+ With no toolchain for that kind, or when the rebuild fails, the engine that+ was here stays, the failure with a line. Returns "present" or "rebuilt"."""+ if not tc.get(CAN_BUILD.get(backend, "can_build")):+ return "present"+ before = os.stat(path).st_mtime_ns+ try:+ build_engine(family, backend, tc, out=out, make_args=make_args, update=True)+ except BuildError as error:+ if not os.path.exists(path):+ raise+ out(f" could not update it (log: {error.log}); "+ f"keeping the {family.engine_artifact} engine that was here")+ return "present"+ if os.stat(path).st_mtime_ns == before:+ out(" up to date")+ return "present"+ out(" rebuilt: the sources changed since the last build")+ return "rebuilt"+++ def refresh_configured(cfg, out=print, tc=None):+ """The configured engine, brought up to date before a start (refresh_engine).+ A rerun of the setup, or `coli start`, used to start the engine as it was, so+ after a `git pull` the old code kept running (#1852). Only an engine built in+ this checkout: a prebuilt one from a release archive has no sources here. Not+ while a server runs: it keeps the engine it started with. Returns+ refresh_engine's answer, or None when there was nothing to check."""+ try:+ family = family_by_id(cfg.get("family"))+ except Exception:+ return None+ engine = cfg.get("engine") or ""+ if (not os.path.isfile(engine)+ or os.path.abspath(engine) != os.path.abspath(engine_path(HERE, family))):+ return None+ if server_status(cfg)["state"] != "stopped":+ return None+ tc = tc or toolchain()+ have = binary_backend(engine)+ # The CUDA variables the setup built it with: they come from the cards it kept.+ make_args = cuda_make_args(family, tc, cfg.get("nvidia") or []) if have == "cuda" else ()+ return refresh_engine(family, have, tc, engine, out=out, make_args=make_args)def resolve_with_fallback(ui, family, entry, decision, tc, hw, requested, failed=()):@@@ -1584,11 -1632,7 +1661,12 @@@ def cmd_setup(a, ui=None)ui.say("The GPU packages are installed now: rebuilding the engine for the GPU.\n")else:ui.say(f"Already set up: {describe_existing(cfg)}")+ try:+ seeded = setup_catalog.by_id((cfg.get("model") or {}).get("id"))+ except KeyError:+ seeded = None # a model given by folder: no catalog entry, no profile+ seed_expert_profile(seeded, cfg.get("model_dir"), out=ui.say)+ refresh_configured(cfg, out=ui.say)return _finish(ui, cfg, a)# A setup that stopped half way (pending), or a finished one whose engine or# files went missing (ready, but not runnable): continue with the same choice.diff --cc c/tests/test_setup_flow.pyindex abe26157,eb9bdef4..22a13fcb--- a/c/tests/test_setup_flow.py+++ b/c/tests/test_setup_flow.py@@@ -835,34 -898,27 +898,53 @@@ class WholeSetup(HomeTestCase)self.assertEqual(code, 0, text)self.assertIn("Already set up: Tiny test model", text)self.assertEqual(len(self.hub.requests), before)- build.assert_not_called()+ build.assert_called_once() # make, with nothing to redo+ self.assertTrue(build.call_args.kwargs["update"])++ def test_a_rerun_after_a_pull_rebuilds_the_engine_it_starts(self):+ code, text = self.run_setup(pick="tiny")+ self.assertEqual(code, 0, text)+ engine = Path(self.engines, "qwen36")+ os.utime(engine, ns=(1_000_000_000, 1_000_000_000))++ def make(family, backend, tc, out=print, make_args=(), update=False):+ self.assertEqual((backend, update), ("vulkan", True))+ engine.write_bytes(b"\x7fELF libvulkan.so.1 with the pulled sources")+ return str(engine)++ with mock.patch.object(setup_flow, "build_engine", side_effect=make):+ code, text = self.run_setup()+ self.assertEqual(code, 0, text)+ self.assertIn("Already set up: Tiny test model", text)+ self.assertIn("rebuilt: the sources changed since the last build", text)+ self.assertIn(b"pulled sources", engine.read_bytes())+ def test_a_fresh_install_gets_the_shipped_expert_profile(self):+ # profiles/<id>.coli_usage is copied into a model that has no history yet,+ # never over one it has, and a model without a profile gets nothing.+ profiles = Path(self.tmp.name, "profiles")+ profiles.mkdir()+ Path(profiles, "tiny.coli_usage").write_text("-1 2 4\n-2 1 7\n0 1 3\n")+ with mock.patch.object(setup_flow, "PROFILES_DIR", str(profiles)):+ code, text = self.run_setup(pick="tiny")+ self.assertEqual(code, 0, text)+ history = Path(self.models, "tiny", ".coli_usage")+ self.assertEqual(history.read_text(), "-1 2 4\n-2 1 7\n0 1 3\n")+ self.assertIn("expert profile: a starting history for Tiny test model", text)+ history.write_text("-1 2 4\n-2 1 7\n1 0 9\n") # the user's own use since+ code, text = self.run_setup()+ self.assertEqual(code, 0, text)+ self.assertEqual(history.read_text(), "-1 2 4\n-2 1 7\n1 0 9\n")+ self.assertNotIn("expert profile", text)+ history.unlink() # a rerun seeds a model left without one+ code, text = self.run_setup()+ self.assertIn("expert profile", text)+ self.assertTrue(history.is_file())+ history.unlink()+ with mock.patch.object(setup_flow, "PROFILES_DIR", str(Path(self.tmp.name, "none"))):+ code, text = self.run_setup()+ self.assertFalse(history.exists())+def test_interrupted_download_continues_on_rerun(self):real = setup_download.download_repoself.hub.cut_after["model-00000.safetensors"] = 150_000