Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
coli: say when --auto-tier is leaving the plan's VRAM tier unused (#1581)
JustVugg8bf3d84d6b83
diff --git a/c/coli b/c/coliindex 0d84debf..7082b4dc 100755--- a/c/coli+++ b/c/coli@@ -583,10 +583,14 @@ def env_for_engine(a, arch, plan=None):plan_must_not_choose_omp=("OMP_NUM_THREADS" not in envand (arch == "deepseek_v4"or env.get("COLI_NO_OMP_TUNE")))- env=environment_for_plan(plan,env,- cuda_enabled=env.get("COLI_CUDA") == "1")+ cuda_on = env.get("COLI_CUDA") == "1"+ env=environment_for_plan(plan,env,cuda_enabled=cuda_on)except (OSError,ValueError,json.JSONDecodeError) as error:sys.exit(f"{C.yel}invalid resource plan:{C.r} {error}")+ # Outside the try on purpose: this only prints, and a display helper+ # that stumbled inside it would be reported as "invalid resource plan"+ # and take the launch down with it.+ if not cuda_on: report_unapplied_vram_tier(plan)if plan_must_not_choose_omp:# environment_for_plan supplies the physical-core default used by# the other engines. V4 deliberately computes logical CPUs minus@@ -702,6 +706,41 @@ def requested_kv_slots(a, env):if value is None: value=int(env.get("COLI_KV_SLOTS","1"))return value+def report_unapplied_vram_tier(plan):+ """Say so when --auto-tier keeps a real VRAM tier out of the launch.++ Dropping it is deliberate, and the contract above explains why: --gpu and+ --vram are what select a CUDA-capable build, so merely asking for the plan+ must never turn a CPU-only sibling binary into an attempted CUDA launch.++ Doing it in SILENCE was not deliberate. `coli plan` and `coli doctor` print+ the VRAM tier as part of the plan, --auto-tier is documented as applying+ that plan, and then the tier vanishes with nothing on screen to connect the+ two. On the box in #1581 the silence cost most of the throughput -- 11.8+ tok/s became 21 once --gpu auto was added -- and the user had no way to see+ that the plan they had just read was not the plan that ran.++ The test for "real" is the planner's own: a device qualified to drive+ placement, and a budget above zero. Anything else was already not going to+ be applied with the GPU on either, and saying so would be noise.+ """+ try:+ from resource_plan import plans_placement, GB+ vram = plan["tiers"]["vram"]+ devices = [d for d in vram["devices"] if plans_placement(d)]+ budget = vram["budget_bytes"]+ except (ImportError, KeyError, TypeError, AttributeError):+ return+ if not devices or budget <= 0:+ return+ where = ", ".join(f"{d['index']}:{d.get('name') or 'GPU'}" for d in devices)+ print(f" {C.yel}[PLAN] the plan's VRAM tier is not being applied{C.r}\n"+ f" {C.dim}{budget / GB:.1f} GB on {where} stays unused. --auto-tier "+ f"applies the RAM plan but never enables the GPU by itself, so a "+ f"CPU-only build is never launched as a CUDA one. Add --gpu auto "+ f"(or --vram N) to use it.{C.r}", file=sys.stderr)++def env_for(a):explicit_env = set(os.environ)e = dict(os.environ, SNAP=a.model)diff --git a/c/tests/test_auto_tier_vram_notice.py b/c/tests/test_auto_tier_vram_notice.pynew file mode 100644index 00000000..c86280fd--- /dev/null+++ b/c/tests/test_auto_tier_vram_notice.py@@ -0,0 +1,84 @@+"""--auto-tier must say when it is leaving the plan's VRAM tier on the floor.++Dropping the tier without --gpu is the contract and stays the contract: --gpu+and --vram are what select a CUDA-capable build, so asking for a plan must not+turn a CPU-only sibling binary into an attempted CUDA launch.++Doing it in silence was the defect. `coli plan` and `coli doctor` print the+VRAM tier, --auto-tier says it applies the plan, and the tier then disappears+with nothing on screen tying the two together -- which on the box in #1581 was+the difference between 11.8 and 21 tok/s with no clue as to why.++So this checks two things and they pull in opposite directions: the notice+appears when the tier was real, and stays quiet when it was not. A warning that+fires on every CPU-only launch is a warning people learn to scroll past.++It also checks that a plan of the wrong shape prints nothing and raises+nothing. This function only writes to a terminal; it must never be the reason a+launch fails.+"""+import contextlib+import importlib.machinery+import importlib.util+import io+import sys+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(HERE))++_loader = importlib.machinery.SourceFileLoader("coli_cli_vram", str(HERE / "coli"))+_spec = importlib.util.spec_from_loader("coli_cli_vram", _loader)+coli = importlib.util.module_from_spec(_spec)+_loader.exec_module(coli)+++def plan(devices, budget_bytes):+ return {"tiers": {"vram": {"devices": devices, "budget_bytes": budget_bytes}}}+++def notice_for(p):+ err = io.StringIO()+ with contextlib.redirect_stderr(err):+ coli.report_unapplied_vram_tier(p)+ return err.getvalue()+++QUALIFIED = [{"index": 0, "name": "NVIDIA GeForce RTX 4060 Ti",+ "free_bytes": 8 * 1000 ** 3}]+++class VramNoticeTest(unittest.TestCase):+ def test_a_real_tier_is_announced(self):+ out = notice_for(plan(QUALIFIED, 5_200_000_000))+ self.assertIn("VRAM tier", out)+ self.assertIn("5.2 GB", out, "the notice should say how much is unused")+ self.assertIn("NVIDIA GeForce RTX 4060 Ti", out, "and on which device")+ self.assertIn("--gpu auto", out, "and what to do about it")++ def test_an_unqualified_device_is_not_worth_a_warning(self):+ """free_bytes None means the planner never qualified that memory as a+ budget, so the tier was not going to be applied with the GPU on either."""+ out = notice_for(plan([{"index": 0, "name": "AMD", "free_bytes": None}],+ 5_200_000_000))+ self.assertEqual(out, "")++ def test_a_zero_budget_is_not_worth_a_warning(self):+ self.assertEqual(notice_for(plan(QUALIFIED, 0)), "")++ def test_no_devices_is_not_worth_a_warning(self):+ self.assertEqual(notice_for(plan([], 5_200_000_000)), "")++ def test_a_plan_of_the_wrong_shape_is_silent_and_harmless(self):+ """It only prints. It must never be why a launch stops."""+ for broken in ({}, {"tiers": {}}, {"tiers": {"vram": {}}},+ {"tiers": {"vram": {"devices": ["not a dict"],+ "budget_bytes": 1}}},+ {"tiers": {"vram": {"devices": None, "budget_bytes": 1}}}):+ with self.subTest(plan=broken):+ self.assertEqual(notice_for(broken), "")+++if __name__ == "__main__":+ unittest.main()