Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

coli: say when --auto-tier is leaving the plan's VRAM tier unused (#1581)

JustVugg8bf3d84d6b83
diff --git a/c/coli b/c/coli
index 0d84debf..7082b4dc 100755
--- a/c/coli
+++ b/c/coli
@@ -583,10 +583,14 @@ def env_for_engine(a, arch, plan=None):
plan_must_not_choose_omp=("OMP_NUM_THREADS" not in env
and (arch == "deepseek_v4"
or env.get("COLI_NO_OMP_TUNE")))
- env=environment_for_plan(plan,env,
- cuda_enabled=env.get("COLI_CUDA") == "1")
+ cuda_on = env.get("COLI_CUDA") == "1"
+ env=environment_for_plan(plan,env,cuda_enabled=cuda_on)
except (OSError,ValueError,json.JSONDecodeError) as error:
sys.exit(f"{C.yel}invalid resource plan:{C.r} {error}")
+ # Outside the try on purpose: this only prints, and a display helper
+ # that stumbled inside it would be reported as "invalid resource plan"
+ # and take the launch down with it.
+ if not cuda_on: report_unapplied_vram_tier(plan)
if plan_must_not_choose_omp:
# environment_for_plan supplies the physical-core default used by
# the other engines. V4 deliberately computes logical CPUs minus
@@ -702,6 +706,41 @@ def requested_kv_slots(a, env):
if value is None: value=int(env.get("COLI_KV_SLOTS","1"))
return value
+def report_unapplied_vram_tier(plan):
+ """Say so when --auto-tier keeps a real VRAM tier out of the launch.
+
+ Dropping it is deliberate, and the contract above explains why: --gpu and
+ --vram are what select a CUDA-capable build, so merely asking for the plan
+ must never turn a CPU-only sibling binary into an attempted CUDA launch.
+
+ Doing it in SILENCE was not deliberate. `coli plan` and `coli doctor` print
+ the VRAM tier as part of the plan, --auto-tier is documented as applying
+ that plan, and then the tier vanishes with nothing on screen to connect the
+ two. On the box in #1581 the silence cost most of the throughput -- 11.8
+ tok/s became 21 once --gpu auto was added -- and the user had no way to see
+ that the plan they had just read was not the plan that ran.
+
+ The test for "real" is the planner's own: a device qualified to drive
+ placement, and a budget above zero. Anything else was already not going to
+ be applied with the GPU on either, and saying so would be noise.
+ """
+ try:
+ from resource_plan import plans_placement, GB
+ vram = plan["tiers"]["vram"]
+ devices = [d for d in vram["devices"] if plans_placement(d)]
+ budget = vram["budget_bytes"]
+ except (ImportError, KeyError, TypeError, AttributeError):
+ return
+ if not devices or budget <= 0:
+ return
+ where = ", ".join(f"{d['index']}:{d.get('name') or 'GPU'}" for d in devices)
+ print(f" {C.yel}[PLAN] the plan's VRAM tier is not being applied{C.r}\n"
+ f" {C.dim}{budget / GB:.1f} GB on {where} stays unused. --auto-tier "
+ f"applies the RAM plan but never enables the GPU by itself, so a "
+ f"CPU-only build is never launched as a CUDA one. Add --gpu auto "
+ f"(or --vram N) to use it.{C.r}", file=sys.stderr)
+
+
def env_for(a):
explicit_env = set(os.environ)
e = dict(os.environ, SNAP=a.model)
diff --git a/c/tests/test_auto_tier_vram_notice.py b/c/tests/test_auto_tier_vram_notice.py
new file mode 100644
index 00000000..c86280fd
--- /dev/null
+++ b/c/tests/test_auto_tier_vram_notice.py
@@ -0,0 +1,84 @@
+"""--auto-tier must say when it is leaving the plan's VRAM tier on the floor.
+
+Dropping the tier without --gpu is the contract and stays the contract: --gpu
+and --vram are what select a CUDA-capable build, so asking for a plan must not
+turn a CPU-only sibling binary into an attempted CUDA launch.
+
+Doing it in silence was the defect. `coli plan` and `coli doctor` print the
+VRAM tier, --auto-tier says it applies the plan, and the tier then disappears
+with nothing on screen tying the two together -- which on the box in #1581 was
+the difference between 11.8 and 21 tok/s with no clue as to why.
+
+So this checks two things and they pull in opposite directions: the notice
+appears when the tier was real, and stays quiet when it was not. A warning that
+fires on every CPU-only launch is a warning people learn to scroll past.
+
+It also checks that a plan of the wrong shape prints nothing and raises
+nothing. This function only writes to a terminal; it must never be the reason a
+launch fails.
+"""
+import contextlib
+import importlib.machinery
+import importlib.util
+import io
+import sys
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(HERE))
+
+_loader = importlib.machinery.SourceFileLoader("coli_cli_vram", str(HERE / "coli"))
+_spec = importlib.util.spec_from_loader("coli_cli_vram", _loader)
+coli = importlib.util.module_from_spec(_spec)
+_loader.exec_module(coli)
+
+
+def plan(devices, budget_bytes):
+ return {"tiers": {"vram": {"devices": devices, "budget_bytes": budget_bytes}}}
+
+
+def notice_for(p):
+ err = io.StringIO()
+ with contextlib.redirect_stderr(err):
+ coli.report_unapplied_vram_tier(p)
+ return err.getvalue()
+
+
+QUALIFIED = [{"index": 0, "name": "NVIDIA GeForce RTX 4060 Ti",
+ "free_bytes": 8 * 1000 ** 3}]
+
+
+class VramNoticeTest(unittest.TestCase):
+ def test_a_real_tier_is_announced(self):
+ out = notice_for(plan(QUALIFIED, 5_200_000_000))
+ self.assertIn("VRAM tier", out)
+ self.assertIn("5.2 GB", out, "the notice should say how much is unused")
+ self.assertIn("NVIDIA GeForce RTX 4060 Ti", out, "and on which device")
+ self.assertIn("--gpu auto", out, "and what to do about it")
+
+ def test_an_unqualified_device_is_not_worth_a_warning(self):
+ """free_bytes None means the planner never qualified that memory as a
+ budget, so the tier was not going to be applied with the GPU on either."""
+ out = notice_for(plan([{"index": 0, "name": "AMD", "free_bytes": None}],
+ 5_200_000_000))
+ self.assertEqual(out, "")
+
+ def test_a_zero_budget_is_not_worth_a_warning(self):
+ self.assertEqual(notice_for(plan(QUALIFIED, 0)), "")
+
+ def test_no_devices_is_not_worth_a_warning(self):
+ self.assertEqual(notice_for(plan([], 5_200_000_000)), "")
+
+ def test_a_plan_of_the_wrong_shape_is_silent_and_harmless(self):
+ """It only prints. It must never be why a launch stops."""
+ for broken in ({}, {"tiers": {}}, {"tiers": {"vram": {}}},
+ {"tiers": {"vram": {"devices": ["not a dict"],
+ "budget_bytes": 1}}},
+ {"tiers": {"vram": {"devices": None, "budget_bytes": 1}}}):
+ with self.subTest(plan=broken):
+ self.assertEqual(notice_for(broken), "")
+
+
+if __name__ == "__main__":
+ unittest.main()