Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(plan): discover Windows AMD GPUs via hipInfo
Kenneth-Javier Ortigoza80370ed9759e
diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 41f0c887..349a4040 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -290,12 +290,129 @@ def _discover_nvidia_gpus():return devices+_HIPINFO_UNITS = {"B": 1, "KB": 1024, "MB": 1024 ** 2,+ "GB": 1024 ** 3, "TB": 1024 ** 4}+++def _hipinfo_executable():+ """Locate hipInfo.exe, preferring the runtime Colibri will actually bind.++ rocm-smi does not exist on Windows -- neither the HIP SDK installer nor a+ source build ships it -- so the rocm-smi probe below finds nothing there and+ every Windows AMD host planned CPU-only. hipInfo.exe is what both shipped+ SDKs do provide, and it sits in the same directory as amdhip64_7.dll.++ Lookup order, and why:++ 1. ``COLI_HIP_RUNTIME_DIR`` -- the directory the loader binds the HIP+ runtime from (docs/windows.md). hipInfo lives beside amdhip64_7.dll+ there, so its answer describes the runtime the engine will actually+ load.+ 2. ``HIP_PATH``\\bin -- the SDK root the Windows HIP SDK installer sets, and+ the same variable c/Makefile derives HIP_SDK_ROOT from.+ 3. ``PATH``.++ The order is the point on a host carrying more than one HIP install: a+ stale ambient HIP_PATH must not describe the hardware through a runtime the+ engine is not going to bind. No install location is hardcoded.+ """+ if sys.platform != "win32":+ return None+ candidates = []+ runtime_dir = os.environ.get("COLI_HIP_RUNTIME_DIR")+ if runtime_dir:+ candidates.append(Path(runtime_dir.strip('"')) / "hipInfo.exe")+ hip_path = os.environ.get("HIP_PATH")+ if hip_path:+ candidates.append(Path(hip_path.strip('"')) / "bin" / "hipInfo.exe")+ for candidate in candidates:+ try:+ if candidate.is_file():+ return candidate+ except OSError:+ continue+ found = shutil.which("hipInfo")+ return Path(found) if found else None+++def _hipinfo_bytes(value):+ """``"89.39 GB"`` -> bytes, or None. hipInfo divides by 1024 (it prints a+ 65536-byte shared block as ``64.00 KB``), so the units are binary."""+ match = re.match(r"([0-9.]+)\s*([KMGT]?B)\b", (value or "").strip())+ if not match:+ return None+ try:+ return int(float(match.group(1)) * _HIPINFO_UNITS[match.group(2)])+ except (ValueError, KeyError, OverflowError):+ return None+++def _parse_hipinfo(text):+ """Devices from hipInfo output, one block per ``device#`` line.++ A block that does not carry both a name and a total is dropped rather than+ completed with zeros: a half-trusted device is worse than no device,+ because the zeros would read as measurements.++ ``memInfo.free`` is deliberately NOT carried into ``free_bytes``. hipInfo+ does report it, and on the validated gfx1151 host it reported 89.24 GB+ "100% free" while Windows itself had 59.3 GiB of physical memory actually+ available -- the same pages, counted twice, ~30 GB apart. Qualifying that+ relationship is a later slice; until then the value is observed and+ discarded, and ``free_bytes`` stays None. See plans_placement().+ """+ blocks = []+ current = None+ for line in text.splitlines():+ stripped = line.strip()+ if stripped.startswith("device#"):+ index = stripped[len("device#"):].strip()+ current = {"index": int(index)} if index.isdigit() else None+ if current is not None:+ blocks.append(current)+ continue+ if current is None:+ continue+ key, sep, value = line.partition(":")+ if sep:+ current[key.strip()] = value.strip()+ devices = []+ for block in blocks:+ name = block.get("Name", "")+ total = _hipinfo_bytes(block.get("memInfo.total")+ or block.get("totalGlobalMem"))+ if not name or not total:+ continue+ devices.append({"index": block["index"], "name": name,+ "arch": block.get("gcnArchName", ""),+ "total_bytes": total,+ "free_bytes": None,+ "unified_memory": block.get("isIntegrated") == "1"})+ return devices+++def _discover_amd_gpus_windows():+ hipinfo = _hipinfo_executable()+ if hipinfo is None:+ return []+ try:+ result = subprocess.run([str(hipinfo)], text=True, capture_output=True,+ check=True, timeout=10)+ except (OSError, subprocess.SubprocessError):+ return []+ return _parse_hipinfo(result.stdout)++def _discover_amd_gpus():- """ROCm/HIP discovery via rocm-smi (#662). Absent on non-AMD hosts, so this- returns [] there. rocm-smi --showmeminfo vram reports BYTES (unlike nvidia-smi's- MiB), so no unit scaling. Column names drift across ROCm versions, so match them- by substring rather than position. VERIFY on AMD hardware (labelled- hardware-owner-needed) -- authored without a ROCm host to test against."""+ """ROCm/HIP discovery. Windows goes through hipInfo (see above); everywhere+ else through rocm-smi (#662), which is absent on non-AMD hosts so this+ returns [] there. rocm-smi --showmeminfo vram reports BYTES (unlike+ nvidia-smi's MiB), so no unit scaling. Column names drift across ROCm+ versions, so match them by substring rather than position. The rocm-smi+ branch remains VERIFY-on-AMD-hardware (labelled hardware-owner-needed) --+ authored without a ROCm host to test against."""+ if sys.platform == "win32":+ return _discover_amd_gpus_windows()command = ["rocm-smi", "--showmeminfo", "vram", "--showproductname", "--csv"]try:result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=5)diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex a576ecfd..a1aa44c5 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -253,9 +253,6 @@ class ResourcePlanTest(unittest.TestCase):self.assertIn("identity only", text)self.assertTrue(any("not qualified as a placement budget" in warningfor warning in plan["warnings"]))- # But it buys no tier.- self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)- self.assertEqual(plan["tiers"]["vram"]["expert_capacity"], 0)def test_plan_wording_is_backend_neutral_without_a_gpu(self):plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,@@ -264,6 +261,9 @@ class ResourcePlanTest(unittest.TestCase):text = format_plan(plan)self.assertIn("no supported GPU detected", text)self.assertNotIn("NVIDIA", text)+ # But it buys no tier.+ self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)+ self.assertEqual(plan["tiers"]["vram"]["expert_capacity"], 0)def test_identity_only_gpu_decides_nothing_a_cpu_only_host_would_not(self):gpu_plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,@@ -330,6 +330,172 @@ class ResourcePlanTest(unittest.TestCase):self.assertEqual(env.get("COLI_GPU"), "1")self.assertNotIn("COLI_GPUS", env)+ # --- Windows AMD discovery through hipInfo ----------------------------+ # Captured from hipInfo.exe shipped with the Windows HIP SDK (TheRock and+ # ROCm 7.1 both emit this layout). Trimmed to the fields the parser reads+ # plus a few it must ignore; the "89.39 GB" spellings are verbatim, and+ # hipInfo divides by 1024 (it prints a 65536-byte shared block as 64.00 KB).+ HIPINFO_INTEGRATED = """\+--------------------------------------------------------------------------------+device# 0+Name: AMD Radeon(TM) 8060S Graphics+pciBusID: 196+totalGlobalMem: 89.39 GB+sharedMemPerBlock: 64.00 KB+isIntegrated: 1+gcnArchName: gfx1151+peers:+non-peers: device#0++memInfo.total: 89.39 GB+memInfo.free: 89.24 GB (100%)+"""++ HIPINFO_DISCRETE = """\+--------------------------------------------------------------------------------+device# 0+Name: AMD Radeon RX 7900 XTX+totalGlobalMem: 24.00 GB+isIntegrated: 0+gcnArchName: gfx1100++memInfo.total: 24.00 GB+memInfo.free: 23.50 GB (97%)+"""++ def _run_hipinfo(self, stdout, returncode=0):+ """Discover with hipInfo located and returning `stdout`."""+ from resource_plan import _discover_amd_gpus+ completed = subprocess.CompletedProcess(args=[], returncode=returncode,+ stdout=stdout, stderr="")+ with mock.patch.object(sys, "platform", "win32"), \+ mock.patch("resource_plan._hipinfo_executable",+ return_value=Path("C:/sdk/bin/hipInfo.exe")), \+ mock.patch("resource_plan.subprocess.run", return_value=completed):+ return _discover_amd_gpus()++ def test_windows_integrated_amd_device_is_identity_only(self):+ devices = self._run_hipinfo(self.HIPINFO_INTEGRATED)+ self.assertEqual(len(devices), 1)+ gpu = devices[0]+ self.assertEqual(gpu["index"], 0)+ self.assertEqual(gpu["name"], "AMD Radeon(TM) 8060S Graphics")+ self.assertEqual(gpu["arch"], "gfx1151")+ self.assertTrue(gpu["unified_memory"])+ self.assertEqual(gpu["total_bytes"], int(89.39 * 1024 ** 3))+ # The whole point of the slice: hipInfo DID report free memory, and it+ # deliberately did not become a planning budget.+ self.assertIsNone(gpu["free_bytes"])++ def test_windows_positive_hipinfo_free_memory_never_becomes_a_budget(self):+ # Belt and braces on the regression that matters most: the fixture says+ # 89.24 GB free (100%), so any leak of that number into planning would+ # show up as a non-zero VRAM tier here.+ self.assertIn("memInfo.free", self.HIPINFO_INTEGRATED)+ devices = self._run_hipinfo(self.HIPINFO_INTEGRATED)+ plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,+ available_disk=1, gpus=devices, physical_cpus=8,+ cpu_sockets=1)+ self.assertEqual(plan["tiers"]["vram"]["budget_bytes"], 0)+ self.assertNotIn("COLI_CUDA_PIPE", plan["tune"])+ self.assertNotIn("CUDA_EXPERT_GB", environment_for_plan(plan))++ def test_windows_discrete_amd_device_is_not_marked_unified(self):+ devices = self._run_hipinfo(self.HIPINFO_DISCRETE)+ self.assertEqual(len(devices), 1)+ self.assertFalse(devices[0]["unified_memory"])+ self.assertEqual(devices[0]["arch"], "gfx1100")+ # Windows AMD free memory is unqualified for every part in this slice,+ # discrete included: we have no discrete Windows AMD host to qualify it+ # against, and guessing is what this slice exists to avoid.+ self.assertIsNone(devices[0]["free_bytes"])++ HIPINFO_SECOND_DEVICE = """\+--------------------------------------------------------------------------------+device# 1+Name: AMD Radeon RX 7900 XTX+totalGlobalMem: 24.00 GB+isIntegrated: 0+gcnArchName: gfx1100++memInfo.total: 24.00 GB+memInfo.free: 23.50 GB (97%)+"""++ def test_windows_parses_every_complete_device_block(self):+ devices = self._run_hipinfo(self.HIPINFO_INTEGRATED+ + self.HIPINFO_SECOND_DEVICE)+ self.assertEqual([gpu["index"] for gpu in devices], [0, 1])+ self.assertEqual([gpu["unified_memory"] for gpu in devices], [True, False])+ self.assertEqual([gpu["free_bytes"] for gpu in devices], [None, None])++ def test_windows_without_hipinfo_reports_no_device(self):+ from resource_plan import _discover_amd_gpus+ with mock.patch.object(sys, "platform", "win32"), \+ mock.patch("resource_plan._hipinfo_executable", return_value=None):+ self.assertEqual(_discover_amd_gpus(), [])++ def test_windows_hipinfo_failure_invents_nothing(self):+ from resource_plan import _discover_amd_gpus+ with mock.patch.object(sys, "platform", "win32"), \+ mock.patch("resource_plan._hipinfo_executable",+ return_value=Path("C:/sdk/bin/hipInfo.exe")), \+ mock.patch("resource_plan.subprocess.run",+ side_effect=subprocess.CalledProcessError(1, "hipInfo")):+ self.assertEqual(_discover_amd_gpus(), [])++ def test_windows_incomplete_device_block_is_not_half_trusted(self):+ # A block that names a device but reports no memory must produce no+ # record at all rather than one with fabricated zeros.+ partial = ("--------------------------------------------------------\n"+ "device# 0\n"+ "Name: AMD Radeon(TM) 8060S Graphics\n"+ "isIntegrated: 1\n")+ self.assertEqual(self._run_hipinfo(partial), [])+ headless = ("--------------------------------------------------------\n"+ "device# 0\n"+ "totalGlobalMem: 89.39 GB\n"+ "memInfo.total: 89.39 GB\n")+ self.assertEqual(self._run_hipinfo(headless), [])++ def test_hipinfo_lookup_prefers_the_colibri_runtime_over_a_stale_install(self):+ # This machine has had two Windows HIP installs at once. The engine+ # binds the runtime COLI_HIP_RUNTIME_DIR names, and hipInfo sits beside+ # amdhip64_7.dll in that same directory, so its answer describes the+ # runtime that will actually be used. An ambient HIP_PATH pointing at a+ # different SDK must not win.+ from resource_plan import _hipinfo_executable+ with tempfile.TemporaryDirectory() as root:+ chosen = Path(root) / "therock" / "bin"+ stale = Path(root) / "sdk" / "bin"+ for directory in (chosen, stale):+ directory.mkdir(parents=True)+ (directory / "hipInfo.exe").write_bytes(b"")+ env = {"COLI_HIP_RUNTIME_DIR": str(chosen),+ "HIP_PATH": str(stale.parent)}+ with mock.patch.object(sys, "platform", "win32"), \+ mock.patch.dict(os.environ, env, clear=False):+ self.assertEqual(_hipinfo_executable(), chosen / "hipInfo.exe")+ # With no Colibri-specific runtime selected, the SDK root is used.+ with mock.patch.object(sys, "platform", "win32"), \+ mock.patch.dict(os.environ, {"HIP_PATH": str(stale.parent)}, clear=True):+ self.assertEqual(_hipinfo_executable(), stale / "hipInfo.exe")++ def test_linux_amd_discovery_still_uses_rocm_smi(self):+ from resource_plan import _discover_amd_gpus+ output = ("device,Card Series,VRAM Total Memory (B),VRAM Total Used Memory (B)\n"+ "card0,Instinct MI300X,68719476736,8589934592\n")+ completed = subprocess.CompletedProcess(args=[], returncode=0,+ stdout=output, stderr="")+ with mock.patch.object(sys, "platform", "linux"), \+ mock.patch("resource_plan.subprocess.run",+ return_value=completed) as run:+ devices = _discover_amd_gpus()+ self.assertEqual(run.call_args[0][0][0], "rocm-smi")+ self.assertEqual(devices[0]["total_bytes"], 68719476736)+ self.assertEqual(devices[0]["free_bytes"], 68719476736 - 8589934592)+ self.assertFalse(devices[0]["unified_memory"])+def test_auto_tier_thread_count_uses_physical_cores(self):# End-to-end for #325: build_plan + environment_for_plan must export the# physical (not logical SMT) core count as OMP_NUM_THREADS. The originaldiff --git a/docs/windows.md b/docs/windows.mdindex e0939627..7484aa2b 100644--- a/docs/windows.md+++ b/docs/windows.md@@ -383,6 +383,78 @@ not been validated on hardware here.**CUDA is unaffected.** A `CUDA_DLL=1` host ignores `COLI_HIP_RUNTIME_DIR`entirely and keeps its existing `coli_cuda.dll` search behaviour unchanged.+### How `coli plan` and `coli doctor` see an AMD GPU here++`rocm-smi` is a Linux tool. Neither the Windows HIP SDK installer nor a source+build ships it, so the ROCm discovery path used everywhere else finds nothing on+Windows and every AMD host planned as if it had no GPU at all.++**Windows AMD discovery therefore goes through `hipInfo.exe`**, which both+shipped SDKs do provide, in the same directory as `amdhip64_7.dll`. It comes+from the HIP environment you already installed to build the backend β no extra+Python package, no third-party dependency.++Colibri looks for it in this order, and stops at the first hit:++1. **`COLI_HIP_RUNTIME_DIR`** β the directory you already point at the HIP+ runtime. `hipInfo.exe` sits beside `amdhip64_7.dll` there, so its answer+ describes *the runtime the engine will actually bind*.+2. **`%HIP_PATH%\bin`** β the SDK root the HIP SDK installer sets, and the same+ variable the Makefile derives `HIP_SDK_ROOT` from.+3. **`PATH`**.++The order matters if you have more than one HIP install, which is common: a+stale machine-wide `HIP_PATH` should not describe your hardware through a+runtime the engine is not going to load. No install location is hardcoded.++If `hipInfo.exe` cannot be found, or fails, or prints a device block missing a+name or a memory total, discovery reports **no device**. It never invents one,+and never completes a partial block with zeros β a zero would read as a+measurement.++#### Reported, but not planned against β and why++`hipInfo` reports the device name, `gcnArchName`, `isIntegrated`, and **both**+`memInfo.total` and `memInfo.free`. Colibri records the identity, marks+`isIntegrated: 1` as unified memory, and takes the total.++It deliberately does **not** use `memInfo.free` as a placement budget. On the+validated gfx1151 host, `hipInfo` reported:++```+memInfo.total: 89.39 GB+memInfo.free: 89.24 GB (100%)+```++while Windows itself had **59.3 GiB** of physical memory actually available.+Those are the same physical pages counted twice, about 30 GB apart, on a part+where the GPU and the host share one pool. Spending the HIP figure as a VRAM+budget would authorise an expert tier the machine cannot back β and the RAM+tier is being sized from that same memory at the same time.++So such a device is carried as **identity only**: free memory is *unknown for+planning*, which is not the same claim as "zero bytes are free". `coli plan`+shows it, marked `(identity only)`, with a warning naming it, and:++- the VRAM tier stays at 0+- no `CUDA_EXPERT_GB`, `PIN_GB`, `COLI_GPU` or `COLI_CUDA` is recommended+- `COLI_CUDA_PIPE` is not switched on+- the bottleneck is classified exactly as it would be on a CPU-only host++No system-RAM figure is substituted for the missing value either; nothing is+fabricated in place of a measurement.++None of this prevents you from running on the GPU. Every environment variable+in this document still works exactly as documented β this governs only what+Colibri will turn on *by itself*. Qualifying a safe automatic budget on shared+memory needs measurement on real hardware, and that is deliberately left to a+later change rather than guessed at here.++A discrete Windows AMD card is marked `unified_memory: false` from+`isIntegrated: 0`, but is also carried as identity-only for now: there is no+discrete Windows AMD host in this validation set, and the same guess would be+just as unqualified there.+### Putting model tensors on the GPU β and checking that it happened`COLI_CUDA=1` initialises the backend. It does **not**, on its own, put a single