FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(moe): report the residency banks actually settle at
Xiaoze Fan831d38a66bc4
diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.pyindex 9cca52d0..9d6df684 100644--- a/python/freetoken/checkpoint/ftw.py+++ b/python/freetoken/checkpoint/ftw.py@@ -582,19 +582,36 @@ def load_ftw_banks(from freetoken.moe.expert_banks import ExpertBanks- unpinned = [i for i, r in enumerate(residency) if r != HostResidency.PINNED.value]+ # a failed mlock leaves a LOCKED layer pageable; the log and labels report what the banks actually settled at+ applied = list(residency)+ for banks in row_hb.values():+ for layer_id, bank in enumerate(banks):+ if (applied[layer_id] == HostResidency.LOCKED.value+ and bank.residency is not HostResidency.LOCKED):+ applied[layer_id] = HostResidency.PAGEABLE.value+ unpinned = [i for i, r in enumerate(applied) if r != HostResidency.PINNED.value]if unpinned:by_layer = [0] * num_layersfor name, banks in row_hb.items():for layer_id, bank in enumerate(banks):by_layer[layer_id] += bank.nbytes+ locked = [i for i in unpinned if applied[i] == HostResidency.LOCKED.value]+ pageable = [i for i in unpinned if i not in set(locked)]pinned_b = sum(b for i, b in enumerate(by_layer) if i not in set(unpinned))- locked_b = sum(by_layer[i] for i in unpinned)+ locked_b = sum(by_layer[i] for i in locked)+ pageable_part = ""+ if pageable:+ pageable_b = sum(by_layer[i] for i in pageable)+ pageable_part = (+ f" + {pageable_b / 2**30:.2f} GiB pageable "+ f"(lock failed, {len(pageable)} CPU layers: {pageable})"+ )logger.info(f"MoE bank split residency: {pinned_b / 2**30:.2f} GiB pinned "f"({'born-pinned cudaHostAlloc' if born else 'cudaHostRegister'}, "f"{num_layers - len(unpinned)} GPU layers) + "- f"{locked_b / 2**30:.2f} GiB OS-locked ({len(unpinned)} CPU layers: {unpinned})"+ f"{locked_b / 2**30:.2f} GiB OS-locked ({len(locked)} CPU layers: {locked})"+ f"{pageable_part}")# alphas are the small per-expert scale vectors, distinguished by their reserved names@@ -602,7 +619,7 @@ def load_ftw_banks(alpha_kw = {n: alpha_hb[n].tensor for n in alpha_hb}return ExpertBanks(reader.meta("quant_format"), sources, **alpha_kw,- layer_residency=list(residency),+ layer_residency=applied,)diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.pyindex b6fadff9..8b6116ba 100644--- a/python/freetoken/moe/expert_banks.py+++ b/python/freetoken/moe/expert_banks.py@@ -500,7 +500,15 @@ def _echo_residency(banks: ExpertBanks, requested, plan) -> ExpertBanks:if plan is not None and plan.applied:import dataclasses- return dataclasses.replace(banks, layer_residency=list(requested))+ labels = [plan.actual.get(i, r) for i, r in enumerate(requested)]+ downgraded = [i for i, r in enumerate(requested) if labels[i] != r]+ if downgraded:+ logger.warning_rank0(+ f"--moe-cpu-layers: layers {downgraded} settled pageable instead of "+ f"OS-locked (lock failed); they still decode on the CPU executor but "+ f"may swap under memory pressure"+ )+ return dataclasses.replace(banks, layer_residency=labels)from freetoken.moe.host_banks import HostResidencyif any(r != HostResidency.PINNED.value for r in requested):diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.pyindex 733349b5..5923c823 100644--- a/python/freetoken/moe/host_banks.py+++ b/python/freetoken/moe/host_banks.py@@ -219,17 +219,23 @@ class _ResidencyPlan:Installed by ``load_expert_banks`` around the provider dispatch so every loader honors --moe-cpu-layers without a new parameter in each signature. ``applied`` flips once a settle point consults the plan."""- __slots__ = ("labels", "applied", "has_unpinned")+ __slots__ = ("labels", "applied", "has_unpinned", "actual")def __init__(self, labels: list[str]):self.labels = list(labels)self.applied = Falseself.has_unpinned = any(r != HostResidency.PINNED.value for r in labels)+ self.actual: dict[int, str] = {}def residency_for(self, layer_id: int) -> str:self.applied = Truereturn self.labels[layer_id]+ def record(self, layer_id: int, achieved: str) -> None:+ """One pageable bank downgrades the whole layer (a failed lock settles PAGEABLE)."""+ if self.actual.get(layer_id) != HostResidency.PAGEABLE.value:+ self.actual[layer_id] = achieved+_requested_residency: _ResidencyPlan | None = None@@ -264,11 +270,13 @@ def pin_banks(banks: dict[str, HostBank | list[HostBank]]) -> None:for bank in banks.values():if isinstance(bank, list):for layer_id, layer_bank in enumerate(bank):- _settle(- layer_bank,+ residency = (HostResidency.PINNED.value if plan is None- else plan.residency_for(layer_id),+ else plan.residency_for(layer_id))+ _settle(layer_bank, residency)+ if plan is not None and residency == HostResidency.LOCKED.value:+ plan.record(layer_id, layer_bank.residency.value)else:bank.pin()@@ -294,14 +302,17 @@ class PinPipeline:returnif self._exc is not None:continue # drain without settling after a failure- bank, residency = item+ bank, residency, plan, layer_id = itemtry:_settle(bank, residency)+ if plan is not None and residency == HostResidency.LOCKED.value:+ plan.record(layer_id, bank.residency.value)except BaseException as exc: # surfaced by wait()/__exit__self._exc = exc- def submit(self, bank: HostBank, residency: str = HostResidency.PINNED.value) -> None:- self._q.put((bank, residency))+ def submit(self, bank: HostBank, residency: str = HostResidency.PINNED.value,+ plan=None, layer_id: int | None = None) -> None:+ self._q.put((bank, residency, plan, layer_id))def __call__(self, layer_id: int, banks: dict[str, HostBank]) -> None:"""Layer-completion sink: queue every bank of the completed layer at its ambient :func:`requested_residency` label."""@@ -310,7 +321,7 @@ class PinPipeline:HostResidency.PINNED.value if plan is None else plan.residency_for(layer_id))for bank in banks.values():- self.submit(bank, residency)+ self.submit(bank, residency, plan, layer_id)def _join(self) -> None:self._q.put(None)diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex 6f2b8bff..422ca867 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -839,3 +839,32 @@ def test_echo_residency_stamps_honored_requests_only():stale = _ResidencyPlan(labels) # never consulted -> keep None + warnassert _echo_residency(banks, labels, stale).layer_residency is Noneassert _echo_residency(banks, None, None) is banks+++def test_lock_failure_downgrades_echoed_residency(monkeypatch):+ # a failed mlock leaves the bank pageable; the plan and the echoed labels must report that instead of the requested LOCKED+ import freetoken.moe.host_banks as hb+ from freetoken.moe.expert_banks import ExpertBanks, _echo_residency++ def boom(addr, nbytes):+ raise OSError(12, "mlock denied")++ monkeypatch.setattr(hb, "_os_lock", boom)+ monkeypatch.setattr(hb, "_os_lock_failed", False)+ monkeypatch.setenv("FREETOKEN_SKIP_BANK_PIN", "1") # keep the pinned layer off CUDA+ labels = [hb.HostResidency.PINNED.value, hb.HostResidency.LOCKED.value]++ banks = {"gate_up": [hb.HostBank((4,), torch.uint8) for _ in range(2)]}+ with hb.requested_residency(labels) as plan:+ hb.pin_banks(banks)+ assert plan.actual == {1: hb.HostResidency.PAGEABLE.value}+ echoed = _echo_residency(ExpertBanks("bf16", {}), labels, plan)+ assert echoed.layer_residency == [+ hb.HostResidency.PINNED.value, hb.HostResidency.PAGEABLE.value,+ ]++ monkeypatch.setattr(hb, "_os_lock_failed", False)+ with hb.requested_residency(labels) as plan2:+ with hb.PinPipeline() as pins:+ pins(1, {"gate_up": hb.HostBank((4,), torch.uint8)})+ assert plan2.actual == {1: hb.HostResidency.PAGEABLE.value}