Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(moe): report the residency banks actually settle at

Xiaoze Fan831d38a66bc4
diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.py
index 9cca52d0..9d6df684 100644
--- a/python/freetoken/checkpoint/ftw.py
+++ b/python/freetoken/checkpoint/ftw.py
@@ -582,19 +582,36 @@ def load_ftw_banks(
from freetoken.moe.expert_banks import ExpertBanks
- unpinned = [i for i, r in enumerate(residency) if r != HostResidency.PINNED.value]
+ # a failed mlock leaves a LOCKED layer pageable; the log and labels report what the banks actually settled at
+ applied = list(residency)
+ for banks in row_hb.values():
+ for layer_id, bank in enumerate(banks):
+ if (applied[layer_id] == HostResidency.LOCKED.value
+ and bank.residency is not HostResidency.LOCKED):
+ applied[layer_id] = HostResidency.PAGEABLE.value
+ unpinned = [i for i, r in enumerate(applied) if r != HostResidency.PINNED.value]
if unpinned:
by_layer = [0] * num_layers
for name, banks in row_hb.items():
for layer_id, bank in enumerate(banks):
by_layer[layer_id] += bank.nbytes
+ locked = [i for i in unpinned if applied[i] == HostResidency.LOCKED.value]
+ pageable = [i for i in unpinned if i not in set(locked)]
pinned_b = sum(b for i, b in enumerate(by_layer) if i not in set(unpinned))
- locked_b = sum(by_layer[i] for i in unpinned)
+ locked_b = sum(by_layer[i] for i in locked)
+ pageable_part = ""
+ if pageable:
+ pageable_b = sum(by_layer[i] for i in pageable)
+ pageable_part = (
+ f" + {pageable_b / 2**30:.2f} GiB pageable "
+ f"(lock failed, {len(pageable)} CPU layers: {pageable})"
+ )
logger.info(
f"MoE bank split residency: {pinned_b / 2**30:.2f} GiB pinned "
f"({'born-pinned cudaHostAlloc' if born else 'cudaHostRegister'}, "
f"{num_layers - len(unpinned)} GPU layers) + "
- f"{locked_b / 2**30:.2f} GiB OS-locked ({len(unpinned)} CPU layers: {unpinned})"
+ f"{locked_b / 2**30:.2f} GiB OS-locked ({len(locked)} CPU layers: {locked})"
+ f"{pageable_part}"
)
# alphas are the small per-expert scale vectors, distinguished by their reserved names
@@ -602,7 +619,7 @@ def load_ftw_banks(
alpha_kw = {n: alpha_hb[n].tensor for n in alpha_hb}
return ExpertBanks(
reader.meta("quant_format"), sources, **alpha_kw,
- layer_residency=list(residency),
+ layer_residency=applied,
)
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index b6fadff9..8b6116ba 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -500,7 +500,15 @@ def _echo_residency(banks: ExpertBanks, requested, plan) -> ExpertBanks:
if plan is not None and plan.applied:
import dataclasses
- return dataclasses.replace(banks, layer_residency=list(requested))
+ labels = [plan.actual.get(i, r) for i, r in enumerate(requested)]
+ downgraded = [i for i, r in enumerate(requested) if labels[i] != r]
+ if downgraded:
+ logger.warning_rank0(
+ f"--moe-cpu-layers: layers {downgraded} settled pageable instead of "
+ f"OS-locked (lock failed); they still decode on the CPU executor but "
+ f"may swap under memory pressure"
+ )
+ return dataclasses.replace(banks, layer_residency=labels)
from freetoken.moe.host_banks import HostResidency
if any(r != HostResidency.PINNED.value for r in requested):
diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py
index 733349b5..5923c823 100644
--- a/python/freetoken/moe/host_banks.py
+++ b/python/freetoken/moe/host_banks.py
@@ -219,17 +219,23 @@ class _ResidencyPlan:
Installed by ``load_expert_banks`` around the provider dispatch so every loader honors --moe-cpu-layers without a new parameter in each signature. ``applied`` flips once a settle point consults the plan."""
- __slots__ = ("labels", "applied", "has_unpinned")
+ __slots__ = ("labels", "applied", "has_unpinned", "actual")
def __init__(self, labels: list[str]):
self.labels = list(labels)
self.applied = False
self.has_unpinned = any(r != HostResidency.PINNED.value for r in labels)
+ self.actual: dict[int, str] = {}
def residency_for(self, layer_id: int) -> str:
self.applied = True
return self.labels[layer_id]
+ def record(self, layer_id: int, achieved: str) -> None:
+ """One pageable bank downgrades the whole layer (a failed lock settles PAGEABLE)."""
+ if self.actual.get(layer_id) != HostResidency.PAGEABLE.value:
+ self.actual[layer_id] = achieved
+
_requested_residency: _ResidencyPlan | None = None
@@ -264,11 +270,13 @@ def pin_banks(banks: dict[str, HostBank | list[HostBank]]) -> None:
for bank in banks.values():
if isinstance(bank, list):
for layer_id, layer_bank in enumerate(bank):
- _settle(
- layer_bank,
+ residency = (
HostResidency.PINNED.value if plan is None
- else plan.residency_for(layer_id),
+ else plan.residency_for(layer_id)
)
+ _settle(layer_bank, residency)
+ if plan is not None and residency == HostResidency.LOCKED.value:
+ plan.record(layer_id, layer_bank.residency.value)
else:
bank.pin()
@@ -294,14 +302,17 @@ class PinPipeline:
return
if self._exc is not None:
continue # drain without settling after a failure
- bank, residency = item
+ bank, residency, plan, layer_id = item
try:
_settle(bank, residency)
+ if plan is not None and residency == HostResidency.LOCKED.value:
+ plan.record(layer_id, bank.residency.value)
except BaseException as exc: # surfaced by wait()/__exit__
self._exc = exc
- def submit(self, bank: HostBank, residency: str = HostResidency.PINNED.value) -> None:
- self._q.put((bank, residency))
+ def submit(self, bank: HostBank, residency: str = HostResidency.PINNED.value,
+ plan=None, layer_id: int | None = None) -> None:
+ self._q.put((bank, residency, plan, layer_id))
def __call__(self, layer_id: int, banks: dict[str, HostBank]) -> None:
"""Layer-completion sink: queue every bank of the completed layer at its ambient :func:`requested_residency` label."""
@@ -310,7 +321,7 @@ class PinPipeline:
HostResidency.PINNED.value if plan is None else plan.residency_for(layer_id)
)
for bank in banks.values():
- self.submit(bank, residency)
+ self.submit(bank, residency, plan, layer_id)
def _join(self) -> None:
self._q.put(None)
diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.py
index 6f2b8bff..422ca867 100644
--- a/tests/moe/test_offload.py
+++ b/tests/moe/test_offload.py
@@ -839,3 +839,32 @@ def test_echo_residency_stamps_honored_requests_only():
stale = _ResidencyPlan(labels) # never consulted -> keep None + warn
assert _echo_residency(banks, labels, stale).layer_residency is None
assert _echo_residency(banks, None, None) is banks
+
+
+def test_lock_failure_downgrades_echoed_residency(monkeypatch):
+ # a failed mlock leaves the bank pageable; the plan and the echoed labels must report that instead of the requested LOCKED
+ import freetoken.moe.host_banks as hb
+ from freetoken.moe.expert_banks import ExpertBanks, _echo_residency
+
+ def boom(addr, nbytes):
+ raise OSError(12, "mlock denied")
+
+ monkeypatch.setattr(hb, "_os_lock", boom)
+ monkeypatch.setattr(hb, "_os_lock_failed", False)
+ monkeypatch.setenv("FREETOKEN_SKIP_BANK_PIN", "1") # keep the pinned layer off CUDA
+ labels = [hb.HostResidency.PINNED.value, hb.HostResidency.LOCKED.value]
+
+ banks = {"gate_up": [hb.HostBank((4,), torch.uint8) for _ in range(2)]}
+ with hb.requested_residency(labels) as plan:
+ hb.pin_banks(banks)
+ assert plan.actual == {1: hb.HostResidency.PAGEABLE.value}
+ echoed = _echo_residency(ExpertBanks("bf16", {}), labels, plan)
+ assert echoed.layer_residency == [
+ hb.HostResidency.PINNED.value, hb.HostResidency.PAGEABLE.value,
+ ]
+
+ monkeypatch.setattr(hb, "_os_lock_failed", False)
+ with hb.requested_residency(labels) as plan2:
+ with hb.PinPipeline() as pins:
+ pins(1, {"gate_up": hb.HostBank((4,), torch.uint8)})
+ assert plan2.actual == {1: hb.HostResidency.PAGEABLE.value}