FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(engine): resolve moe-strategy auto to fused on unified-memory GPUs (GB10) (#445)
Manish Biswalbea8d06f195b
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 2f0c89fb..3848fc70 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -1074,6 +1074,38 @@ def _profile_gpu(index: "int | None" = None) -> Tuple[str | None, str | None]:return ident["name"], ident["uuid"]+def _is_unified_memory_gpu(index: "int | None" = None) -> bool:+ """True when the GPU has no separate device memory (cudaDevAttrIntegrated): host+ banks and the GPU slot cache are the same DRAM, so the offload family's pinned+ staging + slot gather are DRAM-to-DRAM copies with no PCIe link to hide behind.++ The attribute is reliable on true-UMA parts (Jetson, GB10/DGX Spark) but not on+ C2C-linked discrete-HBM parts (GH200 reports integrated=0 despite coherent CPU+ memory), and it has only been verified on GB10 so far --+ FREETOKEN_UNIFIED_MEMORY=0/1 overrides the probe where the attribute lies."""+ env = os.environ.get("FREETOKEN_UNIFIED_MEMORY")+ if env is not None:+ return env.strip().lower() not in {"0", "false", "no", "off"}+ if not torch.cuda.is_available():+ return False+ try:+ dev = torch.cuda.current_device() if index is None else index+ return bool(torch.cuda.get_device_properties(dev).is_integrated)+ except Exception:+ return False+++def _fused_resident_ok(model_config) -> bool:+ """Whether the resident ('fused') MoE path can hold this model's experts.++ FIXME: auto resolves to fused only for bf16 and fp8_block experts; drop this gate once the other quant formats support fused.+ """+ expert_quant = getattr(model_config, "expert_quant", "none")+ if expert_quant not in ("none", "fp8_block"):+ return False+ return getattr(model_config, "moe_weight_format", None) in (None, "bf16")++def _ensure_expandable_segments() -> None:"""Default the CUDA allocator to expandable segments.@@ -1526,6 +1558,21 @@ def _adjust_config(config: EngineConfig):# -- auto never picks it, because nothing here knows whether the experts would fit in# HBM and a wrong guess is a weight-load OOM rather than a slower-but-working run.default_backend = "offload"+ # Unified memory (GB10/DGX Spark, Jetson): there is no host/device boundary, so+ # the offload family stages and gathers between two names for the same DRAM (on+ # GB10 this added a measured ~130 s stall to every request, #369). Resident+ # experts are the safe default here, not the risky one: the model and its banks+ # page from the same pool, so the "wrong guess = weight-load OOM" rationale above+ # does not apply. The benchbw hybrid upgrade is skipped too: CPU execution adds+ # no bandwidth when both sides share one memory. Only formats the resident path+ # can actually hold take this branch; the rest stay on offload as before.+ unified_memory = _is_unified_memory_gpu()+ if unified_memory and _fused_resident_ok(model_config):+ default_backend = "fused"+ logger.info_rank0(+ "Unified-memory GPU detected; auto-selecting 'fused' MoE strategy "+ "(resident experts) instead of offload"+ )# Hardware-adaptive config: a cached `ft bench bw` profile can upgrade# the offload default to hybrid when this machine's CPU MoE bandwidth clears its PCIe# gather bandwidth by the bench threshold (default 2x). hybrid is VRAM-equivalent to@@ -1539,7 +1586,14 @@ def _adjust_config(config: EngineConfig):from freetoken.moe.bench_profile import load_backend_recommendationgpu_name, gpu_uuid = _profile_gpu()- if load_backend_recommendation(bench_fmt, gpu_name=gpu_name, gpu_uuid=gpu_uuid) == "hybrid":+ if (+ default_backend == "offload"+ and not unified_memory+ and load_backend_recommendation(+ bench_fmt, gpu_name=gpu_name, gpu_uuid=gpu_uuid+ )+ == "hybrid"+ ):from freetoken.moe.cpu_executor import compiled_extension_supports_act = getattr(model_config, "hidden_act", "silu")@@ -1585,6 +1639,20 @@ def _adjust_config(config: EngineConfig):f"auto-selected strategy {config.moe_strategy!r}")+ if (+ is_moe+ and config.moe_strategy == "offload"+ and _is_unified_memory_gpu()+ and _fused_resident_ok(model_config)+ ):+ # An explicit offload pick is honored, but on unified memory the user is paying+ # for copies between two names for the same DRAM; say so once at config time.+ logger.warning_rank0(+ "--moe-strategy offload on a unified-memory GPU: expert 'streaming' copies "+ "DRAM to DRAM (there is no PCIe link to overlap it with). If the model fits, "+ "--moe-strategy fused avoids the slot-cache machinery entirely."+ )+if is_moe and config.moe_strategy == "fused":# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The# sizing flags no longer redirect the backend, so ignore them here and say so -- thediff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex b29ce531..cd5f5dc7 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -412,13 +412,18 @@ def test_lru_gpu_cache_assigns_unique_slots_for_large_miss_batch():assert cache.src_indices[:256].tolist() == list(range(256))-def test_adjust_config_converts_moe_cache_rate_to_cache_size():+def test_adjust_config_converts_moe_cache_rate_to_cache_size(monkeypatch):from types import SimpleNamespacefrom freetoken.distributed import DistributedInfofrom freetoken.engine.config import EngineConfig+ import freetoken.engine.engine as engine_modulefrom freetoken.engine.engine import _adjust_config+ # This test exercises the discrete-GPU offload path regardless of the host+ # running the suite (GB10 reports cudaDevAttrIntegrated=1).+ monkeypatch.setattr(engine_module, "_is_unified_memory_gpu", lambda index=None: False)+config = EngineConfig(model_path="/tmp/freetoken-test-model",tp_info=DistributedInfo(rank=0, size=1),