Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(moe): Improved `fast_index_copy_multi_jit` for NVLink-C2C hosts (#610)

SuperHotDogCatc1a504a50b6c
diff --git a/docs/cli.md b/docs/cli.md
index b3cec998..b9be878c 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -204,3 +204,6 @@ profile that `ft serve --moe-strategy auto` and `--moe-hybrid-max-fetch -1` then
- What to measure: `--dtype`, `--model`, `--formats`, `--isa`.
- `--threshold` (default 2.0) sets the call: recommend hybrid when CPU bandwidth beats PCIe
by that factor.
+
+On NVLink-C2C hosts, see [models.md](models.md#nvlink-c2c-hosts-gh200--gb200)
+for `FREETOKEN_H2D_BLOCKS_PER_BANK` if `PCIe-gather` sits well below the linear H2D ceiling.
diff --git a/docs/models.md b/docs/models.md
index efd5268b..2cd3baa3 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -59,6 +59,20 @@ have no resident path and stay on `offload`. Offload-only flags
`--moe-prefill-hit-d2d`, `--disable-moe-prefill-overlap`) are ignored with a
warning; pass `--moe-strategy offload` to use them.
+### NVLink-C2C hosts (GH200 / GB200)
+
+The host link is ~450 GB/s per direction instead of PCIe's ~32-64 GB/s. The
+offload expert gather reads host memory zero-copy and is latency-bound, so it
+needs a wider grid to keep enough loads in flight. Set
+`FREETOKEN_H2D_BLOCKS_PER_BANK`:
+```
+ FREETOKEN_H2D_BLOCKS_PER_BANK=32 ft serve --model <model> --moe-strategy offload
+```
+On GH200 this raises the gather from ~220 to ~410 GB/s (`PCIe-gather` in
+`ft bench bw`), matching the DMA ceiling. Run `ft bench bw` with the variable
+set so the hybrid split is calibrated against the faster gather. Leave it
+unset on PCIe GPUs: wider grids add no bandwidth there.
+
## Notes
- `ft checkpoint` conversion is optional — it pre-converts a checkpoint into
diff --git a/python/freetoken/kernel/aot.py b/python/freetoken/kernel/aot.py
index 5e87f8c9..bbc2d2c6 100644
--- a/python/freetoken/kernel/aot.py
+++ b/python/freetoken/kernel/aot.py
@@ -153,6 +153,7 @@ def default_kernel_specs() -> tuple[KernelSpec, ...]:
for feature_size in DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES
)
specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=8))
+ specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=32))
# prefill hit-D2D gather (HBM-bound: wide grid) + its miss-side batch H2D binding.
specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=64))
specs.append(_batch_memcpy_spec())
diff --git a/python/freetoken/kernel/fast_index_copy.py b/python/freetoken/kernel/fast_index_copy.py
index 1aaa1303..b2030620 100644
--- a/python/freetoken/kernel/fast_index_copy.py
+++ b/python/freetoken/kernel/fast_index_copy.py
@@ -149,6 +149,23 @@ def _jit_fast_index_copy_multi_module(*, num_threads: int, blocks_per_bank: int)
cuda_wrappers=[("launch", f"&MultiIndexCopyKernel<{args}>::run")],
)
+def default_blocks_per_bank(device: torch.device) -> int:
+ """H2D gather grid width per bank. The link's bandwidth-delay product sets how many
+ 16B loads must be in flight: PCIe (~31 GB/s) saturates at ~4K threads/bank, NVLink-C2C
+ (GH200, ~450 GB/s) needs ~32K (measured: 8x1024 -> 222, 32x1024 -> 412 GB/s)."""
+ env = os.environ.get("FREETOKEN_H2D_BLOCKS_PER_BANK")
+ if env is not None:
+ blocks_per_bank = int(
+ env
+ )
+ assert blocks_per_bank > 0, (
+ "FREETOKEN_H2D_BLOCKS_PER_BANK must be greater than 0, "
+ f"got {blocks_per_bank}"
+ )
+ return blocks_per_bank # manual override (validate > 0)
+ if any(k in torch.cuda.get_device_name(device) for k in ("GH200", "GB200", "GB300")):
+ return 32
+ return 8
def fast_index_copy_multi_jit(
dst_ptrs: torch.Tensor,
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index c6f975df..0969f738 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -25,6 +25,7 @@ _FUSED_COPY = os.getenv("FREETOKEN_FUSED_COPY", "1").strip().lower() not in {"0"
# entry the batch sees is >= this size.
_SMALL_BANK_FEAT_BYTES = 256 * 1024
+from freetoken.kernel.fast_index_copy import default_blocks_per_bank
from freetoken.utils import init_logger
logger = init_logger(__name__)
@@ -257,6 +258,7 @@ class OffloadMoeCache:
# Source pointers are per layer (_copy_src_ptrs[layer_id] -> [num_banks] device
# tensor); dst/feat are layer-invariant.
self._copy_fused_ok = False
+ self._copy_blocks_per_bank = 8
self._copy_dst_ptrs: torch.Tensor | None = None
self._copy_src_ptrs: list[torch.Tensor] | None = None
self._copy_feat_bytes: torch.Tensor | None = None
@@ -438,6 +440,7 @@ class OffloadMoeCache:
elif self._gather_bank_ids:
self._gather_dst_ptrs = self._copy_dst_ptrs[self._gather_bank_ids].contiguous()
self._gather_feat_bytes = self._copy_feat_bytes[self._gather_bank_ids].contiguous()
+ self._copy_blocks_per_bank = default_blocks_per_bank(self.device)
self._copy_fused_ok = True
def validate_rebuild(self, cache_size: int) -> None:
@@ -1038,6 +1041,7 @@ class OffloadMoeCache:
self.evict_slots,
self.src_indices,
self.num_indices,
+ blocks_per_bank=self._copy_blocks_per_bank,
)
return