Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(moe): split the offload expert cache into slot pools (#689)

Xiaoze Fanb416f85c4097
diff --git a/docs/cli.md b/docs/cli.md
index 2166e9e8..44b0e30a 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -84,7 +84,9 @@ See [models.md](models.md#moe-strategies) for what each strategy does.
| `--moe-strategy` | auto | `fused`/`offload`/`cpu`/`hybrid`; auto → offload, or hybrid with a `ft bench bw` profile; fused on unified-memory GPUs (GB10), see [models.md](models.md#unified-memory-gpus-gb10--dgx-spark). `--moe-backend` is the deprecated old spelling |
| `--quant-backend` | auto | Kernel per quantized layer type, `layer[.kind]=name` entries: `linear=marlin,moe=b12x` or `moe.nvfp4=triton`. A layer-level entry applies to every kind whose table lists the name |
| `--nvfp4-backend` | — | Deprecated: stands in for `--quant-backend moe.nvfp4=<marlin\|b12x\|triton>` (`flashinfer` means b12x); cannot be combined with `--quant-backend` |
-| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) |
+| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-mem` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / memory such as `12GiB` / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) |
+| `--moe-cache-scope` | unified | `unified`: one LRU pool of slots per expert quant (quant type and bank layout), so all MoE layers share one pool when they share a quant format; `per-layer`: each layer has its own pool. `per-layer` does not combine with `--moe-strategy cpu` or `--moe-cpu-layers` |
+| `--moe-cache-split` | equal | How the pools share the slots: `equal` (in proportion to their layer counts), or `profile:<path>`: every pool keeps `top_k` slots and the rest go to the most used experts in the profile's per-layer expert counts. With `offload`, a pool smaller than the expert count lowers `--max-running-requests` to its slots / top_k |
| `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts |
| `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor |
| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, fp8_block, nvfp4, mxfp4) |
diff --git a/python/freetoken/engine/cache_budget.py b/python/freetoken/engine/cache_budget.py
index b5f07ee8..891ac2ac 100644
--- a/python/freetoken/engine/cache_budget.py
+++ b/python/freetoken/engine/cache_budget.py
@@ -14,8 +14,8 @@ if TYPE_CHECKING:
import torch
-def expert_bytes_per_slot(sources: dict[str, "list[torch.Tensor]"]) -> int:
- """Bytes one expert slot occupies on GPU: summed row bytes over all banks.
+def layer_slot_bytes(sources: dict[str, "list[torch.Tensor]"]) -> list[int]:
+ """Bytes one expert slot of each layer occupies on GPU: its row bytes summed over all banks.
Each bank source is per-layer ``[num_experts, *row_shape]`` tensors and is
already TP-sharded upstream, so the per-row byte count is the per-rank slot
@@ -25,7 +25,14 @@ def expert_bytes_per_slot(sources: dict[str, "list[torch.Tensor]"]) -> int:
# with cache_size), so they are intentionally excluded from the per-slot growth term.
# tensor[0].numel() is the per-row element count (one expert slot); see the matching
# slot-byte idiom in kvcache/linear_state_pool.py and kvcache/dsv4/v4_pool.py.
- return sum(t[0][0].numel() * t[0].element_size() for t in sources.values())
+ num_layers = len(next(iter(sources.values())))
+ return [sum(t[l][0].numel() * t[l].element_size() for t in sources.values()) for l in range(num_layers)]
+
+
+def expert_bytes_per_slot(sources: dict[str, "list[torch.Tensor]"]) -> int:
+ """Bytes one expert slot occupies on GPU, rounded up to the mean over layers when their bank layouts differ."""
+ per_layer = layer_slot_bytes(sources)
+ return -(-sum(per_layer) // len(per_layer))
def net_cache_budget_bytes(
diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py
index b6fa8b89..ae2f1fd6 100644
--- a/python/freetoken/engine/config.py
+++ b/python/freetoken/engine/config.py
@@ -45,6 +45,12 @@ class EngineConfig:
moe_cache_size: int = 0
moe_cache_rate: float | None = None
moe_cache_auto: bool = False
+ # --moe-cache-mem: slot-cache budget in bytes, turned into a slot count once the expert banks are loaded
+ moe_cache_mem: int | None = None
+ # --moe-cache-scope: unified (one LRU pool per expert quant) or per-layer (one pool per layer)
+ moe_cache_scope: str = "unified"
+ # --moe-cache-split: how pools share the slots; equal or profile:<path> (by expert use from a profile file)
+ moe_cache_split: str = "equal"
kv_reserve_tokens: int = 8192 # KV floor for --moe-cache-auto; small by design (MoE-priority)
moe_cache_policy: str = "lru"
moe_prefill_overlap: bool = True
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index feb33fbb..2757b10d 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -23,7 +23,8 @@ from freetoken.models.weight import ftw_lacks_vision
from freetoken.moe import is_offload_moe_strategy
from freetoken.moe.expert_banks import attach_resident_banks, load_expert_banks
from freetoken.moe.host_banks import PinFailed
-from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache
+from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache, bank_layouts
+from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, resolve_profile, split_slots
from freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtype
from .config import EngineConfig
@@ -38,6 +39,25 @@ from freetoken.kvcache.linear_state_pool import (
logger = init_logger(__name__)
+def _clamp_decode_batch_to_pools(config: EngineConfig, pool_sizes: list[int]) -> None:
+ """Clamp the decode batch so every GPU-decoded layer's routed experts fit in its pool."""
+ num_experts = config.model_config.num_experts
+ small = [s for s in pool_sizes if s < num_experts]
+ if not small:
+ return
+ limit = min(small) // config.model_config.num_experts_per_tok
+ if config.max_running_req > limit:
+ logger.warning_rank0(
+ f"the smallest MoE slot pool has {min(small)} slots; lowering max_running_req "
+ f"{config.max_running_req} -> {limit} so a decode step fits in it"
+ )
+ object.__setattr__(config, "max_running_req", limit)
+ if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs > limit:
+ object.__setattr__(config, "cuda_graph_max_bs", limit)
+ if config.cuda_graph_bs is not None:
+ object.__setattr__(config, "cuda_graph_bs", [bs for bs in config.cuda_graph_bs if bs <= limit])
+
+
def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:
"""The offload MoE cache needs at least one slot per expert per layer. A too-small size
(e.g. a bare offload run with moe_cache_size unset and auto disabled) must fail loudly."""
@@ -623,7 +643,7 @@ class Engine:
if parts:
batch.mm_embeds = torch.cat([p.to(self.dtype) for p in parts], dim=0)
- def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None) -> tuple[int, int, bool]:
+ def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None, max_slots=None) -> tuple[int, int, bool]:
"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).
Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable
@@ -647,7 +667,7 @@ class Engine:
prefill_overlap=config.moe_prefill_overlap,
kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),
page_size=page_tokens,
- max_slots=method.slot_limit() if method is not None else None,
+ max_slots=max_slots if max_slots is not None or method is None else method.slot_limit(),
)
def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
@@ -703,6 +723,11 @@ class Engine:
else HostResidency.PINNED.value
for i in range(config.model_config.num_moe_layers)
]
+ # read before the expert load, so a bad profile fails in seconds
+ expert_counts = resolve_profile(
+ config.moe_cache_split, config.model_path,
+ config.model_config.num_moe_layers, config.model_config.num_experts,
+ )
try:
with _weight_load_context():
banks = load_expert_banks(
@@ -718,8 +743,27 @@ class Engine:
)
except PinFailed as exc:
raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc
+ from freetoken.engine.cache_budget import expert_bytes_per_slot, layer_slot_bytes
+
+ self._moe_layer_slot_bytes = layer_slot_bytes(banks.sources)
+ self._moe_pool_groups = group_layers(config.moe_cache_scope, bank_layouts(banks.sources, banks.layer_tags), self._moe_layer_slot_bytes)
+ if len(self._moe_pool_groups) > 1 and cpu_layer_ids:
+ raise ValueError(
+ "several MoE slot pools (--moe-cache-scope per-layer, or experts in several quant formats) need every MoE "
+ "layer to decode from the GPU slot cache; they do not combine with --moe-strategy cpu or --moe-cpu-layers"
+ )
+ self._moe_expert_counts = expert_counts if len(self._moe_pool_groups) > 1 else None
+ if len(self._moe_pool_groups) == 1 and config.moe_cache_split != "equal":
+ logger.warning_rank0(
+ f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "
+ f"(--moe-cache-scope per-layer gives each layer its own)"
+ )
+ self._moe_slot_limit = method.slot_limit() if method is not None else None
+ # with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one token per step
+ pools = self._moe_pools(1)
+ size_cap = sum(p.max_num_slots for p in pools)
if config.moe_cache_auto:
- size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method)
+ size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method, max_slots=size_cap)
object.__setattr__(config, "moe_cache_size", size)
object.__setattr__(config, "moe_prefill_overlap", overlap)
if config.num_page_override is None:
@@ -735,7 +779,28 @@ class Engine:
f"--moe-cache-auto resolved moe_cache_size={size} "
f"num_pages={pages} (prefill_overlap={overlap})"
)
+ elif config.moe_cache_mem is not None:
+ per_slot = expert_bytes_per_slot(banks.sources)
+ size = min(config.moe_cache_mem // per_slot, size_cap)
+ # per_slot is a mean over layers, so with pools of different slot sizes the split can exceed the budget: shrink until it fits
+ while size > 0 and self._moe_pool_bytes(split_slots(size, pools, self._moe_expert_counts)) > config.moe_cache_mem:
+ size -= 1
+ overlap = config.moe_prefill_overlap and size >= 2 * config.model_config.num_experts
+ object.__setattr__(config, "moe_cache_size", size)
+ object.__setattr__(config, "moe_prefill_overlap", overlap)
+ logger.info_rank0(
+ f"--moe-cache-mem {mem_GB(config.moe_cache_mem)} resolved moe_cache_size={size} "
+ f"({per_slot / 2**20:.2f} MiB per slot, prefill_overlap={overlap})"
+ )
_require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)
+ pool_sizes = split_slots(config.moe_cache_size, pools, self._moe_expert_counts)
+ if len(pools) > 1:
+ logger.info_rank0(
+ f"MoE slot pools: {config.moe_cache_scope}, {len(pools)} pools of {min(pool_sizes)}-{max(pool_sizes)} "
+ f"slots ({config.moe_cache_split})"
+ )
+ if decode_target == "gpu":
+ _clamp_decode_batch_to_pools(config, pool_sizes)
layout = max_slots = None
if method is not None:
if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):
@@ -760,10 +825,12 @@ class Engine:
hybrid_max_fetch=config.moe_hybrid_max_fetch,
layout=layout,
max_slots=max_slots,
+ pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),
+ pool_sizes=pool_sizes,
)
# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
cache.cpu_layer_ids = cpu_layer_ids
- cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)
+ cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=banks.layer_tags)
cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
if decode_target == "hybrid":
self._resolve_hybrid_fetch(config, cache)
@@ -772,12 +839,23 @@ class Engine:
cache.collect_stats = config.moe_collect_stats
layers = attach_offload_moe_cache(self.model, cache)
assert len(layers) == config.model_config.num_moe_layers
+ cache.zero_inactive_route_slots = cache.decode_target == "hybrid" and not all(layer.skips_inactive_slots for layer in layers)
if cache.decode_target in ("cpu", "hybrid"):
self._init_cpu_moe_executor(config, cache, layers)
self.ctx.moe_offload_cache = cache
self.moe_offload_cache = cache
return cache
+ def _moe_pool_bytes(self, pool_sizes: list[int]) -> int:
+ return sum(s * self._moe_layer_slot_bytes[g[0]] for g, s in zip(self._moe_pool_groups, pool_sizes))
+
+ def _moe_pools(self, batch: int) -> list:
+ """The slot pools, each with room for the routed experts of ``batch`` decode tokens."""
+ model_config = self.config.model_config
+ num_experts = model_config.num_experts
+ min_num_slots = min(num_experts, model_config.num_experts_per_tok * batch)
+ return make_pools(self._moe_pool_groups, num_experts, min_num_slots, self._moe_slot_limit)
+
def _resolve_hybrid_fetch(self, config: EngineConfig, cache) -> None:
"""Resolve --moe-hybrid-max-fetch -1 (auto) into a bandwidth-matched fetch fraction.
@@ -867,19 +945,18 @@ class Engine:
return min_free_memory, max_free_memory
- def _target_moe_and_expert_bytes(self, moe_cache_size: int | None) -> tuple[int, int]:
- from freetoken.engine.cache_budget import expert_bytes_per_slot
-
- target_moe = (
- moe_cache_size
- if moe_cache_size is not None
- else (self.moe_offload_cache.cache_size if self.moe_offload_cache else 0)
- )
- per_expert_bytes = (
- expert_bytes_per_slot(self.moe_offload_cache.bank_sources)
- if self.moe_offload_cache is not None else 0
- )
- return target_moe, per_expert_bytes
+ def _target_moe_and_expert_bytes(self, moe_cache_size: int | None, pool_sizes: list[int] | None) -> tuple[int, int]:
+ cache = self.moe_offload_cache
+ if cache is None:
+ return moe_cache_size or 0, 0
+ if moe_cache_size is None:
+ target_moe = cache.cache_size
+ nbytes = sum(c.numel() * c.element_size() for _, c in cache.banks)
+ else:
+ target_moe = moe_cache_size
+ nbytes = self._moe_pool_bytes(pool_sizes)
+ # pools of several bank layouts have different slot sizes; round up so target * bytes covers them
+ return target_moe, -(-nbytes // target_moe)
def _resize_kv_pool(self, config, num_pages: int, num_swa_pages: int | None) -> None:
# IN-PLACE, identity-preserving: the CacheManager's swa_pool reference, ctx.kv_cache and
@@ -938,7 +1015,10 @@ class Engine:
"moe_cache_size requested but this model has no MoE offload cache"
)
try:
- self.moe_offload_cache.validate_rebuild(moe_cache_size)
+ # a rebuild does not lower max_running_req again, so with GPU decode every pool must hold a full batch; hybrid caps its fetches instead
+ batch = config.max_running_req if self.moe_offload_cache.decode_target == "gpu" else 1
+ moe_pool_sizes = split_slots(moe_cache_size, self._moe_pools(batch), self._moe_expert_counts)
+ self.moe_offload_cache.validate_rebuild(moe_cache_size, moe_pool_sizes)
except ValueError as e:
raise CacheRebuildRejected(str(e)) from e
if num_pages is not None and num_pages <= 0:
@@ -975,7 +1055,9 @@ class Engine:
# must reject (recoverable) so the old caches stay intact and serving continues,
# rather than freeing and then OOMing into permanent failure. The engine supplies
# the memory account; the pool answers whether its target geometry fits.
- target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(moe_cache_size)
+ target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(
+ moe_cache_size, moe_pool_sizes if moe_cache_size is not None else None
+ )
# Price the sibling GDN state pool at ITS target (physical slots = usable + padding
# sink) and hand the bytes in -- the KV pool only budgets its own tiers.
target_mamba = (
@@ -1019,7 +1101,7 @@ class Engine:
object.__setattr__(config, "swa_num_pages_override", num_swa_pages)
if moe_cache_size is not None:
assert self.moe_offload_cache is not None, "no MoE offload cache to resize"
- self.moe_offload_cache.rebuild(moe_cache_size)
+ self.moe_offload_cache.rebuild(moe_cache_size, moe_pool_sizes)
if num_pages is not None:
# sets self.num_pages (rebuilds KV + window)
self._resize_kv_pool(config, num_pages, num_swa_pages)
@@ -1455,6 +1537,9 @@ _DENSE_MOE_SETTINGS = {
"moe_cache_size": 0,
"moe_cache_rate": None,
"moe_cache_auto": False,
+ "moe_cache_mem": None,
+ "moe_cache_scope": "unified",
+ "moe_cache_split": "equal",
"moe_cpu_layers": None,
"moe_cpu_threads": 0,
"moe_hybrid_max_fetch": -1,
@@ -1750,6 +1835,7 @@ def _adjust_config(config: EngineConfig):
is_offload_moe_strategy(config.moe_strategy)
and config.moe_cache_size <= 0
and config.moe_cache_rate is None
+ and config.moe_cache_mem is None
and not getattr(config, "moe_cache_auto", False)
):
# args.py's "no sizing flag -> default --moe-cache-auto" only fires when the
@@ -1783,11 +1869,13 @@ def _adjust_config(config: EngineConfig):
# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The
# sizing flags no longer redirect the backend, so ignore them here and say so -- the
# geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-
- # rate was already folded into moe_cache_size above, and the three are mutually exclusive.
+ # rate was already folded into moe_cache_size above, and the sizing flags are mutually exclusive.
if config.moe_cache_rate:
inert = f"--moe-cache-rate={config.moe_cache_rate}"
elif config.moe_cache_size:
inert = f"--moe-cache-size={config.moe_cache_size}"
+ elif config.moe_cache_mem is not None:
+ inert = f"--moe-cache-mem={config.moe_cache_mem}"
elif getattr(config, "moe_cache_auto", False):
inert = "--moe-cache-auto"
else:
@@ -1799,6 +1887,7 @@ def _adjust_config(config: EngineConfig):
)
override("moe_cache_size", 0)
override("moe_cache_rate", None)
+ override("moe_cache_mem", None)
override("moe_cache_auto", False)
if is_moe and config.moe_strategy == "cpu":
@@ -1809,6 +1898,7 @@ def _adjust_config(config: EngineConfig):
num_experts = config.model_config.num_experts
if getattr(config, "moe_cache_auto", False):
override("moe_cache_auto", False)
+ override("moe_cache_mem", None)
override("moe_cache_size", 2 * num_experts)
override("moe_prefill_overlap", True)
logger.info_rank0(
diff --git a/python/freetoken/kvcache/cache_status.py b/python/freetoken/kvcache/cache_status.py
index 5b531ce2..8c29dcca 100644
--- a/python/freetoken/kvcache/cache_status.py
+++ b/python/freetoken/kvcache/cache_status.py
@@ -40,6 +40,9 @@ def compute_cache_unit_bytes(engine: "Engine") -> Dict[str, int]:
banks = getattr(engine.moe_offload_cache, "bank_caches", None)
if not banks:
return 0
+ if getattr(engine.moe_offload_cache, "_raw_banks", False):
+ # pools of several bank layouts share byte segments: report the mean slot size
+ return int(sum(t.numel() for t in banks.values()) // engine.moe_offload_cache.cache_size)
# Each bank cache is (cache_size, *row_shape); one slot's bytes = row bytes summed over
# the format's banks (== cache_budget.expert_bytes_per_slot on the source rows).
return int(sum(t[0].numel() * t.element_size() for t in banks.values()))
diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py
index 0595b1d4..79ad9e3a 100644
--- a/python/freetoken/layers/moe.py
+++ b/python/freetoken/layers/moe.py
@@ -189,6 +189,10 @@ class OffloadMoELayer(MoELayer):
)
self.offload_cache: OffloadMoeCache | None = None
+ @property
+ def skips_inactive_slots(self) -> bool:
+ return self.quant_method is not None and self.quant_method.kernel.supports_inactive_slots
+
def forward(
self,
hidden_states: torch.Tensor,
@@ -289,7 +293,7 @@ class OffloadMoELayer(MoELayer):
hidden_states,
topk_weights,
topk_ids,
- views=cache.bank_views(),
+ views=cache.pool_bank_views(self.layer_id),
n=None,
alphas=cache.alphas_for_slots(self.layer_id),
is_prefill=False,
@@ -317,7 +321,8 @@ class OffloadMoELayer(MoELayer):
assert executor is not None, "CPU MoE executor was not initialized"
# the kernel reads every route's slot: give the inactive ones slot 0, which holds a
# complete payload or the bank's zero fill (never uninitialized storage), times weight 0
- clamp = not (self.quant_method is not None and self.quant_method.kernel.supports_inactive_slots)
+ # with several quants this holds only because the cache zeroes staged slot 0 rows (zero_inactive_route_slots)
+ clamp = not self.skips_inactive_slots
gpu_slots, gpu_w, cpu_ids = cache.ensure_experts_hybrid(self.layer_id, topk_ids, topk_weights, clamp=clamp)
pending = executor.decode_submit(self.layer_id, hidden_states, topk_weights, cpu_ids)
@@ -333,7 +338,7 @@ class OffloadMoELayer(MoELayer):
hidden_states,
gpu_w,
gpu_slots,
- views=cache.bank_views(),
+ views=cache.pool_bank_views(self.layer_id),
n=None,
alphas=cache.alphas_for_slots(self.layer_id),
is_prefill=False,
@@ -370,16 +375,18 @@ class OffloadMoELayer(MoELayer):
return out
cache.materialize_layer(self.layer_id)
cache.copy_missing()
- return self._expert_gemm(
+ out = self._expert_gemm(
cache,
hidden_states,
topk_weights,
topk_ids,
- views=cache.bank_views(self.num_experts),
+ views=cache.stage_views(self.layer_id, 0),
n=self.num_experts,
alphas=cache.alphas_for_layer(self.layer_id),
is_prefill=True,
)
+ cache.release_prefill_layer(self.layer_id)
+ return out
def _wait_prefill_overlap(self, cache: OffloadMoeCache) -> tuple[torch.Tensor, ...]:
"""Double-buffer choreography for this layer's overlap prefill: kick off the
diff --git a/python/freetoken/models/deepseek_v4/moe.py b/python/freetoken/models/deepseek_v4/moe.py
index 35a316f1..aadcdc04 100644
--- a/python/freetoken/models/deepseek_v4/moe.py
+++ b/python/freetoken/models/deepseek_v4/moe.py
@@ -120,8 +120,9 @@ class DSV4OffloadMoELayer(OffloadMoELayer):
cache = self.offload_cache
assert cache is not None
# unpinned (LOCKED) layers must take the base materialize path: their copy_missing is the whole-layer pageable branch with position == expert id, which ensure_experts's LRU slot remap would contradict (the GEMM would gather other experts' weights)
+ # one threshold for all layers (the smallest pool): with overlap prefill, the layers of a chunk must all stream or all take the slot path
if (
- hidden_states.shape[0] * self.top_k >= self.num_experts
+ hidden_states.shape[0] * self.top_k >= min(self.num_experts, *cache.pool_sizes)
or cache.is_unpinned_layer(self.layer_id)
):
return super()._prefill_routed(hidden_states, topk_weights, topk_ids)
@@ -134,7 +135,7 @@ class DSV4OffloadMoELayer(OffloadMoELayer):
hidden_states,
topk_weights,
topk_ids,
- views=cache.bank_views(),
+ views=cache.pool_bank_views(self.layer_id),
n=None,
alphas=cache.alphas_for_slots(self.layer_id),
is_prefill=True,
diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.py
index 9430ab2a..cd260cb2 100644
--- a/python/freetoken/moe/benchbw.py
+++ b/python/freetoken/moe/benchbw.py
@@ -413,14 +413,11 @@ def _build_gather_rig(fmt: str, wl: Workload, device: torch.device):
specs = _offload_bank_specs(fmt, H, I)
cache = OffloadMoeCache(num_layers=1, num_experts=E, cache_size=E, device=device, quant_format=fmt)
total_bytes = 0
+ sources = {}
for name, (elems, dtype) in specs.items():
- src = alloc_pinned_tensor(E, elems, dtype=dtype) # cudaHostAlloc (mapped) like the loaders
- dst = torch.empty(E, elems, dtype=dtype, device=device)
- cache.bank_sources[name] = [src]
- cache.bank_caches[name] = dst
- cache.banks.append(([src], dst))
+ sources[name] = [alloc_pinned_tensor(E, elems, dtype=dtype)] # cudaHostAlloc (mapped) like the loaders
total_bytes += E * elems * dtype.itemsize
- cache._build_copy_plan() # enable the fused path (production default) when aligned
+ cache.set_bank_sources(sources) # builds the fused copy plan (production default) when aligned
cache.evict_slots[:E].copy_(torch.arange(E, dtype=torch.int32, device=device))
cache.src_indices[:E].copy_(torch.randperm(E, device=device).to(torch.int32))
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index 7500174a..5521c34e 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -48,6 +48,8 @@ class ExpertBanks:
kind: QuantKind | None = None
kernel: str | None = None
layout: dict | None = None
+ # per-layer quant tag (e.g. the gate_up and down GGUF types), for layers whose banks match in dtype and row shape but not in quant; None = one quant
+ layer_tags: list | None = None
def _dummy_fill(role: str, tensor: torch.Tensor) -> None:
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index 984b7124..701524e0 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -77,7 +77,7 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {
"down_bias",
),
# DeepSeek-V4 FP4: packed e2m1 codes + e8m0 per-32 block scales, no global scale
- # (4 banks). Read by DeepSeek-V4's own DS-FP4 grouped GEMV kernels via bank_views().
+ # (4 banks). Read by DeepSeek-V4's own DS-FP4 grouped GEMV kernels via pool_bank_views().
"ds_fp4": ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale"),
}
@@ -104,6 +104,14 @@ _BANK_BYTES_PER_EXPERT = {
MARLIN_MAX_CACHE_SIZE = 992
+def bank_layouts(sources: dict[str, list[torch.Tensor]], layer_tags: list | None = None) -> list[tuple]:
+ """Each layer's layout key: its quant tag and every bank's (dtype, row shape). A slot pool holds layers of one key."""
+ num_layers = len(next(iter(sources.values())))
+ tags = layer_tags if layer_tags is not None else [None] * num_layers
+ assert len(tags) == num_layers, (len(tags), num_layers)
+ return [(tags[l], tuple((t[l].dtype, tuple(t[l].shape[1:])) for t in sources.values())) for l in range(num_layers)]
+
+
@dataclass
class OffloadMoeCache:
num_layers: int
@@ -148,6 +156,9 @@ class OffloadMoeCache:
# bank layout from the expert kernel (a BankSpec per role); when given it replaces the _BANK_SCHEMAS lookup and the slot cap comes from max_slots
layout: dict | None = None
max_slots: int | None = None
+ # Each layer belongs to one pool: a contiguous slot range with its own LRU scope; pool p owns slots [sum(pool_sizes[:p]), +pool_sizes[p]). Slot ids in slot_for_id are local to the pool. None = one pool for all layers.
+ pool_of_layer: list[int] | None = None
+ pool_sizes: list[int] | None = None
def __post_init__(self) -> None:
policy_ids = {"lru": 0}
@@ -155,6 +166,14 @@ class OffloadMoeCache:
assert self.decode_target in ("gpu", "cpu", "hybrid"), self.decode_target
if self.layout is None:
assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"
+ if self.pool_sizes is None:
+ self.pool_of_layer = [0] * self.num_layers
+ self.pool_sizes = [self.cache_size]
+ self.pool_of_layer = list(self.pool_of_layer)
+ self.pool_sizes = list(self.pool_sizes)
+ assert len(self.pool_of_layer) == self.num_layers, (len(self.pool_of_layer), self.num_layers)
+ assert sorted(set(self.pool_of_layer)) == list(range(len(self.pool_sizes))), self.pool_of_layer
+ assert sum(self.pool_sizes) == self.cache_size and min(self.pool_sizes) > 0, (self.pool_sizes, self.cache_size)
# Attached by the engine for decode_target == "cpu" (CpuMoeExecutor); None
# for the GPU decode path.
self.cpu_executor = None
@@ -162,8 +181,16 @@ class OffloadMoeCache:
# offload/PCIe path. Set by the engine after construction (empty = all-GPU,
# all layers = the plain --moe-strategy cpu case).
self.cpu_layer_ids: frozenset = frozenset()
+ # Bank row layouts, set by set_bank_sources: layouts[g][k] is (dtype, row shape) of bank k (layouts of two quant tags may be equal), layer_layout[l] is layer l's g.
+ # With one (dtype, row shape) per bank a bank cache is typed [cache_size, *row]; with several it is a byte segment each pool views with its own layout.
+ self.layouts: list[tuple[tuple[torch.dtype, tuple[int, ...]], ...]] = []
+ self.layer_layout: list[int] = [0] * self.num_layers
+ self._raw_banks = False
+ # Set by the engine when hybrid decode hands CPU routes to a kernel that reads slot 0 with weight 0 (no supports_inactive_slots).
+ # With several quants a staging buffer can leave another quant's bytes in a pool's slot 0, which may decode to NaN, so the last prefill layer zeroes those rows.
+ self.zero_inactive_route_slots = False
# num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path.
- self.validate_rebuild(self.cache_size)
+ self.validate_rebuild(self.cache_size, self.pool_sizes)
assert not self.prefill_overlap or self.cache_size >= 2 * self.num_experts, (
"Prefill overlap borrows two full expert-layer buffers from the unified MoE "
"cache, so cache_size must be at least 2 * num_experts "
@@ -254,25 +281,28 @@ class OffloadMoeCache:
)
# (per-layer sources, cache) per bank, in schema order. Every piece of cache
# machinery that moves bank bytes (copy_missing, the prefill double buffers,
- # bank_views) iterates this list, so the slot cache is bank-count agnostic.
+ # the pool and staging views) iterates this list, so the slot cache is bank-count agnostic.
self.banks: list[tuple[list[torch.Tensor], torch.Tensor]] = []
# Fused multi-bank copy descriptor (built by set_bank_sources/_build_copy_plan).
# Source pointers are per layer (_copy_src_ptrs[layer_id] -> [num_banks] device
- # tensor); dst/feat are layer-invariant.
+ # tensor); dst pointers are per pool and row bytes per layout.
self._copy_fused_ok = False
self._copy_blocks_per_bank = 8
self._copy_dst_ptrs: torch.Tensor | None = None
+ self._copy_dst_ptrs_pool: list[torch.Tensor] | None = None
self._copy_src_ptrs: list[torch.Tensor] | None = None
- self._copy_feat_bytes: torch.Tensor | None = None
+ self._copy_feat_bytes: list[torch.Tensor] | None = None
# The layer whose misses ensure_experts/materialize_layer staged last; consumed
# by copy_missing to pick the per-layer source (part of the same pending-copy
# state as evict_slots/src_indices/num_indices).
# _pending_whole_layer records WHICH staged it: the pageable branch is only sound after materialize_layer
self._pending_src_layer: int | None = None
self._pending_whole_layer = False
- # Per-bank [2, num_experts, ...] double-buffer views over the slot cache's
- # first 2 * num_experts slots (set up when prefill_overlap is enabled).
- self.prefill_bank_buffers: list[torch.Tensor] = []
+ # The two prefill staging buffers are the first 2 * _stage_bytes[k] bytes of every bank, on top of the first pools.
+ # _stage_views[g][b] are buffer b's per-bank [num_experts, ...] views for layout g; _inval[b] are the global slots it overwrites.
+ self._stage_bytes: list[int] = []
+ self._stage_views: list[list[tuple[torch.Tensor, ...] | None]] = []
+ self._inval: list[slice | torch.Tensor] = []
self.prefill_copy_stream: torch.cuda.Stream | None = None
self.prefill_begin_event: torch.cuda.Event | None = None
self.prefill_ready_events: list[torch.cuda.Event] = []
@@ -282,7 +312,7 @@ class OffloadMoeCache:
self._prefill_buffer_has_release_event: list[bool] = [False, False]
# hit-D2D split state: pinned begin-of-chunk snapshot of slot_for_id (the
# classification input; frozen for the chunk -- no decode runs inside one,
- # and buffer invalidation only clears slot < 2E entries, which classify as
+ # and buffer invalidation only clears slots under the staging buffers, which classify as
# miss regardless), the lazily resolved batch-memcpy entry point (False =
# unavailable), and row counters for cache reports.
self._prefill_slot_snapshot: torch.Tensor | None = None
@@ -292,11 +322,116 @@ class OffloadMoeCache:
self._batch_memcpy = None
self.prefill_hit_rows = 0
self.prefill_total_rows = 0
+ self._slice_pools()
+
+ def _pool_layout(self, pool: int) -> int:
+ return self.layer_layout[self.pool_of_layer.index(pool)]
+
+ def _bank_rows(self, k: int, layout: int, offset: int, nrows: int) -> torch.Tensor:
+ """``nrows`` rows of bank ``k`` in ``layout``, starting ``offset`` bytes into the bank cache."""
+ cache = self.banks[k][1]
+ row = self._row_bytes[layout][k]
+ if not self._raw_banks:
+ first = offset // row
+ return cache if first == 0 and nrows == cache.shape[0] else cache[first : first + nrows]
+ dtype, shape = self.layouts[layout][k]
+ return cache[offset : offset + nrows * row].view(dtype).view(nrows, *shape)
+
+ def _stage_fits(self, buffers: int) -> bool:
+ return all(c.numel() * c.element_size() >= buffers * s for (_, c), s in zip(self.banks, self._stage_bytes))
+
+ def _slice_pools(self) -> None:
+ E = self.num_experts
+ bases = [0]
+ for size in self.pool_sizes[:-1]:
+ bases.append(bases[-1] + size)
+ self.pool_bases = bases
+ spans = list(zip(bases, self.pool_sizes))
+
+ def cut(t: torch.Tensor, b: int, s: int) -> torch.Tensor:
+ return t if s == t.shape[0] else t[b : b + s]
+
+ self._pool_id_of_slot = [cut(self.id_of_slot, b, s) for b, s in spans]
+ self._pool_usage = [cut(self.usage, b, s) for b, s in spans]
+ # rows[k][p]: bytes of one slot of pool p in bank k; before the banks exist, one unit per slot with a stage of E
+ if self.banks:
+ rows = [[self._row_bytes[self._pool_layout(p)][k] for p in range(len(spans))] for k in range(len(self.banks))]
+ stage = self._stage_bytes
+ else:
+ rows, stage = [[1] * len(spans)], [E]
+ offsets = []
+ for row in rows:
+ offsets.append([0])
+ for s, r in zip(self.pool_sizes[:-1], row):
+ offsets[-1].append(offsets[-1][-1] + s * r)
+ self._pool_offsets = offsets
+ # local slots below this index overlap the prefill staging area in some bank
+ self._pool_stable_from = [
+ max(min(max(-(-(2 * stage[k] - offsets[k][p]) // rows[k][p]), 0), s) for k in range(len(rows)))
+ for p, s in enumerate(self.pool_sizes)
+ ]
+ self._inval = []
+ masks = []
+ for b in range(2):
+ mask = torch.zeros(self.cache_size, dtype=torch.bool)
+ masks.append(mask)
+ for k in range(len(rows)):
+ lo, hi = b * stage[k], (b + 1) * stage[k]
+ for p, (base, s) in enumerate(spans):
+ first = max((lo - offsets[k][p]) // rows[k][p], 0)
+ end = min(-(-(hi - offsets[k][p]) // rows[k][p]), s)
+ mask[base + first : base + max(first, end)] = True
+ idx = mask.nonzero().flatten()
+ if idx.numel() == 0 or int(idx[-1] - idx[0]) + 1 == idx.numel():
+ self._inval.append(slice(int(idx[0]), int(idx[-1]) + 1) if idx.numel() else slice(0, 0))
+ else:
+ self._inval.append(idx.to(self.device))
+ self._stage_dirty = [False, False]
+ self._stage_slot0_rows = [[], []]
+ if not self.banks:
+ self._pool_bank_views, self._stage_views = [], []
+ return
+ self._pool_bank_views = [
+ tuple(self._bank_rows(k, self._pool_layout(p), offsets[k][p], s) for k in range(len(self.banks)))
+ for p, s in enumerate(self.pool_sizes)
+ ]
+ self._stage_views = [
+ [
+ tuple(self._bank_rows(k, g, b * stage[k], E) for k in range(len(self.banks))) if self._stage_fits(b + 1) else None
+ for b in range(2)
+ ]
+ for g in range(len(self.layouts))
+ ]
+ if len(self.layouts) > 1:
+ self._stage_slot0_rows = [
+ [self._bank_rows(k, self._pool_layout(p), offsets[k][p], 1) for p, base in enumerate(bases) if masks[b][base] for k in range(len(self.banks))]
+ for b in range(2)
+ ]
+
+ def stage_views(self, layer_id: int, buffer_id: int) -> tuple[torch.Tensor, ...]:
+ """Per-bank ``[num_experts, ...]`` views of prefill staging buffer ``buffer_id`` in ``layer_id``'s layout; row == expert id."""
+ views = self._stage_views[self.layer_layout[layer_id]][buffer_id]
+ assert views is not None, f"the slot cache is too small for staging buffer {buffer_id}"
+ return views
+
+ def pool_state(self, layer_id: int) -> tuple[torch.Tensor, torch.Tensor]:
+ """``(id_of_slot, usage)`` views of the pool that serves ``layer_id``."""
+ p = self.pool_of_layer[layer_id]
+ return self._pool_id_of_slot[p], self._pool_usage[p]
+
+ def pool_slots(self, layer_id: int) -> int:
+ return self.pool_sizes[self.pool_of_layer[layer_id]]
+
+ def pool_bank_views(self, layer_id: int) -> tuple[torch.Tensor, ...]:
+ """Per-bank slot views of the pool that serves ``layer_id``, indexed by its local slot ids."""
+ assert self.banks, "set_bank_sources must register the banks first"
+ return self._pool_bank_views[self.pool_of_layer[layer_id]]
def set_bank_sources(
self,
sources: dict[str, list[torch.Tensor]],
layer_residency: list[str] | None = None,
+ layer_tags: list | None = None,
) -> None:
"""Attach the host (CPU pinned) expert source banks and allocate a GPU slot
cache per bank, following the format's bank schema.
@@ -310,6 +445,8 @@ class OffloadMoeCache:
``layer_residency`` labels each layer with a ``HostResidency`` value (default: all pinned).
Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them.
+
+ ``layer_tags`` names each layer's quant (see :func:`bank_layouts`); layers of different tags never share a pool.
"""
from freetoken.moe.legacy_format import canonical_role
from freetoken.moe.host_banks import HostResidency
@@ -343,43 +480,62 @@ class OffloadMoeCache:
for name in self.bank_schema:
per_layer = sources[name]
assert len(per_layer) == self.num_layers, (name, len(per_layer))
- head = per_layer[0]
- if self.layout is not None:
- spec = self.layout[name]
- if tuple(head.shape[1:]) != tuple(spec.shape) or head.dtype != spec.dtype:
- raise ValueError(
- f"bank {name!r} rows are {tuple(head.shape[1:])} {head.dtype} but the expert kernel's layout "
- f"wants {tuple(spec.shape)} {spec.dtype}; the banks were packed for another kernel"
- )
for layer_id, source in enumerate(per_layer):
assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous"
assert source.size(0) == self.num_experts, (name, layer_id, source.shape)
- assert source.shape == head.shape and source.dtype == head.dtype, (
- name, layer_id, source.shape, source.dtype,
- )
+ if self.layout is not None:
+ spec = self.layout[name]
+ if tuple(source.shape[1:]) != tuple(spec.shape) or source.dtype != spec.dtype:
+ raise ValueError(
+ f"bank {name!r} rows are {tuple(source.shape[1:])} {source.dtype} but the expert kernel's layout "
+ f"wants {tuple(spec.shape)} {spec.dtype}; the banks were packed for another kernel"
+ )
self.bank_sources[name] = list(per_layer)
- self.bank_caches[name] = torch.zeros(
- (self.cache_size, *head.shape[1:]),
- dtype=head.dtype,
- device=self.device,
- )
- self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]
+ keys = bank_layouts(sources, layer_tags)
+ unique = list(dict.fromkeys(keys))
+ self.layouts = [banks for _, banks in unique]
+ self.layer_layout = [unique.index(key) for key in keys]
+ for p in range(len(self.pool_sizes)):
+ mixed = {self.layer_layout[l] for l, q in enumerate(self.pool_of_layer) if q == p}
+ assert len(mixed) == 1, f"slot pool {p} holds layers of {len(mixed)} layouts; a pool needs one"
+ self._raw_banks = len(set(self.layouts)) > 1
+ self._row_bytes = [[math.prod(shape) * dtype.itemsize for dtype, shape in layout] for layout in self.layouts]
+ if self._raw_banks:
+ # every pool view and staging buffer starts on a row boundary of its own layout
+ assert all(r % 16 == 0 for rows in self._row_bytes for r in rows), self._row_bytes
+ self._stage_bytes = [self.num_experts * max(rows[k] for rows in self._row_bytes) for k in range(len(self.bank_schema))]
+ self._alloc_banks()
+ if not self._stage_fits(1):
+ raise ValueError(f"MoE cache of {self.cache_size} slots cannot stage one whole layer of experts")
+ if self.prefill_overlap and not self._stage_fits(2):
+ logger.warning("Disabling MoE prefill overlap: the slot cache cannot hold two whole layers of experts")
+ self.prefill_overlap = False
+ self._slice_pools()
self._build_copy_plan()
if self.prefill_overlap:
self._init_prefill_overlap_buffers()
+ def _alloc_banks(self) -> None:
+ for k, name in enumerate(self.bank_schema):
+ if self._raw_banks:
+ nbytes = sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(self.pool_sizes))
+ self.bank_caches[name] = torch.zeros(nbytes, dtype=torch.uint8, device=self.device)
+ else:
+ dtype, shape = self.layouts[0][k]
+ self.bank_caches[name] = torch.zeros((self.cache_size, *shape), dtype=dtype, device=self.device)
+ self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]
+
def _build_copy_plan(self) -> None:
self._build_fused_copy_plan()
if self._copy_fused_ok or self.device.type != "cuda" or not self.banks:
return
- for name in self.bank_schema:
- cache = self.bank_caches[name]
- feat = math.prod(cache.shape[1:]) * cache.element_size()
- if feat % 128:
- raise RuntimeError(
- f"MoE bank {name!r} rows are {feat} bytes (not a multiple of 128): "
- f"only the fused multi-bank copy can move them, but it is disabled"
- )
+ for k, name in enumerate(self.bank_schema):
+ for rows in self._row_bytes:
+ if rows[k] % 128:
+ raise RuntimeError(
+ f"MoE bank {name!r} rows are {rows[k]} bytes (not a multiple of 128): "
+ f"only the fused multi-bank copy can move them, but it is disabled"
+ )
def _build_fused_copy_plan(self) -> None:
"""Precompute the fused multi-bank copy descriptor (base addrs + per-row bytes).
@@ -391,24 +547,25 @@ class OffloadMoeCache:
"""
self._copy_fused_ok = False
self._copy_dst_ptrs = None
+ self._copy_dst_ptrs_pool = None
self._copy_src_ptrs = None
self._copy_feat_bytes = None
self._copy_dst_ptrs_host: list[int] = []
self._copy_src_ptrs_host: list[list[int]] = []
- self._copy_feat_bytes_host: list[int] = []
- self._gather_bank_ids: list[int] = []
- self._gather_dst_ptrs: torch.Tensor | None = None
- self._gather_feat_bytes: torch.Tensor | None = None
+ self._gather_bank_ids: list[list[int]] = []
+ self._gather_feat_bytes: list[torch.Tensor | None] = []
+ self._gather_stage_ptrs: list[list[torch.Tensor] | None] = []
+ self._gather_pool_ptrs: list[torch.Tensor | None] = []
if not _FUSED_COPY or self.device.type != "cuda" or not self.banks:
return
from freetoken.kernel.pinned import device_ptr
- dst_ptrs, feats = [], []
+ aligned = [self._stage_bytes] + self._row_bytes + self._pool_offsets
+ if any(n % 16 for ns in aligned for n in ns) or any(c.data_ptr() % 16 for _, c in self.banks):
+ return # leave fused disabled; copy_missing uses the per-bank path
+ dst_ptrs = [cache.data_ptr() for _, cache in self.banks]
layer_src_ptrs = [[] for _ in range(self.num_layers)]
- for per_layer, cache in self.banks:
- feat = math.prod(per_layer[0].shape[1:]) * per_layer[0].element_size()
- if feat % 16 != 0 or cache.data_ptr() % 16 != 0:
- return # leave fused disabled; copy_missing uses the per-bank path
+ for per_layer, _ in self.banks:
for layer_id, source in enumerate(per_layer):
if layer_id in self._unpinned_layers:
# unregistered layer: no device alias exists, and the row is never consumed (CPU decode; pageable prefill)
@@ -422,52 +579,65 @@ class OffloadMoeCache:
if src_dev % 16 != 0:
return
layer_src_ptrs[layer_id].append(src_dev)
- dst_ptrs.append(cache.data_ptr())
- feats.append(feat)
- self._copy_dst_ptrs = torch.tensor(dst_ptrs, dtype=torch.int64, device=self.device)
- self._copy_src_ptrs = [
- torch.tensor(ptrs, dtype=torch.int64, device=self.device)
- for ptrs in layer_src_ptrs
+
+ def ptrs(values: list[int]) -> torch.Tensor:
+ return torch.tensor(values, dtype=torch.int64, device=self.device)
+
+ self._copy_dst_ptrs = ptrs(dst_ptrs)
+ self._copy_dst_ptrs_pool = [
+ self._copy_dst_ptrs if all(o[p] == 0 for o in self._pool_offsets)
+ else ptrs([d + o[p] for d, o in zip(dst_ptrs, self._pool_offsets)])
+ for p in range(len(self.pool_sizes))
]
- self._copy_feat_bytes = torch.tensor(feats, dtype=torch.int64, device=self.device)
+ self._copy_src_ptrs = [ptrs(values) for values in layer_src_ptrs]
+ self._copy_feat_bytes = [ptrs(rows) for rows in self._row_bytes]
self._copy_dst_ptrs_host = dst_ptrs
self._copy_src_ptrs_host = layer_src_ptrs
- self._copy_feat_bytes_host = feats
# hit-D2D gather serves only the big banks; small banks are whole-layer
# H2D entries (see _SMALL_BANK_FEAT_BYTES), so their rows never need D2D.
- self._gather_bank_ids = [i for i, f in enumerate(feats) if f >= _SMALL_BANK_FEAT_BYTES]
- if len(self._gather_bank_ids) == len(feats):
- self._gather_dst_ptrs = self._copy_dst_ptrs
- self._gather_feat_bytes = self._copy_feat_bytes
- elif self._gather_bank_ids:
- self._gather_dst_ptrs = self._copy_dst_ptrs[self._gather_bank_ids].contiguous()
- self._gather_feat_bytes = self._copy_feat_bytes[self._gather_bank_ids].contiguous()
+ # The gather copies cache rows of the layer's pool into its staging buffer, row == expert id.
+ self._gather_bank_ids = [[k for k, r in enumerate(rows) if r >= _SMALL_BANK_FEAT_BYTES] for rows in self._row_bytes]
+ for g, ids in enumerate(self._gather_bank_ids):
+ self._gather_feat_bytes.append(ptrs([self._row_bytes[g][k] for k in ids]) if ids else None)
+ self._gather_stage_ptrs.append(
+ [ptrs([dst_ptrs[k] + b * self._stage_bytes[k] for k in ids]) for b in range(2)] if ids else None
+ )
+ for p in range(len(self.pool_sizes)):
+ ids = self._gather_bank_ids[self._pool_layout(p)]
+ self._gather_pool_ptrs.append(ptrs([dst_ptrs[k] + self._pool_offsets[k][p] for k in ids]) if ids else None)
self._copy_blocks_per_bank = default_blocks_per_bank(self.device)
self._copy_fused_ok = True
- def validate_rebuild(self, cache_size: int) -> None:
+ def validate_rebuild(self, cache_size: int, pool_sizes: list[int] | None = None) -> None:
"""Pure geometry validation of a rebuild target (no GPU side effects).
Raises ``ValueError`` if ``cache_size`` is below the ``num_experts`` floor or
- above the marlin slot cap. Called by :meth:`rebuild` and by the engine's
+ a pool is above the expert kernel's slot cap, or the bank byte segments cannot stage one layer. Called by :meth:`rebuild` and by the engine's
pre-teardown check, so an invalid target rejects with the old cache intact
(no destructive free first).
"""
if cache_size < self.num_experts:
raise ValueError(f"cache_size {cache_size} < num_experts {self.num_experts}")
- if self.max_slots is not None and cache_size > self.max_slots:
+ # decode kernels see one pool's slots, so the cap applies per pool
+ largest = cache_size if pool_sizes is None else max(pool_sizes)
+ what = f"moe_cache_size={cache_size}" if largest == cache_size else f"a MoE slot pool of {largest} slots"
+ if self.max_slots is not None and largest > self.max_slots:
raise ValueError(
- f"moe_cache_size={cache_size} exceeds the expert kernel's slot limit of {self.max_slots}; "
+ f"{what} exceeds the expert kernel's slot limit of {self.max_slots}; "
f"pass --moe-cache-size {self.max_slots} or less, or let the default kernel serve the experts"
)
- if self.layout is None and self.quant_format == "nvfp4_marlin" and cache_size > MARLIN_MAX_CACHE_SIZE:
+ if self.layout is None and self.quant_format == "nvfp4_marlin" and largest > MARLIN_MAX_CACHE_SIZE:
raise ValueError(
- f"moe_cache_size={cache_size} exceeds the marlin backend's slot limit of "
+ f"{what} exceeds the marlin backend's slot limit of "
f"{MARLIN_MAX_CACHE_SIZE} (vLLM moe_align_block_size caps padded experts at "
"1024); reduce moe_cache_size or force --quant-backend moe.nvfp4=triton"
)
+ if self._raw_banks and pool_sizes is not None:
+ for k, stage in enumerate(self._stage_bytes):
+ if sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(pool_sizes)) < stage:
+ raise ValueError(f"moe_cache_size={cache_size} cannot stage one whole layer of experts")
- def rebuild(self, cache_size: int) -> None:
+ def rebuild(self, cache_size: int, pool_sizes: list[int] | None = None) -> None:
"""Resize the GPU slot cache + bookkeeping to ``cache_size`` IN PLACE.
Keeps the CPU/pinned ``bank_sources`` and the GPU-resident alphas; never
@@ -477,9 +647,10 @@ class OffloadMoeCache:
``ctx.moe_offload_cache`` stay valid.
"""
assert self.bank_sources, "set_bank_sources must run before rebuild"
- self.validate_rebuild(cache_size)
+ sizes = [cache_size] if pool_sizes is None else list(pool_sizes)
+ assert len(sizes) == len(self.pool_sizes) and sum(sizes) == cache_size and min(sizes) > 0, (sizes, cache_size)
+ self.validate_rebuild(cache_size, sizes)
# 1. Tear down prefill-overlap (its buffer views alias the old bank_caches).
- self.prefill_bank_buffers = []
self.prefill_copy_stream = None
self.prefill_begin_event = None
self.prefill_ready_events = []
@@ -490,22 +661,22 @@ class OffloadMoeCache:
# 2. Drop old GPU tensors (free-before-alloc).
self.banks = []
self.bank_caches = {}
+ self._pool_bank_views = []
+ self._stage_views = []
+ self._stage_slot0_rows = [[], []]
self.cache_size = cache_size
+ self.pool_sizes = sizes
if self.device.type == "cuda":
torch.cuda.synchronize(self.device)
torch.cuda.empty_cache()
# 3. Reallocate the slot cache from the retained host sources.
- for name in self.bank_schema:
- head = self.bank_sources[name][0]
- self.bank_caches[name] = torch.zeros(
- (cache_size, *head.shape[1:]), dtype=head.dtype, device=self.device
- )
- self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]
- self._build_copy_plan() # slot caches were reallocated -> refresh fused-copy addrs
+ self._alloc_banks()
# 4. Reallocate cache_size-shaped bookkeeping; reset the slot map (cold start).
self.slot_for_id.fill_(-1)
self.id_of_slot = torch.full((cache_size,), -1, dtype=torch.int32, device=self.device)
self.usage = torch.zeros((cache_size,), dtype=torch.int64, device=self.device)
+ self._slice_pools()
+ self._build_copy_plan() # slot caches were reallocated -> refresh fused-copy addrs
plan_slots = max(self.num_experts, cache_size)
self.evict_slots = torch.empty((plan_slots,), dtype=torch.int32, device=self.device)
self.src_indices = torch.empty((plan_slots,), dtype=torch.int32, device=self.device)
@@ -529,10 +700,10 @@ class OffloadMoeCache:
self.prefill_total_rows = 0
self._hit_d2d_fallback_logged = False # geometry changed; re-log if still unusable
# 5. Re-evaluate prefill overlap against the new size.
- if self.prefill_overlap and cache_size < 2 * self.num_experts:
+ if self.prefill_overlap and not self._stage_fits(2):
logger.warning(
f"Disabling MoE prefill overlap on rebuild: cache_size {cache_size} "
- f"< 2*num_experts {2 * self.num_experts}."
+ f"cannot hold two whole layers of experts (2*num_experts {2 * self.num_experts} slots of one layout)."
)
self.prefill_overlap = False
if self.prefill_overlap:
@@ -585,8 +756,9 @@ class OffloadMoeCache:
``layer_id`` -- are ever read by the grouped GEMM."""
if self.gate_up_alpha is None:
return None
+ id_of_slot, _ = self.pool_state(layer_id)
idx = layer_id * self.num_experts + (
- self.id_of_slot.clamp(min=0).long() % self.num_experts
+ id_of_slot.clamp(min=0).long() % self.num_experts
)
return self.gate_up_alpha[idx], self.down_alpha[idx]
@@ -600,25 +772,12 @@ class OffloadMoeCache:
hi = lo + self.num_experts
return self.gate_up_alpha[lo:hi], self.down_alpha[lo:hi]
- def bank_views(self, n: int | None = None) -> tuple[torch.Tensor, ...]:
- """Per-bank cache views in registration order: the full ``[S]`` slot cache
- (decode), or its first ``n`` slots (materialized layer)."""
- assert self.banks, "set_bank_sources must register the banks first"
- if n is None:
- return tuple(cache for _, cache in self.banks)
- return tuple(cache[:n] for _, cache in self.banks)
-
def _init_prefill_overlap_buffers(self) -> None:
assert self.banks, "set_bank_sources must register the banks first"
self._prefill_buffer_layer = [None, None]
self._prefill_buffer_released = [True, True]
self._prefill_buffer_has_release_event = [False, False]
- # The double buffers borrow the slot cache's first 2 * num_experts slots
- # (one full expert layer per buffer), one view per registered bank.
- self.prefill_bank_buffers = [
- cache[: 2 * self.num_experts].view(2, self.num_experts, *cache.shape[1:])
- for _, cache in self.banks
- ]
+ # the double buffers are the staging views (_slice_pools), on top of the first pools' slots
if self.device.type == "cuda":
self.prefill_copy_stream = torch.cuda.Stream(device=self.device)
self.prefill_ready_events = [torch.cuda.Event() for _ in range(2)]
@@ -638,14 +797,14 @@ class OffloadMoeCache:
self._prefill_hit_num = torch.zeros((1,), dtype=torch.int64, device=self.device)
def _invalidate_prefill_buffer(self, buffer_id: int) -> None:
- slot_start = buffer_id * self.num_experts
- slot_end = slot_start + self.num_experts
- old_ids = self.id_of_slot[slot_start:slot_end]
+ slots = self._inval[buffer_id]
+ old_ids = self.id_of_slot[slots]
self.slot_for_id.view(-1)[old_ids[old_ids >= 0].long()] = -1
- old_ids.fill_(-1)
+ self.id_of_slot[slots] = -1
# usage=0 makes these slots the oldest, so the argmin(usage) victim selection in
# ensure_experts evicts them first.
- self.usage[slot_start:slot_end].zero_()
+ self.usage[slots] = 0
+ self._stage_dirty[buffer_id] = True
def begin_prefill(self) -> None:
if not self.prefill_overlap:
@@ -676,7 +835,7 @@ class OffloadMoeCache:
if layer_id < 0:
raise ValueError(f"Invalid prefill layer id: {layer_id}")
- assert self.banks and self.prefill_bank_buffers
+ assert self.banks and self._stage_views
buffer_id = layer_id % 2
if self._prefill_buffer_layer[buffer_id] == layer_id:
@@ -688,8 +847,8 @@ class OffloadMoeCache:
def copy() -> None:
self._invalidate_prefill_buffer(buffer_id)
- for (per_layer, _), buffer in zip(self.banks, self.prefill_bank_buffers):
- buffer[buffer_id].copy_(per_layer[layer_id], non_blocking=True)
+ for (per_layer, _), view in zip(self.banks, self.stage_views(layer_id, buffer_id)):
+ view.copy_(per_layer[layer_id], non_blocking=True)
if self._prefill_hit_d2d_active:
self._prefetch_split(layer_id, buffer_id)
@@ -720,10 +879,10 @@ class OffloadMoeCache:
reason = "FREETOKEN_SKIP_FAST_INDEX_COPY is set (the hit gather would be a no-op)"
elif not self._copy_fused_ok:
reason = "the fused copy plan is unavailable (bank alignment or FREETOKEN_FUSED_COPY=0)"
- elif self.cache_size <= 2 * self.num_experts:
+ elif all(stable >= s for stable, s in zip(self._pool_stable_from, self.pool_sizes)):
reason = (
f"cache_size {self.cache_size} leaves no hit region "
- f"(needs > {2 * self.num_experts} slots)"
+ f"(every slot lies under the prefill staging buffers of {2 * self.num_experts} experts)"
)
elif not self._resolve_batch_memcpy():
reason = "cudaMemcpyBatchAsync is unavailable" # resolve logged the specifics
@@ -761,7 +920,7 @@ class OffloadMoeCache:
run list comes from the begin-of-chunk snapshot because the batch API
takes HOST pointer arrays. Live-vs-snapshot cannot disagree: the only
chunk-internal writer (buffer invalidation) rewrites slots already below
- the 2E threshold, and slots < 2E (including -1) are misses on both sides
+ the pool's staging threshold, and those slots (and -1) are misses on both sides
-- the buffers own those slots, so their bytes are volatile within the
chunk. Hit and miss row sets are disjoint, so the streams need no
ordering against each other.
@@ -772,18 +931,20 @@ class OffloadMoeCache:
from freetoken.moe.offload_kernels import prefill_hit_compact
E = self.num_experts
+ layout = self.layer_layout[layer_id]
+ pool = self.pool_of_layer[layer_id]
snap = self._prefill_snapshot_np[layer_id]
- hit_mask = snap >= 2 * E
+ hit_mask = snap >= self._pool_stable_from[pool]
self.prefill_hit_rows += int(hit_mask.sum())
self.prefill_total_rows += E
- if self._gather_dst_ptrs is not None:
- prefill_hit_compact(self, layer_id, buffer_id)
+ if self._gather_feat_bytes[layout] is not None:
+ prefill_hit_compact(self, layer_id)
# blocks_per_bank=64 vs the PCIe-tuned default of 8: HBM D2D needs the
# wider grid (~22 GB/s per 1024-thread block on H100).
fast_index_copy_multi_jit(
- self._gather_dst_ptrs,
- self._gather_dst_ptrs,
- self._gather_feat_bytes,
+ self._gather_stage_ptrs[layout][buffer_id],
+ self._gather_pool_ptrs[pool],
+ self._gather_feat_bytes[layout],
self._prefill_hit_dst,
self._prefill_hit_src,
self._prefill_hit_num,
@@ -799,16 +960,17 @@ class OffloadMoeCache:
starts = miss[run_starts]
lengths = np.diff(np.concatenate((run_starts, [miss.size])))
dst, src, nbytes = [], [], []
- for b, feat in enumerate(self._copy_feat_bytes_host):
+ for b, feat in enumerate(self._row_bytes[layout]):
+ stage = self._copy_dst_ptrs_host[b] + buffer_id * self._stage_bytes[b]
if feat < _SMALL_BANK_FEAT_BYTES:
# Whole layer as one entry, EVEN with zero misses: it keeps every
# batch entry above the driver's async floor and covers the hit
# rows the gather skips for these banks.
- dst.append(self._copy_dst_ptrs_host[b] + buffer_id * E * feat)
+ dst.append(stage)
src.append(self._copy_src_ptrs_host[layer_id][b])
nbytes.append(E * feat)
elif miss.size:
- dst.extend(self._copy_dst_ptrs_host[b] + (buffer_id * E + starts) * feat)
+ dst.extend(stage + starts * feat)
src.extend(self._copy_src_ptrs_host[layer_id][b] + starts * feat)
nbytes.extend(lengths * feat)
if dst:
@@ -826,15 +988,22 @@ class OffloadMoeCache:
marlin/b12x ``(gate_up_packed, gate_up_scale, down_packed, down_scale)``;
nvfp4 native adds the two global banks after each scale bank."""
assert self.prefill_overlap
- assert self.prefill_bank_buffers
+ assert self._stage_views
self.prefetch_prefill_layer(layer_id)
buffer_id = layer_id % 2
assert self._prefill_buffer_layer[buffer_id] == layer_id
if self.prefill_ready_events:
torch.cuda.current_stream(self.device).wait_event(self.prefill_ready_events[buffer_id])
- return tuple(buffer[buffer_id] for buffer in self.prefill_bank_buffers)
+ return self.stage_views(layer_id, buffer_id)
def release_prefill_layer(self, layer_id: int) -> None:
+ if self.zero_inactive_route_slots and layer_id == self.num_layers - 1:
+ # the staged slots were invalidated and nothing registers them within the forward, so their bytes are free to clear
+ for b in range(2):
+ if self._stage_dirty[b]:
+ for row in self._stage_slot0_rows[b]:
+ row.zero_()
+ self._stage_dirty[b] = False
if not self.prefill_overlap:
return
buffer_id = layer_id % 2
@@ -959,6 +1128,8 @@ class OffloadMoeCache:
s, m, a, f = steps[L], missing[L], active[L], fetched[L]
per_layer.append({
"layer": L,
+ "pool": self.pool_of_layer[L],
+ "pool_slots": self.pool_slots(L),
"steps": s,
"active_per_step": (a / s) if s else 0.0,
"missing_per_step": (m / s) if s else 0.0,
@@ -1010,11 +1181,13 @@ class OffloadMoeCache:
f"pageable materialize (position == expert id); ensure_experts's "
f"LRU slot remap cannot be honored without a device alias"
)
- # the only copy a non-pinned layer ever needs is the non-overlap prefill materialize, which schedules the whole layer into slots [0, num_experts) with position == expert id -- a plain synchronous pageable H2D copy
+ # the only copy a non-pinned layer ever needs is the non-overlap prefill materialize, which schedules the whole layer into staging buffer 0 with position == expert id -- a plain synchronous pageable H2D copy
# never CUDA-graph captured: prefill is not captured, and decode never reaches this branch (it routes to the CPU executor)
- for per_layer, cache in self.banks:
- cache[: self.num_experts].copy_(per_layer[layer_id])
+ for (per_layer, _), view in zip(self.banks, self.stage_views(layer_id, 0)):
+ view.copy_(per_layer[layer_id])
return
+ # materialize stages rows of staging buffer 0; ensure stages slots local to the layer's pool
+ whole = self._pending_whole_layer
if self._copy_fused_ok:
from freetoken.kernel.fast_index_copy import fast_index_copy_multi_jit
@@ -1023,9 +1196,9 @@ class OffloadMoeCache:
# src_indices holds layer-local expert rows, resolved against this layer's
# source pointers (layer_id is a static int per captured graph node).
fast_index_copy_multi_jit(
- self._copy_dst_ptrs,
+ self._copy_dst_ptrs if whole else self._copy_dst_ptrs_pool[self.pool_of_layer[layer_id]],
self._copy_src_ptrs[layer_id],
- self._copy_feat_bytes,
+ self._copy_feat_bytes[self.layer_layout[layer_id]],
self.evict_slots,
self.src_indices,
self.num_indices,
@@ -1035,7 +1208,8 @@ class OffloadMoeCache:
from freetoken.kernel import fast_index_copy_jit
- for per_layer, cache in self.banks:
+ views = self.stage_views(layer_id, 0) if whole else self.pool_bank_views(layer_id)
+ for (per_layer, _), cache in zip(self.banks, views):
fast_index_copy_jit(
cache,
self.evict_slots,
diff --git a/python/freetoken/moe/offload_kernels.py b/python/freetoken/moe/offload_kernels.py
index ea12b6fb..1d863a2f 100644
--- a/python/freetoken/moe/offload_kernels.py
+++ b/python/freetoken/moe/offload_kernels.py
@@ -26,11 +26,12 @@ def ensure_experts(cache, layer_id: int, expert_ids: torch.Tensor) -> None:
tensor. ``out_indices`` aliases the input, preserving the in-place rewrite every
downstream GEMM depends on.
"""
+ id_of_slot, usage = cache.pool_state(layer_id)
lru_ensure(
expert_ids,
cache.slot_for_id.view(-1),
- cache.id_of_slot,
- cache.usage,
+ id_of_slot,
+ usage,
cache.step,
expert_ids,
cache.src_indices,
@@ -80,13 +81,13 @@ def ensure_experts_hybrid(
return routes
-def prefill_hit_compact(cache, layer_id: int, buffer_id: int) -> None:
+def prefill_hit_compact(cache, layer_id: int) -> None:
"""Compact this layer's cache-resident experts into gather indices, device-side.
- hit = slot_for_id[layer_id][e] >= 2 * num_experts (the double buffer owns the
+ hit = slot_for_id[layer_id][e] >= the pool's staging threshold (the double buffer owns the
slots below, so those bytes are volatile within a prefill chunk and classify
as miss). Writes fixed-shape ``_prefill_hit_dst``/``_prefill_hit_src`` (buffer
- row / cache slot) and the count into ``_prefill_hit_num``; one launch on the
+ row / pool-local slot) and the count into ``_prefill_hit_num``; one launch on the
current stream, no host sync. Safe against the concurrent buffer invalidation
on the copy stream: that only rewrites entries already below the threshold."""
num_experts = cache.num_experts
@@ -95,15 +96,23 @@ def prefill_hit_compact(cache, layer_id: int, buffer_id: int) -> None:
cache._prefill_hit_dst,
cache._prefill_hit_src,
cache._prefill_hit_num,
- buffer_id * num_experts,
- 2 * num_experts,
+ cache._pool_stable_from[cache.pool_of_layer[layer_id]],
num_experts,
BLOCK=triton.next_power_of_2(num_experts),
)
def materialize_layer(cache, layer_id: int) -> None:
- _materialize_layer_gpu(cache, layer_id)
+ if len(cache.pool_sizes) == 1:
+ _materialize_layer_gpu(cache, layer_id)
+ return
+ # staging buffer 0 may overlap several pools, so it only holds the layer for this forward and registers nothing
+ E = cache.num_experts
+ cache._invalidate_prefill_buffer(0)
+ rows = torch.arange(E, dtype=torch.int32, device=cache.device)
+ cache.evict_slots[:E].copy_(rows)
+ cache.src_indices[:E].copy_(rows)
+ cache.num_indices.fill_(E)
def reset_cache(cache) -> None:
@@ -124,14 +133,16 @@ def _ensure_experts_hybrid_gpu(
max_fetch: int,
frac_q16: int,
) -> None:
+ id_of_slot, usage = cache.pool_state(layer_id)
+ num_slots = id_of_slot.numel()
block_e = triton.next_power_of_2(cache.num_experts)
- block_c = triton.next_power_of_2(cache.cache_size)
+ block_c = triton.next_power_of_2(num_slots)
num_warps = 8 if block_c >= 2048 else 4
_ensure_experts_hybrid_kernel[(1,)](
expert_ids,
cache.slot_for_id,
- cache.id_of_slot,
- cache.usage,
+ id_of_slot,
+ usage,
cache.step,
cache.active_mask,
cache.evict_slots,
@@ -148,7 +159,7 @@ def _ensure_experts_hybrid_gpu(
int(max_fetch),
int(frac_q16),
cache.num_experts,
- cache.cache_size,
+ num_slots,
BLOCK_E=block_e,
BLOCK_C=block_c,
BY_RECENCY=_HYBRID_FETCH_BY_RECENCY,
@@ -257,7 +268,7 @@ def _materialize_layer_kernel(
[email protected](do_not_specialize=["layer_id", "num_active", "max_fetch", "fetch_frac_q16"])
[email protected](do_not_specialize=["layer_id", "num_active", "max_fetch", "fetch_frac_q16", "cache_size", "id_of_slot_ptr", "usage_ptr"])
def _ensure_experts_hybrid_kernel(
expert_ids_ptr,
slot_for_id_ptr,
@@ -280,7 +291,7 @@ def _ensure_experts_hybrid_kernel(
max_fetch,
fetch_frac_q16,
num_experts: tl.constexpr,
- cache_size: tl.constexpr,
+ cache_size,
BLOCK_E: tl.constexpr,
BLOCK_C: tl.constexpr,
BY_RECENCY: tl.constexpr,
@@ -329,10 +340,11 @@ def _ensure_experts_hybrid_kernel(
(lo + 1) * ((1 << 16) - fetch_frac_q16), (num_missing - lo - 1) * fetch_frac_q16
)
max_fetch = tl.where(cost_lo <= cost_hi, lo, lo + 1)
- num_fetch = tl.minimum(num_missing, max_fetch)
+ is_hit = is_active & (slot >= 0)
+ # a pool smaller than num_experts may hold fewer free slots than misses
+ num_fetch = tl.minimum(tl.minimum(num_missing, max_fetch), cache_size - tl.sum(is_hit.to(tl.int32)))
tl.store(num_missing_full_ptr, num_missing.to(tl.int64))
tl.store(num_indices_ptr, num_fetch.to(tl.int64))
- is_hit = is_active & (slot >= 0)
tl.store(usage_ptr + slot, step, mask=is_hit)
# Fetch-selection priority: encode (recency desc, id asc) into one strictly-ordered
@@ -405,14 +417,13 @@ def _ensure_experts_hybrid_kernel(
tl.store(expert_recency_ptr + base + off_e, step_vec, mask=is_active & e_mask)
[email protected](do_not_specialize=["buffer_base"])
[email protected](do_not_specialize=["threshold"])
def _prefill_hit_compact_kernel(
slot_ptr, # [num_experts] int32: this layer's slot_for_id row
- dst_ptr, # [num_experts] int32 out: buffer rows, compacted
- src_ptr, # [num_experts] int32 out: cache slots, compacted
+ dst_ptr, # [num_experts] int32 out: buffer rows (expert ids), compacted
+ src_ptr, # [num_experts] int32 out: pool-local slots, compacted
num_ptr, # [1] int64 out: hit count
- buffer_base, # buffer_id * num_experts
- threshold, # 2 * num_experts
+ threshold, # first pool-local slot outside the staging buffers
num_experts,
BLOCK: tl.constexpr,
):
@@ -421,6 +432,6 @@ def _prefill_hit_compact_kernel(
slots = tl.load(slot_ptr + offs, mask=lane, other=-1)
is_hit = lane & (slots >= threshold)
pos = tl.cumsum(is_hit.to(tl.int32)) - 1
- tl.store(dst_ptr + pos, (buffer_base + offs).to(tl.int32), mask=is_hit)
+ tl.store(dst_ptr + pos, offs.to(tl.int32), mask=is_hit)
tl.store(src_ptr + pos, slots, mask=is_hit)
tl.store(num_ptr, tl.sum(is_hit.to(tl.int64)))
diff --git a/python/freetoken/moe/pool_policy.py b/python/freetoken/moe/pool_policy.py
new file mode 100644
index 00000000..990c18ad
--- /dev/null
+++ b/python/freetoken/moe/pool_policy.py
@@ -0,0 +1,149 @@
+"""Slot pool policy for the offload MoE cache: which layers share a pool, and how many slots each pool gets.
+
+Pure functions. The engine supplies the constraints and hands the result to ``OffloadMoeCache``:
+
+ groups = group_layers(scope, layouts, slot_bytes) # --moe-cache-scope, once per load
+ pools = make_pools(groups, num_experts, min_num_slots, slot_limit) # the minimum follows the decode batch
+ sizes = split_slots(total, pools, counts) # --moe-cache-split, counts from resolve_profile
+ OffloadMoeCache(pool_of_layer=pool_of_layer(pools, num_layers), pool_sizes=sizes, ...)
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import torch
+
+from freetoken.utils import init_logger
+
+logger = init_logger(__name__)
+
+SCOPES = ("unified", "per-layer")
+
+
+@dataclass(frozen=True)
+class PoolSpec:
+ layers: tuple[int, ...]
+ min_num_slots: int
+ max_num_slots: int
+
+
+def group_layers(scope: str, layouts: list, slot_bytes: list[int]) -> list[list[int]]:
+ """The layers of each pool, in placement order.
+
+ ``unified``: one pool per layout key (see ``bank_layouts``); ``per-layer``: one pool per layer.
+ Pools with wider slots come first: the prefill staging buffers sit at the start of every bank, so they then cover fewer slots.
+ """
+ if scope == "unified":
+ groups: dict = {}
+ for layer, layout in enumerate(layouts):
+ groups.setdefault(layout, []).append(layer)
+ pools = list(groups.values())
+ elif scope == "per-layer":
+ pools = [[layer] for layer in range(len(layouts))]
+ else:
+ raise ValueError(f"unknown --moe-cache-scope {scope!r}; want one of {SCOPES}")
+ return sorted(pools, key=lambda layers: (-slot_bytes[layers[0]], layers[0]))
+
+
+def make_pools(groups: list[list[int]], num_experts: int, min_num_slots: int, slot_limit: int | None) -> list[PoolSpec]:
+ """Each group's slot range: at least ``min_num_slots``, at most one slot per expert of its layers and the kernel's ``slot_limit``."""
+ pools = []
+ for layers in groups:
+ max_num_slots = len(layers) * num_experts
+ if slot_limit is not None:
+ max_num_slots = min(max_num_slots, slot_limit)
+ pools.append(PoolSpec(tuple(layers), min(min_num_slots, max_num_slots), max_num_slots))
+ return pools
+
+
+def pool_of_layer(pools: list[PoolSpec], num_layers: int) -> list[int]:
+ """Each layer's pool index, the form ``OffloadMoeCache`` takes."""
+ owner = [-1] * num_layers
+ for p, pool in enumerate(pools):
+ for layer in pool.layers:
+ owner[layer] = p
+ assert min(owner) >= 0, owner
+ return owner
+
+
+def split_slots(total: int, pools: list[PoolSpec], counts: torch.Tensor | None = None) -> list[int]:
+ """Slots per pool, each in [min_num_slots, max_num_slots], summing to ``total``.
+
+ With ``counts`` (per-layer expert use from ``resolve_profile``) the slots follow expert use, else the pools' layer counts.
+ """
+ if len(pools) == 1:
+ return [total]
+ _check_budget(total, pools)
+ return _split_equal(total, pools) if counts is None else _split_hot(total, pools, counts)
+
+
+def _check_budget(total: int, pools: list[PoolSpec]) -> None:
+ need = sum(p.min_num_slots for p in pools)
+ room = sum(p.max_num_slots for p in pools)
+ if total < need:
+ raise ValueError(f"MoE cache of {total} slots is below what the pools need ({need} slots)")
+ if total > room:
+ raise ValueError(f"MoE cache of {total} slots exceeds what the pools can hold ({room} slots)")
+
+
+def _split_equal(total: int, pools: list[PoolSpec]) -> list[int]:
+ sizes = [p.min_num_slots for p in pools]
+ weights = [len(p.layers) for p in pools]
+ left = total - sum(sizes)
+ while left > 0:
+ open_ = [i for i, p in enumerate(pools) if sizes[i] < p.max_num_slots]
+ # give the next slot to the pool with the fewest slots per layer
+ i = min(open_, key=lambda j: (sizes[j] / weights[j], j))
+ sizes[i] += 1
+ left -= 1
+ return sizes
+
+
+def _split_hot(total: int, pools: list[PoolSpec], counts: torch.Tensor) -> list[int]:
+ sizes = [p.min_num_slots for p in pools]
+ # a pool's minimum covers its hottest experts; the rest, up to its maximum, competes across pools
+ cand = []
+ for p, pool in enumerate(pools):
+ hot = counts[list(pool.layers)].flatten().sort(descending=True).values
+ cand.extend((float(c), p) for c in hot[pool.min_num_slots : pool.max_num_slots])
+ cand.sort(key=lambda c: -c[0])
+ for _, p in cand[: total - sum(sizes)]:
+ sizes[p] += 1
+ return sizes
+
+
+def resolve_profile(spec: str, model_path: str, num_layers: int, num_experts: int) -> torch.Tensor | None:
+ """Expert use for ``--moe-cache-split``: None for ``equal``, else the profile at ``profile:<path>``."""
+ if spec == "equal":
+ return None
+ path = Path(spec.removeprefix("profile:"))
+ logger.info_rank0(f"MoE pool profile: {path}")
+ return load_profile(path, num_layers, num_experts, model_path)
+
+
+def load_profile(path: str | Path, num_layers: int, num_experts: int, model_path: str = "") -> torch.Tensor:
+ """Per-layer expert use ``[num_layers, num_experts]`` from a profile, each row scaled to sum to 1."""
+ data = json.loads(Path(path).read_text())
+ model = str(data.get("model", ""))
+ if model and model_path and not any(n.lower().startswith(model.lower()) for n in _model_names(model_path)):
+ logger.warning_rank0(f"MoE pool profile {path} was taken on {model!r}, not on {model_path}")
+ counts = torch.tensor(data["expert_counts"], dtype=torch.float64)
+ if tuple(counts.shape) != (num_layers, num_experts):
+ raise ValueError(
+ f"profile {path} has {tuple(counts.shape)} expert counts; this model needs ({num_layers}, {num_experts})"
+ )
+ # every layer decodes the same tokens, so equal row sums keep the layers comparable
+ return counts / counts.sum(dim=1, keepdim=True).clamp(min=1e-12)
+
+
+def _model_names(model_path: str) -> list[str]:
+ # every path component, so an HF cache snapshot (models--Org--Name/snapshots/<hash>) still finds its name
+ names = []
+ for part in reversed(Path(model_path).parts):
+ name = part.split("--")[-1].removesuffix(".gguf")
+ if name:
+ names.append(name)
+ return names
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index 7c0ed9b0..be244a11 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -134,6 +134,7 @@ def parse_args(
from freetoken.attention import validate_attn_backend
from freetoken.kvcache import SUPPORTED_CACHE_MANAGER
from freetoken.moe import MOE_STRATEGIES
+ from freetoken.moe.pool_policy import SCOPES
def _parse_quant_backend(value: str) -> str:
from freetoken.layers.quantization import QuantBackend
@@ -153,6 +154,27 @@ def parse_args(
raise argparse.ArgumentTypeError("must be in [0, 1]")
return rate
+ def _parse_mem_size(value: str) -> int:
+ units = {
+ "b": 1, "k": 1 << 10, "kb": 1 << 10, "kib": 1 << 10, "m": 1 << 20, "mb": 1 << 20, "mib": 1 << 20,
+ "g": 1 << 30, "gb": 1 << 30, "gib": 1 << 30,
+ }
+ text = value.strip().lower()
+ number = text.rstrip("abgikm")
+ unit = text[len(number):]
+ try:
+ size = int(float(number) * units[unit])
+ except (KeyError, ValueError, OverflowError) as exc:
+ raise argparse.ArgumentTypeError("must be a size with a unit, e.g. 12GiB or 800MiB") from exc
+ if size <= 0:
+ raise argparse.ArgumentTypeError("must be positive")
+ return size
+
+ def _parse_cache_split(value: str) -> str:
+ if value == "equal" or (value.startswith("profile:") and len(value) > len("profile:")):
+ return value
+ raise argparse.ArgumentTypeError("must be equal or profile:<path>")
+
def _positive_int(value: str) -> int:
try:
n = int(value)
@@ -705,6 +727,12 @@ def parse_args(
default=ServerArgs.moe_cache_rate,
help="The fraction of all MoE experts to keep in GPU cache.",
)
+ moe_cache_group.add_argument(
+ "--moe-cache-mem",
+ type=_parse_mem_size,
+ default=ServerArgs.moe_cache_mem,
+ help="GPU memory for the MoE expert slot cache, e.g. 12GiB or 800MiB (G, GB and GiB all mean GiB); the slot count is this over the bytes per slot.",
+ )
moe_cache_group.add_argument(
"--moe-cache-auto",
action="store_true",
@@ -722,6 +750,27 @@ def parse_args(
help="KV-cache token floor reserved before --moe-cache-auto fills experts.",
)
+ parser.add_argument(
+ "--moe-cache-scope",
+ choices=SCOPES,
+ default=ServerArgs.moe_cache_scope,
+ help=(
+ "How MoE layers share the GPU expert slot cache: unified = one LRU pool per expert quant "
+ "(all layers when they share one quant format); per-layer = each layer has its own pool. "
+ "Pools are sized by --moe-cache-split."
+ ),
+ )
+ parser.add_argument(
+ "--moe-cache-split",
+ type=_parse_cache_split,
+ default=ServerArgs.moe_cache_split,
+ help=(
+ "How pools share the slots: equal (by layer count), or profile:<path> = by expert use from a "
+ "profile file. With --moe-strategy offload, a pool smaller than the expert count limits the "
+ "decode batch to its slots / top_k."
+ ),
+ )
+
parser.add_argument(
"--moe-cache-policy",
default=ServerArgs.moe_cache_policy,
@@ -892,6 +941,7 @@ def parse_args(
_no_cache_flag = (
kwargs["moe_cache_size"] == 0
and not kwargs["moe_cache_auto"]
+ and kwargs["moe_cache_mem"] is None
and (kwargs["moe_cache_rate"] is None or kwargs["moe_cache_rate"] == 0)
)
if is_offload_moe_strategy(kwargs["moe_strategy"]) and _no_cache_flag:
diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py
index a77409f0..7b37af9e 100644
--- a/tests/engine/test_cache_budget.py
+++ b/tests/engine/test_cache_budget.py
@@ -523,3 +523,55 @@ def test_uncapped_platform_stays_uncapped(monkeypatch):
if hasattr(os, "uname") and "microsoft" in os.uname().release.lower():
pytest.skip("WSL caps pinning")
assert _pin_budget_bytes(reserved=2**30) is None
+
+
+def test_moe_slot_pool_split(tmp_path):
+ import json
+
+ from freetoken.moe.pool_policy import group_layers, load_profile, make_pools, split_slots
+
+ pools = make_pools(group_layers("per-layer", [0] * 4, [1] * 4), num_experts=16, min_num_slots=8, slot_limit=None)
+ assert split_slots(41, pools) == [11, 10, 10, 10]
+ with pytest.raises(ValueError, match="need"):
+ split_slots(31, pools)
+ with pytest.raises(ValueError, match="hold"):
+ split_slots(65, pools)
+ # one pool takes the whole cache, as before pools existed
+ assert split_slots(100, make_pools(group_layers("unified", [0] * 4, [1] * 4), 16, 8, None)) == [100]
+ # unified pools one bank layout each, widest slots first
+ assert group_layers("unified", ["a", "b", "a", "b"], [1, 3, 1, 3]) == [[1, 3], [0, 2]]
+
+ # layer 0 uses two experts, layer 1 has a hot half and a cold half, layers 2 and 3 use all 16 evenly
+ counts = [[5] * 2 + [0] * 14, [3] * 8 + [1] * 8, [1] * 16, [1] * 16]
+ path = tmp_path / "profile.json"
+ path.write_text(json.dumps({"model": "toy", "expert_counts": counts}))
+ assert split_slots(48, pools, load_profile(path, 4, 16)) == [8, 8, 16, 16]
+ with pytest.raises(ValueError, match="needs"):
+ load_profile(path, 4, 32)
+
+
+
+def test_moe_pool_flags_and_decode_batch_clamp():
+ from unittest.mock import patch
+
+ from freetoken.engine.engine import _clamp_decode_batch_to_pools
+ from freetoken.server.args import parse_args
+
+ hf = SimpleNamespace(to_dict=lambda: {"architectures": ["Qwen3MoeForCausalLM"], "torch_dtype": "bfloat16"})
+ with patch("freetoken.utils.cached_load_hf_config", lambda _path: hf):
+ args, _ = parse_args([
+ "--model", "/models/anon", "--moe-cache-mem", "1.5GiB",
+ "--moe-cache-scope", "per-layer", "--moe-cache-split", "profile:/p.json",
+ ])
+ assert (args.moe_cache_mem, args.moe_cache_scope, args.moe_cache_split) == (3 << 29, "per-layer", "profile:/p.json")
+ for bad in (["--moe-cache-mem", "12"], ["--moe-cache-mem", "1GiB", "--moe-cache-size", "512"]):
+ with pytest.raises(SystemExit):
+ parse_args(["--model", "/models/anon", *bad])
+
+ # GPU decode needs batch * top_k distinct experts to fit the smallest pool below num_experts
+ config = SimpleNamespace(
+ model_config=SimpleNamespace(num_experts=16, num_experts_per_tok=4),
+ max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4],
+ )
+ _clamp_decode_batch_to_pools(config, [12, 16, 9])
+ assert (config.max_running_req, config.cuda_graph_max_bs, config.cuda_graph_bs) == (2, 2, [1, 2])
diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.py
index 0546fbcd..dbf06bcc 100644
--- a/tests/moe/test_nvfp4_backends.py
+++ b/tests/moe/test_nvfp4_backends.py
@@ -187,7 +187,7 @@ def test_marlin_prefill_matches_dequant_reference():
cache.materialize_layer(0)
cache.copy_missing()
g1, g2 = cache.alphas_for_layer(0)
- gu_p, gu_s, dn_p, dn_s = cache.bank_views(E)
+ gu_p, gu_s, dn_p, dn_s = cache.stage_views(0, 0)
out = marlin_fused_experts(
hidden, gu_p, gu_s, g1, dn_p, dn_s, g2,
topk_weights, topk_ids, "silu", False,
@@ -214,7 +214,7 @@ def test_marlin_decode_matches_dequant_reference_after_prefill_stomp():
cache.ensure_experts(layer_id, ids) # rewrites ids -> slots in place
cache.copy_missing()
g1, g2 = cache.alphas_for_slots(layer_id)
- gu_p, gu_s, dn_p, dn_s = cache.bank_views()
+ gu_p, gu_s, dn_p, dn_s = cache.pool_bank_views(layer_id)
out = marlin_fused_experts(
hidden, gu_p, gu_s, g1, dn_p, dn_s, g2,
topk_weights, ids, "silu", False,
@@ -278,7 +278,7 @@ def test_marlin_overlap_prefill_matches_dequant_reference():
assert int(cache.num_indices.item()) == 2, "stale slot mappings survived the prefetch"
cache.copy_missing()
g1, g2 = cache.alphas_for_slots(0)
- gu_p, gu_s, dn_p, dn_s = cache.bank_views()
+ gu_p, gu_s, dn_p, dn_s = cache.pool_bank_views(0)
out = marlin_fused_experts(
dec_hidden, gu_p, gu_s, g1, dn_p, dn_s, g2,
dec_weights, ids, "silu", False,
@@ -403,7 +403,7 @@ def test_triton_decode_marlin_matches_dequant_reference_after_prefill_stomp():
ref = _ref_moe(ref_sources, layer_id, hidden, topk_weights, ids)
cache.ensure_experts(layer_id, ids) # rewrites ids -> slots in place
cache.copy_missing()
- gu_p, gu_s, gu_g, dn_p, dn_s, dn_g = cache.bank_views()
+ gu_p, gu_s, gu_g, dn_p, dn_s, dn_g = cache.pool_bank_views(layer_id)
out = fused_experts_decode_nvfp4_marlin(
hidden, gu_p, gu_s, gu_g, dn_p, dn_s, dn_g, topk_weights, ids, "silu", False
)
@@ -433,7 +433,7 @@ def test_triton_decode_marlin_matches_baseline_kernel():
ids = torch.tensor([[1, 6]], dtype=torch.int32, device=device)
cache.ensure_experts(0, ids)
cache.copy_missing()
- banks = cache.bank_views()
+ banks = cache.pool_bank_views(0)
marlin = fused_experts_decode_nvfp4_marlin(hidden, *banks, topk_weights, ids, "silu", False)
base = fused_experts_decode_nvfp4_serial(hidden, *banks, topk_weights, ids, "silu", False)
torch.testing.assert_close(marlin.float(), base.float(), rtol=2e-3, atol=2e-3)
@@ -461,7 +461,7 @@ def test_b12x_decode_matches_dequant_reference():
cache.ensure_experts(0, ids)
cache.copy_missing()
g1, g2 = cache.alphas_for_slots(0)
- gu_p, gu_s, dn_p, dn_s = cache.bank_views()
+ gu_p, gu_s, dn_p, dn_s = cache.pool_bank_views(0)
out = b12x_fused_experts(
hidden, gu_p, gu_s, g1, dn_p, dn_s, g2, topk_weights, ids, "silu", False
)
diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.py
index cd5f5dc7..d3c8d9d8 100644
--- a/tests/moe/test_offload.py
+++ b/tests/moe/test_offload.py
@@ -209,7 +209,7 @@ def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(mo
assert fused_calls[0]["w2_ptr"] == fused_calls[2]["w2_ptr"]
assert fused_calls[0]["w1_ptr"] != fused_calls[1]["w1_ptr"]
assert fused_calls[0]["w2_ptr"] != fused_calls[1]["w2_ptr"]
- prefill_gate_up_buffer, prefill_down_buffer = cache.prefill_bank_buffers
+ prefill_gate_up_buffer, prefill_down_buffer = cache.stage_views(0, 0)
assert prefill_gate_up_buffer.data_ptr() == cache.bank_caches["gate_up"].data_ptr()
assert prefill_down_buffer.data_ptr() == cache.bank_caches["down"].data_ptr()
@@ -639,6 +639,136 @@ def test_offload_cache_rebuild_resizes_and_preserves_sources():
assert torch.all(cache.id_of_slot == -1)
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
+def test_slot_pools_of_two_bank_layouts_keep_their_rows_through_prefill():
+ from freetoken.moe.offload_cache import OffloadMoeCache
+
+ _init_tp()
+ torch.manual_seed(0)
+ L, E, top_k = 4, 8, 4
+ # layers 1 and 3 keep wider down rows (uint8, like Q6_K next to Q4_K); their pools come first
+ wide = (1, 3)
+ sources = {
+ "gate_up": [torch.randn(E, 1024, 256, dtype=torch.bfloat16).pin_memory() for _ in range(L)],
+ "down": [
+ torch.randint(0, 255, (E, 512, 768), dtype=torch.uint8).pin_memory() if layer in wide
+ else torch.randn(E, 512, 256, dtype=torch.bfloat16).pin_memory()
+ for layer in range(L)
+ ],
+ }
+ cache = OffloadMoeCache(
+ num_layers=L, num_experts=E, cache_size=33, device=torch.device("cuda"),
+ prefill_overlap=True, prefill_hit_d2d=True, pool_of_layer=[2, 0, 3, 1], pool_sizes=[6, 5, 12, 10],
+ )
+ cache.set_bank_sources(sources)
+ assert [v[1].dtype for v in map(cache.pool_bank_views, range(L))] == [torch.bfloat16, torch.uint8] * 2
+ # staging buffers take 2 x 8 rows of every bank: all of the wide pools, the first 8 slots of layer 0's pool
+ assert cache._pool_stable_from == [6, 5, 8, 0]
+
+ def check_resident() -> None:
+ torch.cuda.synchronize()
+ for layer in range(L):
+ for e, s in enumerate(cache.slot_for_id[layer].tolist()):
+ if s >= 0:
+ for (src, _), view in zip(cache.banks, cache.pool_bank_views(layer)):
+ assert torch.equal(view[s].cpu(), src[layer][e]), (layer, e, s)
+
+ def decode(steps: int) -> None:
+ for _ in range(steps):
+ for layer in range(L):
+ cache.ensure_experts(layer, torch.randperm(E)[:top_k].to(torch.int32).view(1, top_k).cuda())
+ cache.copy_missing()
+ check_resident()
+
+ decode(20)
+ cache.zero_inactive_route_slots = True
+ cache.begin_prefill()
+ for layer in range(L):
+ cache.prefetch_prefill_layer(layer)
+ cache.prefetch_prefill_layer(layer + 1)
+ views = cache.wait_prefill_layer(layer)
+ torch.cuda.synchronize()
+ for (src, _), view in zip(cache.banks, views):
+ assert torch.equal(view.cpu(), src[layer])
+ cache.release_prefill_layer(layer)
+ assert cache.prefill_hit_rows > 0
+ for p, stable in enumerate(cache._pool_stable_from):
+ if stable:
+ assert not any(view[0].any() for view in cache.pool_bank_views(cache.pool_of_layer.index(p)))
+ check_resident()
+ decode(20)
+
+ cache.materialize_layer(3)
+ cache.copy_missing()
+ for (src, _), view in zip(cache.banks, cache.stage_views(3, 0)):
+ assert torch.equal(view.cpu(), src[3])
+ check_resident()
+
+ cache.rebuild(32, [5, 5, 10, 12])
+ decode(20)
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
+def test_slot_pools_split_layers_of_one_bank_layout_by_quant_tag():
+ from freetoken.moe.offload_cache import OffloadMoeCache, bank_layouts
+ from freetoken.moe.pool_policy import group_layers
+
+ _init_tp()
+ L, E = 4, 8
+ # uint8 rows of one width, as GGUF Q4_K and IQ4_NL rows are
+ sources = {
+ "gate_up": [torch.randint(1, 255, (E, 64, 144), dtype=torch.uint8).pin_memory() for _ in range(L)],
+ "down": [torch.randint(1, 255, (E, 32, 144), dtype=torch.uint8).pin_memory() for _ in range(L)],
+ }
+ tags = [("ggml", "Q4_K", "Q4_K"), ("ggml", "IQ4_NL", "IQ4_NL")] * 2
+ assert group_layers("unified", bank_layouts(sources, tags), [1] * L) == [[0, 2], [1, 3]]
+
+ def make(pool_of_layer, pool_sizes):
+ return OffloadMoeCache(
+ num_layers=L, num_experts=E, cache_size=sum(pool_sizes), device=torch.device("cuda"),
+ prefill_overlap=True, pool_of_layer=pool_of_layer, pool_sizes=pool_sizes,
+ )
+
+ with pytest.raises(AssertionError, match="layouts"):
+ make([0] * L, [20]).set_bank_sources(sources, layer_tags=tags)
+ cache = make([0, 1, 0, 1], [10, 10])
+ cache.set_bank_sources(sources, layer_tags=tags)
+ assert cache.bank_caches["down"].shape == (20, 32, 144)
+ # the staging buffers cover pool 0 and slots [0, 6) of pool 1, so both slot 0 rows end up with another quant's bytes
+ cache.zero_inactive_route_slots = True
+ cache.begin_prefill()
+ for layer in range(L):
+ cache.prefetch_prefill_layer(layer)
+ cache.prefetch_prefill_layer(layer + 1)
+ cache.wait_prefill_layer(layer)
+ cache.release_prefill_layer(layer)
+ torch.cuda.synchronize()
+ for layer in (0, 1):
+ for view in cache.pool_bank_views(layer):
+ assert not view[0].any()
+
+
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
+def test_offload_cache_rebuild_frees_the_old_slots_first():
+ from freetoken.moe.offload_cache import OffloadMoeCache
+
+ _init_tp()
+ cache = OffloadMoeCache(
+ num_layers=2, num_experts=4, cache_size=32, device=torch.device("cuda"),
+ pool_of_layer=[0, 1], pool_sizes=[16, 16],
+ )
+ cache.set_bank_sources({
+ "gate_up": [torch.zeros(4, 1024, 1024, dtype=torch.bfloat16).pin_memory() for _ in range(2)],
+ "down": [torch.zeros(4, 512, 1024, dtype=torch.bfloat16).pin_memory() for _ in range(2)],
+ })
+ torch.cuda.synchronize()
+ before = torch.cuda.memory_allocated()
+ torch.cuda.reset_peak_memory_stats()
+ cache.rebuild(32, [8, 24])
+ # 96 MiB of slots: holding the old ones while allocating the new ones would double it
+ assert torch.cuda.max_memory_allocated() - before < 8 << 20
+
+
def test_offload_cache_rebuild_disables_prefill_overlap_when_too_small():
from freetoken.moe.offload_cache import OffloadMoeCache
@@ -654,7 +784,7 @@ def test_offload_cache_rebuild_disables_prefill_overlap_when_too_small():
assert cache.cache_size == 5
assert cache.prefill_overlap is False
- assert cache.prefill_bank_buffers == []
+ assert cache._stage_views[0][1] is None # no room for the second staging buffer
def test_offload_cache_rebuild_keeps_overlap_at_boundary():
@@ -690,6 +820,58 @@ def test_offload_cache_validate_rebuild_enforces_marlin_cap_and_floor():
bf16.validate_rebuild(3) # below the num_experts floor
[email protected](not torch.cuda.is_available(), reason="needs CUDA")
[email protected]("target", ["gpu", "hybrid"])
+def test_slot_pools_keep_each_layer_in_its_own_slots(target):
+ from freetoken.moe.offload_cache import OffloadMoeCache
+
+ _init_tp()
+ torch.manual_seed(0)
+ L, E, sizes = 3, 16, [5, 7, 20]
+ # hybrid routes more experts than the smallest pool holds; GPU decode must fit them
+ top_k = 2 if target == "gpu" else 8
+ sources = {
+ "gate_up": [torch.randn(E, 64, 32, dtype=torch.bfloat16).pin_memory() for _ in range(L)],
+ "down": [torch.randn(E, 32, 32, dtype=torch.bfloat16).pin_memory() for _ in range(L)],
+ }
+
+ cache = OffloadMoeCache(
+ num_layers=L, num_experts=E, cache_size=sum(sizes), device=torch.device("cuda"),
+ decode_target=target, hybrid_max_fetch=E, pool_of_layer=[0, 1, 2], pool_sizes=sizes,
+ )
+ cache.set_bank_sources(sources)
+ for _ in range(50):
+ for layer in range(L):
+ experts = torch.randperm(E)[:top_k].to(torch.int32)
+ slots = experts.cuda()
+ if target == "gpu":
+ cache.ensure_experts(layer, slots)
+ else:
+ cache.ensure_experts_hybrid(layer, slots, torch.rand(top_k, device="cuda"))
+ cache.copy_missing()
+ resident = [(e, s) for e, s in zip(experts.tolist(), slots.tolist()) if s >= 0]
+ assert len({s for _, s in resident}) == len(resident)
+ for e, s in resident:
+ assert s < sizes[layer]
+ for (src, _), view in zip(cache.banks, cache.pool_bank_views(layer)):
+ assert torch.equal(view[s].cpu(), src[layer][e])
+
+ # with several pools a whole-layer prefill stages slots [0, E) without registering them in any pool
+ cache.materialize_layer(1)
+ cache.copy_missing()
+ for (src, _), view in zip(cache.banks, cache.stage_views(1, 0)):
+ assert torch.equal(view.cpu(), src[1])
+ slot_for_id, id_of_slot = cache.slot_for_id.cpu(), cache.id_of_slot.cpu()
+ for layer in range(L):
+ for e, s in enumerate(slot_for_id[layer].tolist()):
+ if s >= 0:
+ assert cache.pool_bases[layer] + s >= E
+ assert id_of_slot[cache.pool_bases[layer] + s] == layer * E + e
+
+ cache.rebuild(40, [8, 8, 24])
+ assert [v[0].shape[0] for v in map(cache.pool_bank_views, range(L))] == [8, 8, 24]
+
+
def _make_split_cache(num_layers=2, locked=(1,), prefill_overlap=False, device="cpu"):
"""A [gate_up, down] bf16 cache with the given layers LOCKED (rest pinned)."""
from freetoken.moe.host_banks import HostResidency