FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(moe): split the offload expert cache into slot pools (#689)
Xiaoze Fanb416f85c4097
diff --git a/docs/cli.md b/docs/cli.mdindex 2166e9e8..44b0e30a 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -84,7 +84,9 @@ See [models.md](models.md#moe-strategies) for what each strategy does.| `--moe-strategy` | auto | `fused`/`offload`/`cpu`/`hybrid`; auto → offload, or hybrid with a `ft bench bw` profile; fused on unified-memory GPUs (GB10), see [models.md](models.md#unified-memory-gpus-gb10--dgx-spark). `--moe-backend` is the deprecated old spelling || `--quant-backend` | auto | Kernel per quantized layer type, `layer[.kind]=name` entries: `linear=marlin,moe=b12x` or `moe.nvfp4=triton`. A layer-level entry applies to every kind whose table lists the name || `--nvfp4-backend` | — | Deprecated: stands in for `--quant-backend moe.nvfp4=<marlin\|b12x\|triton>` (`flashinfer` means b12x); cannot be combined with `--quant-backend` |-| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) |+| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-mem` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / memory such as `12GiB` / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) |+| `--moe-cache-scope` | unified | `unified`: one LRU pool of slots per expert quant (quant type and bank layout), so all MoE layers share one pool when they share a quant format; `per-layer`: each layer has its own pool. `per-layer` does not combine with `--moe-strategy cpu` or `--moe-cpu-layers` |+| `--moe-cache-split` | equal | How the pools share the slots: `equal` (in proportion to their layer counts), or `profile:<path>`: every pool keeps `top_k` slots and the rest go to the most used experts in the profile's per-layer expert counts. With `offload`, a pool smaller than the expert count lowers `--max-running-requests` to its slots / top_k || `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts || `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor || `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, fp8_block, nvfp4, mxfp4) |diff --git a/python/freetoken/engine/cache_budget.py b/python/freetoken/engine/cache_budget.pyindex b5f07ee8..891ac2ac 100644--- a/python/freetoken/engine/cache_budget.py+++ b/python/freetoken/engine/cache_budget.py@@ -14,8 +14,8 @@ if TYPE_CHECKING:import torch-def expert_bytes_per_slot(sources: dict[str, "list[torch.Tensor]"]) -> int:- """Bytes one expert slot occupies on GPU: summed row bytes over all banks.+def layer_slot_bytes(sources: dict[str, "list[torch.Tensor]"]) -> list[int]:+ """Bytes one expert slot of each layer occupies on GPU: its row bytes summed over all banks.Each bank source is per-layer ``[num_experts, *row_shape]`` tensors and isalready TP-sharded upstream, so the per-row byte count is the per-rank slot@@ -25,7 +25,14 @@ def expert_bytes_per_slot(sources: dict[str, "list[torch.Tensor]"]) -> int:# with cache_size), so they are intentionally excluded from the per-slot growth term.# tensor[0].numel() is the per-row element count (one expert slot); see the matching# slot-byte idiom in kvcache/linear_state_pool.py and kvcache/dsv4/v4_pool.py.- return sum(t[0][0].numel() * t[0].element_size() for t in sources.values())+ num_layers = len(next(iter(sources.values())))+ return [sum(t[l][0].numel() * t[l].element_size() for t in sources.values()) for l in range(num_layers)]+++def expert_bytes_per_slot(sources: dict[str, "list[torch.Tensor]"]) -> int:+ """Bytes one expert slot occupies on GPU, rounded up to the mean over layers when their bank layouts differ."""+ per_layer = layer_slot_bytes(sources)+ return -(-sum(per_layer) // len(per_layer))def net_cache_budget_bytes(diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.pyindex b6fa8b89..ae2f1fd6 100644--- a/python/freetoken/engine/config.py+++ b/python/freetoken/engine/config.py@@ -45,6 +45,12 @@ class EngineConfig:moe_cache_size: int = 0moe_cache_rate: float | None = Nonemoe_cache_auto: bool = False+ # --moe-cache-mem: slot-cache budget in bytes, turned into a slot count once the expert banks are loaded+ moe_cache_mem: int | None = None+ # --moe-cache-scope: unified (one LRU pool per expert quant) or per-layer (one pool per layer)+ moe_cache_scope: str = "unified"+ # --moe-cache-split: how pools share the slots; equal or profile:<path> (by expert use from a profile file)+ moe_cache_split: str = "equal"kv_reserve_tokens: int = 8192 # KV floor for --moe-cache-auto; small by design (MoE-priority)moe_cache_policy: str = "lru"moe_prefill_overlap: bool = Truediff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex feb33fbb..2757b10d 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -23,7 +23,8 @@ from freetoken.models.weight import ftw_lacks_visionfrom freetoken.moe import is_offload_moe_strategyfrom freetoken.moe.expert_banks import attach_resident_banks, load_expert_banksfrom freetoken.moe.host_banks import PinFailed-from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache+from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache, bank_layouts+from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, resolve_profile, split_slotsfrom freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtypefrom .config import EngineConfig@@ -38,6 +39,25 @@ from freetoken.kvcache.linear_state_pool import (logger = init_logger(__name__)+def _clamp_decode_batch_to_pools(config: EngineConfig, pool_sizes: list[int]) -> None:+ """Clamp the decode batch so every GPU-decoded layer's routed experts fit in its pool."""+ num_experts = config.model_config.num_experts+ small = [s for s in pool_sizes if s < num_experts]+ if not small:+ return+ limit = min(small) // config.model_config.num_experts_per_tok+ if config.max_running_req > limit:+ logger.warning_rank0(+ f"the smallest MoE slot pool has {min(small)} slots; lowering max_running_req "+ f"{config.max_running_req} -> {limit} so a decode step fits in it"+ )+ object.__setattr__(config, "max_running_req", limit)+ if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs > limit:+ object.__setattr__(config, "cuda_graph_max_bs", limit)+ if config.cuda_graph_bs is not None:+ object.__setattr__(config, "cuda_graph_bs", [bs for bs in config.cuda_graph_bs if bs <= limit])++def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:"""The offload MoE cache needs at least one slot per expert per layer. A too-small size(e.g. a bare offload run with moe_cache_size unset and auto disabled) must fail loudly."""@@ -623,7 +643,7 @@ class Engine:if parts:batch.mm_embeds = torch.cat([p.to(self.dtype) for p in parts], dim=0)- def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None) -> tuple[int, int, bool]:+ def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None, max_slots=None) -> tuple[int, int, bool]:"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable@@ -647,7 +667,7 @@ class Engine:prefill_overlap=config.moe_prefill_overlap,kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),page_size=page_tokens,- max_slots=method.slot_limit() if method is not None else None,+ max_slots=max_slots if max_slots is not None or method is None else method.slot_limit(),)def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:@@ -703,6 +723,11 @@ class Engine:else HostResidency.PINNED.valuefor i in range(config.model_config.num_moe_layers)]+ # read before the expert load, so a bad profile fails in seconds+ expert_counts = resolve_profile(+ config.moe_cache_split, config.model_path,+ config.model_config.num_moe_layers, config.model_config.num_experts,+ )try:with _weight_load_context():banks = load_expert_banks(@@ -718,8 +743,27 @@ class Engine:)except PinFailed as exc:raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc+ from freetoken.engine.cache_budget import expert_bytes_per_slot, layer_slot_bytes++ self._moe_layer_slot_bytes = layer_slot_bytes(banks.sources)+ self._moe_pool_groups = group_layers(config.moe_cache_scope, bank_layouts(banks.sources, banks.layer_tags), self._moe_layer_slot_bytes)+ if len(self._moe_pool_groups) > 1 and cpu_layer_ids:+ raise ValueError(+ "several MoE slot pools (--moe-cache-scope per-layer, or experts in several quant formats) need every MoE "+ "layer to decode from the GPU slot cache; they do not combine with --moe-strategy cpu or --moe-cpu-layers"+ )+ self._moe_expert_counts = expert_counts if len(self._moe_pool_groups) > 1 else None+ if len(self._moe_pool_groups) == 1 and config.moe_cache_split != "equal":+ logger.warning_rank0(+ f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "+ f"(--moe-cache-scope per-layer gives each layer its own)"+ )+ self._moe_slot_limit = method.slot_limit() if method is not None else None+ # with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one token per step+ pools = self._moe_pools(1)+ size_cap = sum(p.max_num_slots for p in pools)if config.moe_cache_auto:- size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method)+ size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method, max_slots=size_cap)object.__setattr__(config, "moe_cache_size", size)object.__setattr__(config, "moe_prefill_overlap", overlap)if config.num_page_override is None:@@ -735,7 +779,28 @@ class Engine:f"--moe-cache-auto resolved moe_cache_size={size} "f"num_pages={pages} (prefill_overlap={overlap})")+ elif config.moe_cache_mem is not None:+ per_slot = expert_bytes_per_slot(banks.sources)+ size = min(config.moe_cache_mem // per_slot, size_cap)+ # per_slot is a mean over layers, so with pools of different slot sizes the split can exceed the budget: shrink until it fits+ while size > 0 and self._moe_pool_bytes(split_slots(size, pools, self._moe_expert_counts)) > config.moe_cache_mem:+ size -= 1+ overlap = config.moe_prefill_overlap and size >= 2 * config.model_config.num_experts+ object.__setattr__(config, "moe_cache_size", size)+ object.__setattr__(config, "moe_prefill_overlap", overlap)+ logger.info_rank0(+ f"--moe-cache-mem {mem_GB(config.moe_cache_mem)} resolved moe_cache_size={size} "+ f"({per_slot / 2**20:.2f} MiB per slot, prefill_overlap={overlap})"+ )_require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)+ pool_sizes = split_slots(config.moe_cache_size, pools, self._moe_expert_counts)+ if len(pools) > 1:+ logger.info_rank0(+ f"MoE slot pools: {config.moe_cache_scope}, {len(pools)} pools of {min(pool_sizes)}-{max(pool_sizes)} "+ f"slots ({config.moe_cache_split})"+ )+ if decode_target == "gpu":+ _clamp_decode_batch_to_pools(config, pool_sizes)layout = max_slots = Noneif method is not None:if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):@@ -760,10 +825,12 @@ class Engine:hybrid_max_fetch=config.moe_hybrid_max_fetch,layout=layout,max_slots=max_slots,+ pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),+ pool_sizes=pool_sizes,)# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer setcache.cpu_layer_ids = cpu_layer_ids- cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)+ cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=banks.layer_tags)cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)if decode_target == "hybrid":self._resolve_hybrid_fetch(config, cache)@@ -772,12 +839,23 @@ class Engine:cache.collect_stats = config.moe_collect_statslayers = attach_offload_moe_cache(self.model, cache)assert len(layers) == config.model_config.num_moe_layers+ cache.zero_inactive_route_slots = cache.decode_target == "hybrid" and not all(layer.skips_inactive_slots for layer in layers)if cache.decode_target in ("cpu", "hybrid"):self._init_cpu_moe_executor(config, cache, layers)self.ctx.moe_offload_cache = cacheself.moe_offload_cache = cachereturn cache+ def _moe_pool_bytes(self, pool_sizes: list[int]) -> int:+ return sum(s * self._moe_layer_slot_bytes[g[0]] for g, s in zip(self._moe_pool_groups, pool_sizes))++ def _moe_pools(self, batch: int) -> list:+ """The slot pools, each with room for the routed experts of ``batch`` decode tokens."""+ model_config = self.config.model_config+ num_experts = model_config.num_experts+ min_num_slots = min(num_experts, model_config.num_experts_per_tok * batch)+ return make_pools(self._moe_pool_groups, num_experts, min_num_slots, self._moe_slot_limit)+def _resolve_hybrid_fetch(self, config: EngineConfig, cache) -> None:"""Resolve --moe-hybrid-max-fetch -1 (auto) into a bandwidth-matched fetch fraction.@@ -867,19 +945,18 @@ class Engine:return min_free_memory, max_free_memory- def _target_moe_and_expert_bytes(self, moe_cache_size: int | None) -> tuple[int, int]:- from freetoken.engine.cache_budget import expert_bytes_per_slot-- target_moe = (- moe_cache_size- if moe_cache_size is not None- else (self.moe_offload_cache.cache_size if self.moe_offload_cache else 0)- )- per_expert_bytes = (- expert_bytes_per_slot(self.moe_offload_cache.bank_sources)- if self.moe_offload_cache is not None else 0- )- return target_moe, per_expert_bytes+ def _target_moe_and_expert_bytes(self, moe_cache_size: int | None, pool_sizes: list[int] | None) -> tuple[int, int]:+ cache = self.moe_offload_cache+ if cache is None:+ return moe_cache_size or 0, 0+ if moe_cache_size is None:+ target_moe = cache.cache_size+ nbytes = sum(c.numel() * c.element_size() for _, c in cache.banks)+ else:+ target_moe = moe_cache_size+ nbytes = self._moe_pool_bytes(pool_sizes)+ # pools of several bank layouts have different slot sizes; round up so target * bytes covers them+ return target_moe, -(-nbytes // target_moe)def _resize_kv_pool(self, config, num_pages: int, num_swa_pages: int | None) -> None:# IN-PLACE, identity-preserving: the CacheManager's swa_pool reference, ctx.kv_cache and@@ -938,7 +1015,10 @@ class Engine:"moe_cache_size requested but this model has no MoE offload cache")try:- self.moe_offload_cache.validate_rebuild(moe_cache_size)+ # a rebuild does not lower max_running_req again, so with GPU decode every pool must hold a full batch; hybrid caps its fetches instead+ batch = config.max_running_req if self.moe_offload_cache.decode_target == "gpu" else 1+ moe_pool_sizes = split_slots(moe_cache_size, self._moe_pools(batch), self._moe_expert_counts)+ self.moe_offload_cache.validate_rebuild(moe_cache_size, moe_pool_sizes)except ValueError as e:raise CacheRebuildRejected(str(e)) from eif num_pages is not None and num_pages <= 0:@@ -975,7 +1055,9 @@ class Engine:# must reject (recoverable) so the old caches stay intact and serving continues,# rather than freeing and then OOMing into permanent failure. The engine supplies# the memory account; the pool answers whether its target geometry fits.- target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(moe_cache_size)+ target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(+ moe_cache_size, moe_pool_sizes if moe_cache_size is not None else None+ )# Price the sibling GDN state pool at ITS target (physical slots = usable + padding# sink) and hand the bytes in -- the KV pool only budgets its own tiers.target_mamba = (@@ -1019,7 +1101,7 @@ class Engine:object.__setattr__(config, "swa_num_pages_override", num_swa_pages)if moe_cache_size is not None:assert self.moe_offload_cache is not None, "no MoE offload cache to resize"- self.moe_offload_cache.rebuild(moe_cache_size)+ self.moe_offload_cache.rebuild(moe_cache_size, moe_pool_sizes)if num_pages is not None:# sets self.num_pages (rebuilds KV + window)self._resize_kv_pool(config, num_pages, num_swa_pages)@@ -1455,6 +1537,9 @@ _DENSE_MOE_SETTINGS = {"moe_cache_size": 0,"moe_cache_rate": None,"moe_cache_auto": False,+ "moe_cache_mem": None,+ "moe_cache_scope": "unified",+ "moe_cache_split": "equal","moe_cpu_layers": None,"moe_cpu_threads": 0,"moe_hybrid_max_fetch": -1,@@ -1750,6 +1835,7 @@ def _adjust_config(config: EngineConfig):is_offload_moe_strategy(config.moe_strategy)and config.moe_cache_size <= 0and config.moe_cache_rate is None+ and config.moe_cache_mem is Noneand not getattr(config, "moe_cache_auto", False)):# args.py's "no sizing flag -> default --moe-cache-auto" only fires when the@@ -1783,11 +1869,13 @@ def _adjust_config(config: EngineConfig):# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The# sizing flags no longer redirect the backend, so ignore them here and say so -- the# geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-- # rate was already folded into moe_cache_size above, and the three are mutually exclusive.+ # rate was already folded into moe_cache_size above, and the sizing flags are mutually exclusive.if config.moe_cache_rate:inert = f"--moe-cache-rate={config.moe_cache_rate}"elif config.moe_cache_size:inert = f"--moe-cache-size={config.moe_cache_size}"+ elif config.moe_cache_mem is not None:+ inert = f"--moe-cache-mem={config.moe_cache_mem}"elif getattr(config, "moe_cache_auto", False):inert = "--moe-cache-auto"else:@@ -1799,6 +1887,7 @@ def _adjust_config(config: EngineConfig):)override("moe_cache_size", 0)override("moe_cache_rate", None)+ override("moe_cache_mem", None)override("moe_cache_auto", False)if is_moe and config.moe_strategy == "cpu":@@ -1809,6 +1898,7 @@ def _adjust_config(config: EngineConfig):num_experts = config.model_config.num_expertsif getattr(config, "moe_cache_auto", False):override("moe_cache_auto", False)+ override("moe_cache_mem", None)override("moe_cache_size", 2 * num_experts)override("moe_prefill_overlap", True)logger.info_rank0(diff --git a/python/freetoken/kvcache/cache_status.py b/python/freetoken/kvcache/cache_status.pyindex 5b531ce2..8c29dcca 100644--- a/python/freetoken/kvcache/cache_status.py+++ b/python/freetoken/kvcache/cache_status.py@@ -40,6 +40,9 @@ def compute_cache_unit_bytes(engine: "Engine") -> Dict[str, int]:banks = getattr(engine.moe_offload_cache, "bank_caches", None)if not banks:return 0+ if getattr(engine.moe_offload_cache, "_raw_banks", False):+ # pools of several bank layouts share byte segments: report the mean slot size+ return int(sum(t.numel() for t in banks.values()) // engine.moe_offload_cache.cache_size)# Each bank cache is (cache_size, *row_shape); one slot's bytes = row bytes summed over# the format's banks (== cache_budget.expert_bytes_per_slot on the source rows).return int(sum(t[0].numel() * t.element_size() for t in banks.values()))diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.pyindex 0595b1d4..79ad9e3a 100644--- a/python/freetoken/layers/moe.py+++ b/python/freetoken/layers/moe.py@@ -189,6 +189,10 @@ class OffloadMoELayer(MoELayer):)self.offload_cache: OffloadMoeCache | None = None+ @property+ def skips_inactive_slots(self) -> bool:+ return self.quant_method is not None and self.quant_method.kernel.supports_inactive_slots+def forward(self,hidden_states: torch.Tensor,@@ -289,7 +293,7 @@ class OffloadMoELayer(MoELayer):hidden_states,topk_weights,topk_ids,- views=cache.bank_views(),+ views=cache.pool_bank_views(self.layer_id),n=None,alphas=cache.alphas_for_slots(self.layer_id),is_prefill=False,@@ -317,7 +321,8 @@ class OffloadMoELayer(MoELayer):assert executor is not None, "CPU MoE executor was not initialized"# the kernel reads every route's slot: give the inactive ones slot 0, which holds a# complete payload or the bank's zero fill (never uninitialized storage), times weight 0- clamp = not (self.quant_method is not None and self.quant_method.kernel.supports_inactive_slots)+ # with several quants this holds only because the cache zeroes staged slot 0 rows (zero_inactive_route_slots)+ clamp = not self.skips_inactive_slotsgpu_slots, gpu_w, cpu_ids = cache.ensure_experts_hybrid(self.layer_id, topk_ids, topk_weights, clamp=clamp)pending = executor.decode_submit(self.layer_id, hidden_states, topk_weights, cpu_ids)@@ -333,7 +338,7 @@ class OffloadMoELayer(MoELayer):hidden_states,gpu_w,gpu_slots,- views=cache.bank_views(),+ views=cache.pool_bank_views(self.layer_id),n=None,alphas=cache.alphas_for_slots(self.layer_id),is_prefill=False,@@ -370,16 +375,18 @@ class OffloadMoELayer(MoELayer):return outcache.materialize_layer(self.layer_id)cache.copy_missing()- return self._expert_gemm(+ out = self._expert_gemm(cache,hidden_states,topk_weights,topk_ids,- views=cache.bank_views(self.num_experts),+ views=cache.stage_views(self.layer_id, 0),n=self.num_experts,alphas=cache.alphas_for_layer(self.layer_id),is_prefill=True,)+ cache.release_prefill_layer(self.layer_id)+ return outdef _wait_prefill_overlap(self, cache: OffloadMoeCache) -> tuple[torch.Tensor, ...]:"""Double-buffer choreography for this layer's overlap prefill: kick off thediff --git a/python/freetoken/models/deepseek_v4/moe.py b/python/freetoken/models/deepseek_v4/moe.pyindex 35a316f1..aadcdc04 100644--- a/python/freetoken/models/deepseek_v4/moe.py+++ b/python/freetoken/models/deepseek_v4/moe.py@@ -120,8 +120,9 @@ class DSV4OffloadMoELayer(OffloadMoELayer):cache = self.offload_cacheassert cache is not None# unpinned (LOCKED) layers must take the base materialize path: their copy_missing is the whole-layer pageable branch with position == expert id, which ensure_experts's LRU slot remap would contradict (the GEMM would gather other experts' weights)+ # one threshold for all layers (the smallest pool): with overlap prefill, the layers of a chunk must all stream or all take the slot pathif (- hidden_states.shape[0] * self.top_k >= self.num_experts+ hidden_states.shape[0] * self.top_k >= min(self.num_experts, *cache.pool_sizes)or cache.is_unpinned_layer(self.layer_id)):return super()._prefill_routed(hidden_states, topk_weights, topk_ids)@@ -134,7 +135,7 @@ class DSV4OffloadMoELayer(OffloadMoELayer):hidden_states,topk_weights,topk_ids,- views=cache.bank_views(),+ views=cache.pool_bank_views(self.layer_id),n=None,alphas=cache.alphas_for_slots(self.layer_id),is_prefill=True,diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex 9430ab2a..cd260cb2 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -413,14 +413,11 @@ def _build_gather_rig(fmt: str, wl: Workload, device: torch.device):specs = _offload_bank_specs(fmt, H, I)cache = OffloadMoeCache(num_layers=1, num_experts=E, cache_size=E, device=device, quant_format=fmt)total_bytes = 0+ sources = {}for name, (elems, dtype) in specs.items():- src = alloc_pinned_tensor(E, elems, dtype=dtype) # cudaHostAlloc (mapped) like the loaders- dst = torch.empty(E, elems, dtype=dtype, device=device)- cache.bank_sources[name] = [src]- cache.bank_caches[name] = dst- cache.banks.append(([src], dst))+ sources[name] = [alloc_pinned_tensor(E, elems, dtype=dtype)] # cudaHostAlloc (mapped) like the loaderstotal_bytes += E * elems * dtype.itemsize- cache._build_copy_plan() # enable the fused path (production default) when aligned+ cache.set_bank_sources(sources) # builds the fused copy plan (production default) when alignedcache.evict_slots[:E].copy_(torch.arange(E, dtype=torch.int32, device=device))cache.src_indices[:E].copy_(torch.randperm(E, device=device).to(torch.int32))diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.pyindex 7500174a..5521c34e 100644--- a/python/freetoken/moe/expert_banks.py+++ b/python/freetoken/moe/expert_banks.py@@ -48,6 +48,8 @@ class ExpertBanks:kind: QuantKind | None = Nonekernel: str | None = Nonelayout: dict | None = None+ # per-layer quant tag (e.g. the gate_up and down GGUF types), for layers whose banks match in dtype and row shape but not in quant; None = one quant+ layer_tags: list | None = Nonedef _dummy_fill(role: str, tensor: torch.Tensor) -> None:diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 984b7124..701524e0 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -77,7 +77,7 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {"down_bias",),# DeepSeek-V4 FP4: packed e2m1 codes + e8m0 per-32 block scales, no global scale- # (4 banks). Read by DeepSeek-V4's own DS-FP4 grouped GEMV kernels via bank_views().+ # (4 banks). Read by DeepSeek-V4's own DS-FP4 grouped GEMV kernels via pool_bank_views()."ds_fp4": ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale"),}@@ -104,6 +104,14 @@ _BANK_BYTES_PER_EXPERT = {MARLIN_MAX_CACHE_SIZE = 992+def bank_layouts(sources: dict[str, list[torch.Tensor]], layer_tags: list | None = None) -> list[tuple]:+ """Each layer's layout key: its quant tag and every bank's (dtype, row shape). A slot pool holds layers of one key."""+ num_layers = len(next(iter(sources.values())))+ tags = layer_tags if layer_tags is not None else [None] * num_layers+ assert len(tags) == num_layers, (len(tags), num_layers)+ return [(tags[l], tuple((t[l].dtype, tuple(t[l].shape[1:])) for t in sources.values())) for l in range(num_layers)]++@dataclassclass OffloadMoeCache:num_layers: int@@ -148,6 +156,9 @@ class OffloadMoeCache:# bank layout from the expert kernel (a BankSpec per role); when given it replaces the _BANK_SCHEMAS lookup and the slot cap comes from max_slotslayout: dict | None = Nonemax_slots: int | None = None+ # Each layer belongs to one pool: a contiguous slot range with its own LRU scope; pool p owns slots [sum(pool_sizes[:p]), +pool_sizes[p]). Slot ids in slot_for_id are local to the pool. None = one pool for all layers.+ pool_of_layer: list[int] | None = None+ pool_sizes: list[int] | None = Nonedef __post_init__(self) -> None:policy_ids = {"lru": 0}@@ -155,6 +166,14 @@ class OffloadMoeCache:assert self.decode_target in ("gpu", "cpu", "hybrid"), self.decode_targetif self.layout is None:assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"+ if self.pool_sizes is None:+ self.pool_of_layer = [0] * self.num_layers+ self.pool_sizes = [self.cache_size]+ self.pool_of_layer = list(self.pool_of_layer)+ self.pool_sizes = list(self.pool_sizes)+ assert len(self.pool_of_layer) == self.num_layers, (len(self.pool_of_layer), self.num_layers)+ assert sorted(set(self.pool_of_layer)) == list(range(len(self.pool_sizes))), self.pool_of_layer+ assert sum(self.pool_sizes) == self.cache_size and min(self.pool_sizes) > 0, (self.pool_sizes, self.cache_size)# Attached by the engine for decode_target == "cpu" (CpuMoeExecutor); None# for the GPU decode path.self.cpu_executor = None@@ -162,8 +181,16 @@ class OffloadMoeCache:# offload/PCIe path. Set by the engine after construction (empty = all-GPU,# all layers = the plain --moe-strategy cpu case).self.cpu_layer_ids: frozenset = frozenset()+ # Bank row layouts, set by set_bank_sources: layouts[g][k] is (dtype, row shape) of bank k (layouts of two quant tags may be equal), layer_layout[l] is layer l's g.+ # With one (dtype, row shape) per bank a bank cache is typed [cache_size, *row]; with several it is a byte segment each pool views with its own layout.+ self.layouts: list[tuple[tuple[torch.dtype, tuple[int, ...]], ...]] = []+ self.layer_layout: list[int] = [0] * self.num_layers+ self._raw_banks = False+ # Set by the engine when hybrid decode hands CPU routes to a kernel that reads slot 0 with weight 0 (no supports_inactive_slots).+ # With several quants a staging buffer can leave another quant's bytes in a pool's slot 0, which may decode to NaN, so the last prefill layer zeroes those rows.+ self.zero_inactive_route_slots = False# num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path.- self.validate_rebuild(self.cache_size)+ self.validate_rebuild(self.cache_size, self.pool_sizes)assert not self.prefill_overlap or self.cache_size >= 2 * self.num_experts, ("Prefill overlap borrows two full expert-layer buffers from the unified MoE ""cache, so cache_size must be at least 2 * num_experts "@@ -254,25 +281,28 @@ class OffloadMoeCache:)# (per-layer sources, cache) per bank, in schema order. Every piece of cache# machinery that moves bank bytes (copy_missing, the prefill double buffers,- # bank_views) iterates this list, so the slot cache is bank-count agnostic.+ # the pool and staging views) iterates this list, so the slot cache is bank-count agnostic.self.banks: list[tuple[list[torch.Tensor], torch.Tensor]] = []# Fused multi-bank copy descriptor (built by set_bank_sources/_build_copy_plan).# Source pointers are per layer (_copy_src_ptrs[layer_id] -> [num_banks] device- # tensor); dst/feat are layer-invariant.+ # tensor); dst pointers are per pool and row bytes per layout.self._copy_fused_ok = Falseself._copy_blocks_per_bank = 8self._copy_dst_ptrs: torch.Tensor | None = None+ self._copy_dst_ptrs_pool: list[torch.Tensor] | None = Noneself._copy_src_ptrs: list[torch.Tensor] | None = None- self._copy_feat_bytes: torch.Tensor | None = None+ self._copy_feat_bytes: list[torch.Tensor] | None = None# The layer whose misses ensure_experts/materialize_layer staged last; consumed# by copy_missing to pick the per-layer source (part of the same pending-copy# state as evict_slots/src_indices/num_indices).# _pending_whole_layer records WHICH staged it: the pageable branch is only sound after materialize_layerself._pending_src_layer: int | None = Noneself._pending_whole_layer = False- # Per-bank [2, num_experts, ...] double-buffer views over the slot cache's- # first 2 * num_experts slots (set up when prefill_overlap is enabled).- self.prefill_bank_buffers: list[torch.Tensor] = []+ # The two prefill staging buffers are the first 2 * _stage_bytes[k] bytes of every bank, on top of the first pools.+ # _stage_views[g][b] are buffer b's per-bank [num_experts, ...] views for layout g; _inval[b] are the global slots it overwrites.+ self._stage_bytes: list[int] = []+ self._stage_views: list[list[tuple[torch.Tensor, ...] | None]] = []+ self._inval: list[slice | torch.Tensor] = []self.prefill_copy_stream: torch.cuda.Stream | None = Noneself.prefill_begin_event: torch.cuda.Event | None = Noneself.prefill_ready_events: list[torch.cuda.Event] = []@@ -282,7 +312,7 @@ class OffloadMoeCache:self._prefill_buffer_has_release_event: list[bool] = [False, False]# hit-D2D split state: pinned begin-of-chunk snapshot of slot_for_id (the# classification input; frozen for the chunk -- no decode runs inside one,- # and buffer invalidation only clears slot < 2E entries, which classify as+ # and buffer invalidation only clears slots under the staging buffers, which classify as# miss regardless), the lazily resolved batch-memcpy entry point (False =# unavailable), and row counters for cache reports.self._prefill_slot_snapshot: torch.Tensor | None = None@@ -292,11 +322,116 @@ class OffloadMoeCache:self._batch_memcpy = Noneself.prefill_hit_rows = 0self.prefill_total_rows = 0+ self._slice_pools()++ def _pool_layout(self, pool: int) -> int:+ return self.layer_layout[self.pool_of_layer.index(pool)]++ def _bank_rows(self, k: int, layout: int, offset: int, nrows: int) -> torch.Tensor:+ """``nrows`` rows of bank ``k`` in ``layout``, starting ``offset`` bytes into the bank cache."""+ cache = self.banks[k][1]+ row = self._row_bytes[layout][k]+ if not self._raw_banks:+ first = offset // row+ return cache if first == 0 and nrows == cache.shape[0] else cache[first : first + nrows]+ dtype, shape = self.layouts[layout][k]+ return cache[offset : offset + nrows * row].view(dtype).view(nrows, *shape)++ def _stage_fits(self, buffers: int) -> bool:+ return all(c.numel() * c.element_size() >= buffers * s for (_, c), s in zip(self.banks, self._stage_bytes))++ def _slice_pools(self) -> None:+ E = self.num_experts+ bases = [0]+ for size in self.pool_sizes[:-1]:+ bases.append(bases[-1] + size)+ self.pool_bases = bases+ spans = list(zip(bases, self.pool_sizes))++ def cut(t: torch.Tensor, b: int, s: int) -> torch.Tensor:+ return t if s == t.shape[0] else t[b : b + s]++ self._pool_id_of_slot = [cut(self.id_of_slot, b, s) for b, s in spans]+ self._pool_usage = [cut(self.usage, b, s) for b, s in spans]+ # rows[k][p]: bytes of one slot of pool p in bank k; before the banks exist, one unit per slot with a stage of E+ if self.banks:+ rows = [[self._row_bytes[self._pool_layout(p)][k] for p in range(len(spans))] for k in range(len(self.banks))]+ stage = self._stage_bytes+ else:+ rows, stage = [[1] * len(spans)], [E]+ offsets = []+ for row in rows:+ offsets.append([0])+ for s, r in zip(self.pool_sizes[:-1], row):+ offsets[-1].append(offsets[-1][-1] + s * r)+ self._pool_offsets = offsets+ # local slots below this index overlap the prefill staging area in some bank+ self._pool_stable_from = [+ max(min(max(-(-(2 * stage[k] - offsets[k][p]) // rows[k][p]), 0), s) for k in range(len(rows)))+ for p, s in enumerate(self.pool_sizes)+ ]+ self._inval = []+ masks = []+ for b in range(2):+ mask = torch.zeros(self.cache_size, dtype=torch.bool)+ masks.append(mask)+ for k in range(len(rows)):+ lo, hi = b * stage[k], (b + 1) * stage[k]+ for p, (base, s) in enumerate(spans):+ first = max((lo - offsets[k][p]) // rows[k][p], 0)+ end = min(-(-(hi - offsets[k][p]) // rows[k][p]), s)+ mask[base + first : base + max(first, end)] = True+ idx = mask.nonzero().flatten()+ if idx.numel() == 0 or int(idx[-1] - idx[0]) + 1 == idx.numel():+ self._inval.append(slice(int(idx[0]), int(idx[-1]) + 1) if idx.numel() else slice(0, 0))+ else:+ self._inval.append(idx.to(self.device))+ self._stage_dirty = [False, False]+ self._stage_slot0_rows = [[], []]+ if not self.banks:+ self._pool_bank_views, self._stage_views = [], []+ return+ self._pool_bank_views = [+ tuple(self._bank_rows(k, self._pool_layout(p), offsets[k][p], s) for k in range(len(self.banks)))+ for p, s in enumerate(self.pool_sizes)+ ]+ self._stage_views = [+ [+ tuple(self._bank_rows(k, g, b * stage[k], E) for k in range(len(self.banks))) if self._stage_fits(b + 1) else None+ for b in range(2)+ ]+ for g in range(len(self.layouts))+ ]+ if len(self.layouts) > 1:+ self._stage_slot0_rows = [+ [self._bank_rows(k, self._pool_layout(p), offsets[k][p], 1) for p, base in enumerate(bases) if masks[b][base] for k in range(len(self.banks))]+ for b in range(2)+ ]++ def stage_views(self, layer_id: int, buffer_id: int) -> tuple[torch.Tensor, ...]:+ """Per-bank ``[num_experts, ...]`` views of prefill staging buffer ``buffer_id`` in ``layer_id``'s layout; row == expert id."""+ views = self._stage_views[self.layer_layout[layer_id]][buffer_id]+ assert views is not None, f"the slot cache is too small for staging buffer {buffer_id}"+ return views++ def pool_state(self, layer_id: int) -> tuple[torch.Tensor, torch.Tensor]:+ """``(id_of_slot, usage)`` views of the pool that serves ``layer_id``."""+ p = self.pool_of_layer[layer_id]+ return self._pool_id_of_slot[p], self._pool_usage[p]++ def pool_slots(self, layer_id: int) -> int:+ return self.pool_sizes[self.pool_of_layer[layer_id]]++ def pool_bank_views(self, layer_id: int) -> tuple[torch.Tensor, ...]:+ """Per-bank slot views of the pool that serves ``layer_id``, indexed by its local slot ids."""+ assert self.banks, "set_bank_sources must register the banks first"+ return self._pool_bank_views[self.pool_of_layer[layer_id]]def set_bank_sources(self,sources: dict[str, list[torch.Tensor]],layer_residency: list[str] | None = None,+ layer_tags: list | None = None,) -> None:"""Attach the host (CPU pinned) expert source banks and allocate a GPU slotcache per bank, following the format's bank schema.@@ -310,6 +445,8 @@ class OffloadMoeCache:``layer_residency`` labels each layer with a ``HostResidency`` value (default: all pinned).Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them.++ ``layer_tags`` names each layer's quant (see :func:`bank_layouts`); layers of different tags never share a pool."""from freetoken.moe.legacy_format import canonical_rolefrom freetoken.moe.host_banks import HostResidency@@ -343,43 +480,62 @@ class OffloadMoeCache:for name in self.bank_schema:per_layer = sources[name]assert len(per_layer) == self.num_layers, (name, len(per_layer))- head = per_layer[0]- if self.layout is not None:- spec = self.layout[name]- if tuple(head.shape[1:]) != tuple(spec.shape) or head.dtype != spec.dtype:- raise ValueError(- f"bank {name!r} rows are {tuple(head.shape[1:])} {head.dtype} but the expert kernel's layout "- f"wants {tuple(spec.shape)} {spec.dtype}; the banks were packed for another kernel"- )for layer_id, source in enumerate(per_layer):assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous"assert source.size(0) == self.num_experts, (name, layer_id, source.shape)- assert source.shape == head.shape and source.dtype == head.dtype, (- name, layer_id, source.shape, source.dtype,- )+ if self.layout is not None:+ spec = self.layout[name]+ if tuple(source.shape[1:]) != tuple(spec.shape) or source.dtype != spec.dtype:+ raise ValueError(+ f"bank {name!r} rows are {tuple(source.shape[1:])} {source.dtype} but the expert kernel's layout "+ f"wants {tuple(spec.shape)} {spec.dtype}; the banks were packed for another kernel"+ )self.bank_sources[name] = list(per_layer)- self.bank_caches[name] = torch.zeros(- (self.cache_size, *head.shape[1:]),- dtype=head.dtype,- device=self.device,- )- self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]+ keys = bank_layouts(sources, layer_tags)+ unique = list(dict.fromkeys(keys))+ self.layouts = [banks for _, banks in unique]+ self.layer_layout = [unique.index(key) for key in keys]+ for p in range(len(self.pool_sizes)):+ mixed = {self.layer_layout[l] for l, q in enumerate(self.pool_of_layer) if q == p}+ assert len(mixed) == 1, f"slot pool {p} holds layers of {len(mixed)} layouts; a pool needs one"+ self._raw_banks = len(set(self.layouts)) > 1+ self._row_bytes = [[math.prod(shape) * dtype.itemsize for dtype, shape in layout] for layout in self.layouts]+ if self._raw_banks:+ # every pool view and staging buffer starts on a row boundary of its own layout+ assert all(r % 16 == 0 for rows in self._row_bytes for r in rows), self._row_bytes+ self._stage_bytes = [self.num_experts * max(rows[k] for rows in self._row_bytes) for k in range(len(self.bank_schema))]+ self._alloc_banks()+ if not self._stage_fits(1):+ raise ValueError(f"MoE cache of {self.cache_size} slots cannot stage one whole layer of experts")+ if self.prefill_overlap and not self._stage_fits(2):+ logger.warning("Disabling MoE prefill overlap: the slot cache cannot hold two whole layers of experts")+ self.prefill_overlap = False+ self._slice_pools()self._build_copy_plan()if self.prefill_overlap:self._init_prefill_overlap_buffers()+ def _alloc_banks(self) -> None:+ for k, name in enumerate(self.bank_schema):+ if self._raw_banks:+ nbytes = sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(self.pool_sizes))+ self.bank_caches[name] = torch.zeros(nbytes, dtype=torch.uint8, device=self.device)+ else:+ dtype, shape = self.layouts[0][k]+ self.bank_caches[name] = torch.zeros((self.cache_size, *shape), dtype=dtype, device=self.device)+ self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]+def _build_copy_plan(self) -> None:self._build_fused_copy_plan()if self._copy_fused_ok or self.device.type != "cuda" or not self.banks:return- for name in self.bank_schema:- cache = self.bank_caches[name]- feat = math.prod(cache.shape[1:]) * cache.element_size()- if feat % 128:- raise RuntimeError(- f"MoE bank {name!r} rows are {feat} bytes (not a multiple of 128): "- f"only the fused multi-bank copy can move them, but it is disabled"- )+ for k, name in enumerate(self.bank_schema):+ for rows in self._row_bytes:+ if rows[k] % 128:+ raise RuntimeError(+ f"MoE bank {name!r} rows are {rows[k]} bytes (not a multiple of 128): "+ f"only the fused multi-bank copy can move them, but it is disabled"+ )def _build_fused_copy_plan(self) -> None:"""Precompute the fused multi-bank copy descriptor (base addrs + per-row bytes).@@ -391,24 +547,25 @@ class OffloadMoeCache:"""self._copy_fused_ok = Falseself._copy_dst_ptrs = None+ self._copy_dst_ptrs_pool = Noneself._copy_src_ptrs = Noneself._copy_feat_bytes = Noneself._copy_dst_ptrs_host: list[int] = []self._copy_src_ptrs_host: list[list[int]] = []- self._copy_feat_bytes_host: list[int] = []- self._gather_bank_ids: list[int] = []- self._gather_dst_ptrs: torch.Tensor | None = None- self._gather_feat_bytes: torch.Tensor | None = None+ self._gather_bank_ids: list[list[int]] = []+ self._gather_feat_bytes: list[torch.Tensor | None] = []+ self._gather_stage_ptrs: list[list[torch.Tensor] | None] = []+ self._gather_pool_ptrs: list[torch.Tensor | None] = []if not _FUSED_COPY or self.device.type != "cuda" or not self.banks:returnfrom freetoken.kernel.pinned import device_ptr- dst_ptrs, feats = [], []+ aligned = [self._stage_bytes] + self._row_bytes + self._pool_offsets+ if any(n % 16 for ns in aligned for n in ns) or any(c.data_ptr() % 16 for _, c in self.banks):+ return # leave fused disabled; copy_missing uses the per-bank path+ dst_ptrs = [cache.data_ptr() for _, cache in self.banks]layer_src_ptrs = [[] for _ in range(self.num_layers)]- for per_layer, cache in self.banks:- feat = math.prod(per_layer[0].shape[1:]) * per_layer[0].element_size()- if feat % 16 != 0 or cache.data_ptr() % 16 != 0:- return # leave fused disabled; copy_missing uses the per-bank path+ for per_layer, _ in self.banks:for layer_id, source in enumerate(per_layer):if layer_id in self._unpinned_layers:# unregistered layer: no device alias exists, and the row is never consumed (CPU decode; pageable prefill)@@ -422,52 +579,65 @@ class OffloadMoeCache:if src_dev % 16 != 0:returnlayer_src_ptrs[layer_id].append(src_dev)- dst_ptrs.append(cache.data_ptr())- feats.append(feat)- self._copy_dst_ptrs = torch.tensor(dst_ptrs, dtype=torch.int64, device=self.device)- self._copy_src_ptrs = [- torch.tensor(ptrs, dtype=torch.int64, device=self.device)- for ptrs in layer_src_ptrs++ def ptrs(values: list[int]) -> torch.Tensor:+ return torch.tensor(values, dtype=torch.int64, device=self.device)++ self._copy_dst_ptrs = ptrs(dst_ptrs)+ self._copy_dst_ptrs_pool = [+ self._copy_dst_ptrs if all(o[p] == 0 for o in self._pool_offsets)+ else ptrs([d + o[p] for d, o in zip(dst_ptrs, self._pool_offsets)])+ for p in range(len(self.pool_sizes))]- self._copy_feat_bytes = torch.tensor(feats, dtype=torch.int64, device=self.device)+ self._copy_src_ptrs = [ptrs(values) for values in layer_src_ptrs]+ self._copy_feat_bytes = [ptrs(rows) for rows in self._row_bytes]self._copy_dst_ptrs_host = dst_ptrsself._copy_src_ptrs_host = layer_src_ptrs- self._copy_feat_bytes_host = feats# hit-D2D gather serves only the big banks; small banks are whole-layer# H2D entries (see _SMALL_BANK_FEAT_BYTES), so their rows never need D2D.- self._gather_bank_ids = [i for i, f in enumerate(feats) if f >= _SMALL_BANK_FEAT_BYTES]- if len(self._gather_bank_ids) == len(feats):- self._gather_dst_ptrs = self._copy_dst_ptrs- self._gather_feat_bytes = self._copy_feat_bytes- elif self._gather_bank_ids:- self._gather_dst_ptrs = self._copy_dst_ptrs[self._gather_bank_ids].contiguous()- self._gather_feat_bytes = self._copy_feat_bytes[self._gather_bank_ids].contiguous()+ # The gather copies cache rows of the layer's pool into its staging buffer, row == expert id.+ self._gather_bank_ids = [[k for k, r in enumerate(rows) if r >= _SMALL_BANK_FEAT_BYTES] for rows in self._row_bytes]+ for g, ids in enumerate(self._gather_bank_ids):+ self._gather_feat_bytes.append(ptrs([self._row_bytes[g][k] for k in ids]) if ids else None)+ self._gather_stage_ptrs.append(+ [ptrs([dst_ptrs[k] + b * self._stage_bytes[k] for k in ids]) for b in range(2)] if ids else None+ )+ for p in range(len(self.pool_sizes)):+ ids = self._gather_bank_ids[self._pool_layout(p)]+ self._gather_pool_ptrs.append(ptrs([dst_ptrs[k] + self._pool_offsets[k][p] for k in ids]) if ids else None)self._copy_blocks_per_bank = default_blocks_per_bank(self.device)self._copy_fused_ok = True- def validate_rebuild(self, cache_size: int) -> None:+ def validate_rebuild(self, cache_size: int, pool_sizes: list[int] | None = None) -> None:"""Pure geometry validation of a rebuild target (no GPU side effects).Raises ``ValueError`` if ``cache_size`` is below the ``num_experts`` floor or- above the marlin slot cap. Called by :meth:`rebuild` and by the engine's+ a pool is above the expert kernel's slot cap, or the bank byte segments cannot stage one layer. Called by :meth:`rebuild` and by the engine'spre-teardown check, so an invalid target rejects with the old cache intact(no destructive free first)."""if cache_size < self.num_experts:raise ValueError(f"cache_size {cache_size} < num_experts {self.num_experts}")- if self.max_slots is not None and cache_size > self.max_slots:+ # decode kernels see one pool's slots, so the cap applies per pool+ largest = cache_size if pool_sizes is None else max(pool_sizes)+ what = f"moe_cache_size={cache_size}" if largest == cache_size else f"a MoE slot pool of {largest} slots"+ if self.max_slots is not None and largest > self.max_slots:raise ValueError(- f"moe_cache_size={cache_size} exceeds the expert kernel's slot limit of {self.max_slots}; "+ f"{what} exceeds the expert kernel's slot limit of {self.max_slots}; "f"pass --moe-cache-size {self.max_slots} or less, or let the default kernel serve the experts")- if self.layout is None and self.quant_format == "nvfp4_marlin" and cache_size > MARLIN_MAX_CACHE_SIZE:+ if self.layout is None and self.quant_format == "nvfp4_marlin" and largest > MARLIN_MAX_CACHE_SIZE:raise ValueError(- f"moe_cache_size={cache_size} exceeds the marlin backend's slot limit of "+ f"{what} exceeds the marlin backend's slot limit of "f"{MARLIN_MAX_CACHE_SIZE} (vLLM moe_align_block_size caps padded experts at ""1024); reduce moe_cache_size or force --quant-backend moe.nvfp4=triton")+ if self._raw_banks and pool_sizes is not None:+ for k, stage in enumerate(self._stage_bytes):+ if sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(pool_sizes)) < stage:+ raise ValueError(f"moe_cache_size={cache_size} cannot stage one whole layer of experts")- def rebuild(self, cache_size: int) -> None:+ def rebuild(self, cache_size: int, pool_sizes: list[int] | None = None) -> None:"""Resize the GPU slot cache + bookkeeping to ``cache_size`` IN PLACE.Keeps the CPU/pinned ``bank_sources`` and the GPU-resident alphas; never@@ -477,9 +647,10 @@ class OffloadMoeCache:``ctx.moe_offload_cache`` stay valid."""assert self.bank_sources, "set_bank_sources must run before rebuild"- self.validate_rebuild(cache_size)+ sizes = [cache_size] if pool_sizes is None else list(pool_sizes)+ assert len(sizes) == len(self.pool_sizes) and sum(sizes) == cache_size and min(sizes) > 0, (sizes, cache_size)+ self.validate_rebuild(cache_size, sizes)# 1. Tear down prefill-overlap (its buffer views alias the old bank_caches).- self.prefill_bank_buffers = []self.prefill_copy_stream = Noneself.prefill_begin_event = Noneself.prefill_ready_events = []@@ -490,22 +661,22 @@ class OffloadMoeCache:# 2. Drop old GPU tensors (free-before-alloc).self.banks = []self.bank_caches = {}+ self._pool_bank_views = []+ self._stage_views = []+ self._stage_slot0_rows = [[], []]self.cache_size = cache_size+ self.pool_sizes = sizesif self.device.type == "cuda":torch.cuda.synchronize(self.device)torch.cuda.empty_cache()# 3. Reallocate the slot cache from the retained host sources.- for name in self.bank_schema:- head = self.bank_sources[name][0]- self.bank_caches[name] = torch.zeros(- (cache_size, *head.shape[1:]), dtype=head.dtype, device=self.device- )- self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]- self._build_copy_plan() # slot caches were reallocated -> refresh fused-copy addrs+ self._alloc_banks()# 4. Reallocate cache_size-shaped bookkeeping; reset the slot map (cold start).self.slot_for_id.fill_(-1)self.id_of_slot = torch.full((cache_size,), -1, dtype=torch.int32, device=self.device)self.usage = torch.zeros((cache_size,), dtype=torch.int64, device=self.device)+ self._slice_pools()+ self._build_copy_plan() # slot caches were reallocated -> refresh fused-copy addrsplan_slots = max(self.num_experts, cache_size)self.evict_slots = torch.empty((plan_slots,), dtype=torch.int32, device=self.device)self.src_indices = torch.empty((plan_slots,), dtype=torch.int32, device=self.device)@@ -529,10 +700,10 @@ class OffloadMoeCache:self.prefill_total_rows = 0self._hit_d2d_fallback_logged = False # geometry changed; re-log if still unusable# 5. Re-evaluate prefill overlap against the new size.- if self.prefill_overlap and cache_size < 2 * self.num_experts:+ if self.prefill_overlap and not self._stage_fits(2):logger.warning(f"Disabling MoE prefill overlap on rebuild: cache_size {cache_size} "- f"< 2*num_experts {2 * self.num_experts}."+ f"cannot hold two whole layers of experts (2*num_experts {2 * self.num_experts} slots of one layout).")self.prefill_overlap = Falseif self.prefill_overlap:@@ -585,8 +756,9 @@ class OffloadMoeCache:``layer_id`` -- are ever read by the grouped GEMM."""if self.gate_up_alpha is None:return None+ id_of_slot, _ = self.pool_state(layer_id)idx = layer_id * self.num_experts + (- self.id_of_slot.clamp(min=0).long() % self.num_experts+ id_of_slot.clamp(min=0).long() % self.num_experts)return self.gate_up_alpha[idx], self.down_alpha[idx]@@ -600,25 +772,12 @@ class OffloadMoeCache:hi = lo + self.num_expertsreturn self.gate_up_alpha[lo:hi], self.down_alpha[lo:hi]- def bank_views(self, n: int | None = None) -> tuple[torch.Tensor, ...]:- """Per-bank cache views in registration order: the full ``[S]`` slot cache- (decode), or its first ``n`` slots (materialized layer)."""- assert self.banks, "set_bank_sources must register the banks first"- if n is None:- return tuple(cache for _, cache in self.banks)- return tuple(cache[:n] for _, cache in self.banks)-def _init_prefill_overlap_buffers(self) -> None:assert self.banks, "set_bank_sources must register the banks first"self._prefill_buffer_layer = [None, None]self._prefill_buffer_released = [True, True]self._prefill_buffer_has_release_event = [False, False]- # The double buffers borrow the slot cache's first 2 * num_experts slots- # (one full expert layer per buffer), one view per registered bank.- self.prefill_bank_buffers = [- cache[: 2 * self.num_experts].view(2, self.num_experts, *cache.shape[1:])- for _, cache in self.banks- ]+ # the double buffers are the staging views (_slice_pools), on top of the first pools' slotsif self.device.type == "cuda":self.prefill_copy_stream = torch.cuda.Stream(device=self.device)self.prefill_ready_events = [torch.cuda.Event() for _ in range(2)]@@ -638,14 +797,14 @@ class OffloadMoeCache:self._prefill_hit_num = torch.zeros((1,), dtype=torch.int64, device=self.device)def _invalidate_prefill_buffer(self, buffer_id: int) -> None:- slot_start = buffer_id * self.num_experts- slot_end = slot_start + self.num_experts- old_ids = self.id_of_slot[slot_start:slot_end]+ slots = self._inval[buffer_id]+ old_ids = self.id_of_slot[slots]self.slot_for_id.view(-1)[old_ids[old_ids >= 0].long()] = -1- old_ids.fill_(-1)+ self.id_of_slot[slots] = -1# usage=0 makes these slots the oldest, so the argmin(usage) victim selection in# ensure_experts evicts them first.- self.usage[slot_start:slot_end].zero_()+ self.usage[slots] = 0+ self._stage_dirty[buffer_id] = Truedef begin_prefill(self) -> None:if not self.prefill_overlap:@@ -676,7 +835,7 @@ class OffloadMoeCache:if layer_id < 0:raise ValueError(f"Invalid prefill layer id: {layer_id}")- assert self.banks and self.prefill_bank_buffers+ assert self.banks and self._stage_viewsbuffer_id = layer_id % 2if self._prefill_buffer_layer[buffer_id] == layer_id:@@ -688,8 +847,8 @@ class OffloadMoeCache:def copy() -> None:self._invalidate_prefill_buffer(buffer_id)- for (per_layer, _), buffer in zip(self.banks, self.prefill_bank_buffers):- buffer[buffer_id].copy_(per_layer[layer_id], non_blocking=True)+ for (per_layer, _), view in zip(self.banks, self.stage_views(layer_id, buffer_id)):+ view.copy_(per_layer[layer_id], non_blocking=True)if self._prefill_hit_d2d_active:self._prefetch_split(layer_id, buffer_id)@@ -720,10 +879,10 @@ class OffloadMoeCache:reason = "FREETOKEN_SKIP_FAST_INDEX_COPY is set (the hit gather would be a no-op)"elif not self._copy_fused_ok:reason = "the fused copy plan is unavailable (bank alignment or FREETOKEN_FUSED_COPY=0)"- elif self.cache_size <= 2 * self.num_experts:+ elif all(stable >= s for stable, s in zip(self._pool_stable_from, self.pool_sizes)):reason = (f"cache_size {self.cache_size} leaves no hit region "- f"(needs > {2 * self.num_experts} slots)"+ f"(every slot lies under the prefill staging buffers of {2 * self.num_experts} experts)")elif not self._resolve_batch_memcpy():reason = "cudaMemcpyBatchAsync is unavailable" # resolve logged the specifics@@ -761,7 +920,7 @@ class OffloadMoeCache:run list comes from the begin-of-chunk snapshot because the batch APItakes HOST pointer arrays. Live-vs-snapshot cannot disagree: the onlychunk-internal writer (buffer invalidation) rewrites slots already below- the 2E threshold, and slots < 2E (including -1) are misses on both sides+ the pool's staging threshold, and those slots (and -1) are misses on both sides-- the buffers own those slots, so their bytes are volatile within thechunk. Hit and miss row sets are disjoint, so the streams need noordering against each other.@@ -772,18 +931,20 @@ class OffloadMoeCache:from freetoken.moe.offload_kernels import prefill_hit_compactE = self.num_experts+ layout = self.layer_layout[layer_id]+ pool = self.pool_of_layer[layer_id]snap = self._prefill_snapshot_np[layer_id]- hit_mask = snap >= 2 * E+ hit_mask = snap >= self._pool_stable_from[pool]self.prefill_hit_rows += int(hit_mask.sum())self.prefill_total_rows += E- if self._gather_dst_ptrs is not None:- prefill_hit_compact(self, layer_id, buffer_id)+ if self._gather_feat_bytes[layout] is not None:+ prefill_hit_compact(self, layer_id)# blocks_per_bank=64 vs the PCIe-tuned default of 8: HBM D2D needs the# wider grid (~22 GB/s per 1024-thread block on H100).fast_index_copy_multi_jit(- self._gather_dst_ptrs,- self._gather_dst_ptrs,- self._gather_feat_bytes,+ self._gather_stage_ptrs[layout][buffer_id],+ self._gather_pool_ptrs[pool],+ self._gather_feat_bytes[layout],self._prefill_hit_dst,self._prefill_hit_src,self._prefill_hit_num,@@ -799,16 +960,17 @@ class OffloadMoeCache:starts = miss[run_starts]lengths = np.diff(np.concatenate((run_starts, [miss.size])))dst, src, nbytes = [], [], []- for b, feat in enumerate(self._copy_feat_bytes_host):+ for b, feat in enumerate(self._row_bytes[layout]):+ stage = self._copy_dst_ptrs_host[b] + buffer_id * self._stage_bytes[b]if feat < _SMALL_BANK_FEAT_BYTES:# Whole layer as one entry, EVEN with zero misses: it keeps every# batch entry above the driver's async floor and covers the hit# rows the gather skips for these banks.- dst.append(self._copy_dst_ptrs_host[b] + buffer_id * E * feat)+ dst.append(stage)src.append(self._copy_src_ptrs_host[layer_id][b])nbytes.append(E * feat)elif miss.size:- dst.extend(self._copy_dst_ptrs_host[b] + (buffer_id * E + starts) * feat)+ dst.extend(stage + starts * feat)src.extend(self._copy_src_ptrs_host[layer_id][b] + starts * feat)nbytes.extend(lengths * feat)if dst:@@ -826,15 +988,22 @@ class OffloadMoeCache:marlin/b12x ``(gate_up_packed, gate_up_scale, down_packed, down_scale)``;nvfp4 native adds the two global banks after each scale bank."""assert self.prefill_overlap- assert self.prefill_bank_buffers+ assert self._stage_viewsself.prefetch_prefill_layer(layer_id)buffer_id = layer_id % 2assert self._prefill_buffer_layer[buffer_id] == layer_idif self.prefill_ready_events:torch.cuda.current_stream(self.device).wait_event(self.prefill_ready_events[buffer_id])- return tuple(buffer[buffer_id] for buffer in self.prefill_bank_buffers)+ return self.stage_views(layer_id, buffer_id)def release_prefill_layer(self, layer_id: int) -> None:+ if self.zero_inactive_route_slots and layer_id == self.num_layers - 1:+ # the staged slots were invalidated and nothing registers them within the forward, so their bytes are free to clear+ for b in range(2):+ if self._stage_dirty[b]:+ for row in self._stage_slot0_rows[b]:+ row.zero_()+ self._stage_dirty[b] = Falseif not self.prefill_overlap:returnbuffer_id = layer_id % 2@@ -959,6 +1128,8 @@ class OffloadMoeCache:s, m, a, f = steps[L], missing[L], active[L], fetched[L]per_layer.append({"layer": L,+ "pool": self.pool_of_layer[L],+ "pool_slots": self.pool_slots(L),"steps": s,"active_per_step": (a / s) if s else 0.0,"missing_per_step": (m / s) if s else 0.0,@@ -1010,11 +1181,13 @@ class OffloadMoeCache:f"pageable materialize (position == expert id); ensure_experts's "f"LRU slot remap cannot be honored without a device alias")- # the only copy a non-pinned layer ever needs is the non-overlap prefill materialize, which schedules the whole layer into slots [0, num_experts) with position == expert id -- a plain synchronous pageable H2D copy+ # the only copy a non-pinned layer ever needs is the non-overlap prefill materialize, which schedules the whole layer into staging buffer 0 with position == expert id -- a plain synchronous pageable H2D copy# never CUDA-graph captured: prefill is not captured, and decode never reaches this branch (it routes to the CPU executor)- for per_layer, cache in self.banks:- cache[: self.num_experts].copy_(per_layer[layer_id])+ for (per_layer, _), view in zip(self.banks, self.stage_views(layer_id, 0)):+ view.copy_(per_layer[layer_id])return+ # materialize stages rows of staging buffer 0; ensure stages slots local to the layer's pool+ whole = self._pending_whole_layerif self._copy_fused_ok:from freetoken.kernel.fast_index_copy import fast_index_copy_multi_jit@@ -1023,9 +1196,9 @@ class OffloadMoeCache:# src_indices holds layer-local expert rows, resolved against this layer's# source pointers (layer_id is a static int per captured graph node).fast_index_copy_multi_jit(- self._copy_dst_ptrs,+ self._copy_dst_ptrs if whole else self._copy_dst_ptrs_pool[self.pool_of_layer[layer_id]],self._copy_src_ptrs[layer_id],- self._copy_feat_bytes,+ self._copy_feat_bytes[self.layer_layout[layer_id]],self.evict_slots,self.src_indices,self.num_indices,@@ -1035,7 +1208,8 @@ class OffloadMoeCache:from freetoken.kernel import fast_index_copy_jit- for per_layer, cache in self.banks:+ views = self.stage_views(layer_id, 0) if whole else self.pool_bank_views(layer_id)+ for (per_layer, _), cache in zip(self.banks, views):fast_index_copy_jit(cache,self.evict_slots,diff --git a/python/freetoken/moe/offload_kernels.py b/python/freetoken/moe/offload_kernels.pyindex ea12b6fb..1d863a2f 100644--- a/python/freetoken/moe/offload_kernels.py+++ b/python/freetoken/moe/offload_kernels.py@@ -26,11 +26,12 @@ def ensure_experts(cache, layer_id: int, expert_ids: torch.Tensor) -> None:tensor. ``out_indices`` aliases the input, preserving the in-place rewrite everydownstream GEMM depends on."""+ id_of_slot, usage = cache.pool_state(layer_id)lru_ensure(expert_ids,cache.slot_for_id.view(-1),- cache.id_of_slot,- cache.usage,+ id_of_slot,+ usage,cache.step,expert_ids,cache.src_indices,@@ -80,13 +81,13 @@ def ensure_experts_hybrid(return routes-def prefill_hit_compact(cache, layer_id: int, buffer_id: int) -> None:+def prefill_hit_compact(cache, layer_id: int) -> None:"""Compact this layer's cache-resident experts into gather indices, device-side.- hit = slot_for_id[layer_id][e] >= 2 * num_experts (the double buffer owns the+ hit = slot_for_id[layer_id][e] >= the pool's staging threshold (the double buffer owns theslots below, so those bytes are volatile within a prefill chunk and classifyas miss). Writes fixed-shape ``_prefill_hit_dst``/``_prefill_hit_src`` (buffer- row / cache slot) and the count into ``_prefill_hit_num``; one launch on the+ row / pool-local slot) and the count into ``_prefill_hit_num``; one launch on thecurrent stream, no host sync. Safe against the concurrent buffer invalidationon the copy stream: that only rewrites entries already below the threshold."""num_experts = cache.num_experts@@ -95,15 +96,23 @@ def prefill_hit_compact(cache, layer_id: int, buffer_id: int) -> None:cache._prefill_hit_dst,cache._prefill_hit_src,cache._prefill_hit_num,- buffer_id * num_experts,- 2 * num_experts,+ cache._pool_stable_from[cache.pool_of_layer[layer_id]],num_experts,BLOCK=triton.next_power_of_2(num_experts),)def materialize_layer(cache, layer_id: int) -> None:- _materialize_layer_gpu(cache, layer_id)+ if len(cache.pool_sizes) == 1:+ _materialize_layer_gpu(cache, layer_id)+ return+ # staging buffer 0 may overlap several pools, so it only holds the layer for this forward and registers nothing+ E = cache.num_experts+ cache._invalidate_prefill_buffer(0)+ rows = torch.arange(E, dtype=torch.int32, device=cache.device)+ cache.evict_slots[:E].copy_(rows)+ cache.src_indices[:E].copy_(rows)+ cache.num_indices.fill_(E)def reset_cache(cache) -> None:@@ -124,14 +133,16 @@ def _ensure_experts_hybrid_gpu(max_fetch: int,frac_q16: int,) -> None:+ id_of_slot, usage = cache.pool_state(layer_id)+ num_slots = id_of_slot.numel()block_e = triton.next_power_of_2(cache.num_experts)- block_c = triton.next_power_of_2(cache.cache_size)+ block_c = triton.next_power_of_2(num_slots)num_warps = 8 if block_c >= 2048 else 4_ensure_experts_hybrid_kernel[(1,)](expert_ids,cache.slot_for_id,- cache.id_of_slot,- cache.usage,+ id_of_slot,+ usage,cache.step,cache.active_mask,cache.evict_slots,@@ -148,7 +159,7 @@ def _ensure_experts_hybrid_gpu(int(max_fetch),int(frac_q16),cache.num_experts,- cache.cache_size,+ num_slots,BLOCK_E=block_e,BLOCK_C=block_c,BY_RECENCY=_HYBRID_FETCH_BY_RECENCY,@@ -257,7 +268,7 @@ def _materialize_layer_kernel([email protected](do_not_specialize=["layer_id", "num_active", "max_fetch", "fetch_frac_q16"])[email protected](do_not_specialize=["layer_id", "num_active", "max_fetch", "fetch_frac_q16", "cache_size", "id_of_slot_ptr", "usage_ptr"])def _ensure_experts_hybrid_kernel(expert_ids_ptr,slot_for_id_ptr,@@ -280,7 +291,7 @@ def _ensure_experts_hybrid_kernel(max_fetch,fetch_frac_q16,num_experts: tl.constexpr,- cache_size: tl.constexpr,+ cache_size,BLOCK_E: tl.constexpr,BLOCK_C: tl.constexpr,BY_RECENCY: tl.constexpr,@@ -329,10 +340,11 @@ def _ensure_experts_hybrid_kernel((lo + 1) * ((1 << 16) - fetch_frac_q16), (num_missing - lo - 1) * fetch_frac_q16)max_fetch = tl.where(cost_lo <= cost_hi, lo, lo + 1)- num_fetch = tl.minimum(num_missing, max_fetch)+ is_hit = is_active & (slot >= 0)+ # a pool smaller than num_experts may hold fewer free slots than misses+ num_fetch = tl.minimum(tl.minimum(num_missing, max_fetch), cache_size - tl.sum(is_hit.to(tl.int32)))tl.store(num_missing_full_ptr, num_missing.to(tl.int64))tl.store(num_indices_ptr, num_fetch.to(tl.int64))- is_hit = is_active & (slot >= 0)tl.store(usage_ptr + slot, step, mask=is_hit)# Fetch-selection priority: encode (recency desc, id asc) into one strictly-ordered@@ -405,14 +417,13 @@ def _ensure_experts_hybrid_kernel(tl.store(expert_recency_ptr + base + off_e, step_vec, mask=is_active & e_mask)[email protected](do_not_specialize=["buffer_base"])[email protected](do_not_specialize=["threshold"])def _prefill_hit_compact_kernel(slot_ptr, # [num_experts] int32: this layer's slot_for_id row- dst_ptr, # [num_experts] int32 out: buffer rows, compacted- src_ptr, # [num_experts] int32 out: cache slots, compacted+ dst_ptr, # [num_experts] int32 out: buffer rows (expert ids), compacted+ src_ptr, # [num_experts] int32 out: pool-local slots, compactednum_ptr, # [1] int64 out: hit count- buffer_base, # buffer_id * num_experts- threshold, # 2 * num_experts+ threshold, # first pool-local slot outside the staging buffersnum_experts,BLOCK: tl.constexpr,):@@ -421,6 +432,6 @@ def _prefill_hit_compact_kernel(slots = tl.load(slot_ptr + offs, mask=lane, other=-1)is_hit = lane & (slots >= threshold)pos = tl.cumsum(is_hit.to(tl.int32)) - 1- tl.store(dst_ptr + pos, (buffer_base + offs).to(tl.int32), mask=is_hit)+ tl.store(dst_ptr + pos, offs.to(tl.int32), mask=is_hit)tl.store(src_ptr + pos, slots, mask=is_hit)tl.store(num_ptr, tl.sum(is_hit.to(tl.int64)))diff --git a/python/freetoken/moe/pool_policy.py b/python/freetoken/moe/pool_policy.pynew file mode 100644index 00000000..990c18ad--- /dev/null+++ b/python/freetoken/moe/pool_policy.py@@ -0,0 +1,149 @@+"""Slot pool policy for the offload MoE cache: which layers share a pool, and how many slots each pool gets.++Pure functions. The engine supplies the constraints and hands the result to ``OffloadMoeCache``:++ groups = group_layers(scope, layouts, slot_bytes) # --moe-cache-scope, once per load+ pools = make_pools(groups, num_experts, min_num_slots, slot_limit) # the minimum follows the decode batch+ sizes = split_slots(total, pools, counts) # --moe-cache-split, counts from resolve_profile+ OffloadMoeCache(pool_of_layer=pool_of_layer(pools, num_layers), pool_sizes=sizes, ...)+"""++from __future__ import annotations++import json+from dataclasses import dataclass+from pathlib import Path++import torch++from freetoken.utils import init_logger++logger = init_logger(__name__)++SCOPES = ("unified", "per-layer")+++@dataclass(frozen=True)+class PoolSpec:+ layers: tuple[int, ...]+ min_num_slots: int+ max_num_slots: int+++def group_layers(scope: str, layouts: list, slot_bytes: list[int]) -> list[list[int]]:+ """The layers of each pool, in placement order.++ ``unified``: one pool per layout key (see ``bank_layouts``); ``per-layer``: one pool per layer.+ Pools with wider slots come first: the prefill staging buffers sit at the start of every bank, so they then cover fewer slots.+ """+ if scope == "unified":+ groups: dict = {}+ for layer, layout in enumerate(layouts):+ groups.setdefault(layout, []).append(layer)+ pools = list(groups.values())+ elif scope == "per-layer":+ pools = [[layer] for layer in range(len(layouts))]+ else:+ raise ValueError(f"unknown --moe-cache-scope {scope!r}; want one of {SCOPES}")+ return sorted(pools, key=lambda layers: (-slot_bytes[layers[0]], layers[0]))+++def make_pools(groups: list[list[int]], num_experts: int, min_num_slots: int, slot_limit: int | None) -> list[PoolSpec]:+ """Each group's slot range: at least ``min_num_slots``, at most one slot per expert of its layers and the kernel's ``slot_limit``."""+ pools = []+ for layers in groups:+ max_num_slots = len(layers) * num_experts+ if slot_limit is not None:+ max_num_slots = min(max_num_slots, slot_limit)+ pools.append(PoolSpec(tuple(layers), min(min_num_slots, max_num_slots), max_num_slots))+ return pools+++def pool_of_layer(pools: list[PoolSpec], num_layers: int) -> list[int]:+ """Each layer's pool index, the form ``OffloadMoeCache`` takes."""+ owner = [-1] * num_layers+ for p, pool in enumerate(pools):+ for layer in pool.layers:+ owner[layer] = p+ assert min(owner) >= 0, owner+ return owner+++def split_slots(total: int, pools: list[PoolSpec], counts: torch.Tensor | None = None) -> list[int]:+ """Slots per pool, each in [min_num_slots, max_num_slots], summing to ``total``.++ With ``counts`` (per-layer expert use from ``resolve_profile``) the slots follow expert use, else the pools' layer counts.+ """+ if len(pools) == 1:+ return [total]+ _check_budget(total, pools)+ return _split_equal(total, pools) if counts is None else _split_hot(total, pools, counts)+++def _check_budget(total: int, pools: list[PoolSpec]) -> None:+ need = sum(p.min_num_slots for p in pools)+ room = sum(p.max_num_slots for p in pools)+ if total < need:+ raise ValueError(f"MoE cache of {total} slots is below what the pools need ({need} slots)")+ if total > room:+ raise ValueError(f"MoE cache of {total} slots exceeds what the pools can hold ({room} slots)")+++def _split_equal(total: int, pools: list[PoolSpec]) -> list[int]:+ sizes = [p.min_num_slots for p in pools]+ weights = [len(p.layers) for p in pools]+ left = total - sum(sizes)+ while left > 0:+ open_ = [i for i, p in enumerate(pools) if sizes[i] < p.max_num_slots]+ # give the next slot to the pool with the fewest slots per layer+ i = min(open_, key=lambda j: (sizes[j] / weights[j], j))+ sizes[i] += 1+ left -= 1+ return sizes+++def _split_hot(total: int, pools: list[PoolSpec], counts: torch.Tensor) -> list[int]:+ sizes = [p.min_num_slots for p in pools]+ # a pool's minimum covers its hottest experts; the rest, up to its maximum, competes across pools+ cand = []+ for p, pool in enumerate(pools):+ hot = counts[list(pool.layers)].flatten().sort(descending=True).values+ cand.extend((float(c), p) for c in hot[pool.min_num_slots : pool.max_num_slots])+ cand.sort(key=lambda c: -c[0])+ for _, p in cand[: total - sum(sizes)]:+ sizes[p] += 1+ return sizes+++def resolve_profile(spec: str, model_path: str, num_layers: int, num_experts: int) -> torch.Tensor | None:+ """Expert use for ``--moe-cache-split``: None for ``equal``, else the profile at ``profile:<path>``."""+ if spec == "equal":+ return None+ path = Path(spec.removeprefix("profile:"))+ logger.info_rank0(f"MoE pool profile: {path}")+ return load_profile(path, num_layers, num_experts, model_path)+++def load_profile(path: str | Path, num_layers: int, num_experts: int, model_path: str = "") -> torch.Tensor:+ """Per-layer expert use ``[num_layers, num_experts]`` from a profile, each row scaled to sum to 1."""+ data = json.loads(Path(path).read_text())+ model = str(data.get("model", ""))+ if model and model_path and not any(n.lower().startswith(model.lower()) for n in _model_names(model_path)):+ logger.warning_rank0(f"MoE pool profile {path} was taken on {model!r}, not on {model_path}")+ counts = torch.tensor(data["expert_counts"], dtype=torch.float64)+ if tuple(counts.shape) != (num_layers, num_experts):+ raise ValueError(+ f"profile {path} has {tuple(counts.shape)} expert counts; this model needs ({num_layers}, {num_experts})"+ )+ # every layer decodes the same tokens, so equal row sums keep the layers comparable+ return counts / counts.sum(dim=1, keepdim=True).clamp(min=1e-12)+++def _model_names(model_path: str) -> list[str]:+ # every path component, so an HF cache snapshot (models--Org--Name/snapshots/<hash>) still finds its name+ names = []+ for part in reversed(Path(model_path).parts):+ name = part.split("--")[-1].removesuffix(".gguf")+ if name:+ names.append(name)+ return namesdiff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 7c0ed9b0..be244a11 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -134,6 +134,7 @@ def parse_args(from freetoken.attention import validate_attn_backendfrom freetoken.kvcache import SUPPORTED_CACHE_MANAGERfrom freetoken.moe import MOE_STRATEGIES+ from freetoken.moe.pool_policy import SCOPESdef _parse_quant_backend(value: str) -> str:from freetoken.layers.quantization import QuantBackend@@ -153,6 +154,27 @@ def parse_args(raise argparse.ArgumentTypeError("must be in [0, 1]")return rate+ def _parse_mem_size(value: str) -> int:+ units = {+ "b": 1, "k": 1 << 10, "kb": 1 << 10, "kib": 1 << 10, "m": 1 << 20, "mb": 1 << 20, "mib": 1 << 20,+ "g": 1 << 30, "gb": 1 << 30, "gib": 1 << 30,+ }+ text = value.strip().lower()+ number = text.rstrip("abgikm")+ unit = text[len(number):]+ try:+ size = int(float(number) * units[unit])+ except (KeyError, ValueError, OverflowError) as exc:+ raise argparse.ArgumentTypeError("must be a size with a unit, e.g. 12GiB or 800MiB") from exc+ if size <= 0:+ raise argparse.ArgumentTypeError("must be positive")+ return size++ def _parse_cache_split(value: str) -> str:+ if value == "equal" or (value.startswith("profile:") and len(value) > len("profile:")):+ return value+ raise argparse.ArgumentTypeError("must be equal or profile:<path>")+def _positive_int(value: str) -> int:try:n = int(value)@@ -705,6 +727,12 @@ def parse_args(default=ServerArgs.moe_cache_rate,help="The fraction of all MoE experts to keep in GPU cache.",)+ moe_cache_group.add_argument(+ "--moe-cache-mem",+ type=_parse_mem_size,+ default=ServerArgs.moe_cache_mem,+ help="GPU memory for the MoE expert slot cache, e.g. 12GiB or 800MiB (G, GB and GiB all mean GiB); the slot count is this over the bytes per slot.",+ )moe_cache_group.add_argument("--moe-cache-auto",action="store_true",@@ -722,6 +750,27 @@ def parse_args(help="KV-cache token floor reserved before --moe-cache-auto fills experts.",)+ parser.add_argument(+ "--moe-cache-scope",+ choices=SCOPES,+ default=ServerArgs.moe_cache_scope,+ help=(+ "How MoE layers share the GPU expert slot cache: unified = one LRU pool per expert quant "+ "(all layers when they share one quant format); per-layer = each layer has its own pool. "+ "Pools are sized by --moe-cache-split."+ ),+ )+ parser.add_argument(+ "--moe-cache-split",+ type=_parse_cache_split,+ default=ServerArgs.moe_cache_split,+ help=(+ "How pools share the slots: equal (by layer count), or profile:<path> = by expert use from a "+ "profile file. With --moe-strategy offload, a pool smaller than the expert count limits the "+ "decode batch to its slots / top_k."+ ),+ )+parser.add_argument("--moe-cache-policy",default=ServerArgs.moe_cache_policy,@@ -892,6 +941,7 @@ def parse_args(_no_cache_flag = (kwargs["moe_cache_size"] == 0and not kwargs["moe_cache_auto"]+ and kwargs["moe_cache_mem"] is Noneand (kwargs["moe_cache_rate"] is None or kwargs["moe_cache_rate"] == 0))if is_offload_moe_strategy(kwargs["moe_strategy"]) and _no_cache_flag:diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.pyindex a77409f0..7b37af9e 100644--- a/tests/engine/test_cache_budget.py+++ b/tests/engine/test_cache_budget.py@@ -523,3 +523,55 @@ def test_uncapped_platform_stays_uncapped(monkeypatch):if hasattr(os, "uname") and "microsoft" in os.uname().release.lower():pytest.skip("WSL caps pinning")assert _pin_budget_bytes(reserved=2**30) is None+++def test_moe_slot_pool_split(tmp_path):+ import json++ from freetoken.moe.pool_policy import group_layers, load_profile, make_pools, split_slots++ pools = make_pools(group_layers("per-layer", [0] * 4, [1] * 4), num_experts=16, min_num_slots=8, slot_limit=None)+ assert split_slots(41, pools) == [11, 10, 10, 10]+ with pytest.raises(ValueError, match="need"):+ split_slots(31, pools)+ with pytest.raises(ValueError, match="hold"):+ split_slots(65, pools)+ # one pool takes the whole cache, as before pools existed+ assert split_slots(100, make_pools(group_layers("unified", [0] * 4, [1] * 4), 16, 8, None)) == [100]+ # unified pools one bank layout each, widest slots first+ assert group_layers("unified", ["a", "b", "a", "b"], [1, 3, 1, 3]) == [[1, 3], [0, 2]]++ # layer 0 uses two experts, layer 1 has a hot half and a cold half, layers 2 and 3 use all 16 evenly+ counts = [[5] * 2 + [0] * 14, [3] * 8 + [1] * 8, [1] * 16, [1] * 16]+ path = tmp_path / "profile.json"+ path.write_text(json.dumps({"model": "toy", "expert_counts": counts}))+ assert split_slots(48, pools, load_profile(path, 4, 16)) == [8, 8, 16, 16]+ with pytest.raises(ValueError, match="needs"):+ load_profile(path, 4, 32)++++def test_moe_pool_flags_and_decode_batch_clamp():+ from unittest.mock import patch++ from freetoken.engine.engine import _clamp_decode_batch_to_pools+ from freetoken.server.args import parse_args++ hf = SimpleNamespace(to_dict=lambda: {"architectures": ["Qwen3MoeForCausalLM"], "torch_dtype": "bfloat16"})+ with patch("freetoken.utils.cached_load_hf_config", lambda _path: hf):+ args, _ = parse_args([+ "--model", "/models/anon", "--moe-cache-mem", "1.5GiB",+ "--moe-cache-scope", "per-layer", "--moe-cache-split", "profile:/p.json",+ ])+ assert (args.moe_cache_mem, args.moe_cache_scope, args.moe_cache_split) == (3 << 29, "per-layer", "profile:/p.json")+ for bad in (["--moe-cache-mem", "12"], ["--moe-cache-mem", "1GiB", "--moe-cache-size", "512"]):+ with pytest.raises(SystemExit):+ parse_args(["--model", "/models/anon", *bad])++ # GPU decode needs batch * top_k distinct experts to fit the smallest pool below num_experts+ config = SimpleNamespace(+ model_config=SimpleNamespace(num_experts=16, num_experts_per_tok=4),+ max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4],+ )+ _clamp_decode_batch_to_pools(config, [12, 16, 9])+ assert (config.max_running_req, config.cuda_graph_max_bs, config.cuda_graph_bs) == (2, 2, [1, 2])diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.pyindex 0546fbcd..dbf06bcc 100644--- a/tests/moe/test_nvfp4_backends.py+++ b/tests/moe/test_nvfp4_backends.py@@ -187,7 +187,7 @@ def test_marlin_prefill_matches_dequant_reference():cache.materialize_layer(0)cache.copy_missing()g1, g2 = cache.alphas_for_layer(0)- gu_p, gu_s, dn_p, dn_s = cache.bank_views(E)+ gu_p, gu_s, dn_p, dn_s = cache.stage_views(0, 0)out = marlin_fused_experts(hidden, gu_p, gu_s, g1, dn_p, dn_s, g2,topk_weights, topk_ids, "silu", False,@@ -214,7 +214,7 @@ def test_marlin_decode_matches_dequant_reference_after_prefill_stomp():cache.ensure_experts(layer_id, ids) # rewrites ids -> slots in placecache.copy_missing()g1, g2 = cache.alphas_for_slots(layer_id)- gu_p, gu_s, dn_p, dn_s = cache.bank_views()+ gu_p, gu_s, dn_p, dn_s = cache.pool_bank_views(layer_id)out = marlin_fused_experts(hidden, gu_p, gu_s, g1, dn_p, dn_s, g2,topk_weights, ids, "silu", False,@@ -278,7 +278,7 @@ def test_marlin_overlap_prefill_matches_dequant_reference():assert int(cache.num_indices.item()) == 2, "stale slot mappings survived the prefetch"cache.copy_missing()g1, g2 = cache.alphas_for_slots(0)- gu_p, gu_s, dn_p, dn_s = cache.bank_views()+ gu_p, gu_s, dn_p, dn_s = cache.pool_bank_views(0)out = marlin_fused_experts(dec_hidden, gu_p, gu_s, g1, dn_p, dn_s, g2,dec_weights, ids, "silu", False,@@ -403,7 +403,7 @@ def test_triton_decode_marlin_matches_dequant_reference_after_prefill_stomp():ref = _ref_moe(ref_sources, layer_id, hidden, topk_weights, ids)cache.ensure_experts(layer_id, ids) # rewrites ids -> slots in placecache.copy_missing()- gu_p, gu_s, gu_g, dn_p, dn_s, dn_g = cache.bank_views()+ gu_p, gu_s, gu_g, dn_p, dn_s, dn_g = cache.pool_bank_views(layer_id)out = fused_experts_decode_nvfp4_marlin(hidden, gu_p, gu_s, gu_g, dn_p, dn_s, dn_g, topk_weights, ids, "silu", False)@@ -433,7 +433,7 @@ def test_triton_decode_marlin_matches_baseline_kernel():ids = torch.tensor([[1, 6]], dtype=torch.int32, device=device)cache.ensure_experts(0, ids)cache.copy_missing()- banks = cache.bank_views()+ banks = cache.pool_bank_views(0)marlin = fused_experts_decode_nvfp4_marlin(hidden, *banks, topk_weights, ids, "silu", False)base = fused_experts_decode_nvfp4_serial(hidden, *banks, topk_weights, ids, "silu", False)torch.testing.assert_close(marlin.float(), base.float(), rtol=2e-3, atol=2e-3)@@ -461,7 +461,7 @@ def test_b12x_decode_matches_dequant_reference():cache.ensure_experts(0, ids)cache.copy_missing()g1, g2 = cache.alphas_for_slots(0)- gu_p, gu_s, dn_p, dn_s = cache.bank_views()+ gu_p, gu_s, dn_p, dn_s = cache.pool_bank_views(0)out = b12x_fused_experts(hidden, gu_p, gu_s, g1, dn_p, dn_s, g2, topk_weights, ids, "silu", False)diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex cd5f5dc7..d3c8d9d8 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -209,7 +209,7 @@ def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(moassert fused_calls[0]["w2_ptr"] == fused_calls[2]["w2_ptr"]assert fused_calls[0]["w1_ptr"] != fused_calls[1]["w1_ptr"]assert fused_calls[0]["w2_ptr"] != fused_calls[1]["w2_ptr"]- prefill_gate_up_buffer, prefill_down_buffer = cache.prefill_bank_buffers+ prefill_gate_up_buffer, prefill_down_buffer = cache.stage_views(0, 0)assert prefill_gate_up_buffer.data_ptr() == cache.bank_caches["gate_up"].data_ptr()assert prefill_down_buffer.data_ptr() == cache.bank_caches["down"].data_ptr()@@ -639,6 +639,136 @@ def test_offload_cache_rebuild_resizes_and_preserves_sources():assert torch.all(cache.id_of_slot == -1)[email protected](not torch.cuda.is_available(), reason="needs CUDA")+def test_slot_pools_of_two_bank_layouts_keep_their_rows_through_prefill():+ from freetoken.moe.offload_cache import OffloadMoeCache++ _init_tp()+ torch.manual_seed(0)+ L, E, top_k = 4, 8, 4+ # layers 1 and 3 keep wider down rows (uint8, like Q6_K next to Q4_K); their pools come first+ wide = (1, 3)+ sources = {+ "gate_up": [torch.randn(E, 1024, 256, dtype=torch.bfloat16).pin_memory() for _ in range(L)],+ "down": [+ torch.randint(0, 255, (E, 512, 768), dtype=torch.uint8).pin_memory() if layer in wide+ else torch.randn(E, 512, 256, dtype=torch.bfloat16).pin_memory()+ for layer in range(L)+ ],+ }+ cache = OffloadMoeCache(+ num_layers=L, num_experts=E, cache_size=33, device=torch.device("cuda"),+ prefill_overlap=True, prefill_hit_d2d=True, pool_of_layer=[2, 0, 3, 1], pool_sizes=[6, 5, 12, 10],+ )+ cache.set_bank_sources(sources)+ assert [v[1].dtype for v in map(cache.pool_bank_views, range(L))] == [torch.bfloat16, torch.uint8] * 2+ # staging buffers take 2 x 8 rows of every bank: all of the wide pools, the first 8 slots of layer 0's pool+ assert cache._pool_stable_from == [6, 5, 8, 0]++ def check_resident() -> None:+ torch.cuda.synchronize()+ for layer in range(L):+ for e, s in enumerate(cache.slot_for_id[layer].tolist()):+ if s >= 0:+ for (src, _), view in zip(cache.banks, cache.pool_bank_views(layer)):+ assert torch.equal(view[s].cpu(), src[layer][e]), (layer, e, s)++ def decode(steps: int) -> None:+ for _ in range(steps):+ for layer in range(L):+ cache.ensure_experts(layer, torch.randperm(E)[:top_k].to(torch.int32).view(1, top_k).cuda())+ cache.copy_missing()+ check_resident()++ decode(20)+ cache.zero_inactive_route_slots = True+ cache.begin_prefill()+ for layer in range(L):+ cache.prefetch_prefill_layer(layer)+ cache.prefetch_prefill_layer(layer + 1)+ views = cache.wait_prefill_layer(layer)+ torch.cuda.synchronize()+ for (src, _), view in zip(cache.banks, views):+ assert torch.equal(view.cpu(), src[layer])+ cache.release_prefill_layer(layer)+ assert cache.prefill_hit_rows > 0+ for p, stable in enumerate(cache._pool_stable_from):+ if stable:+ assert not any(view[0].any() for view in cache.pool_bank_views(cache.pool_of_layer.index(p)))+ check_resident()+ decode(20)++ cache.materialize_layer(3)+ cache.copy_missing()+ for (src, _), view in zip(cache.banks, cache.stage_views(3, 0)):+ assert torch.equal(view.cpu(), src[3])+ check_resident()++ cache.rebuild(32, [5, 5, 10, 12])+ decode(20)++[email protected](not torch.cuda.is_available(), reason="needs CUDA")+def test_slot_pools_split_layers_of_one_bank_layout_by_quant_tag():+ from freetoken.moe.offload_cache import OffloadMoeCache, bank_layouts+ from freetoken.moe.pool_policy import group_layers++ _init_tp()+ L, E = 4, 8+ # uint8 rows of one width, as GGUF Q4_K and IQ4_NL rows are+ sources = {+ "gate_up": [torch.randint(1, 255, (E, 64, 144), dtype=torch.uint8).pin_memory() for _ in range(L)],+ "down": [torch.randint(1, 255, (E, 32, 144), dtype=torch.uint8).pin_memory() for _ in range(L)],+ }+ tags = [("ggml", "Q4_K", "Q4_K"), ("ggml", "IQ4_NL", "IQ4_NL")] * 2+ assert group_layers("unified", bank_layouts(sources, tags), [1] * L) == [[0, 2], [1, 3]]++ def make(pool_of_layer, pool_sizes):+ return OffloadMoeCache(+ num_layers=L, num_experts=E, cache_size=sum(pool_sizes), device=torch.device("cuda"),+ prefill_overlap=True, pool_of_layer=pool_of_layer, pool_sizes=pool_sizes,+ )++ with pytest.raises(AssertionError, match="layouts"):+ make([0] * L, [20]).set_bank_sources(sources, layer_tags=tags)+ cache = make([0, 1, 0, 1], [10, 10])+ cache.set_bank_sources(sources, layer_tags=tags)+ assert cache.bank_caches["down"].shape == (20, 32, 144)+ # the staging buffers cover pool 0 and slots [0, 6) of pool 1, so both slot 0 rows end up with another quant's bytes+ cache.zero_inactive_route_slots = True+ cache.begin_prefill()+ for layer in range(L):+ cache.prefetch_prefill_layer(layer)+ cache.prefetch_prefill_layer(layer + 1)+ cache.wait_prefill_layer(layer)+ cache.release_prefill_layer(layer)+ torch.cuda.synchronize()+ for layer in (0, 1):+ for view in cache.pool_bank_views(layer):+ assert not view[0].any()++[email protected](not torch.cuda.is_available(), reason="needs CUDA")+def test_offload_cache_rebuild_frees_the_old_slots_first():+ from freetoken.moe.offload_cache import OffloadMoeCache++ _init_tp()+ cache = OffloadMoeCache(+ num_layers=2, num_experts=4, cache_size=32, device=torch.device("cuda"),+ pool_of_layer=[0, 1], pool_sizes=[16, 16],+ )+ cache.set_bank_sources({+ "gate_up": [torch.zeros(4, 1024, 1024, dtype=torch.bfloat16).pin_memory() for _ in range(2)],+ "down": [torch.zeros(4, 512, 1024, dtype=torch.bfloat16).pin_memory() for _ in range(2)],+ })+ torch.cuda.synchronize()+ before = torch.cuda.memory_allocated()+ torch.cuda.reset_peak_memory_stats()+ cache.rebuild(32, [8, 24])+ # 96 MiB of slots: holding the old ones while allocating the new ones would double it+ assert torch.cuda.max_memory_allocated() - before < 8 << 20++def test_offload_cache_rebuild_disables_prefill_overlap_when_too_small():from freetoken.moe.offload_cache import OffloadMoeCache@@ -654,7 +784,7 @@ def test_offload_cache_rebuild_disables_prefill_overlap_when_too_small():assert cache.cache_size == 5assert cache.prefill_overlap is False- assert cache.prefill_bank_buffers == []+ assert cache._stage_views[0][1] is None # no room for the second staging bufferdef test_offload_cache_rebuild_keeps_overlap_at_boundary():@@ -690,6 +820,58 @@ def test_offload_cache_validate_rebuild_enforces_marlin_cap_and_floor():bf16.validate_rebuild(3) # below the num_experts floor[email protected](not torch.cuda.is_available(), reason="needs CUDA")[email protected]("target", ["gpu", "hybrid"])+def test_slot_pools_keep_each_layer_in_its_own_slots(target):+ from freetoken.moe.offload_cache import OffloadMoeCache++ _init_tp()+ torch.manual_seed(0)+ L, E, sizes = 3, 16, [5, 7, 20]+ # hybrid routes more experts than the smallest pool holds; GPU decode must fit them+ top_k = 2 if target == "gpu" else 8+ sources = {+ "gate_up": [torch.randn(E, 64, 32, dtype=torch.bfloat16).pin_memory() for _ in range(L)],+ "down": [torch.randn(E, 32, 32, dtype=torch.bfloat16).pin_memory() for _ in range(L)],+ }++ cache = OffloadMoeCache(+ num_layers=L, num_experts=E, cache_size=sum(sizes), device=torch.device("cuda"),+ decode_target=target, hybrid_max_fetch=E, pool_of_layer=[0, 1, 2], pool_sizes=sizes,+ )+ cache.set_bank_sources(sources)+ for _ in range(50):+ for layer in range(L):+ experts = torch.randperm(E)[:top_k].to(torch.int32)+ slots = experts.cuda()+ if target == "gpu":+ cache.ensure_experts(layer, slots)+ else:+ cache.ensure_experts_hybrid(layer, slots, torch.rand(top_k, device="cuda"))+ cache.copy_missing()+ resident = [(e, s) for e, s in zip(experts.tolist(), slots.tolist()) if s >= 0]+ assert len({s for _, s in resident}) == len(resident)+ for e, s in resident:+ assert s < sizes[layer]+ for (src, _), view in zip(cache.banks, cache.pool_bank_views(layer)):+ assert torch.equal(view[s].cpu(), src[layer][e])++ # with several pools a whole-layer prefill stages slots [0, E) without registering them in any pool+ cache.materialize_layer(1)+ cache.copy_missing()+ for (src, _), view in zip(cache.banks, cache.stage_views(1, 0)):+ assert torch.equal(view.cpu(), src[1])+ slot_for_id, id_of_slot = cache.slot_for_id.cpu(), cache.id_of_slot.cpu()+ for layer in range(L):+ for e, s in enumerate(slot_for_id[layer].tolist()):+ if s >= 0:+ assert cache.pool_bases[layer] + s >= E+ assert id_of_slot[cache.pool_bases[layer] + s] == layer * E + e++ cache.rebuild(40, [8, 8, 24])+ assert [v[0].shape[0] for v in map(cache.pool_bank_views, range(L))] == [8, 8, 24]++def _make_split_cache(num_layers=2, locked=(1,), prefill_overlap=False, device="cpu"):"""A [gate_up, down] bf16 cache with the given layers LOCKED (rest pinned)."""from freetoken.moe.host_banks import HostResidency