FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(moe): scan the hybrid eviction pool in chunks and add fetch rounding modes (#778)
Xiaoze Fan161ffa5a61bc
diff --git a/python/freetoken/moe/offload_kernels.py b/python/freetoken/moe/offload_kernels.pyindex 5b28c1be..e1fcbcda 100644--- a/python/freetoken/moe/offload_kernels.py+++ b/python/freetoken/moe/offload_kernels.py@@ -15,6 +15,17 @@ from flashlib.kernels.slot_cache import lru_ensure_HYBRID_FETCH_BY_RECENCY = (os.getenv("FREETOKEN_HYBRID_FETCH", "recency").strip().lower() != "lowest_id")+# How fetch_fraction * misses becomes a fetch count: "min-wait" (default) picks the integer+# that makes the PCIe fetch and the CPU compute finish closest together, "stochastic" rounds+# up with probability equal to the fractional part, "prefer-fetch" always rounds up.+_HYBRID_FETCH_ROUNDING = {"min-wait": 0, "stochastic": 1, "prefer-fetch": 2}+_hybrid_fetch_round = (os.getenv("FREETOKEN_HYBRID_FETCH_ROUND") or "min-wait").strip().lower()+if _hybrid_fetch_round not in _HYBRID_FETCH_ROUNDING:+ raise ValueError(f"FREETOKEN_HYBRID_FETCH_ROUND={_hybrid_fetch_round!r}: expected one of {sorted(_HYBRID_FETCH_ROUNDING)}")+_HYBRID_FETCH_ROUND = _HYBRID_FETCH_ROUNDING[_hybrid_fetch_round]+_HYBRID_FETCH_SEED = 0+# Phase 2 of the hybrid ensure kernel scans a pool wider than this in tiles of this width.+_HYBRID_EVICT_CHUNK = 8192def ensure_experts(cache, layer_id: int, expert_ids: torch.Tensor) -> None:@@ -66,9 +77,9 @@ def ensure_experts_hybrid(experts are given a slot and scheduled for copy; the overflow misses staynon-resident and their ``expert_ids`` positions are rewritten to ``-1`` (compute onthe CPU). ``fetch_fraction`` > 0 replaces the fixed cap with the bandwidth-matched- split (fraction = pcie_bw / cpu_bw): fetch ~fraction of the step's misses, rounded to- the integer that makes the PCIe fetch and the CPU overflow compute finish closest to- together. ``num_indices`` = capped fetch count (copy_missing); ``num_missing_full`` =+ split (fraction = pcie_bw / cpu_bw): fetch ~fraction of the step's misses, rounded as+ ``FREETOKEN_HYBRID_FETCH_ROUND`` says (default: the integer that makes the PCIe fetch and the CPU overflow compute finish closest together).+ ``num_indices`` = capped fetch count (copy_missing); ``num_missing_full`` =pre-cap miss count (stats).The same launch splits the routes: ``expert_ids`` becomes the GPU slots in place, and@@ -140,8 +151,9 @@ def _ensure_experts_hybrid_gpu(id_of_slot, usage = cache.pool_state(layer_id)num_slots = id_of_slot.numel()block_e = triton.next_power_of_2(cache.num_experts)- block_c = triton.next_power_of_2(num_slots)- num_warps = 8 if block_c >= 2048 else 4+ chunk = min(_HYBRID_EVICT_CHUNK, triton.next_power_of_2(num_slots))+ # with 4 warps a 4096+ chunk spills on sm80-sm89+ num_warps = 8 if chunk >= 4096 else 4_ensure_experts_hybrid_kernel[(1,)](expert_ids,cache.slot_for_id,@@ -165,9 +177,12 @@ def _ensure_experts_hybrid_gpu(int(fetch_cap),cache.num_experts,num_slots,+ _HYBRID_FETCH_SEED,BLOCK_E=block_e,- BLOCK_C=block_c,+ CHUNK=chunk,+ SMALL=num_slots <= chunk,BY_RECENCY=_HYBRID_FETCH_BY_RECENCY,+ ROUND=_HYBRID_FETCH_ROUND,CLAMP=clamp,STATS=cache.collect_stats,num_warps=num_warps,@@ -298,9 +313,12 @@ def _ensure_experts_hybrid_kernel(fetch_cap,num_experts: tl.constexpr,cache_size,+ seed,BLOCK_E: tl.constexpr,- BLOCK_C: tl.constexpr,+ CHUNK: tl.constexpr,+ SMALL: tl.constexpr,BY_RECENCY: tl.constexpr,+ ROUND: tl.constexpr,CLAMP: tl.constexpr,STATS: tl.constexpr,):@@ -319,7 +337,12 @@ def _ensure_experts_hybrid_kernel(most-recently active before this step (LRU on the expert, via ``expert_recency``),breaking ties toward the lower expert id -- this prioritizes *recurring* misses forcaching, lowering the steady miss rate. Otherwise the lowest expert ids are fetched- (``missing_rank``), the original routing-blind heuristic."""+ (``missing_rank``), the original routing-blind heuristic.++ Victims are the least-used slots, ties to the lowest slot. Every usage is at most the+ step before this launch, so the hit slots stamped with ``step`` in Phase 1 are never+ picked. A pool of at most CHUNK slots stays in registers (SMALL); a larger one is scanned+ from memory in CHUNK tiles per victim."""step = tl.load(step_ptr) + 1tl.store(step_ptr, step)base = layer_id * num_experts@@ -341,11 +364,19 @@ def _ensure_experts_hybrid_kernel(# F * (1 - frac), CPU time with (M - F) * frac; they balance at F = frac * M. Pick# the integer neighbor that minimizes the slower (max) side of the overlap.lo = (num_missing * fetch_frac_q16) >> 16- cost_lo = tl.maximum(lo * ((1 << 16) - fetch_frac_q16), (num_missing - lo) * fetch_frac_q16)- cost_hi = tl.maximum(- (lo + 1) * ((1 << 16) - fetch_frac_q16), (num_missing - lo - 1) * fetch_frac_q16- )- max_fetch = tl.where(cost_lo <= cost_hi, lo, lo + 1)+ if ROUND == 1:+ # step is unique per launch, so each (layer, step) draws its own number+ rem = (num_missing * fetch_frac_q16) & 0xFFFF+ u = tl.rand(seed, (step & 0x7FFFFFFF).to(tl.int32))+ max_fetch = lo + ((u * 65536.0) < rem.to(tl.float32)).to(tl.int32)+ elif ROUND == 2:+ max_fetch = (num_missing * fetch_frac_q16 + 0xFFFF) >> 16+ else:+ cost_lo = tl.maximum(lo * ((1 << 16) - fetch_frac_q16), (num_missing - lo) * fetch_frac_q16)+ cost_hi = tl.maximum(+ (lo + 1) * ((1 << 16) - fetch_frac_q16), (num_missing - lo - 1) * fetch_frac_q16+ )+ max_fetch = tl.where(cost_lo <= cost_hi, lo, lo + 1)is_hit = is_active & (slot >= 0)# a pool smaller than num_experts may hold fewer free slots than missesnum_fetch = tl.minimum(tl.minimum(num_missing, tl.minimum(max_fetch, fetch_cap)), cache_size - tl.sum(is_hit.to(tl.int32)))@@ -366,18 +397,27 @@ def _ensure_experts_hybrid_kernel(# ---- Phase 2: evict victims by argmin(usage), only for the capped fetches ----if num_fetch > 0:- off_c = tl.arange(0, BLOCK_C)- c_mask = off_c < cache_size- oid = tl.load(id_of_slot_ptr + off_c, mask=c_mask, other=-1)- u = tl.load(usage_ptr + off_c, mask=c_mask, other=9223372036854775807).to(tl.int64)- owner_active = c_mask & False- for i in tl.range(num_active):- ei = tl.load(expert_ids_ptr + i)- owner_active = owner_active | (oid == base + ei)- u = tl.where(owner_active | (~c_mask), 9223372036854775807, u)+ off_k = tl.arange(0, CHUNK)+ tl.debug_barrier() # the scan below must see the hit stamps+ if SMALL:+ u_reg = tl.load(usage_ptr + off_k, mask=off_k < cache_size, other=9223372036854775807).to(tl.int64)for i in tl.range(num_fetch):- victim = tl.argmin(u, axis=0).to(tl.int32)- old_id = tl.sum(tl.where(off_c == victim, oid, 0))+ if SMALL:+ victim = tl.argmin(u_reg, axis=0).to(tl.int32)+ u_reg = tl.where(off_k == victim, step, u_reg)+ else:+ best_u = tl.full((), 9223372036854775807, tl.int64)+ best_s = tl.full((), 0, tl.int32)+ for c0 in tl.range(0, cache_size, CHUNK):+ offs = c0 + off_k+ uu = tl.load(usage_ptr + offs, mask=offs < cache_size, other=9223372036854775807).to(tl.int64)+ cmin = tl.min(uu, axis=0)+ carg = (tl.argmin(uu, axis=0) + c0).to(tl.int32)+ take = cmin < best_u+ best_s = tl.where(take, carg, best_s)+ best_u = tl.where(take, cmin, best_u)+ victim = best_s+ old_id = tl.load(id_of_slot_ptr + victim)if old_id >= 0:tl.store(slot_for_id_ptr + old_id, -1)if BY_RECENCY:@@ -390,12 +430,15 @@ def _ensure_experts_hybrid_kernel(tl.store(usage_ptr + victim, step)tl.store(evict_slots_ptr + i, victim)tl.store(src_indices_ptr + i, e) # layer-local row- u = tl.where(off_c == victim, 9223372036854775807, u)+ tl.debug_barrier() # the next scan must see this victim's stamp+ tl.debug_barrier() # Phase 3 reads slot_for_id entries written above# ---- Phase 3: rewrite expert_ids -> slot id (hit/fetched) or -1 (overflow -> CPU) ----for i in tl.range(num_active):e = tl.load(expert_ids_ptr + i)s = tl.load(slot_for_id_ptr + base + e)+ # every warp loads e, one thread stores below; a warp that loads after that store indexes with a slot id+ tl.debug_barrier()on_gpu = s >= 0tl.store(cpu_ids_ptr + i, tl.where(on_gpu, -1, e))tl.store(gpu_w_ptr + i, tl.where(on_gpu, tl.load(weights_ptr + i), 0.0))diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex 0e2706f4..bc8dc26d 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -920,6 +920,98 @@ def test_slot_pools_keep_each_layer_in_its_own_slots(target):assert [v[0].shape[0] for v in map(cache.pool_bank_views, range(L))] == [8, 8, 24]+def _hybrid_ensure_reference(st, layer, ids, weights, num_experts, pool, frac, mode, kernel_fetch):+ """One hybrid ensure call on CPU copies of the cache state, written from the LRU contract."""+ base, size = pool+ step = int(st["step"]) + 1+ slot_for_id, recency = st["slot_for_id"], st["expert_recency"]+ p_ids, p_usage = st["id_of_slot"][base : base + size], st["usage"][base : base + size]+ active = sorted(set(ids.tolist()))+ hits = [e for e in active if slot_for_id[layer, e] >= 0]+ missing = [e for e in active if slot_for_id[layer, e] < 0]+ m, fq = len(missing), min(1 << 16, max(0, round(frac * (1 << 16))))+ lo, room = (m * fq) >> 16, size - len(hits)+ if mode == "min-wait":+ cost_lo = max(lo * ((1 << 16) - fq), (m - lo) * fq)+ cost_hi = max((lo + 1) * ((1 << 16) - fq), (m - lo - 1) * fq)+ num_fetch = min(m, lo if cost_lo <= cost_hi else lo + 1, room)+ elif mode == "prefer-fetch":+ num_fetch = min(m, (m * fq + 0xFFFF) >> 16, room)+ else:+ allowed = {min(m, lo, room)} if (m * fq) & 0xFFFF == 0 else {min(m, lo, room), min(m, lo + 1, room)}+ assert kernel_fetch in allowed+ num_fetch = kernel_fetch+ taken = torch.zeros(size, dtype=torch.bool)+ for e in hits:+ p_usage[slot_for_id[layer, e]] = step+ taken[slot_for_id[layer, e]] = True+ plan = []+ for e in sorted(missing, key=lambda e: (-int(recency[layer, e]), e))[:num_fetch]:+ victim = int(torch.argmin(p_usage.masked_fill(taken, torch.iinfo(torch.int64).max))) # first minimum+ old = int(p_ids[victim])+ if old >= 0:+ slot_for_id.view(-1)[old] = -1+ p_ids[victim] = layer * num_experts + e+ slot_for_id[layer, e] = victim+ p_usage[victim] = step+ taken[victim] = True+ plan.append((victim, e))+ recency[layer, active] = step+ st["step"] = torch.tensor(step)+ slots = slot_for_id[layer, ids.long()]+ on_gpu = slots >= 0+ return st, plan, slots, torch.where(on_gpu, -1, ids), torch.where(on_gpu, weights, 0.0), m, active++[email protected](not torch.cuda.is_available(), reason="needs CUDA")[email protected]("mode", ["min-wait", "stochastic", "prefer-fetch"])[email protected]("clamp", [False, True])[email protected]("layout", ["global", "global-tiled", "per-layer"])+def test_hybrid_ensure_matches_lru_reference(monkeypatch, layout, clamp, mode):+ from freetoken.moe import offload_kernels+ from freetoken.moe.offload_cache import OffloadMoeCache++ _init_tp()+ monkeypatch.setattr(offload_kernels, "_HYBRID_FETCH_ROUND", offload_kernels._HYBRID_FETCH_ROUNDING[mode])+ torch.manual_seed(0)+ # global-tiled: more slots than one Phase 2 tile, so victims come from the tiled scan+ L, E, top_k = (20, 512, 10) if layout == "global-tiled" else (4, 128, 8)+ size = {"global": 300, "global-tiled": offload_kernels._HYBRID_EVICT_CHUNK + 808, "per-layer": 4 * 96}[layout]+ pools = {"pool_of_layer": list(range(L)), "pool_sizes": [96] * L} if layout == "per-layer" else {}+ cache = OffloadMoeCache(+ num_layers=L, num_experts=E, cache_size=size, device=torch.device("cuda"),+ decode_target="hybrid", hybrid_max_fetch=1, **pools,+ )+ popularity = 1.0 / torch.arange(1, E + 1, dtype=torch.float64) ** 0.6+ names = ["slot_for_id", "id_of_slot", "usage", "step", "expert_recency"]++ # fetch every miss until every pool is full, then the benched-style fraction+ for frac, calls in ((1.0, 4000), (0.413, 60 * L)):+ for call in range(calls):+ layer = call % L+ if frac == 1.0 and bool((cache.id_of_slot >= 0).all()) and layer == 0:+ break+ cache.hybrid_fetch_fraction = [frac] * L+ ids = torch.cat([torch.multinomial(popularity, top_k) for _ in range(2)]).to(torch.int32)+ weights = torch.rand(ids.numel())+ st = {n: getattr(cache, n).cpu().clone() for n in names}+ p = cache.pool_of_layer[layer]+ routes = cache.ensure_experts_hybrid(layer, ids.cuda(), weights.cuda(), clamp=clamp)+ n = int(cache.num_indices)+ st, plan, slots, cpu_ids, gpu_w, m, active = _hybrid_ensure_reference(+ st, layer, ids, weights, E, (cache.pool_bases[p], cache.pool_sizes[p]), frac, mode, n+ )+ assert int(cache.num_missing_full) == m and n == len(plan)+ assert cache.evict_slots[:n].cpu().tolist() == [v for v, _ in plan]+ assert cache.src_indices[:n].cpu().tolist() == [e for _, e in plan]+ assert torch.equal(routes.gpu_slots.cpu(), slots.clamp(min=0) if clamp else slots)+ assert torch.equal(routes.cpu_ids.cpu(), cpu_ids) and torch.equal(routes.gpu_w.cpu(), gpu_w)+ assert cache.active_mask.cpu().nonzero().flatten().tolist() == active+ for name in names:+ assert torch.equal(getattr(cache, name).cpu(), st[name]), name+ assert bool((cache.id_of_slot >= 0).all())++def _make_split_cache(num_layers=2, locked=(1,), prefill_overlap=False, device="cpu"):"""A [gate_up, down] bf16 cache with the given layers LOCKED (rest pinned)."""from freetoken.moe.host_banks import HostResidency