FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(moe): enable prefill hit-d2d on rocm (#771)
LZ_QAQcc4aa085b6ea
diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.pyindex 9593667e..23afd357 100644--- a/python/freetoken/engine/config.py+++ b/python/freetoken/engine/config.py@@ -55,8 +55,9 @@ class EngineConfig:moe_cache_policy: str = "lru"moe_prefill_overlap: bool = True# Prefill hit/miss split: serve cache-resident experts D2D during prefill- # prefetch instead of re-streaming the full layer over PCIe. Needs CUDA >= 12.8- # (cudaMemcpyBatchAsync); no-op unless moe_cache_size > 2 * num_experts.+ # prefetch instead of re-streaming the full layer over PCIe. Needs CUDA >= 13.0+ # (cudaMemcpyBatchAsync) or HIP (hipMemcpyBatchAsync);+ # no-op unless moe_cache_size > 2 * num_experts.moe_prefill_hit_d2d: bool = Falsemoe_collect_stats: bool = False # capture decode miss-rate counters into the cuda graph# CPU MoE backend (--moe-strategy cpu): number of CPU worker threads computingdiff --git a/python/freetoken/kernel/batch_memcpy.py b/python/freetoken/kernel/batch_memcpy.pyindex b39e5cde..15f15b7e 100644--- a/python/freetoken/kernel/batch_memcpy.py+++ b/python/freetoken/kernel/batch_memcpy.py@@ -28,6 +28,9 @@ def _probe(fn) -> None:src = torch.arange(16, dtype=torch.uint8).pin_memory()dst = torch.zeros(16, dtype=torch.uint8, device="cuda")stream = torch.cuda.Stream()+ # Order the copy after dst's initialization on the current stream;+ # otherwise the fill can overwrite the copied bytes.+ stream.wait_stream(torch.cuda.current_stream())fn(torch.tensor([dst.data_ptr()]),torch.tensor([src.data_ptr()]),@@ -45,9 +48,12 @@ def load_batch_memcpy():The 8-argument cudaMemcpyBatchAsync signature this binding uses is CUDA 13.0's(12.8/12.9 had an extra failIdx parameter); gate on the torch runtime versionbefore paying for the JIT build, then verify with a real copy.+ HIP uses its own signature, checked by building and probing a real copy."""cuda = torch.version.cuda- if cuda is None or tuple(int(x) for x in cuda.split(".")[:2]) < (13, 0):+ if torch.version.hip is None and (+ cuda is None or tuple(int(x) for x in cuda.split(".")[:2]) < (13, 0)+ ):raise RuntimeError(f"cudaMemcpyBatchAsync binding requires CUDA >= 13.0 (torch built with {cuda})")fn = _jit_batch_memcpy_module().batch_memcpy_probe(fn)@@ -60,10 +66,10 @@ def batch_memcpy_jit(sizes: torch.Tensor,stream: int,) -> None:- """Enqueue one cudaMemcpyBatchAsync of ``len(sizes)`` independent copies.+ """Enqueue one cudaMemcpyBatchAsync/hipMemcpyBatchAsync of ``len(sizes)`` independent copies.``dst_ptrs``/``src_ptrs``/``sizes`` are same-length CPU int64 tensors of raw- addresses and byte counts; ``stream`` is a raw cudaStream_t handle+ addresses and byte counts; ``stream`` is a raw cudaStream_t/hipStream_t handle(``torch.cuda.Stream.cuda_stream``), which must not be the legacy NULL stream."""load_batch_memcpy()(dst_ptrs, src_ptrs, sizes, stream)diff --git a/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh b/python/freetoken/kernel/csrc/jit/batch_memcpy.cuhindex 690eff3d..c8e14f2e 100644--- a/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh+++ b/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh@@ -13,6 +13,7 @@// pointer-to-pointer copies with ONE runtime call, on an explicit (non-legacy)// stream. Callers hand pre-resolved raw addresses; copies within a batch are// unordered, so entries must be pairwise independent.+// ROCm uses hipMemcpyBatchAsync with its nine-argument signature (failIdx).struct BatchMemcpy {static void run(tvm::ffi::TensorView dst_ptrs,@@ -20,7 +21,7 @@ struct BatchMemcpy {tvm::ffi::TensorView sizes,int64_t stream_handle) {-#if CUDART_VERSION >= 13000+#if FREETOKEN_USE_ROCM || CUDART_VERSION >= 13000using namespace host;auto N = SymbolicSize{"batch length"};auto ptr_dtype = SymbolicDType{};@@ -34,7 +35,30 @@ struct BatchMemcpy {if (n == 0) {return;}- RuntimeCheck(stream_handle != 0, "cudaMemcpyBatchAsync rejects the legacy NULL stream");+ RuntimeCheck(stream_handle != 0, "batch memcpy requires an explicit non-NULL stream");+#if FREETOKEN_USE_ROCM+ auto attr = ::hipMemcpyAttributes{};+ attr.srcAccessOrder = ::hipMemcpySrcAccessOrderStream;+ std::size_t attr_idx = 0;+ std::size_t fail_idx = SIZE_MAX;+ const auto status = ::hipMemcpyBatchAsync(+ reinterpret_cast<void**>(dst_ptrs.data_ptr()),+ reinterpret_cast<void**>(src_ptrs.data_ptr()),+ reinterpret_cast<std::size_t*>(sizes.data_ptr()),+ n,+ &attr,+ &attr_idx,+ 1,+ &fail_idx,+ reinterpret_cast<::hipStream_t>(stream_handle)+ );+ if (status != ::hipSuccess) {+ ::host::panic(+ std::source_location::current(),+ "hipMemcpyBatchAsync failed (failIdx=", fail_idx, "): ", ::hipGetErrorString(status)+ );+ }+#elseauto attr = ::cudaMemcpyAttributes{};attr.srcAccessOrder = ::cudaMemcpySrcAccessOrderStream;std::size_t attr_idx = 0;@@ -48,6 +72,7 @@ struct BatchMemcpy {1,reinterpret_cast<::cudaStream_t>(stream_handle)));+#endif#else::host::panic(std::source_location::current(),diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex f555be96..4a6a297d 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -127,7 +127,7 @@ class OffloadMoeCache:prefill_overlap: bool = False# Prefill hit/miss split: experts already resident in the slot cache (slots# >= 2 * num_experts) are gathered device-side into the double buffer instead- # of re-crossing PCIe; only the misses are H2D'd (one cudaMemcpyBatchAsync of+ # of re-crossing PCIe; only the misses are H2D'd (one CUDA/HIP batch memcpy of# coalesced runs). Requires prefill_overlap, cache_size > 2 * num_experts and# the fused copy plan; silently falls back to the full-layer copy otherwise.prefill_hit_d2d: bool = False@@ -935,7 +935,7 @@ class OffloadMoeCache:f"(every slot lies under the prefill staging buffers of {2 * self.num_experts} experts)")elif not self._resolve_batch_memcpy():- reason = "cudaMemcpyBatchAsync is unavailable" # resolve logged the specifics+ reason = "batch memcpy is unavailable" # resolve logged the specificselse:return Trueif not self._hit_d2d_fallback_logged:@@ -965,7 +965,7 @@ class OffloadMoeCache:fixed-shape gather indices (no host round trip), then fast_index_copy_multimoves the rows. Serializing the gather before this layer's GEMMs costs itsplain duration instead of nondeterministic SM contention. Misses cross- PCIe as ONE cudaMemcpyBatchAsync of coalesced expert-id runs on the copy+ PCIe as ONE batch memcpy of coalesced expert-id runs on the copystream, under the existing release/ready event discipline; its host-builtrun list comes from the begin-of-chunk snapshot because the batch APItakes HOST pointer arrays. Live-vs-snapshot cannot disagree: the onlydiff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex d09534a8..cca78e63 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -890,7 +890,7 @@ def parse_args(help=("During prefill prefetch, copy cache-resident experts device-side into ""the double buffer and stream only the misses over PCIe "- "(cudaMemcpyBatchAsync, CUDA >= 13.0). Effective with "+ "(CUDA >= 13.0 or HIP with batch-copy support). Effective with ""--moe-cache-size > 2 * num_experts."),)diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.pyindex 00b31482..44130d35 100644--- a/tests/engine/test_cache_budget.py+++ b/tests/engine/test_cache_budget.py@@ -217,7 +217,7 @@ def test_adjust_config_resolves_num_tokens_generic():cuda_graph_bs = [1, 2]max_seq_len = 1024page_size = 1- attention_backend = "fi"+ attention_backend = "triton" if torch.version.hip is not None else "fi"num_page_override = Nonenum_token_override = 5000@@ -354,7 +354,7 @@ def _offload_engine_config(**overrides):model_path="/tmp/freetoken-test-model",tp_info=DistributedInfo(rank=0, size=1),dtype=torch.bfloat16,- attention_backend="fi",+ attention_backend="triton" if torch.version.hip is not None else "fi",**overrides,)object.__setattr__(diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex 8833cdf9..0e2706f4 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -428,7 +428,7 @@ def test_adjust_config_converts_moe_cache_rate_to_cache_size(monkeypatch):model_path="/tmp/freetoken-test-model",tp_info=DistributedInfo(rank=0, size=1),dtype=torch.float16,- attention_backend="fi",+ attention_backend="triton" if torch.version.hip is not None else "fi",moe_cache_rate=0.3,)object.__setattr__(diff --git a/tests/moe/test_prefill_hit_d2d.py b/tests/moe/test_prefill_hit_d2d.pyindex abe4dfc4..9c6d4189 100644--- a/tests/moe/test_prefill_hit_d2d.py+++ b/tests/moe/test_prefill_hit_d2d.py@@ -26,7 +26,8 @@ def _cuda_at_least(major: int, minor: int) -> bool:return tuple(int(x) for x in cuda.split(".")[:2]) >= (major, minor)BATCH_API = pytest.mark.skipif(- not _cuda_at_least(13, 0), reason="the cudaMemcpyBatchAsync binding needs CUDA >= 13.0"+ torch.version.hip is None and not _cuda_at_least(13, 0),+ reason="the cudaMemcpyBatchAsync binding needs CUDA >= 13.0",)NUM_LAYERS, E, CACHE_SIZE = 3, 8, 24 # hit region = slots [16, 24)@@ -77,6 +78,32 @@ def test_batch_memcpy_roundtrip():assert torch.equal(dst.cpu(), src[perm])+@CUDA+@JIT+@BATCH_API+def test_batch_memcpy_probe_waits_for_current_stream():+ from freetoken.kernel.batch_memcpy import load_batch_memcpy++ # Compile and initialize the binding before queuing work so JIT cannot drain it.+ load_batch_memcpy()+ # Warm probe-sized allocations and the stream pool to avoid first-use syncs.+ for _ in range(4):+ block = torch.zeros(16, dtype=torch.uint8, device="cuda")+ del block+ pinned = torch.arange(16, dtype=torch.uint8).pin_memory()+ del pinned+ streams = [torch.cuda.Stream() for _ in range(40)]+ torch.cuda.synchronize()+ del streams++ try:+ torch.cuda._sleep(1_500_000_000)+ assert not torch.cuda.current_stream().query(), "backlog drained before the probe ran"+ load_batch_memcpy()+ finally:+ torch.cuda.synchronize()++@CUDA@JIT@BATCH_API