Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(moe): enable prefill hit-d2d on rocm (#771)

LZ_QAQcc4aa085b6ea
diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py
index 9593667e..23afd357 100644
--- a/python/freetoken/engine/config.py
+++ b/python/freetoken/engine/config.py
@@ -55,8 +55,9 @@ class EngineConfig:
moe_cache_policy: str = "lru"
moe_prefill_overlap: bool = True
# Prefill hit/miss split: serve cache-resident experts D2D during prefill
- # prefetch instead of re-streaming the full layer over PCIe. Needs CUDA >= 12.8
- # (cudaMemcpyBatchAsync); no-op unless moe_cache_size > 2 * num_experts.
+ # prefetch instead of re-streaming the full layer over PCIe. Needs CUDA >= 13.0
+ # (cudaMemcpyBatchAsync) or HIP (hipMemcpyBatchAsync);
+ # no-op unless moe_cache_size > 2 * num_experts.
moe_prefill_hit_d2d: bool = False
moe_collect_stats: bool = False # capture decode miss-rate counters into the cuda graph
# CPU MoE backend (--moe-strategy cpu): number of CPU worker threads computing
diff --git a/python/freetoken/kernel/batch_memcpy.py b/python/freetoken/kernel/batch_memcpy.py
index b39e5cde..15f15b7e 100644
--- a/python/freetoken/kernel/batch_memcpy.py
+++ b/python/freetoken/kernel/batch_memcpy.py
@@ -28,6 +28,9 @@ def _probe(fn) -> None:
src = torch.arange(16, dtype=torch.uint8).pin_memory()
dst = torch.zeros(16, dtype=torch.uint8, device="cuda")
stream = torch.cuda.Stream()
+ # Order the copy after dst's initialization on the current stream;
+ # otherwise the fill can overwrite the copied bytes.
+ stream.wait_stream(torch.cuda.current_stream())
fn(
torch.tensor([dst.data_ptr()]),
torch.tensor([src.data_ptr()]),
@@ -45,9 +48,12 @@ def load_batch_memcpy():
The 8-argument cudaMemcpyBatchAsync signature this binding uses is CUDA 13.0's
(12.8/12.9 had an extra failIdx parameter); gate on the torch runtime version
before paying for the JIT build, then verify with a real copy.
+ HIP uses its own signature, checked by building and probing a real copy.
"""
cuda = torch.version.cuda
- if cuda is None or tuple(int(x) for x in cuda.split(".")[:2]) < (13, 0):
+ if torch.version.hip is None and (
+ cuda is None or tuple(int(x) for x in cuda.split(".")[:2]) < (13, 0)
+ ):
raise RuntimeError(f"cudaMemcpyBatchAsync binding requires CUDA >= 13.0 (torch built with {cuda})")
fn = _jit_batch_memcpy_module().batch_memcpy
_probe(fn)
@@ -60,10 +66,10 @@ def batch_memcpy_jit(
sizes: torch.Tensor,
stream: int,
) -> None:
- """Enqueue one cudaMemcpyBatchAsync of ``len(sizes)`` independent copies.
+ """Enqueue one cudaMemcpyBatchAsync/hipMemcpyBatchAsync of ``len(sizes)`` independent copies.
``dst_ptrs``/``src_ptrs``/``sizes`` are same-length CPU int64 tensors of raw
- addresses and byte counts; ``stream`` is a raw cudaStream_t handle
+ addresses and byte counts; ``stream`` is a raw cudaStream_t/hipStream_t handle
(``torch.cuda.Stream.cuda_stream``), which must not be the legacy NULL stream.
"""
load_batch_memcpy()(dst_ptrs, src_ptrs, sizes, stream)
diff --git a/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh b/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh
index 690eff3d..c8e14f2e 100644
--- a/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh
+++ b/python/freetoken/kernel/csrc/jit/batch_memcpy.cuh
@@ -13,6 +13,7 @@
// pointer-to-pointer copies with ONE runtime call, on an explicit (non-legacy)
// stream. Callers hand pre-resolved raw addresses; copies within a batch are
// unordered, so entries must be pairwise independent.
+// ROCm uses hipMemcpyBatchAsync with its nine-argument signature (failIdx).
struct BatchMemcpy {
static void run(
tvm::ffi::TensorView dst_ptrs,
@@ -20,7 +21,7 @@ struct BatchMemcpy {
tvm::ffi::TensorView sizes,
int64_t stream_handle
) {
-#if CUDART_VERSION >= 13000
+#if FREETOKEN_USE_ROCM || CUDART_VERSION >= 13000
using namespace host;
auto N = SymbolicSize{"batch length"};
auto ptr_dtype = SymbolicDType{};
@@ -34,7 +35,30 @@ struct BatchMemcpy {
if (n == 0) {
return;
}
- RuntimeCheck(stream_handle != 0, "cudaMemcpyBatchAsync rejects the legacy NULL stream");
+ RuntimeCheck(stream_handle != 0, "batch memcpy requires an explicit non-NULL stream");
+#if FREETOKEN_USE_ROCM
+ auto attr = ::hipMemcpyAttributes{};
+ attr.srcAccessOrder = ::hipMemcpySrcAccessOrderStream;
+ std::size_t attr_idx = 0;
+ std::size_t fail_idx = SIZE_MAX;
+ const auto status = ::hipMemcpyBatchAsync(
+ reinterpret_cast<void**>(dst_ptrs.data_ptr()),
+ reinterpret_cast<void**>(src_ptrs.data_ptr()),
+ reinterpret_cast<std::size_t*>(sizes.data_ptr()),
+ n,
+ &attr,
+ &attr_idx,
+ 1,
+ &fail_idx,
+ reinterpret_cast<::hipStream_t>(stream_handle)
+ );
+ if (status != ::hipSuccess) {
+ ::host::panic(
+ std::source_location::current(),
+ "hipMemcpyBatchAsync failed (failIdx=", fail_idx, "): ", ::hipGetErrorString(status)
+ );
+ }
+#else
auto attr = ::cudaMemcpyAttributes{};
attr.srcAccessOrder = ::cudaMemcpySrcAccessOrderStream;
std::size_t attr_idx = 0;
@@ -48,6 +72,7 @@ struct BatchMemcpy {
1,
reinterpret_cast<::cudaStream_t>(stream_handle)
));
+#endif
#else
::host::panic(
std::source_location::current(),
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index f555be96..4a6a297d 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -127,7 +127,7 @@ class OffloadMoeCache:
prefill_overlap: bool = False
# Prefill hit/miss split: experts already resident in the slot cache (slots
# >= 2 * num_experts) are gathered device-side into the double buffer instead
- # of re-crossing PCIe; only the misses are H2D'd (one cudaMemcpyBatchAsync of
+ # of re-crossing PCIe; only the misses are H2D'd (one CUDA/HIP batch memcpy of
# coalesced runs). Requires prefill_overlap, cache_size > 2 * num_experts and
# the fused copy plan; silently falls back to the full-layer copy otherwise.
prefill_hit_d2d: bool = False
@@ -935,7 +935,7 @@ class OffloadMoeCache:
f"(every slot lies under the prefill staging buffers of {2 * self.num_experts} experts)"
)
elif not self._resolve_batch_memcpy():
- reason = "cudaMemcpyBatchAsync is unavailable" # resolve logged the specifics
+ reason = "batch memcpy is unavailable" # resolve logged the specifics
else:
return True
if not self._hit_d2d_fallback_logged:
@@ -965,7 +965,7 @@ class OffloadMoeCache:
fixed-shape gather indices (no host round trip), then fast_index_copy_multi
moves the rows. Serializing the gather before this layer's GEMMs costs its
plain duration instead of nondeterministic SM contention. Misses cross
- PCIe as ONE cudaMemcpyBatchAsync of coalesced expert-id runs on the copy
+ PCIe as ONE batch memcpy of coalesced expert-id runs on the copy
stream, under the existing release/ready event discipline; its host-built
run list comes from the begin-of-chunk snapshot because the batch API
takes HOST pointer arrays. Live-vs-snapshot cannot disagree: the only
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index d09534a8..cca78e63 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -890,7 +890,7 @@ def parse_args(
help=(
"During prefill prefetch, copy cache-resident experts device-side into "
"the double buffer and stream only the misses over PCIe "
- "(cudaMemcpyBatchAsync, CUDA >= 13.0). Effective with "
+ "(CUDA >= 13.0 or HIP with batch-copy support). Effective with "
"--moe-cache-size > 2 * num_experts."
),
)
diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py
index 00b31482..44130d35 100644
--- a/tests/engine/test_cache_budget.py
+++ b/tests/engine/test_cache_budget.py
@@ -217,7 +217,7 @@ def test_adjust_config_resolves_num_tokens_generic():
cuda_graph_bs = [1, 2]
max_seq_len = 1024
page_size = 1
- attention_backend = "fi"
+ attention_backend = "triton" if torch.version.hip is not None else "fi"
num_page_override = None
num_token_override = 5000
@@ -354,7 +354,7 @@ def _offload_engine_config(**overrides):
model_path="/tmp/freetoken-test-model",
tp_info=DistributedInfo(rank=0, size=1),
dtype=torch.bfloat16,
- attention_backend="fi",
+ attention_backend="triton" if torch.version.hip is not None else "fi",
**overrides,
)
object.__setattr__(
diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.py
index 8833cdf9..0e2706f4 100644
--- a/tests/moe/test_offload.py
+++ b/tests/moe/test_offload.py
@@ -428,7 +428,7 @@ def test_adjust_config_converts_moe_cache_rate_to_cache_size(monkeypatch):
model_path="/tmp/freetoken-test-model",
tp_info=DistributedInfo(rank=0, size=1),
dtype=torch.float16,
- attention_backend="fi",
+ attention_backend="triton" if torch.version.hip is not None else "fi",
moe_cache_rate=0.3,
)
object.__setattr__(
diff --git a/tests/moe/test_prefill_hit_d2d.py b/tests/moe/test_prefill_hit_d2d.py
index abe4dfc4..9c6d4189 100644
--- a/tests/moe/test_prefill_hit_d2d.py
+++ b/tests/moe/test_prefill_hit_d2d.py
@@ -26,7 +26,8 @@ def _cuda_at_least(major: int, minor: int) -> bool:
return tuple(int(x) for x in cuda.split(".")[:2]) >= (major, minor)
BATCH_API = pytest.mark.skipif(
- not _cuda_at_least(13, 0), reason="the cudaMemcpyBatchAsync binding needs CUDA >= 13.0"
+ torch.version.hip is None and not _cuda_at_least(13, 0),
+ reason="the cudaMemcpyBatchAsync binding needs CUDA >= 13.0",
)
NUM_LAYERS, E, CACHE_SIZE = 3, 8, 24 # hit region = slots [16, 24)
@@ -77,6 +78,32 @@ def test_batch_memcpy_roundtrip():
assert torch.equal(dst.cpu(), src[perm])
+@CUDA
+@JIT
+@BATCH_API
+def test_batch_memcpy_probe_waits_for_current_stream():
+ from freetoken.kernel.batch_memcpy import load_batch_memcpy
+
+ # Compile and initialize the binding before queuing work so JIT cannot drain it.
+ load_batch_memcpy()
+ # Warm probe-sized allocations and the stream pool to avoid first-use syncs.
+ for _ in range(4):
+ block = torch.zeros(16, dtype=torch.uint8, device="cuda")
+ del block
+ pinned = torch.arange(16, dtype=torch.uint8).pin_memory()
+ del pinned
+ streams = [torch.cuda.Stream() for _ in range(40)]
+ torch.cuda.synchronize()
+ del streams
+
+ try:
+ torch.cuda._sleep(1_500_000_000)
+ assert not torch.cuda.current_stream().query(), "backlog drained before the probe ran"
+ load_batch_memcpy()
+ finally:
+ torch.cuda.synchronize()
+
+
@CUDA
@JIT
@BATCH_API