Official Python inference and LoRA trainer package for the LTX-2 audio–video generative model.
/**
* @file all2all.cpp
* @brief Implementation of All2All communication primitives for multi-GPU tensor parallelism.
*
* This file implements the All2All class which provides efficient inter-GPU communication
* using CUDA IPC (Inter-Process Communication). The implementation supports:
* - Head redistribution for tensor-parallel attention (send_recv_heads, gather_heads)
* - Sequence gathering for cross-rank aggregation (allgather)
*
* All operations use a barrier-based synchronization protocol where each GPU writes
* directly to remote GPU memory via IPC, then signals completion through atomic
* operations on barrier counters.
*/
#include <ATen/cuda/CUDAContext.h>
#include <ATen/cuda/CUDADataType.h>
#include <c10/cuda/CUDAGuard.h>
#include <chrono>
#include <cuda_runtime.h>
#include <memory>
#include <pybind11/functional.h>
#include <torch/python.h>
#include "all2all.hpp"
#include "cuda/api.cuh"
#include "cuda/configs.cuh"
namespace ltx_kernels {
namespace all2all {
/**
* Constructs the All2All communication manager.
*
* Memory Allocation Strategy:
* The constructor allocates a single contiguous GPU memory block that contains:
* 1. Data buffer (tensor_bytes): Space for tensor data exchange
* 2. Barrier signals (MAX_NUM_PEERS * sizeof(int)): Per-rank completion counters
* 3. Buffer pointers (MAX_NUM_PEERS * sizeof(void*)): GPU-accessible pointer array
* 4. Barrier pointer array (MAX_NUM_PEERS * sizeof(int*)): GPU-accessible signal pointers
*
* This layout minimizes memory allocations and allows the entire region to be
* shared via a single IPC handle.
*/
All2All::All2All(int rank, int world_size, int num_tokens, int hidden_dim, int num_sms, at::ScalarType tensor_dtype,
double timeout_seconds)
: rank(rank), world_size(world_size), num_sms(num_sms), max_tokens(num_tokens), num_elems(0), tensor_bytes(0),
tensor_dtype(tensor_dtype) {
num_elems = int64_t(num_tokens) * int64_t(hidden_dim);
tensor_bytes = num_elems * elementSize(tensor_dtype);
// Derive the barrier timeout from the device's peak SM clock so the wall-clock guard is
// correct on any GPU (the kernel counts SM cycles via clock64). Use cudaDeviceGetAttribute,
// not cudaDeviceProp::clockRate, which was removed in CUDA 13. The attribute is in kHz.
int device = 0;
CUDA_CHECK(cudaGetDevice(&device));
int sm_clock_khz = 0;
CUDA_CHECK(cudaDeviceGetAttribute(&sm_clock_khz, cudaDevAttrClockRate, device));
sm_clock_hz_ = static_cast<double>(sm_clock_khz) * 1e3;
set_timeout_seconds(timeout_seconds);
// Calculate sizes for each region of the shared memory block
int64_t ptrs_bytes = MAX_NUM_PEERS * sizeof(void *);
int64_t barrier_signal_bytes = kNumSignalSlots * sizeof(int);
int64_t barrier_signal_ptrs_bytes = MAX_NUM_PEERS * sizeof(int *);
// Allocate the main shared memory block and create IPC handle
// Layout: [data_buffer | barrier_signals | buffer_ptrs | barrier_signal_ptrs]
CUDA_CHECK(
cudaMalloc(&buffer_ptrs[rank], tensor_bytes + barrier_signal_bytes + ptrs_bytes + barrier_signal_ptrs_bytes));
CUDA_CHECK(cudaIpcGetMemHandle(&ipc_handlers[rank], buffer_ptrs[rank]));
// Set up pointers to each region within the allocated block
buffer_ptrs_gpu =
reinterpret_cast<void **>(static_cast<uint8_t *>(buffer_ptrs[rank]) + tensor_bytes + barrier_signal_bytes);
barrier_signal_ptrs[rank] = reinterpret_cast<int *>(static_cast<uint8_t *>(buffer_ptrs[rank]) + tensor_bytes);
barrier_signal_ptrs_gpu = reinterpret_cast<int **>(static_cast<uint8_t *>(buffer_ptrs[rank]) + tensor_bytes +
barrier_signal_bytes + ptrs_bytes);
// Initialize barrier signals to zero
CUDA_CHECK(cudaMemset(barrier_signal_ptrs[rank], 0, barrier_signal_bytes));
}
All2All::~All2All() noexcept(false) {
if (!destroyed) {
printf("WARNING: destroy() was not called, which can leak resources.\n");
fflush(stdout);
destroy();
}
}
/**
* Releases all allocated resources.
*
* This must be called explicitly before destruction to ensure proper cleanup of:
* - IPC memory mappings to remote GPUs
* - Local GPU memory allocations
*
* The method synchronizes the device to ensure all pending operations complete
* before releasing resources.
*/
void All2All::destroy() {
if (destroyed) {
return;
}
CUDA_CHECK(cudaDeviceSynchronize());
// Close IPC mappings to remote GPU memory (skip our own rank)
// Only close handles that were actually opened via sync()
for (int i = 0; i < world_size; i++) {
if (i != rank && buffer_ptrs[i] != nullptr) {
CUDA_CHECK(cudaIpcCloseMemHandle(buffer_ptrs[i]));
}
}
// Free local GPU memory allocations
CUDA_CHECK(cudaFree(buffer_ptrs[rank]));
destroyed = true;
}
/**
* Opens IPC memory mappings to all peer GPUs.
*
* This method processes IPC handles gathered from all ranks and opens memory
* mappings to enable direct GPU-to-GPU memory access. After calling this method,
* each GPU can read/write directly to any other GPU's buffer via buffer_ptrs.
*
* The barrier_signal_ptrs are also set up to point to the correct offset within
* each peer's shared memory block.
*/
void All2All::sync(const std::vector<std::optional<pybind11::bytearray>> &all_gathered_handles) {
for (int i = 0; i < world_size; i++) {
auto handle_str = std::string(all_gathered_handles[i].value());
EP_HOST_ASSERT(handle_str.size() == CUDA_IPC_HANDLE_SIZE);
if (i != rank) {
// Open IPC mapping to remote GPU's memory
std::memcpy(ipc_handlers[i].reserved, handle_str.c_str(), CUDA_IPC_HANDLE_SIZE);
CUDA_CHECK(cudaIpcOpenMemHandle(&buffer_ptrs[i], ipc_handlers[i], cudaIpcMemLazyEnablePeerAccess));
// Calculate offset to barrier signals in remote buffer
barrier_signal_ptrs[i] = reinterpret_cast<int *>(static_cast<uint8_t *>(buffer_ptrs[i]) + tensor_bytes);
} else {
// Verify our own handle matches what we sent
EP_HOST_ASSERT(std::memcmp(ipc_handlers[i].reserved, handle_str.c_str(), CUDA_IPC_HANDLE_SIZE) == 0);
}
}
// Copy pointer arrays to GPU for kernel access
CUDA_CHECK(cudaMemcpy(buffer_ptrs_gpu, buffer_ptrs, sizeof(void *) * world_size, cudaMemcpyHostToDevice));
CUDA_CHECK(
cudaMemcpy(barrier_signal_ptrs_gpu, barrier_signal_ptrs, sizeof(int *) * world_size, cudaMemcpyHostToDevice));
CUDA_CHECK(cudaDeviceSynchronize());
}
pybind11::bytearray All2All::get_local_ipc_handle() const {
return {ipc_handlers[rank].reserved, CUDA_IPC_HANDLE_SIZE};
}
at::Tensor All2All::local_buffer() const {
return torch::from_blob(buffer_ptrs[rank], {num_elems},
torch::TensorOptions().dtype(tensor_dtype).device(torch::kCUDA, rank));
}
/**
* Creates a tensor from the local IPC buffer.
*
* This helper method returns either a zero-copy view of the IPC buffer or
* a newly allocated tensor with the data copied. The zero-copy mode is more
* efficient but the tensor lifetime is tied to the All2All instance.
*
* @note The buffer pointer is cast to the template type T for proper interpretation.
*/
at::Tensor All2All::get_local_buffer_tensor(at::Tensor &x, int batch_size, int out_tokens, int out_heads, int head_size,
bool should_copy, cudaStream_t stream) {
auto ptr = buffer_ptrs[rank];
if (should_copy) {
// Allocate new tensor and copy data from IPC buffer
auto out_tensor = torch::empty({batch_size, out_tokens, out_heads, head_size}, x.options());
CUDA_CHECK(cudaMemcpyAsync(out_tensor.data_ptr(), ptr,
int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
int64_t(elementSize(x.scalar_type())),
cudaMemcpyDeviceToDevice, stream));
return out_tensor;
} else {
// Return a view directly into the IPC buffer (zero-copy)
int64_t numel = int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size);
return local_buffer().narrow(0, 0, numel).view({batch_size, out_tokens, out_heads, head_size});
}
}
/**
* All2All communication to redistribute attention heads across GPUs.
*
* This operation is used in tensor-parallel transformers to exchange attention heads:
* - Before: Each GPU has all tokens but only a subset of heads
* - After: Each GPU has all tokens with heads redistributed
*
* Tensor Layout Transformation:
* Input: [batch, local_tokens, all_heads, head_size] per GPU
* Output: [batch, all_tokens, heads_per_rank, head_size] per GPU
*
* The operation partitions heads evenly: heads_per_rank = all_heads / world_size
* GPU i receives heads [i*heads_per_rank : (i+1)*heads_per_rank] from all GPUs.
*/
at::Tensor All2All::send_recv_heads(at::Tensor &x, bool copy_output) {
// Validate input tensor properties
EP_HOST_ASSERT(x.dim() == 4 and x.is_contiguous());
EP_HOST_ASSERT(x.dtype() == tensor_dtype);
EP_HOST_ASSERT(x.device().is_cuda());
EP_HOST_ASSERT(x.device().index() == rank);
int batch_size = x.size(0);
int num_tokens = x.size(1);
int num_heads = x.size(2);
int head_size = x.size(3);
// Sequence parallelism pads each rank to the same token count.
int out_tokens = num_tokens * world_size;
int out_heads = num_heads / world_size; // Each rank gets 1/world_size of heads
EP_HOST_ASSERT(int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
int64_t(elementSize(x.scalar_type())) <=
tensor_bytes);
at::cuda::CUDAGuard device_guard{x.device()};
auto stream = at::cuda::getCurrentCUDAStream().stream();
// Launch the All2All kernel
all2all_cuda::all2all_head_launch(buffer_ptrs_gpu, barrier_signal_ptrs_gpu, x.data_ptr(), rank, world_size,
batch_size, out_tokens, num_tokens, num_heads, head_size, stream, num_sms,
tensor_dtype, timeout_cycles_);
return get_local_buffer_tensor(x, batch_size, out_tokens, out_heads, head_size, copy_output, stream);
}
/**
* Inverse All2All to gather heads back to original distribution.
*
* This is the inverse operation of send_recv_heads(). It redistributes data
* so each GPU gets back its original tokens with all attention heads.
*
* Tensor Layout Transformation:
* Input: [batch, all_tokens, heads_per_rank, head_size] per GPU
* Output: [batch, local_tokens, all_heads, head_size] per GPU
*
* Each GPU sends its portion of tokens to the originating rank, reconstructing
* the original head distribution.
*/
at::Tensor All2All::gather_heads(at::Tensor &x, bool copy_output) {
// Validate input tensor properties
EP_HOST_ASSERT(x.dim() == 4 and x.is_contiguous());
EP_HOST_ASSERT(x.dtype() == tensor_dtype);
EP_HOST_ASSERT(x.device().is_cuda());
EP_HOST_ASSERT(x.device().index() == rank);
at::cuda::CUDAGuard device_guard{x.device()};
auto stream = at::cuda::getCurrentCUDAStream().stream();
int batch_size = x.size(0);
int total_tokens = x.size(1);
int num_heads = x.size(2) * world_size; // Reconstruct total head count
int head_size = x.size(3);
EP_HOST_ASSERT(total_tokens % world_size == 0);
int out_tokens = total_tokens / world_size;
int out_heads = num_heads;
EP_HOST_ASSERT(int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
int64_t(elementSize(x.scalar_type())) <=
tensor_bytes);
// Launch the gather kernel
all2all_cuda::all2all_head_gather_launch(buffer_ptrs_gpu, barrier_signal_ptrs_gpu, x.data_ptr(), rank, world_size,
batch_size, total_tokens, num_heads, head_size, stream, num_sms,
tensor_dtype, timeout_cycles_);
return get_local_buffer_tensor(x, batch_size, out_tokens, out_heads, head_size, copy_output, stream);
}
/**
* AllGather operation to collect sequence tokens from all ranks.
*
* Each GPU contributes its local sequence tokens, which are gathered into
* a complete sequence replicated on all GPUs. This is typically used after
* tensor-parallel operations to reconstruct the full sequence.
*
* Tensor Layout Transformation:
* Input: [batch, local_seqlen, heads, head_size] per GPU
* Output: [batch, total_seqlen, heads, head_size] per GPU (identical on all GPUs)
*
* Each GPU's tokens are placed at offset rank * local_seqlen in the output.
*/
at::Tensor All2All::allgather(at::Tensor &x, bool copy_output) {
// Validate input tensor properties
EP_HOST_ASSERT(x.dim() == 4 and x.is_contiguous());
EP_HOST_ASSERT(x.dtype() == tensor_dtype);
EP_HOST_ASSERT(x.device().is_cuda());
EP_HOST_ASSERT(x.device().index() == rank);
at::cuda::CUDAGuard device_guard{x.device()};
auto stream = at::cuda::getCurrentCUDAStream().stream();
int batch_size = x.size(0);
int seqlen = x.size(1);
int num_heads = x.size(2);
int head_size = x.size(3);
int out_tokens = seqlen * world_size;
int out_heads = num_heads;
int hidden_dim = num_heads * head_size;
EP_HOST_ASSERT(int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
int64_t(elementSize(x.scalar_type())) <=
tensor_bytes);
// Launch the allgather kernel
all2all_cuda::allgather_launch(buffer_ptrs_gpu, barrier_signal_ptrs_gpu, x.data_ptr(), rank, world_size, batch_size,
seqlen, hidden_dim, out_tokens, stream, num_sms, tensor_dtype, timeout_cycles_);
return get_local_buffer_tensor(x, batch_size, out_tokens, out_heads, head_size, copy_output, stream);
}
void All2All::rope_send_recv_heads(at::Tensor &x, at::Tensor &sin_freqs, at::Tensor &cos_freqs, at::Tensor &weights,
double eps, bool barrier) {
EP_HOST_ASSERT(x.dim() == 3 and x.scalar_type() == at::ScalarType::BFloat16);
EP_HOST_ASSERT(tensor_dtype == at::ScalarType::BFloat16);
EP_HOST_ASSERT(x.device().is_cuda() and x.device().index() == rank);
EP_HOST_ASSERT(weights.scalar_type() == at::ScalarType::BFloat16);
EP_HOST_ASSERT(cos_freqs.dim() == 4 and sin_freqs.dim() == 4);
// The kernel walks x and weights from one base pointer each, so a strided input is made contiguous,
// as ops/rms_norm_split_rope.cpp does for x.
if (!x.is_contiguous()) {
x = x.contiguous();
}
if (!weights.is_contiguous()) {
weights = weights.contiguous();
}
int b = x.size(0);
int s = x.size(1);
int h = x.size(2);
int n = cos_freqs.size(1);
EP_HOST_ASSERT(int64_t(b) * s * world_size * (h / world_size) * int64_t(elementSize(tensor_dtype)) <= tensor_bytes);
// Same freq-table handling as ops/rms_norm_split_rope.cpp: innermost dim contiguous, outer strides
// forwarded, a batch-shared table expanded (stride 0) rather than copied.
if (cos_freqs.stride(-1) != 1) {
cos_freqs = cos_freqs.contiguous();
}
if (sin_freqs.stride(-1) != 1) {
sin_freqs = sin_freqs.contiguous();
}
if (cos_freqs.size(0) == 1 && b > 1) {
cos_freqs = cos_freqs.expand({b, n, cos_freqs.size(2), cos_freqs.size(3)});
}
if (sin_freqs.size(0) == 1 && b > 1) {
sin_freqs = sin_freqs.expand({b, n, sin_freqs.size(2), sin_freqs.size(3)});
}
at::cuda::CUDAGuard device_guard{x.device()};
auto stream = at::cuda::getCurrentCUDAStream().stream();
all2all_cuda::rope_scatter_heads_launch(x.data_ptr(), sin_freqs.data_ptr(), cos_freqs.data_ptr(), weights.data_ptr(),
static_cast<float>(eps), b, s, n, h, cos_freqs.stride(0), cos_freqs.stride(1),
cos_freqs.stride(2), sin_freqs.stride(0), sin_freqs.stride(1),
sin_freqs.stride(2), buffer_ptrs_gpu, barrier_signal_ptrs_gpu, rank,
world_size, timeout_cycles_, barrier, stream);
}
void All2All::linear_send_recv_heads(at::Tensor &x, at::Tensor &weight, at::Tensor &bias, bool barrier) {
// A blockwise-fp8 exchange quantizes in its own copy kernel; this path writes the GEMM's bf16 output.
EP_HOST_ASSERT(tensor_dtype == at::ScalarType::BFloat16);
EP_HOST_ASSERT(x.dim() == 3 and x.scalar_type() == tensor_dtype);
EP_HOST_ASSERT(weight.dim() == 2 and weight.scalar_type() == tensor_dtype);
EP_HOST_ASSERT(bias.dim() == 1 and bias.scalar_type() == tensor_dtype);
EP_HOST_ASSERT(x.device().is_cuda() and x.device().index() == rank);
int b = x.size(0);
int s = x.size(1);
int h = weight.size(0);
EP_HOST_ASSERT(h % world_size == 0);
int64_t cols = h / world_size;
int64_t total_tokens = int64_t(s) * world_size;
EP_HOST_ASSERT(int64_t(b) * total_tokens * cols * int64_t(elementSize(tensor_dtype)) <= tensor_bytes);
at::cuda::CUDAGuard device_guard{x.device()};
auto stream = at::cuda::getCurrentCUDAStream().stream();
auto options = x.options();
// Peers first, so the NVLink-bound writes start as early as possible.
for (int step = 1; step <= world_size; step++) {
int target = (rank + step) % world_size;
auto w = weight.narrow(0, target * cols, cols);
auto bias_slice = bias.narrow(0, target * cols, cols);
for (int batch = 0; batch < b; batch++) {
int64_t offset = (int64_t(batch) * total_tokens + int64_t(rank) * s) * cols;
// A peer buffer is IPC-mapped into this device's address space; pinning the target device skips the
// pointer-owner lookup, which would report the peer.
auto out =
at::for_blob(static_cast<uint8_t *>(buffer_ptrs[target]) + offset * elementSize(tensor_dtype), {s, cols})
.options(options)
.target_device(x.device())
.make_tensor();
at::addmm_out(out, bias_slice, x.select(0, batch), w.t());
}
}
if (barrier) {
all2all_cuda::fused_barrier_launch(barrier_signal_ptrs_gpu, rank, world_size, timeout_cycles_, stream);
}
}
void All2All::exchange_barrier() {
at::cuda::CUDAGuard device_guard{static_cast<c10::DeviceIndex>(rank)};
auto stream = at::cuda::getCurrentCUDAStream().stream();
all2all_cuda::fused_barrier_launch(barrier_signal_ptrs_gpu, rank, world_size, timeout_cycles_, stream);
}
} // namespace all2all
} // namespace ltx_kernels
/**
* Python bindings for the All2All communication library.
*
* Usage from Python:
* import all2all_cpp
*
* # Create instance (one per GPU)
* comm = all2all_cpp.All2All(rank, world_size, max_tokens, hidden_dim, num_sms, dtype)
*
* # Exchange IPC handles and synchronize
* handle = comm.get_local_ipc_handle()
* # ... gather handles via NCCL ...
* comm.sync(all_handles)
*
* # Perform operations
* output = comm.send_recv_heads(input_tensor, copy_output=False)
*
* # Cleanup
* comm.destroy()
*/
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.doc() = "High-performance All2All communication library for multi-GPU tensor parallelism.\n\n"
"This library provides IPC-based All2All operations optimized for transformer models.\n"
"Supported operations:\n"
" - send_recv_heads: Redistribute attention heads across GPUs\n"
" - gather_heads: Inverse of send_recv_heads\n"
" - allgather: Gather sequence tokens from all ranks\n";
pybind11::class_<ltx_kernels::all2all::All2All>(
m, "All2All",
"Manages All2All communication state for multi-GPU operations.\n\n"
"Args:\n"
" rank: This GPU's rank (0 to world_size-1)\n"
" world_size: Total number of GPUs\n"
" num_tokens: Maximum tokens per rank\n"
" hidden_dim: Hidden dimension (heads * head_size)\n"
" num_sms: Number of SMs for kernel launches\n"
" tensor_dtype: Tensor data type (torch.bfloat16 or torch.float8_e4m3fn)\n"
" timeout_seconds: Optional initial barrier timeout in seconds (defaults to the kernel default)")
.def(pybind11::init<int, int, int, int, int, at::ScalarType>())
.def(pybind11::init<int, int, int, int, int, at::ScalarType, double>())
.def("get_local_ipc_handle", <x_kernels::all2all::All2All::get_local_ipc_handle,
"Returns the IPC handle for this rank's buffer.")
.def("sync", <x_kernels::all2all::All2All::sync, "Opens IPC mappings to all peer GPUs using gathered handles.")
.def("local_buffer", <x_kernels::all2all::All2All::local_buffer,
"Returns the local data buffer, which every operation writes its output into, as a flat tensor.")
.def("destroy", <x_kernels::all2all::All2All::destroy,
"Releases all GPU resources. Must be called before destruction.")
.def("send_recv_heads", <x_kernels::all2all::All2All::send_recv_heads,
"All2All operation to redistribute attention heads.")
.def("gather_heads", <x_kernels::all2all::All2All::gather_heads,
"Inverse All2All to gather heads back to original distribution.")
.def("allgather", <x_kernels::all2all::All2All::allgather, "Gathers sequence tokens from all ranks.")
.def("rope_send_recv_heads", <x_kernels::all2all::All2All::rope_send_recv_heads,
"Fused RMS-norm + split RoPE written straight into every rank's head-exchange buffer.")
.def("linear_send_recv_heads", <x_kernels::all2all::All2All::linear_send_recv_heads,
"Linear projection written straight into every rank's head-exchange buffer.")
.def("exchange_barrier", <x_kernels::all2all::All2All::exchange_barrier,
"Waits until every rank's earlier writes on this stream are visible to every rank.")
.def("set_timeout_seconds", <x_kernels::all2all::All2All::set_timeout_seconds,
"Sets the barrier timeout in seconds (converted to cycles via the device peak SM clock).");
}