Skip to content

armgit / LTX-2

Public Imported from GitHub
0

Official Python inference and LoRA trainer package for the LTX-2 audio–video generative model.

21.4 KB
Raw Blame
/**
 * @file all2all.cpp
 * @brief Implementation of All2All communication primitives for multi-GPU tensor parallelism.
 *
 * This file implements the All2All class which provides efficient inter-GPU communication
 * using CUDA IPC (Inter-Process Communication). The implementation supports:
 * - Head redistribution for tensor-parallel attention (send_recv_heads, gather_heads)
 * - Sequence gathering for cross-rank aggregation (allgather)
 *
 * All operations use a barrier-based synchronization protocol where each GPU writes
 * directly to remote GPU memory via IPC, then signals completion through atomic
 * operations on barrier counters.
 */

#include <ATen/cuda/CUDAContext.h>
#include <ATen/cuda/CUDADataType.h>
#include <c10/cuda/CUDAGuard.h>

#include <chrono>
#include <cuda_runtime.h>
#include <memory>
#include <pybind11/functional.h>
#include <torch/python.h>

#include "all2all.hpp"
#include "cuda/api.cuh"
#include "cuda/configs.cuh"

namespace ltx_kernels {
namespace all2all {

/**
 * Constructs the All2All communication manager.
 *
 * Memory Allocation Strategy:
 * The constructor allocates a single contiguous GPU memory block that contains:
 *   1. Data buffer (tensor_bytes): Space for tensor data exchange
 *   2. Barrier signals (MAX_NUM_PEERS * sizeof(int)): Per-rank completion counters
 *   3. Buffer pointers (MAX_NUM_PEERS * sizeof(void*)): GPU-accessible pointer array
 *   4. Barrier pointer array (MAX_NUM_PEERS * sizeof(int*)): GPU-accessible signal pointers
 *
 * This layout minimizes memory allocations and allows the entire region to be
 * shared via a single IPC handle.
 */
All2All::All2All(int rank, int world_size, int num_tokens, int hidden_dim, int num_sms, at::ScalarType tensor_dtype,
                 double timeout_seconds)
    : rank(rank), world_size(world_size), num_sms(num_sms), max_tokens(num_tokens), num_elems(0), tensor_bytes(0),
      tensor_dtype(tensor_dtype) {
  num_elems = int64_t(num_tokens) * int64_t(hidden_dim);
  tensor_bytes = num_elems * elementSize(tensor_dtype);

  // Derive the barrier timeout from the device's peak SM clock so the wall-clock guard is
  // correct on any GPU (the kernel counts SM cycles via clock64). Use cudaDeviceGetAttribute,
  // not cudaDeviceProp::clockRate, which was removed in CUDA 13. The attribute is in kHz.
  int device = 0;
  CUDA_CHECK(cudaGetDevice(&device));
  int sm_clock_khz = 0;
  CUDA_CHECK(cudaDeviceGetAttribute(&sm_clock_khz, cudaDevAttrClockRate, device));
  sm_clock_hz_ = static_cast<double>(sm_clock_khz) * 1e3;
  set_timeout_seconds(timeout_seconds);

  // Calculate sizes for each region of the shared memory block
  int64_t ptrs_bytes = MAX_NUM_PEERS * sizeof(void *);
  int64_t barrier_signal_bytes = kNumSignalSlots * sizeof(int);
  int64_t barrier_signal_ptrs_bytes = MAX_NUM_PEERS * sizeof(int *);

  // Allocate the main shared memory block and create IPC handle
  // Layout: [data_buffer | barrier_signals | buffer_ptrs | barrier_signal_ptrs]
  CUDA_CHECK(
      cudaMalloc(&buffer_ptrs[rank], tensor_bytes + barrier_signal_bytes + ptrs_bytes + barrier_signal_ptrs_bytes));
  CUDA_CHECK(cudaIpcGetMemHandle(&ipc_handlers[rank], buffer_ptrs[rank]));

  // Set up pointers to each region within the allocated block
  buffer_ptrs_gpu =
      reinterpret_cast<void **>(static_cast<uint8_t *>(buffer_ptrs[rank]) + tensor_bytes + barrier_signal_bytes);
  barrier_signal_ptrs[rank] = reinterpret_cast<int *>(static_cast<uint8_t *>(buffer_ptrs[rank]) + tensor_bytes);
  barrier_signal_ptrs_gpu = reinterpret_cast<int **>(static_cast<uint8_t *>(buffer_ptrs[rank]) + tensor_bytes +
                                                     barrier_signal_bytes + ptrs_bytes);

  // Initialize barrier signals to zero
  CUDA_CHECK(cudaMemset(barrier_signal_ptrs[rank], 0, barrier_signal_bytes));
}

All2All::~All2All() noexcept(false) {
  if (!destroyed) {
    printf("WARNING: destroy() was not called, which can leak resources.\n");
    fflush(stdout);
    destroy();
  }
}

/**
 * Releases all allocated resources.
 *
 * This must be called explicitly before destruction to ensure proper cleanup of:
 * - IPC memory mappings to remote GPUs
 * - Local GPU memory allocations
 *
 * The method synchronizes the device to ensure all pending operations complete
 * before releasing resources.
 */
void All2All::destroy() {
  if (destroyed) {
    return;
  }
  CUDA_CHECK(cudaDeviceSynchronize());

  // Close IPC mappings to remote GPU memory (skip our own rank)
  // Only close handles that were actually opened via sync()
  for (int i = 0; i < world_size; i++) {
    if (i != rank && buffer_ptrs[i] != nullptr) {
      CUDA_CHECK(cudaIpcCloseMemHandle(buffer_ptrs[i]));
    }
  }

  // Free local GPU memory allocations
  CUDA_CHECK(cudaFree(buffer_ptrs[rank]));
  destroyed = true;
}

/**
 * Opens IPC memory mappings to all peer GPUs.
 *
 * This method processes IPC handles gathered from all ranks and opens memory
 * mappings to enable direct GPU-to-GPU memory access. After calling this method,
 * each GPU can read/write directly to any other GPU's buffer via buffer_ptrs.
 *
 * The barrier_signal_ptrs are also set up to point to the correct offset within
 * each peer's shared memory block.
 */
void All2All::sync(const std::vector<std::optional<pybind11::bytearray>> &all_gathered_handles) {
  for (int i = 0; i < world_size; i++) {
    auto handle_str = std::string(all_gathered_handles[i].value());
    EP_HOST_ASSERT(handle_str.size() == CUDA_IPC_HANDLE_SIZE);

    if (i != rank) {
      // Open IPC mapping to remote GPU's memory
      std::memcpy(ipc_handlers[i].reserved, handle_str.c_str(), CUDA_IPC_HANDLE_SIZE);
      CUDA_CHECK(cudaIpcOpenMemHandle(&buffer_ptrs[i], ipc_handlers[i], cudaIpcMemLazyEnablePeerAccess));
      // Calculate offset to barrier signals in remote buffer
      barrier_signal_ptrs[i] = reinterpret_cast<int *>(static_cast<uint8_t *>(buffer_ptrs[i]) + tensor_bytes);
    } else {
      // Verify our own handle matches what we sent
      EP_HOST_ASSERT(std::memcmp(ipc_handlers[i].reserved, handle_str.c_str(), CUDA_IPC_HANDLE_SIZE) == 0);
    }
  }

  // Copy pointer arrays to GPU for kernel access
  CUDA_CHECK(cudaMemcpy(buffer_ptrs_gpu, buffer_ptrs, sizeof(void *) * world_size, cudaMemcpyHostToDevice));
  CUDA_CHECK(
      cudaMemcpy(barrier_signal_ptrs_gpu, barrier_signal_ptrs, sizeof(int *) * world_size, cudaMemcpyHostToDevice));
  CUDA_CHECK(cudaDeviceSynchronize());
}

pybind11::bytearray All2All::get_local_ipc_handle() const {
  return {ipc_handlers[rank].reserved, CUDA_IPC_HANDLE_SIZE};
}

at::Tensor All2All::local_buffer() const {
  return torch::from_blob(buffer_ptrs[rank], {num_elems},
                          torch::TensorOptions().dtype(tensor_dtype).device(torch::kCUDA, rank));
}

/**
 * Creates a tensor from the local IPC buffer.
 *
 * This helper method returns either a zero-copy view of the IPC buffer or
 * a newly allocated tensor with the data copied. The zero-copy mode is more
 * efficient but the tensor lifetime is tied to the All2All instance.
 *
 * @note The buffer pointer is cast to the template type T for proper interpretation.
 */
at::Tensor All2All::get_local_buffer_tensor(at::Tensor &x, int batch_size, int out_tokens, int out_heads, int head_size,
                                            bool should_copy, cudaStream_t stream) {
  auto ptr = buffer_ptrs[rank];
  if (should_copy) {
    // Allocate new tensor and copy data from IPC buffer
    auto out_tensor = torch::empty({batch_size, out_tokens, out_heads, head_size}, x.options());
    CUDA_CHECK(cudaMemcpyAsync(out_tensor.data_ptr(), ptr,
                               int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
                                   int64_t(elementSize(x.scalar_type())),
                               cudaMemcpyDeviceToDevice, stream));
    return out_tensor;
  } else {
    // Return a view directly into the IPC buffer (zero-copy)
    int64_t numel = int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size);
    return local_buffer().narrow(0, 0, numel).view({batch_size, out_tokens, out_heads, head_size});
  }
}

/**
 * All2All communication to redistribute attention heads across GPUs.
 *
 * This operation is used in tensor-parallel transformers to exchange attention heads:
 *   - Before: Each GPU has all tokens but only a subset of heads
 *   - After: Each GPU has all tokens with heads redistributed
 *
 * Tensor Layout Transformation:
 *   Input:  [batch, local_tokens, all_heads, head_size]  per GPU
 *   Output: [batch, all_tokens, heads_per_rank, head_size]  per GPU
 *
 * The operation partitions heads evenly: heads_per_rank = all_heads / world_size
 * GPU i receives heads [i*heads_per_rank : (i+1)*heads_per_rank] from all GPUs.
 */
at::Tensor All2All::send_recv_heads(at::Tensor &x, bool copy_output) {
  // Validate input tensor properties
  EP_HOST_ASSERT(x.dim() == 4 and x.is_contiguous());
  EP_HOST_ASSERT(x.dtype() == tensor_dtype);
  EP_HOST_ASSERT(x.device().is_cuda());
  EP_HOST_ASSERT(x.device().index() == rank);

  int batch_size = x.size(0);
  int num_tokens = x.size(1);
  int num_heads = x.size(2);
  int head_size = x.size(3);

  // Sequence parallelism pads each rank to the same token count.
  int out_tokens = num_tokens * world_size;
  int out_heads = num_heads / world_size; // Each rank gets 1/world_size of heads

  EP_HOST_ASSERT(int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
                     int64_t(elementSize(x.scalar_type())) <=
                 tensor_bytes);

  at::cuda::CUDAGuard device_guard{x.device()};
  auto stream = at::cuda::getCurrentCUDAStream().stream();

  // Launch the All2All kernel
  all2all_cuda::all2all_head_launch(buffer_ptrs_gpu, barrier_signal_ptrs_gpu, x.data_ptr(), rank, world_size,
                                    batch_size, out_tokens, num_tokens, num_heads, head_size, stream, num_sms,
                                    tensor_dtype, timeout_cycles_);

  return get_local_buffer_tensor(x, batch_size, out_tokens, out_heads, head_size, copy_output, stream);
}

/**
 * Inverse All2All to gather heads back to original distribution.
 *
 * This is the inverse operation of send_recv_heads(). It redistributes data
 * so each GPU gets back its original tokens with all attention heads.
 *
 * Tensor Layout Transformation:
 *   Input:  [batch, all_tokens, heads_per_rank, head_size]  per GPU
 *   Output: [batch, local_tokens, all_heads, head_size]  per GPU
 *
 * Each GPU sends its portion of tokens to the originating rank, reconstructing
 * the original head distribution.
 */
at::Tensor All2All::gather_heads(at::Tensor &x, bool copy_output) {
  // Validate input tensor properties
  EP_HOST_ASSERT(x.dim() == 4 and x.is_contiguous());
  EP_HOST_ASSERT(x.dtype() == tensor_dtype);
  EP_HOST_ASSERT(x.device().is_cuda());
  EP_HOST_ASSERT(x.device().index() == rank);

  at::cuda::CUDAGuard device_guard{x.device()};
  auto stream = at::cuda::getCurrentCUDAStream().stream();

  int batch_size = x.size(0);
  int total_tokens = x.size(1);
  int num_heads = x.size(2) * world_size; // Reconstruct total head count
  int head_size = x.size(3);

  EP_HOST_ASSERT(total_tokens % world_size == 0);
  int out_tokens = total_tokens / world_size;
  int out_heads = num_heads;

  EP_HOST_ASSERT(int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
                     int64_t(elementSize(x.scalar_type())) <=
                 tensor_bytes);

  // Launch the gather kernel
  all2all_cuda::all2all_head_gather_launch(buffer_ptrs_gpu, barrier_signal_ptrs_gpu, x.data_ptr(), rank, world_size,
                                           batch_size, total_tokens, num_heads, head_size, stream, num_sms,
                                           tensor_dtype, timeout_cycles_);

  return get_local_buffer_tensor(x, batch_size, out_tokens, out_heads, head_size, copy_output, stream);
}

/**
 * AllGather operation to collect sequence tokens from all ranks.
 *
 * Each GPU contributes its local sequence tokens, which are gathered into
 * a complete sequence replicated on all GPUs. This is typically used after
 * tensor-parallel operations to reconstruct the full sequence.
 *
 * Tensor Layout Transformation:
 *   Input:  [batch, local_seqlen, heads, head_size]  per GPU
 *   Output: [batch, total_seqlen, heads, head_size]  per GPU (identical on all GPUs)
 *
 * Each GPU's tokens are placed at offset rank * local_seqlen in the output.
 */
at::Tensor All2All::allgather(at::Tensor &x, bool copy_output) {
  // Validate input tensor properties
  EP_HOST_ASSERT(x.dim() == 4 and x.is_contiguous());
  EP_HOST_ASSERT(x.dtype() == tensor_dtype);
  EP_HOST_ASSERT(x.device().is_cuda());
  EP_HOST_ASSERT(x.device().index() == rank);

  at::cuda::CUDAGuard device_guard{x.device()};
  auto stream = at::cuda::getCurrentCUDAStream().stream();

  int batch_size = x.size(0);
  int seqlen = x.size(1);
  int num_heads = x.size(2);
  int head_size = x.size(3);

  int out_tokens = seqlen * world_size;
  int out_heads = num_heads;
  int hidden_dim = num_heads * head_size;

  EP_HOST_ASSERT(int64_t(batch_size) * int64_t(out_tokens) * int64_t(out_heads) * int64_t(head_size) *
                     int64_t(elementSize(x.scalar_type())) <=
                 tensor_bytes);

  // Launch the allgather kernel
  all2all_cuda::allgather_launch(buffer_ptrs_gpu, barrier_signal_ptrs_gpu, x.data_ptr(), rank, world_size, batch_size,
                                 seqlen, hidden_dim, out_tokens, stream, num_sms, tensor_dtype, timeout_cycles_);

  return get_local_buffer_tensor(x, batch_size, out_tokens, out_heads, head_size, copy_output, stream);
}

void All2All::rope_send_recv_heads(at::Tensor &x, at::Tensor &sin_freqs, at::Tensor &cos_freqs, at::Tensor &weights,
                                   double eps, bool barrier) {
  EP_HOST_ASSERT(x.dim() == 3 and x.scalar_type() == at::ScalarType::BFloat16);
  EP_HOST_ASSERT(tensor_dtype == at::ScalarType::BFloat16);
  EP_HOST_ASSERT(x.device().is_cuda() and x.device().index() == rank);
  EP_HOST_ASSERT(weights.scalar_type() == at::ScalarType::BFloat16);
  EP_HOST_ASSERT(cos_freqs.dim() == 4 and sin_freqs.dim() == 4);
  // The kernel walks x and weights from one base pointer each, so a strided input is made contiguous,
  // as ops/rms_norm_split_rope.cpp does for x.
  if (!x.is_contiguous()) {
    x = x.contiguous();
  }
  if (!weights.is_contiguous()) {
    weights = weights.contiguous();
  }
  int b = x.size(0);
  int s = x.size(1);
  int h = x.size(2);
  int n = cos_freqs.size(1);
  EP_HOST_ASSERT(int64_t(b) * s * world_size * (h / world_size) * int64_t(elementSize(tensor_dtype)) <= tensor_bytes);

  // Same freq-table handling as ops/rms_norm_split_rope.cpp: innermost dim contiguous, outer strides
  // forwarded, a batch-shared table expanded (stride 0) rather than copied.
  if (cos_freqs.stride(-1) != 1) {
    cos_freqs = cos_freqs.contiguous();
  }
  if (sin_freqs.stride(-1) != 1) {
    sin_freqs = sin_freqs.contiguous();
  }
  if (cos_freqs.size(0) == 1 && b > 1) {
    cos_freqs = cos_freqs.expand({b, n, cos_freqs.size(2), cos_freqs.size(3)});
  }
  if (sin_freqs.size(0) == 1 && b > 1) {
    sin_freqs = sin_freqs.expand({b, n, sin_freqs.size(2), sin_freqs.size(3)});
  }

  at::cuda::CUDAGuard device_guard{x.device()};
  auto stream = at::cuda::getCurrentCUDAStream().stream();
  all2all_cuda::rope_scatter_heads_launch(x.data_ptr(), sin_freqs.data_ptr(), cos_freqs.data_ptr(), weights.data_ptr(),
                                          static_cast<float>(eps), b, s, n, h, cos_freqs.stride(0), cos_freqs.stride(1),
                                          cos_freqs.stride(2), sin_freqs.stride(0), sin_freqs.stride(1),
                                          sin_freqs.stride(2), buffer_ptrs_gpu, barrier_signal_ptrs_gpu, rank,
                                          world_size, timeout_cycles_, barrier, stream);
}

void All2All::linear_send_recv_heads(at::Tensor &x, at::Tensor &weight, at::Tensor &bias, bool barrier) {
  // A blockwise-fp8 exchange quantizes in its own copy kernel; this path writes the GEMM's bf16 output.
  EP_HOST_ASSERT(tensor_dtype == at::ScalarType::BFloat16);
  EP_HOST_ASSERT(x.dim() == 3 and x.scalar_type() == tensor_dtype);
  EP_HOST_ASSERT(weight.dim() == 2 and weight.scalar_type() == tensor_dtype);
  EP_HOST_ASSERT(bias.dim() == 1 and bias.scalar_type() == tensor_dtype);
  EP_HOST_ASSERT(x.device().is_cuda() and x.device().index() == rank);
  int b = x.size(0);
  int s = x.size(1);
  int h = weight.size(0);
  EP_HOST_ASSERT(h % world_size == 0);
  int64_t cols = h / world_size;
  int64_t total_tokens = int64_t(s) * world_size;
  EP_HOST_ASSERT(int64_t(b) * total_tokens * cols * int64_t(elementSize(tensor_dtype)) <= tensor_bytes);

  at::cuda::CUDAGuard device_guard{x.device()};
  auto stream = at::cuda::getCurrentCUDAStream().stream();
  auto options = x.options();
  // Peers first, so the NVLink-bound writes start as early as possible.
  for (int step = 1; step <= world_size; step++) {
    int target = (rank + step) % world_size;
    auto w = weight.narrow(0, target * cols, cols);
    auto bias_slice = bias.narrow(0, target * cols, cols);
    for (int batch = 0; batch < b; batch++) {
      int64_t offset = (int64_t(batch) * total_tokens + int64_t(rank) * s) * cols;
      // A peer buffer is IPC-mapped into this device's address space; pinning the target device skips the
      // pointer-owner lookup, which would report the peer.
      auto out =
          at::for_blob(static_cast<uint8_t *>(buffer_ptrs[target]) + offset * elementSize(tensor_dtype), {s, cols})
              .options(options)
              .target_device(x.device())
              .make_tensor();
      at::addmm_out(out, bias_slice, x.select(0, batch), w.t());
    }
  }
  if (barrier) {
    all2all_cuda::fused_barrier_launch(barrier_signal_ptrs_gpu, rank, world_size, timeout_cycles_, stream);
  }
}

void All2All::exchange_barrier() {
  at::cuda::CUDAGuard device_guard{static_cast<c10::DeviceIndex>(rank)};
  auto stream = at::cuda::getCurrentCUDAStream().stream();
  all2all_cuda::fused_barrier_launch(barrier_signal_ptrs_gpu, rank, world_size, timeout_cycles_, stream);
}

} // namespace all2all
} // namespace ltx_kernels

/**
 * Python bindings for the All2All communication library.
 *
 * Usage from Python:
 *   import all2all_cpp
 *
 *   # Create instance (one per GPU)
 *   comm = all2all_cpp.All2All(rank, world_size, max_tokens, hidden_dim, num_sms, dtype)
 *
 *   # Exchange IPC handles and synchronize
 *   handle = comm.get_local_ipc_handle()
 *   # ... gather handles via NCCL ...
 *   comm.sync(all_handles)
 *
 *   # Perform operations
 *   output = comm.send_recv_heads(input_tensor, copy_output=False)
 *
 *   # Cleanup
 *   comm.destroy()
 */
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
  m.doc() = "High-performance All2All communication library for multi-GPU tensor parallelism.\n\n"
            "This library provides IPC-based All2All operations optimized for transformer models.\n"
            "Supported operations:\n"
            "  - send_recv_heads: Redistribute attention heads across GPUs\n"
            "  - gather_heads: Inverse of send_recv_heads\n"
            "  - allgather: Gather sequence tokens from all ranks\n";

  pybind11::class_<ltx_kernels::all2all::All2All>(
      m, "All2All",
      "Manages All2All communication state for multi-GPU operations.\n\n"
      "Args:\n"
      "    rank: This GPU's rank (0 to world_size-1)\n"
      "    world_size: Total number of GPUs\n"
      "    num_tokens: Maximum tokens per rank\n"
      "    hidden_dim: Hidden dimension (heads * head_size)\n"
      "    num_sms: Number of SMs for kernel launches\n"
      "    tensor_dtype: Tensor data type (torch.bfloat16 or torch.float8_e4m3fn)\n"
      "    timeout_seconds: Optional initial barrier timeout in seconds (defaults to the kernel default)")
      .def(pybind11::init<int, int, int, int, int, at::ScalarType>())
      .def(pybind11::init<int, int, int, int, int, at::ScalarType, double>())
      .def("get_local_ipc_handle", &ltx_kernels::all2all::All2All::get_local_ipc_handle,
           "Returns the IPC handle for this rank's buffer.")
      .def("sync", &ltx_kernels::all2all::All2All::sync, "Opens IPC mappings to all peer GPUs using gathered handles.")
      .def("local_buffer", &ltx_kernels::all2all::All2All::local_buffer,
           "Returns the local data buffer, which every operation writes its output into, as a flat tensor.")
      .def("destroy", &ltx_kernels::all2all::All2All::destroy,
           "Releases all GPU resources. Must be called before destruction.")
      .def("send_recv_heads", &ltx_kernels::all2all::All2All::send_recv_heads,
           "All2All operation to redistribute attention heads.")
      .def("gather_heads", &ltx_kernels::all2all::All2All::gather_heads,
           "Inverse All2All to gather heads back to original distribution.")
      .def("allgather", &ltx_kernels::all2all::All2All::allgather, "Gathers sequence tokens from all ranks.")
      .def("rope_send_recv_heads", &ltx_kernels::all2all::All2All::rope_send_recv_heads,
           "Fused RMS-norm + split RoPE written straight into every rank's head-exchange buffer.")
      .def("linear_send_recv_heads", &ltx_kernels::all2all::All2All::linear_send_recv_heads,
           "Linear projection written straight into every rank's head-exchange buffer.")
      .def("exchange_barrier", &ltx_kernels::all2all::All2All::exchange_barrier,
           "Waits until every rank's earlier writes on this stream are visible to every rank.")
      .def("set_timeout_seconds", &ltx_kernels::all2all::All2All::set_timeout_seconds,
           "Sets the barrier timeout in seconds (converted to cycles via the device peak SM clock).");
}