Skip to content

armgit / LTX-2

Public Imported from GitHub
0

Official Python inference and LoRA trainer package for the LTX-2 audio–video generative model.

16.1 KB
Raw Blame
// NVFP4 block-scaled GEMM on cuBLASLt (SM >= 10.0).
//
// Row-major D(M,N) = alpha * A(M,K) @ B(N,K)^T + bias, all in FP4 E2M1 with 1x16 E4M3
// block scales. cuBLASLt is column-major, so we compute the transposed problem
//     D_col(N,M) = op_T(Bw)(N,K) @ op_N(Ax)(K,M)
// which is the TN layout FP4 requires: cuBLAS "A" is our weight, cuBLAS "B" our
// activation, and D_col(N,M) with ld=N *is* row-major D(M,N).
//
// Descriptors + heuristic are cached per (M,N,K,bias,out dtype); pointers are bound per
// call. `alpha` is a device fp32 scalar (CUBLASLT_POINTER_MODE_DEVICE) so a dynamic
// activation scale costs neither a host sync nor an extra pass over D.

#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include <cublasLt.h>

#include <cstdlib>
#include <mutex>
#include <sstream>
#include <unordered_map>

#include "nvfp4.h"

namespace ltx_nvfp4 {
namespace {

#define LTX_CUBLAS_CHECK(expr)                                                                                         \
  do {                                                                                                                 \
    cublasStatus_t status_ = (expr);                                                                                   \
    TORCH_CHECK(status_ == CUBLAS_STATUS_SUCCESS, "cuBLASLt call failed (", #expr, ") with status ",                   \
                static_cast<int>(status_));                                                                            \
  } while (0)

// 32 MiB matches torch's Lt workspace default. Override with LTX_NVFP4_WORKSPACE_MB to
// let the heuristic consider split-k algos with bigger scratch needs.
size_t workspace_bytes() {
  static const size_t bytes = [] {
    const char *env = std::getenv("LTX_NVFP4_WORKSPACE_MB");
    const long mb = env != nullptr ? std::strtol(env, nullptr, 10) : 32;
    return static_cast<size_t>(mb > 0 ? mb : 32) << 20;
  }();
  return bytes;
}

struct PlanKey {
  int64_t m, n, k;
  int out_dtype;
  bool has_bias;

  bool operator==(const PlanKey &o) const {
    return m == o.m && n == o.n && k == o.k && out_dtype == o.out_dtype && has_bias == o.has_bias;
  }
};

struct PlanKeyHash {
  size_t operator()(const PlanKey &p) const {
    size_t h = std::hash<int64_t>()(p.m);
    h = h * 1000003 ^ std::hash<int64_t>()(p.n);
    h = h * 1000003 ^ std::hash<int64_t>()(p.k);
    h = h * 1000003 ^ std::hash<int>()(p.out_dtype);
    h = h * 1000003 ^ std::hash<bool>()(p.has_bias);
    return h;
  }
};

constexpr int kMaxAlgos = 8;

struct Plan {
  cublasLtMatmulDesc_t op = nullptr;
  cublasLtMatrixLayout_t a = nullptr; // weight, (K, N) col-major
  cublasLtMatrixLayout_t b = nullptr; // activation, (K, M) col-major
  cublasLtMatrixLayout_t d = nullptr; // output, (N, M) col-major
  cublasLtMatmulAlgo_t algo{};
  size_t workspace = 0;
  // Remaining heuristic candidates, timed on the first eager call (cuBLASLt's own ranking
  // is not always right for these FP4 shapes; picking by measurement is worth a few %).
  cublasLtMatmulAlgo_t candidates[kMaxAlgos]{};
  int num_candidates = 0;
  bool tuned = false;
};

std::mutex g_mutex;
std::unordered_map<PlanKey, Plan, PlanKeyHash> g_plans;

// Persistent per-device scratch: [0] = beta (always 0).
torch::Tensor &beta_zero(const torch::Device &device) {
  static std::unordered_map<int, torch::Tensor> cache;
  auto it = cache.find(device.index());
  if (it == cache.end()) {
    it = cache.emplace(device.index(), torch::zeros({1}, torch::TensorOptions().dtype(torch::kFloat32).device(device)))
             .first;
  }
  return it->second;
}

torch::Tensor &workspace_for(const torch::Device &device) {
  static std::unordered_map<int, torch::Tensor> cache;
  auto it = cache.find(device.index());
  if (it == cache.end()) {
    it = cache
             .emplace(device.index(), torch::empty({static_cast<int64_t>(workspace_bytes())},
                                                   torch::TensorOptions().dtype(torch::kUInt8).device(device)))
             .first;
  }
  return it->second;
}

cudaDataType_t cuda_dtype(torch::ScalarType t) {
  switch (t) {
  case torch::kBFloat16:
    return CUDA_R_16BF;
  case torch::kHalf:
    return CUDA_R_16F;
  case torch::kFloat32:
    return CUDA_R_32F;
  default:
    TORCH_CHECK(false, "unsupported NVFP4 GEMM output dtype ", t);
  }
}

// Pointers cuBLASLt inspects at heuristic time (it derives alignment from them), rebound
// on every call.
struct Bindings {
  const void *a_scale = nullptr; // cuBLAS A == our weight
  const void *b_scale = nullptr;
  const void *bias = nullptr;
};

bool g_alpha_on_device = true;
bool g_autotune = false;

// Times every heuristic candidate once and keeps the fastest. Called at most once per
// cached plan, never during cudagraph capture.
template <typename RunFn> void tune_plan(Plan &plan, const RunFn &run, cudaStream_t stream) {
  cudaEvent_t start, stop;
  if (cudaEventCreate(&start) != cudaSuccess)
    return;
  if (cudaEventCreate(&stop) != cudaSuccess) {
    cudaEventDestroy(start);
    return;
  }
  float best_ms = -1.0f;
  int best = 0;
  for (int i = 0; i < plan.num_candidates; ++i) {
    if (run(plan.candidates[i], stream) != CUBLAS_STATUS_SUCCESS)
      continue; // warmup
    if (cudaEventRecord(start, stream) != cudaSuccess)
      break;
    bool ok = true;
    for (int rep = 0; rep < 3; ++rep) {
      ok = ok && run(plan.candidates[i], stream) == CUBLAS_STATUS_SUCCESS;
    }
    cudaEventRecord(stop, stream);
    if (cudaEventSynchronize(stop) != cudaSuccess || !ok)
      continue;
    float ms = 0.0f;
    if (cudaEventElapsedTime(&ms, start, stop) != cudaSuccess)
      continue;
    if (best_ms < 0.0f || ms < best_ms) {
      best_ms = ms;
      best = i;
    }
  }
  cudaEventDestroy(start);
  cudaEventDestroy(stop);
  plan.algo = plan.candidates[best];
}

Plan &get_plan(const PlanKey &key, torch::ScalarType out_dtype, cudaDataType_t bias_dtype, const Bindings &binds) {
  auto it = g_plans.find(key);
  if (it != g_plans.end())
    return it->second;

  Plan plan;
  const cudaDataType_t d_type = cuda_dtype(out_dtype);
  LTX_CUBLAS_CHECK(cublasLtMatmulDescCreate(&plan.op, CUBLAS_COMPUTE_32F, CUDA_R_32F));

  const cublasOperation_t op_t = CUBLAS_OP_T;
  const cublasOperation_t op_n = CUBLAS_OP_N;
  LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_TRANSA, &op_t, sizeof(op_t)));
  LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_TRANSB, &op_n, sizeof(op_n)));

  const int32_t scale_mode = CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3;
  LTX_CUBLAS_CHECK(
      cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_A_SCALE_MODE, &scale_mode, sizeof(scale_mode)));
  LTX_CUBLAS_CHECK(
      cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_B_SCALE_MODE, &scale_mode, sizeof(scale_mode)));

  LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &binds.a_scale,
                                                  sizeof(binds.a_scale)));
  LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &binds.b_scale,
                                                  sizeof(binds.b_scale)));

  if (g_alpha_on_device) {
    const int32_t pointer_mode = CUBLASLT_POINTER_MODE_DEVICE;
    LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_POINTER_MODE, &pointer_mode,
                                                    sizeof(pointer_mode)));
  }

  if (key.has_bias) {
    const cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_BIAS;
    LTX_CUBLAS_CHECK(
        cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)));
    LTX_CUBLAS_CHECK(
        cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_BIAS_DATA_TYPE, &bias_dtype, sizeof(bias_dtype)));
    LTX_CUBLAS_CHECK(
        cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &binds.bias, sizeof(binds.bias)));
  }

  LTX_CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&plan.a, CUDA_R_4F_E2M1, key.k, key.n, key.k));
  LTX_CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&plan.b, CUDA_R_4F_E2M1, key.k, key.m, key.k));
  LTX_CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&plan.d, d_type, key.n, key.m, key.n));

  cublasLtMatmulPreference_t pref = nullptr;
  LTX_CUBLAS_CHECK(cublasLtMatmulPreferenceCreate(&pref));
  size_t ws = workspace_bytes();
  LTX_CUBLAS_CHECK(
      cublasLtMatmulPreferenceSetAttribute(pref, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &ws, sizeof(ws)));

  cublasLtMatmulHeuristicResult_t heuristics[kMaxAlgos]{};
  int returned = 0;
  const cublasStatus_t heur_status =
      cublasLtMatmulAlgoGetHeuristic(at::cuda::getCurrentCUDABlasLtHandle(), plan.op, plan.a, plan.b, plan.d, plan.d,
                                     pref, kMaxAlgos, heuristics, &returned);
  cublasLtMatmulPreferenceDestroy(pref);
  TORCH_CHECK(heur_status == CUBLAS_STATUS_SUCCESS && returned > 0, "cuBLASLt found no NVFP4 GEMM algo for M=", key.m,
              " N=", key.n, " K=", key.k, " (status ", static_cast<int>(heur_status),
              "). NVFP4 needs SM >= 10.0, K % 32 == 0", " and N % 8 == 0.");
  plan.algo = heuristics[0].algo;
  plan.workspace = heuristics[0].workspaceSize;
  plan.num_candidates = returned;
  for (int i = 0; i < returned; ++i)
    plan.candidates[i] = heuristics[i].algo;

  return g_plans.emplace(key, plan).first->second;
}

} // namespace

torch::Tensor scaled_mm_nvfp4_cublaslt(const torch::Tensor &a, const torch::Tensor &b,
                                       const torch::Tensor &block_scale_a, const torch::Tensor &block_scale_b,
                                       const torch::Tensor &alpha, const std::optional<torch::Tensor> &bias,
                                       torch::Tensor &out) {
  TORCH_CHECK(a.is_cuda() && b.is_cuda(), "NVFP4 GEMM operands must be CUDA tensors");
  TORCH_CHECK(a.dim() == 2 && b.dim() == 2, "NVFP4 GEMM operands must be 2-D");
  TORCH_CHECK(a.is_contiguous() && b.is_contiguous(), "NVFP4 GEMM operands must be contiguous");
  TORCH_CHECK(a.size(1) == b.size(1), "packed K mismatch: ", a.size(1), " vs ", b.size(1));
  TORCH_CHECK(alpha.is_cuda() && alpha.scalar_type() == torch::kFloat32 && alpha.numel() == 1,
              "alpha must be a CUDA fp32 scalar");

  const int64_t m = a.size(0);
  const int64_t n = b.size(0);
  const int64_t k = a.size(1) * 2;
  TORCH_CHECK(k % 32 == 0, "NVFP4 GEMM needs K % 32 == 0, got K=", k);
  TORCH_CHECK(n % 8 == 0, "NVFP4 GEMM needs N % 8 == 0, got N=", n);
  TORCH_CHECK(out.size(0) == m && out.size(1) == n, "output shape mismatch");

  const c10::cuda::CUDAGuard guard(a.device());
  const bool has_bias = bias.has_value() && bias->numel() > 0;
  const cudaDataType_t bias_dtype = has_bias ? cuda_dtype(bias->scalar_type()) : CUDA_R_16BF;
  const PlanKey key{m, n, k, static_cast<int>(out.scalar_type()), has_bias};

  Bindings binds;
  binds.a_scale = block_scale_b.data_ptr(); // cuBLAS A == our weight
  binds.b_scale = block_scale_a.data_ptr();
  binds.bias = has_bias ? bias->data_ptr() : nullptr;

  std::lock_guard<std::mutex> lock(g_mutex);
  Plan &plan = get_plan(key, out.scalar_type(), bias_dtype, binds);

  LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &binds.a_scale,
                                                  sizeof(binds.a_scale)));
  LTX_CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &binds.b_scale,
                                                  sizeof(binds.b_scale)));
  if (has_bias) {
    LTX_CUBLAS_CHECK(
        cublasLtMatmulDescSetAttribute(plan.op, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &binds.bias, sizeof(binds.bias)));
  }

  // Host alpha fallback: cuBLASLt without device pointer mode gets alpha = 1 and the
  // per-tensor product is folded in afterwards by the caller.
  const float host_alpha = 1.0f;
  const float host_beta = 0.0f;
  const void *alpha_ptr = g_alpha_on_device ? alpha.data_ptr() : static_cast<const void *>(&host_alpha);
  const void *beta_ptr = g_alpha_on_device ? beta_zero(a.device()).data_ptr() : static_cast<const void *>(&host_beta);

  torch::Tensor &ws = workspace_for(a.device());
  auto run = [&](const cublasLtMatmulAlgo_t &algo, cudaStream_t stream) {
    return cublasLtMatmul(at::cuda::getCurrentCUDABlasLtHandle(), plan.op, alpha_ptr,
                          b.data_ptr(), // cuBLAS A: weight, (K, N) col-major
                          plan.a,
                          a.data_ptr(), // cuBLAS B: activation, (K, M) col-major
                          plan.b, beta_ptr, out.data_ptr(), plan.d, out.data_ptr(), plan.d, &algo, ws.data_ptr(),
                          workspace_bytes(), stream);
  };

  auto stream = at::cuda::getCurrentCUDAStream();
  if (g_autotune && !plan.tuned && plan.num_candidates > 1 &&
      at::cuda::currentStreamCaptureStatusMayInitCtx() == at::cuda::CaptureStatus::None) {
    plan.tuned = true;
    tune_plan(plan, run, stream);
  }
  LTX_CUBLAS_CHECK(run(plan.algo, stream));
  return out;
}

std::string gemm_alpha_mode() { return g_alpha_on_device ? "device_scalar" : "host_scalar"; }

void set_gemm_autotune(bool enabled) {
  std::lock_guard<std::mutex> lock(g_mutex);
  g_autotune = enabled;
  g_plans.clear();
}

void set_gemm_alpha_on_device(bool on_device) {
  std::lock_guard<std::mutex> lock(g_mutex);
  if (on_device != g_alpha_on_device) {
    g_plans.clear(); // pointer mode is baked into the cached descriptors
    g_alpha_on_device = on_device;
  }
}

// Diagnostic sweep: which cuBLASLt FP4 configurations this driver/library accepts.
std::string probe_gemm_support(int64_t m, int64_t n, int64_t k, int64_t scale_ptr_value) {
  std::ostringstream report;
  const void *fake_scale = reinterpret_cast<const void *>(scale_ptr_value);
  for (int pointer_dev = 0; pointer_dev < 2; ++pointer_dev) {
    for (int with_scales = 0; with_scales < 2; ++with_scales) {
      for (int with_scale_ptr = 0; with_scale_ptr < 2; ++with_scale_ptr) {
        cublasLtMatmulDesc_t op = nullptr;
        cublasLtMatrixLayout_t la = nullptr, lb = nullptr, ld = nullptr;
        cublasLtMatmulDescCreate(&op, CUBLAS_COMPUTE_32F, CUDA_R_32F);
        const cublasOperation_t op_t = CUBLAS_OP_T, op_n = CUBLAS_OP_N;
        cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_TRANSA, &op_t, sizeof(op_t));
        cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_TRANSB, &op_n, sizeof(op_n));
        if (with_scales) {
          const int32_t mode = CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3;
          cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_A_SCALE_MODE, &mode, sizeof(mode));
          cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_B_SCALE_MODE, &mode, sizeof(mode));
        }
        if (with_scale_ptr) {
          cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &fake_scale, sizeof(fake_scale));
          cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &fake_scale, sizeof(fake_scale));
        }
        if (pointer_dev) {
          const int32_t pm = CUBLASLT_POINTER_MODE_DEVICE;
          cublasLtMatmulDescSetAttribute(op, CUBLASLT_MATMUL_DESC_POINTER_MODE, &pm, sizeof(pm));
        }
        cublasLtMatrixLayoutCreate(&la, CUDA_R_4F_E2M1, k, n, k);
        cublasLtMatrixLayoutCreate(&lb, CUDA_R_4F_E2M1, k, m, k);
        cublasLtMatrixLayoutCreate(&ld, CUDA_R_16BF, n, m, n);

        cublasLtMatmulPreference_t pref = nullptr;
        cublasLtMatmulPreferenceCreate(&pref);
        size_t ws = workspace_bytes();
        cublasLtMatmulPreferenceSetAttribute(pref, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &ws, sizeof(ws));
        cublasLtMatmulHeuristicResult_t heur{};
        int returned = 0;
        const cublasStatus_t st = cublasLtMatmulAlgoGetHeuristic(at::cuda::getCurrentCUDABlasLtHandle(), op, la, lb, ld,
                                                                 ld, pref, 1, &heur, &returned);
        report << "alpha_device=" << pointer_dev << " scale_mode=" << with_scales << " scale_ptr=" << with_scale_ptr
               << " -> status=" << static_cast<int>(st) << " algos=" << returned << "\n";
        cublasLtMatmulPreferenceDestroy(pref);
        cublasLtMatrixLayoutDestroy(la);
        cublasLtMatrixLayoutDestroy(lb);
        cublasLtMatrixLayoutDestroy(ld);
        cublasLtMatmulDescDestroy(op);
      }
    }
  }
  return report.str();
}

} // namespace ltx_nvfp4