LLM inference in C/C++
C++ ggml
cuda: support Nemotron 3 Puzzle state size 96 for ssm scan (#28717)
Anav Prasad95887577ab5f
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex cf9a587f71..b50d05d5f7 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5487,8 +5487,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gif (op->src[3]->ne[0] == 1) {// Mamba2- // (kernel only supports (d_state == 128 || d_state == 256) && d_head % 16 == 0)- return (op->src[0]->ne[0] == 128 || op->src[0]->ne[0] == 256) && op->src[0]->ne[1] % 16 == 0;+ // (kernel only supports (d_state == 96 || d_state == 128 || d_state == 256) && d_head % 16 == 0)+ const int64_t d_state = op->src[0]->ne[0];+ return (d_state == 96 || d_state == 128 || d_state == 256) && op->src[0]->ne[1] % 16 == 0;} else {if (K > 1) {return false;diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cuindex e9a1043fb6..f6a41b4077 100644--- a/ggml/src/ggml-cuda/ssm-scan.cu+++ b/ggml/src/ggml-cuda/ssm-scan.cu@@ -163,13 +163,19 @@ __global__ void __launch_bounds__(d_state, 1)const int lane = threadIdx.x % WARP_SIZE;const int warp_idx = blockIdx.x * c_factor + warp;+ ggml_cuda_pdl_sync();++ // the last block can have unused warps when n_head*d_head is not a multiple of c_factor+ if (warp_idx >= n_head * d_head) {+ return;+ }+const int head_idx = warp_idx / d_head;const int head_off = (warp_idx % d_head) * sizeof(float);const int seq_idx = blockIdx.y;const int group_off = (head_idx / (n_head / n_group)) * d_state * sizeof(float);- ggml_cuda_pdl_sync();// TODO: refactor strides to be in elements/floats instead of bytes to be cleaner and consistent with the rest of the codebaseconst float * s0_warp = (const float *) ((const char *) src0 + src6[seq_idx] * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);const float * x_warp = (const float *) ((const char *) src1 + (seq_idx * src1_nb3) + (warp_idx * sizeof(float)));@@ -246,7 +252,17 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa// NOTE: if you change conditions here, be sure to update the corresponding supports_op condition!if (src3_nb1 == sizeof(float)) {// Mamba-2- if (d_state == 128) {+ if (d_state == 96) {+ constexpr int threads = 96;+ constexpr int num_warps = threads/WARP_SIZE;++ const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1);+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream);+ ggml_cuda_kernel_launch(ssm_scan_f32_group<96/WARP_SIZE, 96>, launch_params,+ src0, src1, src2, src3, src4, src5, src6, dst,+ src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,+ src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K);+ } else if (d_state == 128) {constexpr int threads = 128;constexpr int num_warps = threads/WARP_SIZE;@@ -267,7 +283,7 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floasrc0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K);} else {- GGML_ABORT("doesn't support d_state!=(128 or 256).");+ GGML_ABORT("doesn't support d_state!=(96, 128 or 256).");}} else {// Mamba-1diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex c15c5c8b1a..0ef71c67df 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9979,6 +9979,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 16, 1, 1024, 1, 32, 4)); // Mamba-1+ test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 96, 64, 128, 8, 1, 1)); // Nemotron-3-Puzzle decode (scan path, unused warp in last block)+ test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 96, 64, 16, 8, 300, 2, false, /*K=*/1, /*weak_decay=*/true)); // d_state=96 SSD multi-chunk (partial 2nd chunk, 2 seqs)+ test_cases.emplace_back(new test_ssm_scan_rollback(GGML_TYPE_F32, 96, 64, 16, 8, 8, 2, /*K=*/3)); // d_state=96 rollback snapshots match prefix statestest_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 32, 4)); // Mamba-2test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 256, 64, 8, 2, 32, 4)); // Falcon-H1test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 128, 4, 4, 16, 2, true)); // x/B/C overlap