LLM inference in C/C++
C++ ggml
CUDA: refactor MMQ kernel configuration (#24127)
Johannes Gäßler6eddde06a4f2
diff --git a/ggml/src/ggml-cuda/mmq-config-ampere.cuh b/ggml/src/ggml-cuda/mmq-config-ampere.cuhnew file mode 100644index 0000000000..0037bac3d0--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-config-ampere.cuh@@ -0,0 +1,366 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_ampere(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/ggml/src/ggml-cuda/mmq-config-blackwell.cuh b/ggml/src/ggml-cuda/mmq-config-blackwell.cuhnew file mode 100644index 0000000000..9fbe32b697--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-config-blackwell.cuh@@ -0,0 +1,37 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_blackwell(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);++ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);++ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+}diff --git a/ggml/src/ggml-cuda/mmq-config-cdna.cuh b/ggml/src/ggml-cuda/mmq-config-cdna.cuhnew file mode 100644index 0000000000..46ec6aa9d5--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-config-cdna.cuh@@ -0,0 +1,177 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_cdna(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/ggml/src/ggml-cuda/mmq-config-pascal.cuh b/ggml/src/ggml-cuda/mmq-config-pascal.cuhnew file mode 100644index 0000000000..8f0faac889--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-config-pascal.cuh@@ -0,0 +1,261 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh b/ggml/src/ggml-cuda/mmq-config-rdna2.cuhnew file mode 100644index 0000000000..de4db0a3db--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh@@ -0,0 +1,261 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna2(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh b/ggml/src/ggml-cuda/mmq-config-rdna4.cuhnew file mode 100644index 0000000000..6280e80ee4--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh@@ -0,0 +1,282 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna4(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuhnew file mode 100644index 0000000000..3978b1baa7--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh@@ -0,0 +1,1671 @@+#pragma once++#include "vecdotq.cuh"++#include "mmq.cuh"++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0;+ constexpr int threads_per_row = blocks_per_iter * QI1_0;+ constexpr int nrows = warp_size / threads_per_row;+ constexpr int scale_entries_per_block = QK1_0 / QK8_1;+ constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;++ const int txi = threadIdx.x % threads_per_row;+ const int kbx = txi / QI1_0;+ const int kqsx = txi % QI1_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx;+ const int qs_offset = 4*kqsx;+ const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) |+ (bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24);++ int unpacked_bytes[8];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int shift = j * 4;+ const int bits4 = (qs0 >> shift) & 0x0F;+ const int b0 = (bits4 & 0x01) ? 1 : -1;+ const int b1 = (bits4 & 0x02) ? 1 : -1;+ const int b2 = (bits4 & 0x04) ? 1 : -1;+ const int b3 = (bits4 & 0x08) ? 1 : -1;+ unpacked_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24);+ }++ const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + dst_offset + j] = unpacked_bytes[j];+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j];+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }++ const int ksx = threadIdx.x % scale_entries_per_row;+ const int scale_block = ksx / scale_entries_per_block;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps) {+ int i = i0 + threadIdx.y;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + ksx] = bxi->d;+#else+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI4_0;+ const int kqsx = txi % QI4_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx;+ const int qs0 = get_int_b2(bxi->qs, kqsx);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + 0] = __vsubss4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808);+ x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsubss4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808);+#else+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = bxi->d;+#else+ x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI4_1;+ const int kqsx = txi % QI4_1;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx;+ const int qs0 = get_int_b4(bxi->qs, kqsx);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F;+ x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F;+#else+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_dm[i*sram_stride + kbxd] = bxi->dm;+#else+ x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI5_0;+ const int kqsx = txi % QI5_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx;++ const int ql = get_int_b2(bxi->qs, kqsx);+ const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx);++ int qs0 = (ql >> 0) & 0x0F0F0F0F;+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28+ qs0 = __vsubss4(qs0, 0x10101010); // subtract 16++ int qs1 = (ql >> 4) & 0x0F0F0F0F;+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28+ qs1 = __vsubss4(qs1, 0x10101010); // subtract 16++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + 0] = qs0;+ x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = bxi->d;+#else+ x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI5_1;+ const int kqsx = txi % QI5_1;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx;++ const int ql = get_int_b4(bxi->qs, kqsx);+ const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx);++ int qs0 = (ql >> 0) & 0x0F0F0F0F;+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28++ int qs1 = (ql >> 4) & 0x0F0F0F0F;+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + 0] = qs0;+ x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_dm[i*sram_stride + kbxd] = bxi->dm;+#else+ x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp+ constexpr int threads_per_row = 32;+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI8_0;+ const int kqsx = txi % QI8_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);+ x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = bxi->d;+#else+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K);+ constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride;++ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);++#pragma unroll+ for (int l = 0; l < QR2_K; ++l) {+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;++ const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k] = x_qs_k;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int sc_m = bxi->scales[kqsx];+#ifdef FAST_FP16_AVAILABLE+ const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4));+#else+ const float2 bxi_dmf = __half22float2(bxi->dm);+ const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4));+#endif // FAST_FP16_AVAILABLE++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_dm[i*sram_stride + kqsx] = x_dm_ik;+#else+ x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_df + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;++ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);+ const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2)));++#pragma unroll+ for (int l = 0; l < QR3_K; ++l) {+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;++ const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303;+ const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404;++ const int x_qs_k = __vsubss4(x_ql_k | x_qh_k, 0x04040404);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k] = x_qs_k;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }++ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;++ const int ksc = threadIdx.x % 4;++ const int ksc_low = ksc % (QI3_K/8);+ const int shift_low = 4 * (ksc / (QI3_K/8));+ const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;++ const int ksc_high = QI3_K/8;+ const int shift_high = 2 * ksc;+ const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;++ const int sc = __vsubss4(sc_low | sc_high, 0x20202020);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ const int8_t * sc8 = (const int8_t *) ≻+ const float d = bxi->d;++#pragma unroll+ for (int l = 0; l < int(sizeof(int)); ++l) {+ x_df[i*sram_stride + sizeof(int)*ksc + l] = d*sc8[l];+ }+#else+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE))+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;++ x_df[i] = bxi->d;+ }+#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE)+}++static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) {+ // scale arrangement after the following two lines:+ // - ksc == 0: sc0, sc1, sc2, sc3+ // - ksc == 1: sc4, sc5, sc6, sc7+ // - ksc == 2: m0, m1, m2, m3+ // - ksc == 3: m4, m5, m6, m7+ return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits+ ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_dm + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;+ const int qs0 = get_int_b4(bxi->qs, txi);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F;+ x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F;+#else+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int rows_per_warp = warp_size / 2;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ // Need if on AMD instead of % because warp_size == 64+ // This causes double work and throughput loss (MI300X)+ // H100 loses about 100 t/s with 'if' condition over '%'+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;+ if (i < I) {+#else+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I;+ {+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;++ const int * scales = (const int *) bxi->scales;+ const int ksc = threadIdx.x % 2;++ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);++ const uint8_t * sc8 = (const uint8_t *) &sc32;+ const uint8_t * m8 = (const uint8_t *) &m32;++ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);++ #pragma unroll+ for (int l = 0; l < sizeof(int); ++l) {+ x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);+ }+ }+ }+#else+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;++ x_dm[i] = bxi->dm;+ }+ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);++ const int * scales = (const int *) bxi->scales;++ const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);+ const int scales8 = unpack_scales_q45_K(scales, ksc);++ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_dm + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;+ const int ky = QR5_K*txi;++ const int ql = get_int_b4(bxi->qs, txi);+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;++ const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4));+ const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010;+ const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010;++ const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0;+ const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kq0] = ql0 | qh0;+ x_qs[i*sram_stride + kq1] = ql1 | qh1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int rows_per_warp = warp_size / 2;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+#if defined(AMD_MFMA_AVAILABLE)+ // Need if on AMD instead of % because warp_size == 64+ // This causes double work and throughput loss (MI300X)+ // H100 loses about 100 t/s with 'if' condition over '%'+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;+ if (i < I) {+#else+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I;+ {+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;++ const int * scales = (const int *) bxi->scales;+ const int ksc = threadIdx.x % 2;++ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);++ const uint8_t * sc8 = (const uint8_t *) &sc32;+ const uint8_t * m8 = (const uint8_t *) &m32;++ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);++#pragma unroll+ for (int l = 0; l < int(sizeof(int)); ++l) {+ x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);+ }+ }+ }+#else+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;++ x_dm[i] = bxi->dm;+ }++ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;++ const int * scales = (const int *) bxi->scales;++ const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);+ const int scales8 = unpack_scales_q45_K(scales, ksc);++ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+ int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_df + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;++ const int ql = get_int_b2(bxi->ql, txi);+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;++ const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4));+ const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030;+ const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030;++ const int kq0 = 2*txi - txi % (QI6_K/2) + 0;+ const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kq0] = __vsubss4(ql0 | qh0, 0x20202020);+ x_qs[i*sram_stride + kq1] = __vsubss4(ql1 | qh1, 0x20202020);+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride] = bxi->d;+#else+ x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8));+#else+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8));+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I);+ int * x_qs = (int *) x_tile;+ half2 * x_ds = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride;++ const int qs_packed = get_int_b2(bxi->qs, kqsx);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[kqsx];++ #pragma unroll+ for (int l = 0; l < QR1_S/2; ++l) {+ const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)];++ const int grid0 = (grid >> 0) & 0x0F0F0F0F;+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0;+ x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1);+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta);+#else+ x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride;++ const int q2 = get_int_b2(bxi->qs, 2*kqsx+0);+ const uint8_t * aux8 = (const uint8_t *) &q2;+ const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1);++#pragma unroll+ for (int l = 0; l < QR2_XXS; ++l) {+ const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]];+ const uint32_t signs = unpack_ksigns(aux32 >> (7 * l));++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride;++ const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));+ const uint16_t * q2 = (const uint16_t *) &q2_packed;++ #pragma unroll+ for (int l = 0; l < QR2_XS; ++l) {+ const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF];+ const uint32_t signs = unpack_ksigns(q2[l] >> 9);++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = bxi->scales[kqsx];+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#else+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride;++ const int qs_packed = get_int_b2(bxi->qs, kqsx);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[kqsx];++ const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx);+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;++#pragma unroll+ for (int l = 0; l < QR2_S; ++l) {+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300)));++ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);++ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = bxi->scales[kqsx];+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#else+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride;++ const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));+ const uint8_t * q3 = (const uint8_t *) &q3_packed;+ const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx);++#pragma unroll+ for (int l = 0; l < QR3_XXS; ++l) {+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]);+ const uint32_t signs = unpack_ksigns(aux32 >> (7*l));++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = aux32 >> 28;+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = (ls*d + d/2)/2;+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride;++ const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[kqsx];++ const int signs_packed_32 = get_int_b2(bxi->signs, kqsx);+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;++#pragma unroll+ for (int l = 0; l < QR3_S; ++l) {+ const int2 grid_pos = make_int2(+ iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)],+ iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]);++ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);++ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F);+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = ls*d;+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;++ const int aux_q4 = get_int_b4(bxi->qs, kqsx);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);+ const int k0 = 8 * (kqsx / 4) + kqsx % 4;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k0 + 0] = v.x;+ x_qs[i*sram_stride + k0 + 4] = v.y;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int rows_per_warp = warp_size / 8;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;++ const float d = __half2float(bxi->d);++ const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F)+ | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + threadIdx.x % 8] = d * (ls - 32);+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI4_NL;+ const int kqsx = txi % QI4_NL;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx;++ const int aux_q4 = get_int_b2(bxi->qs, kqsx);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);+ const int k0 = kbx * (2 * QI4_NL) + kqsx;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k0 + 0] = v.x;+ x_qs[i*sram_stride + k0 + QI4_NL] = v.y;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = __half2float(bxi->d);+#else+ x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI_MXFP4;+ const int kqsx = txi % QI_MXFP4;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx;++ const int aux_q4 = get_int_b1(bxi->qs, kqsx);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);+ const int k0 = kbx * (2 * QI_MXFP4) + kqsx;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k0 + 0] = v.x;+ x_qs[i*sram_stride + k0 + QI_MXFP4] = v.y;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;+#else+ x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++ int * x_qs = (int *) x_tile;+ uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);++ const int txi = threadIdx.x;++ constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback);++ constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block+ constexpr int rows_per_warp = warp_size / threads_per_row;+ const int kbx = txi % threads_per_row;+ const int row_in_warp = txi / threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;++ if constexpr (fallback) {+ i = min(i, i_max);+ }++ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx;++ // quantize_mxfp4_mmq permutes nibbles to match the quantized format+ const int k0 = kbx * 4;+ memcpy(x_qs + i*sram_stride + k0, bxi->qs, 16);++ // Load E8M0 scales: pack 2 consecutive scales into one uint32+ if (kbx % 2 == 0) {+ uint32_t e = bxi->e;+ e |= ((bxi + 1)->e << 8);+ x_sc[i*sram_stride + kbx / 2] = e;+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4;+ constexpr int rows_per_warp = warp_size / threads_per_row;+ const int kbx = threadIdx.x % threads_per_row;+ const int row_in_warp = threadIdx.x / threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;++ if constexpr (fallback) {+ i = min(i, i_max);+ }++ const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx;+ const uint32_t * __restrict__ src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);+ const int kqs = 16 * kbx;+ const int ksc = 4 * kbx;++#pragma unroll+ for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {+ const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4);+ const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kqs + 4 * sub + 0] = q0.x;+ x_qs[i*sram_stride + kqs + 4 * sub + 1] = q1.x;+ x_qs[i*sram_stride + kqs + 4 * sub + 2] = q0.y;+ x_qs[i*sram_stride + kqs + 4 * sub + 3] = q1.y;+ x_df[i*sram_stride + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);+#else+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x;+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x;+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y;+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y;+ x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback);+ constexpr int threads_per_row = iter_k / QK_NVFP4; // each thread processes 1 block+ constexpr int rows_per_warp = warp_size / threads_per_row;+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++ uint32_t * x_u32 = (uint32_t *) x_tile;++ const int txi = threadIdx.x;+ const int kbx = txi % threads_per_row;+ const int row_in_warp = txi / threads_per_row;++ const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx;+ uint32_t * x_u32_scale = x_u32 + 64 + kbx;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;++ if constexpr (fallback) {+ i = min(i, i_max);+ }++ const block_nvfp4 * bxi = bxi_base + i * stride;++ const uint32_t * src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);++#pragma unroll+ for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {+ x_u32[i*sram_stride + 8*kbx + 2 * sub + 0] = src_qs[2 * sub + 0];+ x_u32[i*sram_stride + 8*kbx + 2 * sub + 1] = src_qs[2 * sub + 1];+ }++ x_u32_scale[i*sram_stride] = get_int_b4(bxi->d, 0);+ }+}diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuhnew file mode 100644index 0000000000..d573433865--- /dev/null+++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh@@ -0,0 +1,1251 @@+#pragma once++#include "vecdotq.cuh"+#include "mma.cuh"++using namespace ggml_cuda_mma;++#include "mmq.cuh"++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);++ int u[2*VDR_Q4_0_Q8_1_MMQ];++ constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();+ constexpr int mcpy_int = max_cpy / sizeof(int);+ static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");++ int tmp0[4], tmp1[4];++ #pragma unroll+ for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {+ ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );+ ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]);+ }++ u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];+ u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>+ (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u,+ x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);++ int u[2*VDR_Q4_1_Q8_1_MMQ];++ constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();+ constexpr int mcpy_int = max_cpy / sizeof(int);+ static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");++ int tmp0[4], tmp1[4];++ #pragma unroll+ for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {+ ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );+ ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]);+ }++ u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];+ u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>+ (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u,+ x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMQ>+ (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K],+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback, mmq_q8_1_ds_layout ds_layout>+static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 8, int, input_layout> tile_A;+ typedef tile<16, 8, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;+ const half2 * y_ds = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ float dB;+ const int j = j0 + tile_C::get_j(0);+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {+ dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ } else {+ dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(l);+ const float dA = x_df[i*sram_stride + k0/QI8_0];+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB;+ }+ }+ }+ }+#else+ typedef tile<16, 8, int> tile_A;+ typedef tile< 8, 8, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;+ const half2 * y_ds = (const half2 *) y;++ tile_A A[ntx][MMQ_TILE_NE_K/QI8_0];+ float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0];++ const int i0 = (threadIdx.y/ntx)*rows_per_warp;++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++ load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++ dA[n][l][k01/QI8_0] = x_df[i*sram_stride + k0/QI8_0];+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ tile_B B;+ float dB[tile_C::ne/2];++ load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ } else {+ dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n][k01/QI8_0], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2];+ }+ }+ }+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}+++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>+ (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],+ x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 8, int, input_layout> tile_A;+ typedef tile<16, 8, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_dm = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(l);+ float2 dmA = __half22float2(x_dm[i*sram_stride + k0/QI8_1]);+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l];+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y;+ }+ }+ }+ }+#else+ typedef tile<16, 8, int> tile_A;+ typedef tile< 8, 8, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_dm = (const half2 *) y;++ tile_A A[ntx][MMQ_TILE_NE_K/QI8_1];+ float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1];++ const int i0 = (threadIdx.y/ntx)*rows_per_warp;++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*sram_stride + k0/QI8_1]);+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ tile_B B;+ float2 dsB[tile_C::ne/2];++ load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n][k01/QI8_1], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l];+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y;+ }+ }+ }+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}++// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl<QI8_0>(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0],+ &y_qs[j*MMQ_TILE_Y_K + k01],+ &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)],+ y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++// Used for Q3_K, IQ2_S, and IQ2_XS:+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 4, int, input_layout> tile_A;+ typedef tile<16, 4, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*sram_stride + k0/4] * dB;+ }+ }+ }+ }+#elif defined(TURING_MMA_AVAILABLE)++ typedef tile<16, 4, int> tile_A;+ typedef tile<16, 8, int> tile_A_8;+ typedef tile< 8, 4, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);++ tile_A A[ntx][8];+ float dA[ntx][tile_C::ne/2][8];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {+ const int k0 = k00 + k01;++ load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ dA[n][l][k01/4] = x_df[i*sram_stride + k0/4];+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {+ tile_B B[2];+ float dB[tile_C::ne/2];++ // Here load_generic is faster than load_ldmatrix.+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C[2];+ mma(C[0], A[n][k01/4 + 0], B[0]);+ mma(C[1], A[n][k01/4 + 1], B[1]);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]);+ }+ }+ }+ }+#else+ GGML_UNUSED_VARS(x, y, sum, k00);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++ float2 y_df[J/nwarps];+#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ constexpr int ns = 2;+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],+ &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);+ }+ }+ }++ // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop.+ // As a workaround 2 separate loops are used instead.+#pragma unroll+ for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ constexpr int ns = 1;+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],+ &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 4, int, input_layout> tile_A;+ typedef tile<16, 4, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y;+ const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0+ : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y+ : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x);++ tile_C Cm;+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tile_A A1;+#pragma unroll+ for (int l = 0; l < tile_A::ne; ++l) {+ A1.x[l] = 0x01010101;+ }+ mma(Cm, A1, B);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C Cd;+ mma(Cd, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);+ const float2 dm = __half22float2(x_dm[i*sram_stride + k0/4]);+ float tmp = Cd.x[l]*dm.x;+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tmp -= Cm.x[l]*dm.y;+ }+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB;+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB;+ }+ }+ }+ }+#elif defined(TURING_MMA_AVAILABLE)++ typedef tile<16, 4, int> tile_A;+ typedef tile<16, 8, int> tile_A_8;+ typedef tile< 8, 4, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);++ tile_A A[ntx][8];+ float dA[ntx][tile_C::ne/2][8];+ float mA[ntx][tile_C::ne/2][8];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) {+ const int k0 = k00 + k01;++ const float2 dm = __half22float2(x_dm[i*sram_stride + k0/(QI8_1/2)]);++ dA[n][l][k01/(QI8_1/2)] = dm.x;+ mA[n][l][k01/(QI8_1/2)] = dm.y;+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ float2 dB[tile_C::ne/2];++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ tile_B B[2];++ // Here load_generic is faster than load_ldmatrix.+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);++ tile_C Cm[2];+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tile_A A1;+ A1.x[0] = 0x01010101;+ A1.x[1] = 0x01010101;+ mma(Cm[0], A1, B[0]);+ mma(Cm[1], A1, B[1]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C Cd[2];++ mma(Cd[0], A[n][k01/4 + 0], B[0]);+ mma(Cd[1], A[n][k01/4 + 1], B[1]);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1];+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1];+ }+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y);+ }+ }+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) {+ float2 sB[tile_C::ne/2];++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x;+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y;+ }+ }+ }+ }+#else+ GGML_UNUSED_VARS(x, y, sum, k00);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_df + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales,+ x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_dm + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16);++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq(+ &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_dm + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16);++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq(+ &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_df + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]);++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq(+ &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc,+ x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 4, int, input_layout> tile_A;+ typedef tile<16, 4, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);+ const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16);+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB;+ }+ }+ }+ }+#elif defined(TURING_MMA_AVAILABLE)++ typedef tile<16, 4, int> tile_A;+ typedef tile< 8, 4, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);++ tile_A A[ntx][8];+ int scA[ntx][tile_C::ne/2][8];+ float dA[ntx][tile_C::ne/2];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {+ const int k0 = k00 + k01;++ load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + 0), sram_stride);+ load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + tile_A::J), sram_stride);+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++ const int sc_packed = x_sc[i*sram_stride + k0/16];+ const int8_t * sc = (const int8_t *) &sc_packed;++#pragma unroll+ for (int ksc = 0; ksc < sizeof(int); ++ksc) {+ scA[n][l][k01/4 + ksc] = sc[ksc];+ }+ }+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++ dA[n][l] = x_df[i*sram_stride];+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ float tmp[ntx][tile_C::ne] = {{0.0f}};++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {+ tile_B B[2];+ float dB[tile_C::ne/2];++ // Here load_generic is faster than load_ldmatrix.+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K);+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K);++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C[2];+ mma(C[0], A[n][k01/4 + 0], B[0]);+ mma(C[1], A[n][k01/4 + 1], B[1]);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2];+ }+ }+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2];+ }+ }+ }+#else+ GGML_UNUSED_VARS(x, y, sum, k00);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE+}++// ---------------------------------------------------------------------------------------------++// Shared MMA kernel for MXFP4 and NVFP4 on Blackwell.+// Both quantizations encode values as e2m1 (FP4) and produce one uint32 scale per+// m16n8k64 MMA call; only the PTX kind (scale_vec::2X ue8m0 vs scale_vec::4X ue4m3)+// and the per-type stride constant differ.+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_fp4_fp4_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {++ typedef tile<16, 8, int> tile_A;+ typedef tile<8, 8, int> tile_B;+ typedef tile<16, 8, float> tile_C;++ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp / tile_C::I;+ constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J;++ y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);+ const int * y_qs = (const int *) y + 4;+ const uint32_t * y_sc = (const uint32_t *) y;++ // 2 threads per quad supply the packed scale register to the block_scale MMA,+ // see https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling+ const int tidx_A = threadIdx.x / 4 + (threadIdx.x % 2) * 8;+ const int tidx_B = threadIdx.x / 4;+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ tile_A A[ntx][nfrags];+ uint32_t scaleA[ntx][nfrags];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int frag = 0; frag < nfrags; ++frag) {+ const int k0 = k00 + frag * tile_A::J;+ load_ldmatrix(A[n][frag], x_qs + (i0 + n * tile_A::I) * sram_stride + k0, sram_stride);+ scaleA[n][frag] = x_sc[(i0 + n * tile_A::I + tidx_A) * sram_stride + k0 / tile_A::J];+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx * tile_C::J) {+ tile_B B[nfrags];+ uint32_t scaleB[nfrags];++#pragma unroll+ for (int frag = 0; frag < nfrags; ++frag) {+ const int k0 = frag * tile_B::J;+ load_generic(B[frag], y_qs + j0 * MMQ_TILE_Y_K + k0, MMQ_TILE_Y_K);+ scaleB[frag] = y_sc[(j0 + tidx_B) * MMQ_TILE_Y_K + frag];+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int frag = 0; frag < nfrags; ++frag) {+ tile_C C = {};+ mma_block_scaled_fp4<type>(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]);+#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0 / tile_C::J + n) * tile_C::ne + l] += C.x[l];+ }+ }+ }+ }+}+diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cuindex 6b3b0d064a..bf9f5d5264 100644--- a/ggml/src/ggml-cuda/mmq.cu+++ b/ggml/src/ggml-cuda/mmq.cu@@ -3,6 +3,8 @@#include "quantize.cuh"#include "mmid.cuh"+#include <cstdint>+static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {switch (args.type_x) {case GGML_TYPE_Q1_0:@@ -118,15 +120,14 @@ void ggml_cuda_mul_mat_q(const int64_t s03 = src0->nb[3] / ts_src0;const int64_t s3 = dst->nb[3] / ts_dst;- const bool use_stream_k = (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA)- || GGML_CUDA_CC_IS_CDNA(cc);+ const bool fallback = ne01 % 128 != 0;// TODO: tighter pool buffer size vs q8 pathconst bool use_native_fp4 = blackwell_mma_available(cc) && (src0->type == GGML_TYPE_MXFP4 || src0->type == GGML_TYPE_NVFP4);if (!ids) {const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1 +- get_mmq_x_max_host(cc)*sizeof(block_q8_1_mmq);+ ggml_cuda_mmq_get_J_max(src0->type, fallback, cc, ne11) * sizeof(block_q8_1_mmq);ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);{@@ -156,7 +157,7 @@ void ggml_cuda_mul_mat_q(ne00, ne01, ne1, s01, ne11, s1,ne02, ne12, s02, s12, s2,ne03, ne13, s03, s13, s3,- use_stream_k, ne1};+ ne1};ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);return;}@@ -184,7 +185,7 @@ void ggml_cuda_mul_mat_q(}const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * sizeof(block_q8_1)/QK8_1 +- get_mmq_x_max_host(cc)*sizeof(block_q8_1_mmq);+ ggml_cuda_mmq_get_J_max(src0->type, fallback, cc, ne11) * sizeof(block_q8_1_mmq);ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);const int64_t ne11_flat = ne12*n_expert_used;@@ -217,51 +218,9 @@ void ggml_cuda_mul_mat_q(ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,ne02, ne02, s02, s12, s2,ne03, ne13, s03, s13, s3,- use_stream_k, ne12};-- ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);-}--void ggml_cuda_op_mul_mat_q(- ggml_backend_cuda_context & ctx,- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream) {-- const int64_t ne00 = src0->ne[0];-- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- GGML_ASSERT(ne10 % QK8_1 == 0);-- const int64_t ne0 = dst->ne[0];-- const int64_t row_diff = row_high - row_low;- const int64_t stride01 = ne00 / ggml_blck_size(src0->type);-- const int id = ggml_cuda_get_device();- const int cc = ggml_cuda_info().devices[id].cc;-- // the main device has a larger memory buffer to hold the results from all GPUs- // nrows_dst == nrows of the matrix that the kernel writes into- const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;-- // The stream-k decomposition is only faster for recent NVIDIA GPUs.- // Also its fixup needs to allocate a temporary buffer in the memory pool.- // There are multiple parallel CUDA streams for src1_ncols != ne11 which would introduce a race condition for this buffer.- const bool use_stream_k = ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA)- || GGML_CUDA_CC_IS_CDNA(cc))- && src1_ncols == ne11;- const mmq_args args = {- src0_dd_i, src0->type, (const int *) src1_ddq_i, nullptr, nullptr, dst_dd_i,- ne00, row_diff, src1_ncols, stride01, ne11, nrows_dst,- 1, 1, 0, 0, 0,- 1, 1, 0, 0, 0,- use_stream_k, src1_ncols};+ ne12};ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);-- GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_padded_row_size);}bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) {diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuhindex edf546d8f1..607e433bf1 100644--- a/ggml/src/ggml-cuda/mmq.cuh+++ b/ggml/src/ggml-cuda/mmq.cuh@@ -1,22 +1,18 @@#pragma once#include "common.cuh"-#include "vecdotq.cuh"-#include "mma.cuh"#include <climits>#include <cstdint>-using namespace ggml_cuda_mma;-#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available.#define MMQ_ITER_K 256#define MMQ_ITER_K_FP4 512#define MMQ_NWARPS 8-typedef void (*load_tiles_mmq_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);-typedef void (*vec_dot_mmq_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);-typedef void (*mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,+typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);+typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);+typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,float * __restrict__ dst, const int stride, const int i_max, const int j_max);enum mmq_q8_1_ds_layout {@@ -100,3098 +96,327 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {}}-struct tile_x_sizes {- int qs;- int dm;- int sc;-};--static int get_mmq_x_max_host(const int cc) {- return (turing_mma_available(cc) || amd_wmma_available(cc)) ? 128 :- GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA ?-#ifdef GGML_CUDA_FORCE_MMQ- 128 : 64;-#else- MMQ_DP4A_MAX_BATCH_SIZE : 64;-#endif // GGML_CUDA_FORCE_MMQ-}--static constexpr __device__ int get_mmq_x_max_device() {-#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- return 128;-#else // defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)--#if defined(GGML_USE_HIP)- return 64;-#else // defined(GGML_USE_HIP)--#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA-#ifdef GGML_CUDA_FORCE_MMQ- return 128;-#else // GGML_CUDA_FORCE_MMQ- return MMQ_DP4A_MAX_BATCH_SIZE;-#endif // GGML_CUDA_FORCE_MMQ-#else // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA- return 64;-#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA--#endif // defined(GGML_USE_HIP)-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-}--static int get_mmq_y_host(const int cc) {- return GGML_CUDA_CC_IS_AMD(cc) ? (GGML_CUDA_CC_IS_RDNA1(cc) ? 64 : 128) :- ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) ? 128 : 64);-}--static constexpr __device__ int get_iter_k([[maybe_unused]] const ggml_type type) {-#if defined(BLACKWELL_MMA_AVAILABLE)-if (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4) {- return MMQ_ITER_K_FP4;-}-#endif // defined(BLACKWELL_MMA_AVAILABLE)- return MMQ_ITER_K;-}--static constexpr __device__ int get_mmq_y_device() {-#if defined(GGML_USE_HIP)-#if defined(RDNA1)- return 64;-#else- return 128;-#endif // defined RDNA1-#else-#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA- return 128;-#else- return 64;-#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA-#endif // defined(GGML_USE_HIP)-}--// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes.-// The K dimension of the tiles has either,-// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K),-// 32 bit elements for the quantized data (does not include scales).-// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K.-// The final tile size in K direction is padded to avoid shared memory bank conflicts,-// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma.-#define MMQ_TILE_NE_K 32--#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_0 + mmq_y/QI4_0, 0}-#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_1 + mmq_y/QI4_1, 0}-#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*2/QI8_0 + mmq_y/(QI8_0/2), 0}-#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*4/QI8_0 + mmq_y/(QI8_0/4), 0}-#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*2/QI8_1 + mmq_y/(QI8_1/2), 0}-#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K + mmq_y, 0}-#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}-#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}-#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K/QI5_K + mmq_y/QI5_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}-#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K/QI6_K + mmq_y/QI6_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8}--static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int mmq_y) {- switch (type) {- case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;- case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;- case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1;- case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1;- case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16;- case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K;- case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K;- case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K;- case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K;- case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K;- case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;- case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16;- case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;- case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;- default: return tile_x_sizes{0, 0, 0};- }-}--#define MMQ_MMA_TILE_X_K_Q8_0 (2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4)-#define MMQ_MMA_TILE_X_K_FP4 (2*MMQ_TILE_NE_K + 8 + 4) // MXFP4 and NVFP4 Blackwell-#define MMQ_MMA_TILE_X_K_NVFP4 (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4) // NVFP4 Generic-#define MMQ_MMA_TILE_X_K_Q8_1 (2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4)-#define MMQ_MMA_TILE_X_K_Q2_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4)-#define MMQ_MMA_TILE_X_K_Q3_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4)-#define MMQ_MMA_TILE_X_K_Q6_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7)--static_assert(MMQ_MMA_TILE_X_K_Q8_0 % 8 == 4, "Wrong padding.");-static_assert(MMQ_MMA_TILE_X_K_Q8_1 % 8 == 4, "Wrong padding.");-static_assert(MMQ_MMA_TILE_X_K_Q2_K % 8 == 4, "Wrong padding.");-static_assert(MMQ_MMA_TILE_X_K_Q3_K % 8 == 4, "Wrong padding.");-static_assert(MMQ_MMA_TILE_X_K_Q6_K % 8 == 4, "Wrong padding.");-static_assert(MMQ_MMA_TILE_X_K_FP4 % 8 == 4, "Wrong padding.");-static_assert(MMQ_MMA_TILE_X_K_FP4 == MMQ_MMA_TILE_X_K_Q8_1, "Wrong tile size for MXFP4");-static_assert(MMQ_MMA_TILE_X_K_NVFP4 % 8 == 4, "Wrong padding.");---static constexpr __host__ __device__ int mmq_get_mma_tile_x_k(ggml_type type) {- switch (type) {- case GGML_TYPE_Q1_0: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_Q4_0: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_Q4_1: return MMQ_MMA_TILE_X_K_Q8_1;- case GGML_TYPE_Q5_0: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_Q5_1: return MMQ_MMA_TILE_X_K_Q8_1;- case GGML_TYPE_Q8_0: return MMQ_MMA_TILE_X_K_Q8_0;- // tile sizes are the same for Q8_1 and FP4 for blackwell- case GGML_TYPE_MXFP4: return MMQ_MMA_TILE_X_K_Q8_1;-#if defined(BLACKWELL_MMA_AVAILABLE)- case GGML_TYPE_NVFP4: return MMQ_MMA_TILE_X_K_FP4;-#else- case GGML_TYPE_NVFP4: return MMQ_MMA_TILE_X_K_NVFP4;-#endif // defined(BLACKWELL_MMA_AVAILABLE)- case GGML_TYPE_Q2_K: return MMQ_MMA_TILE_X_K_Q2_K;- case GGML_TYPE_Q3_K: return MMQ_MMA_TILE_X_K_Q3_K;- case GGML_TYPE_Q4_K: return MMQ_MMA_TILE_X_K_Q8_1;- case GGML_TYPE_Q5_K: return MMQ_MMA_TILE_X_K_Q8_1;- case GGML_TYPE_Q6_K: return MMQ_MMA_TILE_X_K_Q6_K;- case GGML_TYPE_IQ2_XXS: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_IQ2_XS: return MMQ_MMA_TILE_X_K_Q3_K;- case GGML_TYPE_IQ2_S: return MMQ_MMA_TILE_X_K_Q3_K;- case GGML_TYPE_IQ3_XXS: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_IQ3_S: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_IQ1_S: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_IQ4_XS: return MMQ_MMA_TILE_X_K_Q8_0;- case GGML_TYPE_IQ4_NL: return MMQ_MMA_TILE_X_K_Q8_0;- default: return 0;- }-}--// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales)-#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1)-#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K--static int mmq_get_granularity_host(const int mmq_x, const int cc) {- if (amd_mfma_available(cc) || amd_wmma_available(cc)) {- return mmq_x >= 128 ? 32 : 16;- } else if (turing_mma_available(cc) && mmq_x >= 48) {- return 16;- } else {- return 8;- }-}--#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-static constexpr __device__ int mmq_get_granularity_device(const int mmq_x) {- return mmq_x >= 128 ? 32 : 16;-}-#elif defined(TURING_MMA_AVAILABLE)-static constexpr __device__ int mmq_get_granularity_device(const int mmq_x) {- return mmq_x >= 48 ? 16 : 8;-}-#else-static constexpr __device__ int mmq_get_granularity_device(const int /*mmq_x*/) {- return 8;-}-#endif // AMD_MFMA_AVAILABLE--#if defined(GGML_USE_HIP)-static int mmq_get_nwarps_host(const int cc, const int warp_size) {- return amd_mfma_available(cc) ? 8 : 256/warp_size;-}-#else-static int mmq_get_nwarps_host(const int /*cc*/, const int warp_size) {- return 256/warp_size;-}-#endif // (GGML_USE_HIP)--static constexpr __device__ int mmq_get_nwarps_device() {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- return 8;-#else- return 256/ggml_cuda_get_physical_warp_size();-#endif // AMD_MFMA_AVAILABLE-}--// --------------------------------------------------------------template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q1_0(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0;- constexpr int threads_per_row = blocks_per_iter * QI1_0;- constexpr int nrows = warp_size / threads_per_row;- constexpr int scale_entries_per_block = QK1_0 / QK8_1;- constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;-- const int txi = threadIdx.x % threads_per_row;- const int kbx = txi / QI1_0;- const int kqsx = txi % QI1_0;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx;- const int qs_offset = 4*kqsx;- const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) |- (bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24);-- int unpacked_bytes[8];-#pragma unroll- for (int j = 0; j < 8; ++j) {- const int shift = j * 4;- const int bits4 = (qs0 >> shift) & 0x0F;- const int b0 = (bits4 & 0x01) ? 1 : -1;- const int b1 = (bits4 & 0x02) ? 1 : -1;- const int b2 = (bits4 & 0x04) ? 1 : -1;- const int b3 = (bits4 & 0x08) ? 1 : -1;- unpacked_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24);- }-- const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;-#pragma unroll- for (int j = 0; j < 8; ++j) {-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + dst_offset + j] = unpacked_bytes[j];-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j];-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }- }-- const int ksx = threadIdx.x % scale_entries_per_row;- const int scale_block = ksx / scale_entries_per_block;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + threadIdx.y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + ksx] = bxi->d;-#else- x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q4_0(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI4_0;- const int kqsx = txi % QI4_0;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx;- const int qs0 = get_int_b2(bxi->qs, kqsx);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + 0] = __vsubss4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808);- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsubss4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808);-#else- x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;-#else- x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q4_0_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y);- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + txs.qs;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;- const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);-- int u[2*VDR_Q4_0_Q8_1_MMQ];-- constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();- constexpr int mcpy_int = max_cpy / sizeof(int);- static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");-- int tmp0[4], tmp1[4];-- #pragma unroll- for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {- ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );- ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]);- }-- u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];- u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>- (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u,- x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q4_1(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y);- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI4_1;- const int kqsx = txi % QI4_1;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx;- const int qs0 = get_int_b4(bxi->qs, kqsx);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F;-#else- x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm;-#else- x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q4_1_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y);- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + txs.qs;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;- const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);-- int u[2*VDR_Q4_1_Q8_1_MMQ];-- constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();- constexpr int mcpy_int = max_cpy / sizeof(int);- static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");-- int tmp0[4], tmp1[4];-- #pragma unroll- for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {- ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );- ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]);- }-- u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];- u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>- (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u,- x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q5_0(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI5_0;- const int kqsx = txi % QI5_0;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx;-- const int ql = get_int_b2(bxi->qs, kqsx);- const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx);-- int qs0 = (ql >> 0) & 0x0F0F0F0F;- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28- qs0 = __vsubss4(qs0, 0x10101010); // subtract 16-- int qs1 = (ql >> 4) & 0x0F0F0F0F;- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28- qs1 = __vsubss4(qs1, 0x10101010); // subtract 16--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + 0] = qs0;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;-#else- x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q5_1(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y);- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI5_1;- const int kqsx = txi % QI5_1;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx;-- const int ql = get_int_b4(bxi->qs, kqsx);- const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx);-- int qs0 = (ql >> 0) & 0x0F0F0F0F;- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28-- int qs1 = (ql >> 4) & 0x0F0F0F0F;- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + 0] = qs0;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm;-#else- x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q8_0(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp- constexpr int threads_per_row = 32;- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI8_0;- const int kqsx = txi % QI8_0;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);- x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d;-#else- x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_mxfp4(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI_MXFP4;- const int kqsx = txi % QI_MXFP4;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx;-- const int aux_q4 = get_int_b1(bxi->qs, kqsx);- const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);- const int k0 = kbx * (2 * QI_MXFP4) + kqsx;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + k0 + 0] = v.x;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + k0 + QI_MXFP4] = v.y;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;-#else- x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check>-static __device__ __forceinline__ void load_tiles_mxfp4_fp4(const char * __restrict__ x,- int * __restrict__ x_tile,- const int kbx0,- const int i_max,- const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- int * x_qs = (int *) x_tile;- uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);-- const int txi = threadIdx.x;-- constexpr int iter_k = get_iter_k(GGML_TYPE_MXFP4);-- constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block- constexpr int rows_per_warp = warp_size / threads_per_row;- const int kbx = txi % threads_per_row;- const int row_in_warp = txi / threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += rows_per_warp * nwarps) {- int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;-- if constexpr (need_check) {- i = min(i, i_max);- }-- const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx;-- // quantize_mxfp4_mmq permutes nibbles to match the quantized format- const int k0 = kbx * 4;- memcpy(x_qs + i * MMQ_MMA_TILE_X_K_FP4 + k0, bxi->qs, 16);-- // Load E8M0 scales: pack 2 consecutive scales into one uint32- if (kbx % 2 == 0) {- uint32_t e = bxi->e;- e |= ((bxi + 1)->e << 8);- x_sc[i * MMQ_MMA_TILE_X_K_FP4 + kbx / 2] = e;- }- }-}--#ifdef BLACKWELL_MMA_AVAILABLE-template <int mmq_y, bool need_check>-static __device__ __forceinline__ void load_tiles_nvfp4_nvfp4(const char * __restrict__ x,- int * __restrict__ x_tile,- const int kbx0,- const int i_max,- const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();- constexpr int iter_k = get_iter_k(GGML_TYPE_NVFP4);- constexpr int threads_per_row = iter_k / QK_NVFP4; // each thread processes 1 block- constexpr int rows_per_warp = warp_size / threads_per_row;-- uint32_t * x_u32 = (uint32_t *) x_tile;-- const int txi = threadIdx.x;- const int kbx = txi % threads_per_row;- const int row_in_warp = txi / threads_per_row;-- const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx;- uint32_t * x_u32_scale = x_u32 + 64 + kbx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += rows_per_warp * nwarps) {- int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;-- if constexpr (need_check) {- i = min(i, i_max);- }-- const block_nvfp4 * bxi = bxi_base + i * stride;- const int row_base = i * MMQ_MMA_TILE_X_K_FP4;- const int q_base = row_base + 8 * kbx;-- const uint32_t * src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);--#pragma unroll- for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {- x_u32[q_base + 2 * sub + 0] = src_qs[2 * sub + 0];- x_u32[q_base + 2 * sub + 1] = src_qs[2 * sub + 1];- }-- x_u32_scale[row_base] = get_int_b4(bxi->d, 0);- }-}--// Shared MMA kernel for MXFP4 and NVFP4 on Blackwell.-// Both quantizations encode values as e2m1 (FP4) and produce one uint32 scale per-// m16n8k64 MMA call; only the PTX kind (scale_vec::2X ue8m0 vs scale_vec::4X ue4m3)-// and the per-type stride constant differ.-template <int mmq_x, int mmq_y, ggml_type type>-static __device__ __forceinline__ void vec_dot_fp4_fp4_mma(const int * __restrict__ x,- const int * __restrict__ y,- float * __restrict__ sum,- const int k00) {- static_assert(type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4,- "vec_dot_fp4_fp4_mma: type must be MXFP4 or NVFP4");-- typedef tile<16, 8, int> tile_A;- typedef tile<8, 8, int> tile_B;- typedef tile<16, 8, float> tile_C;-- constexpr int stride = MMQ_MMA_TILE_X_K_FP4;- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = 2 * granularity;- constexpr int ntx = rows_per_warp / tile_C::I;- constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J;-- y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);- const int * y_qs = (const int *) y + 4;- const uint32_t * y_sc = (const uint32_t *) y;-- // 2 threads per quad supply the packed scale register to the block_scale MMA,- // see https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling- const int tidx_A = threadIdx.x / 4 + (threadIdx.x % 2) * 8;- const int tidx_B = threadIdx.x / 4;- const int i0 = (threadIdx.y / ntx) * rows_per_warp;-- tile_A A[ntx][nfrags];- uint32_t scaleA[ntx][nfrags];--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int frag = 0; frag < nfrags; ++frag) {- const int k0 = k00 + frag * tile_A::J;- load_ldmatrix(A[n][frag], x_qs + (i0 + n * tile_A::I) * stride + k0, stride);- scaleA[n][frag] = x_sc[(i0 + n * tile_A::I + tidx_A) * stride + k0 / tile_A::J];- }- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx * tile_C::J) {- tile_B B[nfrags];- uint32_t scaleB[nfrags];--#pragma unroll- for (int frag = 0; frag < nfrags; ++frag) {- const int k0 = frag * tile_B::J;- load_generic(B[frag], y_qs + j0 * MMQ_TILE_Y_K + k0, MMQ_TILE_Y_K);- scaleB[frag] = y_sc[(j0 + tidx_B) * MMQ_TILE_Y_K + frag];- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int frag = 0; frag < nfrags; ++frag) {- tile_C C = {};- mma_block_scaled_fp4<type>(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]);-#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0 / tile_C::J + n) * tile_C::ne + l] += C.x[l];- }- }- }- }-}-#endif // BLACKWELL_MMA_AVAILABLE---template <int mmq_y, bool need_check>-static __device__ __forceinline__ void load_tiles_nvfp4(const char * __restrict__ x,- int * __restrict__ x_tile,- const int kb0,- const int i_max,- const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4;- constexpr int rows_per_warp = warp_size / threads_per_row;- const int kbx = threadIdx.x % threads_per_row;- const int row_in_warp = threadIdx.x / threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += rows_per_warp * nwarps) {- int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;-- if constexpr (need_check) {- i = min(i, i_max);- }-- const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx;- const uint32_t * __restrict__ src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);- const int kqs = 16 * kbx;- const int ksc = 4 * kbx;--#pragma unroll- for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {- const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4);- const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 0] = q0.x;- x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 1] = q1.x;- x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 2] = q0.y;- x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 3] = q1.y;- x_df[i * MMQ_MMA_TILE_X_K_NVFP4 + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);-#else- x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x;- x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x;- x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y;- x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y;- x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q8_0_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y);- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + txs.qs;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMQ>- (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K],- x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]);- }- }- }-}--template <int mmq_x, int mmq_y, mmq_q8_1_ds_layout ds_layout>-static __device__ __forceinline__ void vec_dot_q8_0_q8_1_mma(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr data_layout input_layout = get_input_data_layout();- typedef tile<16, 8, int, input_layout> tile_A;- typedef tile<16, 8, int, input_layout> tile_B;- typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;- const half2 * y_ds = (const half2 *) y;-- const int i0 = (threadIdx.y / ntx) * rows_per_warp;-- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {- const int k0 = k00 + k01;-- tile_A A[ntx];-#pragma unroll- for (int n = 0; n < ntx; ++n) {- load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0);- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- tile_B B;- load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);-- float dB;- const int j = j0 + tile_C::get_j(0);- if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {- dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];- } else {- dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C;- mma(C, A[n], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- const int i = i0 + n*tile_A::I + tile_C::get_i(l);- const float dA = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0];- sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB;- }- }- }- }-#else- typedef tile<16, 8, int> tile_A;- typedef tile< 8, 8, int> tile_B;- typedef tile<16, 8, int> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = 2 * granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;- const half2 * y_ds = (const half2 *) y;-- tile_A A[ntx][MMQ_TILE_NE_K/QI8_0];- float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0];-- const int i0 = (threadIdx.y/ntx)*rows_per_warp;--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {- const int k0 = k00 + k01;-- load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0);- }--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {- const int k0 = k00 + k01;-- dA[n][l][k01/QI8_0] = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0];- }- }- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {- tile_B B;- float dB[tile_C::ne/2];-- load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int j = j0 + tile_C::get_j(l);-- if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {- dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];- } else {- dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C;- mma(C, A[n][k01/QI8_0], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2];- }- }- }- }-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-}---template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q8_1_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y);- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + txs.qs;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>- (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],- x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q8_1_q8_1_mma(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr data_layout input_layout = get_input_data_layout();- typedef tile<16, 8, int, input_layout> tile_A;- typedef tile<16, 8, int, input_layout> tile_B;- typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;- const int * y_qs = (const int *) y + 4;- const half2 * y_dm = (const half2 *) y;-- const int i0 = (threadIdx.y / ntx) * rows_per_warp;-- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {- const int k0 = k00 + k01;-- tile_A A[ntx];-#pragma unroll- for (int n = 0; n < ntx; ++n) {- load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1);- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- tile_B B;- load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);-- const int j = j0 + tile_C::get_j(0);- const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C;- mma(C, A[n], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- const int i = i0 + n*tile_A::I + tile_C::get_i(l);- float2 dmA = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]);- sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l];- sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y;- }- }- }- }-#else- typedef tile<16, 8, int> tile_A;- typedef tile< 8, 8, int> tile_B;- typedef tile<16, 8, int> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = 2 * granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;- const int * y_qs = (const int *) y + 4;- const half2 * y_dm = (const half2 *) y;-- tile_A A[ntx][MMQ_TILE_NE_K/QI8_1];- float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1];-- const int i0 = (threadIdx.y/ntx)*rows_per_warp;--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {- const int k0 = k00 + k01;-- load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1);- }--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {- const int k0 = k00 + k01;-- dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]);- }- }- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {- tile_B B;- float2 dsB[tile_C::ne/2];-- load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int j = j0 + tile_C::get_j(l);-- dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C;- mma(C, A[n][k01/QI8_1], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l];- sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y;- }- }- }- }-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-}--// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS-template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16;- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + txs.qs;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl<QI8_0>(- &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0],- &y_qs[j*MMQ_TILE_Y_K + k01],- &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)],- y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--// Used for Q3_K, IQ2_S, and IQ2_XS:-template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_mma(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr data_layout input_layout = get_input_data_layout();- typedef tile<16, 4, int, input_layout> tile_A;- typedef tile<16, 4, int, input_layout> tile_B;- typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;-- const int i0 = (threadIdx.y / ntx) * rows_per_warp;-- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {- const int k0 = k00 + k01;-- tile_A A[ntx];-#pragma unroll- for (int n = 0; n < ntx; ++n) {- load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K);- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- tile_B B;- load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);-- const int j = j0 + tile_C::get_j(0);- const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C;- mma(C, A[n], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(l);- sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4] * dB;- }- }- }- }-#elif defined(TURING_MMA_AVAILABLE)-- typedef tile<16, 4, int> tile_A;- typedef tile<16, 8, int> tile_A_8;- typedef tile< 8, 4, int> tile_B;- typedef tile<16, 8, int> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = 2 * granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;-- const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);-- tile_A A[ntx][8];- float dA[ntx][tile_C::ne/2][8];--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {- const int k0 = k00 + k01;-- load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K);- }--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {- const int k0 = k00 + k01;-- dA[n][l][k01/4] = x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4];- }- }- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {- tile_B B[2];- float dB[tile_C::ne/2];-- // Here load_generic is faster than load_ldmatrix.- load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);- load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int j = j0 + tile_C::get_j(l);-- dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C[2];- mma(C[0], A[n][k01/4 + 0], B[0]);- mma(C[1], A[n][k01/4 + 1], B[1]);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]);- }- }- }- }-#else- GGML_UNUSED_VARS(x, y, sum, k00);- NO_DEVICE_CODE;-#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q2_K(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y);- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K);- constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride;-- const int x_ql_0 = get_int_b2(bxi->qs, kqsx);--#pragma unroll- for (int l = 0; l < QR2_K; ++l) {- const int k = (kqsx/8)*32 + l*8 + kqsx % 8;-- const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q2_K + k] = x_qs_k;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- const int sc_m = bxi->scales[kqsx];-#ifdef FAST_FP16_AVAILABLE- const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4));-#else- const float2 bxi_dmf = __half22float2(bxi->dm);- const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4));-#endif // FAST_FP16_AVAILABLE--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + kqsx] = x_dm_ik;-#else- x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q2_K_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y);- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + txs.qs;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;-- float2 y_df[mmq_x/nwarps];-#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;-- y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);- }--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- constexpr int ns = 2;- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(- &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],- &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,- &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);- }- }- }-- // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop.- // As a workaround 2 separate loops are used instead.-#pragma unroll- for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- constexpr int ns = 1;- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(- &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],- &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,- &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);- }- }- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q2_K_q8_1_mma(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr data_layout input_layout = get_input_data_layout();- typedef tile<16, 4, int, input_layout> tile_A;- typedef tile<16, 4, int, input_layout> tile_B;- typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;-- const int i0 = (threadIdx.y / ntx) * rows_per_warp;-- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {- const int k0 = k00 + k01;-- tile_A A[ntx];-#pragma unroll- for (int n = 0; n < ntx; ++n) {- load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K);- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- tile_B B;- load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);-- const int j = j0 + tile_C::get_j(0);- const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y;- const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0- : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y- : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x);-- tile_C Cm;- if (k01 >= MMQ_TILE_NE_K * 3/4) {- tile_A A1;-#pragma unroll- for (int l = 0; l < tile_A::ne; ++l) {- A1.x[l] = 0x01010101;- }- mma(Cm, A1, B);- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C Cd;- mma(Cd, A[n], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(l);- const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/4]);- float tmp = Cd.x[l]*dm.x;- if (k01 >= MMQ_TILE_NE_K * 3/4) {- tmp -= Cm.x[l]*dm.y;- }- sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB;- sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB;- }- }- }- }-#elif defined(TURING_MMA_AVAILABLE)-- typedef tile<16, 4, int> tile_A;- typedef tile<16, 8, int> tile_A_8;- typedef tile< 8, 4, int> tile_B;- typedef tile<16, 8, int> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = 2 * granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;-- const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);-- tile_A A[ntx][8];- float dA[ntx][tile_C::ne/2][8];- float mA[ntx][tile_C::ne/2][8];--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {- const int k0 = k00 + k01;-- load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K);- }- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) {- const int k0 = k00 + k01;-- const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/(QI8_1/2)]);-- dA[n][l][k01/(QI8_1/2)] = dm.x;- mA[n][l][k01/(QI8_1/2)] = dm.y;- }- }- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- float2 dB[tile_C::ne/2];--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int j = j0 + tile_C::get_j(l);-- dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);- }--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {- tile_B B[2];-- // Here load_generic is faster than load_ldmatrix.- load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);- load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);-- tile_C Cm[2];- if (k01 >= MMQ_TILE_NE_K * 3/4) {- tile_A A1;- A1.x[0] = 0x01010101;- A1.x[1] = 0x01010101;- mma(Cm[0], A1, B[0]);- mma(Cm[1], A1, B[1]);- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C Cd[2];-- mma(Cd[0], A[n][k01/4 + 0], B[0]);- mma(Cd[1], A[n][k01/4 + 1], B[1]);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1];- if (k01 >= MMQ_TILE_NE_K * 3/4) {- tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1];- }- sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y);- }- }- }--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) {- float2 sB[tile_C::ne/2];--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int j = j0 + tile_C::get_j(l);-- sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x;- sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y;- }- }- }- }-#else- GGML_UNUSED_VARS(x, y, sum, k00);- NO_DEVICE_CODE;-#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q3_K(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);- int * x_sc = (int *) (x_df + txs.dm);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K);- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;-- const int x_ql_0 = get_int_b2(bxi->qs, kqsx);- const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2)));--#pragma unroll- for (int l = 0; l < QR3_K; ++l) {- const int k = (kqsx/8)*32 + l*8 + kqsx % 8;-- const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303;- const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404;-- const int x_qs_k = __vsubss4(x_ql_k | x_qh_k, 0x04040404);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + k] = x_qs_k;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }- }-- constexpr int rows_per_warp = warp_size / 4;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {- int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4;-- if (need_check) {- i = min(i, i_max);- }-- const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;-- const int ksc = threadIdx.x % 4;-- const int ksc_low = ksc % (QI3_K/8);- const int shift_low = 4 * (ksc / (QI3_K/8));- const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;-- const int ksc_high = QI3_K/8;- const int shift_high = 2 * ksc;- const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;-- const int sc = __vsubss4(sc_low | sc_high, 0x20202020);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- const int8_t * sc8 = (const int8_t *) ≻- const float d = bxi->d;--#pragma unroll- for (int l = 0; l < int(sizeof(int)); ++l) {- x_df[i*MMQ_MMA_TILE_X_K_Q3_K + sizeof(int)*ksc + l] = d*sc8[l];- }-#else- x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }--#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE))-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {- int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;-- x_df[i] = bxi->d;- }-#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE)-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q3_K_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y);- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + txs.qs;- const int * x_sc = (const int *) x_df + txs.dm;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4;-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq(- &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales,- x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) {- // scale arrangement after the following two lines:- // - ksc == 0: sc0, sc1, sc2, sc3- // - ksc == 1: sc4, sc5, sc6, sc7- // - ksc == 2: m0, m1, m2, m3- // - ksc == 3: m4, m5, m6, m7- return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits- ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q4_K(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y);- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + txs.qs);- int * x_sc = (int *) (x_dm + txs.dm);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;- const int qs0 = get_int_b4(bxi->qs, txi);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F;-#else- x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr int rows_per_warp = warp_size / 2;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- // Need if on AMD instead of % because warp_size == 64- // This causes double work and throughput loss (MI300X)- // H100 loses about 100 t/s with 'if' condition over '%'- int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;- if (i < mmq_y) {-#else- int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % mmq_y;- {-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- if (need_check) {- i = min(i, i_max);- }-- const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;-- const int * scales = (const int *) bxi->scales;- const int ksc = threadIdx.x % 2;-- const int sc32 = unpack_scales_q45_K(scales, ksc + 0);- const int m32 = unpack_scales_q45_K(scales, ksc + 2);-- const uint8_t * sc8 = (const uint8_t *) &sc32;- const uint8_t * m8 = (const uint8_t *) &m32;-- const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);-- #pragma unroll- for (int l = 0; l < sizeof(int); ++l) {- x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);- }- }- }-#else-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {- int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;-- x_dm[i] = bxi->dm;- }- constexpr int rows_per_warp = warp_size / 4;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {- int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);-- const int * scales = (const int *) bxi->scales;-- const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);- const int scales8 = unpack_scales_q45_K(scales, ksc);-- x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;- }-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q4_K_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y);- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + txs.qs;- const int * x_sc = (const int *) x_dm + txs.dm;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16);-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq(- &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,- x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q5_K(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y);- int * x_qs = (int *) x_tile;- half2 * x_dm = (half2 *) (x_qs + txs.qs);- int * x_sc = (int *) (x_dm + txs.dm);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;- const int ky = QR5_K*txi;-- const int ql = get_int_b4(bxi->qs, txi);- const int ql0 = (ql >> 0) & 0x0F0F0F0F;- const int ql1 = (ql >> 4) & 0x0F0F0F0F;-- const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4));- const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010;- const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010;-- const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0;- const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq0] = ql0 | qh0;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq1] = ql1 | qh1;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr int rows_per_warp = warp_size / 2;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {-#if defined(AMD_MFMA_AVAILABLE)- // Need if on AMD instead of % because warp_size == 64- // This causes double work and throughput loss (MI300X)- // H100 loses about 100 t/s with 'if' condition over '%'- int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;- if (i < mmq_y) {-#else- int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % mmq_y;- {-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;-- const int * scales = (const int *) bxi->scales;- const int ksc = threadIdx.x % 2;-- const int sc32 = unpack_scales_q45_K(scales, ksc + 0);- const int m32 = unpack_scales_q45_K(scales, ksc + 2);-- const uint8_t * sc8 = (const uint8_t *) &sc32;- const uint8_t * m8 = (const uint8_t *) &m32;-- const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);--#pragma unroll- for (int l = 0; l < int(sizeof(int)); ++l) {- x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);- }- }- }-#else-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {- int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;-- x_dm[i] = bxi->dm;- }-- constexpr int rows_per_warp = warp_size / 4;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {- int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;-- const int * scales = (const int *) bxi->scales;-- const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);- const int scales8 = unpack_scales_q45_K(scales, ksc);-- x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;- }-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q5_K_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y);- const int * x_qs = (const int *) x;- const half2 * x_dm = (const half2 *) x_qs + txs.qs;- const int * x_sc = (const int *) x_dm + txs.dm;- const int * y_qs = (const int *) y + 4;- const half2 * y_ds = (const half2 *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16);-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq(- &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,- x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_q6_K(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);- int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);- int * x_sc = (int *) (x_df + txs.dm);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;-- const int ql = get_int_b2(bxi->ql, txi);- const int ql0 = (ql >> 0) & 0x0F0F0F0F;- const int ql1 = (ql >> 4) & 0x0F0F0F0F;-- const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4));- const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030;- const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030;-- const int kq0 = 2*txi - txi % (QI6_K/2) + 0;- const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq0] = __vsubss4(ql0 | qh0, 0x20202020);- x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq1] = __vsubss4(ql1 | qh1, 0x20202020);-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);- x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) {- int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q6_K] = bxi->d;-#else- x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int rows_per_warp = warp_size / 4;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) {- int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8));-#else- x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8));-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q6_K_q8_1_dp4a(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y);- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + txs.qs;- const int * x_sc = (const int *) x_df + txs.dm;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;--// #pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) {- const int k0 = k00 + k01;--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {- const int j = j0 + threadIdx.y;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {- const int i = i0 + threadIdx.x;-- const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]);-- sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq(- &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc,- x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);- }- }- }-}--template <int mmq_x, int mmq_y>-static __device__ __forceinline__ void vec_dot_q6_K_q8_1_mma(- const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr data_layout input_layout = get_input_data_layout();- typedef tile<16, 4, int, input_layout> tile_A;- typedef tile<16, 4, int, input_layout> tile_B;- typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;- const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;-- const int i0 = (threadIdx.y / ntx) * rows_per_warp;-- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {- const int k0 = k00 + k01;-- tile_A A[ntx];-#pragma unroll- for (int n = 0; n < ntx; ++n) {- load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + k0, MMQ_MMA_TILE_X_K_Q6_K);- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- tile_B B;- load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);-- const int j = j0 + tile_C::get_j(0);- const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C;- mma(C, A[n], B);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(l);- const int8_t * sc = (const int8_t *) (x_sc + i*MMQ_MMA_TILE_X_K_Q6_K + k00/16);- sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*MMQ_MMA_TILE_X_K_Q6_K] * dB;- }- }- }- }-#elif defined(TURING_MMA_AVAILABLE)-- typedef tile<16, 4, int> tile_A;- typedef tile< 8, 4, int> tile_B;- typedef tile<16, 8, int> tile_C;-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int rows_per_warp = 2 * granularity;- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.-- y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);-- const int * x_qs = (const int *) x;- const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;- const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;- const int * y_qs = (const int *) y + 4;- const float * y_df = (const float *) y;-- const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);-- tile_A A[ntx][8];- int scA[ntx][tile_C::ne/2][8];- float dA[ntx][tile_C::ne/2];--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {- const int k0 = k00 + k01;-- load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + 0), MMQ_MMA_TILE_X_K_Q6_K);- load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + tile_A::J), MMQ_MMA_TILE_X_K_Q6_K);- }--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) {- const int k0 = k00 + k01;--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);-- const int sc_packed = x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + k0/16];- const int8_t * sc = (const int8_t *) &sc_packed;--#pragma unroll- for (int ksc = 0; ksc < sizeof(int); ++ksc) {- scA[n][l][k01/4 + ksc] = sc[ksc];- }- }- }--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);-- dA[n][l] = x_df[i*MMQ_MMA_TILE_X_K_Q6_K];- }- }--#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {- float tmp[ntx][tile_C::ne] = {{0.0f}};--#pragma unroll- for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {- tile_B B[2];- float dB[tile_C::ne/2];-- // Here load_generic is faster than load_ldmatrix.- load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K);- load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K);--#pragma unroll- for (int l = 0; l < tile_C::ne/2; ++l) {- const int j = j0 + tile_C::get_j(l);-- dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {- tile_C C[2];- mma(C[0], A[n][k01/4 + 0], B[0]);- mma(C[1], A[n][k01/4 + 1], B[1]);--#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2];- }- }- }--#pragma unroll- for (int n = 0; n < ntx; ++n) {-#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2];- }- }- }-#else- GGML_UNUSED_VARS(x, y, sum, k00);- NO_DEVICE_CODE;-#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq4_nl(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL);- constexpr int nrows = warp_size / threads_per_row;- const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;- const int kbx = txi / QI4_NL;- const int kqsx = txi % QI4_NL;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);-- if (need_check) {- i = min(i, i_max);- }-- const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx;-- const int aux_q4 = get_int_b2(bxi->qs, kqsx);- const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);- const int k0 = kbx * (2 * QI4_NL) + kqsx;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + QI4_NL] = v.y;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL;- constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;- const int kbxd = threadIdx.x % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd;--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = __half2float(bxi->d);-#else- x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq2_xxs(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2;- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride;-- const int q2 = get_int_b2(bxi->qs, 2*kqsx+0);- const uint8_t * aux8 = (const uint8_t *) &q2;- const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1);--#pragma unroll- for (int l = 0; l < QR2_XXS; ++l) {- const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]];- const uint32_t signs = unpack_ksigns(aux32 >> (7 * l));-- const int signs0 = __vcmpne4(signs & 0x08040201, 0);- const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);-- const int signs1 = __vcmpne4(signs & 0x80402010, 0);- const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid0;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid1;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)- const float d = bxi->d;-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4-#else- x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq2_xs(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16;- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2;- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride;-- const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));- const uint16_t * q2 = (const uint16_t *) &q2_packed;-- #pragma unroll- for (int l = 0; l < QR2_XS; ++l) {- const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF];- const uint32_t signs = unpack_ksigns(q2[l] >> 9);-- const int signs0 = __vcmpne4(signs & 0x08040201, 0);- const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);-- const int signs1 = __vcmpne4(signs & 0x80402010, 0);- const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l;- x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- const int ls = bxi->scales[kqsx];- const float d = bxi->d;-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;- x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;-#else- x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;- x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-}--template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq2_s(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2;- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride;-- const int qs_packed = get_int_b2(bxi->qs, kqsx);- const uint8_t * qs = (const uint8_t *) &qs_packed;-- const int qh = bxi->qh[kqsx];-- const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx);- const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;--#pragma unroll- for (int l = 0; l < QR2_S; ++l) {- const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300)));+struct tile_x_sizes {+ int qs;+ int dm;+ int sc;+};- const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);- const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);+// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes.+// The K dimension of the tiles has either,+// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K),+// 32 bit elements for the quantized data (does not include scales).+// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K.+// The final tile size in K direction is padded to avoid shared memory bank conflicts,+// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma.+#define MMQ_TILE_NE_K 32- const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);+// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales)+#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1)+#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l;- x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }+enum ggml_cuda_mmq_sram_layout {+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K,+ GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell.+ GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4+};- const int ls = bxi->scales[kqsx];- const float d = bxi->d;-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;- x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_sram_layout sram_layout) {+ switch (sram_layout) {+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0:+ return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1:+ return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_1 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4:+ return 2*MMQ_TILE_NE_K + 8 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;+ default:+ return -1;+ }+}++static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding.");++static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) == ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1), "Wrong tile size for MXFP4");++// Config options for the MMQ kernel.+// Should not affect results, only speed/register pressure/shared memory use.+struct ggml_cuda_mmq_config {+ ggml_type type; // src0->type+ int nthreads; // Number of threads per CUDA block.+ int occupancy; // Targeted occupancy for the MMA kernel.+ int I; // SRAM tile width in src0->ne[1]/dst->ne[0] direction.+ int J; // SRAM tile width in src1->ne[1]/dst->ne[1] direction.+ ggml_cuda_mmq_sram_layout sram_layout; // SRAM tile length in src0->ne[0]/src1->ne[0] direction (physical 32 bit elements).+ int K_vram; // VRAM tile length in src0->ne[0]/src1->ne[0] direction (logical elements).+ bool stream_k; // Whether or not to use stream-k decomposition.+ bool fallback; // Whether a fallback for out-of-bounds check in src0->ne[1] direction is needed.++ constexpr __host__ __device__ ggml_cuda_mmq_config(+ ggml_type type, int nthreads, int occupancy, int I, int J, ggml_cuda_mmq_sram_layout sram_layout, int K_vram, bool stream_k, bool fallback) :+ type(type), nthreads(nthreads), occupancy(occupancy), I(I), J(J), sram_layout(sram_layout), K_vram(K_vram), stream_k(stream_k), fallback(fallback) {}++ constexpr __device__ int rows_per_warp() const {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ return 16;#else- x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;- x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ return J >= 48 && J % 16 == 0 ? 32 : 16;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)}-}-template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq3_xxs(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ // TODO transition all combinations of GPUs and quantizations to the MMA data layout.+ __host__ int use_mma_data_layout(const int cc) const {+ if (amd_mfma_available(cc) || amd_wmma_available(cc) || turing_mma_available(cc)) {+ return true;+ }+ return false;+ }-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+ constexpr __device__ bool use_mma_data_layout() const {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)+ return true;#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2;- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }+ return false;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)+ }- const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride;+};- const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));- const uint8_t * q3 = (const uint8_t *) &q3_packed;- const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx);+#define CASE(type_, nthreads_, occupancy_, I_, J_, sram_layout_, K_vram_, stream_k_, fallback_) \+ if (type == (type_) && J == (J_) && fallback == (fallback_)) { \+ static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \+ static_assert( (occupancy_) <= 8, "bad occupancy"); \+ static_assert((I_) % 32 == 0, "bad I"); \+ static_assert((J_) % 8 == 0, "bad J"); \+ static_assert((K_vram_) % 256 == 0, "bad K_vram"); \+ return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \+ } \-#pragma unroll- for (int l = 0; l < QR3_XXS; ++l) {- const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]);- const uint32_t signs = unpack_ksigns(aux32 >> (7*l));+#include "mmq-config-pascal.cuh"+#include "mmq-config-ampere.cuh"+#include "mmq-config-blackwell.cuh"- const int signs0 = __vcmpne4(signs & 0x08040201, 0);- const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);+#include "mmq-config-cdna.cuh"+#include "mmq-config-rdna2.cuh"+#include "mmq-config-rdna4.cuh"- const int signs1 = __vcmpne4(signs & 0x80402010, 0);- const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);+#undef CASE-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid_l;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid_h;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) {+ if (GGML_CUDA_CC_IS_AMD(cc)) {+ if (GGML_CUDA_CC_IS_CDNA(cc)) {+ return ggml_cuda_mmq_get_config_cdna(type, J, fallback);}-- const int ls = aux32 >> 28;- const float d = bxi->d;-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = (ls*d + d/2)/2;-#else- x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ if (amd_wmma_available(cc)) {+ return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);+ }+ return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);}+ if (blackwell_mma_available(cc)) {+ return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);+ }+ if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) {+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+ }+ return ggml_cuda_mmq_get_config_pascal(type, J, fallback);}-template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq3_s(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) {+#ifdef GGML_USE_HIP+#ifdef CDNA+ return ggml_cuda_mmq_get_config_cdna(type, J, fallback);+#elif defined(AMD_WMMA_AVAILABLE)+ return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)-- constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2;- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride;-- const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));- const uint8_t * qs = (const uint8_t *) &qs_packed;-- const int qh = bxi->qh[kqsx];-- const int signs_packed_32 = get_int_b2(bxi->signs, kqsx);- const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;--#pragma unroll- for (int l = 0; l < QR3_S; ++l) {- const int2 grid_pos = make_int2(- iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)],- iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]);-- const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);- const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);-- const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);- const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+0)] = grid_l;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+1)] = grid_h;+ return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);+#endif // CDNA#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }-- const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F);- const float d = bxi->d;-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = ls*d;+#ifdef BLACKWELL_MMA_AVAILABLE+ return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);+#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);#else- x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }+ return ggml_cuda_mmq_get_config_pascal(type, J, fallback);+#endif // BLACKWELL_MMA_AVAILABLE+#endif // GGML_USE_HIP+ GGML_UNUSED_VARS(type, J, fallback);}-template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq1_s(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y);- int * x_qs = (int *) x_tile;- half2 * x_ds = (half2 *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).type;+}- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S);- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;+static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).type;+}-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) {- int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;+static __host__ int ggml_cuda_mmq_get_nthreads(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).nthreads;+}- if (need_check) {- i = min(i, i_max);- }+static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).nthreads;+}- const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride;+static __host__ int ggml_cuda_mmq_get_occupancy(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).occupancy;+}- const int qs_packed = get_int_b2(bxi->qs, kqsx);- const uint8_t * qs = (const uint8_t *) &qs_packed;+static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).occupancy;+}- const int qh = bxi->qh[kqsx];+static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).I;+}- #pragma unroll- for (int l = 0; l < QR1_S/2; ++l) {- const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)];+static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).I;+}- const int grid0 = (grid >> 0) & 0x0F0F0F0F;- const int grid1 = (grid >> 4) & 0x0F0F0F0F;+static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).J;+}-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+0)] = grid0;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+1)] = grid1;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }+static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).J;+}- const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1);- const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);+static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).sram_layout;+}-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_ds[i*MMQ_MMA_TILE_X_K_Q8_1 + kqsx] = make_half2(d1q, d1q*delta);-#else- x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }+static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).sram_layout;}-template <int mmq_y, bool need_check> static __device__ __forceinline__ void load_tiles_iq4_xs(- const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {- constexpr int nwarps = mmq_get_nwarps_device();- constexpr int warp_size = ggml_cuda_get_physical_warp_size();+static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).K_vram;+}-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);-#else- constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, mmq_y);- int * x_qs = (int *) x_tile;- float * x_df = (float *) (x_qs + txs.qs);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).K_vram;+}- constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS);- constexpr int nrows = warp_size / threads_per_row;- const int kqsx = threadIdx.x % threads_per_row;+static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).stream_k;+}-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) {- int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);+static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).stream_k;+}- if (need_check) {- i = min(i, i_max);- }+static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc).fallback;+}- const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;+static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).fallback;+}- const int aux_q4 = get_int_b4(bxi->qs, kqsx);- const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);- const int k0 = 8 * (kqsx / 4) + kqsx % 4;+// ----------------------------------------------------------------------------------------------#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x;- x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 4] = v.y;-#else- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;- x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- }+static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc) {+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc));+}- constexpr int rows_per_warp = warp_size / 8;-#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) {- int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4);+static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback));+}- if (need_check) {- i = min(i, i_max);+static __host__ int ggml_cuda_mmq_get_J_max(const ggml_type type, const bool fallback, const int cc, const int64_t ne11) {+ int ret = std::min(ne11, int64_t(512));+ ret -= ret % 8;+ for (;ret > 0; ret -= 8) {+ if (ggml_cuda_mmq_get_config(type, ret, fallback, cc).type != GGML_TYPE_COUNT) {+ return ret;}+ }+ return ret;+}- const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;+static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback) {+ return ggml_cuda_mmq_get_config(type, J, fallback).rows_per_warp();+}- const float d = __half2float(bxi->d);+#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0}+#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_1 + I/QI4_1, 0}+#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_0 + I/(QI8_0/2), 0}+#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*4/QI8_0 + I/(QI8_0/4), 0}+#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_1 + I/(QI8_1/2), 0}+#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K + I, 0}+#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I, I*MMQ_TILE_NE_K/8 + I/8}+#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_K, I*MMQ_TILE_NE_K/8 + I/8}+#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI5_K + I/QI5_K, I*MMQ_TILE_NE_K/8 + I/8}+#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI6_K + I/QI6_K, I*MMQ_TILE_NE_K/8 + I/8}- const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F)- | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4);+static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) {+ switch (type) {+ case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;+ case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;+ case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1;+ case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1;+ case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K;+ case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K;+ case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K;+ case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K;+ case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K;+ case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;+ default: return tile_x_sizes{0, 0, 0};+ }+}-#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + threadIdx.x % 8] = d * (ls - 32);-#else- x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+// FIXME temporary until all combinations of data types and GPUs can use the MMA data layout+static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config & config, const int cc) {+ if (config.use_mma_data_layout(cc)) {+ return config.I * ggml_cuda_mmq_get_sram_stride(config.sram_layout) * 4;}+ const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(config.type, config.I);+ return (txs.qs + txs.dm + txs.sc) * 4;}-template<int mmq_x, int mmq_y, bool need_check>-static __device__ __forceinline__ void mmq_write_back_dp4a(+// ------------------------------------------------------------++#include "mmq-load-tiles.cuh"+#include "mmq-vec-dot.cuh"++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,const int stride, const int i_max, const int j_max) {- constexpr int nwarps = mmq_get_nwarps_device();constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {+ for (int j0 = 0; j0 < J; j0 += nwarps) {const int j = j0 + threadIdx.y;if (j > j_max) {@@ -3199,45 +424,38 @@ static __device__ __forceinline__ void mmq_write_back_dp4a(}#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += warp_size) {+ for (int i0 = 0; i0 < I; i0 += warp_size) {const int i = i0 + threadIdx.x;- if (need_check && i > i_max) {+ if (fallback && i > i_max) {continue;}- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size];+ dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];}}}-template<ggml_type type, int mmq_x, int mmq_y, bool need_check>-static __device__ __forceinline__ void mmq_write_back_mma(- const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,- const int stride, const int i_max, const int j_max) {-- constexpr int granularity = mmq_get_granularity_device(mmq_x);- constexpr int nwarps = mmq_get_nwarps_device();-+template<ggml_type type, int J, bool fallback>+static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(+ const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,+ const int stride, const int i_max, const int j_max) {#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr int tileC_IJ = mmq_get_granularity_device(0);- typedef tile<tileC_IJ, tileC_IJ, int, DATA_LAYOUT_J_MAJOR> tile_C;- constexpr int rows_per_warp = granularity;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;#else- typedef tile<16, 8, int> tile_C;- constexpr int rows_per_warp = 2 * granularity;-#endif // defined(AMD_MFMA_AVAILABLE)- constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.+ typedef tile<16, 8, int> tile_C;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I);-#if defined(TURING_MMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- static_assert(nwarps*tile_C::I == mmq_y, "nwarps*tile_C::I != mmq_y");-#else- GGML_UNUSED(nwarps);-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) {+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {#pragma unrollfor (int n = 0; n < ntx; ++n) {#pragma unroll@@ -3250,7 +468,7 @@ static __device__ __forceinline__ void mmq_write_back_mma(const int i = i0 + n*tile_C::I + tile_C::get_i(l);- if (need_check && i > i_max) {+ if (fallback && i > i_max) {continue;}@@ -3262,188 +480,339 @@ static __device__ __forceinline__ void mmq_write_back_mma(// --------------------------------------------------------------------------------------------------------------------------------------template <int mmq_x, int mmq_y, bool need_check, ggml_type type>-struct mmq_type_traits;--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q1_0> {- static constexpr int vdr = VDR_Q1_0_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q1_0<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q4_0> {- static constexpr int vdr = VDR_Q4_0_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_0<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_DS4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_0_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q4_1> {- static constexpr int vdr = VDR_Q4_1_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_1<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_1_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q5_0> {- static constexpr int vdr = VDR_Q5_0_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_0<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q5_1> {- static constexpr int vdr = VDR_Q5_1_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_1<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a<mmq_x, mmq_y>;-};+// TODO remove this struct and use ggml_cuda_mmq_sram_layout instead.+struct ggml_cuda_mmq_util_funcs {+ int vdr;+ ggml_cuda_mmq_load_tiles_t load_tiles;+ ggml_cuda_mmq_vec_dot_t vec_dot;+ ggml_cuda_mmq_write_back_t write_back;-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q8_0> {- static constexpr int vdr = VDR_Q8_0_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q8_0<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;+ constexpr __host__ __device__ ggml_cuda_mmq_util_funcs(+ int vdr, ggml_cuda_mmq_load_tiles_t load_tiles, ggml_cuda_mmq_vec_dot_t vec_dot, ggml_cuda_mmq_write_back_t write_back) :+ vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {}};-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_MXFP4> {- static constexpr int vdr = VDR_MXFP4_Q8_1_MMQ;-#ifdef BLACKWELL_MMA_AVAILABLE- static constexpr load_tiles_mmq_t load_tiles = load_tiles_mxfp4_fp4<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_fp4_fp4_mma<mmq_x, mmq_y, GGML_TYPE_MXFP4>;-#else- static constexpr load_tiles_mmq_t load_tiles = load_tiles_mxfp4<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;-#endif // BLACKWELL_MMA_AVAILABLE- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};+template <ggml_type type, int J, bool fallback>+static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() {+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);++ if (!ggml_cuda_mmq_get_config(type, J, fallback).use_mma_data_layout()) {+ switch (type) {+ case GGML_TYPE_Q1_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q1_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q4_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q4_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q4_1:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q4_1_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q5_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q5_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q5_1:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q5_1_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q8_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q8_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_Q2_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q2_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q3_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q3_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q4_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q4_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q5_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q5_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_Q6_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q6_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_IQ1_S:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ1_S_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ2_XXS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_XXS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ2_XS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_XS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ2_S:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_S_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ3_XXS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ3_XXS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ3_S:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ3_S_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ4_XS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ4_XS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_IQ4_NL:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ4_NL_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_MXFP4:+ return ggml_cuda_mmq_util_funcs(+ VDR_MXFP4_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ case GGML_TYPE_NVFP4:+ return ggml_cuda_mmq_util_funcs(+ VDR_NVFP4_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);+ default:+ return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);+ }+ }++// ----------------------------------------------------------------------------------------------template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_NVFP4> {- static constexpr int vdr = VDR_NVFP4_Q8_1_MMQ;#ifdef BLACKWELL_MMA_AVAILABLE- static constexpr load_tiles_mmq_t load_tiles = load_tiles_nvfp4_nvfp4<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_fp4_fp4_mma<mmq_x, mmq_y, GGML_TYPE_NVFP4>;-#else- static constexpr load_tiles_mmq_t load_tiles = load_tiles_nvfp4<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y>;+ switch (type) {+ case GGML_TYPE_MXFP4:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_mxfp4_fp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_NVFP4:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ default:+ break;+ }#endif // BLACKWELL_MMA_AVAILABLE- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a<mmq_x, mmq_y>;-};-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q2_K> {- static constexpr int vdr = VDR_Q2_K_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q2_K<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q2_K_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q2_K_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q3_K> {- static constexpr int vdr = VDR_Q3_K_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q3_K<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q3_K_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q4_K> {- static constexpr int vdr = VDR_Q4_K_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_K<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_K_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q5_K> {- static constexpr int vdr = VDR_Q5_K_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_K<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q5_K_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_Q6_K> {- static constexpr int vdr = VDR_Q6_K_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_q6_K<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q6_K_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q6_K_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ2_XXS> {- static constexpr int vdr = VDR_IQ2_XXS_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xxs<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};--template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ2_XS> {- static constexpr int vdr = VDR_IQ2_XS_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xs<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a<mmq_x, mmq_y>;-};+// ----------------------------------------------------------------------------------------------template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ2_S> {- static constexpr int vdr = VDR_IQ2_S_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_s<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a<mmq_x, mmq_y>;-};+ switch (type) {+ case GGML_TYPE_Q1_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q4_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_DS4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q4_1:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q5_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q5_1:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q8_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_Q2_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q3_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q4_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q5_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_Q6_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_IQ1_S:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ2_XXS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ2_XS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ2_S:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ3_XXS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ3_S:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ4_XS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_IQ4_NL:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_MXFP4:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ case GGML_TYPE_NVFP4:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback>);+ default:+ return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);+ }+}-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ3_XXS> {- static constexpr int vdr = VDR_IQ3_XXS_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_xxs<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};+template <ggml_type type, int J, bool fallback>+static constexpr __device__ int ggml_cuda_mmq_get_vdr() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().vdr;+}-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ3_S> {- static constexpr int vdr = VDR_IQ3_S_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_s<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};+template <ggml_type type, int J, bool fallback>+static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().load_tiles;+}-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ1_S> {- static constexpr int vdr = VDR_IQ1_S_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq1_s<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma<mmq_x, mmq_y>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a<mmq_x, mmq_y>;-};+template <ggml_type type, int J, bool fallback>+static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().vec_dot;+}-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ4_NL> {- static constexpr int vdr = VDR_IQ4_NL_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_nl<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};+template <ggml_type type, int J, bool fallback>+static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().write_back;+}-template <int mmq_x, int mmq_y, bool need_check>-struct mmq_type_traits<mmq_x, mmq_y, need_check, GGML_TYPE_IQ4_XS> {- static constexpr int vdr = VDR_IQ4_XS_Q8_1_MMQ;- static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_xs<mmq_y, need_check>;- static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma<mmq_x, mmq_y, MMQ_Q8_1_DS_LAYOUT_D4>;- static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a<mmq_x, mmq_y>;-};+// ----------------------------------------------------------------------------------------------template <ggml_type type, int mmq_x, bool need_check, bool fixup>+template <ggml_type type, int J, bool fallback, bool fixup>static __device__ __forceinline__ void mul_mat_q_process_tile(const char * __restrict__ x, const int offset_x, const int * __restrict__ y,const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,@@ -3451,22 +820,16 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {constexpr int warp_size = ggml_cuda_get_physical_warp_size();- constexpr int nwarps = mmq_get_nwarps_device();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;constexpr int qk = ggml_cuda_type_traits<type>::qk;- constexpr int mmq_y = get_mmq_y_device();- constexpr load_tiles_mmq_t load_tiles = mmq_type_traits<mmq_x, mmq_y, need_check, type>::load_tiles;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback>();+ constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot<type, J, fallback>();+ constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback>();extern __shared__ int data_mul_mat_q[];- int * tile_y = data_mul_mat_q + mmq_x;- int * tile_x = tile_y + GGML_PAD(mmq_x*MMQ_TILE_Y_K, nwarps*warp_size);--#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)- constexpr vec_dot_mmq_t vec_dot = mmq_type_traits<mmq_x, mmq_y, need_check, type>::vec_dot_mma;- constexpr mmq_write_back_t write_back = mmq_write_back_mma<type, mmq_x, mmq_y, need_check>;-#else- constexpr vec_dot_mmq_t vec_dot = mmq_type_traits<mmq_x, mmq_y, need_check, type>::vec_dot_dp4a;- constexpr mmq_write_back_t write_back = mmq_write_back_dp4a<mmq_x, mmq_y, need_check>;-#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * tile_y = data_mul_mat_q + J;+ int * tile_x = tile_y + GGML_PAD(J*MMQ_TILE_Y_K, nwarps*warp_size);#if defined(BLACKWELL_MMA_AVAILABLE)// FP4 tile stores 8 blocks@@ -3475,10 +838,10 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(constexpr int ne_block = 4 * QK8_1;#endif // defined(BLACKWELL_MMA_AVAILABLE)- constexpr int ITER_K = get_iter_k(type);+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback);constexpr int blocks_per_iter = ITER_K / qk;- float sum[mmq_x*mmq_y / (nwarps*warp_size)] = {0.0f};+ float sum[J*I / (nwarps*warp_size)] = {0.0f};constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);@@ -3487,7 +850,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile({const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz;#pragma unroll- for (int l0 = 0; l0 < mmq_x * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {+ for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {int l = l0 + threadIdx.y*warp_size + threadIdx.x;tile_y[l] = by0[l];@@ -3503,7 +866,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile({const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz);#pragma unroll- for (int l0 = 0; l0 < mmq_x * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {+ for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {int l = l0 + threadIdx.y*warp_size + threadIdx.x;tile_y[l] = by0[l];@@ -3518,7 +881,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(}if (fixup) {- write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(mmq_x*mmq_y), mmq_y, mmq_y, mmq_x);+ write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), I, I, J);} else {write_back(sum, ids_dst, dst, stride_col_dst, tile_x_max_i, tile_y_max_j);}@@ -3527,18 +890,8 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598-template <ggml_type type, int mmq_x, bool need_check>-#if defined(GGML_USE_HIP)-#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)- __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2)-#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)-#else-#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA- __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 1)-#else- __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2)-#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA-#endif // defined(GGML_USE_HIP)+template <ggml_type type, int J, bool fallback>+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback), ggml_cuda_mmq_get_occupancy(type, J, fallback))static __global__ void mul_mat_q(const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,@@ -3548,28 +901,27 @@ static __global__ void mul_mat_q(const uint3 ntx) {// Skip unused template specializations for faster compilation:- if (mmq_x > get_mmq_x_max_device() || mmq_x % mmq_get_granularity_device(mmq_x) != 0) {+ if (ggml_cuda_mmq_get_config(type, J, fallback).type == GGML_TYPE_COUNT) {NO_DEVICE_CODE;return;}- constexpr int nwarps = mmq_get_nwarps_device();constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);- constexpr int qk = ggml_cuda_type_traits<type>::qk;- constexpr int mmq_y = get_mmq_y_device();-- const uint32_t nty = (nrows_x + mmq_y - 1) / mmq_y; // Number of tiles y+ const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y// Initialize the ids for writing back data with just the index.// For regular matrix multiplications this is never changed.// For MoE the correct indices are loaded from ids_dst.extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory.#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {const int j = j0 + threadIdx.y*warp_size + threadIdx.x;- if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {+ if (j0 + nwarps*warp_size > J && j >= J) {break;}@@ -3577,9 +929,7 @@ static __global__ void mul_mat_q(}__syncthreads();- // On non-CDNA AMD or old CUDA the performance with stream-k was worse, use conventional tiling instead:-#if (defined(GGML_USE_HIP) && !defined(CDNA)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA- {+ if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback)) {const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);const int wt = tmp2.x;const int zt = tmp2.y;@@ -3591,7 +941,7 @@ static __global__ void mul_mat_q(int col_high = ncols_dst;int col_diff = ncols_dst;int offset_y = wt*stride_sample_y + zt*stride_channel_y;- int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst;+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;if (ids_dst) {col_low = expert_bounds[zt + 0];@@ -3601,41 +951,40 @@ static __global__ void mul_mat_q(offset_y = 0;offset_dst = 0;- if (jt*mmq_x >= col_diff) {+ if (jt*J >= col_diff) {return;}// __syncthreads(); // There is no previous tile that could cause a race condition.#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {const int j = j0 + threadIdx.y*warp_size + threadIdx.x;- if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {+ if (j0 + nwarps*warp_size > J && j >= J) {break;}- ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j];+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];}__syncthreads();}- offset_y += (col_low + jt*mmq_x)*(sizeof(block_q8_1_mmq)/sizeof(int));- offset_dst += it*mmq_y;+ offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int));+ offset_dst += it*I;- const int tile_x_max_i = nrows_x - it*mmq_y - 1;- const int tile_y_max_j = col_diff - jt*mmq_x - 1;+ const int tile_x_max_i = nrows_x - it*I - 1;+ const int tile_y_max_j = col_diff - jt*J - 1;- const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;+ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;constexpr bool fixup = false;- mul_mat_q_process_tile<type, mmq_x, need_check, fixup>+ mul_mat_q_process_tile<type, J, fallback, fixup>(x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z);return;}-#endif // (defined(GGML_USE_HIP) && !defined(CDNA4) && !defined(CDNA3)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA- constexpr int ITER_K = get_iter_k(type);+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback);constexpr int blocks_per_iter = ITER_K / qk;// kbc == k block continuous, current index in continuous ijk space.@@ -3665,7 +1014,7 @@ static __global__ void mul_mat_q(int col_high = ncols_dst;int col_diff = ncols_dst;int offset_y = wt*stride_sample_y + zt*stride_channel_y;- int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst;+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;if (ids_dst) {col_low = expert_bounds[zt + 0];@@ -3675,7 +1024,7 @@ static __global__ void mul_mat_q(offset_y = 0;offset_dst = 0;- if (jt*mmq_x >= col_diff) {+ if (jt*J >= col_diff) {kbc += blocks_per_ne00.z;kbc -= fastmodulo(kbc, blocks_per_ne00);@@ -3687,28 +1036,28 @@ static __global__ void mul_mat_q(__syncthreads();#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {const int j = j0 + threadIdx.y*warp_size + threadIdx.x;- if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {+ if (j0 + nwarps*warp_size > J && j >= J) {break;}- ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j];+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];}__syncthreads();}- offset_y += (col_low + jt * mmq_x) * (sizeof(block_q8_1_mmq) / sizeof(int));- offset_dst += it*mmq_y;+ offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));+ offset_dst += it*I;- const int tile_x_max_i = nrows_x - it*mmq_y - 1;- const int tile_y_max_j = col_diff - jt*mmq_x - 1;+ const int tile_x_max_i = nrows_x - it*I - 1;+ const int tile_y_max_j = col_diff - jt*J - 1;- const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;+ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.- mul_mat_q_process_tile<type, mmq_x, need_check, fixup>+ mul_mat_q_process_tile<type, J, fallback, fixup>(x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);@@ -3739,7 +1088,7 @@ static __global__ void mul_mat_q(int col_high = ncols_dst;int col_diff = ncols_dst;int offset_y = wt*stride_sample_y + zt*stride_channel_y;- int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst;+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;if (ids_dst) {col_low = expert_bounds[zt + 0];@@ -3749,17 +1098,17 @@ static __global__ void mul_mat_q(offset_y = 0;offset_dst = 0;- if (jt*mmq_x >= col_diff) {+ if (jt*J >= col_diff) {return;}// The memory layout for the fixup buffer is always contiguous, therefore reset ids:__syncthreads();#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) {+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {const int j = j0 + threadIdx.y*warp_size + threadIdx.x;- if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) {+ if (j0 + nwarps*warp_size > J && j >= J) {break;}@@ -3768,39 +1117,38 @@ static __global__ void mul_mat_q(__syncthreads();}- offset_y += (col_low + jt * mmq_x) * (sizeof(block_q8_1_mmq) / sizeof(int));- offset_dst += it*mmq_y;+ offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));+ offset_dst += it*I;- const int tile_x_max_i = nrows_x - it*mmq_y - 1;- const int tile_y_max_j = col_diff - jt*mmq_x - 1;+ const int tile_x_max_i = nrows_x - it*I - 1;+ const int tile_y_max_j = col_diff - jt*J - 1;- const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;+ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.- mul_mat_q_process_tile<type, mmq_x, need_check, fixup>+ mul_mat_q_process_tile<type, J, fallback, fixup>(x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);}-template <ggml_type type, int mmq_x, bool need_check>-__launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device()/2, 1)+template <ggml_type type, int J, bool fallback>+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback)/2, 1)static __global__ void mul_mat_q_stream_k_fixup(const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst,float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst,const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y,const int stride_sample_dst, const uint3 ntx) {- constexpr int mmq_y = get_mmq_y_device();+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback) / 2) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);constexpr int qk = ggml_cuda_type_traits<type>::qk;- constexpr int ITER_K = get_iter_k(type);+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback);constexpr int blocks_per_iter = ITER_K / qk;- constexpr int nwarps = mmq_get_nwarps_device()/2;- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- float sum[mmq_x / nwarps] = {0.0f};+ float sum[J / nwarps] = {0.0f};const int i = blockIdx.y*warp_size + threadIdx.x;- const int nty = (nrows_x + mmq_y - 1) / mmq_y;+ const int nty = (nrows_x + I - 1) / I;const int bidx0 = blockIdx.x;@@ -3838,10 +1186,10 @@ static __global__ void mul_mat_q_stream_k_fixup(#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {+ for (int j0 = 0; j0 < J; j0 += nwarps) {const int j = j0 + threadIdx.y;- sum[j0/nwarps] += tmp_last_tile[bidx*(mmq_x*mmq_y) + j*mmq_y + i];+ sum[j0/nwarps] += tmp_last_tile[bidx*(J*I) + j*I + i];}// If this block started in a previous tile we are done and don't need to combine additional partial results.@@ -3868,17 +1216,17 @@ static __global__ void mul_mat_q_stream_k_fixup(const int it = tmp2.x;if (!ids_dst) {- const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst + it*mmq_y;+ const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;dst += offset_dst;- const int i_max = nrows_x - it*mmq_y - 1;- const int j_max = ncols_dst - jt*mmq_x - 1;- if (need_check && i > i_max) {+ const int i_max = nrows_x - it*I - 1;+ const int j_max = ncols_dst - jt*J - 1;+ if (fallback && i > i_max) {return;}#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {+ for (int j0 = 0; j0 < J; j0 += nwarps) {const int j = j0 + threadIdx.y;if (j > j_max) {@@ -3890,27 +1238,27 @@ static __global__ void mul_mat_q_stream_k_fixup(return;}- __shared__ int ids_dst_shared[mmq_x];+ __shared__ int ids_dst_shared[J];const int col_low = expert_bounds[zt + 0];const int col_high = expert_bounds[zt + 1];const int col_diff = col_high - col_low;- for (int j = threadIdx.y*warp_size + threadIdx.x; j < mmq_x; j += nwarps*warp_size) {- ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j];+ for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) {+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];}__syncthreads();- const int offset_dst = it*mmq_y;+ const int offset_dst = it*I;dst += offset_dst;- const int i_max = nrows_x - it*mmq_y - 1;- const int j_max = col_diff - jt*mmq_x - 1;- if (need_check && i > i_max) {+ const int i_max = nrows_x - it*I - 1;+ const int j_max = col_diff - jt*J - 1;+ if (fallback && i > i_max) {return;}#pragma unroll- for (int j0 = 0; j0 < mmq_x; j0 += nwarps) {+ for (int j0 = 0; j0 < J; j0 += nwarps) {const int j = j0 + threadIdx.y;if (j > j_max) {@@ -3926,37 +1274,35 @@ struct mmq_args {int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;- bool use_stream_k; int64_t ncols_max;+ int64_t ncols_max;};-template<ggml_type type>-static size_t mmq_get_nbytes_shared(const int mmq_x, const int mmq_y, const int cc, const int warp_size, const int nwarps) {- const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, mmq_y);- const int mmq_tile_x_k = mmq_get_mma_tile_x_k(type);- const size_t nbs_ids = mmq_x*sizeof(int);- const size_t nbs_x = (turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) ? mmq_y*mmq_tile_x_k*sizeof(int) : txs.qs*sizeof(int) + txs.dm*sizeof(half2) + txs.sc*sizeof(int);- const size_t nbs_y = mmq_x * (sizeof(block_q8_1_mmq));- return nbs_ids + nbs_x + GGML_PAD(nbs_y, nwarps*warp_size*sizeof(int));+static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {+ const size_t nbs_ids = config.J*sizeof(int);+ const size_t nbs_x = ggml_cuda_mmq_get_nbytes_shared_x(config, cc);+ const size_t nbs_y = config.J * (sizeof(block_q8_1_mmq));+ return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int));}-template <ggml_type type, int mmq_x>+template <ggml_type type, int J, bool fallback>static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {const int id = ggml_cuda_get_device();const int cc = ggml_cuda_info().devices[id].cc;const int nsm = ggml_cuda_info().devices[id].nsm;const int warp_size = ggml_cuda_info().devices[id].warp_size;- const int nwarps = mmq_get_nwarps_host(cc, warp_size);- const int mmq_y = get_mmq_y_host(cc);- const dim3 block_dims(warp_size, nwarps, 1);+ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc);+ GGML_ASSERT(config.nthreads % warp_size == 0);+ const int nwarps = config.nthreads / warp_size;+ const int nbytes_shared = mmq_get_nbytes_shared(config, cc);- const int nbytes_shared = mmq_get_nbytes_shared<type>(mmq_x, mmq_y, cc, warp_size, nwarps);+ const dim3 block_dims(warp_size, nwarps, 1);- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, mmq_x, false>), nbytes_shared);- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, mmq_x, true>), nbytes_shared);+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false>), nbytes_shared);+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true>), nbytes_shared);- const int nty = (args.nrows_x + mmq_y - 1) / mmq_y;- const int ntx = (args.ncols_max + mmq_x - 1) / mmq_x;+ const int nty = (args.nrows_x + config.I - 1) / config.I;+ const int ntx = (args.ncols_max + config.J - 1) / config.J;const int ntzw = args.nchannels_y * args.nsamples_y;const dim3 block_nums_xy_tiling(nty, ntx, ntzw);@@ -3972,24 +1318,13 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & aconst uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);- if (!args.use_stream_k) {- if (args.nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- mul_mat_q<type, mmq_x, need_check><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,- ntx_fd);- } else {- constexpr bool need_check = true;- mul_mat_q<type, mmq_x, need_check><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,- ntx_fd);- }+ if (!ggml_cuda_mmq_get_stream_k(type, J, fallback, cc)) {+ mul_mat_q<type, J, fallback><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>+ (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,+ blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,+ channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,+ sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,+ ntx_fd);return;}@@ -4007,136 +1342,124 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & aggml_cuda_pool & pool = ctx.pool(id);ggml_cuda_pool_alloc<float> tmp_fixup(pool);if (fixup_needed) {- tmp_fixup.alloc(block_nums_stream_k.x * mmq_x*mmq_y);+ tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);}- const dim3 block_nums_fixup(block_nums_stream_k.x, mmq_y/warp_size, 1);+ const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);- if (args.nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- mul_mat_q<type, mmq_x, need_check><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,- ntx_fd);-- if (!fixup_needed) {- return;- }-- CUDA_CHECK(cudaGetLastError());- mul_mat_q_stream_k_fixup<type, mmq_x, need_check><<<block_nums_fixup, block_dims_fixup, 0, stream>>>- (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,- args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,- ntx_fd);- } else {- constexpr bool need_check = true;- mul_mat_q<type, mmq_x, need_check><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,- ntx_fd);-- if (!fixup_needed) {- return;- }+ mul_mat_q<type, J, fallback><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>+ (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,+ blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,+ channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,+ sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,+ ntx_fd);- CUDA_CHECK(cudaGetLastError());- mul_mat_q_stream_k_fixup<type, mmq_x, need_check><<<block_nums_fixup, block_dims_fixup, 0, stream>>>- (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,- args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,- ntx_fd);+ if (!fixup_needed) {+ return;}-}-template <ggml_type type>-void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {- const int id = ggml_cuda_get_device();- const int cc = ggml_cuda_info().devices[id].cc;- const size_t smpbo = ggml_cuda_info().devices[id].smpbo;- const int warp_size = ggml_cuda_info().devices[id].warp_size;- const int nwarps = mmq_get_nwarps_host(cc, warp_size);+ CUDA_CHECK(cudaGetLastError());+ mul_mat_q_stream_k_fixup<type, J, fallback><<<block_nums_fixup, block_dims_fixup, 0, stream>>>+ (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,+ args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,+ ntx_fd);+}- const int mmq_x_max = get_mmq_x_max_host(cc);- const int mmq_y = get_mmq_y_host(cc);+template <ggml_type type, bool fallback>+void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {+ const int id = ggml_cuda_get_device();+ const int cc = ggml_cuda_info().devices[id].cc;+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;- int mmq_x_best = 0;- int ntiles_x_best = INT_MAX;+ int J_best = 0;+ int ntiles_J_best = INT_MAX;- for (int mmq_x = 8; mmq_x <= mmq_x_max && ntiles_x_best > 1; mmq_x += 8) {- const int granularity = mmq_get_granularity_host(mmq_x, cc);+ for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {+ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc);+ if (config.type == GGML_TYPE_COUNT) {+ continue;+ }- if (mmq_x % granularity != 0 || mmq_get_nbytes_shared<type>(mmq_x, mmq_y, cc, warp_size, nwarps) > smpbo) {+ if (mmq_get_nbytes_shared(config, cc) > smpbo) {continue;}- const int ntiles_x = (args.ncols_max + mmq_x - 1) / mmq_x;+ const int ntiles_x = (args.ncols_max + config.J - 1) / config.J;- if (ntiles_x < ntiles_x_best) {- mmq_x_best = mmq_x;- ntiles_x_best = ntiles_x;+ if (ntiles_x < ntiles_J_best) {+ J_best = J;+ ntiles_J_best = ntiles_x;}}- switch (mmq_x_best) {+ switch (J_best) {case 8:- launch_mul_mat_q<type, 8>(ctx, args, stream);+ launch_mul_mat_q<type, 8, fallback>(ctx, args, stream);break;case 16:- launch_mul_mat_q<type, 16>(ctx, args, stream);+ launch_mul_mat_q<type, 16, fallback>(ctx, args, stream);break;case 24:- launch_mul_mat_q<type, 24>(ctx, args, stream);+ launch_mul_mat_q<type, 24, fallback>(ctx, args, stream);break;case 32:- launch_mul_mat_q<type, 32>(ctx, args, stream);+ launch_mul_mat_q<type, 32, fallback>(ctx, args, stream);break;case 40:- launch_mul_mat_q<type, 40>(ctx, args, stream);+ launch_mul_mat_q<type, 40, fallback>(ctx, args, stream);break;case 48:- launch_mul_mat_q<type, 48>(ctx, args, stream);+ launch_mul_mat_q<type, 48, fallback>(ctx, args, stream);break;case 56:- launch_mul_mat_q<type, 56>(ctx, args, stream);+ launch_mul_mat_q<type, 56, fallback>(ctx, args, stream);break;case 64:- launch_mul_mat_q<type, 64>(ctx, args, stream);+ launch_mul_mat_q<type, 64, fallback>(ctx, args, stream);break;case 72:- launch_mul_mat_q<type, 72>(ctx, args, stream);+ launch_mul_mat_q<type, 72, fallback>(ctx, args, stream);break;case 80:- launch_mul_mat_q<type, 80>(ctx, args, stream);+ launch_mul_mat_q<type, 80, fallback>(ctx, args, stream);break;case 88:- launch_mul_mat_q<type, 88>(ctx, args, stream);+ launch_mul_mat_q<type, 88, fallback>(ctx, args, stream);break;case 96:- launch_mul_mat_q<type, 96>(ctx, args, stream);+ launch_mul_mat_q<type, 96, fallback>(ctx, args, stream);break;case 104:- launch_mul_mat_q<type, 104>(ctx, args, stream);+ launch_mul_mat_q<type, 104, fallback>(ctx, args, stream);break;case 112:- launch_mul_mat_q<type, 112>(ctx, args, stream);+ launch_mul_mat_q<type, 112, fallback>(ctx, args, stream);break;case 120:- launch_mul_mat_q<type, 120>(ctx, args, stream);+ launch_mul_mat_q<type, 120, fallback>(ctx, args, stream);break;case 128:- launch_mul_mat_q<type, 128>(ctx, args, stream);+ launch_mul_mat_q<type, 128, fallback>(ctx, args, stream);break;default:- fprintf(stderr, "mmq_x_best=%d\n", mmq_x_best);+ fprintf(stderr, "J_best=%d\n", J_best);GGML_ABORT("fatal error");break;}}+template <ggml_type type>+void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {+ if (args.nrows_x % 128 == 0) {+ constexpr bool fallback = false;+ mul_mat_q_switch_J<type, fallback>(ctx, args, stream);+ } else {+ constexpr bool fallback = true;+ mul_mat_q_switch_J<type, fallback>(ctx, args, stream);+ }+}+#define DECL_MMQ_CASE(type) \template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \@@ -4166,11 +1489,4 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);void ggml_cuda_mul_mat_q(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);-void ggml_cuda_op_mul_mat_q(- ggml_backend_cuda_context & ctx,- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream);-bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);-