LLM inference in C/C++
C++ ggml
CUDA: Fuse MMVQ post-scale for NVFP4 (#24481)
Oliver Simons3899b39ce2ac
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuhindex e6e50e0411..290dc4aff2 100644--- a/ggml/src/ggml-cuda/common.cuh+++ b/ggml/src/ggml-cuda/common.cuh@@ -1505,12 +1505,16 @@ struct ggml_cuda_mm_fusion_args_host {const ggml_tensor * x_bias = nullptr;const ggml_tensor * gate = nullptr;const ggml_tensor * gate_bias = nullptr;+ const ggml_tensor * x_scale = nullptr;+ const ggml_tensor * gate_scale = nullptr;ggml_glu_op glu_op;};struct ggml_cuda_mm_fusion_args_device {const void * x_bias = nullptr;const void * gate = nullptr;const void * gate_bias = nullptr;+ const void * x_scale = nullptr;+ const void * gate_scale = nullptr;ggml_glu_op glu_op;};diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 71cc2a6199..674381537c 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -1582,12 +1582,18 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,const ggml_tensor * ffn_gate,const ggml_tensor * glu,const ggml_tensor * ffn_up_bias = nullptr,- const ggml_tensor * ffn_gate_bias = nullptr) {+ const ggml_tensor * ffn_gate_bias = nullptr,+ const ggml_tensor * ffn_up_scale = nullptr,+ const ggml_tensor * ffn_gate_scale = nullptr) {const bool has_bias = ffn_up_bias != nullptr || ffn_gate_bias != nullptr;+ const bool has_scale = ffn_up_scale != nullptr || ffn_gate_scale != nullptr;if (has_bias && (!ffn_up_bias || !ffn_gate_bias)) {return false;}+ if (has_scale && (!ffn_up_scale || !ffn_gate_scale)) {+ return false;+ }const bool is_mul_mat = ffn_up->op == GGML_OP_MUL_MAT && ffn_gate->op == GGML_OP_MUL_MAT && glu->op == GGML_OP_GLU;const bool is_mul_mat_id = ffn_up->op == GGML_OP_MUL_MAT_ID && ffn_gate->op == GGML_OP_MUL_MAT_ID && glu->op == GGML_OP_GLU;@@ -1599,34 +1605,45 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,}const ggml_op expected_bias_op = is_mul_mat ? GGML_OP_ADD : GGML_OP_ADD_ID;+ const ggml_tensor * ffn_up_bias_src = has_scale ? ffn_up_scale : ffn_up;+ const ggml_tensor * ffn_gate_bias_src = has_scale ? ffn_gate_scale : ffn_gate;+ const ggml_tensor * ffn_up_out = has_bias ? ffn_up_bias : ffn_up_bias_src;+ const ggml_tensor * ffn_gate_out = has_bias ? ffn_gate_bias : ffn_gate_bias_src;- if (has_bias) {- if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) {+ if (glu->src[0] != ffn_gate_out || glu->src[1] != ffn_up_out) {+ return false;+ }++ if (has_scale) {+ if (ffn_up_scale->op != GGML_OP_MUL || ffn_gate_scale->op != GGML_OP_MUL) {+ return false;+ }+ const bool up_has_mm = ffn_up_scale->src[0] == ffn_up || ffn_up_scale->src[1] == ffn_up;+ const bool gate_has_mm = ffn_gate_scale->src[0] == ffn_gate || ffn_gate_scale->src[1] == ffn_gate;+ if (!up_has_mm || !gate_has_mm) {return false;}+ }- if (glu->src[0] != ffn_gate_bias || glu->src[1] != ffn_up_bias) {+ if (has_bias) {+ if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) {return false;}if (expected_bias_op == GGML_OP_ADD) {- const bool up_has_mul = ffn_up_bias->src[0] == ffn_up || ffn_up_bias->src[1] == ffn_up;- const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate || ffn_gate_bias->src[1] == ffn_gate;+ const bool up_has_mul = ffn_up_bias->src[0] == ffn_up_bias_src || ffn_up_bias->src[1] == ffn_up_bias_src;+ const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate_bias_src || ffn_gate_bias->src[1] == ffn_gate_bias_src;if (!up_has_mul || !gate_has_mul) {return false;}} else { // GGML_OP_ADD_ID- if (ffn_up_bias->src[0] != ffn_up || ffn_gate_bias->src[0] != ffn_gate) {+ if (ffn_up_bias->src[0] != ffn_up_bias_src || ffn_gate_bias->src[0] != ffn_gate_bias_src) {return false;}if (ffn_up_bias->src[2] != ffn_up->src[2] || ffn_gate_bias->src[2] != ffn_gate->src[2]) {return false;}}- } else {- if (glu->src[0] != ffn_gate && glu->src[1] != ffn_up) {- return false;- }}if (ffn_up->src[0]->type != ffn_gate->src[0]->type || !ggml_are_same_shape(ffn_up->src[0], ffn_gate->src[0]) ||@@ -1638,7 +1655,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,return false;}- if (ffn_up->src[2] && (ffn_up->src[2] != ffn_gate->src[2])) {+ if (is_mul_mat_id && ffn_up->src[2] != ffn_gate->src[2]) {return false;}@@ -3204,10 +3221,240 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphbool fused_mul_mat_vec = false;int fused_node_count = 0;- // gate + glu + up+ auto get_mul_mat_scale = [](const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * {+ const bool scale_lhs_mm = scale_node->src[0] == mm_node;+ const bool scale_rhs_mm = scale_node->src[1] == mm_node;+ if (!scale_lhs_mm && !scale_rhs_mm) {+ return nullptr;+ }++ const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0];+ if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 ||+ scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 ||+ !ggml_are_same_shape(scale_node, mm_node)) {+ return nullptr;+ }++ return scale;+ };++ auto get_mul_mat_id_scale = [](const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows,+ const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * {+ if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) {+ return nullptr;+ }+ if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) ||+ (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) {+ return nullptr;+ }++ const ggml_tensor * scale = reshape->src[0];+ if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 ||+ scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] ||+ !ggml_are_same_shape(scale_node, mm_node)) {+ return nullptr;+ }++ return scale;+ };++ auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) -> const ggml_tensor * {+ if (op_bias == GGML_OP_ADD) {+ if (bias_node->src[0] == mul_node) {+ return bias_node->src[1];+ }+ if (bias_node->src[1] == mul_node) {+ return bias_node->src[0];+ }+ return nullptr;+ }+ GGML_ASSERT(op_bias == GGML_OP_ADD_ID);+ GGML_ASSERT(bias_node->src[0] == mul_node);+ return bias_node->src[1];+ };++ // gate + glu + up, with optional scale/bias on both lanes.for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) {const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID;+ if (op == GGML_OP_MUL_MAT) {+ for (const bool with_bias : { false, true }) {+ const int gate_idx = i;+ const int gate_scale_idx = i + 1;+ const int gate_bias_idx = with_bias ? i + 2 : -1;+ const int up_idx = with_bias ? i + 3 : i + 2;+ const int up_scale_idx = up_idx + 1;+ const int up_bias_idx = with_bias ? up_idx + 2 : -1;+ const int glu_idx = with_bias ? up_idx + 3 : up_idx + 2;++ const int out_nodes[] = { glu_idx };+ ggml_op ops[7];+ if (with_bias) {+ ops[0] = op;+ ops[1] = GGML_OP_MUL;+ ops[2] = bias_op;+ ops[3] = op;+ ops[4] = GGML_OP_MUL;+ ops[5] = bias_op;+ ops[6] = GGML_OP_GLU;+ } else {+ ops[0] = op;+ ops[1] = GGML_OP_MUL;+ ops[2] = op;+ ops[3] = GGML_OP_MUL;+ ops[4] = GGML_OP_GLU;+ }+ const int n_ops = with_bias ? 7 : 5;++ if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) ||+ !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) {+ continue;+ }++ ggml_tensor * gate_n = cgraph->nodes[gate_idx];+ ggml_tensor * gate_scale_n = cgraph->nodes[gate_scale_idx];+ ggml_tensor * gate_out_n = with_bias ? cgraph->nodes[gate_bias_idx] : gate_scale_n;+ ggml_tensor * up_n = cgraph->nodes[up_idx];+ ggml_tensor * up_scale_n = cgraph->nodes[up_scale_idx];+ ggml_tensor * up_out_n = with_bias ? cgraph->nodes[up_bias_idx] : up_scale_n;+ const ggml_tensor * glu = cgraph->nodes[glu_idx];++ if (!ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu,+ with_bias ? up_out_n : nullptr, with_bias ? gate_out_n : nullptr, up_scale_n, gate_scale_n)) {+ continue;+ }++ const ggml_tensor * gate_scale = get_mul_mat_scale(gate_scale_n, gate_n);+ const ggml_tensor * up_scale = get_mul_mat_scale(up_scale_n, up_n);+ if (!gate_scale || !up_scale) {+ continue;+ }++ const ggml_tensor * up_bias = with_bias ? get_bias_tensor(up_out_n, up_scale_n, bias_op) : nullptr;+ const ggml_tensor * gate_bias = with_bias ? get_bias_tensor(gate_out_n, gate_scale_n, bias_op) : nullptr;+ if (with_bias && (!ggml_are_same_shape(gate_out_n->src[0], gate_out_n->src[1]) ||+ !ggml_are_same_shape(up_out_n->src[0], up_out_n->src[1]))) {+ continue;+ }++ const ggml_tensor * src0 = up_n->src[0];+ const ggml_tensor * src1 = up_n->src[1];+ const ggml_tensor * ids = up_n->src[2];++ ggml_cuda_mm_fusion_args_host fusion_data{};+ fusion_data.gate = gate_n->src[0];+ fusion_data.x_bias = up_bias;+ fusion_data.gate_bias = gate_bias;+ fusion_data.x_scale = up_scale;+ fusion_data.gate_scale = gate_scale;+ fusion_data.glu_op = ggml_get_glu_op(glu);++ if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {+ ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);+ fused_mul_mat_vec = true;+ fused_node_count = n_ops;+ break;+ }+ }++ if (fused_mul_mat_vec) {+ break;+ }+ } else {+ for (const bool with_bias : { false, true }) {+ const int gate_idx = i;+ const int gate_scale_idx = i + 4;+ const int gate_bias_idx = with_bias ? i + 5 : -1;+ const int up_idx = with_bias ? i + 6 : i + 5;+ const int up_scale_idx = up_idx + 4;+ const int up_bias_idx = with_bias ? up_idx + 5 : -1;+ const int glu_idx = with_bias ? up_idx + 6 : up_idx + 5;++ const int out_nodes[] = { glu_idx };+ ggml_op ops[13];+ if (with_bias) {+ ops[0] = op;+ ops[1] = GGML_OP_RESHAPE;+ ops[2] = GGML_OP_REPEAT;+ ops[3] = GGML_OP_GET_ROWS;+ ops[4] = GGML_OP_MUL;+ ops[5] = bias_op;+ ops[6] = op;+ ops[7] = GGML_OP_RESHAPE;+ ops[8] = GGML_OP_REPEAT;+ ops[9] = GGML_OP_GET_ROWS;+ ops[10] = GGML_OP_MUL;+ ops[11] = bias_op;+ ops[12] = GGML_OP_GLU;+ } else {+ ops[0] = op;+ ops[1] = GGML_OP_RESHAPE;+ ops[2] = GGML_OP_REPEAT;+ ops[3] = GGML_OP_GET_ROWS;+ ops[4] = GGML_OP_MUL;+ ops[5] = op;+ ops[6] = GGML_OP_RESHAPE;+ ops[7] = GGML_OP_REPEAT;+ ops[8] = GGML_OP_GET_ROWS;+ ops[9] = GGML_OP_MUL;+ ops[10] = GGML_OP_GLU;+ }+ const int n_ops = with_bias ? 13 : 11;++ if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) ||+ !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) {+ continue;+ }++ ggml_tensor * gate_n = cgraph->nodes[gate_idx];+ ggml_tensor * gate_scale_n = cgraph->nodes[gate_scale_idx];+ ggml_tensor * gate_out_n = with_bias ? cgraph->nodes[gate_bias_idx] : gate_scale_n;+ ggml_tensor * up_n = cgraph->nodes[up_idx];+ ggml_tensor * up_scale_n = cgraph->nodes[up_scale_idx];+ ggml_tensor * up_out_n = with_bias ? cgraph->nodes[up_bias_idx] : up_scale_n;+ const ggml_tensor * glu = cgraph->nodes[glu_idx];++ if (!ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu,+ with_bias ? up_out_n : nullptr, with_bias ? gate_out_n : nullptr, up_scale_n, gate_scale_n)) {+ continue;+ }++ const ggml_tensor * gate_scale = get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2],+ cgraph->nodes[gate_idx + 3], gate_scale_n, gate_n);+ const ggml_tensor * up_scale = get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2],+ cgraph->nodes[up_idx + 3], up_scale_n, up_n);+ if (!gate_scale || !up_scale) {+ continue;+ }++ const ggml_tensor * up_bias = with_bias ? get_bias_tensor(up_out_n, up_scale_n, bias_op) : nullptr;+ const ggml_tensor * gate_bias = with_bias ? get_bias_tensor(gate_out_n, gate_scale_n, bias_op) : nullptr;++ const ggml_tensor * src0 = up_n->src[0];+ const ggml_tensor * src1 = up_n->src[1];+ const ggml_tensor * ids = up_n->src[2];++ ggml_cuda_mm_fusion_args_host fusion_data{};+ fusion_data.gate = gate_n->src[0];+ fusion_data.x_bias = up_bias;+ fusion_data.gate_bias = gate_bias;+ fusion_data.x_scale = up_scale;+ fusion_data.gate_scale = gate_scale;+ fusion_data.glu_op = ggml_get_glu_op(glu);++ if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {+ ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);+ fused_mul_mat_vec = true;+ fused_node_count = n_ops;+ break;+ }+ }++ if (fused_mul_mat_vec) {+ break;+ }+ }+if (ggml_cuda_can_fuse(cgraph, i, { op, bias_op, op, bias_op, GGML_OP_GLU }, {})) {ggml_tensor * glu = cgraph->nodes[i + 4];ggml_tensor * gate_bias_n = glu->src[0];@@ -3227,23 +3474,8 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphcontinue;}- auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) {- if (op_bias == GGML_OP_ADD) {- if (bias_node->src[0] == mul_node) {- return bias_node->src[1];- }- if (bias_node->src[1] == mul_node) {- return bias_node->src[0];- }- return (ggml_tensor *) nullptr;- }- GGML_ASSERT(op_bias == GGML_OP_ADD_ID);- GGML_ASSERT(bias_node->src[0] == mul_node);- return bias_node->src[1];- };-- ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op);- ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op);+ const ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op);+ const ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op);if (!up_bias_tensor || !gate_bias_tensor) {continue;@@ -3331,7 +3563,95 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraphfused_mul_mat_vec = false;fused_node_count = 0;- // gate + add + glu + up + add+ // mul_mat + scale + optional bias+ for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) {+ const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID;++ for (const bool with_bias : { false, true }) {+ const int n_ops = op == GGML_OP_MUL_MAT ? (with_bias ? 3 : 2) : (with_bias ? 6 : 5);+ const int out_nodes[] = { i + n_ops - 1 };+ ggml_op ops[6];+ if (op == GGML_OP_MUL_MAT) {+ if (with_bias) {+ ops[0] = op;+ ops[1] = GGML_OP_MUL;+ ops[2] = bias_op;+ } else {+ ops[0] = op;+ ops[1] = GGML_OP_MUL;+ }+ } else {+ if (with_bias) {+ ops[0] = op;+ ops[1] = GGML_OP_RESHAPE;+ ops[2] = GGML_OP_REPEAT;+ ops[3] = GGML_OP_GET_ROWS;+ ops[4] = GGML_OP_MUL;+ ops[5] = bias_op;+ } else {+ ops[0] = op;+ ops[1] = GGML_OP_RESHAPE;+ ops[2] = GGML_OP_REPEAT;+ ops[3] = GGML_OP_GET_ROWS;+ ops[4] = GGML_OP_MUL;+ }+ }++ if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) ||+ !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) {+ continue;+ }++ ggml_tensor * mm_node = cgraph->nodes[i];+ ggml_tensor * scale_node = op == GGML_OP_MUL_MAT ? cgraph->nodes[i + 1] : cgraph->nodes[i + 4];+ ggml_tensor * out_node = with_bias ? cgraph->nodes[i + n_ops - 1] : scale_node;++ const ggml_tensor * scale = nullptr;+ if (op == GGML_OP_MUL_MAT) {+ scale = get_mul_mat_scale(scale_node, mm_node);+ } else {+ scale = get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node);+ }+ if (!scale) {+ continue;+ }++ const ggml_tensor * bias = with_bias ? get_bias_tensor(out_node, scale_node, bias_op) : nullptr;+ if (with_bias && !bias) {+ continue;+ }+ if (with_bias && bias_op == GGML_OP_ADD && !ggml_are_same_shape(out_node->src[0], out_node->src[1])) {+ continue;+ }+ if (with_bias && bias_op == GGML_OP_ADD_ID && out_node->src[2] != mm_node->src[2]) {+ continue;+ }++ const ggml_tensor * src0 = mm_node->src[0];+ const ggml_tensor * src1 = mm_node->src[1];+ const ggml_tensor * ids = mm_node->src[2];++ ggml_cuda_mm_fusion_args_host fusion_data{};+ fusion_data.x_bias = bias;+ fusion_data.x_scale = scale;++ if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) {+ ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);+ fused_mul_mat_vec = true;+ fused_node_count = n_ops;+ break;+ }+ }+ if (fused_mul_mat_vec) {+ break;+ }+ }++ if (fused_mul_mat_vec) {+ return fused_node_count - 1;+ }++ // mul_mat + addfor (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) {const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID;@@ -3562,12 +3882,6 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud}}-#ifdef GGML_CUDA_DEBUG- const int nodes_fused = i - prev_i - 1;- if (nodes_fused > 0) {- GGML_LOG_INFO("nodes_fused: %d\n", nodes_fused);- }-#endifprev_i = i;if (ggml_cuda_is_view_or_noop(node)) {@@ -3581,6 +3895,12 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cudint nodes_to_skip = ggml_cuda_try_fuse(cuda_ctx, cgraph, i);if (nodes_to_skip != 0) {+#ifdef GGML_CUDA_DEBUG+ const int last_fused = i + nodes_to_skip;+ GGML_LOG_INFO("nodes_fused: %d, first: %s (%s), last: %s (%s)\n",+ nodes_to_skip + 1, ggml_op_name(node->op), node->name,+ ggml_op_name(cgraph->nodes[last_fused]->op), cgraph->nodes[last_fused]->name);+#endifi += nodes_to_skip;continue;}diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cuindex 2278c7d9bd..a48cc48b2b 100644--- a/ggml/src/ggml-cuda/mmvq.cu+++ b/ggml/src/ggml-cuda/mmvq.cu@@ -521,9 +521,13 @@ static __global__ void mul_mat_vec_q(bool use_gate = false;bool use_bias = false;bool use_gate_bias = false;+ bool use_scale = false;+ bool use_gate_scale = false;[[maybe_unused]] const void * vgate = nullptr;const float * x_bias = nullptr;const float * gate_bias = nullptr;+ const float * x_scale = nullptr;+ const float * gate_scale = nullptr;ggml_glu_op active_glu;if constexpr (has_fusion) {@@ -534,34 +538,47 @@ static __global__ void mul_mat_vec_q(x_bias = (const float *) fusion.x_bias;gate_bias = (const float *) fusion.gate_bias;active_glu = fusion.glu_op;+ if constexpr (type == GGML_TYPE_NVFP4) {+ use_scale = fusion.x_scale != nullptr;+ use_gate_scale = fusion.gate_scale != nullptr && use_gate;+ x_scale = (const float *) fusion.x_scale;+ gate_scale = (const float *) fusion.gate_scale;+ }}[[maybe_unused]] float x_biases[ncols_dst] = { 0.0f };[[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f };+ [[maybe_unused]] float x_scales;+ [[maybe_unused]] float gate_scales;if constexpr (has_fusion) {+ // 1. Hide latency by prefetching bias, gates and scales here+ // 2. load only on threads that won't die after partial sum calculationconst uint32_t channel_bias = ids ? channel_x : channel_dst;- if (use_bias) {- x_bias = x_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0;- // 1. Hide latency by prefetching bias and gate here- // 2. load only on threads that won't die after partial sum calculation- if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&- (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {+ if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&+ (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {+ if (use_bias) {+ x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;#pragma unrollfor (int j = 0; j < ncols_dst; ++j) {x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x];}}- }- if (use_gate_bias) {- gate_bias = gate_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0;- if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&- (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {+ if (use_gate_bias) {+ gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;#pragma unrollfor (int j = 0; j < ncols_dst; ++j) {gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];}}+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (use_scale) {+ x_scales = x_scale[ids ? channel_x : 0];+ }+ if (use_gate_scale) {+ gate_scales = gate_scale[ids ? channel_x : 0];+ }+ }}}@@ -643,11 +660,21 @@ static __global__ void mul_mat_vec_q(if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {float result = tmp[j][threadIdx.x];if constexpr (has_fusion) {+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (use_scale) {+ result *= x_scales;+ }+ }if (use_bias) {result += x_biases[j];}if (use_gate) {float gate_value = tmp_gate[j][threadIdx.x];+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (use_gate_scale) {+ gate_value *= gate_scales;+ }+ }if (use_gate_bias) {gate_value += gate_biases[j];}@@ -673,7 +700,10 @@ static __global__ void mul_mat_vec_q(}if constexpr (!has_fusion) {- GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, gate_bias, x_bias, tmp_gate);+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);+ }+ if constexpr (type != GGML_TYPE_NVFP4) {+ GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales);}}@@ -769,7 +799,8 @@ static void mul_mat_vec_q_switch_fusion(const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared,const uint32_t ids_stride, cudaStream_t stream) {- const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;+ const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;if constexpr (c_ncols_dst == 1) {if (has_fusion) {const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);@@ -834,7 +865,6 @@ static void mul_mat_vec_q_switch_ncols_dst(const int warp_size = ggml_cuda_info().devices[device].warp_size;const mmvq_parameter_table_id table_id = get_device_table_id(cc);- const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;const bool has_ids = ids != nullptr;const auto should_use_small_k = [&](int c_ncols_dst) {@@ -973,8 +1003,6 @@ static void mul_mat_vec_q_switch_ncols_dst(GGML_ABORT("fatal error");break;}-- GGML_UNUSED(has_fusion);}static void mul_mat_vec_q_switch_type(const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,@@ -1154,6 +1182,9 @@ void ggml_cuda_mul_mat_vec_q(if (fusion) {GGML_ASSERT( !ids || dst->ne[2] == 1);GGML_ASSERT( ids || dst->ne[1] == 1);+ // Scale fusion is only allowed for NVFP4 currently as the cost of checking this at run-time in the prologue is+ // non-negligible for some models such as gpt-oss-20b+ GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_NVFP4);if (fusion->x_bias) {GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);@@ -1171,6 +1202,18 @@ void ggml_cuda_mul_mat_vec_q(GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);fusion_local.gate_bias = fusion->gate_bias->data;}+ if (fusion->x_scale) {+ GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);+ GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));+ fusion_local.x_scale = fusion->x_scale->data;+ }+ if (fusion->gate_scale) {+ GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);+ GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));+ fusion_local.gate_scale = fusion->gate_scale->data;+ }fusion_local.glu_op = fusion->glu_op;}diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex 440095f805..fc46ab0aa0 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -7419,6 +7419,10 @@ static int ggml_node_list_find_tensor(const struct ggml_cgraph * cgraph,return -1;}+static bool ggml_is_constant(const struct ggml_tensor * tensor) {+ return tensor->buffer != NULL && ggml_backend_buffer_get_usage(tensor->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && (tensor->flags & GGML_TENSOR_FLAG_PARAM) == 0;+}+bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph,const int * node_idxs,int count,@@ -7464,10 +7468,11 @@ bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph,return false;}- // if node is a view, check if the view_src and all it's parent view_srcs are within the subgraph+ // if node is a view, check if the view_src and all its parent view_srcs are within the subgraph.+ // external view sources are allowed only for weight tensors, which are constant for this graph execution.struct ggml_tensor * view_src = node->view_src;while (view_src) {- if (ggml_node_list_find_tensor(cgraph, node_idxs, count, view_src) == -1) {+ if (ggml_node_list_find_tensor(cgraph, node_idxs, count, view_src) == -1 && !ggml_is_constant(view_src)) {return false;}view_src = view_src->view_src;diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex c21675ef54..82a9d9546b 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -1137,6 +1137,10 @@ struct test_case {}virtual ggml_tensor * build_graph(ggml_context * ctx) = 0;+ virtual ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) {+ GGML_UNUSED(ctx_weights);+ return build_graph(ctx);+ }virtual double max_nmse_err() {return 1e-7;@@ -1213,6 +1217,7 @@ struct test_case {virtual bool run_whole_graph() { return false; }virtual std::vector<ggml_tensor *> fusion_test_nodes() { return {}; }+ virtual bool use_weight_context() { return false; }ggml_cgraph * gf = nullptr;ggml_cgraph * gb = nullptr;@@ -1319,20 +1324,28 @@ struct test_case {/* .mem_base = */ NULL,/* .no_alloc = */ true,};+ const bool use_weights = use_weight_context();+ggml_context * ctx = ggml_init(params);GGML_ASSERT(ctx);+ ggml_context * ctx_weights = use_weights ? ggml_init(params) : nullptr;+ GGML_ASSERT(!use_weights || ctx_weights);gf = ggml_new_graph(ctx);// pre-graph sentineladd_sentinel(ctx);+ if (ctx_weights) {+ add_sentinel(ctx_weights);+ }- ggml_tensor * out = build_graph(ctx);+ ggml_tensor * out = build_graph(ctx, ctx_weights);current_op_name = op_desc(out);check_for_f16_tensor(ctx);if (!matches_filter(out, op_names_filter)) {//printf(" %s: skipping\n", op_desc(out).c_str());+ ggml_free(ctx_weights);ggml_free(ctx);return test_status_t::SKIPPED;}@@ -1355,18 +1368,36 @@ struct test_case {print_test_result_locked(output_printer, result);+ ggml_free(ctx_weights);ggml_free(ctx);return test_status_t::NOT_SUPPORTED;}// post-graph sentineladd_sentinel(ctx);+ if (ctx_weights) {+ add_sentinel(ctx_weights);+ }++ ggml_backend_buffer_t buf_weights = nullptr;+ if (ctx_weights) {+ buf_weights = ggml_backend_alloc_ctx_tensors(ctx_weights, backend1);+ if (buf_weights == NULL) {+ printf("failed to allocate weight tensors [%s] ", ggml_backend_name(backend1));+ ggml_free(ctx_weights);+ ggml_free(ctx);+ return test_status_t::FAIL;+ }+ ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);+ }// allocateggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend1);if (buf == NULL) {printf("failed to allocate tensors [%s] ", ggml_backend_name(backend1));+ ggml_backend_buffer_free(buf_weights);+ ggml_free(ctx_weights);ggml_free(ctx);return test_status_t::FAIL;}@@ -1381,6 +1412,9 @@ struct test_case {// randomize tensorsinitialize_tensors(ctx);+ if (ctx_weights) {+ initialize_tensors(ctx_weights);+ }// comparestruct callback_userdata {@@ -1466,7 +1500,8 @@ struct test_case {fused_nodes_to_verify.size());ggml_backend_buffer_free(buf);-+ ggml_backend_buffer_free(buf_weights);+ ggml_free(ctx_weights);ggml_free(ctx);// Create test result@@ -1490,10 +1525,14 @@ struct test_case {/* .mem_base = */ NULL,/* .no_alloc = */ true,};+ const bool use_weights = use_weight_context();+ggml_context_ptr ctx(ggml_init(params)); // smart ptrGGML_ASSERT(ctx);+ ggml_context_ptr ctx_weights(use_weights ? ggml_init(params) : nullptr);+ GGML_ASSERT(!use_weights || ctx_weights);- ggml_tensor * out = build_graph(ctx.get());+ ggml_tensor * out = build_graph(ctx.get(), ctx_weights.get());current_op_name = op_desc(out);if (!matches_filter(out, op_names_filter)) {//printf(" %s: skipping\n", op_desc(out).c_str());@@ -1510,6 +1549,16 @@ struct test_case {return true;}+ ggml_backend_buffer_ptr buf_weights(nullptr);+ if (ctx_weights) {+ buf_weights.reset(ggml_backend_alloc_ctx_tensors(ctx_weights.get(), backend));+ if (buf_weights == NULL) {+ printf("failed to allocate weight tensors\n");+ return false;+ }+ ggml_backend_buffer_set_usage(buf_weights.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);+ }+// allocateggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr@@ -1520,6 +1569,9 @@ struct test_case {// randomize tensorsinitialize_tensors(ctx.get());+ if (ctx_weights) {+ initialize_tensors(ctx_weights.get());+ }// build graphggml_cgraph * gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false);@@ -5848,19 +5900,21 @@ struct test_mul_mat_vec_fusion : public test_case {const bool b; // broadcast b matrix (only for use_id)const bool with_bias;const bool with_gate;+ const bool with_lane_scale;std::array<int64_t, 2> batch_dims;test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k,bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true,- std::array<int64_t, 2> batch_dims = {4, 2})- : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), with_gate(with_gate), batch_dims(batch_dims) {+ bool with_lane_scale = false, std::array<int64_t, 2> batch_dims = {4, 2})+ : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias),+ with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims) {if (use_id) {GGML_ASSERT(n_used <= n_mats);}}std::string vars() override {- return VARS_TO_STR12(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, batch_dims);+ return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims);}std::string op_desc(ggml_tensor * t) override {@@ -5869,6 +5923,7 @@ struct test_mul_mat_vec_fusion : public test_case {}bool run_whole_graph() override { return true; }+ bool use_weight_context() override { return use_id && with_lane_scale; }ggml_tensor * build_gate(ggml_context * ctx, ggml_tensor * ffn_gate, ggml_tensor * ffn_up) {ggml_tensor * out = nullptr;@@ -5884,7 +5939,26 @@ struct test_mul_mat_vec_fusion : public test_case {return out;}+ ggml_tensor * build_lane_scale_dense(ggml_context * ctx, ggml_tensor * out) {+ ggml_tensor * scale = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);+ return ggml_mul(ctx, out, scale);+ }++ ggml_tensor * build_lane_scale_id(ggml_context * ctx, ggml_context * ctx_weights, ggml_tensor * out, ggml_tensor * ids) {+ GGML_ASSERT(ctx_weights);+ ggml_tensor * scale = ggml_new_tensor_1d(ctx_weights, GGML_TYPE_F32, n_mats);+ ggml_tensor * s = ggml_reshape_3d(ctx, scale, 1, n_mats, 1);+ s = ggml_repeat_4d(ctx, s, 1, n_mats, m, 1);+ s = ggml_get_rows(ctx, s, ids);+ return ggml_mul(ctx, out, s);+ }+ggml_tensor * build_graph(ggml_context * ctx) override {+ GGML_ASSERT(!use_weight_context());+ return build_graph(ctx, nullptr);+ }++ ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override {if (!use_id) {const int channels = batch_dims[0];const int samples = batch_dims[1];@@ -5895,19 +5969,34 @@ struct test_mul_mat_vec_fusion : public test_case {ggml_tensor * gate = with_gate ? ggml_new_tensor(ctx, type, 4, ne0.data()) : nullptr;ggml_tensor * up = ggml_new_tensor(ctx, type, 4, ne0.data());- ggml_tensor * ffn_up = ggml_mul_mat(ctx, up, cur);- if (with_bias) {- std::array<int64_t, 4> bias_ne = { ffn_up->ne[0], 1, channels, samples };- ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());- ffn_up = ggml_add(ctx, ffn_up, up_bias);- }+ auto build_lane_up = [&]() {+ ggml_tensor * ffn_up = ggml_mul_mat(ctx, up, cur);+ if (with_lane_scale) {+ ffn_up = build_lane_scale_dense(ctx, ffn_up);+ }+ if (with_bias) {+ std::array<int64_t, 4> bias_ne = { ffn_up->ne[0], 1, channels, samples };+ ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());+ ffn_up = ggml_add(ctx, ffn_up, up_bias);+ }+ return ffn_up;+ };- ggml_tensor * ffn_gate = with_gate ? ggml_mul_mat(ctx, gate, cur) : nullptr;- if (with_bias && with_gate) {- std::array<int64_t, 4> bias_ne = { ffn_gate->ne[0], 1, channels, samples };- ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());- ffn_gate = ggml_add(ctx, ffn_gate, gate_bias);- }+ auto build_lane_gate = [&]() {+ ggml_tensor * ffn_gate = ggml_mul_mat(ctx, gate, cur);+ if (with_lane_scale) {+ ffn_gate = build_lane_scale_dense(ctx, ffn_gate);+ }+ if (with_bias) {+ std::array<int64_t, 4> bias_ne = { ffn_gate->ne[0], 1, channels, samples };+ ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());+ ffn_gate = ggml_add(ctx, ffn_gate, gate_bias);+ }+ return ffn_gate;+ };++ ggml_tensor * ffn_up = build_lane_up();+ ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr;ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up;@@ -5929,17 +6018,32 @@ struct test_mul_mat_vec_fusion : public test_case {ggml_tensor * cur = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, this->b ? 1 : n_used, m);ggml_set_name(cur, "cur");- ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, ups, cur, ids);- if (with_bias) {- ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats);- ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids);- }+ auto build_lane_up = [&]() {+ ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, ups, cur, ids);+ if (with_lane_scale) {+ ffn_up = build_lane_scale_id(ctx, ctx_weights, ffn_up, ids);+ }+ if (with_bias) {+ ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats);+ ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids);+ }+ return ffn_up;+ };- ggml_tensor * ffn_gate = with_gate? ggml_mul_mat_id(ctx, gates, cur, ids) : nullptr;- if (with_bias && with_gate) {- ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats);- ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids);- }+ auto build_lane_gate = [&]() {+ ggml_tensor * ffn_gate = ggml_mul_mat_id(ctx, gates, cur, ids);+ if (with_lane_scale) {+ ffn_gate = build_lane_scale_id(ctx, ctx_weights, ffn_gate, ids);+ }+ if (with_bias) {+ ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats);+ ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids);+ }+ return ffn_gate;+ };++ ggml_tensor * ffn_up = build_lane_up();+ ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr;ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up;@@ -9202,10 +9306,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {if (!with_gate && glu_op != GGML_GLU_OP_SWIGLU) {continue;}- test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,- use_id, 16, 8, b, with_bias, with_gate));- test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,- use_id, 16, 8, b, with_bias, with_gate, {1, 1}));+ for (bool with_lane_scale : {false, true}) {+ if (with_lane_scale && type != GGML_TYPE_NVFP4) {+ continue;+ }+ test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,+ use_id, 16, 8, b, with_bias, with_gate, with_lane_scale));+ test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,+ use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}));+ }}}}@@ -9823,6 +9932,13 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo}if (mode == MODE_GRAD) {+ test_cases.erase(+ std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr<test_case> & tc) {+ return tc->run_whole_graph();+ }),+ test_cases.end()+ );+size_t n_ok = 0;for (auto & test : test_cases) {if (test->eval_grad(backend, op_names_filter, output_printer)) {