Skip to content

LLM inference in C/C++

C++ ggml

mtmd/ggml: add ggml_build_forward_order (#26649)

Pascalc8e03ce8122b
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 35f0c44ec4..5cb49d0ee4 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -2788,6 +2788,12 @@ extern "C" {
struct ggml_cgraph * cgraph,
struct ggml_tensor * tensor);
+ // add the tensor and its parents to the graph without marking them for compute
+ // the flag is set later, when the tensor is reached from a node that computes
+ GGML_API void ggml_build_forward_order(
+ struct ggml_cgraph * cgraph,
+ struct ggml_tensor * tensor);
+
GGML_API void ggml_build_backward_expand(
struct ggml_context * ctx, // context for gradient computation
struct ggml_cgraph * cgraph,
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index 59191c663e..da7f3a5f2e 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -7200,6 +7200,10 @@ void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor *
ggml_build_forward_impl(cgraph, tensor, true, true);
}
+void ggml_build_forward_order(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {
+ ggml_build_forward_impl(cgraph, tensor, true, false);
+}
+
void ggml_build_backward_expand(
struct ggml_context * ctx,
struct ggml_cgraph * cgraph,
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index c77e7cc9d3..b1360fd7d3 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -708,9 +708,10 @@ ggml_tensor * clip_graph::build_attn(
ggml_tensor * sinks) const {
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
- ggml_build_forward_expand(gf, q_cur);
- ggml_build_forward_expand(gf, k_cur);
- ggml_build_forward_expand(gf, v_cur);
+ // the order is fixed without the compute flag, so an unselected branch stays out of the compute set
+ ggml_build_forward_order(gf, q_cur);
+ ggml_build_forward_order(gf, k_cur);
+ ggml_build_forward_order(gf, v_cur);
ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3);
//cb(q, "q", il);