Skip to content

LLM inference in C/C++

C++ ggml

llama: refactor fused ops (#24646)

Aman Gupta90e0f5cfcb6c
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 0465430df4..5edfc85abf 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -17,6 +17,7 @@
#include <cstring>
#include <limits>
#include <stdexcept>
+#include <string>
//
// llama_context
@@ -30,6 +31,30 @@ static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {
throw std::runtime_error("Unsupported ctx type");
}
+struct llm_fused_op_probe {
+ llm_fused_op op;
+ const char * name;
+ uint32_t n_tokens_per_seq;
+};
+
+static const llm_fused_op_probe llm_fused_op_flash_attn_probe = {
+ /*.op =*/ LLM_FUSED_OP_FLASH_ATTN,
+ /*.name =*/ "Flash Attention",
+ /*.n_tokens_per_seq =*/ 1,
+};
+
+static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = {
+ /*.op =*/ LLM_FUSED_OP_GDN_AR,
+ /*.name =*/ "fused Gated Delta Net (autoregressive)",
+ /*.n_tokens_per_seq =*/ 1,
+};
+
+static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = {
+ /*.op =*/ LLM_FUSED_OP_GDN_CH,
+ /*.name =*/ "fused Gated Delta Net (chunked)",
+ /*.n_tokens_per_seq =*/ 16,
+};
+
llama_context::llama_context(
const llama_model & model,
llama_context_params params) :
@@ -436,6 +461,69 @@ llama_context::~llama_context() {
ggml_opt_free(opt_ctx);
}
+void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs) {
+ const char * func = __func__;
+ auto resolve = [&](const llm_fused_op_probe & probe, bool & enabled) {
+ if (!enabled) {
+ return;
+ }
+
+ const uint32_t n_tokens_probe = probe.n_tokens_per_seq*n_seqs;
+
+ auto * gf = graph_reserve(n_tokens_probe, n_seqs, n_tokens_probe, mctx, true);
+ if (!gf) {
+ throw std::runtime_error(std::string("failed to reserve graph for ") + probe.name + " check");
+ }
+
+ bool device_mismatch = false;
+ for (const auto & node : get_gf_res_reserve()->get_fused_nodes()) {
+ if (node.op != probe.op) {
+ continue;
+ }
+
+ GGML_ASSERT(node.il >= 0);
+
+ ggml_backend_t backend_fused = ggml_backend_sched_get_tensor_backend(sched.get(), node.tensor);
+ ggml_backend_dev_t device_fused = backend_fused ? ggml_backend_get_device(backend_fused) : nullptr;
+
+ // TODO: make this descriptor-specific; model.dev_layer() preserves the current behavior,
+ // but is still wrong for cases like --no-kv-offload.
+ ggml_backend_dev_t device_layer = model.dev_layer(node.il);
+
+ if (device_fused != device_layer) {
+ LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but %s "
+ "is assigned to device %s (usually due to missing support)\n",
+ func, node.il,
+ device_layer ? ggml_backend_dev_name(device_layer) : "none",
+ probe.name,
+ device_fused ? ggml_backend_dev_name(device_fused) : "none");
+ device_mismatch = true;
+ break;
+ }
+ }
+
+ if (device_mismatch) {
+ enabled = false;
+ LLAMA_LOG_WARN("%s: %s not supported, set to disabled\n", func, probe.name);
+ } else {
+ enabled = true;
+ LLAMA_LOG_INFO("%s: %s enabled\n", func, probe.name);
+ }
+ };
+
+ if (cparams.auto_fa) {
+ resolve(llm_fused_op_flash_attn_probe, cparams.flash_attn);
+ cparams.auto_fa = false;
+ }
+
+ if (cparams.auto_fgdn) {
+ LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func);
+ resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar);
+ resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch);
+ cparams.auto_fgdn = false;
+ }
+}
+
void llama_context::sched_reserve() {
if (!sched_need_reserve) {
return;
@@ -475,128 +563,7 @@ void llama_context::sched_reserve() {
LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);
- // resolve automatic Flash Attention use
- if (cparams.auto_fa) {
- auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);
- if (!gf) {
- throw std::runtime_error("failed to reserve graph for Flash Attention check");
- }
-
- const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN) + 1;
- bool fa_device_mismatch = false;
- for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
- ggml_tensor * n = ggml_graph_node(gf, i);
- if (n->op != GGML_OP_FLASH_ATTN_EXT) {
- continue;
- }
- ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
-
- // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer
- GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0);
- const int il = std::stoi(n->name + prefix_len);
- ggml_backend_dev_t device_kv = model.dev_layer(il);
- if (device_fa != device_kv) {
- LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor "
- "is assigned to device %s (usually due to missing support)\n",
- __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa));
- // FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways
- fa_device_mismatch = true;
- break;
- }
- }
-
- if (fa_device_mismatch) {
- cparams.flash_attn = false;
- LLAMA_LOG_WARN("%s: Flash Attention was auto, set to disabled\n", __func__);
- } else {
- cparams.flash_attn = true;
- LLAMA_LOG_INFO("%s: Flash Attention was auto, set to enabled\n", __func__);
- }
-
- cparams.auto_fa = false;
- }
-
- if (cparams.auto_fgdn) {
- LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", __func__);
-
- if (cparams.fused_gdn_ar) {
- auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);
- if (!gf) {
- throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (autoregressive)");
- }
-
- const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR) + 1;
- bool gdn_device_mismatch = false;
- for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
- ggml_tensor * n = ggml_graph_node(gf, i);
- if (n->op != GGML_OP_GATED_DELTA_NET) {
- continue;
- }
- ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
-
- GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0);
- const int il = std::stoi(n->name + prefix_len);
- ggml_backend_dev_t device_kv = model.dev_layer(il);
- if (device_gdn != device_kv) {
- LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "
- "is assigned to device %s (usually due to missing support)\n",
- __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));
- gdn_device_mismatch = true;
- break;
- }
- }
-
- if (gdn_device_mismatch) {
- cparams.fused_gdn_ar = false;
- LLAMA_LOG_WARN("%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n", __func__);
- } else {
- LLAMA_LOG_INFO("%s: fused Gated Delta Net (autoregressive) enabled\n", __func__);
- }
- }
-
- if (cparams.fused_gdn_ch) {
- // more than one token in the batch per sequence in order to take the chunked path
- // note: n_outputs must match n_tokens for embedding models with mean/rank pooling,
- // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies
- // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens,
- // the ggml_mul_mat assertion fails.
- const uint32_t n_tokens_ch = 16*n_seqs;
- auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true);
- if (!gf) {
- throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (chunked)");
- }
-
- const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH) + 1;
- bool gdn_device_mismatch = false;
- for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
- ggml_tensor * n = ggml_graph_node(gf, i);
- if (n->op != GGML_OP_GATED_DELTA_NET) {
- continue;
- }
- ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
-
- GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0);
- const int il = std::stoi(n->name + prefix_len);
- ggml_backend_dev_t device_kv = model.dev_layer(il);
- if (device_gdn != device_kv) {
- LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "
- "is assigned to device %s (usually due to missing support)\n",
- __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));
- gdn_device_mismatch = true;
- break;
- }
- }
-
- if (gdn_device_mismatch) {
- cparams.fused_gdn_ch = false;
- LLAMA_LOG_WARN("%s: fused Gated Delta Net (chunked) not supported, set to disabled\n", __func__);
- } else {
- LLAMA_LOG_INFO("%s: fused Gated Delta Net (chunked) enabled\n", __func__);
- }
- }
-
- cparams.auto_fgdn = false;
- }
+ resolve_fused_ops(mctx.get(), n_seqs);
// reserve worst-case graph
int n_splits_pp = -1;
diff --git a/src/llama-context.h b/src/llama-context.h
index f8b7805871..bf91daa8b5 100644
--- a/src/llama-context.h
+++ b/src/llama-context.h
@@ -262,6 +262,10 @@ private:
llm_graph_cb graph_get_cb() const;
+ // disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device
+ // that differs from the layer it belongs to (usually due to missing backend support)
+ void resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs);
+
// TODO: read/write lora adapters and cvec
size_t state_write_data(llama_io_write_i & io);
size_t state_read_data (llama_io_read_i & io);
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index d78ddf827b..8c9228b38f 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1192,6 +1192,7 @@ void llm_graph_result::reset() {
params = {};
inputs.clear();
+ fused_nodes.clear();
buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
@@ -1293,6 +1294,10 @@ llm_graph_input_i * llm_graph_result::add_input(llm_graph_input_ptr input) {
return inputs.back().get();
}
+void llm_graph_result::add_fused_node(llm_graph_fused_node result) {
+ fused_nodes.push_back(result);
+}
+
void llm_graph_result::set_params(const llm_graph_params & params) {
this->params = params;
}
@@ -1352,6 +1357,8 @@ void llm_graph_context::cb(ggml_tensor * cur, const char * name, int il) const {
}
}
+
+
ggml_tensor * llm_graph_context::build_cvec(
ggml_tensor * cur,
int il) const {
@@ -2402,7 +2409,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias,
hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f);
- cb(cur, LLAMA_TENSOR_NAME_FATTN, il);
+ res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il});
ggml_flash_attn_ext_add_sinks(cur, sinks);
ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 4b5b75c632..97141ef93b 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -38,6 +38,12 @@ enum llm_graph_type {
LLM_GRAPH_TYPE_DECODER_MTP,
};
+enum llm_fused_op {
+ LLM_FUSED_OP_FLASH_ATTN,
+ LLM_FUSED_OP_GDN_AR,
+ LLM_FUSED_OP_GDN_CH,
+};
+
enum llm_ffn_op_type : int {
LLM_FFN_NONE = 0, // sentinel: unset; archs must assign before use
LLM_FFN_SILU,
@@ -775,6 +781,12 @@ struct llm_graph_params {
}
};
+struct llm_graph_fused_node {
+ llm_fused_op op;
+ ggml_tensor * tensor;
+ int il;
+};
+
class llm_graph_result {
public:
llm_graph_result(int64_t max_nodes);
@@ -808,6 +820,10 @@ public:
llm_graph_input_i * add_input(llm_graph_input_ptr input);
+ void add_fused_node(llm_graph_fused_node result);
+
+ const std::vector<llm_graph_fused_node> & get_fused_nodes() const { return fused_nodes; }
+
void set_params(const llm_graph_params & params);
// important graph nodes
@@ -826,6 +842,7 @@ public:
std::map<llama_seq_id, ggml_tensor *> t_sampled_probs;
std::vector<llm_graph_input_ptr> inputs;
+ std::vector<llm_graph_fused_node> fused_nodes;
ggml_context_ptr ctx_compute;
diff --git a/src/llama-impl.h b/src/llama-impl.h
index 2d06752c94..4988b06d2c 100644
--- a/src/llama-impl.h
+++ b/src/llama-impl.h
@@ -103,7 +103,3 @@ std::string llama_format_tensor_shape(const std::vector<int64_t> & ne);
std::string llama_format_tensor_shape(const struct ggml_tensor * t);
std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i);
-
-#define LLAMA_TENSOR_NAME_FATTN "__fattn__"
-#define LLAMA_TENSOR_NAME_FGDN_AR "__fgdn_ar__"
-#define LLAMA_TENSOR_NAME_FGDN_CH "__fgdn_ch__"
diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp
index cf5e380950..ad66126477 100644
--- a/src/models/delta-net-base.cpp
+++ b/src/models/delta-net-base.cpp
@@ -401,9 +401,9 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne
// K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs].
ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
if (n_tokens == 1) {
- cb(result, LLAMA_TENSOR_NAME_FGDN_AR, il);
+ res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il});
} else {
- cb(result, LLAMA_TENSOR_NAME_FGDN_CH, il);
+ res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il});
}
ggml_tensor * output = ggml_view_4d(ctx0, result,
@@ -566,9 +566,9 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
// state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output.
ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
if (n_seq_tokens > 1) {
- cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_CH, il);
+ res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il});
} else {
- cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_AR, il);
+ res->add_fused_node({LLM_FUSED_OP_GDN_AR, gdn_out, il});
}
const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs;