LLM inference in C/C++
C++ ggml
graph : avoid reallocations in the "no cache" path (#30439)
Georgi Gerganov98801578818b
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 231b34dc28..9f044de03e 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -2719,8 +2719,8 @@ void llama_context::sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraphlctx->n_reallocs_same++;}- LLAMA_LOG_INFO("%s: graph reallocation: nodes = %d, splits = %d, n_same = %d, n_diff = %d\n",- __func__, ggml_graph_n_nodes(gf), ggml_backend_sched_get_n_splits(sched),+ LLAMA_LOG_INFO("%s: graph reallocation: nodes = %d (changed = %d), splits = %d, n_same = %d, n_diff = %d\n",+ __func__, ggml_graph_n_nodes(gf), ggml_backend_sched_get_n_splits(sched), graph_size_changed,lctx->n_reallocs_same, lctx->n_reallocs_diff);}diff --git a/src/llama-context.h b/src/llama-context.hindex 277aff71d5..b89a519f77 100644--- a/src/llama-context.h+++ b/src/llama-context.h@@ -280,7 +280,7 @@ private:static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data);// ggml_backend_sched realloc callback, counts and traces the graph reallocations of this context- static void sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);+ static void sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * gf, bool backend_ids_changed, bool graph_size_changed, void * user_data);// disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device// that differs from the layer it belongs to (usually due to missing backend support)diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex f06b94d15c..91af274731 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -3058,10 +3058,9 @@ ggml_tensor * llm_graph_context::build_attn(// the KV values are padded with zeros up to the KV dimension of the maskconst int64_t n_kv_pad = kq_mask->ne[0] - k->ne[2];- if (n_kv_pad > 0) {- k = ggml_pad(ctx0, k, 0, 0, n_kv_pad, 0);- v = ggml_pad(ctx0, v, 0, 0, n_kv_pad, 0);- }++ k = ggml_pad(ctx0, k, 0, 0, n_kv_pad, 0);+ v = ggml_pad(ctx0, v, 0, 0, n_kv_pad, 0);ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);diff --git a/src/models/bert.cpp b/src/models/bert.cppindex 9cc03dc56a..25640445a1 100644--- a/src/models/bert.cpp+++ b/src/models/bert.cpp@@ -73,11 +73,17 @@ llama_model_bert::graph::graph(const llama_model & model, const llm_graph_paramsggml_tensor * cur;ggml_tensor * inpL;ggml_tensor * inp_pos = nullptr;+ ggml_tensor * embd_pos = nullptr;if (model.arch != LLM_ARCH_JINA_BERT_V2) {inp_pos = build_inp_pos();}+ if (model.arch == LLM_ARCH_BERT) {+ embd_pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);+ ggml_build_forward_order(gf, embd_pos);+ }+// construct input embeddings (token, type, position)inpL = build_inp_embd(model.tok_embd);@@ -87,7 +93,7 @@ llama_model_bert::graph::graph(const llama_model & model, const llm_graph_paramsinpL = ggml_add(ctx0, inpL, type_row0);}if (model.arch == LLM_ARCH_BERT) {- inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL);+ inpL = ggml_add(ctx0, embd_pos, inpL);}cb(inpL, "inp_embd", -1);