LLM inference in C/C++
C++ ggml
llama-context : sync pending async copies before clearing embd_seq (#25676)
o7si432d7ffe2c3b
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 9b399d6096..3e0af3f317 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -474,6 +474,9 @@ llama_context::llama_context(}llama_context::~llama_context() {+ // wait for any pending asynchronous copies into the output buffers before they are freed+ synchronize();+if (!model.hparams.no_alloc) {for (size_t i = 0; i < backend_ptrs.size(); ++i) {ggml_backend_t backend = backend_ptrs[i];@@ -1417,13 +1420,17 @@ int llama_context::encode(const llama_batch & batch_inp) {// micro-batching is not possible for non-causal encoding, so we process the batch in a single shotGGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");+ // TODO: this clear of the buffer can easily be forgotten - need something better+ // sync first so any in-flight async copies into embd_seq complete before it is freed+ if (!embd_seq.empty()) {+ synchronize();+ }+ embd_seq.clear();+if (t_compute_start_us == 0) {t_compute_start_us = ggml_time_us();}- // TODO: this clear of the buffer can easily be forgotten - need something better- embd_seq.clear();-sched_reserve();n_queued_tokens += n_tokens;@@ -1762,13 +1769,18 @@ int llama_context::decode(const llama_batch & batch_inp) {GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");+ // TODO: this clear of the buffer can easily be forgotten - need something better+ // sync first so any in-flight async copies into embd_seq complete before it is freed+ if (!embd_seq.empty()) {+ synchronize();+ }+ embd_seq.clear();+if (t_compute_start_us == 0) {t_compute_start_us = ggml_time_us();}n_queued_tokens += n_tokens_all;- // TODO: this clear of the buffer can easily be forgotten - need something better- embd_seq.clear();output_swaps.clear();sched_reserve();