Skip to content

LLM inference in C/C++

C++ ggml

llama-context : sync pending async copies before clearing embd_seq (#25676)

o7si432d7ffe2c3b
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 9b399d6096..3e0af3f317 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -474,6 +474,9 @@ llama_context::llama_context(
}
llama_context::~llama_context() {
+ // wait for any pending asynchronous copies into the output buffers before they are freed
+ synchronize();
+
if (!model.hparams.no_alloc) {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
@@ -1417,13 +1420,17 @@ int llama_context::encode(const llama_batch & batch_inp) {
// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");
+ // TODO: this clear of the buffer can easily be forgotten - need something better
+ // sync first so any in-flight async copies into embd_seq complete before it is freed
+ if (!embd_seq.empty()) {
+ synchronize();
+ }
+ embd_seq.clear();
+
if (t_compute_start_us == 0) {
t_compute_start_us = ggml_time_us();
}
- // TODO: this clear of the buffer can easily be forgotten - need something better
- embd_seq.clear();
-
sched_reserve();
n_queued_tokens += n_tokens;
@@ -1762,13 +1769,18 @@ int llama_context::decode(const llama_batch & batch_inp) {
GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");
+ // TODO: this clear of the buffer can easily be forgotten - need something better
+ // sync first so any in-flight async copies into embd_seq complete before it is freed
+ if (!embd_seq.empty()) {
+ synchronize();
+ }
+ embd_seq.clear();
+
if (t_compute_start_us == 0) {
t_compute_start_us = ggml_time_us();
}
n_queued_tokens += n_tokens_all;
- // TODO: this clear of the buffer can easily be forgotten - need something better
- embd_seq.clear();
output_swaps.clear();
sched_reserve();