LLM inference in C/C++
C++ ggml
kv-cache : optimize restoring non-contiguous cells (#27991)
itsnotoger2d8d612e4c68
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex a6d2f264fc..a920c42311 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -2907,17 +2907,28 @@ public:}if (mbuf_cur.n_tensors == mbuf.n_tensors) {- // same chunking: copy 1:1 by index+ // an equal tensor count does not imply the same chunking, e.g. save ranges [2,1] vs restore runs [1,2]+ bool same_chunking = true;for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {- GGML_ASSERT(ggml_nbytes(mbuf_cur.cpy[i]) == ggml_nbytes(mbuf.org[i]));- ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);+ if (ggml_nbytes(mbuf_cur.cpy[i]) != ggml_nbytes(mbuf.org[i])) {+ same_chunking = false;+ break;+ }+ }++ if (same_chunking) {+ // same chunking: copy 1:1 by index+ for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {+ ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);+ }+ continue;}- continue;}// different chunking: copy the write-side data (mbuf_cur.cpy) into the read-side targets (mbuf.org)// with a byte cursor. Write and read enumerate the same logical data in the same order but may chunk- // it differently, so copy across tensor boundaries rather than 1:1 by index.+ // it differently (even with an equal number of tensors), so copy across tensor boundaries rather than+ // 1:1 by index.const size_t total = mbuf_cur.total_size;ggml_init_params params_scratch = {diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cppindex 65afbd8c37..0e095f8b6b 100644--- a/src/llama-kv-cache.cpp+++ b/src/llama-kv-cache.cpp@@ -2533,6 +2533,24 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32_t cell_count, const slot_info & sinfo) {auto & cells = v_cells[strm];+ // batch the scatter reads per contiguous run of destination indices+ // from inclusive, to exclusive - same convention as cell_ranges_t+ // contiguous cells yield a single run covering the whole block+ struct cell_run { uint32_t from; uint32_t to; };+ std::vector<cell_run> runs;+ if (cell_count > 0) {+ const auto & idxs = sinfo.idxs[0];+ uint32_t i0 = 0;+ while (i0 < cell_count) {+ uint32_t i1 = i0 + 1;+ while (i1 < cell_count && idxs[i1] == idxs[i1 - 1] + 1) {+ ++i1;+ }+ runs.push_back({idxs[i0], idxs[i1 - 1] + 1});+ i0 = i1;+ }+ }+uint32_t v_trans;uint32_t n_layer;@@ -2580,17 +2598,8 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32return false;}- if (cell_count) {- if (sinfo.is_contiguous()) {- // Fast path: contiguous cells, single memcpy- io.read_tensor(k, sinfo.head() * k_size_row, cell_count * k_size_row);- } else {- // Slow path: scatter to non-contiguous positions- for (uint32_t i = 0; i < cell_count; ++i) {- const size_t dst_offset = sinfo.idxs[0][i] * k_size_row;- io.read_tensor(k, dst_offset, k_size_row);- }- }+ for (const auto & r : runs) {+ io.read_tensor(k, (size_t) r.from * k_size_row, (size_t) (r.to - r.from) * k_size_row);}}@@ -2623,17 +2632,8 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32return false;}- if (cell_count) {- if (sinfo.is_contiguous()) {- // Fast path: contiguous cells, single memcpy- io.read_tensor(v, sinfo.head() * v_size_row, cell_count * v_size_row);- } else {- // Slow path: scatter to non-contiguous positions- for (uint32_t i = 0; i < cell_count; ++i) {- const size_t dst_offset = sinfo.idxs[0][i] * v_size_row;- io.read_tensor(v, dst_offset, v_size_row);- }- }+ for (const auto & r : runs) {+ io.read_tensor(v, (size_t) r.from * v_size_row, (size_t) (r.to - r.from) * v_size_row);}}} else {@@ -2674,22 +2674,10 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32return false;}- if (cell_count) {- if (sinfo.is_contiguous()) {- // Fast path: contiguous cells- const uint32_t h = sinfo.head();- for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {- const size_t dst_offset = (h + j * cells.size()) * v_size_el;- io.read_tensor(v, dst_offset, cell_count * v_size_el);- }- } else {- // Slow path: scatter to non-contiguous positions- for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {- for (uint32_t i = 0; i < cell_count; ++i) {- const size_t dst_offset = (sinfo.idxs[0][i] + j * cells.size()) * v_size_el;- io.read_tensor(v, dst_offset, v_size_el);- }- }+ for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {+ for (const auto & r : runs) {+ const size_t dst_offset = ((size_t) r.from + j * cells.size()) * v_size_el;+ io.read_tensor(v, dst_offset, (size_t) (r.to - r.from) * v_size_el);}}}diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cppindex 0ceab7c545..35c7690587 100644--- a/tests/test-save-load-state.cpp+++ b/tests/test-save-load-state.cpp@@ -355,7 +355,101 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p}-// Run the full save/load test suite (tests 1-5) for a single model.+// Test 6/7: seq copy (scatter)+// - decode the same prefix on two sequences, interleaving seq 0 cells between the seq 1 cells+// - save the seq 1 state, free the interleaved seq 0 cells, and restore via the given io path+// - the restore destination is non-contiguous: scatter reads are batched per contiguous run+// - save again on the host and compare the two blobs byte for byte+static bool test_seq_cp_scatter(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, int test_num, bool on_device) {+ auto params_ctx = common_context_params_to_llama(params);+ params_ctx.n_ctx = 256;+ params_ctx.n_seq_max = 2;+ params_ctx.kv_unified = true;+ auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};++ LOG("\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host");++ const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;++ auto decode_one = [&](llama_token tok, int pos, llama_seq_id seq) {+ llama_batch_ptr batch(1, 0, 1);+ common_batch_add(batch.get(), tok, pos, { seq }, false);+ return llama_decode(ctx.get(), batch.get()) == 0;+ };++ // seq 0 cells 0,1,4 interleave the seq 1 cells 2,3,5+ if (!decode_one(tokens[0], 0, 0) ||+ !decode_one(tokens[1], 1, 0) ||+ !decode_one(tokens[0], 0, 1) ||+ !decode_one(tokens[1], 1, 1) ||+ !decode_one(tokens[2], 2, 0) ||+ !decode_one(tokens[2], 2, 1)) {+ LOG_ERR("%s: failed to build interleaved state\n", __func__);+ return false;+ }++ const auto get_seq_state = [&](llama_seq_id seq_id, uint32_t fl, std::vector<uint8_t> & state) {+ const size_t state_size = llama_state_seq_get_size_ext(ctx.get(), seq_id, fl);+ if (state_size == 0) {+ LOG_ERR("%s: sequence state is empty\n", __func__);+ return false;+ }++ state.resize(state_size);+ const size_t ncopy = llama_state_seq_get_data_ext(ctx.get(), state.data(), state.size(), seq_id, fl);+ if (ncopy != state.size()) {+ LOG_ERR("%s: sequence state length %zu does not match expected length %zu\n",+ __func__, ncopy, state.size());+ return false;+ }++ return true;+ };++ // host blob: contains the KV data, used for the byte-for-byte comparison+ std::vector<uint8_t> state_before;+ if (!get_seq_state(1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {+ return false;+ }++ // save via the io path under test+ std::vector<uint8_t> state_save;+ if (!get_seq_state(1, flags, state_save)) {+ return false;+ }+ LOG_TRC("%s: seq 1 saved via %s, %zu bytes\n", __func__, on_device ? "device" : "host", state_save.size());++ // free seq 0's cells so the ring is fragmented: the restore destination (seq 1's interleaved cells) stays non-contiguous+ if (!llama_memory_seq_rm(llama_get_memory(ctx.get()), 0, -1, -1)) {+ LOG_ERR("%s: failed to remove sequence 0\n", __func__);+ return false;+ }++ // restore via the io path under test+ const size_t nset = llama_state_seq_set_data_ext(ctx.get(), state_save.data(), state_save.size(), 1, flags);+ if (nset != state_save.size()) {+ LOG_ERR("%s: seq set data length %zu does not match expected length %zu\n", __func__, nset, state_save.size());+ return false;+ }+ LOG_TRC("%s: seq 1 restored via %s, %zu bytes\n", __func__, on_device ? "device" : "host", nset);++ std::vector<uint8_t> state_after;+ if (!get_seq_state(1, LLAMA_STATE_SEQ_FLAGS_NONE, state_after)) {+ return false;+ }++ // the blob is serialized in sequence cell order, so identical bytes iff the restore wrote the same KV+ if (state_before.size() != state_after.size() || memcmp(state_before.data(), state_after.data(), state_before.size()) != 0) {+ LOG_ERR("\n%s: error: restored KV state is not byte-identical to the saved state\n", __func__);+ return false;+ }++ LOG("\nPASS\n");+ return true;+}+++// Run the full save/load test suite (tests 1-7) for a single model.// Returns true if all tests pass, false otherwise.static bool run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {struct common_params params = base_params;@@ -422,6 +516,16 @@ static bool run_save_load_tests_for_model(const std::string & model_path, constreturn false;}+ // Test 6: seq copy (host, scatter)+ if (!test_seq_cp_scatter(model, params, tokens, 6, false)) {+ return false;+ }++ // Test 7: seq copy (device, scatter)+ if (!test_seq_cp_scatter(model, params, tokens, 7, true)) {+ return false;+ }+LOG("\nAll tests passed.\n");return true;