LLM inference in C/C++
C++ ggml
tests : run test-save-load-state across all architectures (#27755)
Georgi Gerganov4e97ac86ebe2
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex fb88919f9d..179c526c29 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -661,11 +661,19 @@ void llama_context::sched_reserve() {// reserve again with pp graph to avoid ggml-alloc reallocations during inference{- // TODO: not sure if the following graph would be worst case for multi-stream KV caches:- //- // auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());- //- auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);+ // TODO: the worst case graph is not always reached for `n_seqs > 1`+ // need to implement a more robust mechanism that tries a few different inputs and analyzes the results+ ggml_cgraph * gf = nullptr;+ switch (model.arch) {+ case LLM_ARCH_MINIMAX_01:+ // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which+ // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`+ gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);+ break;+ default:+ gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);+ };+if (!gf) {throw std::runtime_error("failed to allocate compute pp buffers");}@@ -2892,13 +2900,83 @@ public:for (auto & [buft, mbuf] : mbufs_new) {const auto & mbuf_cur = mbufs.at(buft);- if (!mbuf_cur.buf || mbuf_cur.n_tensors != mbuf.n_tensors || mbuf_cur.total_size != mbuf.total_size) {+ if (!mbuf_cur.buf || mbuf_cur.total_size != mbuf.total_size) {GGML_ABORT("%s: memory buffer mismatch\n", __func__);}- for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {- ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);+ if (mbuf_cur.n_tensors == mbuf.n_tensors) {+ // same chunking: copy 1:1 by index+ for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {+ GGML_ASSERT(ggml_nbytes(mbuf_cur.cpy[i]) == ggml_nbytes(mbuf.org[i]));+ ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);+ }+ continue;}++ // different chunking: copy the write-side data (mbuf_cur.cpy) into the read-side targets (mbuf.org)+ // with a byte cursor. Write and read enumerate the same logical data in the same order but may chunk+ // it differently, so copy across tensor boundaries rather than 1:1 by index.+ const size_t total = mbuf_cur.total_size;++ ggml_init_params params_scratch = {+ /*.mem_size =*/ 2*(mbuf_cur.cpy.size() + mbuf.org.size())*ggml_tensor_overhead(),+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ true,+ };+ ggml_context * ctx_scratch = ggml_init(params_scratch);++ size_t src_pos = 0;+ size_t dst_pos = 0;+ size_t src_j = 0;+ size_t dst_i = 0;+ size_t src_base = 0;+ size_t dst_base = 0;++ while (src_pos < total) {+ const auto & src_t = mbuf_cur.cpy[src_j];+ const auto & dst_t = mbuf.org[dst_i];++ const size_t src_size = ggml_nbytes(src_t);+ const size_t dst_size = ggml_nbytes(dst_t);++ const size_t src_off = src_pos - src_base;+ const size_t dst_off = dst_pos - dst_base;++ const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off);++ const size_t el = ggml_element_size(src_t);+ const int64_t n_el = (int64_t) (n_copy / el);++ auto * src_v = ggml_view_1d(ctx_scratch, src_t, n_el, src_off);+ ggml_backend_view_init(src_v);+ auto * dst_v = ggml_view_1d(ctx_scratch, dst_t, n_el, dst_off);+ ggml_backend_view_init(dst_v);++ ggml_backend_tensor_copy(src_v, dst_v);++ src_pos += n_copy;+ dst_pos += n_copy;++ if (src_pos - src_base == src_size) {+ src_base = src_pos;+ ++src_j;+ }+ if (dst_pos - dst_base == dst_size) {+ dst_base = dst_pos;+ ++dst_i;+ }+ }++ GGML_ASSERT(src_pos == total && dst_pos == total);+ // any tensors left unvisited hold no data+ for (size_t i = src_j; i < mbuf_cur.cpy.size(); ++i) {+ GGML_ASSERT(ggml_nbytes(mbuf_cur.cpy[i]) == 0);+ }+ for (size_t i = dst_i; i < mbuf.org.size(); ++i) {+ GGML_ASSERT(ggml_nbytes(mbuf.org[i]) == 0);+ }++ ggml_free(ctx_scratch);}GGML_ASSERT(buf_size == 0);diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cppindex f14626b2c7..361114acc3 100644--- a/src/models/minimax-01.cpp+++ b/src/models/minimax-01.cpp@@ -181,7 +181,7 @@ public:return res;}- const llama_hparams & hparams;+ const llama_hparams hparams;ggml_tensor * inp_slopes = nullptr; // F32 [n_head]ggml_tensor * inp_q_decay = nullptr; // F32 [1, n_head, n_batch]diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex b9f9d4b78a..fe3d14ffc5 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -149,6 +149,7 @@ if (LLAMA_LLGUIDANCE)endif ()llama_build(test-recurrent-state-rollback.cpp)+llama_build(test-save-load-state.cpp)if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)# these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries)@@ -237,6 +238,14 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)set_tests_properties(test-recurrent-state-rollback-dsv4 PROPERTIESFIXTURES_REQUIRED generate-models)++ # Test state save/load functionality across all architectures, using the generated dummy models+ llama_test(+ test-save-load-state+ LABEL main+ ARGS --models "${MODEL_DIR}"+ )+ set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED generate-models)endif()llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp)@@ -299,10 +308,6 @@ llama_build_and_test(test-backend-sampler.cpp LABEL "model")llama_build_and_test(test-state-restore-fragmented.cpp LABEL "model" ARGS -m "${MODEL_DEST}")set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED test-download-model)-# Test state save/load functionality-llama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}")-set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model)-if (APPLE)llama_build(test-rset-release.cpp)endif()diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex d58d90952e..35a3286e4a 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -65,7 +65,7 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {}static void usage(char ** argv) {- printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-v/--verbose]\n", argv[0]);+ printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v/--verbose] [-h/--help]\n", argv[0]);}static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){@@ -82,7 +82,7 @@ static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {gguf_context_ptr ret(gguf_init_empty());llama_model_saver ms(arch, ret.get());- const uint32_t n_ctx = 128;+ const uint32_t n_ctx = 256;uint32_t n_vocab = 128;uint32_t n_embd = 256;@@ -256,10 +256,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>(n_layer, 4));}- ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 || arch == LLM_ARCH_DEEPSEEK4 ? n_head : uint32_t(1));+ // minimax-m3 keeps one indexer head per GQA head; the rest use a fixed 64 to match the fused+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(64));// qwen4exp ropes indexer keys with the main rotary width, so its head can't be < n_rotms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,- arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(64));+ arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));+ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));@@ -762,6 +764,10 @@ int main(int argc, char ** argv) {std::string out;for (int i = 1; i < argc; i++) {+ if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {+ usage(argv);+ return 0;+ }if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {if (i + 1 < argc) {const std::string arch_name = argv[++i];diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cppindex 6e93ce6fb8..0ceab7c545 100644--- a/tests/test-save-load-state.cpp+++ b/tests/test-save-load-state.cpp@@ -3,8 +3,12 @@#include "log.h"#include "llama-cpp.h"+#include <algorithm>#include <clocale>+#include <cstring>+#include <filesystem>#include <random>+#include <string>#include <vector>struct llama_batch_ptr {@@ -53,7 +57,9 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i// - decode the last token// - generate n_predict tokensstatic llama_tokens test_baseline(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens) {- auto ctx = llama_context_ptr{llama_init_from_model(model, common_context_params_to_llama(params))};+ auto params_ctx = common_context_params_to_llama(params);+ params_ctx.n_seq_max = 2;+ auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};auto sparams = llama_sampler_chain_default_params();auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};@@ -161,7 +167,9 @@ static bool test_seq_rm_isolated(// - replay the last prompt token// - generate n_predict tokens and compare against expected resultstatic bool test_state_load(struct llama_model * model, const struct common_params & params, const llama_tokens & tokens, const llama_tokens & expected_result) {- auto ctx = llama_context_ptr{llama_init_from_model(model, common_context_params_to_llama(params))};+ auto params_ctx = common_context_params_to_llama(params);+ params_ctx.n_seq_max = 2;+ auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};auto sparams = llama_sampler_chain_default_params();auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};@@ -347,38 +355,18 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p}-int main(int argc, char ** argv) {- std::setlocale(LC_NUMERIC, "C");-- common_params params;- params.prompt = "";- params.n_batch = 100;- params.out_file = "dump_state.bin";- params.sampling.seed = 1234;-- common_init();-- if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {- return 1;- }-- if (params.n_parallel == 1) {- LOG_TRC("%s: n_parallel == 1, enabling unified kv cache\n", __func__);- params.kv_unified = true;- }-- if (params.n_predict < 0) {- params.n_predict = 16;- }-- ggml_backend_load_all();+// Run the full save/load test suite (tests 1-5) for a single model.+// Returns true if all tests pass, false otherwise.+static bool run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {+ struct common_params params = base_params;+ params.model.path = model_path;auto llama_init = common_init_from_params(params, true);auto * model = llama_init->model();if (model == nullptr) {- LOG_ERR("%s: failed to init\n", __func__);- return 1;+ LOG_ERR("%s: failed to init model '%s'\n", __func__, model_path.c_str());+ return false;}GGML_ASSERT(llama_init->context() == nullptr);@@ -411,30 +399,127 @@ int main(int argc, char ** argv) {// Test 1: baseline (saves state to disk)auto result_baseline = test_baseline(model, params, tokens);if (result_baseline.empty()) {- return 1;+ return false;}// Test 2: sequence removal isolationif (!test_seq_rm_isolated(model, params, tokens)) {- return 1;+ return false;}// Test 3: state loadif (!test_state_load(model, params, tokens, result_baseline)) {- return 1;+ return false;}// Test 4: seq copy (host)if (!test_seq_cp_host(model, params, tokens, result_baseline)) {- return 1;+ return false;}// Test 5: seq copy (device)if (!test_seq_cp_device(model, params, tokens, result_baseline)) {- return 1;+ return false;}LOG("\nAll tests passed.\n");- return 0;+ return true;+}+++int main(int argc, char ** argv) {+ std::setlocale(LC_NUMERIC, "C");++ common_params params;+ params.prompt = "";+ params.n_batch = 100;+ params.out_file = "dump_state.bin";+ params.sampling.seed = 1234;++ common_init();++ // extract our own --models DIR option before handing the rest to the common arg parser+ std::string models_dir;+ std::vector<char *> filtered_argv;+ filtered_argv.push_back(argv[0]);+ for (int i = 1; i < argc; i++) {+ if (strcmp(argv[i], "--models") == 0) {+ if (i + 1 >= argc) {+ LOG_ERR("%s: --models requires a directory argument\n", __func__);+ return 1;+ }+ models_dir = argv[i + 1];+ i++;+ } else {+ filtered_argv.push_back(argv[i]);+ }+ }+ filtered_argv.push_back(nullptr);+ const int fargc = (int)filtered_argv.size() - 1;++ // in --models mode there is no single model; set a placeholder so the common parser's+ // "--model is required" check passes (each model is set individually inside the loop)+ if (!models_dir.empty()) {+ params.model.path = models_dir;+ }++ if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {+ return 1;+ }++ if (params.n_parallel == 1) {+ LOG_TRC("%s: n_parallel == 1, enabling unified kv cache\n", __func__);+ params.kv_unified = true;+ }++ if (params.n_predict < 0) {+ params.n_predict = 16;+ }++ ggml_backend_load_all();++ if (!models_dir.empty()) {+ // run the suite over every dummy model in the directory+ if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {+ LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());+ return 1;+ }++ std::vector<std::string> models;+ for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {+ if (entry.is_regular_file() && entry.path().extension() == ".gguf") {+ models.push_back(entry.path().string());+ }+ }+ std::sort(models.begin(), models.end());++ if (models.empty()) {+ LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());+ return 1;+ }++ LOG_INF("%s: running save/load tests over %zu models in '%s'\n", __func__, models.size(), models_dir.c_str());++ size_t n_pass = 0;+ size_t n_fail = 0;+ for (const auto & model_path : models) {+ LOG("\n================================================================\n");+ LOG_INF("%s: model %s\n", __func__, model_path.c_str());++ if (run_save_load_tests_for_model(model_path, params)) {+ n_pass++;+ } else {+ n_fail++;+ }+ }++ LOG("\n================================================================\n");+ LOG_INF("%s: summary: %zu passed, %zu failed (of %zu)\n", __func__, n_pass, n_fail, models.size());++ return n_fail == 0 ? 0 : 1;+ }++ // single-model mode+ return run_save_load_tests_for_model(params.model.path, params) ? 0 : 1;}