LLM inference in C/C++
C++ ggml
tests : refactor test-recurrent-state-rollback (#29426)
Georgi Gerganovd77dd0806dc2
diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cppindex eac3bd6fef..e55b01503d 100644--- a/ggml/src/ggml-metal/ggml-metal-fusion.cpp+++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp@@ -418,7 +418,8 @@ static bool ggml_metal_fusion_check_topk_moe(const int64_t n_tokens = logits->ne[1];const int64_t n_expert_used = ids->ne[0];- if (n_expert <= 0 || n_tokens <= 0 || n_expert_used <= 0 || n_expert_used > n_expert ||+ // note: n_tokens == 0 (no-output batch) must match so that the packing stays shape-independent+ if (n_expert <= 0 || n_expert_used <= 0 || n_expert_used > n_expert ||n_expert > GGML_METAL_TOPK_MOE_MAX_EXPERTS || n_expert_used > GGML_METAL_TOPK_MOE_MAX_EXPERTS) {return false;}@@ -545,7 +546,8 @@ static bool ggml_metal_fusion_match_moe_reduce(const int64_t n_embd = experts->ne[0];const int64_t n_tokens = experts->ne[2];- if (n_embd <= 0 || n_tokens <= 0 || experts->ne[1] != n_expert_used || experts->ne[3] != 1 ||+ // note: n_tokens == 0 (no-output batch) must match so that the packing stays shape-independent+ if (n_embd <= 0 || experts->ne[1] != n_expert_used || experts->ne[3] != 1 ||weights->ne[0] != 1 || weights->ne[1] != n_expert_used || weights->ne[2] != n_tokens || weights->ne[3] != 1 ||dst->ne[0] != n_embd || dst->ne[1] != n_tokens || dst->ne[2] != 1 || dst->ne[3] != 1) {return false;@@ -1079,16 +1081,17 @@ const ggml_metal_fusion * ggml_metal_fusion_next(// transparent) node sequence that the compute phase uses, so the returned count is the raw index// span from idx to the last matched node (intermediate views are packed along).int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) {- // an empty/view node cannot start a pattern - pack it alone- if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) {+ // a view node cannot start a pattern - pack it alone+ if (ggml_op_is_empty(gf->nodes[idx]->op)) {return 1;}- // collect the non-empty node indices starting at idx+ // collect the non-view node indices starting at idx; 0-element tensors are included so+ // that empty graphs pack like their non-empty counterparts (see ggml_metal_fusion_filter_ops)int idxs[GGML_METAL_FUSION_MAX];int n_idxs = 0;for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) {- if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) {+ if (!ggml_op_is_empty(gf->nodes[i]->op)) {idxs[n_idxs++] = i;}}diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex 01af4ae373..96d910a752 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -207,42 +207,13 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)FIXTURES_SETUP generate-models)+ # Test recurrent-state rollback across all architectures, using the generated dummy modelsllama_test(test-recurrent-state-rollbackLABEL main- ARGS -m "${MODEL_DIR}/qwen35-dense.gguf"- )- set_tests_properties(test-recurrent-state-rollback PROPERTIES- FIXTURES_REQUIRED generate-models- )-- llama_test(- test-recurrent-state-rollback- NAME test-recurrent-state-rollback-nemotron-h- LABEL main- ARGS -m "${MODEL_DIR}/nemotron_h-dense.gguf"- )- set_tests_properties(test-recurrent-state-rollback-nemotron-h PROPERTIES- FIXTURES_REQUIRED generate-models- )- llama_test(- test-recurrent-state-rollback- NAME test-recurrent-state-rollback-dsv4- LABEL main- ARGS -m "${MODEL_DIR}/deepseek4-moe.gguf"- )- set_tests_properties(test-recurrent-state-rollback-dsv4 PROPERTIES- FIXTURES_REQUIRED generate-models- )- llama_test(- test-recurrent-state-rollback- NAME test-recurrent-state-rollback-kimi-k3- LABEL main- ARGS -m "${MODEL_DIR}/kimi-k3-moe.gguf"- )- set_tests_properties(test-recurrent-state-rollback-kimi-k3 PROPERTIES- FIXTURES_REQUIRED generate-models+ ARGS --models "${MODEL_DIR}")+ set_tests_properties(test-recurrent-state-rollback PROPERTIES FIXTURES_REQUIRED generate-models)# Test state save/load functionality across all architectures, using the generated dummy modelsllama_test(diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cppindex 1944a441cb..4ad0d6f9ef 100644--- a/tests/test-recurrent-state-rollback.cpp+++ b/tests/test-recurrent-state-rollback.cpp@@ -1,6 +1,8 @@#include "arg.h"#include "common.h"#include "ggml-backend.h"+#include "log.h"+#include "llama-cpp.h"#include "llama.h"#include "../src/llama-io.h"@@ -10,10 +12,28 @@#include <clocale>#include <cmath>#include <cstdio>+#include <cstring>+#include <filesystem>#include <limits>#include <set>+#include <string>#include <vector>+enum class test_status {+ PASS,+ FAIL,+ SKIP,+};++static const char * test_status_str(test_status status) {+ switch (status) {+ case test_status::PASS: return "\033[1;32mPASS\033[m";+ case test_status::FAIL: return "\033[1;31mFAIL\033[m";+ case test_status::SKIP: return "\033[1;33mSKIP\033[m";+ }+ return "";+}+static bool decode_tokens(llama_context * ctx, const std::vector<llama_token> & tokens, uint32_t count) {llama_batch batch = llama_batch_init(count, 0, 1);for (uint32_t pos = 0; pos < count; ++pos) {@@ -50,25 +70,23 @@ struct cache_buffer_collector : llama_io_write_i {}};-static llama_context * init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {- llama_context * ctx = llama_init_from_model(model, cparams);- if (ctx == nullptr || fill == 0) {+static llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {+ llama_context_ptr ctx{llama_init_from_model(model, cparams)};+ if (!ctx || fill == 0) {return ctx;}// Use a full ubatch so buffer discovery preserves prefill allocation sizes.- const uint32_t n_tokens = llama_n_ubatch(ctx);- if (!decode_tokens(ctx, std::vector<llama_token>(n_tokens, 0), n_tokens)) {- llama_free(ctx);+ const uint32_t n_tokens = llama_n_ubatch(ctx.get());+ if (!decode_tokens(ctx.get(), std::vector<llama_token>(n_tokens, 0), n_tokens)) {return nullptr;}- llama_synchronize(ctx);+ llama_synchronize(ctx.get());cache_buffer_collector collector;- llama_get_memory(ctx)->state_write(collector);- llama_memory_clear(llama_get_memory(ctx), true);+ llama_get_memory(ctx.get())->state_write(collector);+ llama_memory_clear(llama_get_memory(ctx.get()), true);if (collector.buffers.empty()) {- fprintf(stderr, "%s : no cache buffers found\n", __func__);- llama_free(ctx);+ LOG_ERR("%s: no cache buffers found\n", __func__);return nullptr;}for (auto * buffer : collector.buffers) {@@ -77,7 +95,7 @@ static llama_context * init_ctx(llama_model * model, llama_context_params cparamreturn ctx;}-static llama_context * make_ctx(const common_params & params, llama_model * model, uint8_t fill) {+static llama_context_ptr make_ctx(const common_params & params, llama_model * model, uint8_t fill) {auto cparams = common_context_params_to_llama(params);cparams.n_seq_max = 1;cparams.n_rs_seq = 8;@@ -109,7 +127,9 @@ static double nmse(const float * a, const float * b, int n) {// ubatches while its rollback restore is still pending. Compared against a// reference context that never advanced past the rollback point and decodes// the identical replay batch.-static bool test_multi_seq_split_replay(const common_params & params, llama_model * model, const int n_vocab, uint8_t fill) {+static test_status test_multi_seq_split_replay(const common_params & params, llama_model * model, uint8_t fill) {+ const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));+constexpr uint32_t n_seqs = 2;constexpr uint32_t n_ubatch = 16;constexpr uint32_t n_prompt = 19;@@ -128,22 +148,16 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modereturn init_ctx(model, cparams, fill);};- llama_context * ctx_roll = make_ctx_multi();- llama_context * ctx_ref = make_ctx_multi();- if (ctx_roll == nullptr || ctx_ref == nullptr) {- fprintf(stderr, "%s : failed to init multi-seq contexts\n", __func__);- return false;+ llama_context_ptr ctx_roll = make_ctx_multi();+ llama_context_ptr ctx_ref = make_ctx_multi();+ if (!ctx_roll || !ctx_ref) {+ LOG_ERR("%s: failed to init multi-seq contexts\n", __func__);+ return test_status::FAIL;}- const auto cleanup = [&]() {- llama_free(ctx_roll);- llama_free(ctx_ref);- };-- if (llama_n_rs_seq(ctx_roll) < n_rollback) {- fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__);- cleanup();- return true;+ if (llama_n_rs_seq(ctx_roll.get()) < n_rollback) {+ LOG_INF("%s: skipping because n_rs_seq is too small\n", __func__);+ return test_status::SKIP;}const auto tok = [&](uint32_t seq, llama_pos pos) {@@ -159,25 +173,24 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modefor (llama_pos pos = 0; pos < (llama_pos) p0; ++pos) {common_batch_add(batch, tok(s, pos), pos, { (llama_seq_id) s }, false);}- ok = ok && llama_decode(ctx_roll, batch) == 0;- ok = ok && llama_decode(ctx_ref, batch) == 0;+ ok = ok && llama_decode(ctx_roll.get(), batch) == 0;+ ok = ok && llama_decode(ctx_ref.get(), batch) == 0;common_batch_clear(batch);for (llama_pos pos = p0; pos < (llama_pos) n_prompt; ++pos) {common_batch_add(batch, tok(s, pos), pos, { (llama_seq_id) s }, false);}- ok = ok && llama_decode(ctx_roll, batch) == 0;+ ok = ok && llama_decode(ctx_roll.get(), batch) == 0;llama_batch_free(batch);- ok = ok && llama_memory_seq_rm(llama_get_memory(ctx_roll), (llama_seq_id) s, p0, -1);+ ok = ok && llama_memory_seq_rm(llama_get_memory(ctx_roll.get()), (llama_seq_id) s, p0, -1);// a second partial removal while one is pending must be refused- ok = ok && !llama_memory_seq_rm(llama_get_memory(ctx_roll), (llama_seq_id) s, p0 - 1, -1);+ ok = ok && !llama_memory_seq_rm(llama_get_memory(ctx_roll.get()), (llama_seq_id) s, p0 - 1, -1);}if (!ok) {- fprintf(stderr, "%s : multi-seq prefill/rollback failed\n", __func__);- cleanup();- return false;+ LOG_ERR("%s: multi-seq prefill/rollback failed\n", __func__);+ return test_status::FAIL;}llama_batch batch = llama_batch_init(n_seqs*n_replay, 0, 1);@@ -187,18 +200,17 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modecommon_batch_add(batch, tok(s, pos), pos, { (llama_seq_id) s }, true);}}- ok = llama_decode(ctx_roll, batch) == 0;- ok = ok && llama_decode(ctx_ref, batch) == 0;+ ok = llama_decode(ctx_roll.get(), batch) == 0;+ ok = ok && llama_decode(ctx_ref.get(), batch) == 0;llama_batch_free(batch);if (!ok) {- fprintf(stderr, "%s : multi-seq replay decode failed\n", __func__);- cleanup();- return false;+ LOG_ERR("%s: multi-seq replay decode failed\n", __func__);+ return test_status::FAIL;}- // identical ubatch shapes should produce identical states, but the larger- // stdev makes the model sensitive to backend scheduling/rounding noise- constexpr float nmse_eps = 1e-5f;+ // both contexts decode identical batches, so the logits should match;+ // random dummy models can still drift up to ~1.7e-5, so the bound is 1e-4+ constexpr float nmse_eps = 1e-4f;float diff_max = 0.0f;uint32_t seq_first = 0;@@ -206,12 +218,11 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modedouble nmse_ab = 0.0;double nmse_a0 = 0.0;for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {- const float * l_roll = llama_get_logits_ith(ctx_roll, i);- const float * l_ref = llama_get_logits_ith(ctx_ref, i);+ const float * l_roll = llama_get_logits_ith(ctx_roll.get(), i);+ const float * l_ref = llama_get_logits_ith(ctx_ref.get(), i);if (l_roll == nullptr || l_ref == nullptr) {- fprintf(stderr, "%s : missing multi-seq logits at index %u\n", __func__, i);- cleanup();- return false;+ LOG_ERR("%s: missing multi-seq logits at index %u\n", __func__, i);+ return test_status::FAIL;}for (int t = 0; t < n_vocab; ++t) {const float r = l_roll[t];@@ -235,13 +246,12 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modeconst double nmse_val = nmse_a0 == 0.0 ? (nmse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_ab/nmse_a0;if (nmse_val > nmse_eps) {- fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, nmse %g, first at seq %u pos %d)\n",+ LOG_ERR("%s: multi-seq split replay logits mismatch (max diff %g, nmse %g, first at seq %u pos %d)\n",__func__, (double) diff_max, nmse_val, seq_first, pos_first);- cleanup();- return false;+ return test_status::FAIL;}- fprintf(stderr, "%s : multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff_max, nmse_val);+ LOG_INF("%s: multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff_max, nmse_val);// seq-1-only decodes must be independent of seq 0's content: diverge seq 0// in ctx_ref only, then compare identical seq-1-only continuations bitwise@@ -253,7 +263,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modeconst llama_pos pos = p0 + (llama_pos) (n_replay + i);common_batch_add(batch_tail, tok(0, pos + 7), pos, { 0 }, false);}- ok = llama_decode(ctx_ref, batch_tail) == 0;+ ok = llama_decode(ctx_ref.get(), batch_tail) == 0;llama_batch_free(batch_tail);}@@ -264,15 +274,15 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modeconst llama_pos pos = p0 + (llama_pos) (n_replay + i);llama_batch batch_one = llama_batch_init(1, 0, 1);common_batch_add(batch_one, tok(1, pos), pos, { 1 }, true);- ok = llama_decode(ctx_roll, batch_one) == 0;- ok = ok && llama_decode(ctx_ref, batch_one) == 0;+ ok = llama_decode(ctx_roll.get(), batch_one) == 0;+ ok = ok && llama_decode(ctx_ref.get(), batch_one) == 0;llama_batch_free(batch_one);if (!ok) {break;}- const float * l_roll = llama_get_logits_ith(ctx_roll, 0);- const float * l_ref = llama_get_logits_ith(ctx_ref, 0);+ const float * l_roll = llama_get_logits_ith(ctx_roll.get(), 0);+ const float * l_ref = llama_get_logits_ith(ctx_ref.get(), 0);ok = l_roll != nullptr && l_ref != nullptr;for (int t = 0; ok && t < n_vocab; ++t) {const float r = l_roll[t];@@ -291,53 +301,48 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modeconst double nmse_tail = nmse_tail_a0 == 0.0 ? (nmse_tail_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_tail_ab/nmse_tail_a0;if (!ok || nmse_tail > nmse_eps) {- fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g, nmse %g)\n",+ LOG_ERR("%s: seq-1-only decode leaked seq 0 state (ok=%d, max diff %g, nmse %g)\n",__func__, ok ? 1 : 0, (double) diff_tail, nmse_tail);- cleanup();- return false;+ return test_status::FAIL;}- fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g, nmse %g)\n", __func__, (double) diff_tail, nmse_tail);- cleanup();- return true;+ LOG_INF("%s: seq-1-only decode independent of seq 0 (max diff %g, nmse %g)\n", __func__, (double) diff_tail, nmse_tail);+ return test_status::PASS;}-static int test_rollback(const common_params & params, llama_model * model, uint8_t fill) {+// Save a rolled-back single-seq state, restore it into fresh and dirty+// contexts, and verify exact logit matches on replay.+static test_status test_rollback(const common_params & params, llama_model * model, uint8_t fill) {const llama_vocab * vocab = llama_model_get_vocab(model);const int n_vocab = llama_vocab_n_tokens(vocab);- // TODO: use smart pointers- llama_context * ctx_src = make_ctx(params, model, fill);- llama_context * ctx_dst = make_ctx(params, model, fill);- if (ctx_src == nullptr || ctx_dst == nullptr) {- fprintf(stderr, "%s : failed to init contexts\n", __func__);- return 1;+ llama_context_ptr ctx_src = make_ctx(params, model, fill);+ llama_context_ptr ctx_dst = make_ctx(params, model, fill);+ if (!ctx_src || !ctx_dst) {+ LOG_ERR("%s: failed to init contexts\n", __func__);+ return test_status::FAIL;}- if (llama_n_rs_seq(ctx_src) == 0) {- fprintf(stderr, "%s : skipping because n_rs_seq is disabled\n", __func__);- llama_free(ctx_src);- llama_free(ctx_dst);- return 0;+ if (llama_n_rs_seq(ctx_src.get()) == 0) {+ LOG_INF("%s: skipping because n_rs_seq is disabled\n", __func__);+ return test_status::SKIP;}std::vector<llama_token> tokens;if (llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_NONE) {tokens = { 1, 2, 3, 4, 5, 6, 7, 8, 9 };} else {- tokens = common_tokenize(ctx_src, "The quick brown fox jumps over the lazy dog", true);+ tokens = common_tokenize(ctx_src.get(), "The quick brown fox jumps over the lazy dog", true);}- const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src);+ const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src.get());constexpr uint32_t n_rollback = 3;if (n_rs_seq < n_rollback) {- fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__);- llama_free(ctx_src);- llama_free(ctx_dst);- return 0;+ LOG_INF("%s: skipping because n_rs_seq is too small\n", __func__);+ return test_status::SKIP;}if (tokens.empty()) {- fprintf(stderr, "%s : not enough prompt tokens\n", __func__);- return 1;+ LOG_ERR("%s: not enough prompt tokens\n", __func__);+ return test_status::FAIL;}tokens.resize(n_rs_seq + 1, tokens.back());@@ -347,35 +352,35 @@ static int test_rollback(const common_params & params, llama_model * model, uint// Decode the full prompt on the source, then roll back three positions.// Replaying them crosses DSV4's ratio-4 compressor boundary.// Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).- if (!decode_tokens(ctx_src, tokens, n_tokens)) {- fprintf(stderr, "%s : failed to decode prompt\n", __func__);- return 1;+ if (!decode_tokens(ctx_src.get(), tokens, n_tokens)) {+ LOG_ERR("%s: failed to decode prompt\n", __func__);+ return test_status::FAIL;}- if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1)) {- fprintf(stderr, "%s : rollback failed\n", __func__);- return 1;+ if (!llama_memory_seq_rm(llama_get_memory(ctx_src.get()), 0, rollback_pos, -1)) {+ LOG_ERR("%s: rollback failed\n", __func__);+ return test_status::FAIL;}// Save the rolled-back state and restore it into a fresh context.common_prompt_checkpoint ckpt;- ckpt.update_tgt(ctx_src, 0, 0);- ckpt.load_tgt(ctx_dst, 0, 0);+ ckpt.update_tgt(ctx_src.get(), 0, 0);+ ckpt.load_tgt(ctx_dst.get(), 0, 0);constexpr float nmse_eps = 0.0;std::vector<std::vector<float>> logits_src_replay(n_rollback);const auto replay_and_compare = [&](const char * mode) {for (uint32_t i = 0; i < n_rollback; ++i) {const llama_pos pos = rollback_pos + i;- if (!decode_one(ctx_src, tokens[pos], pos) ||- !decode_one(ctx_dst, tokens[pos], pos)) {- fprintf(stderr, "%s : %s replay failed at position %d\n", __func__, mode, pos);+ if (!decode_one(ctx_src.get(), tokens[pos], pos) ||+ !decode_one(ctx_dst.get(), tokens[pos], pos)) {+ LOG_ERR("%s: %s replay failed at position %d\n", __func__, mode, pos);return false;}- const float * logits_src = llama_get_logits_ith(ctx_src, 0);- const float * logits_dst = llama_get_logits_ith(ctx_dst, 0);+ const float * logits_src = llama_get_logits_ith(ctx_src.get(), 0);+ const float * logits_dst = llama_get_logits_ith(ctx_dst.get(), 0);if (logits_src == nullptr || logits_dst == nullptr) {- fprintf(stderr, "%s : missing %s logits at position %d\n", __func__, mode, pos);+ LOG_ERR("%s: missing %s logits at position %d\n", __func__, mode, pos);return false;}@@ -388,7 +393,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint}}if (nmse_val > nmse_eps) {- fprintf(stderr, "%s : %s logits mismatch at position %d, first token %d, nmse %g\n",+ LOG_ERR("%s: %s logits mismatch at position %d, first token %d, nmse %g\n",__func__, mode, pos, token_first, nmse_val);return false;}@@ -396,7 +401,7 @@ static int test_rollback(const common_params & params, llama_model * model, uintreturn true;};if (!replay_and_compare("full")) {- return 1;+ return test_status::FAIL;}// TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens@@ -419,10 +424,10 @@ static int test_rollback(const common_params & params, llama_model * model, uint// Repeat the load into a context that already has its own rollback state:// groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is// non-zero at load time. The restore must wipe that state and still match.- llama_context * ctx_dirty = make_ctx(params, model, fill);- if (ctx_dirty == nullptr) {- fprintf(stderr, "%s : failed to init dirty ctx\n", __func__);- return 1;+ llama_context_ptr ctx_dirty = make_ctx(params, model, fill);+ if (!ctx_dirty) {+ LOG_ERR("%s: failed to init dirty ctx\n", __func__);+ return test_status::FAIL;}std::vector<llama_token> noise = tokens;@@ -432,28 +437,28 @@ static int test_rollback(const common_params & params, llama_model * model, uintt = 0;}}- if (!decode_tokens(ctx_dirty, noise, n_tokens)) {- fprintf(stderr, "%s : dirty prompt decode failed\n", __func__);- return 1;+ if (!decode_tokens(ctx_dirty.get(), noise, n_tokens)) {+ LOG_ERR("%s: dirty prompt decode failed\n", __func__);+ return test_status::FAIL;}- if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, rollback_pos, -1)) {- fprintf(stderr, "%s : dirty rollback failed\n", __func__);- return 1;+ if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty.get()), 0, rollback_pos, -1)) {+ LOG_ERR("%s: dirty rollback failed\n", __func__);+ return test_status::FAIL;}- ckpt.load_tgt(ctx_dirty, 0, 0);+ ckpt.load_tgt(ctx_dirty.get(), 0, 0);for (uint32_t i = 0; i < n_rollback; ++i) {const llama_pos pos = rollback_pos + i;- if (!decode_one(ctx_dirty, tokens[pos], pos)) {- fprintf(stderr, "%s : dirty replay failed at position %d\n", __func__, pos);- return 1;+ if (!decode_one(ctx_dirty.get(), tokens[pos], pos)) {+ LOG_ERR("%s: dirty replay failed at position %d\n", __func__, pos);+ return test_status::FAIL;}- const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0);+ const float * logits_dirty = llama_get_logits_ith(ctx_dirty.get(), 0);if (logits_dirty == nullptr) {- fprintf(stderr, "%s : missing dirty logits at position %d\n", __func__, pos);- return 1;+ LOG_ERR("%s: missing dirty logits at position %d\n", __func__, pos);+ return test_status::FAIL;}const double nmse_dirty = nmse(logits_src_replay[i].data(), logits_dirty, n_vocab);@@ -464,22 +469,75 @@ static int test_rollback(const common_params & params, llama_model * model, uint}}if (nmse_dirty > nmse_eps) {- fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, first token %d, nmse %g\n",+ LOG_ERR("%s: dirty-ctx logits mismatch at position %d, first token %d, nmse %g\n",__func__, pos, token_first, nmse_dirty);- return 1;+ return test_status::FAIL;}}- fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__);- llama_free(ctx_src);- llama_free(ctx_dst);- llama_free(ctx_dirty);+ LOG_INF("%s: recurrent rollback checkpoint restored successfully\n", __func__);+ return test_status::PASS;+}- if (!test_multi_seq_split_replay(params, model, n_vocab, fill)) {- return 1;+static test_status merge_status(test_status a, test_status b) {+ if (a == test_status::FAIL || b == test_status::FAIL) {+ return test_status::FAIL;+ }+ if (a == test_status::PASS || b == test_status::PASS) {+ return test_status::PASS;+ }+ return test_status::SKIP;+}++struct test_results {+ test_status rollback = test_status::SKIP;+ test_status replay = test_status::SKIP;+};++// Run every test for an initialized model over both cache fills.+static test_results run_tests(const common_params & params, llama_model * model) {+ test_results res;+ for (uint8_t fill : { 0, 0x3e }) {+ LOG_INF("%s: testing with cache fill 0x%02x\n", __func__, fill);+ const test_status rb = test_rollback(params, model, fill);+ const test_status rp = test_multi_seq_split_replay(params, model, fill);+ res.rollback = merge_status(res.rollback, rb);+ res.replay = merge_status(res.replay, rp);+ if (rb == test_status::FAIL || rp == test_status::FAIL) {+ break;+ }+ }+ return res;+}++// Run the tests for a single model file.+// Returns the per-test statuses.+static test_results run_tests_for_model(const std::string & model_path, const struct common_params & base_params) {+ struct common_params params = base_params;+ params.model.path = model_path;++ auto llama_init = common_init_from_params(params, true);+ auto * model = llama_init->model();++ if (model == nullptr) {+ LOG_ERR("%s: failed to init model '%s'\n", __func__, model_path.c_str());+ // a model that cannot be loaded is a failure, not a skip+ return { test_status::FAIL, test_status::FAIL };}- return 0;+ if (!llama_model_is_recurrent(model) && !llama_model_is_hybrid(model)) {+ LOG_INF("%s: skipping for non-recurrent model\n", __func__);+ return {};+ }++ return run_tests(params, model);+}++static void print_usage(int /* argc */, char ** argv) {+ LOG("\nexample usage:\n");+ LOG("\n %s -m your_model.gguf\n", argv[0]);+ LOG("\n %s --models tests/test-models\n", argv[0]);+ LOG("\n");}int main(int argc, char ** argv) {@@ -491,30 +549,107 @@ int main(int argc, char ** argv) {common_init();- if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {- return 1;+ // extract our own --models DIR option before handing the rest to the common arg parser+ std::string models_dir;+ std::vector<char *> filtered_argv;+ filtered_argv.push_back(argv[0]);+ for (int i = 1; i < argc; i++) {+ if (strcmp(argv[i], "--models") == 0) {+ if (i + 1 >= argc) {+ LOG_ERR("%s: --models requires a directory argument\n", __func__);+ return 1;+ }+ models_dir = argv[i + 1];+ i++;+ } else {+ filtered_argv.push_back(argv[i]);+ }}+ filtered_argv.push_back(nullptr);+ const int fargc = (int)filtered_argv.size() - 1;- llama_backend_init();+ // in --models mode there is no single model; set a placeholder so the common parser's+ // "--model is required" check passes (each model is set individually inside the loop)+ if (!models_dir.empty()) {+ params.model.path = models_dir;+ }- common_init_result_ptr llama_init = common_init_from_params(params);- llama_model * model = llama_init->model();- if (model == nullptr) {- fprintf(stderr, "%s : failed to init model\n", __func__);+ if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON, print_usage)) {return 1;}- if (!llama_model_is_recurrent(model) && !llama_model_is_hybrid(model)) {- fprintf(stderr, "%s : skipping for non-recurrent model\n", __func__);- return 0;- }+ llama_backend_init();- for (uint8_t fill : { 0, 0x3e }) {- fprintf(stderr, "%s : testing with cache fill 0x%02x\n", __func__, fill);- if (test_rollback(params, model, fill) != 0) {+ if (!models_dir.empty()) {+ // run every test over each dummy model in the directory+ if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {+ LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());return 1;}++ std::vector<std::string> models;+ for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {+ if (entry.is_regular_file() && entry.path().extension() == ".gguf") {+ models.push_back(entry.path().string());+ }+ }+ std::sort(models.begin(), models.end());++ if (models.empty()) {+ LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());+ return 1;+ }++ size_t name_width = 5; // "Model"+ for (const auto & model_path : models) {+ name_width = std::max(name_width, std::filesystem::path(model_path).filename().string().size());+ }++ // silence everything but the table itself (LOG has verbosity LOG_LEVEL_OUTPUT = 0)+ common_log_set_verbosity_thold(0);++ LOG("%-*s %-8s %s\n", (int) name_width, "Model", "rollback", "split replay");+ common_log_flush(common_log_main());++ size_t n_pass[2] = { 0, 0 };+ size_t n_skip[2] = { 0, 0 };+ size_t n_fail[2] = { 0, 0 };+ for (const auto & model_path : models) {+ const auto name = std::filesystem::path(model_path).filename().string();++ LOG("%-*s", (int) name_width, name.c_str());++ const test_results res = run_tests_for_model(model_path, params);++ // all status strings have the same raw length, so the columns line up;+ // pad the first status to the width of the "rollback" header + separator+ LOG(" %s %s", test_status_str(res.rollback), test_status_str(res.replay));+ LOG("\n");+ common_log_flush(common_log_main());++ const test_status all[2] = { res.rollback, res.replay };+ for (int t = 0; t < 2; ++t) {+ switch (all[t]) {+ case test_status::PASS: n_pass[t]++; break;+ case test_status::FAIL: n_fail[t]++; break;+ case test_status::SKIP: n_skip[t]++; break;+ }+ }+ }++ common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA);+ common_log_flush(common_log_main());++ LOG_INF("%s: rollback: %zu passed, %zu skipped, %zu failed (of %zu)\n",+ __func__, n_pass[0], n_skip[0], n_fail[0], models.size());+ LOG_INF("%s: split replay: %zu passed, %zu skipped, %zu failed (of %zu)\n",+ __func__, n_pass[1], n_skip[1], n_fail[1], models.size());++ return (n_fail[0] + n_fail[1]) == 0 ? 0 : 1;}- return 0;+ // single-model mode+ const test_results res = run_tests_for_model(params.model.path, params);++ return (res.rollback == test_status::FAIL || res.replay == test_status::FAIL) ? 1 : 0;}