LLM inference in C/C++
C++ ggml
test-llama-archs : make tensor data stdev configurable and improve help (#29133)
Georgi Gerganov161755f29e41
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cppindex f3b64b210d..53caca1a5c 100644--- a/src/models/minimax-m3.cpp+++ b/src/models/minimax-m3.cpp@@ -23,7 +23,12 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size);ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);- msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks };++ msa_p = {+ /*.blk =*/ (int) hparams.indexer_block_size,+ /*.topk_blocks =*/ (int) hparams.indexer_top_k,+ /*.local =*/ (int) hparams.indexer_local_blocks,+ };GGML_ASSERT(hparams.indexer_block_size > 0); // avoid div by zerodiff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex 80a0451855..488770297a 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -12,16 +12,25 @@#include "../src/llama-model-saver.h"#include <cinttypes>+#include <cmath>#include <cstddef>#include <cstdio>#include <cstring>#include <cstdint>#include <random>+#include <regex>#include <stdexcept>#include <string>#include <utility>#include <vector>+static bool arch_matches(const std::string & filter, llm_arch arch) {+ if (filter.empty()) {+ return true;+ }+ return std::regex_search(llm_arch_name(arch), std::regex(filter));+}+// normalized mean squared error = mse(a, b) / mse(a, 0)static double nmse(const std::vector<float> & a, const std::vector<float> & b) {GGML_ASSERT(a.size() == b.size());@@ -39,24 +48,36 @@ static double nmse(const std::vector<float> & a, const std::vector<float> & b) {return mse_a_b / mse_a_0;}+struct tensor_data_params {+ size_t seed;+ float stdev;+};+static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {- size_t seed = *(const size_t *) userdata;+ const tensor_data_params & params = *(const tensor_data_params *) userdata;+ size_t seed = params.seed;std::hash<std::string> hasher;seed ^= hasher(tensor->name);std::mt19937 gen(seed);- std::normal_distribution<float> dis(0.0f, 1.0e-2f);+ std::normal_distribution<float> dis(0.0f, params.stdev);++ // TODO: refactor per-tensor initialization logic in a cleaner way+ // note: Mamba A must be negative (state decay)+ const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr;const int64_t ne = ggml_nelements(tensor);if (tensor->type == GGML_TYPE_F32) {std::vector<float> tmp(ne);for (int64_t i = 0; i < ne; i++) {- tmp[i] = dis(gen);+ float val = dis(gen);+ tmp[i] = is_ssm_a ? -fabsf(val) : val;}ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));} else if (tensor->type == GGML_TYPE_F16) {std::vector<ggml_fp16_t> tmp(ne);for (int64_t i = 0; i < ne; i++) {- tmp[i] = ggml_fp32_to_fp16(dis(gen));+ float val = dis(gen);+ tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val);}ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));} else {@@ -65,7 +86,19 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {}static void usage(char ** argv) {- printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help]\n", argv[0]);+ LOG("Usage: %s [options]\n\n", argv[0]);+ LOG("Options:\n");+ LOG(" -a, --arch <arch|regex> Run only matching LLM architectures (default: all supported)\n");+ LOG(" -s, --seed <seed> Set the random seed for tensor initialization and token generation\n");+ LOG(" -d, --stdev <stdev> Set the standard deviation of the tensor initialization distribution (default: 0.1f)\n");+ LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n");+ LOG(" -v <N> Set log verbosity level\n");+ LOG(" -h, --help Show this help message\n\n");+ LOG("Examples:\n");+ LOG(" %s\n", argv[0]);+ LOG(" %s -a qwen35moe\n", argv[0]);+ LOG(" %s -a deepseek4 -o tests/test-models/\n", argv[0]);+ LOG(" %s -a cohere2moe -v 5\n", argv[0]);}static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){@@ -292,7 +325,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));- ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));+ // note: using a realistic top-k here makes the results unstable and hard to match between CPU and GPU+ // a large value makes things deterministic since all data is selected by the indexer+ //ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(131072));+ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));@@ -314,13 +351,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {}if (arch == LLM_ARCH_DEEPSEEK4) {- ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));- ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));- ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));- ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);- ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));- ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));- ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);+ ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));+ ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));+ ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));+ ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);+ ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));+ ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));+ ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);@@ -381,19 +418,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));- ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);- ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);+ ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);+ ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);if (arch == LLM_ARCH_BAILINGMOE3) {ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));}- ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);- ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));- ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);- ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));- ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);+ ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);+ ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));+ ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);+ ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));+ ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);- ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);+ ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);for (uint32_t il = 0; il < n_layer; il++) {ggml_tensor t;@@ -416,7 +453,8 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/)}static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(- struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const std::vector<ggml_backend_dev_t> & devs,+ struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,+ const std::vector<ggml_backend_dev_t> & devs,const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false) {GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));llama_model_params model_params = llama_model_default_params();@@ -434,9 +472,9 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(ctx_params.n_ubatch = 64;}- size_t tmp = seed;+ tensor_data_params tensor_params = { seed, stdev };llama_model_ptr model(gguf_ctx != nullptr ?- llama_model_init_from_user(gguf_ctx, set_tensor_data, &tmp, model_params) :+ llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :llama_model_load_from_file_ptr(file, model_params));if (!model) {throw std::runtime_error("failed to create llama model");@@ -608,7 +646,7 @@ static bool arch_supported(const llm_arch arch) {return true;}-static int save_models(const llm_arch target_arch, const size_t seed, const int verbosity, const std::string & dir) {+static int save_models(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const std::string & dir) {struct user_data_t {struct {ggml_log_callback callback;@@ -635,7 +673,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const intif (arch == LLM_ARCH_UNKNOWN) {continue;}- if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {+ if (!arch_matches(arch_filter, arch)) {continue;}if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {@@ -656,7 +694,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const intcontinue;}gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);- auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {});+ auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {});const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());@@ -666,7 +704,7 @@ static int save_models(const llm_arch target_arch, const size_t seed, const intreturn 0;}-static int test_backends(const llm_arch target_arch, const size_t seed, const int verbosity) {+static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity) {struct user_data_t {struct {ggml_log_callback callback;@@ -731,22 +769,24 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const inconst std::string template_row_res = "%15s %10s|%20s|\n";bool all_ok = true;+ size_t n_tests = 0;+ size_t n_failed = 0;common_log_flush(common_log_main());- printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");- printf("|");+ LOG(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");+ LOG("|");for (size_t i = 0; i < max_arch_name_length; i++) {- printf("-");+ LOG("-");}- printf("|");+ LOG("|");for (size_t i = 0; i < max_device_label_length; i++) {- printf("-");+ LOG("-");}- printf("|------|---------------|---------|\n");+ LOG("|------|---------------|---------|\n");for (const llm_arch & arch : llm_arch_all()) {if (arch == LLM_ARCH_UNKNOWN) {continue;}- if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {+ if (!arch_matches(arch_filter, arch)) {continue;}if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {@@ -773,8 +813,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const instd::vector<float> logits_cpu;for (device_config & dc : dev_configs) {// print test config first; should anything fail during model loading or inference, at least we know which test case caused it- printf(template_row_cfg.c_str(),- llm_arch_name(arch), dc.label.c_str(), config_name.c_str());+ LOG(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str());fflush(stdout);std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;@@ -784,19 +823,22 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const inchar nmse_str[12] = {0};bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());+ bool test_executed = false;+ bool test_ok = true;if (!skip) {if (logits_cpu.empty()) {- model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode);+ model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}, LLAMA_SPLIT_MODE_LAYER, encode);logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);}if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {- model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, dc.devs, dc.split_mode, encode);+ test_executed = true;+ model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode);logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);const double nmse_val = nmse(logits_cpu, logits_dev);snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);status_nmse = "\033[1;32mOK\033[0m";if (nmse_val > 1e-4) {- all_ok = false;+ test_ok = false;status_nmse = "\033[1;31mFAIL\033[0m";}}@@ -805,6 +847,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in// FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend// in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatibleif (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {+ test_executed = true;GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get());ms.add_kv_from_model();@@ -812,14 +855,14 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const inms.save(file);rewind(file);- auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode);+ auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode);const std::vector<float> logits_roundtrip = get_logits(model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);status_roundtrip = "\033[1;32mOK\033[0m";GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());for (size_t i = 0; i < logits_roundtrip.size(); i++) {if (logits_roundtrip[i] != logits_dev[i]) {- all_ok = false;+ test_ok = false;status_roundtrip = "\033[1;31mFAIL\033[0m";break;}@@ -827,12 +870,28 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in}}+ if (test_executed) {+ n_tests++;+ if (!test_ok) {+ n_failed++;+ all_ok = false;+ }+ }+// log the results for this test case- printf(template_row_res.c_str(),- status_nmse.c_str(), nmse_str, status_roundtrip.c_str());+ LOG(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str());}}}++ if (n_tests == 0) {+ LOG("Summary: no tests executed\n");+ } else if (n_failed == 0) {+ LOG("Summary: all %zu test(s) passed\n", n_tests);+ } else {+ LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed);+ }+llama_log_set(ud.log_old.callback, ud.log_old.user_data);return all_ok ? 0 : 1;}@@ -844,8 +903,9 @@ int main(int argc, char ** argv) {std::random_device rd;- llm_arch arch = LLM_ARCH_UNKNOWN;+ std::string arch_filter;size_t seed = rd();+ float stdev = 0.1f;std::string out;int verbosity = LOG_LEVEL_ERROR;@@ -854,52 +914,70 @@ int main(int argc, char ** argv) {if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {usage(argv);return 0;- }- if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {+ } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {if (i + 1 < argc) {const std::string arch_name = argv[++i];- arch = llm_arch_from_string(arch_name);- if (arch == LLM_ARCH_UNKNOWN) {- LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str());- return 1;+ if (llm_arch_from_string(arch_name) != LLM_ARCH_UNKNOWN) {+ // exact architecture name+ arch_filter = "^" + arch_name + "$";+ } else {+ try {+ std::regex re(arch_name);+ arch_filter = arch_name;+ } catch (const std::regex_error & err) {+ LOG_ERR("%s: invalid architecture regex: %s (%s)\n", __func__, arch_name.c_str(), err.what());+ return 1;+ }}} else {usage(argv);return 1;}- }- if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {+ } else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {if (i + 1 < argc) {seed = std::stoull(argv[++i]);} else {usage(argv);return 1;}- }- if (strcmp(argv[i], "-v") == 0) {+ } else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--stdev") == 0) {+ if (i + 1 < argc) {+ stdev = std::stof(argv[++i]);+ } else {+ usage(argv);+ return 1;+ }+ } else if (strcmp(argv[i], "-v") == 0) {if (i + 1 < argc) {verbosity = std::stoull(argv[++i]);} else {usage(argv);return 1;}- }- if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {+ } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {if (i + 1 < argc) {out = argv[++i];} else {usage(argv);return 1;}+ } else {+ LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]);+ usage(argv);+ return 1;}}- printf("%s: using seed %zu\n", __func__, seed);+ if (stdev <= 0.0f) {+ LOG_ERR("%s: stdev must be > 0\n", __func__);+ return 1;+ }+ LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);try {if (!out.empty()) {- return save_models(arch, seed, verbosity, out);+ return save_models(arch_filter, seed, stdev, verbosity, out);}- return test_backends(arch, seed, verbosity);+ return test_backends(arch_filter, seed, stdev, verbosity);} catch (const std::exception & err) {fprintf(stderr, "encountered runtime error: %s\n", err.what());return -1;diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cppindex ef05de67d0..fdac344d89 100644--- a/tests/test-recurrent-state-rollback.cpp+++ b/tests/test-recurrent-state-rollback.cpp@@ -90,6 +90,20 @@ static float logit_diff(float a, float b) {return std::isfinite(a) && std::isfinite(b) ? std::fabs(a - b) : std::numeric_limits<float>::infinity();}+static double nmse(const float * a, const float * b, int n) {+ double mse_ab = 0.0;+ double mse_a0 = 0.0;+ for (int i = 0; i < n; i++) {+ if (!std::isfinite(a[i]) || !std::isfinite(b[i])) {+ return std::numeric_limits<double>::infinity();+ }+ const double diff = (double) a[i] - b[i];+ mse_ab += diff*diff;+ mse_a0 += (double) a[i]*a[i];+ }+ return mse_a0 == 0.0 ? (mse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : mse_ab/mse_a0;+}+// Roll back multiple sequences, then replay them in a single batch whose// per-seq token count exceeds n_ubatch: each seq's replay spans several// ubatches while its rollback restore is still pending. Compared against a@@ -182,13 +196,15 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modereturn false;}- // identical ubatch shapes from bit-exact states: a correct implementation- // matches bitwise, so eps only allows backend scheduling noise- constexpr float eps = 1e-7f;+ // identical ubatch shapes should produce identical states, but the larger+ // stdev makes the model sensitive to backend scheduling/rounding noise+ constexpr float nmse_eps = 1e-5f;float diff_max = 0.0f;uint32_t seq_first = 0;int32_t pos_first = -1;+ double nmse_ab = 0.0;+ double nmse_a0 = 0.0;for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {const float * l_roll = llama_get_logits_ith(ctx_roll, i);const float * l_ref = llama_get_logits_ith(ctx_ref, i);@@ -198,23 +214,34 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modereturn false;}for (int t = 0; t < n_vocab; ++t) {- const float diff = logit_diff(l_roll[t], l_ref[t]);- if (diff > eps && pos_first < 0) {+ const float r = l_roll[t];+ const float f = l_ref[t];+ const float diff = logit_diff(r, f);+ if (diff > 0.0f && pos_first < 0) {seq_first = i/n_replay;pos_first = p0 + (int32_t) (i%n_replay);}diff_max = std::max(diff_max, diff);+ if (std::isfinite(r) && std::isfinite(f)) {+ const double d = (double) r - f;+ nmse_ab += d*d;+ nmse_a0 += (double) r*r;+ } else {+ nmse_ab = std::numeric_limits<double>::infinity();+ nmse_a0 = 1.0;+ }}}+ const double nmse_val = nmse_a0 == 0.0 ? (nmse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_ab/nmse_a0;- if (diff_max > eps) {- fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n",- __func__, (double) diff_max, seq_first, pos_first);+ if (nmse_val > nmse_eps) {+ fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, nmse %g, first at seq %u pos %d)\n",+ __func__, (double) diff_max, nmse_val, seq_first, pos_first);cleanup();return false;}- fprintf(stderr, "%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max);+ fprintf(stderr, "%s : multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff_max, nmse_val);// seq-1-only decodes must be independent of seq 0's content: diverge seq 0// in ctx_ref only, then compare identical seq-1-only continuations bitwise@@ -231,6 +258,8 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode}float diff_tail = 0.0f;+ double nmse_tail_ab = 0.0;+ double nmse_tail_a0 = 0.0;for (uint32_t i = 0; i < n_tail && ok; ++i) {const llama_pos pos = p0 + (llama_pos) (n_replay + i);llama_batch batch_one = llama_batch_init(1, 0, 1);@@ -246,18 +275,29 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_modeconst float * l_ref = llama_get_logits_ith(ctx_ref, 0);ok = l_roll != nullptr && l_ref != nullptr;for (int t = 0; ok && t < n_vocab; ++t) {- diff_tail = std::max(diff_tail, logit_diff(l_roll[t], l_ref[t]));+ const float r = l_roll[t];+ const float f = l_ref[t];+ diff_tail = std::max(diff_tail, logit_diff(r, f));+ if (std::isfinite(r) && std::isfinite(f)) {+ const double d = (double) r - f;+ nmse_tail_ab += d*d;+ nmse_tail_a0 += (double) r*r;+ } else {+ nmse_tail_ab = std::numeric_limits<double>::infinity();+ nmse_tail_a0 = 1.0;+ }}}+ const double nmse_tail = nmse_tail_a0 == 0.0 ? (nmse_tail_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity()) : nmse_tail_ab/nmse_tail_a0;- if (!ok || diff_tail > eps) {- fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n",- __func__, ok ? 1 : 0, (double) diff_tail);+ if (!ok || nmse_tail > nmse_eps) {+ fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g, nmse %g)\n",+ __func__, ok ? 1 : 0, (double) diff_tail, nmse_tail);cleanup();return false;}- fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail);+ fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g, nmse %g)\n", __func__, (double) diff_tail, nmse_tail);cleanup();return true;}@@ -266,6 +306,7 @@ static int test_rollback(const common_params & params, llama_model * model, uintconst llama_vocab * vocab = llama_model_get_vocab(model);const int n_vocab = llama_vocab_n_tokens(vocab);+ // TODO: use smart pointersllama_context * ctx_src = make_ctx(params, model, fill);llama_context * ctx_dst = make_ctx(params, model, fill);if (ctx_src == nullptr || ctx_dst == nullptr) {@@ -320,7 +361,7 @@ static int test_rollback(const common_params & params, llama_model * model, uintckpt.update_tgt(ctx_src, 0, 0);ckpt.load_tgt(ctx_dst, 0, 0);- constexpr float eps = 1e-5f;+ constexpr float nmse_eps = 0.0;std::vector<std::vector<float>> logits_src_replay(n_rollback);const auto replay_and_compare = [&](const char * mode) {for (uint32_t i = 0; i < n_rollback; ++i) {@@ -339,13 +380,18 @@ static int test_rollback(const common_params & params, llama_model * model, uint}logits_src_replay[i].assign(logits_src, logits_src + n_vocab);+ const double nmse_val = nmse(logits_src, logits_dst, n_vocab);+ int token_first = -1;for (int token = 0; token < n_vocab; ++token) {- if (logit_diff(logits_src[token], logits_dst[token]) > eps) {- fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n",- __func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]);- return false;+ if (logit_diff(logits_src[token], logits_dst[token]) > 0.0f && token_first < 0) {+ token_first = token;}}+ if (nmse_val > nmse_eps) {+ fprintf(stderr, "%s : %s logits mismatch at position %d, first token %d, nmse %g\n",+ __func__, mode, pos, token_first, nmse_val);+ return false;+ }}return true;};@@ -353,20 +399,22 @@ static int test_rollback(const common_params & params, llama_model * model, uintreturn 1;}- if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||- !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {- fprintf(stderr, "%s : partial rollback failed\n", __func__);- return 1;- }+ // TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens+ // this is not the case here. add asserts and guardrails to prevent such attempts+ //if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||+ // !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {+ // fprintf(stderr, "%s : partial rollback failed\n", __func__);+ // return 1;+ //}- constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;- common_prompt_checkpoint ckpt_partial;- ckpt_partial.update_tgt(ctx_src, 0, partial_flags);- ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);+ //constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;+ //common_prompt_checkpoint ckpt_partial;+ //ckpt_partial.update_tgt(ctx_src, 0, partial_flags);+ //ckpt_partial.load_tgt(ctx_dst, 0, partial_flags);- if (!replay_and_compare("partial")) {- return 1;- }+ //if (!replay_and_compare("partial")) {+ // return 1;+ //}// Repeat the load into a context that already has its own rollback state:// groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is@@ -408,13 +456,18 @@ static int test_rollback(const common_params & params, llama_model * model, uintreturn 1;}+ const double nmse_dirty = nmse(logits_src_replay[i].data(), logits_dirty, n_vocab);+ int token_first = -1;for (int token = 0; token < n_vocab; ++token) {- if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > eps) {- fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n",- __func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]);- return 1;+ if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > 0.0f && token_first < 0) {+ token_first = token;}}+ if (nmse_dirty > nmse_eps) {+ fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, first token %d, nmse %g\n",+ __func__, pos, token_first, nmse_dirty);+ return 1;+ }}fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__);