LLM inference in C/C++
C++ ggml
test-save-load-state : print a per-model results table in --models mode (#29316)
Georgi Gerganov4c5957c2779c
diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cppindex 08c7c67727..33ee7143af 100644--- a/tests/test-save-load-state.cpp+++ b/tests/test-save-load-state.cpp@@ -13,6 +13,21 @@constexpr double NMSE_THRESHOLD = 1e-5;+enum class test_status {+ PASS,+ FAIL,+ SKIP,+};++static const char * test_status_str(test_status status) {+ switch (status) {+ case test_status::PASS: return "\033[1;32mPASS\033[0m";+ case test_status::FAIL: return "\033[1;31mFAIL\033[0m";+ case test_status::SKIP: return "\033[1;33mSKIP\033[0m";+ }+ return "";+}+// normalized mean squared error = mse(a, b) / mse(a, 0)static double nmse(const std::vector<float> & a, const std::vector<float> & b) {GGML_ASSERT(a.size() == b.size());@@ -78,7 +93,8 @@ static generation_result generate_tokens(llama_context * ctx, llama_sampler * smauto next_token = llama_sampler_sample(smpl, ctx, -1);- LOG("%d ", next_token);+ // LOG_LEVEL_INFO gate: visible in single-model mode, silenced in --models table mode+ LOGV(LOG_LEVEL_INFO, "%d ", next_token);result.tokens.push_back(next_token);result.logits.push_back(std::move(logits));@@ -126,7 +142,7 @@ static bool generate_tokens_compare(const auto next_token = llama_sampler_sample(smpl, ctx, -1);const auto expected_token = expected.tokens[i];- LOG("%d ", next_token);+ LOGV(LOG_LEVEL_INFO, "%d ", next_token);if (next_token != expected_token) {LOG_TRC("%s: sampled token %d differs from expected %d, using expected token\n", __func__, next_token, expected_token);}@@ -164,14 +180,14 @@ static generation_result test_baseline(struct llama_model * model, const structreturn {};}- LOG("\n=== Test 1: baseline ===\n");+ LOGV(LOG_LEVEL_INFO, "\n=== Test 1: baseline ===\n");auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 0);if (result.empty()) {return {};}- LOG("\n");+ LOGV(LOG_LEVEL_INFO, "\n");return result;}@@ -196,7 +212,7 @@ static bool test_seq_rm_isolated(return false;}- LOG("\n=== Test 2: sequence removal isolation ===\n");+ LOGV(LOG_LEVEL_INFO, "\n=== Test 2: sequence removal isolation ===\n");const size_t n_tokens = tokens.size() < 128 ? tokens.size() : 128;for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) {@@ -249,7 +265,7 @@ static bool test_seq_rm_isolated(return false;}- LOG("PASS\n");+ LOGV(LOG_LEVEL_INFO, "PASS\n");return true;}@@ -268,7 +284,7 @@ static bool test_state_load(struct llama_model * model, const struct common_paraauto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed));- LOG("\n=== Test 3: state load ===\n");+ LOGV(LOG_LEVEL_INFO, "\n=== Test 3: state load ===\n");// Load state from filellama_tokens unused_sts(tokens.size());@@ -293,7 +309,7 @@ static bool test_state_load(struct llama_model * model, const struct common_parareturn false;}- LOG("\nPASS\n");+ LOGV(LOG_LEVEL_INFO, "\nPASS\n");return true;}@@ -313,7 +329,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_parauto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed));- LOG("\n=== Test 4: seq copy (host) ===\n");+ LOGV(LOG_LEVEL_INFO, "\n=== Test 4: seq copy (host) ===\n");// Load state from filellama_tokens unused_sts(tokens.size());@@ -359,7 +375,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_parreturn false;}- LOG("\nPASS\n");+ LOGV(LOG_LEVEL_INFO, "\nPASS\n");return true;}@@ -379,7 +395,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_pauto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed));- LOG("\n=== Test 5: seq copy (device) ===\n");+ LOGV(LOG_LEVEL_INFO, "\n=== Test 5: seq copy (device) ===\n");// Load state from filellama_tokens unused_sts(tokens.size());@@ -425,7 +441,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_preturn false;}- LOG("\nPASS\n");+ LOGV(LOG_LEVEL_INFO, "\nPASS\n");return true;}@@ -442,7 +458,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_params_ctx.kv_unified = true;auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};- LOG("\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host");+ LOGV(LOG_LEVEL_INFO, "\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host");const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;@@ -519,7 +535,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_return false;}- LOG("\nPASS\n");+ LOGV(LOG_LEVEL_INFO, "\nPASS\n");return true;}@@ -530,7 +546,7 @@ static bool test_state_roundtrip(struct llama_model * model, const struct commonauto params_ctx = common_context_params_to_llama(params);auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};- LOG("\n=== Test 8: state blob round-trip ===\n");+ LOGV(LOG_LEVEL_INFO, "\n=== Test 8: state blob round-trip ===\n");if (llama_decode(ctx.get(), llama_batch_get_one(const_cast<llama_token *>(tokens.data()), (int32_t) tokens.size()))) {LOG_ERR("\n%s: failed to decode prompt\n", __func__);@@ -578,14 +594,29 @@ static bool test_state_roundtrip(struct llama_model * model, const struct commonreturn false;}- LOG("\nPASS\n");+ LOGV(LOG_LEVEL_INFO, "\nPASS\n");return true;}+struct test_suite {+ std::vector<test_status> results;++ bool all_passed() const {+ return std::all_of(results.begin(), results.end(), [](test_status s) { return s == test_status::PASS; });+ }+};++// column headers for the --models table, one per test, in the order they are run+static const std::vector<const char *> test_names = {+ "baseline", "seq_rm", "state_load", "cp_h", "cp_d", "cp_h_s", "cp_d_s", "rt",+};+// Run the full save/load test suite (tests 1-8) for a single model.-// Returns true if all tests pass, false otherwise.-static bool run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {+// Returns the per-test results.+static test_suite run_save_load_tests_for_model(const std::string & model_path, const struct common_params & base_params) {+ test_suite suite;+struct common_params params = base_params;params.model.path = model_path;@@ -594,7 +625,8 @@ static bool run_save_load_tests_for_model(const std::string & model_path, constif (model == nullptr) {LOG_ERR("%s: failed to init model '%s'\n", __func__, model_path.c_str());- return false;+ suite.results.assign(test_names.size(), test_status::SKIP);+ return suite;}GGML_ASSERT(llama_init->context() == nullptr);@@ -626,51 +658,48 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const// Test 1: baseline (saves state to disk)auto result_baseline = test_baseline(model, params, tokens);- if (result_baseline.empty()) {- return false;- }+ suite.results.push_back(result_baseline.empty() ? test_status::FAIL : test_status::PASS);// Test 2: sequence removal isolation- if (!test_seq_rm_isolated(model, params, tokens)) {- return false;- }+ suite.results.push_back(test_seq_rm_isolated(model, params, tokens) ? test_status::PASS : test_status::FAIL);- // Test 3: state load- if (!test_state_load(model, params, tokens, result_baseline)) {- return false;- }+ if (!result_baseline.empty()) {+ // Test 3: state load+ suite.results.push_back(test_state_load(model, params, tokens, result_baseline) ? test_status::PASS : test_status::FAIL);- // Test 4: seq copy (host)- if (!test_seq_cp_host(model, params, tokens, result_baseline)) {- return false;- }+ // Test 4: seq copy (host)+ suite.results.push_back(test_seq_cp_host(model, params, tokens, result_baseline) ? test_status::PASS : test_status::FAIL);- // Test 5: seq copy (device)- if (!test_seq_cp_device(model, params, tokens, result_baseline)) {- return false;+ // Test 5: seq copy (device)+ suite.results.push_back(test_seq_cp_device(model, params, tokens, result_baseline) ? test_status::PASS : test_status::FAIL);+ } else {+ // tests 3-5 depend on the baseline result and the state file it saves+ suite.results.push_back(test_status::SKIP);+ suite.results.push_back(test_status::SKIP);+ suite.results.push_back(test_status::SKIP);}// Test 6: seq copy (host, scatter)- if (!test_seq_cp_scatter(model, params, tokens, 6, false)) {- return false;- }+ suite.results.push_back(test_seq_cp_scatter(model, params, tokens, 6, false) ? test_status::PASS : test_status::FAIL);// Test 7: seq copy (device, scatter)- if (!test_seq_cp_scatter(model, params, tokens, 7, true)) {- return false;- }+ suite.results.push_back(test_seq_cp_scatter(model, params, tokens, 7, true) ? test_status::PASS : test_status::FAIL);// Test 8: state blob round-trip- if (!test_state_roundtrip(model, params, tokens)) {- return false;- }+ suite.results.push_back(test_state_roundtrip(model, params, tokens) ? test_status::PASS : test_status::FAIL);- LOG("\nAll tests passed.\n");-- return true;+ return suite;}+static void print_usage(int /* argc */, char ** argv) {+ LOG("\nexample usage:\n");+ LOG("\n %s -m your_model.gguf\n", argv[0]);+ LOG("\n %s --models tests/test-models\n", argv[0]);+ LOG("\n %s -m your_model.gguf -lv 5\n", argv[0]);+ LOG("\n");+}+int main(int argc, char ** argv) {std::setlocale(LC_NUMERIC, "C");@@ -707,7 +736,7 @@ int main(int argc, char ** argv) {params.model.path = models_dir;}- if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {+ if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON, print_usage)) {return 1;}@@ -742,27 +771,59 @@ int main(int argc, char ** argv) {return 1;}- LOG_INF("%s: running save/load tests over %zu models in '%s'\n", __func__, models.size(), models_dir.c_str());+ auto col_width = [](const char * name) { return (int) std::max(strlen(name), (size_t) 4); };++ size_t name_width = 5; // "Model"+ for (const auto & model_path : models) {+ name_width = std::max(name_width, std::filesystem::path(model_path).filename().string().size());+ }++ // silence everything but the table itself (LOG has verbosity LOG_LEVEL_OUTPUT = 0)+ common_log_set_verbosity_thold(0);++ LOG("%-*s", (int) name_width, "Model");+ for (const auto & name : test_names) {+ LOG(" %-*s", col_width(name), name);+ }+ LOG("\n");+ common_log_flush(common_log_main());size_t n_pass = 0;size_t n_fail = 0;for (const auto & model_path : models) {- LOG("\n================================================================\n");- LOG_INF("%s: model %s\n", __func__, model_path.c_str());+ const auto name = std::filesystem::path(model_path).filename().string();++ LOG("%-*s", (int) name_width, name.c_str());+ common_log_flush(common_log_main());- if (run_save_load_tests_for_model(model_path, params)) {+ const test_suite suite = run_save_load_tests_for_model(model_path, params);++ for (size_t i = 0; i < suite.results.size(); i++) {+ LOG(" %s%*s", test_status_str(suite.results[i]), col_width(test_names[i]) - 4, "");+ }+ LOG("\n");+ common_log_flush(common_log_main());++ if (suite.all_passed()) {n_pass++;} else {n_fail++;}}- LOG("\n================================================================\n");+ common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA);+ common_log_flush(common_log_main());+LOG_INF("%s: summary: %zu passed, %zu failed (of %zu)\n", __func__, n_pass, n_fail, models.size());return n_fail == 0 ? 0 : 1;}// single-model mode- return run_save_load_tests_for_model(params.model.path, params) ? 0 : 1;+ const test_suite suite = run_save_load_tests_for_model(params.model.path, params);+ const bool all_passed = suite.all_passed();+ if (all_passed) {+ LOG("\nAll tests passed.\n");+ }+ return all_passed ? 0 : 1;}