LLM inference in C/C++
C++ ggml
tests : catch vanished rows in the fusion baseline (#30618)
Georgi Gerganov88aaf325a702
diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.mdindex b086d72162..927224fbfc 100644--- a/.pi/gg/SYSTEM.md+++ b/.pi/gg/SYSTEM.md@@ -11,6 +11,7 @@ General:- Never read the `AGENTS.md` fileCoding:+- Never write inline comments in the code unless the user tells you to- When in doubt, always refer to the CONTRIBUTING.md file of the project- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments- When referencing issues or PRs in comments, use the format:@@ -23,13 +24,15 @@ Pull requests (PRs):- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)- When creating a pull request, look for the repository's PR template and follow it- For the AI usage disclosure section, write "YES", then these 4 bullets:- - Model: `<model>`, linked to its Hugging Face page - the link text is the plain model name- Harness: `pi:llama.cpp`- Hardware: `<hardware>`+ - Model: `<model>`, linked to its Hugging Face page - the link text is the plain model name- Stats: `<stats>`- Run `~/.pi/stats.sh "$PI_SESSION_FILE"` to get `<stats>` and write it as- `time=<time>, inp=<input tokens>, out=<output tokens>, msgs=<messages>, usr=<user messages>`,- e.g. `time=12.3m, inp=120k, out=45k, msgs=87, usr=6`+ `◷ <time> · ↑ <input tokens> · ↓ <output tokens> · ≡ <messages> · ⌨ <user messages>`,+ e.g. `◷ 12.3m · ↑ 120k · ↓ 45k · ≡ 87 · ⌨ 6`+- Write these symbols as literal unicode in the PR description, never `:shortcodes:`, and keep+ commit trailers ASCII- If `PI_MODEL_NAME` or `PI_HARDWARE` is not set, ask the user and use their answer for `<model>` or `<hardware>`- Always create the pull requests in draft mode- Never reply to review comments or post comments on issues/PRs without explicit permission from the userdiff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csvindex e839e20ce3..d31a9f23e5 100644--- a/tests/fusion/MTL.csv+++ b/tests/fusion/MTL.csv@@ -156,6 +156,10 @@ llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,llama4 ,0 ,any ,ADD+ADD , 2llama4 ,0 ,any ,RMS_NORM+MUL , 9maincoder ,0 ,any ,RMS_NORM+MUL , 9+maion-coder ,0 ,any ,MUL+ADD , 4+maion-coder ,0 ,any ,RMS_NORM+MUL , 17+maion-coder ,0 ,any ,RMS_NORM+MUL+ADD , 8+maion-coder ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2mamba ,0 ,any ,RMS_NORM+MUL , 3mamba2 ,0 ,any ,RMS_NORM+MUL , 5maple ,0 ,any ,MUL+ADD , 4@@ -189,6 +193,8 @@ mistral4 ,0 ,any ,RMS_NORM+MUL , 9mpt ,0 ,any ,NORM+MUL+ADD , 5muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3+muse-spark ,0 ,any ,MUL+ADD , 4+muse-spark ,0 ,any ,RMS_NORM+MUL , 14nanbeige ,0 ,any ,RMS_NORM+MUL , 5nemotron ,0 ,any ,NORM+MUL+ADD , 5nemotron_h ,0 ,any ,RMS_NORM+MUL , 5@@ -251,7 +257,6 @@ qwen3vlmoe ,1 ,any ,MUL+ADD , 4qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2qwen4exp ,0 ,any ,ADD+ADD+ADD , 2-qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1qwen4exp ,0 ,any ,MUL+ADD , 4qwen4exp ,0 ,any ,RMS_NORM+MUL , 13diff --git a/tests/fusion/README.md b/tests/fusion/README.mdindex 3ac02b061e..9951304fbf 100644--- a/tests/fusion/README.md+++ b/tests/fusion/README.md@@ -22,5 +22,5 @@ rm -rf build-ci-models && mkdir -p build-ci-models```Non-zero exit means a row differs from the baseline. Use `--model FILE` to run a single-architecture. Note `--check` only sees present rows — a fusion that stops matching is not-reported, so diff the recorded CSV to catch removed patterns.+architecture. `--check` also reports baseline rows that the run no longer produces, so a fusion+that stops matching fails the run instead of silently dropping out of the CSV.diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cppindex 55e4f9df46..c268472ca7 100644--- a/tests/test-fusion.cpp+++ b/tests/test-fusion.cpp@@ -32,6 +32,7 @@#include <iostream>#include <map>#include <random>+#include <set>#include <string>#include <vector>@@ -191,6 +192,14 @@ static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_tapi_stats_get(finfo, labels.data(), counts.data(), n);}+struct baseline_row {+ std::string arch;+ bool moe;+ std::string mode;+ std::string label;+ uint64_t count;+};+// one row of the per-label reportstruct fusion_row {std::string arch;@@ -329,8 +338,8 @@ int main(int argc, char ** argv) {const bool has_counts = true;- // load the baseline (if any): key arch|moe|mode|label -> expected count- std::map<std::string, uint64_t> baseline;+ // load the baseline (if any): key arch|moe|mode|label -> expected row+ std::map<std::string, baseline_row> baseline;if (!check_path.empty()) {std::ifstream in(check_path);if (!in) {@@ -352,12 +361,16 @@ int main(int argc, char ** argv) {if (cols.size() != 5) {continue;}- baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]);+ const std::string key = cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3];+ baseline[key] = { cols[0], cols[1] == "1", cols[2], cols[3], std::stoull(cols[4]) };}}std::vector<fusion_row> rows;+ std::set<std::string> run_arches;+ std::set<std::string> produced;+LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());const size_t seed = 1;@@ -378,6 +391,8 @@ int main(int argc, char ** argv) {continue;}+ run_arches.insert(arch);+struct mode_cfg {std::string name;std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);@@ -480,7 +495,8 @@ int main(int argc, char ** argv) {if (match) {// one "any" row; use the worst NMSE across the two modesconst std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label;- const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0;+ produced.insert(any_key);+ const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key).count : 0;const bool ok_count = check_path.empty() || d[0].count_fused == expected;const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse;const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus);@@ -495,7 +511,8 @@ int main(int argc, char ** argv) {}const mode_data & a = d[mi];const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label;- const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0;+ produced.insert(mode_key);+ const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key).count : 0;const bool ok_count = check_path.empty() || a.count_fused == expected;rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused,expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse });@@ -507,6 +524,18 @@ int main(int argc, char ** argv) {LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());}+ if (!check_path.empty()) {+ for (const auto & kv : baseline) {+ if (produced.count(kv.first) || !run_arches.count(kv.second.arch)) {+ continue;+ }+ const baseline_row & b = kv.second;+ LOG_WRN("%s: %s: fusion %s x%llu is no longer matched\n", __func__, b.arch.c_str(),+ b.label.c_str(), (unsigned long long) b.count);+ rows.push_back({ b.arch, b.moe, b.mode, b.label, 0, 0, b.count, 0.0, 0.0, false, true });+ }+ }+// print the report{std::ofstream out(record_path);diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex f2c1ca9035..eb72553b3e 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -448,7 +448,20 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : 2));ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));- ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid++ uint32_t gating_func = 2;+ switch (arch) {+ case LLM_ARCH_MELLUM:+ gating_func = 1;+ break;+ case LLM_ARCH_DEEPSEEK4:+ gating_func = 4;+ break;+ default:+ break;+ }++ ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, gating_func); // sigmoid : softmax : sqrtsoftplusms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));if (arch == LLM_ARCH_K2_HORIZON) {