Skip to content

LLM inference in C/C++

C++ ggml

tests : catch vanished rows in the fusion baseline (#30618)

Georgi Gerganov88aaf325a702
diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.md
index b086d72162..927224fbfc 100644
--- a/.pi/gg/SYSTEM.md
+++ b/.pi/gg/SYSTEM.md
@@ -11,6 +11,7 @@ General:
- Never read the `AGENTS.md` file
Coding:
+- Never write inline comments in the code unless the user tells you to
- When in doubt, always refer to the CONTRIBUTING.md file of the project
- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments
- When referencing issues or PRs in comments, use the format:
@@ -23,13 +24,15 @@ Pull requests (PRs):
- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
- When creating a pull request, look for the repository's PR template and follow it
- For the AI usage disclosure section, write "YES", then these 4 bullets:
- - Model: `<model>`, linked to its Hugging Face page - the link text is the plain model name
- Harness: `pi:llama.cpp`
- Hardware: `<hardware>`
+ - Model: `<model>`, linked to its Hugging Face page - the link text is the plain model name
- Stats: `<stats>`
- Run `~/.pi/stats.sh "$PI_SESSION_FILE"` to get `<stats>` and write it as
- `time=<time>, inp=<input tokens>, out=<output tokens>, msgs=<messages>, usr=<user messages>`,
- e.g. `time=12.3m, inp=120k, out=45k, msgs=87, usr=6`
+ `◷ <time> · ↑ <input tokens> · ↓ <output tokens> · ≡ <messages> · ⌨ <user messages>`,
+ e.g. `◷ 12.3m · ↑ 120k · ↓ 45k · ≡ 87 · ⌨ 6`
+- Write these symbols as literal unicode in the PR description, never `:shortcodes:`, and keep
+ commit trailers ASCII
- If `PI_MODEL_NAME` or `PI_HARDWARE` is not set, ask the user and use their answer for `<model>` or `<hardware>`
- Always create the pull requests in draft mode
- Never reply to review comments or post comments on issues/PRs without explicit permission from the user
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index e839e20ce3..d31a9f23e5 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -156,6 +156,10 @@ llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,
llama4 ,0 ,any ,ADD+ADD , 2
llama4 ,0 ,any ,RMS_NORM+MUL , 9
maincoder ,0 ,any ,RMS_NORM+MUL , 9
+maion-coder ,0 ,any ,MUL+ADD , 4
+maion-coder ,0 ,any ,RMS_NORM+MUL , 17
+maion-coder ,0 ,any ,RMS_NORM+MUL+ADD , 8
+maion-coder ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
mamba ,0 ,any ,RMS_NORM+MUL , 3
mamba2 ,0 ,any ,RMS_NORM+MUL , 5
maple ,0 ,any ,MUL+ADD , 4
@@ -189,6 +193,8 @@ mistral4 ,0 ,any ,RMS_NORM+MUL , 9
mpt ,0 ,any ,NORM+MUL+ADD , 5
muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
+muse-spark ,0 ,any ,MUL+ADD , 4
+muse-spark ,0 ,any ,RMS_NORM+MUL , 14
nanbeige ,0 ,any ,RMS_NORM+MUL , 5
nemotron ,0 ,any ,NORM+MUL+ADD , 5
nemotron_h ,0 ,any ,RMS_NORM+MUL , 5
@@ -251,7 +257,6 @@ qwen3vlmoe ,1 ,any ,MUL+ADD , 4
qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9
qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
qwen4exp ,0 ,any ,ADD+ADD+ADD , 2
-qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9
qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1
qwen4exp ,0 ,any ,MUL+ADD , 4
qwen4exp ,0 ,any ,RMS_NORM+MUL , 13
diff --git a/tests/fusion/README.md b/tests/fusion/README.md
index 3ac02b061e..9951304fbf 100644
--- a/tests/fusion/README.md
+++ b/tests/fusion/README.md
@@ -22,5 +22,5 @@ rm -rf build-ci-models && mkdir -p build-ci-models
```
Non-zero exit means a row differs from the baseline. Use `--model FILE` to run a single
-architecture. Note `--check` only sees present rows — a fusion that stops matching is not
-reported, so diff the recorded CSV to catch removed patterns.
+architecture. `--check` also reports baseline rows that the run no longer produces, so a fusion
+that stops matching fails the run instead of silently dropping out of the CSV.
diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp
index 55e4f9df46..c268472ca7 100644
--- a/tests/test-fusion.cpp
+++ b/tests/test-fusion.cpp
@@ -32,6 +32,7 @@
#include <iostream>
#include <map>
#include <random>
+#include <set>
#include <string>
#include <vector>
@@ -191,6 +192,14 @@ static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t
api_stats_get(finfo, labels.data(), counts.data(), n);
}
+struct baseline_row {
+ std::string arch;
+ bool moe;
+ std::string mode;
+ std::string label;
+ uint64_t count;
+};
+
// one row of the per-label report
struct fusion_row {
std::string arch;
@@ -329,8 +338,8 @@ int main(int argc, char ** argv) {
const bool has_counts = true;
- // load the baseline (if any): key arch|moe|mode|label -> expected count
- std::map<std::string, uint64_t> baseline;
+ // load the baseline (if any): key arch|moe|mode|label -> expected row
+ std::map<std::string, baseline_row> baseline;
if (!check_path.empty()) {
std::ifstream in(check_path);
if (!in) {
@@ -352,12 +361,16 @@ int main(int argc, char ** argv) {
if (cols.size() != 5) {
continue;
}
- baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]);
+ const std::string key = cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3];
+ baseline[key] = { cols[0], cols[1] == "1", cols[2], cols[3], std::stoull(cols[4]) };
}
}
std::vector<fusion_row> rows;
+ std::set<std::string> run_arches;
+ std::set<std::string> produced;
+
LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());
const size_t seed = 1;
@@ -378,6 +391,8 @@ int main(int argc, char ** argv) {
continue;
}
+ run_arches.insert(arch);
+
struct mode_cfg {
std::string name;
std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);
@@ -480,7 +495,8 @@ int main(int argc, char ** argv) {
if (match) {
// one "any" row; use the worst NMSE across the two modes
const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label;
- const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0;
+ produced.insert(any_key);
+ const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key).count : 0;
const bool ok_count = check_path.empty() || d[0].count_fused == expected;
const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse;
const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus);
@@ -495,7 +511,8 @@ int main(int argc, char ** argv) {
}
const mode_data & a = d[mi];
const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label;
- const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0;
+ produced.insert(mode_key);
+ const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key).count : 0;
const bool ok_count = check_path.empty() || a.count_fused == expected;
rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused,
expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse });
@@ -507,6 +524,18 @@ int main(int argc, char ** argv) {
LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());
}
+ if (!check_path.empty()) {
+ for (const auto & kv : baseline) {
+ if (produced.count(kv.first) || !run_arches.count(kv.second.arch)) {
+ continue;
+ }
+ const baseline_row & b = kv.second;
+ LOG_WRN("%s: %s: fusion %s x%llu is no longer matched\n", __func__, b.arch.c_str(),
+ b.label.c_str(), (unsigned long long) b.count);
+ rows.push_back({ b.arch, b.moe, b.mode, b.label, 0, 0, b.count, 0.0, 0.0, false, true });
+ }
+ }
+
// print the report
{
std::ofstream out(record_path);
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index f2c1ca9035..eb72553b3e 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -448,7 +448,20 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : 2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
- ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid
+
+ uint32_t gating_func = 2;
+ switch (arch) {
+ case LLM_ARCH_MELLUM:
+ gating_func = 1;
+ break;
+ case LLM_ARCH_DEEPSEEK4:
+ gating_func = 4;
+ break;
+ default:
+ break;
+ }
+
+ ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, gating_func); // sigmoid : softmax : sqrtsoftplus
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
if (arch == LLM_ARCH_K2_HORIZON) {