LLM inference in C/C++
C++ ggml
metal : single-source fusion table + fusion debug rework (#28164)
Georgi Gerganova2878d30df01
diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.ymlnew file mode 100644index 0000000000..ad7d5ab60e--- /dev/null+++ b/.github/workflows/fusion.yml@@ -0,0 +1,67 @@+name: Fusion++on:+ workflow_dispatch: # allows manual triggering+ push:+ branches:+ - master+ paths: [+ '.github/workflows/fusion.yml',+ 'ggml/**',+ 'tests/fusion/**',+ 'tests/test-fusion.cpp'+ ]++ pull_request:+ types: [opened, synchronize, reopened]+ paths: [+ '.github/workflows/fusion.yml',+ 'ggml/**',+ 'tests/fusion/**',+ 'tests/test-fusion.cpp'+ ]++concurrency:+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}+ cancel-in-progress: true++env:+ GGML_NLOOP: 3+ GGML_N_THREADS: 1+ LLAMA_ARG_LOG_COLORS: 1+ LLAMA_ARG_LOG_PREFIX: 1+ LLAMA_ARG_LOG_TIMESTAMPS: 1++jobs:+ # TODO: add jobs for other backends as they adopt the fusion debug API+ metal:+ runs-on: [self-hosted, macOS, ARM64]++ steps:+ - name: Clone+ id: checkout+ uses: actions/checkout@v6++ - name: Build+ id: cmake_build+ run: |+ cmake -B build \+ -DCMAKE_BUILD_TYPE=Release \+ -DLLAMA_FATAL_WARNINGS=ON \+ -DLLAMA_OPENSSL=OFF \+ -DGGML_SCHED_NO_REALLOC=ON \+ -DGGML_BLAS=OFF \+ -DGGML_METAL=ON+ time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)+ time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)++ - name: Generate models+ id: generate_models+ run: |+ rm -rf build-ci-models && mkdir -p build-ci-models+ ./build/bin/test-llama-archs -o build-ci-models++ - name: Test fusion+ id: test_fusion+ run: |+ ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csvdiff --git a/ci/run.sh b/ci/run.shindex 5463597274..294cbe57bb 100755--- a/ci/run.sh+++ b/ci/run.sh@@ -334,6 +334,35 @@ function gg_sum_test_llama_archs_tensor_split {gg_printf '```\n'}+# test_llama_archs_models++function gg_run_test_llama_archs_models {+ cd ${SRC}++ set -e++ # TODO: fix and re-enable `test-llama-archs` on OpenVINO+ # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists+ if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then+ rm -rf build-ci-models && mkdir -p build-ci-models++ # generate the dummy models used by the model-dependent tests+ ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1+ fi++ set +e+}++function gg_sum_test_llama_archs_models {+ gg_printf '### %s\n\n' "${ci}"++ gg_printf 'Generates the dummy models used by the model-dependent tests\n'+ gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"+ gg_printf '```\n'+ gg_printf '%s\n' "$(cat $OUT/${ci}.log)"+ gg_printf '```\n'+}+# test_scriptsfunction gg_run_test_scripts {@@ -790,6 +819,7 @@ ret=0test $ret -eq 0 && gg_run ctest_debugtest $ret -eq 0 && gg_run ctest_release+test $ret -eq 0 && gg_run test_llama_archs_modelstest $ret -eq 0 && gg_run test_llama_archs_tensor_splitif [ ! -z ${GG_BUILD_HIGH_PERF} ]; thendiff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txtindex a661e710a2..e7afdb6957 100644--- a/ggml/src/ggml-metal/CMakeLists.txt+++ b/ggml/src/ggml-metal/CMakeLists.txt@@ -10,6 +10,7 @@ ggml_add_backend_library(ggml-metalggml-metal-device.cppggml-metal-common.cppggml-metal-context.m+ ggml-metal-fusion.cppggml-metal-ops.cppggml-metal-tuning.cpp)diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cppindex 6f1638a114..05755eb3b2 100644--- a/ggml/src/ggml-metal/ggml-metal-common.cpp+++ b/ggml/src/ggml-metal/ggml-metal-common.cpp@@ -1,4 +1,5 @@#include "ggml-metal-common.h"+#include "ggml-metal-fusion.h"#include "ggml.h"#include "ggml-impl.h"@@ -390,59 +391,31 @@ static std::vector<int> ggml_metal_graph_optimize_reorder(const std::vector<node}void ggml_graph_optimize(ggml_cgraph * gf) {- constexpr int MAX_FUSE = 16;-const int n = gf->n_nodes;- enum ggml_op ops[MAX_FUSE];-std::vector<node_info> nodes;nodes.reserve(gf->n_nodes);// fuse nodes:// we don't want to make reorders that break fusing, so we first pack all fusable tensors// and perform the reorder over the fused nodes. after the reorder is done, we unfuse+ //+ // the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the+ // packing here is driven by the same patterns that the op encoders will later usefor (int i = 0; i < n; i++) {node_info node = {/*.node =*/ gf->nodes[i],/*.fused =*/ {},};- // fuse only ops that start with these operations- // can be expanded when needed- if (node.op() == GGML_OP_ADD ||- node.op() == GGML_OP_NORM ||- node.op() == GGML_OP_RMS_NORM) {- ops[0] = node.op();-- int f = i + 1;- while (f < n && f < i + MAX_FUSE) {- // conservatively allow fusing only these ops- // can be expanded when needed- if (gf->nodes[f]->op != GGML_OP_ADD &&- gf->nodes[f]->op != GGML_OP_MUL &&- gf->nodes[f]->op != GGML_OP_NORM &&- gf->nodes[f]->op != GGML_OP_RMS_NORM) {- break;- }- ops[f - i] = gf->nodes[f]->op;- f++;- }-- f -= i;- for (; f > 1; f--) {- if (ggml_can_fuse(gf, i, ops, f)) {- break;- }- }+ const int f = ggml_metal_fusion_max(gf, i);- // add the fused tensors into the node info so we can unfuse them later- for (int k = 1; k < f; k++) {- ++i;+ // add the fused tensors into the node info so we can unfuse them later+ for (int k = 1; k < f; k++) {+ ++i;- // the .dst() becomes the last fused tensor- node.add_fused(gf->nodes[i]);- }+ // the .dst() becomes the last fused tensor+ node.add_fused(gf->nodes[i]);}nodes.push_back(std::move(node));diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.hindex abf4b06ed2..b538b1ad20 100644--- a/ggml/src/ggml-metal/ggml-metal-context.h+++ b/ggml/src/ggml-metal/ggml-metal-context.h@@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx);void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb);void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data);+bool ggml_metal_supports_family (ggml_metal_t ctx, int family);void ggml_metal_capture_next_compute(ggml_metal_t ctx);diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.mindex 6cdc4006bc..bf4fe2dcd5 100644--- a/ggml/src/ggml-metal/ggml-metal-context.m+++ b/ggml/src/ggml-metal/ggml-metal-context.m@@ -6,6 +6,7 @@#import "ggml-metal-impl.h"#import "ggml-metal-common.h"#import "ggml-metal-ops.h"+#import "ggml-metal-fusion.h"#import <Foundation/Foundation.h>@@ -36,15 +37,12 @@ struct ggml_metal {// additional, inference-time compiled pipelinesggml_metal_pipelines_t pipelines_ext;- bool use_fusion;bool use_concurrency;bool use_graph_optimize;int debug_graph;- int debug_fusion;- // how many times a given op was fused- uint64_t fuse_cnt[GGML_OP_COUNT];+ struct ggml_metal_fusion_info * finfo;// capture stateint capture_compute;@@ -139,7 +137,6 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);- res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;{@@ -147,20 +144,19 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {res->debug_graph = val ? atoi(val) : 0;}- {- const char * val = getenv("GGML_METAL_FUSION_DEBUG");- res->debug_fusion = val ? atoi(val) : 0;- }-res->use_graph_optimize = true;if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) {res->use_graph_optimize = false;}- memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt));+ res->finfo = ggml_metal_device_get_fusion_info(dev);+ if (ggml_metal_fusion_info_stats(res->finfo)) {+ ggml_metal_fusion_info_labels_init(res->finfo);+ res->n_cb = 0;+ }- GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false");+ GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false");GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false");GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false");@@ -222,15 +218,18 @@ void ggml_metal_free(ggml_metal_t ctx) {ctx->pipelines_ext = nil;}- if (ctx->debug_fusion > 0) {+ if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) {GGML_LOG_DEBUG("%s: fusion stats:\n", __func__);- for (int i = 0; i < GGML_OP_COUNT; i++) {- if (ctx->fuse_cnt[i] == 0) {++ const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo);+ for (int i = 0; i < n_fusions; i++) {+ const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i);+ if (count == 0) {continue;}// note: cannot use ggml_log here- GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]);+ GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count);}}@@ -481,10 +480,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph *@autoreleasepool {ctx->gf = gf;- ctx->n_nodes_0 = MIN(n_main, gf->n_nodes);- ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0;+ if (ctx->n_cb == 0) {+ // single-threaded encoding: the whole graph is encoded by one command buffer+ ctx->n_nodes_0 = gf->n_nodes;+ ctx->n_nodes_1 = 0;+ ctx->n_nodes_per_cb = 0;+ } else {+ ctx->n_nodes_0 = MIN(n_main, gf->n_nodes);+ ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0;- ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb;+ ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb;+ }if (ctx->capture_compute >= 0) {ctx->capture_compute--;@@ -682,6 +688,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) {}void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {+ // when fusion stats are collected the graph must be encoded by a single thread so the+ // counters are race-free; override whatever the caller requested+ if (ggml_metal_fusion_info_stats(ctx->finfo)) {+ n_cb = 0;+ }+if (ctx->n_cb != n_cb) {ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS);@@ -717,13 +729,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {ctx->dev,cmd_buf,ctx->gf,+ ctx->finfo,idx_start,idx_end,- ctx->use_fusion,ctx->use_concurrency,ctx->capture_compute,- ctx->debug_graph,- ctx->debug_fusion);+ ctx->debug_graph);for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) {const int res = ggml_metal_op_encode(ctx_op, idx);diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex 31fc07d44d..ced33aadfb 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -325,6 +325,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_teconst struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev);+struct ggml_metal_fusion_info;++// the device-owned fusion debugging context (NULL unless fusion debugging is enabled)+struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev);+//// device buffers//diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex afd6f52101..5654c50040 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1,4 +1,5 @@#import "ggml-metal-device.h"+#import "ggml-metal-fusion.h"#import "ggml-impl.h"#import "ggml-backend-impl.h"@@ -896,6 +897,9 @@ struct ggml_metal_device {struct ggml_metal_device_props props;+ // shared fusion debugging context+ struct ggml_metal_fusion_info * finfo;+// virtual address for GPU memory allocationsatomic_uintptr_t addr_virt;};@@ -1274,6 +1278,13 @@ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;}+ {+ const char * val = getenv("GGML_METAL_FUSION_DEBUG");+ dev->finfo = ggml_metal_fusion_info_init(+ getenv("GGML_METAL_FUSION_DISABLE") == nil,+ val ? atoi(val) : 0);+ }+snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);const char * gpu_name = [[dev->mtl_device name] UTF8String];if (n_devices > 1) {@@ -1348,6 +1359,8 @@ void ggml_metal_device_free(ggml_metal_device_t dev) {assert(dev != NULL);@autoreleasepool {+ ggml_metal_fusion_info_free(dev->finfo);+ggml_metal_rsets_free(dev->rsets);ggml_metal_library_free(dev->library);@@ -1935,6 +1948,10 @@ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {dev->props.has_tensor = false;}+struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) {+ return dev->finfo;+}+//// device buffers//diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cppnew file mode 100644index 0000000000..ac3ac04148--- /dev/null+++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp@@ -0,0 +1,502 @@+#include "ggml-metal-fusion.h"++#include "ggml-backend-impl.h"+#include "ggml-metal-device.h"++#include <algorithm>+#include <string>+#include <vector>++// ---- helpers -------------------------------------------------------------++// true if two tensors live in the same Metal buffer+static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) {+ if (!a || !b) {+ return false;+ }++ ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer;+ ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer;++ ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context;+ ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context;++ return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal;+}++// ---- pattern checks ------------------------------------------------------++// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input+// width, be contiguous rows, and the fused outputs must stay F32+static bool ggml_metal_fusion_check_norm(+ const ggml_metal_fusion * fusion,+ const ggml_tensor * const * nodes,+ ggml_metal_fusion_mode mode) {+ GGML_UNUSED(mode);++ GGML_ASSERT(fusion->n_ops >= 2);++ for (int j = 1; j < fusion->n_ops; j++) {+ // the fused MUL/ADD must read the previous node as src0+ if (nodes[j]->src[0] != nodes[j - 1]) {+ return false;+ }++ // the weight/bias must have the same row width as the norm input+ if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) {+ return false;+ }++ if (!ggml_is_contiguous_rows(nodes[j]->src[1])) {+ return false;+ }++ if (nodes[j]->type != GGML_TYPE_F32) {+ return false;+ }+ }++ return true;+}++// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout+// (and, in FULL mode, live in the same Metal buffer)+static bool ggml_metal_fusion_check_add_chain(+ const ggml_metal_fusion * fusion,+ const ggml_tensor * const * nodes,+ ggml_metal_fusion_mode mode) {+ GGML_ASSERT(fusion->n_ops >= 2);++ for (int j = 1; j < fusion->n_ops; j++) {+ if (nodes[j]->src[0] != nodes[j - 1]) {+ return false;+ }++ if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) {+ return false;+ }++ if (mode == GGML_METAL_FUSION_FULL) {+ if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) {+ return false;+ }+ }+ }++ return true;+}++// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent+// cache, so the gdn kernel writes them straight to the cache and the cpy is elided.+// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the+// attn scores view), so unlike the other patterns this is not an elision chain: the structural+// checks live entirely in this callback (unsafe = true).+static bool ggml_metal_fusion_check_gdn_cache(+ const ggml_metal_fusion * fusion,+ const ggml_tensor * const * nodes,+ ggml_metal_fusion_mode mode) {+ GGML_UNUSED(fusion);++ const ggml_tensor * gdn = nodes[0];+ const ggml_tensor * cpy = nodes[1];++ // the kernel skips the snapshot tail, so the gdn output must not be a graph output+ if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) {+ return false;+ }++ if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) {+ return false;+ }++ const int64_t S_v = gdn->src[2]->ne[0];+ const int64_t H = gdn->src[2]->ne[1];+ const int64_t n_tokens = gdn->src[2]->ne[2];+ const int64_t n_seqs = gdn->src[2]->ne[3];+ const int64_t K = ggml_get_op_params_i32(gdn, 0);+ const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs);++ const int64_t D = S_v * S_v * H;+ const int64_t n_written = std::min<int64_t>(n_tokens, K);++ const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view+ const ggml_tensor * dst = cpy->src[1]; // cache view++ // src must be this gdn's snapshot tail (contiguous, at the tail offset)+ if (src->op != GGML_OP_VIEW || src->view_src != gdn ||+ src->view_offs != tail_off || !ggml_is_contiguous(src)) {+ return false;+ }++ const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 };+ if (dst->type != GGML_TYPE_F32 ||+ !std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) ||+ dst->nb[0] != ggml_type_size(GGML_TYPE_F32) ||+ dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) {+ return false;+ }++ if (mode == GGML_METAL_FUSION_FULL) {+ // the cache must be allocated so the kernel can write straight to its buffer+ if (dst->data == nullptr) {+ return false;+ }+ }++ return true;+}++// MUL + SIN + SQR + MUL + ADD (snake activation)+static bool ggml_metal_fusion_check_snake(+ const ggml_metal_fusion * fusion,+ const ggml_tensor * const * nodes,+ ggml_metal_fusion_mode mode) {+ GGML_UNUSED(fusion);+ GGML_UNUSED(mode);++ const ggml_tensor * mul0 = nodes[0];+ const ggml_tensor * sin_node = nodes[1];+ const ggml_tensor * sqr = nodes[2];+ const ggml_tensor * mul1 = nodes[3];+ const ggml_tensor * add = nodes[4];++ // x carries the full activation shape, a is the broadcast operand+ const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1];+ const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0];++ // mul1 reads sqr and inv_b in either operand order+ const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0];++ // closure check: the trailing add reads the same x as the leading mul+ const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0];++ // x is in the supported whitelist and every chain intermediate shares x's type.+ // a and inv_b bind as device const float * in the kernel, so they stay F32.+ const bool types_ok =+ (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) &&+ (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) &&+ (mul0->type == x->type) && (sin_node->type == x->type) &&+ (sqr->type == x->type) && (mul1->type == x->type) &&+ (add->type == x->type);++ // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D+ const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1];+ const bool dim_ok =+ (x->ne[2] == 1) && (x->ne[3] == 1) &&+ (add->ne[2] == 1) && (add->ne[3] == 1) &&+ (a->ne[2] == 1) && (a->ne[3] == 1) &&+ (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1);++ // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous+ const bool contig_ok =+ ggml_is_contiguous(x) && ggml_is_contiguous(add) &&+ ggml_is_contiguous(a) && ggml_is_contiguous(inv_b);++ return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x;+}++// ---- patterns ------------------------------------------------------------++static const ggml_op ops_norm_mul[] = { GGML_OP_NORM, GGML_OP_MUL };+static const ggml_op ops_norm_mul_add[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD };+static const ggml_op ops_rms_norm_mul[] = { GGML_OP_RMS_NORM, GGML_OP_MUL };+static const ggml_op ops_rms_norm_mul_add[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD };++static const ggml_op ops_add_2[] = { GGML_OP_ADD, GGML_OP_ADD };+static const ggml_op ops_add_3[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };+static const ggml_op ops_add_4[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };+static const ggml_op ops_add_5[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };+static const ggml_op ops_add_6[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };+static const ggml_op ops_add_7[] = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD };+static const ggml_op ops_snake[] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD };++static const ggml_op ops_gdn_cache[] = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY };++static const ggml_metal_fusion ggml_metal_fusions[] = {+ { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, 2, false, ggml_metal_fusion_check_norm },+ { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, 3, false, ggml_metal_fusion_check_norm },+ { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, 2, false, ggml_metal_fusion_check_norm },+ { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, 3, false, ggml_metal_fusion_check_norm },+ { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, 2, false, ggml_metal_fusion_check_add_chain },+ { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, 3, false, ggml_metal_fusion_check_add_chain },+ { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, 4, false, ggml_metal_fusion_check_add_chain },+ { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, 5, false, ggml_metal_fusion_check_add_chain },+ { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, 6, false, ggml_metal_fusion_check_add_chain },+ { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, 7, false, ggml_metal_fusion_check_add_chain },+ { GGML_METAL_FUSION_SNAKE, ops_snake, 5, false, ggml_metal_fusion_check_snake },+ { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, 2, true, ggml_metal_fusion_check_gdn_cache },+};++const ggml_metal_fusion * ggml_metal_fusion_all(int * n) {+ *n = (int) sizeof(ggml_metal_fusions) / sizeof(ggml_metal_fusions[0]);++ return ggml_metal_fusions;+}++// ---- shared fusion info ---------------------------------------------------++static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) {+ GGML_ASSERT(fusion != nullptr);++ std::string label;+ for (int j = 0; j < fusion->n_ops; j++) {+ if (j > 0) {+ label += '+';+ }+ label += ggml_op_name(fusion->ops[j]);+ }+ return label;+}++struct ggml_metal_fusion_info {+ std::vector<std::string> labels;+ std::vector<uint64_t> counts;+ bool enabled;+ bool stats;+ bool labels_set;+ int debug;+};++struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) {+ ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info;+ finfo->enabled = enabled;+ finfo->stats = debug > 0;+ finfo->labels_set = false;+ finfo->debug = debug;++ if (finfo->stats) {+ ggml_metal_fusion_info_labels_init(finfo);+ }++ return finfo;+}++void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo) {+ delete finfo;+}++bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo) {+ return finfo->enabled;+}++bool ggml_metal_fusion_info_stats(const struct ggml_metal_fusion_info * finfo) {+ return finfo->stats;+}++int ggml_metal_fusion_info_debug(const struct ggml_metal_fusion_info * finfo) {+ return finfo->debug;+}++int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo) {+ return (int) finfo->labels.size();+}++const char * ggml_metal_fusion_info_label(const struct ggml_metal_fusion_info * finfo, int idx) {+ GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size());+ return finfo->labels[idx].c_str();+}++uint64_t ggml_metal_fusion_info_count(const struct ggml_metal_fusion_info * finfo, int idx) {+ GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size());+ return finfo->counts[idx];+}++void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion) {+ if (!finfo->stats || fusion == nullptr) {+ return;+ }++ int n = 0;+ const ggml_metal_fusion * all = ggml_metal_fusion_all(&n);++ int idx = -1;+ for (int i = 0; i < n; i++) {+ if (&all[i] == fusion) {+ idx = i;+ break;+ }+ }++ if (idx >= 0 && idx < (int) finfo->counts.size()) {+ finfo->counts[idx]++;+ }+}++void ggml_metal_fusion_info_set_enabled(struct ggml_metal_fusion_info * finfo, bool enabled) {+ finfo->enabled = enabled;+}++void ggml_metal_fusion_info_labels_init(struct ggml_metal_fusion_info * finfo) {+ if (finfo->labels_set) {+ return;+ }++ int n = 0;+ const ggml_metal_fusion * all = ggml_metal_fusion_all(&n);++ finfo->labels.clear();+ finfo->counts.assign(n, 0);+ finfo->labels.reserve(n);++ for (int i = 0; i < n; i++) {+ finfo->labels.emplace_back(ggml_metal_fusion_label(&all[i]));+ }++ finfo->labels_set = true;+}++void ggml_metal_fusion_info_stats_init(struct ggml_metal_fusion_info * finfo) {+ finfo->stats = true;+ ggml_metal_fusion_info_labels_init(finfo);+}++void ggml_metal_fusion_info_stats_reset(struct ggml_metal_fusion_info * finfo) {+ std::fill(finfo->counts.begin(), finfo->counts.end(), 0);+}++int ggml_metal_fusion_info_stats_get(const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) {+ const int n_fusions = (int) finfo->labels.size();++ if (labels == nullptr) {+ return n_fusions;+ }++ const int n_fill = std::min(n, n_fusions);+ for (int i = 0; i < n_fill; i++) {+ labels[i] = finfo->labels[i].c_str();+ if (counts != nullptr) {+ counts[i] = finfo->counts[i];+ }+ }++ return n_fill;+}++// ---- queries -------------------------------------------------------------++// find the longest pattern matching the node sequence starting at idx+// (idx is a position in node_idxs, which maps to graph node indices)+const ggml_metal_fusion * ggml_metal_fusion_next(+ const ggml_cgraph * gf,+ const int * node_idxs,+ int n_idxs,+ int idx,+ ggml_metal_fusion_mode mode,+ int * n_out) {+ int n = 0;+ const ggml_metal_fusion * all = ggml_metal_fusion_all(&n);++ const ggml_metal_fusion * res = nullptr;+ int best = 1;++ for (int i = 0; i < n; i++) {+ const ggml_metal_fusion * fusion = &all[i];++ // only look for a longer match than the current best+ if (fusion->n_ops <= best) {+ continue;+ }+ if (idx + fusion->n_ops > n_idxs) {+ continue;+ }++ const ggml_tensor * nodes[GGML_METAL_FUSION_MAX];++ // the op sequence must match exactly+ bool ok = true;+ for (int j = 0; j < fusion->n_ops; j++) {+ nodes[j] = gf->nodes[node_idxs[idx + j]];+ if (nodes[j]->op != fusion->ops[j]) {+ ok = false;+ break;+ }+ }+ if (!ok) {+ continue;+ }++ if (!fusion->unsafe) {+ // common element-wise chain constraints: each node reads the previous one,+ // and all nodes have the same shape+ for (int j = 1; j < fusion->n_ops && ok; j++) {+ if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) {+ ok = false;+ break;+ }+ if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) {+ ok = false;+ break;+ }+ }+ if (!ok) {+ continue;+ }++ // all current fusions are single-output elision chains, so the last node is the only output+ // TODO: multi-output fusions: store pattern-relative offsets in the table and translate them here+ int outputs_buf[1];+ outputs_buf[0] = node_idxs[idx + fusion->n_ops - 1];++ // structural subgraph checks (op sequence, elidable uses, view containment)+ if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, fusion->n_ops, fusion->ops, outputs_buf, 1)) {+ continue;+ }+ }++ // pattern-specific checks (the sole validator for unsafe patterns)+ if (fusion->check && !fusion->check(fusion, nodes, mode)) {+ continue;+ }++ best = fusion->n_ops;+ res = fusion;+ }++ *n_out = best;++ return res;+}++// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that+// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view+// transparent) node sequence that the compute phase uses, so the returned count is the raw index+// span from idx to the last matched node (intermediate views are packed along).+int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) {+ // an empty/view node cannot start a pattern - pack it alone+ if (ggml_op_is_empty(gf->nodes[idx]->op) || ggml_is_empty(gf->nodes[idx])) {+ return 1;+ }++ // collect the non-empty node indices starting at idx+ int idxs[GGML_METAL_FUSION_MAX];+ int n_idxs = 0;+ for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) {+ if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) {+ idxs[n_idxs++] = i;+ }+ }+ if (n_idxs == 0) {+ return 1;+ }++ int total = 0;+ int i_f = 0;++ while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) {+ int len = 1;+ const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len);+ if (!fusion || total + len > GGML_METAL_FUSION_MAX) {+ break;+ }++ total += len;+ i_f += len;+ }++ if (i_f == 0) {+ return 1;+ }++ // map the matched non-empty nodes back to the raw index span (views are included)+ return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1);+}diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.hnew file mode 100644index 0000000000..e8515bdeca--- /dev/null+++ b/ggml/src/ggml-metal/ggml-metal-fusion.h@@ -0,0 +1,104 @@+// single source of truth for the fusions supported by the Metal backend+//+// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in+// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max)+// and the op encoders (ggml_metal_fusion_next) consult this same table, so the two+// phases can never disagree about what can be fused.++#pragma once++#include "ggml-impl.h"++#include <stdbool.h>++#ifdef __cplusplus+extern "C" {+#endif++// the maximum number of nodes that can be fused in a single kernel+// (also the maximum length of a packed fusion group during graph optimization)+#define GGML_METAL_FUSION_MAX 16++typedef enum ggml_metal_fusion_mode {+ // structural checks only; used by the graph optimizer, at which point the graph+ // tensors are not allocated yet, so buffer placement cannot be verified+ GGML_METAL_FUSION_STRUCTURAL = 0,+ // full checks, including buffer placement; used by the op encoders+ GGML_METAL_FUSION_FULL,+} ggml_metal_fusion_mode;++// identifier of each fusion pattern so the op encoders know which kernel to use+typedef enum ggml_metal_fusion_id {+ GGML_METAL_FUSION_NONE = 0,+ GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL+ GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD+ GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7])+ GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD+ GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache)+} ggml_metal_fusion_id;++struct ggml_metal_fusion {+ ggml_metal_fusion_id id;++ const enum ggml_op * ops; // op sequence (fixed length)+ int n_ops; // number of ops++ // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the+ // check callback below is the sole validator (used for patterns that are not elision chains,+ // e.g. the gdn + cache-cpy write-through fusion)+ bool unsafe;++ // extra backend constraints on top of ggml_can_fuse_subgraph+ // nodes[j] is the j-th node of the pattern+ bool (*check)(const struct ggml_metal_fusion * fusion,+ const struct ggml_tensor * const * nodes,+ ggml_metal_fusion_mode mode);+};++typedef struct ggml_metal_fusion ggml_metal_fusion;++// the single table of all fusions supported by the Metal backend+const ggml_metal_fusion * ggml_metal_fusion_all(int * n);++// ---- shared fusion info ---------------------------------------------------++// shared fusion debugging context, owned by the device; newly created backend contexts for that+// device register with it so the fusion counters are race-free and accumulate across contexts.+struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp++struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug);+void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo);++bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo);+bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo);+int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo);++int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo);+const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx);+uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx);++void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion);+void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled);++void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo);+void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo);+int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n);+void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo);++// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode`.+// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed.+const ggml_metal_fusion * ggml_metal_fusion_next(+ const struct ggml_cgraph * gf,+ const int * node_idxs,+ int n_idxs,+ int idx,+ ggml_metal_fusion_mode mode,+ int * n_out);++// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that+// could be fused, chaining patterns back-to-back. returns at least 1.+int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx);++#ifdef __cplusplus+}+#endifdiff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.hindex 28a9ba101c..7ad21341e4 100644--- a/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ggml/src/ggml-metal/ggml-metal-impl.h@@ -985,6 +985,7 @@ typedef struct {uint64_t nb1;uint64_t nb2;uint64_t nb3;+ uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused)} ggml_metal_kargs_gated_delta_net;typedef struct {diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 3db8bca437..b4e87cb2cc 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -7,6 +7,7 @@#include "ggml-metal-impl.h"#include "ggml-metal-common.h"#include "ggml-metal-device.h"+#include "ggml-metal-fusion.h"#include "ggml-metal-tuning.h"#include <cassert>@@ -31,24 +32,22 @@ struct ggml_metal_op {ggml_metal_device_t dev,ggml_metal_cmd_buf_t cmd_buf,ggml_cgraph * gf,+ ggml_metal_fusion_info * finfo,int idx_start,int idx_end,- bool use_fusion,bool use_concurrency,bool use_capture,- int debug_graph,- int debug_fusion) {+ int debug_graph) {this->dev = dev;this->lib = ggml_metal_device_get_library(dev);this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency);this->mem_ranges = ggml_mem_ranges_init(debug_graph);+ this->finfo = finfo;this->idx_start = idx_start;this->idx_end = idx_end;- this->use_fusion = use_fusion;this->use_concurrency = use_concurrency;this->use_capture = use_capture;this->debug_graph = debug_graph;- this->debug_fusion = debug_fusion;this->gf = gf;idxs.reserve(gf->n_nodes);@@ -78,15 +77,24 @@ struct ggml_metal_op {return ggml_graph_node(gf, idxs[i]);}- bool can_fuse(int i0, const ggml_op * ops, int n_ops) const {- assert(use_fusion);+ // consult the fusion table for the longest pattern starting at i0+ // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes+ const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const {+ assert(use_fusion());assert(i0 >= 0 && i0 < n_nodes());- if (i0 + n_ops > n_nodes()) {- return false;- }+ return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out);+ }++ // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned+ // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test)+ bool use_fusion() const {+ return ggml_metal_fusion_info_enabled(finfo);+ }- return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops);+ // record that a fusion fired, indexed by the matching table entry+ void count_fusions(const ggml_metal_fusion * fusion) const {+ ggml_metal_fusion_info_count_fusion(finfo, fusion);}ggml_metal_device_t dev;@@ -94,12 +102,13 @@ struct ggml_metal_op {ggml_metal_encoder_t enc;ggml_mem_ranges_t mem_ranges;- bool use_fusion;+ // shared fusion debugging context+ ggml_metal_fusion_info * finfo;+bool use_concurrency;bool use_capture;int debug_graph;- int debug_fusion;private:ggml_cgraph * gf;@@ -115,24 +124,22 @@ ggml_metal_op_t ggml_metal_op_init(ggml_metal_device_t dev,ggml_metal_cmd_buf_t cmd_buf,ggml_cgraph * gf,+ ggml_metal_fusion_info * finfo,int idx_start,int idx_end,- bool use_fusion,bool use_concurrency,bool use_capture,- int debug_graph,- int debug_fusion) {+ int debug_graph) {ggml_metal_op_t res = new ggml_metal_op(dev,cmd_buf,gf,+ finfo,idx_start,idx_end,- use_fusion,use_concurrency,use_capture,- debug_graph,- debug_fusion);+ debug_graph);return res;}@@ -1868,6 +1875,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {ggml_metal_library_t lib = ctx->lib;ggml_metal_encoder_t enc = ctx->enc;+ const bool use_fusion = ctx->use_fusion();+ const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo);GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);@@ -1880,6 +1889,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op);+ // when fused with the trailing cache cpy, the snapshots are written straight into the+ // recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE)+ ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op);+ uint64_t nb_out = 0;+ int n_fuse = 1;++ if (use_fusion) {+ int n = 1;+ const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);++ if (fusion && fusion->id == GGML_METAL_FUSION_GDN_CACHE) {+ const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view++ bid_out = ggml_metal_get_buffer_id(dst_cache);+ nb_out = dst_cache->nb[2]/sizeof(float);+ n_fuse = 2;++ ctx->count_fusions(fusion);++ if (debug_fusion > 1) {+ GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__);+ }+ }+ }+int ida = 0;ggml_metal_kargs_gated_delta_net args = {@@ -1918,23 +1952,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {/*.nb1 =*/ nb1,/*.nb2 =*/ nb2,/*.nb3 =*/ nb3,+ /*.nb_out =*/ nb_out,};ggml_metal_encoder_set_pipeline(enc, pipeline);- ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++);+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // argsggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // qggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // kggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // vggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gateggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // betaggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn)+ ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_outconst int nsg = pipeline.nsg;ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1);- return 1;+ return n_fuse;}int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) {@@ -3718,56 +3754,20 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {return 1;}-// Snake activation autofuse: mul -> sin -> sqr -> mul -> add-static bool ggml_metal_op_can_fuse_snake(ggml_metal_op_t ctx, int idx) {- static constexpr ggml_op snake_ops[5] = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD };-- if (ctx->node(idx)->op != GGML_OP_MUL || !ctx->can_fuse(idx, snake_ops, 5)) {- return false;- }-- const ggml_tensor * mul0 = ctx->node(idx + 0);- const ggml_tensor * sin_node = ctx->node(idx + 1);- const ggml_tensor * sqr = ctx->node(idx + 2);- const ggml_tensor * mul1 = ctx->node(idx + 3);- const ggml_tensor * add = ctx->node(idx + 4);-- // x carries the full activation shape, a is the broadcast operand- const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1];- const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0];+int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {+ int n_fuse = 1;+ const ggml_metal_fusion * fusion = nullptr;- // mul1 reads sqr and inv_b in either operand order- const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0];-- // closure check: the trailing add reads the same x as the leading mul- const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0];-- // x is in the supported whitelist and every chain intermediate shares x's type.- // a and inv_b bind as device const float * in the kernel, so they stay F32.- const bool types_ok =- (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) &&- (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) &&- (mul0->type == x->type) && (sin_node->type == x->type) &&- (sqr->type == x->type) && (mul1->type == x->type) &&- (add->type == x->type);- // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D- const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1];- const bool dim_ok =- (x->ne[2] == 1) && (x->ne[3] == 1) &&- (add->ne[2] == 1) && (add->ne[3] == 1) &&- (a->ne[2] == 1) && (a->ne[3] == 1) &&- (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1);- // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous- const bool contig_ok =- ggml_is_contiguous(x) && ggml_is_contiguous(add) &&- ggml_is_contiguous(a) && ggml_is_contiguous(inv_b);-- return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x;-}+ if (ctx->use_fusion()) {+ int n = 1;+ fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);+ n_fuse = n;-int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {- if (ctx->use_fusion && ggml_metal_op_can_fuse_snake(ctx, idx)) {- return ggml_metal_op_snake_fused(ctx, idx);+ // snake activation autofuse: mul -> sin -> sqr -> mul -> add+ if (fusion && fusion->id == GGML_METAL_FUSION_SNAKE) {+ ctx->count_fusions(fusion);+ return ggml_metal_op_snake_fused(ctx, idx);+ }}ggml_tensor * op = ctx->node(idx);@@ -3775,9 +3775,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {ggml_metal_library_t lib = ctx->lib;ggml_metal_encoder_t enc = ctx->enc;- const bool use_fusion = ctx->use_fusion;+ const bool use_fusion = ctx->use_fusion();- const int debug_fusion = ctx->debug_fusion;+ const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo);GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);@@ -3822,57 +3822,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {/*.o1 =*/ { bid_src1.offs },};- ggml_op fops[8];-- int n_fuse = 1;-// c[0] = add(a, b[0])// c[1] = add(c[0], b[1])// c[2] = add(c[1], b[2])// ...- if (use_fusion) {- fops[0] = GGML_OP_ADD;- fops[1] = GGML_OP_ADD;- fops[2] = GGML_OP_ADD;- fops[3] = GGML_OP_ADD;- fops[4] = GGML_OP_ADD;- fops[5] = GGML_OP_ADD;- fops[6] = GGML_OP_ADD;- fops[7] = GGML_OP_ADD;-- // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops- // across splits. idx_end indicates the last node in the current split- for (n_fuse = 0; n_fuse <= 6; ++n_fuse) {- if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) {- break;- }-- ggml_tensor * f0 = ctx->node(idx + n_fuse);- ggml_tensor * f1 = ctx->node(idx + n_fuse + 1);-- if (f0 != f1->src[0]) {- break;- }-- // b[0] === b[1] === ...- if (!ggml_are_same_layout(f0->src[1], f1->src[1])) {- break;- }-- // only fuse ops if src1 is in the same Metal buffer- ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]);- if (bid_fuse.metal != bid_src1.metal) {- break;- }-- //ctx->fuse_cnt[ops[n_fuse + 1]->op]++;-- args.o1[n_fuse + 1] = bid_fuse.offs;+ if (use_fusion && fusion && fusion->id == GGML_METAL_FUSION_ADD_CHAIN) {+ // the offsets of the fused addends are relative to the start of the src1 buffer+ for (int i = 1; i < n_fuse; i++) {+ args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs;}- ++n_fuse;+ ctx->count_fusions(fusion);- if (debug_fusion > 1 && n_fuse > 1) {+ if (debug_fusion > 1) {GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse);}}@@ -4080,9 +4042,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {ggml_metal_library_t lib = ctx->lib;ggml_metal_encoder_t enc = ctx->enc;- const bool use_fusion = ctx->use_fusion;+ const bool use_fusion = ctx->use_fusion();- const int debug_fusion = ctx->debug_fusion;+ const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo);GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);@@ -4110,8 +4072,6 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {/*.nbf3 =*/ { nb03 },};- ggml_op fops[8];-int n_fuse = 1;ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 };@@ -4120,55 +4080,35 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {// d[1] = mul(d[0], b)// d[2] = add(d[1], c)if (use_fusion) {- fops[0] = op->op;- fops[1] = GGML_OP_MUL;- fops[2] = GGML_OP_ADD;+ int n = 1;+ const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);- for (n_fuse = 0; n_fuse <= 1; ++n_fuse) {- if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) {- break;- }+ if (fusion && (fusion->id == GGML_METAL_FUSION_NORM_MUL || fusion->id == GGML_METAL_FUSION_NORM_MUL_ADD)) {+ n_fuse = n;- ggml_tensor * f0 = ctx->node(idx + n_fuse);- ggml_tensor * f1 = ctx->node(idx + n_fuse + 1);+ ctx->count_fusions(fusion);- if (f0 != f1->src[0]) {- break;- }+ for (int i = 1; i < n_fuse; i++) {+ const ggml_tensor * fn = ctx->node(idx + i);- if (f1->src[1]->ne[0] != op->ne[0]) {- break;- }+ bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]);- if (!ggml_is_contiguous_rows(f1->src[1])) {- break;- }+ args.nef1[i] = fn->src[1]->ne[1];+ args.nef2[i] = fn->src[1]->ne[2];+ args.nef3[i] = fn->src[1]->ne[3];- if (f1->type != GGML_TYPE_F32) {- break;+ args.nbf1[i] = fn->src[1]->nb[1];+ args.nbf2[i] = fn->src[1]->nb[2];+ args.nbf3[i] = fn->src[1]->nb[3];}- //ctx->fuse_cnt[f1->op]++;-- bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]);-- args.nef1[n_fuse + 1] = f1->src[1]->ne[1];- args.nef2[n_fuse + 1] = f1->src[1]->ne[2];- args.nef3[n_fuse + 1] = f1->src[1]->ne[3];-- args.nbf1[n_fuse + 1] = f1->src[1]->nb[1];- args.nbf2[n_fuse + 1] = f1->src[1]->nb[2];- args.nbf3[n_fuse + 1] = f1->src[1]->nb[3];- }-- ++n_fuse;-- if (debug_fusion > 1 && n_fuse > 1) {- if (n_fuse == 2) {- GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op));- }- if (n_fuse == 3) {- GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op));+ if (debug_fusion > 1) {+ if (n_fuse == 2) {+ GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op));+ }+ if (n_fuse == 3) {+ GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op));+ }}}}diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.hindex f8fe50b468..4dd8ce7af6 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.h+++ b/ggml/src/ggml-metal/ggml-metal-ops.h@@ -8,17 +8,18 @@ extern "C" {typedef struct ggml_metal_op * ggml_metal_op_t;+struct ggml_metal_fusion; // forward decl (ggml-metal-device.h)+ggml_metal_op_t ggml_metal_op_init(ggml_metal_device_t dev,ggml_metal_cmd_buf_t cmd_buf,struct ggml_cgraph * gf,+ struct ggml_metal_fusion_info * finfo,int idx_start,int idx_end,- bool use_fusion,bool use_concurrency,bool use_capture,- int debug_graph,- int debug_fusion);+ int debug_graph);void ggml_metal_op_free(ggml_metal_op_t ctx);diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex 3bd6abd06f..4cbec8645a 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -4,6 +4,7 @@#include "ggml-backend-impl.h"#include "ggml-metal-device.h"+#include "ggml-metal-fusion.h"#include "ggml-metal-context.h"#include "ggml-metal-ops.h"#include "ggml-metal-tuning.h"@@ -906,6 +907,30 @@ static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t dereturn ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id);}+// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device+// fusion context once and passes that opaque handle to the rest of the functions+typedef void * ggml_backend_fusion_t;++static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) {+ return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context);+}++static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) {+ ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo);+}++static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) {+ ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo);+}++static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) {+ return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n);+}++static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) {+ ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled);+}+static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) {if (strcmp(name, "ggml_backend_get_features") == 0) {return (void *)ggml_backend_metal_get_features;@@ -928,6 +953,23 @@ static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, constif (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) {return (void *)ggml_backend_metal_tuning_device_token;}+ // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official+ // ggml backend interface yet; a backend that adopts it exports these exact names)+ if (strcmp(name, "ggml_backend_fusion_get") == 0) {+ return (void *)ggml_backend_metal_fusion_get;+ }+ if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) {+ return (void *)ggml_backend_metal_fusion_stats_init;+ }+ if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) {+ return (void *)ggml_backend_metal_fusion_stats_reset;+ }+ if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) {+ return (void *)ggml_backend_metal_fusion_stats_get;+ }+ if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) {+ return (void *)ggml_backend_metal_fusion_set_enabled;+ }return NULL;diff --git a/ggml/src/ggml-metal/kernels/gated_delta_net.metal b/ggml/src/ggml-metal/kernels/gated_delta_net.metalindex 8422d8e29f..5e4861ece3 100644--- a/ggml/src/ggml-metal/kernels/gated_delta_net.metal+++ b/ggml/src/ggml-metal/kernels/gated_delta_net.metal@@ -15,6 +15,7 @@ kernel void kernel_gated_delta_net_impl(device const char * b,device const char * s,device char * dst,+ device char * dst_fuse,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],uint3 ntg[[threads_per_threadgroup]]) {@@ -65,6 +66,12 @@ kernel void kernel_gated_delta_net_impl(// per-(seq,head) offset within a slotconst uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v;+ // when fused with the cache cpy, write the snapshots straight into the cache buffer using+ // the slot stride; otherwise append them after the attn scores (nb_out == 0)+ const bool fused = args.nb_out > 0;+ const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size;+ const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap;+for (short t = 0; t < args.ne22; t++) {float s_k = 0.0f;@@ -116,7 +123,7 @@ kernel void kernel_gated_delta_net_impl(if (K > 1) {const int target_slot = (int)args.ne22 - 1 - (int)t;if (target_slot >= 0 && target_slot < (int)K) {- device float * dst_state = (device float *) (dst) + attn_size + (uint)target_slot * state_size_per_snap + state_out_base;+ device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base;FOR_UNROLL (short j = 0; j < NSG; j++) {const short is = tx*NSG + j;dst_state[is] = ls[j];@@ -126,7 +133,7 @@ kernel void kernel_gated_delta_net_impl(}if (K == 1) {- device float * dst_state = (device float *) (dst) + attn_size + state_out_base;+ device float * dst_state = (device float *)state_out + state_out_base;FOR_UNROLL (short j = 0; j < NSG; j++) {const short is = tx*NSG + j;dst_state[is] = ls[j];@@ -158,6 +165,7 @@ kernel void kernel_gated_delta_net_impl(device const char * b,device const char * s,device char * dst,+ device char * dst_fuse,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],uint3 ntg[[threads_per_threadgroup]]) {@@ -230,7 +238,13 @@ kernel void kernel_gated_delta_net_impl(dst_attn += args.ne21*S_v;}- device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20;+ // when fused with the cache cpy, write the snapshots straight into the cache buffer using+ // the slot stride; otherwise append them after the attn scores (nb_out == 0)+ const bool fused = args.nb_out > 0;+ const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v;+ const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v;++ device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20;device T * dstt_state = (device T *) (dst_state);FOR_UNROLL (short j = 0; j < NSG; j++) {diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 21501574a9..6334f3ccab 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -666,7 +666,9 @@ void llama_context::sched_reserve() {// need to implement a more robust mechanism that tries a few different inputs and analyzes the resultsggml_cgraph * gf = nullptr;switch (model.arch) {+ case LLM_ARCH_KIMI_LINEAR:case LLM_ARCH_MINIMAX_01:+ // [TAG_RESERVE_DIAG_DECAY]// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cppindex 361114acc3..9fa2e8fc01 100644--- a/src/models/minimax-01.cpp+++ b/src/models/minimax-01.cpp@@ -229,6 +229,7 @@ llama_model_minimax_01::graph::graph(const llama_model & model, const llm_graph_ggml_set_input(inp->inp_k_decay);cb(inp->inp_k_decay, "k_decay_exp", -1);+ // [TAG_RESERVE_DIAG_DECAY]inp->inp_diag_decay = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_seq_tokens, n_seq_tokens, n_head, n_seqs);ggml_set_input(inp->inp_diag_decay);cb(inp->inp_diag_decay, "diag_decay_exp", -1);diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cppindex d946b3cff6..ba1cea1465 100644--- a/src/models/plamo2.cpp+++ b/src/models/plamo2.cpp@@ -142,6 +142,11 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_paracur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il);}+ if (il == n_layer - 1 && inp_out_ids) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ residual = ggml_get_rows(ctx0, residual, inp_out_ids);+ }+// post_mixer_normcur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);cb(cur, "attn_post_norm", il);@@ -167,11 +172,6 @@ llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_paracur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);cb(cur, "ffn_post_norm", il);- if (il == n_layer - 1 && inp_out_ids) {- cur = ggml_get_rows(ctx0, cur, inp_out_ids);- residual = ggml_get_rows(ctx0, residual, inp_out_ids);- }-// residual connectioncur = ggml_add(ctx0, cur, residual);cb(cur, "ffn_residual", il);diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cppindex 5596620f07..30c08ed35e 100644--- a/src/models/qwen3vl.cpp+++ b/src/models/qwen3vl.cpp@@ -18,6 +18,7 @@ void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) {int64_t n_vocab_out = n_vocab;if (arch == LLM_ARCH_QWEN3TTS) {+ // [TAG_LLAMA_N_VOCAB_OUT]n_vocab_out = 3072;}diff --git a/tests/.gitignore b/tests/.gitignoreindex 52b292b1f8..04095c9ddb 100644--- a/tests/.gitignore+++ b/tests/.gitignore@@ -1,6 +1,7 @@*!*.*!snapshots/+!fusion/*.oggml-common.h**/*.swpdiff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex 5531c4ce3c..920c58c738 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -196,7 +196,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)# llama_build_and_test(test-double-float.cpp) # SLOW- llama_build_and_test(test-llama-archs.cpp)+ llama_build(test-llama-archs.cpp)set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/")file(MAKE_DIRECTORY "${MODEL_DIR}")@@ -255,6 +255,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)ARGS --models "${MODEL_DIR}")set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED generate-models)++ llama_build(test-fusion.cpp)endif()llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp)diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csvnew file mode 100644index 0000000000..067316abfe--- /dev/null+++ b/tests/fusion/MTL.csv@@ -0,0 +1,154 @@+# test-fusion baseline for device MTL+# arch ,moe ,mode ,label , count+arcee ,0 ,any ,RMS_NORM+MUL , 5+arctic ,0 ,any ,RMS_NORM+MUL , 7+baichuan ,0 ,any ,RMS_NORM+MUL , 5+bailingmoe ,1 ,any ,ADD+ADD , 2+bailingmoe ,1 ,any ,RMS_NORM+MUL , 5+bailingmoe2 ,1 ,any ,ADD+ADD , 1+bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9+bailingmoe3 ,1 ,any ,ADD+ADD , 1+bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1+bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8+bloom ,0 ,any ,NORM+MUL+ADD , 6+chatglm ,0 ,any ,RMS_NORM+MUL , 5+codeshell ,0 ,any ,NORM+MUL+ADD , 5+cogvlm ,0 ,any ,RMS_NORM+MUL , 5+command-r ,0 ,any ,NORM+MUL , 3+dbrx ,0 ,any ,NORM+MUL , 5+deci ,0 ,any ,RMS_NORM+MUL , 5+deepseek ,0 ,any ,ADD+ADD , 1+deepseek ,0 ,any ,RMS_NORM+MUL , 5+deepseek2 ,0 ,any ,ADD+ADD , 1+deepseek2 ,0 ,any ,RMS_NORM+MUL , 9+deepseek32 ,0 ,any ,ADD+ADD , 1+deepseek32 ,0 ,any ,NORM+MUL+ADD , 2+deepseek32 ,0 ,any ,RMS_NORM+MUL , 9+deepseek4 ,0 ,any ,RMS_NORM+MUL , 20+dots1 ,0 ,any ,ADD+ADD , 1+dots1 ,0 ,any ,RMS_NORM+MUL , 9+dream ,0 ,any ,RMS_NORM+MUL , 5+ernie4_5-moe ,1 ,any ,ADD+ADD , 1+ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5+ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5+exaone ,0 ,any ,RMS_NORM+MUL , 5+exaone4 ,0 ,any ,RMS_NORM+MUL , 5+exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4+falcon ,0 ,any ,ADD+ADD , 2+falcon ,0 ,any ,NORM+MUL+ADD , 5+falcon-h1 ,0 ,any ,ADD+ADD , 2+falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9+gemma ,0 ,any ,RMS_NORM+MUL , 5+gemma2 ,0 ,any ,RMS_NORM+MUL , 5+gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4+glm-dsa ,0 ,any ,ADD+ADD , 1+glm-dsa ,0 ,any ,NORM+MUL+ADD , 2+glm-dsa ,0 ,any ,RMS_NORM+MUL , 9+glm4 ,0 ,any ,RMS_NORM+MUL , 5+glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4+glm4moe ,1 ,any ,ADD+ADD , 1+glm4moe ,1 ,any ,RMS_NORM+MUL , 9+gpt-oss ,0 ,any ,RMS_NORM+MUL , 5+gpt2 ,0 ,any ,NORM+MUL+ADD , 5+gptneox ,0 ,any ,NORM+MUL+ADD , 5+granite ,0 ,any ,RMS_NORM+MUL , 5+granite ,0 ,any ,RMS_NORM+MUL , 5+granitehybrid ,0 ,any ,RMS_NORM+MUL , 6+granitemoe ,1 ,any ,RMS_NORM+MUL , 5+granitemoe ,1 ,any ,RMS_NORM+MUL , 5+grok ,0 ,any ,RMS_NORM+MUL , 5+grok ,0 ,any ,RMS_NORM+MUL+ADD , 4+grovemoe ,1 ,any ,ADD+ADD , 2+grovemoe ,1 ,any ,RMS_NORM+MUL , 9+hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9+hunyuan-moe ,1 ,any ,ADD+ADD , 2+hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9+hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9+hy_v3 ,0 ,any ,ADD+ADD , 2+hy_v3 ,0 ,any ,RMS_NORM+MUL , 9+hy_v4 ,0 ,any ,NORM+MUL+ADD , 1+hy_v4 ,0 ,any ,RMS_NORM+MUL , 9+internlm2 ,0 ,any ,RMS_NORM+MUL , 5+jais ,0 ,any ,NORM+MUL+ADD , 5+jais2 ,0 ,any ,NORM+MUL+ADD , 5+jamba ,0 ,any ,RMS_NORM+MUL , 8+kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1+kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17+kimi-linear ,0 ,any ,ADD+ADD , 1+kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1+kimi-linear ,0 ,any ,RMS_NORM+MUL , 7+lfm2 ,0 ,any ,RMS_NORM+MUL , 7+lfm2moe ,1 ,any ,RMS_NORM+MUL , 7+llada ,0 ,any ,RMS_NORM+MUL , 5+llada-moe ,1 ,any ,RMS_NORM+MUL , 9+llama ,0 ,any ,RMS_NORM+MUL , 5+llama ,0 ,any ,RMS_NORM+MUL , 5+llama4 ,0 ,any ,ADD+ADD , 2+llama4 ,0 ,any ,RMS_NORM+MUL , 9+maincoder ,0 ,any ,RMS_NORM+MUL , 9+mamba ,0 ,any ,RMS_NORM+MUL , 3+mamba2 ,0 ,any ,RMS_NORM+MUL , 5+minicpm ,0 ,any ,RMS_NORM+MUL , 5+minicpm ,0 ,any ,RMS_NORM+MUL , 5+minicpm3 ,0 ,any ,RMS_NORM+MUL , 9+minimax-01 ,0 ,any ,RMS_NORM+MUL , 6+minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9+minimax-m3 ,0 ,any ,ADD+ADD , 1+minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11+mistral3 ,0 ,any ,RMS_NORM+MUL , 5+mistral3 ,0 ,any ,RMS_NORM+MUL , 5+mistral4 ,0 ,any ,ADD+ADD , 1+mistral4 ,0 ,any ,RMS_NORM+MUL , 9+mpt ,0 ,any ,NORM+MUL+ADD , 5+nanbeige ,0 ,any ,RMS_NORM+MUL , 5+nemotron ,0 ,any ,NORM+MUL+ADD , 5+nemotron_h ,0 ,any ,RMS_NORM+MUL , 5+nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5+olmoe ,1 ,any ,RMS_NORM+MUL , 9+openelm ,0 ,any ,RMS_NORM+MUL , 9+orion ,0 ,any ,NORM+MUL+ADD , 5+paddleocr ,0 ,any ,RMS_NORM+MUL , 5+pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5+phi2 ,0 ,any ,ADD+ADD , 2+phi2 ,0 ,any ,NORM+MUL+ADD , 3+phi3 ,0 ,any ,RMS_NORM+MUL , 5+phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5+plamo ,0 ,any ,ADD+ADD , 2+plamo ,0 ,any ,RMS_NORM+MUL , 3+plamo2 ,0 ,any ,RMS_NORM+MUL , 10+plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4+pockettts ,0 ,any ,NORM+MUL+ADD , 5+qwen ,0 ,any ,RMS_NORM+MUL , 5+qwen2 ,0 ,any ,RMS_NORM+MUL , 5+qwen2moe ,1 ,any ,ADD+ADD , 2+qwen2moe ,1 ,any ,RMS_NORM+MUL , 5+qwen2vl ,0 ,any ,RMS_NORM+MUL , 5+qwen3 ,0 ,any ,RMS_NORM+MUL , 9+qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1+qwen35 ,0 ,any ,RMS_NORM+MUL , 8+qwen35moe ,1 ,any ,ADD+ADD , 2+qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1+qwen35moe ,1 ,any ,RMS_NORM+MUL , 8+qwen3moe ,1 ,any ,RMS_NORM+MUL , 9+qwen3next ,0 ,any ,ADD+ADD , 2+qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1+qwen3next ,0 ,any ,RMS_NORM+MUL , 8+qwen3tts ,0 ,any ,RMS_NORM+MUL , 9+qwen3vl ,0 ,any ,RMS_NORM+MUL , 9+qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9+qwen4exp ,0 ,any ,ADD+ADD+ADD , 5+qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9+qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1+qwen4exp ,0 ,any ,RMS_NORM+MUL , 5+refact ,0 ,any ,RMS_NORM+MUL , 5+refact ,0 ,any ,RMS_NORM+MUL , 5+rnd1 ,0 ,any ,RMS_NORM+MUL , 9+seed_oss ,0 ,any ,RMS_NORM+MUL , 5+smallthinker ,0 ,any ,RMS_NORM+MUL , 5+smollm3 ,0 ,any ,RMS_NORM+MUL , 5+stablelm ,0 ,any ,NORM+MUL , 4+stablelm ,0 ,any ,NORM+MUL+ADD , 5+starcoder ,0 ,any ,NORM+MUL+ADD , 5+starcoder2 ,0 ,any ,NORM+MUL+ADD , 5+talkie ,0 ,any ,ADD+ADD , 2+xverse ,0 ,any ,RMS_NORM+MUL , 5diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex ef4fc30cec..15c42a1081 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -4638,6 +4638,122 @@ struct test_gated_delta_net : public test_case {}};+// GGML_OP_GATED_DELTA_NET + GGML_OP_CPY (recurrent cache fusion)+struct test_gated_delta_net_cache_fusion : public test_case {+ const ggml_type type;++ const int64_t head_count;+ const int64_t head_size;+ const int64_t n_seq_tokens;+ const int64_t n_seqs;+ const int64_t K; // snapshot slot count (>1)++ ggml_tensor * cpy_node = nullptr;++ std::string vars() override {+ return VARS_TO_STR6(type, head_count, head_size, n_seq_tokens, n_seqs, K);+ }++ test_gated_delta_net_cache_fusion(ggml_type type = GGML_TYPE_F32,+ int64_t head_count = 4, int64_t head_size = 32, int64_t n_seq_tokens = 2, int64_t n_seqs = 1,+ int64_t K = 2)+ : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), K(K) {}++ ggml_tensor * build_graph(ggml_context * ctx) override {+ const int64_t S_v = head_size;+ const int64_t H_v = head_count;+ const int64_t H_k = head_count;+ const int64_t D = S_v * S_v * H_v;+ const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);++ ggml_tensor * q = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs);+ ggml_tensor * k = ggml_new_tensor_4d(ctx, type, head_size, H_k, n_seq_tokens, n_seqs);+ ggml_tensor * v = ggml_new_tensor_4d(ctx, type, head_size, H_v, n_seq_tokens, n_seqs);+ ggml_set_name(q, "q");+ ggml_set_name(k, "k");+ ggml_set_name(v, "v");+ ggml_tensor * g = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs);+ ggml_tensor * beta = ggml_new_tensor_4d(ctx, type, 1, H_v, n_seq_tokens, n_seqs);+ ggml_tensor * state = ggml_new_tensor_4d(ctx, type, head_size, head_size, H_v, n_seqs);+ ggml_set_name(g, "g");+ ggml_set_name(beta, "beta");+ ggml_set_name(state, "state");++ q = ggml_l2_norm(ctx, q, 1e-6f);+ k = ggml_l2_norm(ctx, k, 1e-6f);++ ggml_tensor * gdn_out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K);+ ggml_set_name(gdn_out, "gdn_out");++ // attn scores view (first part of the gdn output)+ ggml_tensor * attn = ggml_view_4d(ctx, gdn_out,+ S_v, H_v, n_seq_tokens, n_seqs,+ ggml_row_size(gdn_out->type, S_v),+ ggml_row_size(gdn_out->type, S_v * H_v),+ ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens), 0);+ ggml_set_name(attn, "attn");++ // snapshot tail view [D, n_seqs, n_written]+ const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs;+ ggml_tensor * src = ggml_view_3d(ctx, gdn_out,+ D, n_seqs, n_written,+ ggml_row_size(gdn_out->type, D),+ ggml_row_size(gdn_out->type, D * n_seqs),+ ggml_row_size(gdn_out->type, attn_score_elems));++ // recurrent cache view [D, n_seqs, n_written]+ ggml_tensor * cache = ggml_new_tensor_3d(ctx, type, D, n_seqs, n_written);+ ggml_set_name(cache, "cache");+ ggml_tensor * dst = ggml_view_3d(ctx, cache,+ D, n_seqs, n_written,+ ggml_row_size(cache->type, D),+ ggml_row_size(cache->type, D * n_seqs), 0);++ ggml_tensor * cpy = ggml_cpy(ctx, src, dst);+ ggml_set_name(cpy, "gdn_cache_cpy");+ cpy_node = cpy;++ // read the cpy output (not the plain dst view, which would not pull the cpy into the graph)+ // so that neither the gdn nor the cpy is the graph output+ ggml_tensor * out = ggml_sum(ctx, cpy);+ return out;+ }++ std::string op_desc(ggml_tensor * t) override {+ GGML_UNUSED(t);+ return "GATED_DELTA_NET_CACHE_FUSION";+ }++ bool run_whole_graph() override { return true; }+ std::vector<ggml_tensor *> fusion_test_nodes() override { return { cpy_node }; }++ uint64_t op_flops(ggml_tensor * t) override {+ GGML_UNUSED(t);+ const uint64_t S_v = head_size;+ const uint64_t H_v = head_count;+ const uint64_t T = n_seq_tokens;+ const uint64_t B = n_seqs;+ return (4ull*S_v + 2ull*S_v*S_v) * H_v * T * B;+ }++ void initialize_tensors(ggml_context * ctx) override {+ for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {+ if (ggml_is_view_op(t->op)) { continue; }+ if (strcmp(t->name, "g") == 0) {+ init_tensor_uniform(t, -20.0f, -1e-4f);+ } else if (strcmp(t->name, "beta") == 0) {+ init_tensor_uniform(t, 0.0f, 1.0f);+ } else if (strcmp(t->name, "v") == 0) {+ init_tensor_uniform(t, -0.3f, 5.0f);+ } else if (strcmp(t->name, "cache") == 0) {+ init_tensor_uniform(t, 0.0f, 0.0f);+ } else {+ init_tensor_uniform(t);+ }+ }+ }+};+// GGML_OP_GATED_LINEAR_ATTNstruct test_gla : public test_case {const ggml_type type;@@ -10741,6 +10857,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 32, 8, 1, 1, false, false, /*K=*/3));test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 16, 2, 1, false, false, /*K=*/4));+ // gdn + cache cpy fusion (K > 1)+ test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 2, 1, 2));+ test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 64, 4, 1, 2));+ test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 4, 1, 4));+ test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 8, 32, 4, 2, 4));+ test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 8, 1, 4));+#if 0// these tests are disabled to save execution time, sbut they can be handy for debuggingtest_cases.emplace_back(new test_llama(2, true));diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cppnew file mode 100644index 0000000000..467248f0f1--- /dev/null+++ b/tests/test-fusion.cpp@@ -0,0 +1,565 @@+// test-fusion: verify the backend fusion logic against a per-device baseline.+//+// for every dummy model generated by test-llama-archs, the tool runs the model on a single+// device with fusion enabled and disabled, and reports:+// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the+// per-graph counts match)+// - the NMSE between the fused and unfused logits+// - the NMSE between the device and a CPU reference+//+// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a+// fusion pattern that silently stops matching (or fires when it should not) is caught as a+// regression.+//+// usage:+// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline+// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it+// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model++#include "common.h"+#include "log.h"+#include "llama-cpp.h"++#include "ggml.h"+#include "gguf.h"++#include <algorithm>+#include <array>+#include <cstring>+#include <filesystem>+#include <fstream>+#include <iomanip>+#include <iostream>+#include <map>+#include <random>+#include <string>+#include <vector>++// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism+// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging+// exports these exact names.+typedef void * ggml_backend_fusion_t;++typedef ggml_backend_fusion_t ( * fusion_get_t) (ggml_backend_dev_t);+typedef void ( * fusion_stats_init_t) (ggml_backend_fusion_t);+typedef void ( * fusion_stats_reset_t) (ggml_backend_fusion_t);+typedef int ( * fusion_stats_get_t) (ggml_backend_fusion_t, const char **, uint64_t *, int);+typedef void ( * fusion_set_enabled_t) (ggml_backend_fusion_t, bool);++static bool silent_model_load_progress(float, void *) {+ return true;+}++struct gguf_context_ptr {+ gguf_context * ctx;+ gguf_context_ptr(gguf_context * c) : ctx(c) {}+ ~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } }+ gguf_context * get() const { return ctx; }+ gguf_context_ptr(const gguf_context_ptr &) = delete;+ gguf_context_ptr & operator=(const gguf_context_ptr &) = delete;+};++// NMSE between two vectors (same as tests/test-llama-archs.cpp)+static double nmse(const std::vector<float> & a, const std::vector<float> & b) {+ GGML_ASSERT(a.size() == b.size());+ double mse_a_b = 0.0;+ double mse_a_0 = 0.0;++ for (size_t i = 0; i < a.size(); i++) {+ const float a_i = a[i];+ const float b_i = b[i];++ mse_a_b += (a_i - b_i) * (a_i - b_i);+ mse_a_0 += a_i * a_i;+ }++ return mse_a_b / mse_a_0;+}++// deterministic token sequence+static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {+ std::mt19937 gen(seed);+ std::uniform_int_distribution<> dis(0, n_vocab - 1);+ std::vector<llama_token> ret;+ ret.reserve(n_tokens);+ for (uint32_t i = 0; i < n_tokens; i++) {+ ret.push_back(dis(gen));+ }+ return ret;+}++// trim leading/trailing whitespace (used when parsing padded CSV columns)+static std::string trim(const std::string & s) {+ const size_t b = s.find_first_not_of(" \t\r\n");+ if (b == std::string::npos) {+ return "";+ }+ const size_t e = s.find_last_not_of(" \t\r\n");+ return s.substr(b, e - b + 1);+}++static std::string get_arch(const std::string & path) {+ gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr };+ gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params));+ if (!ctx.get()) {+ throw std::runtime_error("failed to read gguf: " + path);+ }+ const int idx = gguf_find_key(ctx.get(), "general.architecture");+ if (idx < 0) {+ return "unknown";+ }+ const char * val = gguf_get_val_str(ctx.get(), idx);+ return val ? val : "unknown";+}++static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) {+ llama_model_params model_params = llama_model_default_params();+ model_params.progress_callback = silent_model_load_progress;+ std::vector<ggml_backend_dev_t> devs = { dev, nullptr };+ model_params.devices = devs.data();+ model_params.split_mode = LLAMA_SPLIT_MODE_LAYER;++ llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params));+ if (!model) {+ throw std::runtime_error("failed to load model: " + path);+ }+ return model;+}++// a fresh context (fresh state) from an already-loaded model+static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) {+ llama_context_params ctx_params = llama_context_default_params();+ ctx_params.n_ctx = 0;+ ctx_params.n_threads = 4;+ ctx_params.n_threads_batch = 4;+ ctx_params.n_ubatch = n_ubatch;+ ctx_params.n_batch = n_ubatch;++ llama_context_ptr lctx(llama_init_from_model(model, ctx_params));+ if (!lctx) {+ throw std::runtime_error("failed to init context");+ }+ return lctx;+}++// decode all tokens in one batch; returns the logits of every token+static std::vector<float> decode_prefill(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {+ const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));+ llama_batch batch = llama_batch_init(tokens.size(), 0, 1);+ for (size_t i = 0; i < tokens.size(); i++) {+ common_batch_add(batch, tokens[i], i, { 0 }, true);+ }+ batch.n_tokens = tokens.size();+ if (llama_decode(lctx, batch)) {+ llama_batch_free(batch);+ throw std::runtime_error("prefill decode failed");+ }++ std::vector<float> ret;+ ret.reserve(tokens.size() * n_vocab);+ for (size_t i = 0; i < tokens.size(); i++) {+ const float * logits_ith = llama_get_logits_ith(lctx, i);+ for (uint32_t j = 0; j < n_vocab; j++) {+ ret.push_back(logits_ith[j]);+ }+ }+ llama_batch_free(batch);+ return ret;+}++// decode one token at a time; returns the logits of the last token of each step+static std::vector<float> decode_gen(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {+ const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));+ llama_batch batch = llama_batch_init(1, 0, 1);+ std::vector<float> ret;+ for (size_t i = 0; i < tokens.size(); i++) {+ common_batch_clear(batch);+ common_batch_add(batch, tokens[i], i, { 0 }, true);+ if (llama_decode(lctx, batch)) {+ llama_batch_free(batch);+ throw std::runtime_error("decode failed");+ }+ const float * logits = llama_get_logits_ith(lctx, 0);+ for (uint32_t j = 0; j < n_vocab; j++) {+ ret.push_back(logits[j]);+ }+ }+ llama_batch_free(batch);+ return ret;+}++static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t finfo,+ std::vector<const char *> & labels, std::vector<uint64_t> & counts) {+ const int n = api_stats_get(finfo, nullptr, nullptr, 0);+ labels.assign(n, nullptr);+ counts.assign(n, 0);+ api_stats_get(finfo, labels.data(), counts.data(), n);+}++// one row of the per-label report+struct fusion_row {+ std::string arch;+ bool moe;+ std::string mode;+ std::string label;+ uint64_t count_fused;+ uint64_t count_unfused;+ uint64_t expected;+ double nmse_fus;+ double nmse_dev;+ bool ok_count; // counts match the baseline+ bool ok_nmse; // nmse within epsilon+};++static void usage(const char * argv0) {+ printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0);+ printf("usage: %s [options]\n\n", argv0);+ printf("options:\n");+ printf(" --models DIR run over all .gguf models in a directory\n");+ printf(" --model FILE run over a single model file (mutually exclusive with --models)\n");+ printf(" --device NAME device to run on (e.g. MTL0, CPU)\n");+ printf(" --record CSV write the golden baseline\n");+ printf(" --check CSV validate the counters against a baseline (default)\n");+ printf(" -h, --help show this message and exit\n");+}++int main(int argc, char ** argv) {+ std::string models_dir;+ std::string model_file;+ std::string device_name;+ std::string record_path;+ std::string check_path;++ for (int i = 1; i < argc; i++) {+ const std::string arg = argv[i];+ const auto next = [&](const char * name) -> std::string {+ if (i + 1 >= argc) {+ LOG_ERR("%s: %s requires an argument\n", __func__, name);+ exit(1);+ }+ return argv[++i];+ };+ if (arg == "-h" || arg == "--help") {+ usage(argv[0]);+ exit(0);+ }+ if (arg == "--models") { models_dir = next("--models"); }+ else if (arg == "--model") { model_file = next("--model"); }+ else if (arg == "--device"){ device_name = next("--device"); }+ else if (arg == "--record"){ record_path = next("--record"); }+ else if (arg == "--check") { check_path = next("--check"); }+ else {+ LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str());+ return 1;+ }+ }++ if (device_name.empty()) {+ LOG_ERR("%s: --device NAME is required\n", __func__);+ return 1;+ }+ if (models_dir.empty() && model_file.empty()) {+ LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__);+ return 1;+ }+ if (!models_dir.empty() && !model_file.empty()) {+ LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__);+ return 1;+ }+ if (!record_path.empty() && !check_path.empty()) {+ LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__);+ return 1;+ }++ std::vector<std::string> models;+ if (!model_file.empty()) {+ if (!std::filesystem::is_regular_file(model_file)) {+ LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str());+ return 1;+ }+ models.push_back(model_file);+ } else {+ if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {+ LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());+ return 1;+ }+ for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {+ if (entry.is_regular_file() && entry.path().extension() == ".gguf") {+ models.push_back(entry.path().string());+ }+ }+ std::sort(models.begin(), models.end());++ if (models.empty()) {+ LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());+ return 1;+ }+ }++ common_init();+ ggml_backend_load_all();++ ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str());+ if (!dev) {+ LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n",+ __func__, device_name.c_str());+ return 0;+ }++ // resolve the generic fusion debugging functions through the ad-hoc get_proc_address+ // mechanism; a backend that does not adopt fusion debugging exports none of them+ auto * reg = ggml_backend_dev_backend_reg(dev);++ // output naming uses the backend base name (e.g. "MTL") rather than the specific device+ // name (e.g. "MTL0") the test was invoked with+ const std::string base_name = ggml_backend_reg_name(reg);++ auto api_get = (fusion_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_get");+ auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init");+ auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset");+ auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get");+ auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled");++ if (!api_get || !api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) {+ LOG_ERR("%s: device '%s' does not export the generic fusion debugging API "+ "(ggml_backend_fusion_*) - cannot run the fusion regression test\n",+ __func__, device_name.c_str());+ return 1;+ }++ ggml_backend_fusion_t finfo = api_get(dev);++ // enable fusions stats+ api_stats_init(finfo);++ const bool has_counts = true;++ // load the baseline (if any): key arch|moe|mode|label -> expected count+ std::map<std::string, uint64_t> baseline;+ if (!check_path.empty()) {+ std::ifstream in(check_path);+ if (!in) {+ LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str());+ return 1;+ }+ std::string line;+ while (std::getline(in, line)) {+ if (line.empty() || line[0] == '#') {+ continue;+ }+ std::vector<std::string> cols;+ size_t pos = 0;+ while ((pos = line.find(',')) != std::string::npos) {+ cols.push_back(trim(line.substr(0, pos)));+ line.erase(0, pos + 1);+ }+ cols.push_back(trim(line));+ if (cols.size() != 5) {+ continue;+ }+ baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]);+ }+ }++ std::vector<fusion_row> rows;++ LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());++ const size_t seed = 1;++ for (const auto & model_path : models) {+ const std::string arch = get_arch(model_path);+ const bool moe = arch.find("moe") != std::string::npos;++ llama_model_ptr model;+ llama_model_ptr model_cpu;+ uint32_t n_vocab = 0;+ try {+ model = load_model(model_path, dev);+ model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU"));+ n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get()));+ } catch (const std::exception & e) {+ LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what());+ continue;+ }++ struct mode_cfg {+ std::string name;+ std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);+ int n_tokens;+ int n_graphs; // graph runs per mode (prefill=1, decode=16)+ };+ const mode_cfg modes[] = {+ { "prefill", decode_prefill, 32, 1 },+ { "decode", decode_gen, 16, 16 },+ };++ // per-label, per-mode data for this model; prefill and decode are merged into a single+ // "any" row when their per-graph counts match+ struct mode_data {+ bool present;+ uint64_t count_fused; // per graph+ uint64_t count_unfused; // per graph+ double nmse_fus;+ double nmse_dev;+ bool ok_nmse;+ };+ std::map<std::string, std::array<mode_data, 2>> mdata;++ for (int mi = 0; mi < 2; mi++) {+ const mode_cfg & mode = modes[mi];+ const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed);++ // CPU reference for this mode (fresh context, fresh state)+ std::vector<float> logits_cpu;+ try {+ llama_context_ptr ctx = create_ctx(model_cpu.get(), 32);+ logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens);+ } catch (const std::exception & e) {+ LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what());+ }++ // fused run on a fresh context (fresh state)+ std::vector<float> logits_fused;+ std::vector<const char *> labels;+ std::vector<uint64_t> counts_fused;+ {+ llama_context_ptr ctx = create_ctx(model.get(), 32);+ if (has_counts) {+ api_set_enabled(finfo, true);+ api_stats_reset(finfo);+ }+ logits_fused = mode.decode(model.get(), ctx.get(), tokens);+ if (has_counts) {+ read_counts(api_stats_get, finfo, labels, counts_fused);+ }+ }++ // unfused run on another fresh context (fresh state)+ std::vector<float> logits_unfused;+ std::vector<uint64_t> counts_unfused;+ {+ llama_context_ptr ctx = create_ctx(model.get(), 32);+ if (has_counts) {+ api_set_enabled(finfo, false);+ api_stats_reset(finfo);+ }+ logits_unfused = mode.decode(model.get(), ctx.get(), tokens);+ if (has_counts) {+ read_counts(api_stats_get, finfo, labels, counts_unfused);+ }+ }++ const double nmse_fus = nmse(logits_fused, logits_unfused);+ const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu);++ if (has_counts) {+ for (int i = 0; i < (int) labels.size(); i++) {+ const uint64_t fused = counts_fused[i] / mode.n_graphs;+ const uint64_t unfused = counts_unfused[i] / mode.n_graphs;+ if (fused == 0 && unfused == 0) {+ continue;+ }+ auto & d = mdata[labels[i]][mi];+ d.present = true;+ d.count_fused = fused;+ d.count_unfused = unfused;+ d.nmse_fus = nmse_fus;+ d.nmse_dev = nmse_dev;+ d.ok_nmse = nmse_fus <= 1e-4;+ }+ } else {+ rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true, nmse_fus <= 1e-4 });+ }+ }++ // build the per-label rows, merging prefill and decode into "any" when the per-graph+ // counts match (they always do for the deterministic fusion table)+ if (has_counts) {+ for (auto & kv : mdata) {+ const std::string & label = kv.first;+ const auto & d = kv.second;+ const bool both = d[0].present && d[1].present;+ const bool match = both && d[0].count_fused == d[1].count_fused;++ if (match) {+ // one "any" row; use the worst NMSE across the two modes+ const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label;+ const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0;+ const bool ok_count = check_path.empty() || d[0].count_fused == expected;+ const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse;+ const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus);+ const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev);+ rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused,+ expected, nmse_fus, nmse_dev, ok_count, ok_nmse });+ } else {+ // counts differ - keep a separate row per mode+ for (int mi = 0; mi < 2; mi++) {+ if (!d[mi].present) {+ continue;+ }+ const mode_data & a = d[mi];+ const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label;+ const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0;+ const bool ok_count = check_path.empty() || a.count_fused == expected;+ rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused,+ expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse });+ }+ }+ }+ }++ LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());+ }++ // print the report+ {+ std::ofstream out(record_path);+ std::ostream & os = record_path.empty() ? std::cout : out;+ if (!record_path.empty()) {+ os << "# test-fusion baseline for device " << base_name << "\n";+ os << "# " << std::left+ << std::setw(18) << "arch" << ','+ << std::setw(4) << "moe" << ','+ << std::setw(8) << "mode" << ','+ << std::setw(28) << "label" << ','+ << std::right << std::setw(7) << "count" << '\n';+ }++ LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n",+ "arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status");+ int n_ok = 0;+ int n_bad = 0;+ for (const auto & r : rows) {+ const bool ok = r.ok_count && r.ok_nmse;+ const char * status = ok ? "ok" : "FAIL";+ if (ok) { n_ok++; } else { n_bad++; }+ LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n",+ r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(),+ (unsigned long long) r.count_fused, (unsigned long long) r.count_unfused,+ (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status);+ if (!record_path.empty()) {+ os << std::left+ << std::setw(20) << r.arch << ','+ << std::setw(4) << (r.moe ? "1" : "0") << ','+ << std::setw(8) << r.mode << ','+ << std::setw(28) << r.label << ','+ << std::right << std::setw(7) << r.count_fused << '\n';+ }+ }+ LOG_INF("summary: %d ok, %d failed\n", n_ok, n_bad);+ if (!record_path.empty()) {+ LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str());+ }++ if (n_bad && !models_dir.empty() && !check_path.empty()) {+ LOG_WRN("%s: if the fusion counts are expected to change, run with --record to update the baseline:\n"+ "\n"+ "./bin/test-llama-archs -o %s\n"+ "%s --device %s --models %s --record %s\n",+ __func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str());+ }++ return n_bad;+ }+}diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex dbed9846f9..3496f72e49 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -128,7 +128,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {} else if (arch == LLM_ARCH_CHAMELEON) {n_vocab = 10240;} else if (arch == LLM_ARCH_QWEN3TTS) {- n_vocab = 4096; // must be >= the hard-coded codec head size (3072)+ //n_vocab = 4096; // must be >= the hard-coded codec head size (3072)+ n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT]}uint32_t n_head_kv = n_head;diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cppindex 6179e6c108..74d1ba6c21 100644--- a/tests/test-save-load-state.cpp+++ b/tests/test-save-load-state.cpp@@ -109,7 +109,7 @@ static bool test_seq_rm_isolated(for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) {llama_batch_ptr batch(n_tokens, 0, 1);for (size_t i = 0; i < n_tokens; ++i) {- common_batch_add(batch.get(), tokens[i], i, { seq_id }, false);+ common_batch_add(batch.get(), tokens[i], i, { seq_id }, i == n_tokens - 1);}if (llama_decode(ctx.get(), batch.get())) {@@ -373,7 +373,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_auto decode_one = [&](llama_token tok, int pos, llama_seq_id seq) {llama_batch_ptr batch(1, 0, 1);- common_batch_add(batch.get(), tok, pos, { seq }, false);+ common_batch_add(batch.get(), tok, pos, { seq }, true);return llama_decode(ctx.get(), batch.get()) == 0;};