Skip to content

LLM inference in C/C++

C++ ggml

ggml: avoid KleidiAI buffer type init on dispatch (#27891)

Aman Chadha(IVIXMMI)b81c99b479d4
diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
index 92d7fd644f..dbd1987807 100644
--- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
+++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
@@ -1823,7 +1823,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
const bool src0_is_kleidiai =
op->src[0]->buffer &&
(ggml_n_dims(op->src[0]) == 2) &&
- op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type() &&
+ op->src[0]->buffer->buft->context == this &&
slot_total > 0;
if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) &&
@@ -1862,7 +1862,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {
if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) {
- if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type()) {
+ if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) {
return (ggml::cpu::tensor_traits *) op->src[0]->extra;
} else {
// KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any