LLM inference in C/C++
C++ ggml
ggml : fix ggml_backend_buft_get_alloc_size() guard (#28038)
Georgi Gerganov6d1479c14889
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppindex fec7d7c92b..8f40fb2f7b 100644--- a/ggml/src/ggml-backend.cpp+++ b/ggml/src/ggml-backend.cpp@@ -70,6 +70,7 @@ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const s// if you hit this assert, update ggml_backend_op_alloc_size_may_expand() accordinglyGGML_ASSERT(size <= ggml_nbytes(tensor) ||ggml_op_is_empty(tensor->op) ||+ ggml_is_quantized(tensor->type) || // [TAG_ALLOC_SIZE_EXPAND]ggml_backend_op_alloc_size_may_expand(tensor->op));return size;diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 7c2d3cad20..a6fc655c41 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -915,6 +915,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t: ggml_nbytes(tensor);int64_t ne0 = tensor->ne[0];+ // [TAG_ALLOC_SIZE_EXPAND]if (ggml_is_quantized(tensor->type)) {if (ne0 % MATRIX_ROW_PADDING != 0) {GGML_ASSERT(tensor->nb[0] == ggml_element_size(tensor));