LLM inference in C/C++
C++ ggml
model-loader : fix quantized reshaped tensor strides (#26672)
Georgi Gerganov3db4ff877dae
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex b31e92e2da..71bc9f7ef0 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -1249,7 +1249,13 @@ struct ggml_tensor * llama_model_loader::create_tensor(for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;GGML_ASSERT(t_meta.ne[dim] >= 1);- t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];+ if (dim == 0) {+ t_meta.nb[dim] = ggml_type_size(type);+ } else if (dim == 1) {+ t_meta.nb[dim] = ggml_row_size(type, t_meta.ne[dim-1]);+ } else {+ t_meta.nb[dim] = t_meta.nb[dim-1]*t_meta.ne[dim-1];+ }GGML_ASSERT(t_meta.nb[dim] >= 1);}ggml_set_name(&t_meta, tn.str().c_str());@@ -1272,10 +1278,18 @@ struct ggml_tensor * llama_model_loader::create_tensor(if (flags & TENSOR_ALLOW_RESHAPE) {for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;- t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];+ if (dim == 0) {+ t_meta.nb[dim] = ggml_type_size(t_meta.type);+ } else if (dim == 1) {+ t_meta.nb[dim] = ggml_row_size(t_meta.type, t_meta.ne[dim-1]);+ } else {+ t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1];+ }}}+ GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur));+ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);if (buft == nullptr) {return nullptr;