Skip to content

LLM inference in C/C++

C++ ggml

model-loader : fix quantized reshaped tensor strides (#26672)

Georgi Gerganov3db4ff877dae
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index b31e92e2da..71bc9f7ef0 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -1249,7 +1249,13 @@ struct ggml_tensor * llama_model_loader::create_tensor(
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
GGML_ASSERT(t_meta.ne[dim] >= 1);
- t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
+ if (dim == 0) {
+ t_meta.nb[dim] = ggml_type_size(type);
+ } else if (dim == 1) {
+ t_meta.nb[dim] = ggml_row_size(type, t_meta.ne[dim-1]);
+ } else {
+ t_meta.nb[dim] = t_meta.nb[dim-1]*t_meta.ne[dim-1];
+ }
GGML_ASSERT(t_meta.nb[dim] >= 1);
}
ggml_set_name(&t_meta, tn.str().c_str());
@@ -1272,10 +1278,18 @@ struct ggml_tensor * llama_model_loader::create_tensor(
if (flags & TENSOR_ALLOW_RESHAPE) {
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
- t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
+ if (dim == 0) {
+ t_meta.nb[dim] = ggml_type_size(t_meta.type);
+ } else if (dim == 1) {
+ t_meta.nb[dim] = ggml_row_size(t_meta.type, t_meta.ne[dim-1]);
+ } else {
+ t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1];
+ }
}
}
+ GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur));
+
ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);
if (buft == nullptr) {
return nullptr;