LLM inference in C/C++
C++ ggml
Move duplicated imatrix code into single common imatrix-loader.cpp (#22445)
Bartowskie7bcf1c3a851
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txtindex 1a56c25857..c42320c46b 100644--- a/common/CMakeLists.txt+++ b/common/CMakeLists.txt@@ -78,6 +78,8 @@ add_library(${TARGET}hf-cache.cpphf-cache.hhttp.h+ imatrix-loader.cpp+ imatrix-loader.hjson-partial.cppjson-partial.hjson-schema-to-grammar.cppdiff --git a/common/imatrix-loader.cpp b/common/imatrix-loader.cppnew file mode 100644index 0000000000..efe9aecee3--- /dev/null+++ b/common/imatrix-loader.cpp@@ -0,0 +1,165 @@+#include "imatrix-loader.h"+#include "common.h"+#include "log.h"+#include "gguf.h"++#include <cmath>+#include <cstring>+#include <fstream>++static bool common_imatrix_load_legacy(const std::string & fname, common_imatrix & imatrix) {+ std::ifstream in(fname, std::ios::binary);+ if (!in) {+ LOG_ERR("%s: failed to open %s\n", __func__, fname.c_str());+ return false;+ }++ int n_entries;+ in.read((char *) &n_entries, sizeof(n_entries));+ if (in.fail() || n_entries < 1) {+ LOG_ERR("%s: no data in file %s\n", __func__, fname.c_str());+ return false;+ }++ for (int i = 0; i < n_entries; ++i) {+ int32_t len = 0;+ in.read((char *) &len, sizeof(len));+ std::vector<char> name_as_vec(len + 1);+ in.read((char *) name_as_vec.data(), len);+ if (in.fail()) {+ LOG_ERR("%s: failed reading name for entry %d from %s\n", __func__, i + 1, fname.c_str());+ return false;+ }+ name_as_vec[len] = 0;+ std::string name{ name_as_vec.data() };++ int32_t ncall = 0;+ in.read((char *) &ncall, sizeof(ncall));+ int32_t nval = 0;+ in.read((char *) &nval, sizeof(nval));+ if (in.fail() || nval < 1) {+ LOG_ERR("%s: failed reading number of values for entry %d\n", __func__, i);+ return false;+ }++ auto & e = imatrix.entries[std::move(name)];+ e.sums.resize(nval);+ in.read((char *) e.sums.data(), nval * sizeof(float));+ if (in.fail()) {+ LOG_ERR("%s: failed reading data for entry %d\n", __func__, i);+ return false;+ }++ e.counts.resize(1);+ e.counts[0] = ncall;+ }++ // the trailing data (chunk count + dataset name) is optional+ if (in.peek() != EOF) {+ int32_t n_calls = 0;+ in.read((char *) &n_calls, sizeof(n_calls));+ imatrix.chunk_count = n_calls;++ if (!in.fail()) {+ int32_t len = 0;+ in.read((char *) &len, sizeof(len));+ if (!in.fail() && len > 0) {+ std::vector<char> dataset(len + 1, 0);+ in.read(dataset.data(), len);+ if (!in.fail()) {+ imatrix.datasets.push_back(dataset.data());+ }+ }+ }+ }++ imatrix.chunk_size = 0;+ imatrix.is_legacy = true;++ return true;+}++bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {+ struct ggml_context * ctx = nullptr;+ struct gguf_init_params meta_gguf_params = {+ /* .no_alloc = */ false,+ /* .ctx = */ &ctx,+ };+ struct gguf_context * ctx_gguf = gguf_init_from_file(fname.c_str(), meta_gguf_params);+ if (!ctx_gguf) {+ return common_imatrix_load_legacy(fname, imatrix);+ }++ const int32_t n_entries = gguf_get_n_tensors(ctx_gguf);+ if (n_entries < 1) {+ LOG_ERR("%s: no data in file %s\n", __func__, fname.c_str());+ gguf_free(ctx_gguf);+ ggml_free(ctx);+ return false;+ }++ const int64_t datasets_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);+ const int64_t chunk_count_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);+ const int64_t chunk_size_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);++ if (datasets_key != -1 && gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) {+ const int64_t n = gguf_get_arr_n(ctx_gguf, datasets_key);+ imatrix.datasets.reserve(imatrix.datasets.size() + n);+ for (int64_t i = 0; i < n; ++i) {+ imatrix.datasets.push_back(gguf_get_arr_str(ctx_gguf, datasets_key, i));+ }+ }++ imatrix.has_metadata = (datasets_key != -1 && chunk_count_key != -1 && chunk_size_key != -1);+ imatrix.chunk_count = (chunk_count_key != -1) ? gguf_get_val_u32(ctx_gguf, chunk_count_key) : 0;+ imatrix.chunk_size = (chunk_size_key != -1) ? gguf_get_val_u32(ctx_gguf, chunk_size_key) : 0;++ const std::string in_sum2_suffix{ ".in_sum2" };+ const std::string counts_suffix{ ".counts" };++ std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;++ for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {+ std::string name = cur->name;++ if (name.empty()) { continue; }++ if (string_remove_suffix(name, in_sum2_suffix)) {+ sums_counts_for[std::move(name)].first = cur;+ } else if (string_remove_suffix(name, counts_suffix)) {+ sums_counts_for[std::move(name)].second = cur;+ }+ }++ for (const auto & sc : sums_counts_for) {+ const std::string & name = sc.first;+ const struct ggml_tensor * in_sum2 = sc.second.first;+ const struct ggml_tensor * counts = sc.second.second;++ if (!in_sum2 || !counts) {+ LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str());+ gguf_free(ctx_gguf);+ ggml_free(ctx);+ return false;+ }++ auto & e = imatrix.entries[name];++ const int64_t nval = ggml_nelements(in_sum2);+ const int64_t ncounts = ggml_nelements(counts);++ e.sums.resize(nval);+ for (int64_t j = 0; j < nval; ++j) {+ e.sums[j] = ((const float *) in_sum2->data)[j];+ }++ e.counts.resize(ncounts);+ for (int64_t j = 0; j < ncounts; ++j) {+ e.counts[j] = std::lround(((const float *) counts->data)[j]);+ }+ }++ gguf_free(ctx_gguf);+ ggml_free(ctx);+ return true;+}diff --git a/common/imatrix-loader.h b/common/imatrix-loader.hnew file mode 100644index 0000000000..ed00d724ac--- /dev/null+++ b/common/imatrix-loader.h@@ -0,0 +1,26 @@+#pragma once++#include <cstdint>+#include <map>+#include <string>+#include <vector>++inline constexpr const char * LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";+inline constexpr const char * LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";+inline constexpr const char * LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";++struct common_imatrix_entry {+ std::vector<float> sums;+ std::vector<int64_t> counts;+};++struct common_imatrix {+ std::map<std::string, common_imatrix_entry> entries;+ std::vector<std::string> datasets;+ int32_t chunk_count = 0;+ int32_t chunk_size = 0;+ bool is_legacy = false;+ bool has_metadata = false;+};++bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix);diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cppindex 3f7f3a11df..3431a4eca8 100644--- a/tools/imatrix/imatrix.cpp+++ b/tools/imatrix/imatrix.cpp@@ -1,5 +1,6 @@#include "arg.h"#include "common.h"+#include "imatrix-loader.h"#include "log.h"#include "llama.h"#include "gguf.h"@@ -34,10 +35,6 @@ static void print_usage(int, char ** argv) {LOG("\n");}-static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";-static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";-static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";-struct Stats {std::vector<float> values;std::vector<int64_t> counts;@@ -65,7 +62,6 @@ public:bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data);void save_imatrix_legacy(int32_t ncall = -1) const;void save_imatrix(int32_t n_chunk = -1) const;- bool load_imatrix_legacy(const char * fname);bool load_imatrix(const char * file_name);const std::unordered_map<std::string, Stats> & get_mstats() const { return m_stats; }private:@@ -624,204 +620,63 @@ void IMatrixCollector::save_imatrix(int32_t n_chunk) const {ggml_free(ctx);}-bool IMatrixCollector::load_imatrix_legacy(const char * fname) {- std::ifstream in(fname, std::ios::binary);- if (!in) {- LOG_ERR("%s: failed to open %s\n", __func__, fname);- return false;- }- int n_entries;- in.read((char *) &n_entries, sizeof(n_entries));- if (in.fail() || n_entries < 1) {- LOG_ERR("%s: no data in file %s\n", __func__, fname);+bool IMatrixCollector::load_imatrix(const char * file_name) {+ common_imatrix loaded;+ if (!common_imatrix_load(file_name, loaded)) {return false;}- // Guess the chunk size because it's not stored in the file- const int32_t chunk_size = m_params.n_ctx / m_params.n_parallel;-- for (int i = 0; i < n_entries; ++i) {- int32_t len = 0;- in.read((char *) &len, sizeof(len));- std::vector<char> name_as_vec(len + 1);- in.read((char *) name_as_vec.data(), len);- if (in.fail()) {- LOG_ERR("%s: failed reading name for entry %d from %s\n", __func__, i + 1, fname);- return false;- }- name_as_vec[len] = 0;- std::string name{ name_as_vec.data() };- auto & e = m_stats[std::move(name)];- int32_t ncall = 0;- in.read((char *) &ncall, sizeof(ncall));- int32_t nval = 0;- in.read((char *) &nval, sizeof(nval));- if (in.fail() || nval < 1) {- LOG_ERR("%s: failed reading number of values for entry %d\n", __func__, i);- m_stats = {};- return false;- }- if (e.values.empty()) {- e.values.resize(nval, 0.0f);- e.counts.resize(1, 0);- }-- std::vector<float> tmp(nval);- in.read((char *) tmp.data(), nval * sizeof(float));- if (in.fail()) {- LOG_ERR("%s: failed reading data for entry %d\n", __func__, i);- m_stats = {};- return false;- }+ const int32_t chunk_size = m_params.n_ctx / m_params.n_parallel;+ const bool is_legacy = loaded.is_legacy;- // Recreate the state as expected by save_imatrix(), and correct for weighted sum.- for (int i = 0; i < nval; i++) {- e.values[i] += tmp[i] * chunk_size;- }- // The legacy format doesn't distinguish the counts for different experts- for (size_t j = 0; j < e.counts.size(); ++j) {- e.counts[j] += ncall * chunk_size;- }- }+ for (auto & [name, entry] : loaded.entries) {+ auto & e = m_stats[name];- {- // TODO: extract into its own method; this is also used by the GGUF-based format- // Calculate the last chunk count- int64_t max_count = 0;- for (const auto & stats : m_stats) {- for (int64_t count : stats.second.counts) {- if (count > max_count) {- max_count = count;- }+ if (is_legacy) {+ // Legacy format: sums contain (raw_sum/raw_count)*ncall, counts contain {ncall}+ // Reconstruct raw form by multiplying by chunk_size+ if (e.values.empty()) {+ e.values.resize(entry.sums.size(), 0.0f);+ e.counts.resize(1, 0);}- }- m_last_chunk = max_count / (chunk_size);- }-- {- // Read the number of calls the matrix was computed with- int32_t n_calls;- in.read((char *) &n_calls, sizeof(n_calls));- // ignore it because it's not important- }-- // Read the dataset path to include it when writing to GGUF- if (!in.fail()){- int32_t len = 0;- in.read((char *) &len, sizeof(len));- if (!in.fail()) {- std::vector<char> dataset;- dataset.resize(len + 1, 0);- in.read(dataset.data(), len);- if (!in.fail()) {- m_datasets.push_back(dataset.data());+ for (size_t j = 0; j < entry.sums.size(); ++j) {+ e.values[j] += entry.sums[j] * chunk_size;+ }+ for (size_t j = 0; j < e.counts.size(); ++j) {+ e.counts[j] += entry.counts[0] * chunk_size;}- }- }-- return true;-}--// Using GGUF as the file format, for greater extensibility-bool IMatrixCollector::load_imatrix(const char * file_name) {- struct ggml_context * ctx = nullptr;- struct gguf_init_params meta_gguf_params = {- /* .no_alloc = */ false, // the data is needed- /* .ctx = */ &ctx,- };- struct gguf_context * ctx_gguf = gguf_init_from_file(file_name, meta_gguf_params);- if (!ctx_gguf) {- return this->load_imatrix_legacy(file_name);- }- const int32_t n_entries = gguf_get_n_tensors(ctx_gguf);- if (n_entries < 1) {- LOG_ERR("%s: no data in file %s\n", __func__, file_name);- gguf_free(ctx_gguf);- ggml_free(ctx);- return false;- }-- const int64_t datasets_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);- if (datasets_key != -1 && gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) {- const int64_t n = gguf_get_arr_n(ctx_gguf, datasets_key);- m_datasets.reserve(m_datasets.size() + n);- for (int64_t i = 0; i < n; ++i) {- m_datasets.push_back(gguf_get_arr_str(ctx_gguf, datasets_key, i));- }- }-- const std::string in_sum2_suffix{ ".in_sum2" };- const std::string counts_suffix{ ".counts" };-- // Could re-use m_stats instead, but this allows- // checking for completeness of *each* loaded imatrix file- // and also makes it easier to re-use a similar implementation in quantize.cpp- // Using an ordered map to get a deterministic iteration order.- std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;-- for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {- std::string name = cur->name;-- if (name.empty()) { continue; }-- if (string_remove_suffix(name, in_sum2_suffix)) {- // in_sum2- sums_counts_for[std::move(name)].first = cur;- } else if (string_remove_suffix(name, counts_suffix)) {- // counts- sums_counts_for[std::move(name)].second = cur;} else {- // ignore other tensors- }- }-- for (const auto & sc : sums_counts_for) {- const std::string & name = sc.first;- const struct ggml_tensor * in_sum2 = sc.second.first;- const struct ggml_tensor * counts = sc.second.second;-- if (!in_sum2 || !counts) {- LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str());- gguf_free(ctx_gguf);- ggml_free(ctx);- return false;- }-- auto & e = m_stats[name];-- int64_t nval = ggml_nelements(in_sum2);- if (e.values.empty()) {- e.values.resize(nval, 0.0f);- } else if ((size_t) nval != e.values.size()) {- LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size());- gguf_free(ctx_gguf);- ggml_free(ctx);- return false;- }+ // GGUF format: raw sums and counts, accumulate directly+ const int64_t nval = entry.sums.size();+ const int64_t ncounts = entry.counts.size();++ if (e.values.empty()) {+ e.values.resize(nval, 0.0f);+ } else if ((size_t) nval != e.values.size()) {+ LOG_ERR("%s: mismatched sums size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) nval, e.values.size());+ return false;+ }- int64_t ncounts = ggml_nelements(counts);- if (e.counts.empty()) {- e.counts.resize(ncounts, 0);- } else if (e.counts.size() == 1 && ncounts > 1) {- // broadcast, when loading an old imatrix- e.counts.resize(ncounts, e.counts[0]);- } else if ((size_t) ncounts != e.counts.size()) {- LOG_ERR("%s: mismatched counts size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) ncounts, e.counts.size());- gguf_free(ctx_gguf);- ggml_free(ctx);- return false;- }+ if (e.counts.empty()) {+ e.counts.resize(ncounts, 0);+ } else if (e.counts.size() == 1 && ncounts > 1) {+ e.counts.resize(ncounts, e.counts[0]);+ } else if ((size_t) ncounts != e.counts.size()) {+ LOG_ERR("%s: mismatched counts size for %s: %zu != %zu\n", __func__, name.c_str(), (size_t) ncounts, e.counts.size());+ return false;+ }- // Recreate the state as expected by save_imatrix()- for (int64_t j = 0; j < nval; j++) {- e.values[j] += ((const float *) in_sum2->data)[j];- }- for (int64_t j = 0; j < ncounts; j++) {- e.counts[j] += std::lround(((const float *) counts->data)[j]);+ for (int64_t j = 0; j < nval; ++j) {+ e.values[j] += entry.sums[j];+ }+ for (int64_t j = 0; j < ncounts; ++j) {+ e.counts[j] += entry.counts[j];+ }}}- // TODO: extract into its own method; this is also used by the legacy format+ m_datasets.insert(m_datasets.end(), loaded.datasets.begin(), loaded.datasets.end());+// Calculate the last chunk countint64_t max_count = 0;for (const auto & stats : m_stats) {@@ -831,10 +686,8 @@ bool IMatrixCollector::load_imatrix(const char * file_name) {}}}- m_last_chunk = max_count / (m_params.n_ctx / m_params.n_parallel);+ m_last_chunk = max_count / chunk_size;- gguf_free(ctx_gguf);- ggml_free(ctx);return true;}@@ -1218,6 +1071,9 @@ int main(int argc, char ** argv) {return 1;}+ // set_params before show_statistics so load_imatrix has valid n_ctx/n_parallel+ g_collector.set_params(params);+if (params.show_statistics) {if (!show_statistics(params)) {return 1;diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cppindex 7292bda6f4..840eefc2f5 100644--- a/tools/quantize/quantize.cpp+++ b/tools/quantize/quantize.cpp@@ -2,6 +2,7 @@#include "build-info.h"#include "common.h"+#include "imatrix-loader.h"#include "gguf.h"@@ -14,7 +15,6 @@#include <vector>#include <string>#include <unordered_map>-#include <map>#include <fstream>#include <filesystem>@@ -78,11 +78,6 @@ static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrixstatic const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";-// TODO: share with imatrix.cpp-static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";-static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";-static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";-static bool striequals(const char * a, const char * b) {while (*a && *b) {if (std::tolower(*a) != std::tolower(*b)) {@@ -181,184 +176,84 @@ static void usage(const char * executable) {exit(1);}-static int load_legacy_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {- std::ifstream in(imatrix_file.c_str(), std::ios::binary);- if (!in) {- printf("%s: failed to open %s\n",__func__, imatrix_file.c_str());- exit(1);- }- int n_entries;- in.read((char *)&n_entries, sizeof(n_entries));- if (in.fail() || n_entries < 1) {- printf("%s: no data in file %s\n", __func__, imatrix_file.c_str());- exit(1);- }- for (int i = 0; i < n_entries; ++i) {- int len; in.read((char *)&len, sizeof(len));- std::vector<char> name_as_vec(len+1);- in.read((char *)name_as_vec.data(), len);- if (in.fail()) {- printf("%s: failed reading name for entry %d from %s\n", __func__, i+1, imatrix_file.c_str());- exit(1);- }- name_as_vec[len] = 0;- std::string name{name_as_vec.data()};- auto & e = imatrix_data[name];- int ncall;- in.read((char *)&ncall, sizeof(ncall));- int nval;- in.read((char *)&nval, sizeof(nval));- if (in.fail() || nval < 1) {- printf("%s: failed reading number of values for entry %d\n", __func__, i);- imatrix_data = {};- exit(1);- }- e.resize(nval);- in.read((char *)e.data(), nval*sizeof(float));- if (in.fail()) {- printf("%s: failed reading data for entry %d\n", __func__, i);- imatrix_data = {};- exit(1);- }- if (ncall > 0) {- for (auto & v : e) {- v /= ncall;- }- }-- if (getenv("LLAMA_TRACE")) {- printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n", __func__, int(e.size()), ncall, name.c_str());- }- }-- // latest legacy imatrix version contains the dataset filename at the end of the file- int m_last_call = 0;- if (in.peek() != EOF) {- in.read((char *)&m_last_call, sizeof(m_last_call));- int dataset_len;- in.read((char *)&dataset_len, sizeof(dataset_len));- std::vector<char> dataset_as_vec(dataset_len);- in.read(dataset_as_vec.data(), dataset_len);- imatrix_datasets.resize(1);- imatrix_datasets[0].assign(dataset_as_vec.begin(), dataset_as_vec.end());- printf("%s: imatrix dataset='%s'\n", __func__, imatrix_datasets[0].c_str());- }- printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_call);- return m_last_call;-}-static int load_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {-- struct ggml_context * ctx = nullptr;- struct gguf_init_params meta_gguf_params = {- /* .no_alloc = */ false, // the data is needed- /* .ctx = */ &ctx,- };- struct gguf_context * ctx_gguf = gguf_init_from_file(imatrix_file.c_str(), meta_gguf_params);- if (!ctx_gguf) {- fprintf(stderr, "%s: imatrix file '%s' is using old format\n", __func__, imatrix_file.c_str());- return load_legacy_imatrix(imatrix_file, imatrix_datasets, imatrix_data);- }- const int32_t n_entries = gguf_get_n_tensors(ctx_gguf);- if (n_entries < 1) {- fprintf(stderr, "%s: no data in file %s\n", __func__, imatrix_file.c_str());- gguf_free(ctx_gguf);- ggml_free(ctx);+ common_imatrix loaded;+ if (!common_imatrix_load(imatrix_file, loaded)) {+ fprintf(stderr, "%s: failed to load imatrix from '%s'\n", __func__, imatrix_file.c_str());exit(1);}- const int dataset_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);- const int chunk_count_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);- const int chunk_size_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);- if (dataset_idx < 0 || chunk_count_idx < 0 || chunk_size_idx < 0) {+ if (!loaded.is_legacy && !loaded.has_metadata) {fprintf(stderr, "%s: missing imatrix metadata in file %s\n", __func__, imatrix_file.c_str());- gguf_free(ctx_gguf);- ggml_free(ctx);exit(1);}- const uint32_t chunk_size = gguf_get_val_u32(ctx_gguf, chunk_size_idx);-- const std::string sums_suffix{ ".in_sum2" };- const std::string counts_suffix{ ".counts" };-- // Using an ordered map to get a deterministic iteration order.- std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;-- for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {- std::string name = cur->name;-- if (name.empty()) { continue; }+ for (const auto & [name, entry] : loaded.entries) {+ auto & e = imatrix_data[name];+ e.resize(entry.sums.size());++ if (!loaded.is_legacy) {+ // GGUF format: normalize by per-expert counts+ const int64_t ncounts = entry.counts.size();+ const int64_t ne0 = (int64_t) entry.sums.size() / ncounts;++ for (int64_t j = 0; j < ncounts; ++j) {+ const float count = (float) entry.counts[j];+ if (count > 0.0f) {+ for (int64_t i = 0; i < ne0; ++i) {+ e[j*ne0 + i] = entry.sums[j*ne0 + i] / count;+ }+ } else {+ for (int64_t i = 0; i < ne0; ++i) {+ e[j*ne0 + i] = 1;+ }+ }+ }- if (string_remove_suffix(name, sums_suffix)) {- // in_sum2- sums_counts_for[std::move(name)].first = cur;- } else if (string_remove_suffix(name, counts_suffix)) {- // counts- sums_counts_for[std::move(name)].second = cur;+ if (getenv("LLAMA_TRACE")) {+ float max_count = 0.0f;+ for (int64_t j = 0; j < ncounts; ++j) {+ const float count = (float) entry.counts[j];+ if (count > max_count) {+ max_count = count;+ }+ }+ printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n",+ __func__, int(e.size()), int(max_count), int(max_count / loaded.chunk_size), name.c_str());+ }} else {- // ignore other tensors- }- }-- for (const auto & sc : sums_counts_for) {- const std::string & name = sc.first;- const struct ggml_tensor * sums = sc.second.first;- const struct ggml_tensor * counts = sc.second.second;-- if (!sums || !counts) {- fprintf(stderr, "%s: mismatched sums and counts for %s\n", __func__, name.c_str());- gguf_free(ctx_gguf);- ggml_free(ctx);- exit(1);- }-- const int64_t ne0 = sums->ne[0];- const int64_t ne1 = sums->ne[1];-- auto & e = imatrix_data[name];- e.resize(ggml_nelements(sums));- float max_count = 0.0f;- for (int64_t j = 0; j < ne1; ++j) {- const float count = ((const float *) counts->data)[j];- if (count > 0.0f) {- for (int64_t i = 0; i < ne0; ++i) {- e[j*ne0 + i] = ((const float *) sums->data)[j*ne0 + i] / count;+ // Legacy format: sums contain (raw/count)*ncall, divide by ncall+ const int64_t ncall = entry.counts.empty() ? 0 : entry.counts[0];+ if (ncall > 0) {+ for (size_t i = 0; i < entry.sums.size(); ++i) {+ e[i] = entry.sums[i] / ncall;}} else {- // Partial imatrix data, this tensor never got any input during calibration- for (int64_t i = 0; i < ne0; ++i) {- e[j*ne0 + i] = 1;+ for (size_t i = 0; i < entry.sums.size(); ++i) {+ e[i] = entry.sums[i];}}- if (count > max_count) {- max_count = count;++ if (getenv("LLAMA_TRACE")) {+ printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n",+ __func__, int(e.size()), int(ncall), name.c_str());}}- if (getenv("LLAMA_TRACE")) {- printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n", __func__, int(e.size()), int(max_count), int(max_count / chunk_size), name.c_str());- }}- int m_last_chunk = gguf_get_val_u32(ctx_gguf, chunk_count_idx);+ imatrix_datasets = std::move(loaded.datasets);- int64_t n_datasets = gguf_get_arr_n(ctx_gguf, dataset_idx);- imatrix_datasets.reserve(n_datasets);- for (int64_t i = 0; i < n_datasets; ++i) {- imatrix_datasets.push_back(gguf_get_arr_str(ctx_gguf, dataset_idx, i));- }- printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());- for (size_t i = 1; i < imatrix_datasets.size(); ++i) {- printf(", '%s'", imatrix_datasets[i].c_str());+ if (!imatrix_datasets.empty()) {+ printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());+ for (size_t i = 1; i < imatrix_datasets.size(); ++i) {+ printf(", '%s'", imatrix_datasets[i].c_str());+ }+ printf("]\n");}- printf("]\n");-- printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_chunk);- gguf_free(ctx_gguf);- ggml_free(ctx);+ printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), loaded.chunk_count);- return m_last_chunk;+ return loaded.chunk_count;}static int prepare_imatrix(const std::string & imatrix_file,