LLM inference in C/C++
C++ ggml
llama : add llama_model_ftype_name() (#25134)
Adrien Gallouëtfdb1db877c52
diff --git a/include/llama.h b/include/llama.hindex f723c9f60c..b89ab758b2 100644--- a/include/llama.h+++ b/include/llama.h@@ -159,6 +159,9 @@ extern "C" {LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file};+ // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"+ LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);+enum llama_rope_scaling_type {LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,LLAMA_ROPE_SCALING_TYPE_NONE = 0,@@ -606,6 +609,9 @@ extern "C" {// Get a string describing the model typeLLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);+ // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0+ LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model);+// Returns the total size of all the tensors in the model in bytesLLAMA_API uint64_t llama_model_size(const struct llama_model * model);diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex e07b0d2313..55554735da 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -27,52 +27,54 @@ const char * llama_file_version_name(llama_fver version) {return "unknown";}-static std::string llama_model_ftype_name(llama_ftype ftype) {- if (ftype & LLAMA_FTYPE_GUESSED) {- return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";- }-- switch (ftype) {- case LLAMA_FTYPE_ALL_F32: return "all F32";- case LLAMA_FTYPE_MOSTLY_F16: return "F16";- case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";- case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0";- case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";- case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";- case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";- case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";- case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";- case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE";- case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4";- case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";- case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";- case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";- case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";- case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";- case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";- case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";- case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";- case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";- case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";- case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";- case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";- case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";- case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";- case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";- case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";- case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";- case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";- case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";- case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";-- default: return "unknown, may not work";+#define LLAMA_FTYPE_PREFIX "(guessed) "++const char * llama_ftype_name(llama_ftype ftype) {+ static constexpr size_t guessed_prefix_len = sizeof(LLAMA_FTYPE_PREFIX) - 1;+ const char * name;+ switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) {+ case LLAMA_FTYPE_ALL_F32: name = LLAMA_FTYPE_PREFIX "all F32"; break;+ case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break;+ case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break;+ case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break;+ case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break;+ case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break;+ case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break;+ case LLAMA_FTYPE_MOSTLY_Q5_1: name = LLAMA_FTYPE_PREFIX "Q5_1"; break;+ case LLAMA_FTYPE_MOSTLY_Q8_0: name = LLAMA_FTYPE_PREFIX "Q8_0"; break;+ case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = LLAMA_FTYPE_PREFIX "MXFP4 MoE"; break;+ case LLAMA_FTYPE_MOSTLY_NVFP4: name = LLAMA_FTYPE_PREFIX "NVFP4"; break;+ case LLAMA_FTYPE_MOSTLY_Q2_K: name = LLAMA_FTYPE_PREFIX "Q2_K - Medium"; break;+ case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = LLAMA_FTYPE_PREFIX "Q2_K - Small"; break;+ case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = LLAMA_FTYPE_PREFIX "Q3_K - Small"; break;+ case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = LLAMA_FTYPE_PREFIX "Q3_K - Medium"; break;+ case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = LLAMA_FTYPE_PREFIX "Q3_K - Large"; break;+ case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = LLAMA_FTYPE_PREFIX "Q4_K - Small"; break;+ case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = LLAMA_FTYPE_PREFIX "Q4_K - Medium"; break;+ case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = LLAMA_FTYPE_PREFIX "Q5_K - Small"; break;+ case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = LLAMA_FTYPE_PREFIX "Q5_K - Medium"; break;+ case LLAMA_FTYPE_MOSTLY_Q6_K: name = LLAMA_FTYPE_PREFIX "Q6_K"; break;+ case LLAMA_FTYPE_MOSTLY_TQ1_0: name = LLAMA_FTYPE_PREFIX "TQ1_0 - 1.69 bpw ternary"; break;+ case LLAMA_FTYPE_MOSTLY_TQ2_0: name = LLAMA_FTYPE_PREFIX "TQ2_0 - 2.06 bpw ternary"; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = LLAMA_FTYPE_PREFIX "IQ2_XXS - 2.0625 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = LLAMA_FTYPE_PREFIX "IQ2_XS - 2.3125 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_S: name = LLAMA_FTYPE_PREFIX "IQ3_S - 3.4375 bpw"; break;+ case LLAMA_FTYPE_MOSTLY_IQ3_M: name = LLAMA_FTYPE_PREFIX "IQ3_S mix - 3.66 bpw"; break;+ default: name = LLAMA_FTYPE_PREFIX "unknown, may not work"; break;}+ return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len;}+#undef LLAMA_FTYPE_PREFIX+// return a list of splits for a given path// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splitsstatic std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {@@ -1693,12 +1695,12 @@ bool llama_model_loader::load_all_data(}std::string llama_model_loader::ftype_name() const {- return llama_model_ftype_name(ftype);+ return llama_ftype_name(ftype);}void llama_model_loader::print_info() const {LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver));- LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str());+ LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_ftype_name(ftype));if (n_bytes < GiB) {LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements);} else {diff --git a/src/llama-model.cpp b/src/llama-model.cppindex d58ebac28b..e07f6e9863 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -987,6 +987,8 @@ struct llama_model::impl {std::string desc_str;+ llama_ftype ftype = LLAMA_FTYPE_ALL_F32;+// model memory mapped filesllama_mmaps mappings;@@ -1200,6 +1202,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name();+ pimpl->ftype = ml.ftype;+if (hparams.f_max_alibi_bias > 0.0f) {hparams.use_alibi = true;}@@ -1646,6 +1650,10 @@ std::string llama_model::desc() const {return pimpl->desc_str;}+llama_ftype llama_model::ftype() const {+ return pimpl->ftype;+}+size_t llama_model::size() const {return pimpl->n_bytes;}@@ -2616,6 +2624,10 @@ int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size)return snprintf(buf, buf_size, "%s", model->desc().c_str());}+llama_ftype llama_model_ftype(const llama_model * model) {+ return model->ftype();+}+uint64_t llama_model_size(const llama_model * model) {return model->size();}diff --git a/src/llama-model.h b/src/llama-model.hindex 4800d2928c..45b054cedf 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -637,6 +637,8 @@ struct llama_model {std::string desc() const;+ llama_ftype ftype() const;+size_t size() const; // file sizesize_t n_tensors() const;size_t n_devices() const;diff --git a/tools/cli/cli.cpp b/tools/cli/cli.cppindex 8b7b58693f..d974a40195 100644--- a/tools/cli/cli.cpp+++ b/tools/cli/cli.cpp@@ -448,6 +448,9 @@ int llama_cli(int argc, char ** argv) {console::log("%s\n", LLAMA_ASCII_LOGO);console::log("build : %s\n", inf.build_info.c_str());console::log("model : %s\n", inf.model_name.c_str());+ if (!inf.model_ftype.empty()) {+ console::log("ftype : %s\n", inf.model_ftype.c_str());+ }console::log("modalities : %s\n", modalities.c_str());if (!params.system_prompt.empty()) {console::log("using custom system prompt\n");diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 39aa20b325..20e93258fc 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -3989,6 +3989,8 @@ server_context_meta server_context::get_meta() const {auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, bos_id, true) : "";auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, eos_id, true) : "";+ const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt));+return server_context_meta {/* build_info */ std::string(llama_build_info()),/* model_name */ impl->model_name,@@ -4023,6 +4025,7 @@ server_context_meta server_context::get_meta() const {/* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt),/* model_n_params */ llama_model_n_params(impl->model_tgt),/* model_size */ llama_model_size(impl->model_tgt),+ /* model_ftype */ ftype_name,};}@@ -5118,6 +5121,7 @@ json server_routes::get_model_info() const {{"n_embd", meta->model_n_embd_inp},{"n_params", meta->model_n_params},{"size", meta->model_size},+ {"ftype", meta->model_ftype},}},};}diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex 952f825f72..f9ab1132b1 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -50,6 +50,7 @@ struct server_context_meta {int32_t model_n_embd_inp;uint64_t model_n_params;uint64_t model_size;+ std::string model_ftype;};enum server_state {